2023-08-27T20:31:49.678 INFO:root:teuthology version: 0.0.1.dev185+g449a1bc 2023-08-27T20:31:49.678 DEBUG:teuthology.run:Teuthology command: teuthology --name teuthology-2023-08-26_14:23:13-upgrade:quincy-x-reef-distro-default-smithi --verbose --owner scheduled_teuthology@teuthology --description upgrade:quincy-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} --archive /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:quincy-x-reef-distro-default-smithi/7381782 -- /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:quincy-x-reef-distro-default-smithi/7381782/orig.config.yaml 2023-08-27T20:31:49.752 DEBUG:teuthology.report:Pushing job info to https://paddles.front.sepia.ceph.com/ 2023-08-27T20:31:49.843 INFO:teuthology.run:Config: archive_path: /home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:quincy-x-reef-distro-default-smithi/7381782 branch: reef description: upgrade:quincy-x/stress-split/{0-distro/centos_8.stream_container_tools_crun 0-roles 1-start 2-first-half-tasks/readwrite 3-stress-tasks/{radosbench rbd-cls rbd-import-export rbd_api readwrite snaps-few-objects} 4-second-half-tasks/radosbench mon_election/connectivity} email: ceph-qa@ceph.io first-half-sequence: - cephadm.shell: env: - sha1 mon.a: - ceph config set mgr mgr/cephadm/daemon_cache_timeout 60 - ceph config set global log_to_journald false --force - ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1 - ceph orch ps - echo wait for minority of mons to upgrade - while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for majority of mons to upgrade - 'while ! ceph mon versions | grep $sha1 | egrep '': [23]'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for all mons to upgrade - 'while ! ceph mon versions | grep $sha1 | grep '': 3'' ; do sleep 2 ; done' - ceph orch ps - ceph orch upgrade pause - sleep 60 - ceph orch upgrade resume - echo wait for half of osds to upgrade - 'while ! ceph osd versions | grep $sha1 | egrep '': [45678]''; do sleep 2 ; done' - ceph orch upgrade pause - ceph orch ps - ceph orch ps - ceph versions first-half-tasks: - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' first_in_suite: false job_id: '7381782' kernel: kdb: true sha1: distro last_in_suite: false machine_type: smithi meta: - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' - desc: 'run basic cls tests for rbd ' - desc: 'run basic import/export cli tests for rbd ' - desc: 'librbd C and C++ api tests ' - desc: 'randomized correctness test for rados operations on a replicated pool, using only reads, writes, and deletes ' - desc: 'randomized correctness test for rados operations on a replicated pool with snapshot operations ' - desc: 'run randomized correctness test for rados operations generate write load with rados bench ' name: teuthology-2023-08-26_14:23:13-upgrade:quincy-x-reef-distro-default-smithi no_nested_subset: false nuke-on-error: true openstack: - volumes: count: 4 size: 10 os_type: centos os_version: 8.stream overrides: admin_socket: branch: reef ceph: conf: global: mon election default strategy: 3 mgr: debug mgr: 20 debug ms: 1 mon: debug mon: 20 debug ms: 1 debug paxos: 20 osd: debug ms: 1 debug osd: 20 osd shutdown pgref assert: true create_rbd_pool: true flavor: default log-ignorelist: - \(MDS_ALL_DOWN\) - \(MDS_UP_LESS_THAN_MAX\) sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 ceph-deploy: conf: client: log file: /var/log/ceph/ceph-$name.$pid.log mon: {} install: ceph: flavor: default sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 selinux: whitelist: - scontext=system_u:system_r:logrotate_t:s0 workunit: branch: reef sha1: 81fef490342f7b9544e89fe32a9e51929064494d owner: scheduled_teuthology@teuthology priority: 100 repo: https://github.com/ceph/ceph.git roles: - - mon.a - mon.c - mgr.y - osd.0 - osd.1 - osd.2 - osd.3 - client.0 - node-exporter.a - alertmanager.a - - mon.b - mgr.x - osd.4 - osd.5 - osd.6 - osd.7 - client.1 - prometheus.a - grafana.a - node-exporter.b second-half-sequence: sequential: - cephadm.shell: env: - sha1 mon.a: - ceph orch upgrade resume - sleep 60 - echo wait for upgrade to complete - while ceph orch upgrade status | jq '.in_progress' | grep true ; do ceph orch ps ; ceph versions ; sleep 30 ; done - echo upgrade complete - ceph orch ps - ceph versions - ceph versions | jq -e '.overall | length == 1' - ceph versions | jq -e '.overall | keys' | grep $sha1 second-half-tasks: - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' seed: 6846 sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 sleep_before_teardown: 0 stress-tasks: - thrashosds: aggressive_pg_num_changes: false chance_force_recovery: 0 chance_pgnum_grow: 1 chance_pgpnum_fix: 1 chance_thrash_cluster_full: 0 chance_thrash_pg_upmap: 0 chance_thrash_pg_upmap_items: 0 disable_objectstore_tool_tests: true timeout: 1200 - full_sequential: - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - radosbench: clients: - client.0 time: 90 - print: '**** done end radosbench.yaml' - workunit: branch: quincy clients: client.0: - cls/test_cls_rbd.sh - print: '**** done cls/test_cls_rbd.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/import_export.sh env: RBD_CREATE_ARGS: --new-format - print: '**** done rbd/import_export.sh 5-workload' - workunit: branch: quincy clients: client.0: - rbd/test_librbd.sh - print: '**** done rbd/test_librbd.sh 7-workload' - full_sequential: - rados: clients: - client.0 objects: 500 op_weights: delete: 10 read: 45 write: 45 ops: 4000 write_append_excl: false - print: '**** done rados/readwrite 5-workload' - full_sequential: - rados: clients: - client.0 objects: 50 op_weights: delete: 50 read: 100 rollback: 50 snap_create: 50 snap_remove: 50 write: 100 ops: 4000 write_append_excl: false - print: '**** done rados/snaps-few-objects 5-workload' suite: upgrade:quincy-x suite_branch: reef suite_path: /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa suite_relpath: qa suite_repo: https://github.com/ceph/ceph.git suite_sha1: 81fef490342f7b9544e89fe32a9e51929064494d targets: smithi093.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDftJQ4o/mSCzjI2iFAnBJLzpzuTGR4UZ9XWEVWBgF8DajelCvrVOlD10JTjrX0zAels91dW5yJH+fRrO/QbFI03A/rvfMFudttn9NEYzqg3bCmWoneKttJEXfSF/ZVtLNTVn71BN4JXeH9WeDZL7/Thn3Mj730U+hYiHdkwMjrM5aacCNfXkRsJfPjrcmn4Q1XCj9f8tiqxjO702bWu5MGajjNZVcRsN1Apwg9nctNqawGxV/CWhFpNOi2BIvV50Ywa5qJIRfwMCT8ZJ3bqu8QXHyTNQkL8YhP+UtOwtN0AnXY0JBACBndukmUMqMzvtOPZtCV6WBJgxq46GvupMeSXdagx0qX/2Pvu0kwFC0Svc/mrQis3fW6f5WbgNECpReF28eAvk/Lyq33nPExsujX8zML/ibefVNZp394d5eXpeUrHyRqnr4qAlrqxhpTI023Y+NjCm1XyjcDwVGBqktzUrzec3JkGntmNix1DK5EGPKffSccj+kKHor/RY1t2kc= smithi136.front.sepia.ceph.com: ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQCkwsEwHn3SANYfpNQER3SrVM3UzjmKwMwLznJvBzkVcAZun2HNbWrsktOXthpqqiVvMEp9T3V2q89QqB0nYuDxT836OD57uteZ51zjijFLPWqNrIB81H94Q8mF7Wk3hryIQnV3P86AE2nDsOn8fZBlk9dDXxcd7R8HZjUSeljazcd5M5CGSh7y1zMxSlGO3lG3LM3J7WBe/hEUasb2Sl5bLYM0xKQN0VgKWYWkSkBiCwkKcHrq+YDhbQwa2XMDk7ZmgFBDSwf0u/vzfpKwRsv4PnXD2KZiASqMIqRMZcA621qJ/BIJfgbdK3I7B9aq8wYqB9kXMt45Iy+RhkVjAbNzp8vzp6z3xuWTfjBTskfvWmTaXkKWHCl5XB7lZu9cC7K7ya3mTSa/muHuIE3SHubBB0mtujBwXosb4pXP8HSlvl/iBJhZCDJZcf1hb3Q5nBO9QXRXfPl3WYMSj6GMUZTQqvL96yWnVlsx09gmmFfOf+jGX0WxA8kCVSn2xebc1lM= tasks: - pexec: all: - sudo cp /etc/containers/registries.conf /etc/containers/registries.conf.backup - sudo dnf -y module reset container-tools - sudo dnf -y module install container-tools --allowerasing --nobest - sudo cp /etc/containers/registries.conf.backup /etc/containers/registries.conf - sudo sed -i 's/runtime = "runc"/#runtime = "runc"/g' /usr/share/containers/containers.conf - sudo sed -i 's/#runtime = "crun"/runtime = "crun"/g' /usr/share/containers/containers.conf - install: branch: quincy exclude_packages: - ceph-volume - cephadm: cephadm_branch: quincy cephadm_git_url: https://github.com/ceph/ceph conf: osd: osd_class_default_list: '*' osd_class_load_list: '*' image: quay.ceph.io/ceph-ci/ceph:quincy - cephadm.shell: mon.a: - ceph fs volume create foo - ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force - ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force - ceph.healthy: null - print: '**** upgrading first half of cluster, with stress ****' - parallel: - first-half-tasks - first-half-sequence - print: '**** done upgrading first half of cluster ****' - ceph.healthy: null - print: '**** applying stress + thrashing to mixed-version cluster ****' - parallel: - stress-tasks - ceph.healthy: null - print: '**** finishing upgrade ****' - parallel: - second-half-tasks - second-half-sequence - ceph.healthy: null teuthology: fragments_dropped: [] meta: {} postmerge: [] teuthology_branch: main teuthology_sha1: 449a1bc2027504e7b3c3d7b30fa4178906581da7 timestamp: 2023-08-26_14:23:13 tube: smithi user: teuthology verbose: true worker_log: /home/teuthworker/archive/worker_logs/dispatcher.smithi.29856 2023-08-27T20:31:49.844 INFO:teuthology.run:suite_path is set to /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa; will attempt to use it 2023-08-27T20:31:49.845 INFO:teuthology.run:Found tasks at /home/teuthworker/src/github.com_ceph_ceph_81fef490342f7b9544e89fe32a9e51929064494d/qa/tasks 2023-08-27T20:31:49.845 INFO:teuthology.run_tasks:Running task internal.check_packages... 2023-08-27T20:31:49.847 INFO:teuthology.task.internal:Checking packages... 2023-08-27T20:31:49.871 INFO:teuthology.task.internal:Checking packages for os_type 'centos', flavor 'default' and ceph hash '9cf208207dfe23ffccb33308b22e51570cb45e51' 2023-08-27T20:31:49.871 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T20:31:49.871 INFO:teuthology.packaging:ref: None 2023-08-27T20:31:49.871 INFO:teuthology.packaging:tag: None 2023-08-27T20:31:49.871 INFO:teuthology.packaging:branch: reef 2023-08-27T20:31:49.871 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:31:49.871 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=reef 2023-08-27T20:31:50.084 INFO:teuthology.task.internal:Found packages for ceph version 18.2.0-191.g02042ef2 2023-08-27T20:31:50.084 INFO:teuthology.run_tasks:Running task internal.buildpackages_prep... 2023-08-27T20:31:50.092 INFO:teuthology.task.internal:no buildpackages task found 2023-08-27T20:31:50.092 INFO:teuthology.run_tasks:Running task internal.save_config... 2023-08-27T20:31:50.101 INFO:teuthology.task.internal:Saving configuration 2023-08-27T20:31:50.123 INFO:teuthology.run_tasks:Running task internal.check_lock... 2023-08-27T20:31:50.130 INFO:teuthology.task.internal.check_lock:Checking locks... 2023-08-27T20:31:50.155 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi093.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:quincy-x-reef-distro-default-smithi/7381782', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-08-27 20:26:44.785333', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBLsOYTOC4B4/m11zyhcPKaGZxkp46CJvdjSZhJEd53mzwo6GaYCej7UGYWPlOpRDao49p6e57komdPlUiSkOK9E='} 2023-08-27T20:31:50.175 DEBUG:teuthology.task.internal.check_lock:machine status is {'name': 'smithi136.front.sepia.ceph.com', 'description': '/home/teuthworker/archive/teuthology-2023-08-26_14:23:13-upgrade:quincy-x-reef-distro-default-smithi/7381782', 'up': True, 'machine_type': 'smithi', 'is_vm': False, 'vm_host': None, 'os_type': 'centos', 'os_version': '8', 'arch': 'x86_64', 'locked': True, 'locked_since': '2023-08-27 20:26:44.787395', 'locked_by': 'scheduled_teuthology@teuthology', 'mac_address': None, 'ssh_pub_key': 'ecdsa-sha2-nistp256 AAAAE2VjZHNhLXNoYTItbmlzdHAyNTYAAAAIbmlzdHAyNTYAAABBBE7NpyuX49xZ/E9jySE9tWIn8zaXbn33O2jdlw01nxtRW2ImitUa87g5oDid7EYRQ3yH4EUaitq8B4cYxq31ZRw='} 2023-08-27T20:31:50.176 INFO:teuthology.run_tasks:Running task internal.add_remotes... 2023-08-27T20:31:50.182 INFO:teuthology.task.internal:roles: ubuntu@smithi093.front.sepia.ceph.com - ['mon.a', 'mon.c', 'mgr.y', 'osd.0', 'osd.1', 'osd.2', 'osd.3', 'client.0', 'node-exporter.a', 'alertmanager.a'] 2023-08-27T20:31:50.182 INFO:teuthology.task.internal:roles: ubuntu@smithi136.front.sepia.ceph.com - ['mon.b', 'mgr.x', 'osd.4', 'osd.5', 'osd.6', 'osd.7', 'client.1', 'prometheus.a', 'grafana.a', 'node-exporter.b'] 2023-08-27T20:31:50.182 INFO:teuthology.run_tasks:Running task console_log... 2023-08-27T20:31:50.254 DEBUG:teuthology.exit:Installing handler: Handler(exiter=, func=.kill_console_loggers at 0x7f904cef6ee0>, signals=[15]) 2023-08-27T20:31:50.254 INFO:teuthology.run_tasks:Running task internal.connect... 2023-08-27T20:31:50.268 INFO:teuthology.task.internal:Opening connections... 2023-08-27T20:31:50.268 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi093.front.sepia.ceph.com 2023-08-27T20:31:50.269 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi093.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:31:50.342 DEBUG:teuthology.task.internal:connecting to ubuntu@smithi136.front.sepia.ceph.com 2023-08-27T20:31:50.342 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:31:50.409 INFO:teuthology.run_tasks:Running task internal.push_inventory... 2023-08-27T20:31:50.416 DEBUG:teuthology.orchestra.run.smithi093:> uname -m 2023-08-27T20:31:50.439 INFO:teuthology.orchestra.run.smithi093.stdout:x86_64 2023-08-27T20:31:50.439 DEBUG:teuthology.orchestra.run.smithi093:> cat /etc/os-release 2023-08-27T20:31:50.500 INFO:teuthology.orchestra.run.smithi093.stdout:NAME="CentOS Stream" 2023-08-27T20:31:50.500 INFO:teuthology.orchestra.run.smithi093.stdout:VERSION="8" 2023-08-27T20:31:50.500 INFO:teuthology.orchestra.run.smithi093.stdout:ID="centos" 2023-08-27T20:31:50.500 INFO:teuthology.orchestra.run.smithi093.stdout:ID_LIKE="rhel fedora" 2023-08-27T20:31:50.500 INFO:teuthology.orchestra.run.smithi093.stdout:VERSION_ID="8" 2023-08-27T20:31:50.501 INFO:teuthology.orchestra.run.smithi093.stdout:PLATFORM_ID="platform:el8" 2023-08-27T20:31:50.501 INFO:teuthology.orchestra.run.smithi093.stdout:PRETTY_NAME="CentOS Stream 8" 2023-08-27T20:31:50.501 INFO:teuthology.orchestra.run.smithi093.stdout:ANSI_COLOR="0;31" 2023-08-27T20:31:50.501 INFO:teuthology.orchestra.run.smithi093.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-08-27T20:31:50.501 INFO:teuthology.orchestra.run.smithi093.stdout:HOME_URL="https://centos.org/" 2023-08-27T20:31:50.501 INFO:teuthology.orchestra.run.smithi093.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-08-27T20:31:50.501 INFO:teuthology.orchestra.run.smithi093.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-08-27T20:31:50.502 INFO:teuthology.orchestra.run.smithi093.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-08-27T20:31:50.502 INFO:teuthology.lock.ops:Updating smithi093.front.sepia.ceph.com on lock server 2023-08-27T20:31:50.526 DEBUG:teuthology.orchestra.run.smithi136:> uname -m 2023-08-27T20:31:50.548 INFO:teuthology.orchestra.run.smithi136.stdout:x86_64 2023-08-27T20:31:50.548 DEBUG:teuthology.orchestra.run.smithi136:> cat /etc/os-release 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:NAME="CentOS Stream" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION="8" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:ID="centos" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:ID_LIKE="rhel fedora" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:VERSION_ID="8" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:PLATFORM_ID="platform:el8" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:PRETTY_NAME="CentOS Stream 8" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:ANSI_COLOR="0;31" 2023-08-27T20:31:50.609 INFO:teuthology.orchestra.run.smithi136.stdout:CPE_NAME="cpe:/o:centos:centos:8" 2023-08-27T20:31:50.610 INFO:teuthology.orchestra.run.smithi136.stdout:HOME_URL="https://centos.org/" 2023-08-27T20:31:50.610 INFO:teuthology.orchestra.run.smithi136.stdout:BUG_REPORT_URL="https://bugzilla.redhat.com/" 2023-08-27T20:31:50.610 INFO:teuthology.orchestra.run.smithi136.stdout:REDHAT_SUPPORT_PRODUCT="Red Hat Enterprise Linux 8" 2023-08-27T20:31:50.610 INFO:teuthology.orchestra.run.smithi136.stdout:REDHAT_SUPPORT_PRODUCT_VERSION="CentOS Stream" 2023-08-27T20:31:50.610 INFO:teuthology.lock.ops:Updating smithi136.front.sepia.ceph.com on lock server 2023-08-27T20:31:50.634 INFO:teuthology.run_tasks:Running task internal.serialize_remote_roles... 2023-08-27T20:31:50.644 INFO:teuthology.run_tasks:Running task internal.check_conflict... 2023-08-27T20:31:50.652 INFO:teuthology.task.internal:Checking for old test directory... 2023-08-27T20:31:50.653 DEBUG:teuthology.orchestra.run.smithi093:> test '!' -e /home/ubuntu/cephtest 2023-08-27T20:31:50.655 DEBUG:teuthology.orchestra.run.smithi136:> test '!' -e /home/ubuntu/cephtest 2023-08-27T20:31:50.674 INFO:teuthology.run_tasks:Running task internal.check_ceph_data... 2023-08-27T20:31:50.680 INFO:teuthology.task.internal:Checking for non-empty /var/lib/ceph... 2023-08-27T20:31:50.681 DEBUG:teuthology.orchestra.run.smithi093:> test -z $(ls -A /var/lib/ceph) 2023-08-27T20:31:50.716 DEBUG:teuthology.orchestra.run.smithi136:> test -z $(ls -A /var/lib/ceph) 2023-08-27T20:31:50.748 INFO:teuthology.run_tasks:Running task internal.vm_setup... 2023-08-27T20:31:50.799 INFO:teuthology.run_tasks:Running task kernel... 2023-08-27T20:31:50.834 INFO:teuthology.task.kernel:normalize config orig: {'kdb': True, 'sha1': 'distro'} 2023-08-27T20:31:50.834 INFO:teuthology.task.kernel:config {'mon.a': {'kdb': True, 'sha1': 'distro'}, 'mon.b': {'kdb': True, 'sha1': 'distro'}}, timeout 300 2023-08-27T20:31:50.834 DEBUG:teuthology.orchestra.run.smithi093:> test -f /run/.containerenv -o -f /.dockerenv 2023-08-27T20:31:50.835 DEBUG:teuthology.orchestra.run.smithi136:> test -f /run/.containerenv -o -f /.dockerenv 2023-08-27T20:31:50.854 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-27T20:31:50.854 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2023-08-27T20:31:50.855 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-27T20:31:50.856 DEBUG:teuthology.orchestra.run.smithi093:> uname -r 2023-08-27T20:31:50.916 INFO:teuthology.orchestra.run.smithi136.stdout:4.18.0-489.el8.x86_64 2023-08-27T20:31:50.916 INFO:teuthology.task.kernel:Running kernel on smithi136: 4.18.0-489.el8.x86_64 2023-08-27T20:31:50.916 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum install -y kernel 2023-08-27T20:31:50.918 INFO:teuthology.orchestra.run.smithi093.stdout:4.18.0-489.el8.x86_64 2023-08-27T20:31:50.918 INFO:teuthology.task.kernel:Running kernel on smithi093: 4.18.0-489.el8.x86_64 2023-08-27T20:31:50.918 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum install -y kernel 2023-08-27T20:31:55.405 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - AppStream 16 MB/s | 33 MB 00:02 2023-08-27T20:31:55.655 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - AppStream 16 MB/s | 33 MB 00:02 2023-08-27T20:32:03.588 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - BaseOS 20 MB/s | 46 MB 00:02 2023-08-27T20:32:03.770 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - BaseOS 23 MB/s | 46 MB 00:02 2023-08-27T20:32:10.862 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - Extras 25 kB/s | 18 kB 00:00 2023-08-27T20:32:10.894 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - Extras 35 kB/s | 18 kB 00:00 2023-08-27T20:32:11.454 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - Extras common packages 16 kB/s | 6.8 kB 00:00 2023-08-27T20:32:11.519 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - Extras common packages 15 kB/s | 6.8 kB 00:00 2023-08-27T20:32:11.748 INFO:teuthology.orchestra.run.smithi093.stdout:Copr repo for python3-asyncssh owned by ceph 24 kB/s | 3.5 kB 00:00 2023-08-27T20:32:11.830 INFO:teuthology.orchestra.run.smithi136.stdout:Copr repo for python3-asyncssh owned by ceph 22 kB/s | 3.5 kB 00:00 2023-08-27T20:32:13.130 INFO:teuthology.orchestra.run.smithi136.stdout:Extra Packages for Enterprise Linux 14 MB/s | 16 MB 00:01 2023-08-27T20:32:13.800 INFO:teuthology.orchestra.run.smithi093.stdout:Extra Packages for Enterprise Linux 8.3 MB/s | 16 MB 00:01 2023-08-27T20:32:16.894 INFO:teuthology.orchestra.run.smithi136.stdout:lab-extras 396 kB/s | 24 kB 00:00 2023-08-27T20:32:17.517 INFO:teuthology.orchestra.run.smithi093.stdout:lab-extras 500 kB/s | 24 kB 00:00 2023-08-27T20:32:20.345 INFO:teuthology.orchestra.run.smithi136.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-08-27T20:32:20.345 INFO:teuthology.orchestra.run.smithi136.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-08-27T20:32:20.584 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:32:20.585 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:32:20.585 INFO:teuthology.orchestra.run.smithi136.stdout: Package Architecture Version Repository Size 2023-08-27T20:32:20.585 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:32:20.585 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout: kernel x86_64 4.18.0-512.el8 baseos 10 M 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-core x86_64 4.18.0-512.el8 baseos 42 M 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules x86_64 4.18.0-512.el8 baseos 35 M 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout:Install 3 Packages 2023-08-27T20:32:20.586 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:32:20.587 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 88 M 2023-08-27T20:32:20.587 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 96 M 2023-08-27T20:32:20.587 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:32:20.999 INFO:teuthology.orchestra.run.smithi093.stdout:Package kernel-4.18.0-348.el8.x86_64 is already installed. 2023-08-27T20:32:20.999 INFO:teuthology.orchestra.run.smithi093.stdout:Package kernel-4.18.0-489.el8.x86_64 is already installed. 2023-08-27T20:32:21.237 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:32:21.237 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:32:21.237 INFO:teuthology.orchestra.run.smithi093.stdout: Package Architecture Version Repository Size 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout: kernel x86_64 4.18.0-512.el8 baseos 10 M 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout: kernel-core x86_64 4.18.0-512.el8 baseos 42 M 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout: kernel-modules x86_64 4.18.0-512.el8 baseos 35 M 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:32:21.238 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:32:21.239 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:32:21.239 INFO:teuthology.orchestra.run.smithi093.stdout:Install 3 Packages 2023-08-27T20:32:21.239 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:32:21.239 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 88 M 2023-08-27T20:32:21.240 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 96 M 2023-08-27T20:32:21.240 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:32:21.939 INFO:teuthology.orchestra.run.smithi136.stdout:(1/3): kernel-4.18.0-512.el8.x86_64.rpm 8.6 MB/s | 10 MB 00:01 2023-08-27T20:32:22.851 INFO:teuthology.orchestra.run.smithi093.stdout:(1/3): kernel-4.18.0-512.el8.x86_64.rpm 6.4 MB/s | 10 MB 00:01 2023-08-27T20:32:23.264 INFO:teuthology.orchestra.run.smithi136.stdout:(2/3): kernel-modules-4.18.0-512.el8.x86_64.rpm 14 MB/s | 35 MB 00:02 2023-08-27T20:32:23.897 INFO:teuthology.orchestra.run.smithi136.stdout:(3/3): kernel-core-4.18.0-512.el8.x86_64.rpm 14 MB/s | 42 MB 00:03 2023-08-27T20:32:23.898 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:32:23.898 INFO:teuthology.orchestra.run.smithi136.stdout:Total 27 MB/s | 88 MB 00:03 2023-08-27T20:32:23.935 INFO:teuthology.orchestra.run.smithi093.stdout:(2/3): kernel-modules-4.18.0-512.el8.x86_64.rpm 13 MB/s | 35 MB 00:02 2023-08-27T20:32:24.410 INFO:teuthology.orchestra.run.smithi093.stdout:(3/3): kernel-core-4.18.0-512.el8.x86_64.rpm 14 MB/s | 42 MB 00:03 2023-08-27T20:32:24.410 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:32:24.411 INFO:teuthology.orchestra.run.smithi093.stdout:Total 28 MB/s | 88 MB 00:03 2023-08-27T20:32:24.564 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:32:24.988 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:32:24.988 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:32:25.084 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:32:25.502 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:32:25.502 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:32:26.229 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:32:26.252 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:32:26.712 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:32:26.732 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:32:28.979 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:32:29.478 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:32:29.559 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-core-4.18.0-512.el8.x86_64 1/3 2023-08-27T20:32:30.017 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : kernel-core-4.18.0-512.el8.x86_64 1/3 2023-08-27T20:32:33.120 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-core-4.18.0-512.el8.x86_64 1/3 2023-08-27T20:32:33.508 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-modules-4.18.0-512.el8.x86_64 2/3 2023-08-27T20:32:33.581 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: kernel-core-4.18.0-512.el8.x86_64 1/3 2023-08-27T20:32:33.938 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : kernel-modules-4.18.0-512.el8.x86_64 2/3 2023-08-27T20:32:36.885 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-modules-4.18.0-512.el8.x86_64 2/3 2023-08-27T20:32:37.198 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : kernel-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:37.309 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: kernel-modules-4.18.0-512.el8.x86_64 2/3 2023-08-27T20:32:37.567 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : kernel-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:58.868 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-core-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:58.892 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-modules-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:59.238 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: kernel-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:59.238 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-4.18.0-512.el8.x86_64 1/3 2023-08-27T20:32:59.238 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-core-4.18.0-512.el8.x86_64 2/3 2023-08-27T20:32:59.242 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: kernel-core-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:59.267 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: kernel-modules-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:59.606 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: kernel-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:59.607 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : kernel-4.18.0-512.el8.x86_64 1/3 2023-08-27T20:32:59.607 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : kernel-core-4.18.0-512.el8.x86_64 2/3 2023-08-27T20:32:59.967 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : kernel-modules-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:32:59.967 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:32:59.967 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:32:59.968 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-4.18.0-512.el8.x86_64 kernel-core-4.18.0-512.el8.x86_64 2023-08-27T20:32:59.968 INFO:teuthology.orchestra.run.smithi136.stdout: kernel-modules-4.18.0-512.el8.x86_64 2023-08-27T20:32:59.968 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:32:59.968 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:33:00.114 DEBUG:teuthology.orchestra.run.smithi136:> rpm -q kernel | sort -rV | head -n 1 2023-08-27T20:33:00.326 INFO:teuthology.orchestra.run.smithi136.stdout:kernel-4.18.0-512.el8.x86_64 2023-08-27T20:33:00.326 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.326 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.326 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-08-27T20:33:00.326 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-08-27T20:33:00.327 INFO:teuthology.task.kernel:Installing distro kernel on mon.b... 2023-08-27T20:33:00.327 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi136.front.sepia.ceph.com, path=None, version=distro) 2023-08-27T20:33:00.327 DEBUG:teuthology.orchestra.run.smithi136:> rpm -q kernel | sort -rV | head -n 1 2023-08-27T20:33:00.334 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : kernel-modules-4.18.0-512.el8.x86_64 3/3 2023-08-27T20:33:00.334 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:33:00.334 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:33:00.334 INFO:teuthology.orchestra.run.smithi093.stdout: kernel-4.18.0-512.el8.x86_64 kernel-core-4.18.0-512.el8.x86_64 2023-08-27T20:33:00.334 INFO:teuthology.orchestra.run.smithi093.stdout: kernel-modules-4.18.0-512.el8.x86_64 2023-08-27T20:33:00.334 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:33:00.334 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:33:00.476 DEBUG:teuthology.orchestra.run.smithi093:> rpm -q kernel | sort -rV | head -n 1 2023-08-27T20:33:00.534 INFO:teuthology.orchestra.run.smithi136.stdout:kernel-4.18.0-512.el8.x86_64 2023-08-27T20:33:00.535 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.535 DEBUG:teuthology.orchestra.run.smithi136:> sudo rpm -qi grub2-tools 2023-08-27T20:33:00.568 INFO:teuthology.orchestra.run.smithi136.stdout:Name : grub2-tools 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Epoch : 1 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Version : 2.02 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Release : 129.el8 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Architecture: x86_64 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Group : System Environment/Base 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Size : 9265586 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:License : GPLv3+ 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-08-27T20:33:00.569 INFO:teuthology.orchestra.run.smithi136.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:Build Host : kbuilder.bsys.centos.org 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:Relocations : (not relocatable) 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:Packager : CentOS BuildSystem 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:Vendor : CentOS 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:URL : http://www.gnu.org/software/grub/ 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:Summary : Support tools for GRUB. 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:Description : 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:customizable bootloader with modular architecture. It supports a rich 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:variety of kernel formats, file systems, computer architectures and 2023-08-27T20:33:00.570 INFO:teuthology.orchestra.run.smithi136.stdout:hardware devices. 2023-08-27T20:33:00.571 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:33:00.571 INFO:teuthology.orchestra.run.smithi136.stdout:This subpackage provides tools for support of all platforms. 2023-08-27T20:33:00.571 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-08-27T20:33:00.572 INFO:teuthology.task.kernel:Updating grub on smithi136 to boot 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.572 DEBUG:teuthology.orchestra.run.smithi136:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-08-27T20:33:00.695 INFO:teuthology.orchestra.run.smithi093.stdout:kernel-4.18.0-512.el8.x86_64 2023-08-27T20:33:00.695 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.695 INFO:teuthology.task.kernel:Not newest distro kernel. Current: 4.18.0-489.el8.x86_64 Expected: 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.695 INFO:teuthology.task.kernel:Skipping firmware on distro kernel 2023-08-27T20:33:00.695 DEBUG:teuthology.task.kernel:src is distro, skipping download 2023-08-27T20:33:00.695 INFO:teuthology.task.kernel:Installing distro kernel on mon.a... 2023-08-27T20:33:00.695 DEBUG:teuthology.task.kernel:install_kernel(remote=ubuntu@smithi093.front.sepia.ceph.com, path=None, version=distro) 2023-08-27T20:33:00.696 DEBUG:teuthology.orchestra.run.smithi093:> rpm -q kernel | sort -rV | head -n 1 2023-08-27T20:33:00.902 INFO:teuthology.orchestra.run.smithi093.stdout:kernel-4.18.0-512.el8.x86_64 2023-08-27T20:33:00.903 DEBUG:teuthology.task.kernel:get_latest_image_version_rpm: 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.903 DEBUG:teuthology.orchestra.run.smithi093:> sudo rpm -qi grub2-tools 2023-08-27T20:33:00.936 INFO:teuthology.orchestra.run.smithi093.stdout:Name : grub2-tools 2023-08-27T20:33:00.936 INFO:teuthology.orchestra.run.smithi093.stdout:Epoch : 1 2023-08-27T20:33:00.936 INFO:teuthology.orchestra.run.smithi093.stdout:Version : 2.02 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Release : 129.el8 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Architecture: x86_64 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Install Date: Thu 04 May 2023 10:17:48 PM UTC 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Group : System Environment/Base 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Size : 9265586 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:License : GPLv3+ 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Signature : RSA/SHA256, Tue 19 Jul 2022 02:52:06 PM UTC, Key ID 05b555b38483c65d 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Source RPM : grub2-2.02-129.el8.src.rpm 2023-08-27T20:33:00.937 INFO:teuthology.orchestra.run.smithi093.stdout:Build Date : Tue 19 Jul 2022 02:30:39 PM UTC 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout:Build Host : kbuilder.bsys.centos.org 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout:Relocations : (not relocatable) 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout:Packager : CentOS BuildSystem 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout:Vendor : CentOS 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout:URL : http://www.gnu.org/software/grub/ 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout:Summary : Support tools for GRUB. 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout:Description : 2023-08-27T20:33:00.938 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:33:00.939 INFO:teuthology.orchestra.run.smithi093.stdout:The GRand Unified Bootloader (GRUB) is a highly configurable and 2023-08-27T20:33:00.939 INFO:teuthology.orchestra.run.smithi093.stdout:customizable bootloader with modular architecture. It supports a rich 2023-08-27T20:33:00.939 INFO:teuthology.orchestra.run.smithi093.stdout:variety of kernel formats, file systems, computer architectures and 2023-08-27T20:33:00.939 INFO:teuthology.orchestra.run.smithi093.stdout:hardware devices. 2023-08-27T20:33:00.939 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:33:00.939 INFO:teuthology.orchestra.run.smithi093.stdout:This subpackage provides tools for support of all platforms. 2023-08-27T20:33:00.940 INFO:teuthology.task.kernel:Updating Grub Version: grub2 2023-08-27T20:33:00.940 INFO:teuthology.task.kernel:Updating grub on smithi093 to boot 4.18.0-512.el8.x86_64 2023-08-27T20:33:00.940 DEBUG:teuthology.orchestra.run.smithi093:> sudo grub2-mkconfig -o /boot/grub2/grub.cfg 2023-08-27T20:33:02.270 INFO:teuthology.orchestra.run.smithi136.stderr:Generating grub configuration file ... 2023-08-27T20:33:02.545 INFO:teuthology.orchestra.run.smithi093.stderr:Generating grub configuration file ... 2023-08-27T20:33:04.208 INFO:teuthology.orchestra.run.smithi136.stderr:done 2023-08-27T20:33:04.210 DEBUG:teuthology.orchestra.run.smithi136:> mktemp 2023-08-27T20:33:04.232 INFO:teuthology.orchestra.run.smithi136.stdout:/tmp/tmp.8FGsxTzFNh 2023-08-27T20:33:04.232 DEBUG:teuthology.orchestra.run.smithi136:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.8FGsxTzFNh 2023-08-27T20:33:04.309 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 0666 /tmp/tmp.8FGsxTzFNh 2023-08-27T20:33:04.436 INFO:teuthology.orchestra.run.smithi093.stderr:done 2023-08-27T20:33:04.437 DEBUG:teuthology.orchestra.run.smithi093:> mktemp 2023-08-27T20:33:04.448 DEBUG:teuthology.orchestra.remote:smithi136:/tmp/tmp.8FGsxTzFNh is 6KB 2023-08-27T20:33:04.459 INFO:teuthology.orchestra.run.smithi093.stdout:/tmp/tmp.3Vj6Xu1BlJ 2023-08-27T20:33:04.461 DEBUG:teuthology.orchestra.run.smithi093:> sudo cp /boot/grub2/grub.cfg /tmp/tmp.3Vj6Xu1BlJ 2023-08-27T20:33:04.495 DEBUG:teuthology.orchestra.run.smithi136:> rm -fr /tmp/tmp.8FGsxTzFNh 2023-08-27T20:33:04.512 DEBUG:teuthology.orchestra.run.smithi136:> sudo /bin/ls /boot/loader/entries || true 2023-08-27T20:33:04.533 DEBUG:teuthology.orchestra.run.smithi093:> sudo chmod 0666 /tmp/tmp.3Vj6Xu1BlJ 2023-08-27T20:33:04.585 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-08-27T20:33:04.585 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-08-27T20:33:04.585 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-08-27T20:33:04.585 INFO:teuthology.orchestra.run.smithi136.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-512.el8.x86_64.conf 2023-08-27T20:33:04.586 DEBUG:teuthology.orchestra.run.smithi136:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-512.el8.x86_64 2023-08-27T20:33:04.675 DEBUG:teuthology.orchestra.remote:smithi093:/tmp/tmp.3Vj6Xu1BlJ is 6KB 2023-08-27T20:33:04.723 DEBUG:teuthology.orchestra.run.smithi093:> rm -fr /tmp/tmp.3Vj6Xu1BlJ 2023-08-27T20:33:04.740 DEBUG:teuthology.orchestra.run.smithi093:> sudo /bin/ls /boot/loader/entries || true 2023-08-27T20:33:04.750 DEBUG:teuthology.orchestra.run.smithi136:> sudo shutdown -r now 2023-08-27T20:33:04.815 INFO:teuthology.orchestra.run.smithi093.stdout:a1b29d6c289747bdb22930598d39874a-0-rescue.conf 2023-08-27T20:33:04.815 INFO:teuthology.orchestra.run.smithi093.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-348.el8.x86_64.conf 2023-08-27T20:33:04.816 INFO:teuthology.orchestra.run.smithi093.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-489.el8.x86_64.conf 2023-08-27T20:33:04.816 INFO:teuthology.orchestra.run.smithi093.stdout:a1b29d6c289747bdb22930598d39874a-4.18.0-512.el8.x86_64.conf 2023-08-27T20:33:04.817 DEBUG:teuthology.orchestra.run.smithi093:> sudo grub2-set-default a1b29d6c289747bdb22930598d39874a-4.18.0-512.el8.x86_64 2023-08-27T20:33:04.975 DEBUG:teuthology.orchestra.run.smithi093:> sudo shutdown -r now 2023-08-27T20:33:34.779 INFO:teuthology.task.kernel:Checking client mon.b for new kernel version... 2023-08-27T20:33:34.779 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2023-08-27T20:33:34.780 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:33:34.978 INFO:teuthology.task.kernel:Checking client mon.a for new kernel version... 2023-08-27T20:33:34.979 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi093.front.sepia.ceph.com' 2023-08-27T20:33:34.979 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi093.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:33:53.263 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2023-08-27T20:33:53.519 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.93 2023-08-27T20:34:02.273 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2023-08-27T20:34:02.274 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:34:02.520 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi093.front.sepia.ceph.com' 2023-08-27T20:34:02.520 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi093.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:34:05.355 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2023-08-27T20:34:05.579 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.93 2023-08-27T20:34:17.368 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2023-08-27T20:34:17.369 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:34:17.579 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi093.front.sepia.ceph.com' 2023-08-27T20:34:17.580 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi093.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:34:35.759 DEBUG:teuthology.orchestra.remote:[Errno None] Unable to connect to port 22 on 172.21.15.136 2023-08-27T20:34:49.915 DEBUG:teuthology.orchestra.run.smithi093:> true 2023-08-27T20:34:50.424 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi093.front.sepia.ceph.com' 2023-08-27T20:34:50.424 INFO:teuthology.task.kernel:Checking kernel version of mon.a, want "4.18.0-512.el8.x86_64"... 2023-08-27T20:34:50.424 DEBUG:teuthology.orchestra.run.smithi093:> uname -r 2023-08-27T20:34:50.442 INFO:teuthology.orchestra.run.smithi093.stdout:4.18.0-512.el8.x86_64 2023-08-27T20:34:50.442 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-512.el8.x86_64 vs 4.18.0-512.el8.x86_64 2023-08-27T20:34:50.442 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-08-27T20:34:50.443 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-08-27T20:34:50.948 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2023-08-27T20:34:50.949 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:34:51.443 INFO:teuthology.task.kernel:Enabling kdb on mon.a... 2023-08-27T20:34:51.444 DEBUG:teuthology.orchestra.run.smithi093:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-08-27T20:34:51.512 INFO:teuthology.orchestra.run.smithi093.stdout:ttyS1 2023-08-27T20:34:51.525 DEBUG:teuthology.parallel:result is None 2023-08-27T20:34:51.842 DEBUG:teuthology.orchestra.run.smithi136:> true 2023-08-27T20:34:53.160 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi136.front.sepia.ceph.com' 2023-08-27T20:34:53.160 INFO:teuthology.task.kernel:Checking kernel version of mon.b, want "4.18.0-512.el8.x86_64"... 2023-08-27T20:34:53.160 DEBUG:teuthology.orchestra.run.smithi136:> uname -r 2023-08-27T20:34:53.180 INFO:teuthology.orchestra.run.smithi136.stdout:4.18.0-512.el8.x86_64 2023-08-27T20:34:53.180 DEBUG:teuthology.task.kernel:current kernel version is 4.18.0-512.el8.x86_64 vs 4.18.0-512.el8.x86_64 2023-08-27T20:34:53.180 DEBUG:teuthology.task.kernel:utsrelease strings match, do not need to install 2023-08-27T20:34:53.180 DEBUG:teuthology.task.kernel:Distro of this test job: centos 2023-08-27T20:34:54.182 INFO:teuthology.task.kernel:Enabling kdb on mon.b... 2023-08-27T20:34:54.182 DEBUG:teuthology.orchestra.run.smithi136:> echo ttyS1 | sudo tee /sys/module/kgdboc/parameters/kgdboc 2023-08-27T20:34:54.255 INFO:teuthology.orchestra.run.smithi136.stdout:ttyS1 2023-08-27T20:34:54.266 DEBUG:teuthology.parallel:result is None 2023-08-27T20:34:54.266 INFO:teuthology.run_tasks:Running task internal.base... 2023-08-27T20:34:54.276 INFO:teuthology.task.internal:Creating test directory... 2023-08-27T20:34:54.276 DEBUG:teuthology.orchestra.run.smithi093:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-08-27T20:34:54.278 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest 2023-08-27T20:34:54.299 INFO:teuthology.run_tasks:Running task internal.archive_upload... 2023-08-27T20:34:54.340 INFO:teuthology.run_tasks:Running task internal.archive... 2023-08-27T20:34:54.354 INFO:teuthology.task.internal:Creating archive directory... 2023-08-27T20:34:54.354 DEBUG:teuthology.orchestra.run.smithi093:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-08-27T20:34:54.356 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive 2023-08-27T20:34:54.403 INFO:teuthology.run_tasks:Running task internal.coredump... 2023-08-27T20:34:54.416 INFO:teuthology.task.internal:Enabling coredump saving... 2023-08-27T20:34:54.416 DEBUG:teuthology.orchestra.run.smithi093:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-08-27T20:34:54.422 DEBUG:teuthology.orchestra.run.smithi136:> install -d -m0755 -- /home/ubuntu/cephtest/archive/coredump && sudo sysctl -w kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core && echo kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core | sudo tee -a /etc/sysctl.conf 2023-08-27T20:34:54.459 INFO:teuthology.orchestra.run.smithi093.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T20:34:54.471 INFO:teuthology.orchestra.run.smithi093.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T20:34:54.478 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern = /home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T20:34:54.488 INFO:teuthology.orchestra.run.smithi136.stdout:kernel.core_pattern=/home/ubuntu/cephtest/archive/coredump/%t.%p.core 2023-08-27T20:34:54.490 INFO:teuthology.run_tasks:Running task internal.sudo... 2023-08-27T20:34:54.501 INFO:teuthology.task.internal:Configuring sudo... 2023-08-27T20:34:54.501 DEBUG:teuthology.orchestra.run.smithi093:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-08-27T20:34:54.515 DEBUG:teuthology.orchestra.run.smithi136:> sudo sed -i.orig.teuthology -e 's/^\([^#]*\) \(requiretty\)/\1 !\2/g' -e 's/^\([^#]*\) !\(visiblepw\)/\1 \2/g' /etc/sudoers 2023-08-27T20:34:54.558 INFO:teuthology.run_tasks:Running task internal.syslog... 2023-08-27T20:34:54.570 INFO:teuthology.task.internal.syslog:Starting syslog monitoring... 2023-08-27T20:34:54.570 DEBUG:teuthology.orchestra.run.smithi093:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-08-27T20:34:54.589 DEBUG:teuthology.orchestra.run.smithi136:> mkdir -p -m0755 -- /home/ubuntu/cephtest/archive/syslog 2023-08-27T20:34:54.616 DEBUG:teuthology.orchestra.run.smithi093:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-27T20:34:54.697 DEBUG:teuthology.orchestra.run.smithi093:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-27T20:34:54.754 DEBUG:teuthology.orchestra.run.smithi093:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-27T20:34:54.843 DEBUG:teuthology.orchestra.run.smithi093:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-27T20:34:54.893 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:34:54.894 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-08-27T20:34:54.967 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-27T20:34:55.007 DEBUG:teuthology.orchestra.run.smithi136:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/kern.log 2023-08-27T20:34:55.072 DEBUG:teuthology.orchestra.run.smithi136:> install -m 666 /dev/null /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-27T20:34:55.112 DEBUG:teuthology.orchestra.run.smithi136:> sudo chcon system_u:object_r:var_log_t:s0 /home/ubuntu/cephtest/archive/syslog/misc.log 2023-08-27T20:34:55.159 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:34:55.159 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/rsyslog.d/80-cephtest.conf 2023-08-27T20:34:55.226 DEBUG:teuthology.orchestra.run.smithi093:> sudo service rsyslog restart 2023-08-27T20:34:55.228 DEBUG:teuthology.orchestra.run.smithi136:> sudo service rsyslog restart 2023-08-27T20:34:55.286 INFO:teuthology.orchestra.run.smithi093.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-08-27T20:34:55.320 INFO:teuthology.orchestra.run.smithi136.stderr:Redirecting to /bin/systemctl restart rsyslog.service 2023-08-27T20:34:55.769 INFO:teuthology.run_tasks:Running task internal.timer... 2023-08-27T20:34:55.777 INFO:teuthology.task.internal:Starting timer... 2023-08-27T20:34:55.777 INFO:teuthology.run_tasks:Running task pcp... 2023-08-27T20:34:55.789 INFO:teuthology.run_tasks:Running task selinux... 2023-08-27T20:34:55.799 DEBUG:teuthology.task:Applying overrides for task selinux: {'whitelist': ['scontext=system_u:system_r:logrotate_t:s0']} 2023-08-27T20:34:55.799 DEBUG:teuthology.orchestra.run.smithi093:> sudo service auditd rotate 2023-08-27T20:34:55.897 INFO:teuthology.orchestra.run.smithi093.stdout:Rotating logs: 2023-08-27T20:34:55.898 DEBUG:teuthology.orchestra.run.smithi136:> sudo service auditd rotate 2023-08-27T20:34:55.973 INFO:teuthology.orchestra.run.smithi136.stdout:Rotating logs: 2023-08-27T20:34:55.974 DEBUG:teuthology.task.selinux:Getting current SELinux state 2023-08-27T20:34:55.974 DEBUG:teuthology.orchestra.run.smithi093:> /usr/sbin/getenforce 2023-08-27T20:34:56.005 INFO:teuthology.orchestra.run.smithi093.stdout:Permissive 2023-08-27T20:34:56.006 DEBUG:teuthology.orchestra.run.smithi136:> /usr/sbin/getenforce 2023-08-27T20:34:56.071 INFO:teuthology.orchestra.run.smithi136.stdout:Permissive 2023-08-27T20:34:56.071 DEBUG:teuthology.task.selinux:Existing SELinux modes: {'ubuntu@smithi093.front.sepia.ceph.com': 'permissive', 'ubuntu@smithi136.front.sepia.ceph.com': 'permissive'} 2023-08-27T20:34:56.072 DEBUG:teuthology.orchestra.run.smithi093:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-08-27T20:34:56.100 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-27T20:34:56.100 DEBUG:teuthology.orchestra.run.smithi136:> sudo grep -a 'avc: .*denied' /var/log/audit/audit.log | grep -av -e 'comm="dmidecode"' -e chronyd.service -e 'name="cephtest"' -e scontext=system_u:system_r:nrpe_t:s0 -e scontext=system_u:system_r:pcp_pmlogger_t -e scontext=system_u:system_r:pcp_pmcd_t:s0 -e 'comm="rhsmd"' -e scontext=system_u:system_r:syslogd_t:s0 -e tcontext=system_u:system_r:nrpe_t:s0 -e 'comm="updatedb"' -e 'comm="smartd"' -e 'comm="rhsmcertd-worke"' -e 'comm="setroubleshootd"' -e 'comm="rpm"' -e tcontext=system_u:object_r:container_runtime_exec_t:s0 -e 'comm="ksmtuned"' -e 'comm="sssd"' -e 'comm="sss_cache"' -e context=system_u:system_r:NetworkManager_dispatcher_t:s0 2023-08-27T20:34:56.127 DEBUG:teuthology.orchestra.run:got remote process result: 1 2023-08-27T20:34:56.128 INFO:teuthology.task.selinux:Putting SELinux into permissive mode 2023-08-27T20:34:56.128 DEBUG:teuthology.orchestra.run.smithi093:> sudo /usr/sbin/setenforce permissive 2023-08-27T20:34:56.169 DEBUG:teuthology.orchestra.run.smithi136:> sudo /usr/sbin/setenforce permissive 2023-08-27T20:34:56.197 INFO:teuthology.run_tasks:Running task ansible.cephlab... 2023-08-27T20:34:56.208 INFO:teuthology.repo_utils:/home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main was just updated or references a specific commit; assuming it is current 2023-08-27T20:34:56.208 INFO:teuthology.repo_utils:Resetting repo at /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main to origin/main 2023-08-27T20:34:56.226 INFO:teuthology.task.ansible:Playbook: [{'import_playbook': 'ansible_managed.yml'}, {'import_playbook': 'teuthology.yml'}, {'hosts': 'testnodes', 'tasks': [{'set_fact': {'ran_from_cephlab_playbook': True}}]}, {'import_playbook': 'testnodes.yml'}, {'import_playbook': 'container-host.yml'}, {'import_playbook': 'cobbler.yml'}, {'import_playbook': 'paddles.yml'}, {'import_playbook': 'pulpito.yml'}, {'hosts': 'testnodes', 'become': True, 'tasks': [{'name': 'Touch /ceph-qa-ready', 'file': {'path': '/ceph-qa-ready', 'state': 'touch'}, 'when': 'ran_from_cephlab_playbook|bool'}]}] 2023-08-27T20:34:56.227 DEBUG:teuthology.task.ansible:Running ansible-playbook -v --extra-vars '{"ansible_ssh_user": "ubuntu"}' -i /etc/ansible/hosts --limit smithi093.front.sepia.ceph.com,smithi136.front.sepia.ceph.com /home/teuthworker/src/git.ceph.com_ceph-cm-ansible_main/cephlab.yml 2023-08-27T20:42:09.006 DEBUG:teuthology.task.ansible:Reconnecting to [Remote(name='ubuntu@smithi093.front.sepia.ceph.com'), Remote(name='ubuntu@smithi136.front.sepia.ceph.com')] 2023-08-27T20:42:09.007 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi093.front.sepia.ceph.com' 2023-08-27T20:42:09.007 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi093.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:42:09.077 DEBUG:teuthology.orchestra.run.smithi093:> true 2023-08-27T20:42:09.146 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi093.front.sepia.ceph.com' 2023-08-27T20:42:09.147 INFO:teuthology.orchestra.remote:Trying to reconnect to host 'ubuntu@smithi136.front.sepia.ceph.com' 2023-08-27T20:42:09.147 DEBUG:teuthology.orchestra.connection:{'hostname': 'smithi136.front.sepia.ceph.com', 'username': 'ubuntu', 'timeout': 60} 2023-08-27T20:42:09.213 DEBUG:teuthology.orchestra.run.smithi136:> true 2023-08-27T20:42:09.283 INFO:teuthology.orchestra.remote:Successfully reconnected to host 'ubuntu@smithi136.front.sepia.ceph.com' 2023-08-27T20:42:09.283 INFO:teuthology.run_tasks:Running task clock... 2023-08-27T20:42:09.292 INFO:teuthology.task.clock:Syncing clocks and checking initial clock skew... 2023-08-27T20:42:09.293 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-08-27T20:42:09.293 DEBUG:teuthology.orchestra.run.smithi093:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-08-27T20:42:09.295 INFO:teuthology.orchestra.run:Running command with timeout 360 2023-08-27T20:42:09.296 DEBUG:teuthology.orchestra.run.smithi136:> sudo systemctl stop ntp.service || sudo systemctl stop ntpd.service || sudo systemctl stop chronyd.service ; sudo ntpd -gq || sudo chronyc makestep ; sudo systemctl start ntp.service || sudo systemctl start ntpd.service || sudo systemctl start chronyd.service ; PATH=/usr/bin:/usr/sbin ntpq -p || PATH=/usr/bin:/usr/sbin chronyc sources || true 2023-08-27T20:42:09.326 INFO:teuthology.orchestra.run.smithi093.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-08-27T20:42:09.341 INFO:teuthology.orchestra.run.smithi093.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-08-27T20:42:09.356 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to stop ntp.service: Unit ntp.service not loaded. 2023-08-27T20:42:09.370 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to stop ntpd.service: Unit ntpd.service not loaded. 2023-08-27T20:42:09.406 INFO:teuthology.orchestra.run.smithi093.stderr:sudo: ntpd: command not found 2023-08-27T20:42:09.418 INFO:teuthology.orchestra.run.smithi093.stdout:506 Cannot talk to daemon 2023-08-27T20:42:09.432 INFO:teuthology.orchestra.run.smithi136.stderr:sudo: ntpd: command not found 2023-08-27T20:42:09.432 INFO:teuthology.orchestra.run.smithi093.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-08-27T20:42:09.443 INFO:teuthology.orchestra.run.smithi136.stdout:506 Cannot talk to daemon 2023-08-27T20:42:09.445 INFO:teuthology.orchestra.run.smithi093.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-08-27T20:42:09.457 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to start ntp.service: Unit ntp.service not found. 2023-08-27T20:42:09.471 INFO:teuthology.orchestra.run.smithi136.stderr:Failed to start ntpd.service: Unit ntpd.service not found. 2023-08-27T20:42:09.522 INFO:teuthology.orchestra.run.smithi093.stderr:bash: ntpq: command not found 2023-08-27T20:42:09.526 INFO:teuthology.orchestra.run.smithi093.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-08-27T20:42:09.526 INFO:teuthology.orchestra.run.smithi093.stdout:=============================================================================== 2023-08-27T20:42:09.526 INFO:teuthology.orchestra.run.smithi093.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.526 INFO:teuthology.orchestra.run.smithi093.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.526 INFO:teuthology.orchestra.run.smithi093.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.526 INFO:teuthology.orchestra.run.smithi093.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.549 INFO:teuthology.orchestra.run.smithi136.stderr:bash: ntpq: command not found 2023-08-27T20:42:09.552 INFO:teuthology.orchestra.run.smithi136.stdout:MS Name/IP address Stratum Poll Reach LastRx Last sample 2023-08-27T20:42:09.553 INFO:teuthology.orchestra.run.smithi136.stdout:=============================================================================== 2023-08-27T20:42:09.553 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv01.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.553 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv02.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.553 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv03.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.553 INFO:teuthology.orchestra.run.smithi136.stdout:^? hv04.front.sepia.ceph.com 0 6 0 - +0ns[ +0ns] +/- 0ns 2023-08-27T20:42:09.554 INFO:teuthology.run_tasks:Running task pexec... 2023-08-27T20:42:09.563 INFO:teuthology.task.pexec:Executing custom commands... 2023-08-27T20:42:09.563 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi093.front.sepia.ceph.com 2023-08-27T20:42:09.563 DEBUG:teuthology.orchestra.run.smithi093:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-08-27T20:42:09.564 INFO:teuthology.task.pexec:Running commands on host ubuntu@smithi136.front.sepia.ceph.com 2023-08-27T20:42:09.564 DEBUG:teuthology.orchestra.run.smithi136:> TESTDIR=/home/ubuntu/cephtest bash -s 2023-08-27T20:42:10.443 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:35 ago on Sun 27 Aug 2023 08:40:35 PM UTC. 2023-08-27T20:42:10.467 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:35 ago on Sun 27 Aug 2023 08:40:35 PM UTC. 2023-08-27T20:42:12.321 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout: Package Architecture Version Repository Size 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout:Resetting modules: 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout: container-tools 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:42:12.322 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:12.323 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:12.346 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:42:12.347 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:12.347 INFO:teuthology.orchestra.run.smithi136.stdout: Package Architecture Version Repository Size 2023-08-27T20:42:12.347 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:12.347 INFO:teuthology.orchestra.run.smithi136.stdout:Resetting modules: 2023-08-27T20:42:12.347 INFO:teuthology.orchestra.run.smithi136.stdout: container-tools 2023-08-27T20:42:12.347 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:12.348 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:42:12.348 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:12.348 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:12.570 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:42:12.604 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:42:13.549 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:38 ago on Sun 27 Aug 2023 08:40:35 PM UTC. 2023-08-27T20:42:13.587 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:38 ago on Sun 27 Aug 2023 08:40:35 PM UTC. 2023-08-27T20:42:15.421 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:42:15.460 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout:Installing group/module packages: 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout: buildah x86_64 1:1.31.2-1.module_el8+643+8db347f4 appstream 8.8 M 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout: cockpit-podman noarch 74-1.module_el8+643+8db347f4 appstream 741 k 2023-08-27T20:42:15.566 INFO:teuthology.orchestra.run.smithi093.stdout: crun x86_64 1.8.6-1.module_el8+593+980a18f9 appstream 239 k 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: skopeo x86_64 2:1.13.2-1.module_el8+643+8db347f4 appstream 8.1 M 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout:Installing weak dependencies: 2023-08-27T20:42:15.567 INFO:teuthology.orchestra.run.smithi093.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout:Installing module profiles: 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout: container-tools/common 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout:Enabling module streams: 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout: container-tools rhel8 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout:Install 13 Packages 2023-08-27T20:42:15.568 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:15.569 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 26 M 2023-08-27T20:42:15.569 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 87 M 2023-08-27T20:42:15.569 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:42:15.604 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:15.604 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:42:15.604 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:15.604 INFO:teuthology.orchestra.run.smithi136.stdout:Installing group/module packages: 2023-08-27T20:42:15.604 INFO:teuthology.orchestra.run.smithi136.stdout: aardvark-dns x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 1.0 M 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: buildah x86_64 1:1.31.2-1.module_el8+643+8db347f4 appstream 8.8 M 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: cockpit-podman noarch 74-1.module_el8+643+8db347f4 appstream 741 k 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: crun x86_64 1.8.6-1.module_el8+593+980a18f9 appstream 239 k 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: netavark x86_64 2:1.7.0-1.module_el8+487+8e42a277 appstream 3.7 M 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: python3-podman noarch 4.6.0-1.module_el8+582+a2558ae2 appstream 177 k 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: skopeo x86_64 2:1.13.2-1.module_el8+643+8db347f4 appstream 8.1 M 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: toolbox x86_64 0.0.99.4-5.module_el8+649+e2deac1d appstream 2.5 M 2023-08-27T20:42:15.605 INFO:teuthology.orchestra.run.smithi136.stdout: udica noarch 0.2.6-20.module_el8+487+8e42a277 appstream 53 k 2023-08-27T20:42:15.606 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:42:15.606 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pytoml noarch 0.1.14-5.git7dea353.el8 appstream 25 k 2023-08-27T20:42:15.606 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyxdg noarch 0.25-16.el8 appstream 94 k 2023-08-27T20:42:15.606 INFO:teuthology.orchestra.run.smithi136.stdout: yajl x86_64 2.1.0-12.el8 appstream 41 k 2023-08-27T20:42:15.606 INFO:teuthology.orchestra.run.smithi136.stdout:Installing weak dependencies: 2023-08-27T20:42:15.606 INFO:teuthology.orchestra.run.smithi136.stdout: criu-libs x86_64 3.15-3.module_el8.7.0+1216+b022c01d appstream 38 k 2023-08-27T20:42:15.606 INFO:teuthology.orchestra.run.smithi136.stdout:Installing module profiles: 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout: container-tools/common 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout:Enabling module streams: 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout: container-tools rhel8 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:15.607 INFO:teuthology.orchestra.run.smithi136.stdout:Install 13 Packages 2023-08-27T20:42:15.608 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:15.608 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 26 M 2023-08-27T20:42:15.609 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 87 M 2023-08-27T20:42:15.609 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:42:16.276 INFO:teuthology.orchestra.run.smithi093.stdout:(1/13): cockpit-podman-74-1.module_el8+643+8db3 1.2 MB/s | 741 kB 00:00 2023-08-27T20:42:16.310 INFO:teuthology.orchestra.run.smithi093.stdout:(2/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.5 MB/s | 1.0 MB 00:00 2023-08-27T20:42:16.335 INFO:teuthology.orchestra.run.smithi136.stdout:(1/13): cockpit-podman-74-1.module_el8+643+8db3 1.2 MB/s | 741 kB 00:00 2023-08-27T20:42:16.368 INFO:teuthology.orchestra.run.smithi136.stdout:(2/13): aardvark-dns-1.7.0-1.module_el8+487+8e4 1.5 MB/s | 1.0 MB 00:00 2023-08-27T20:42:16.385 INFO:teuthology.orchestra.run.smithi093.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 349 kB/s | 38 kB 00:00 2023-08-27T20:42:16.410 INFO:teuthology.orchestra.run.smithi093.stdout:(4/13): crun-1.8.6-1.module_el8+593+980a18f9.x8 2.3 MB/s | 239 kB 00:00 2023-08-27T20:42:16.435 INFO:teuthology.orchestra.run.smithi136.stdout:(3/13): criu-libs-3.15-3.module_el8.7.0+1216+b0 378 kB/s | 38 kB 00:00 2023-08-27T20:42:16.485 INFO:teuthology.orchestra.run.smithi136.stdout:(4/13): crun-1.8.6-1.module_el8+593+980a18f9.x8 2.0 MB/s | 239 kB 00:00 2023-08-27T20:42:16.519 INFO:teuthology.orchestra.run.smithi093.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 1.6 MB/s | 177 kB 00:00 2023-08-27T20:42:16.594 INFO:teuthology.orchestra.run.smithi136.stdout:(5/13): python3-podman-4.6.0-1.module_el8+582+a 1.6 MB/s | 177 kB 00:00 2023-08-27T20:42:16.619 INFO:teuthology.orchestra.run.smithi093.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 248 kB/s | 25 kB 00:00 2023-08-27T20:42:16.686 INFO:teuthology.orchestra.run.smithi136.stdout:(6/13): python3-pytoml-0.1.14-5.git7dea353.el8. 270 kB/s | 25 kB 00:00 2023-08-27T20:42:16.770 INFO:teuthology.orchestra.run.smithi136.stdout:(7/13): netavark-1.7.0-1.module_el8+487+8e42a27 11 MB/s | 3.7 MB 00:00 2023-08-27T20:42:16.795 INFO:teuthology.orchestra.run.smithi093.stdout:(7/13): buildah-1.31.2-1.module_el8+643+8db347f 7.8 MB/s | 8.8 MB 00:01 2023-08-27T20:42:16.795 INFO:teuthology.orchestra.run.smithi136.stdout:(8/13): python3-pyxdg-0.25-16.el8.noarch.rpm 863 kB/s | 94 kB 00:00 2023-08-27T20:42:16.845 INFO:teuthology.orchestra.run.smithi093.stdout:(8/13): netavark-1.7.0-1.module_el8+487+8e42a27 8.2 MB/s | 3.7 MB 00:00 2023-08-27T20:42:16.870 INFO:teuthology.orchestra.run.smithi093.stdout:(9/13): python3-pyxdg-0.25-16.el8.noarch.rpm 374 kB/s | 94 kB 00:00 2023-08-27T20:42:16.895 INFO:teuthology.orchestra.run.smithi136.stdout:(9/13): buildah-1.31.2-1.module_el8+643+8db347f 7.5 MB/s | 8.8 MB 00:01 2023-08-27T20:42:16.996 INFO:teuthology.orchestra.run.smithi093.stdout:(10/13): udica-0.2.6-20.module_el8+487+8e42a277 421 kB/s | 53 kB 00:00 2023-08-27T20:42:16.996 INFO:teuthology.orchestra.run.smithi136.stdout:(10/13): udica-0.2.6-20.module_el8+487+8e42a277 527 kB/s | 53 kB 00:00 2023-08-27T20:42:17.038 INFO:teuthology.orchestra.run.smithi093.stdout:(11/13): toolbox-0.0.99.4-5.module_el8+649+e2de 13 MB/s | 2.5 MB 00:00 2023-08-27T20:42:17.071 INFO:teuthology.orchestra.run.smithi136.stdout:(11/13): toolbox-0.0.99.4-5.module_el8+649+e2de 9.2 MB/s | 2.5 MB 00:00 2023-08-27T20:42:17.088 INFO:teuthology.orchestra.run.smithi093.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 445 kB/s | 41 kB 00:00 2023-08-27T20:42:17.105 INFO:teuthology.orchestra.run.smithi136.stdout:(12/13): yajl-2.1.0-12.el8.x86_64.rpm 377 kB/s | 41 kB 00:00 2023-08-27T20:42:17.205 INFO:teuthology.orchestra.run.smithi136.stdout:(13/13): skopeo-1.13.2-1.module_el8+643+8db347f 19 MB/s | 8.1 MB 00:00 2023-08-27T20:42:17.205 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:42:17.205 INFO:teuthology.orchestra.run.smithi136.stdout:Total 16 MB/s | 26 MB 00:01 2023-08-27T20:42:17.322 INFO:teuthology.orchestra.run.smithi093.stdout:(13/13): skopeo-1.13.2-1.module_el8+643+8db347f 15 MB/s | 8.1 MB 00:00 2023-08-27T20:42:17.322 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:42:17.322 INFO:teuthology.orchestra.run.smithi093.stdout:Total 15 MB/s | 26 MB 00:01 2023-08-27T20:42:17.466 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:42:17.487 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:42:17.487 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:42:17.586 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:42:17.607 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:42:17.607 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:42:17.743 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:42:17.746 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:42:17.853 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:42:17.856 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:42:18.435 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:42:18.554 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:42:18.660 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-08-27T20:42:18.770 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : yajl-2.1.0-12.el8.x86_64 1/13 2023-08-27T20:42:18.831 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-08-27T20:42:18.957 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pyxdg-0.25-16.el8.noarch 2/13 2023-08-27T20:42:19.073 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-08-27T20:42:19.159 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 3/13 2023-08-27T20:42:19.283 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-08-27T20:42:19.415 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-08-27T20:42:19.808 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-08-27T20:42:19.991 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 5/13 2023-08-27T20:42:20.763 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-08-27T20:42:20.958 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-08-27T20:42:20.997 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : buildah-1:1.31.2-1.module_el8+643+8db347f4.x86_64 7/13 2023-08-27T20:42:21.215 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : buildah-1:1.31.2-1.module_el8+643+8db347f4.x86_64 7/13 2023-08-27T20:42:21.245 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : crun-1.8.6-1.module_el8+593+980a18f9.x86_64 8/13 2023-08-27T20:42:21.421 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : crun-1.8.6-1.module_el8+593+980a18f9.x86_64 8/13 2023-08-27T20:42:21.493 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-08-27T20:42:21.669 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 9/13 2023-08-27T20:42:21.912 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-08-27T20:42:22.087 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : udica-0.2.6-20.module_el8+487+8e42a277.noarch 10/13 2023-08-27T20:42:22.807 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-08-27T20:42:22.959 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-08-27T20:42:22.997 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : skopeo-2:1.13.2-1.module_el8+643+8db347f4.x86_64 12/13 2023-08-27T20:42:23.146 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : cockpit-podman-74-1.module_el8+643+8db347f4.noarch 13/13 2023-08-27T20:42:23.214 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : skopeo-2:1.13.2-1.module_el8+643+8db347f4.x86_64 12/13 2023-08-27T20:42:23.397 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : cockpit-podman-74-1.module_el8+643+8db347f4.noarch 13/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: cockpit-podman-74-1.module_el8+643+8db347f4.noarch 13/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : buildah-1:1.31.2-1.module_el8+643+8db347f4.x86_64 2/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : cockpit-podman-74-1.module_el8+643+8db347f4.noarch 3/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : crun-1.8.6-1.module_el8+593+980a18f9.x86_64 5/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-08-27T20:42:23.575 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-08-27T20:42:23.576 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-08-27T20:42:23.576 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : skopeo-2:1.13.2-1.module_el8+643+8db347f4.x86_64 10/13 2023-08-27T20:42:23.576 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-08-27T20:42:23.576 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: cockpit-podman-74-1.module_el8+643+8db347f4.noarch 13/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86 1/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : buildah-1:1.31.2-1.module_el8+643+8db347f4.x86_64 2/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : cockpit-podman-74-1.module_el8+643+8db347f4.noarch 3/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_ 4/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : crun-1.8.6-1.module_el8+593+980a18f9.x86_64 5/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 6/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-podman-4.6.0-1.module_el8+582+a2558ae2.noa 7/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pytoml-0.1.14-5.git7dea353.el8.noarch 8/13 2023-08-27T20:42:23.822 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pyxdg-0.25-16.el8.noarch 9/13 2023-08-27T20:42:23.823 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : skopeo-2:1.13.2-1.module_el8+643+8db347f4.x86_64 10/13 2023-08-27T20:42:23.823 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 11/13 2023-08-27T20:42:23.823 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : udica-0.2.6-20.module_el8+487+8e42a277.noarch 12/13 2023-08-27T20:42:24.332 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-08-27T20:42:24.332 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:24.332 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:42:24.332 INFO:teuthology.orchestra.run.smithi136.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-08-27T20:42:24.332 INFO:teuthology.orchestra.run.smithi136.stdout: buildah-1:1.31.2-1.module_el8+643+8db347f4.x86_64 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: cockpit-podman-74-1.module_el8+643+8db347f4.noarch 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: crun-1.8.6-1.module_el8+593+980a18f9.x86_64 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: skopeo-2:1.13.2-1.module_el8+643+8db347f4.x86_64 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-08-27T20:42:24.333 INFO:teuthology.orchestra.run.smithi136.stdout: yajl-2.1.0-12.el8.x86_64 2023-08-27T20:42:24.334 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:24.334 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:42:24.625 DEBUG:teuthology.parallel:result is None 2023-08-27T20:42:24.633 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : yajl-2.1.0-12.el8.x86_64 13/13 2023-08-27T20:42:24.633 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:24.633 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: aardvark-dns-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: buildah-1:1.31.2-1.module_el8+643+8db347f4.x86_64 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: cockpit-podman-74-1.module_el8+643+8db347f4.noarch 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: criu-libs-3.15-3.module_el8.7.0+1216+b022c01d.x86_64 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: crun-1.8.6-1.module_el8+593+980a18f9.x86_64 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: netavark-2:1.7.0-1.module_el8+487+8e42a277.x86_64 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: python3-podman-4.6.0-1.module_el8+582+a2558ae2.noarch 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pytoml-0.1.14-5.git7dea353.el8.noarch 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyxdg-0.25-16.el8.noarch 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: skopeo-2:1.13.2-1.module_el8+643+8db347f4.x86_64 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: toolbox-0.0.99.4-5.module_el8+649+e2deac1d.x86_64 2023-08-27T20:42:24.634 INFO:teuthology.orchestra.run.smithi093.stdout: udica-0.2.6-20.module_el8+487+8e42a277.noarch 2023-08-27T20:42:24.635 INFO:teuthology.orchestra.run.smithi093.stdout: yajl-2.1.0-12.el8.x86_64 2023-08-27T20:42:24.635 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:24.635 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:42:24.957 DEBUG:teuthology.parallel:result is None 2023-08-27T20:42:24.957 INFO:teuthology.run_tasks:Running task install... 2023-08-27T20:42:24.966 DEBUG:teuthology.task.install:project ceph 2023-08-27T20:42:24.967 DEBUG:teuthology.task.install:INSTALL overrides: {'ceph': {'flavor': 'default', 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'}} 2023-08-27T20:42:24.967 DEBUG:teuthology.task.install:config {'branch': 'quincy', 'exclude_packages': ['ceph-volume'], 'flavor': 'default', 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'} 2023-08-27T20:42:24.967 INFO:teuthology.task.install:Using flavor: default 2023-08-27T20:42:24.972 DEBUG:teuthology.task.install:Package list is: {'deb': ['ceph', 'cephadm', 'ceph-mds', 'ceph-mgr', 'ceph-common', 'ceph-fuse', 'ceph-test', 'radosgw', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'libcephfs2', 'libcephfs-dev', 'librados2', 'librbd1', 'rbd-fuse'], 'rpm': ['ceph-radosgw', 'ceph-test', 'ceph', 'ceph-base', 'cephadm', 'ceph-immutable-object-cache', 'ceph-mgr', 'ceph-mgr-dashboard', 'ceph-mgr-diskprediction-local', 'ceph-mgr-rook', 'ceph-mgr-cephadm', 'ceph-fuse', 'librados-devel', 'libcephfs2', 'libcephfs-devel', 'librados2', 'librbd1', 'python3-rados', 'python3-rgw', 'python3-cephfs', 'python3-rbd', 'rbd-fuse', 'rbd-mirror', 'rbd-nbd']} 2023-08-27T20:42:24.973 INFO:teuthology.task.install:extra packages: [] 2023-08-27T20:42:24.974 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51', 'tag': None, 'wait_for_package': False} 2023-08-27T20:42:24.974 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T20:42:24.974 INFO:teuthology.packaging:ref: None 2023-08-27T20:42:24.974 INFO:teuthology.packaging:tag: None 2023-08-27T20:42:24.974 INFO:teuthology.packaging:branch: quincy 2023-08-27T20:42:24.974 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:42:24.974 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-08-27T20:42:24.976 DEBUG:teuthology.task.install.rpm:_update_package_list_and_install: config is {'branch': 'quincy', 'cleanup': None, 'debuginfo': None, 'downgrade_packages': [], 'exclude_packages': ['ceph-volume'], 'extra_packages': [], 'extra_system_packages': [], 'extras': None, 'enable_coprs': [], 'flavor': 'default', 'install_ceph_packages': True, 'packages': {}, 'project': 'ceph', 'repos_only': False, 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51', 'tag': None, 'wait_for_package': False} 2023-08-27T20:42:24.976 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T20:42:24.976 INFO:teuthology.packaging:ref: None 2023-08-27T20:42:24.976 INFO:teuthology.packaging:tag: None 2023-08-27T20:42:24.976 INFO:teuthology.packaging:branch: quincy 2023-08-27T20:42:24.977 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:42:24.977 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-08-27T20:42:25.281 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/ 2023-08-27T20:42:25.281 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1016.g4fe39952 2023-08-27T20:42:25.299 INFO:teuthology.task.install.rpm:Pulling from https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/ 2023-08-27T20:42:25.299 INFO:teuthology.task.install.rpm:Package version is 17.2.6-1016.g4fe39952 2023-08-27T20:42:25.416 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-08-27T20:42:25.417 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:42:25.417 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-08-27T20:42:25.447 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-08-27T20:42:25.447 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T20:42:25.448 INFO:teuthology.packaging:ref: None 2023-08-27T20:42:25.448 INFO:teuthology.packaging:tag: None 2023-08-27T20:42:25.448 INFO:teuthology.packaging:branch: quincy 2023-08-27T20:42:25.448 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:42:25.448 DEBUG:teuthology.orchestra.run.smithi136:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-08-27T20:42:25.448 INFO:teuthology.packaging:Writing yum repo: [ceph] name=ceph packages for $basearch baseurl=https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/$basearch enabled=1 gpgcheck=0 type=rpm-md [ceph-noarch] name=ceph noarch packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/noarch enabled=1 gpgcheck=0 type=rpm-md [ceph-source] name=ceph source packages baseurl=https://3.chacra.ceph.com/r/ceph/quincy/4fe39952b691f82409c13a09ef1217f15a8c4218/centos/8/flavors/default/SRPMS enabled=1 gpgcheck=0 type=rpm-md 2023-08-27T20:42:25.449 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:42:25.449 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/etc/yum.repos.d/ceph.repo 2023-08-27T20:42:25.476 INFO:teuthology.task.install.rpm:Installing packages: ceph-radosgw, ceph-test, ceph, ceph-base, cephadm, ceph-immutable-object-cache, ceph-mgr, ceph-mgr-dashboard, ceph-mgr-diskprediction-local, ceph-mgr-rook, ceph-mgr-cephadm, ceph-fuse, librados-devel, libcephfs2, libcephfs-devel, librados2, librbd1, python3-rados, python3-rgw, python3-cephfs, python3-rbd, rbd-fuse, rbd-mirror, rbd-nbd on remote rpm x86_64 2023-08-27T20:42:25.477 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T20:42:25.477 INFO:teuthology.packaging:ref: None 2023-08-27T20:42:25.477 INFO:teuthology.packaging:tag: None 2023-08-27T20:42:25.477 INFO:teuthology.packaging:branch: quincy 2023-08-27T20:42:25.477 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:42:25.477 DEBUG:teuthology.orchestra.run.smithi093:> if test -f /etc/yum.repos.d/ceph.repo ; then sudo sed -i -e ':a;N;$!ba;s/enabled=1\ngpg/enabled=1\npriority=1\ngpg/g' -e 's;ref/[a-zA-Z0-9_-]*/;ref/quincy/;g' /etc/yum.repos.d/ceph.repo ; fi 2023-08-27T20:42:25.517 DEBUG:teuthology.orchestra.run.smithi136:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-08-27T20:42:25.545 DEBUG:teuthology.orchestra.run.smithi093:> sudo touch -a /etc/yum/pluginconf.d/priorities.conf ; test -e /etc/yum/pluginconf.d/priorities.conf.orig || sudo cp -af /etc/yum/pluginconf.d/priorities.conf /etc/yum/pluginconf.d/priorities.conf.orig 2023-08-27T20:42:25.595 DEBUG:teuthology.orchestra.run.smithi136:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-08-27T20:42:25.622 DEBUG:teuthology.orchestra.run.smithi093:> grep check_obsoletes /etc/yum/pluginconf.d/priorities.conf && sudo sed -i 's/check_obsoletes.*0/check_obsoletes = 1/g' /etc/yum/pluginconf.d/priorities.conf || echo 'check_obsoletes = 1' | sudo tee -a /etc/yum/pluginconf.d/priorities.conf 2023-08-27T20:42:25.662 INFO:teuthology.orchestra.run.smithi136.stdout:check_obsoletes = 1 2023-08-27T20:42:25.663 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum clean all 2023-08-27T20:42:25.688 INFO:teuthology.orchestra.run.smithi093.stdout:check_obsoletes = 1 2023-08-27T20:42:25.689 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum clean all 2023-08-27T20:42:25.937 INFO:teuthology.orchestra.run.smithi136.stdout:76 files removed 2023-08-27T20:42:25.959 INFO:teuthology.orchestra.run.smithi093.stdout:76 files removed 2023-08-27T20:42:25.968 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-radosgw 2023-08-27T20:42:26.005 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-radosgw 2023-08-27T20:42:26.674 INFO:teuthology.orchestra.run.smithi136.stdout:ceph packages for x86_64 263 kB/s | 78 kB 00:00 2023-08-27T20:42:26.705 INFO:teuthology.orchestra.run.smithi093.stdout:ceph packages for x86_64 268 kB/s | 78 kB 00:00 2023-08-27T20:42:26.964 INFO:teuthology.orchestra.run.smithi093.stdout:ceph noarch packages 41 kB/s | 9.2 kB 00:00 2023-08-27T20:42:27.000 INFO:teuthology.orchestra.run.smithi136.stdout:ceph noarch packages 32 kB/s | 9.2 kB 00:00 2023-08-27T20:42:27.267 INFO:teuthology.orchestra.run.smithi136.stdout:ceph source packages 6.1 kB/s | 1.4 kB 00:00 2023-08-27T20:42:27.282 INFO:teuthology.orchestra.run.smithi093.stdout:ceph source packages 5.0 kB/s | 1.4 kB 00:00 2023-08-27T20:42:29.250 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - AppStream 17 MB/s | 33 MB 00:01 2023-08-27T20:42:29.698 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - AppStream 14 MB/s | 33 MB 00:02 2023-08-27T20:42:37.393 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - BaseOS 22 MB/s | 46 MB 00:02 2023-08-27T20:42:37.941 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - BaseOS 21 MB/s | 46 MB 00:02 2023-08-27T20:42:44.551 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - Extras 34 kB/s | 18 kB 00:00 2023-08-27T20:42:45.001 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - Extras common packages 25 kB/s | 6.8 kB 00:00 2023-08-27T20:42:45.341 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - Extras 27 kB/s | 18 kB 00:00 2023-08-27T20:42:45.627 INFO:teuthology.orchestra.run.smithi136.stdout:CentOS Stream 8 - PowerTools 14 MB/s | 6.5 MB 00:00 2023-08-27T20:42:45.892 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - Extras common packages 18 kB/s | 6.8 kB 00:00 2023-08-27T20:42:46.687 INFO:teuthology.orchestra.run.smithi093.stdout:CentOS Stream 8 - PowerTools 10 MB/s | 6.5 MB 00:00 2023-08-27T20:42:47.029 INFO:teuthology.orchestra.run.smithi136.stdout:Copr repo for python3-asyncssh owned by ceph 29 kB/s | 3.5 kB 00:00 2023-08-27T20:42:48.130 INFO:teuthology.orchestra.run.smithi093.stdout:Copr repo for python3-asyncssh owned by ceph 27 kB/s | 3.5 kB 00:00 2023-08-27T20:42:48.688 INFO:teuthology.orchestra.run.smithi136.stdout:Extra Packages for Enterprise Linux 10 MB/s | 16 MB 00:01 2023-08-27T20:42:49.366 INFO:teuthology.orchestra.run.smithi093.stdout:Extra Packages for Enterprise Linux 15 MB/s | 16 MB 00:01 2023-08-27T20:42:52.663 INFO:teuthology.orchestra.run.smithi136.stdout:lab-extras 142 kB/s | 24 kB 00:00 2023-08-27T20:42:53.181 INFO:teuthology.orchestra.run.smithi093.stdout:lab-extras 545 kB/s | 24 kB 00:00 2023-08-27T20:42:56.968 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:42:56.971 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:56.971 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:42:56.971 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:56.971 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:42:56.971 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-radosgw x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 17 M 2023-08-27T20:42:56.971 INFO:teuthology.orchestra.run.smithi136.stdout:Upgrading: 2023-08-27T20:42:56.971 INFO:teuthology.orchestra.run.smithi136.stdout: librados2 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.5 M 2023-08-27T20:42:56.972 INFO:teuthology.orchestra.run.smithi136.stdout: librbd1 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.6 M 2023-08-27T20:42:56.972 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:42:56.972 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-base x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 5.2 M 2023-08-27T20:42:56.972 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-common x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 23 M 2023-08-27T20:42:56.972 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-selinux x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 24 k 2023-08-27T20:42:56.972 INFO:teuthology.orchestra.run.smithi136.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-08-27T20:42:56.972 INFO:teuthology.orchestra.run.smithi136.stdout: libcephfs2 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 796 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: libradosstriper1 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 506 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: librgw2 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 5.5 M 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: python3-ceph-argparse x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 45 k 2023-08-27T20:42:56.973 INFO:teuthology.orchestra.run.smithi136.stdout: python3-ceph-common x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 115 k 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cephfs x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 190 k 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rados x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 370 k 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rbd x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 358 k 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rgw x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 107 k 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:56.974 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:42:56.975 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:42:56.975 INFO:teuthology.orchestra.run.smithi136.stdout:Install 21 Packages 2023-08-27T20:42:56.975 INFO:teuthology.orchestra.run.smithi136.stdout:Upgrade 2 Packages 2023-08-27T20:42:56.975 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:42:56.975 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 63 M 2023-08-27T20:42:56.975 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:42:57.411 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:42:57.414 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:57.414 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:42:57.414 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:57.414 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-radosgw x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 17 M 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout:Upgrading: 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: librados2 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.5 M 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: librbd1 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.6 M 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-base x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 5.2 M 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-common x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 23 M 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-selinux x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 24 k 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: gperftools-libs x86_64 1:2.7-9.el8 epel 306 k 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: libcephfs2 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 796 k 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: liboath x86_64 2.6.2-3.el8 epel 59 k 2023-08-27T20:42:57.415 INFO:teuthology.orchestra.run.smithi093.stdout: librabbitmq x86_64 0.9.0-4.el8 baseos 47 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: libradosstriper1 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 506 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: librdkafka x86_64 0.11.4-3.el8 appstream 354 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: librgw2 x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 5.5 M 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: libunwind x86_64 1.3.1-3.el8 epel 75 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: lttng-ust x86_64 2.8.1-11.el8 appstream 259 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: python3-ceph-argparse x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 45 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: python3-ceph-common x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 115 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cephfs x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 190 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: python3-prettytable noarch 0.7.2-14.el8 appstream 44 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rados x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 370 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rbd x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 358 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rgw x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 107 k 2023-08-27T20:42:57.416 INFO:teuthology.orchestra.run.smithi093.stdout: thrift x86_64 0.13.0-2.el8 epel 1.7 M 2023-08-27T20:42:57.417 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:57.417 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:42:57.417 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:42:57.417 INFO:teuthology.orchestra.run.smithi093.stdout:Install 21 Packages 2023-08-27T20:42:57.417 INFO:teuthology.orchestra.run.smithi093.stdout:Upgrade 2 Packages 2023-08-27T20:42:57.417 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:42:57.417 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 63 M 2023-08-27T20:42:57.418 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:42:58.140 INFO:teuthology.orchestra.run.smithi136.stdout:(1/23): ceph-base-17.2.6-1016.g4fe39952.el8.x86 6.7 MB/s | 5.2 MB 00:00 2023-08-27T20:42:58.240 INFO:teuthology.orchestra.run.smithi136.stdout:(2/23): ceph-selinux-17.2.6-1016.g4fe39952.el8. 243 kB/s | 24 kB 00:00 2023-08-27T20:42:58.973 INFO:teuthology.orchestra.run.smithi136.stdout:(3/23): libcephfs2-17.2.6-1016.g4fe39952.el8.x8 1.1 MB/s | 796 kB 00:00 2023-08-27T20:42:59.540 INFO:teuthology.orchestra.run.smithi136.stdout:(4/23): libradosstriper1-17.2.6-1016.g4fe39952. 893 kB/s | 506 kB 00:00 2023-08-27T20:42:59.849 INFO:teuthology.orchestra.run.smithi136.stdout:(5/23): ceph-common-17.2.6-1016.g4fe39952.el8.x 9.4 MB/s | 23 MB 00:02 2023-08-27T20:42:59.941 INFO:teuthology.orchestra.run.smithi136.stdout:(6/23): python3-ceph-argparse-17.2.6-1016.g4fe3 488 kB/s | 45 kB 00:00 2023-08-27T20:43:00.041 INFO:teuthology.orchestra.run.smithi136.stdout:(7/23): python3-ceph-common-17.2.6-1016.g4fe399 1.1 MB/s | 115 kB 00:00 2023-08-27T20:43:00.166 INFO:teuthology.orchestra.run.smithi136.stdout:(8/23): python3-cephfs-17.2.6-1016.g4fe39952.el 1.5 MB/s | 190 kB 00:00 2023-08-27T20:43:00.392 INFO:teuthology.orchestra.run.smithi136.stdout:(9/23): python3-rados-17.2.6-1016.g4fe39952.el8 1.6 MB/s | 370 kB 00:00 2023-08-27T20:43:00.517 INFO:teuthology.orchestra.run.smithi136.stdout:(10/23): python3-rbd-17.2.6-1016.g4fe39952.el8. 2.8 MB/s | 358 kB 00:00 2023-08-27T20:43:00.559 INFO:teuthology.orchestra.run.smithi136.stdout:(11/23): python3-rgw-17.2.6-1016.g4fe39952.el8. 2.5 MB/s | 107 kB 00:00 2023-08-27T20:43:00.648 INFO:teuthology.orchestra.run.smithi093.stdout:(1/23): ceph-radosgw-17.2.6-1016.g4fe39952.el8. 5.4 MB/s | 17 MB 00:03 2023-08-27T20:43:00.882 INFO:teuthology.orchestra.run.smithi093.stdout:(2/23): ceph-common-17.2.6-1016.g4fe39952.el8.x 7.0 MB/s | 23 MB 00:03 2023-08-27T20:43:00.907 INFO:teuthology.orchestra.run.smithi093.stdout:(3/23): ceph-selinux-17.2.6-1016.g4fe39952.el8. 94 kB/s | 24 kB 00:00 2023-08-27T20:43:01.007 INFO:teuthology.orchestra.run.smithi093.stdout:(4/23): libcephfs2-17.2.6-1016.g4fe39952.el8.x8 6.2 MB/s | 796 kB 00:00 2023-08-27T20:43:01.032 INFO:teuthology.orchestra.run.smithi093.stdout:(5/23): libradosstriper1-17.2.6-1016.g4fe39952. 3.9 MB/s | 506 kB 00:00 2023-08-27T20:43:01.067 INFO:teuthology.orchestra.run.smithi136.stdout:(12/23): librdkafka-0.11.4-3.el8.x86_64.rpm 696 kB/s | 354 kB 00:00 2023-08-27T20:43:01.108 INFO:teuthology.orchestra.run.smithi093.stdout:(6/23): ceph-base-17.2.6-1016.g4fe39952.el8.x86 1.4 MB/s | 5.2 MB 00:03 2023-08-27T20:43:01.133 INFO:teuthology.orchestra.run.smithi093.stdout:(7/23): python3-ceph-argparse-17.2.6-1016.g4fe3 444 kB/s | 45 kB 00:00 2023-08-27T20:43:01.168 INFO:teuthology.orchestra.run.smithi136.stdout:(13/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 2.5 MB/s | 259 kB 00:00 2023-08-27T20:43:01.175 INFO:teuthology.orchestra.run.smithi093.stdout:(8/23): python3-ceph-common-17.2.6-1016.g4fe399 1.7 MB/s | 115 kB 00:00 2023-08-27T20:43:01.200 INFO:teuthology.orchestra.run.smithi093.stdout:(9/23): python3-cephfs-17.2.6-1016.g4fe39952.el 2.8 MB/s | 190 kB 00:00 2023-08-27T20:43:01.276 INFO:teuthology.orchestra.run.smithi136.stdout:(14/23): python3-prettytable-0.7.2-14.el8.noarc 405 kB/s | 44 kB 00:00 2023-08-27T20:43:01.284 INFO:teuthology.orchestra.run.smithi093.stdout:(10/23): python3-rbd-17.2.6-1016.g4fe39952.el8. 4.2 MB/s | 358 kB 00:00 2023-08-27T20:43:01.343 INFO:teuthology.orchestra.run.smithi093.stdout:(11/23): python3-rgw-17.2.6-1016.g4fe39952.el8. 1.8 MB/s | 107 kB 00:00 2023-08-27T20:43:01.351 INFO:teuthology.orchestra.run.smithi136.stdout:(15/23): librgw2-17.2.6-1016.g4fe39952.el8.x86_ 3.1 MB/s | 5.5 MB 00:01 2023-08-27T20:43:01.368 INFO:teuthology.orchestra.run.smithi093.stdout:(12/23): python3-rados-17.2.6-1016.g4fe39952.el 1.9 MB/s | 370 kB 00:00 2023-08-27T20:43:01.536 INFO:teuthology.orchestra.run.smithi136.stdout:(16/23): gperftools-libs-2.7-9.el8.x86_64.rpm 1.6 MB/s | 306 kB 00:00 2023-08-27T20:43:01.586 INFO:teuthology.orchestra.run.smithi136.stdout:(17/23): liboath-2.6.2-3.el8.x86_64.rpm 1.2 MB/s | 59 kB 00:00 2023-08-27T20:43:01.618 INFO:teuthology.orchestra.run.smithi093.stdout:(13/23): librgw2-17.2.6-1016.g4fe39952.el8.x86_ 9.1 MB/s | 5.5 MB 00:00 2023-08-27T20:43:01.661 INFO:teuthology.orchestra.run.smithi136.stdout:(18/23): libunwind-1.3.1-3.el8.x86_64.rpm 1.0 MB/s | 75 kB 00:00 2023-08-27T20:43:01.686 INFO:teuthology.orchestra.run.smithi136.stdout:(19/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 114 kB/s | 47 kB 00:00 2023-08-27T20:43:01.787 INFO:teuthology.orchestra.run.smithi136.stdout:(20/23): thrift-0.13.0-2.el8.x86_64.rpm 14 MB/s | 1.7 MB 00:00 2023-08-27T20:43:01.843 INFO:teuthology.orchestra.run.smithi093.stdout:(14/23): librdkafka-0.11.4-3.el8.x86_64.rpm 707 kB/s | 354 kB 00:00 2023-08-27T20:43:01.898 INFO:teuthology.orchestra.run.smithi093.stdout:(15/23): lttng-ust-2.8.1-11.el8.x86_64.rpm 490 kB/s | 259 kB 00:00 2023-08-27T20:43:01.944 INFO:teuthology.orchestra.run.smithi093.stdout:(16/23): python3-prettytable-0.7.2-14.el8.noarc 135 kB/s | 44 kB 00:00 2023-08-27T20:43:02.219 INFO:teuthology.orchestra.run.smithi093.stdout:(17/23): liboath-2.6.2-3.el8.x86_64.rpm 219 kB/s | 59 kB 00:00 2023-08-27T20:43:02.220 INFO:teuthology.orchestra.run.smithi136.stdout:(21/23): librbd1-17.2.6-1016.g4fe39952.el8.x86_ 8.3 MB/s | 3.6 MB 00:00 2023-08-27T20:43:02.245 INFO:teuthology.orchestra.run.smithi093.stdout:(18/23): librabbitmq-0.9.0-4.el8.x86_64.rpm 117 kB/s | 47 kB 00:00 2023-08-27T20:43:02.270 INFO:teuthology.orchestra.run.smithi093.stdout:(19/23): gperftools-libs-2.7-9.el8.x86_64.rpm 825 kB/s | 306 kB 00:00 2023-08-27T20:43:02.320 INFO:teuthology.orchestra.run.smithi093.stdout:(20/23): libunwind-1.3.1-3.el8.x86_64.rpm 731 kB/s | 75 kB 00:00 2023-08-27T20:43:02.362 INFO:teuthology.orchestra.run.smithi136.stdout:(22/23): librados2-17.2.6-1016.g4fe39952.el8.x8 5.1 MB/s | 3.5 MB 00:00 2023-08-27T20:43:02.754 INFO:teuthology.orchestra.run.smithi093.stdout:(21/23): thrift-0.13.0-2.el8.x86_64.rpm 3.4 MB/s | 1.7 MB 00:00 2023-08-27T20:43:02.838 INFO:teuthology.orchestra.run.smithi093.stdout:(22/23): librbd1-17.2.6-1016.g4fe39952.el8.x86_ 6.9 MB/s | 3.6 MB 00:00 2023-08-27T20:43:03.188 INFO:teuthology.orchestra.run.smithi093.stdout:(23/23): librados2-17.2.6-1016.g4fe39952.el8.x8 3.8 MB/s | 3.5 MB 00:00 2023-08-27T20:43:03.192 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:43:03.192 INFO:teuthology.orchestra.run.smithi093.stdout:Total 11 MB/s | 63 MB 00:05 2023-08-27T20:43:03.204 INFO:teuthology.orchestra.run.smithi136.stdout:(23/23): ceph-radosgw-17.2.6-1016.g4fe39952.el8 2.9 MB/s | 17 MB 00:05 2023-08-27T20:43:03.207 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:43:03.207 INFO:teuthology.orchestra.run.smithi136.stdout:Total 10 MB/s | 63 MB 00:06 2023-08-27T20:43:03.235 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:43:03.248 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:43:03.280 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:43:03.280 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:43:03.294 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:43:03.294 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:43:03.631 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:43:03.634 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:43:03.647 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:43:03.650 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:43:04.677 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:43:04.691 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:43:04.879 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-08-27T20:43:04.941 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : liboath-2.6.2-3.el8.x86_64 1/25 2023-08-27T20:43:05.046 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-08-27T20:43:05.142 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-08-27T20:43:05.433 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-08-27T20:43:05.507 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: lttng-ust-2.8.1-11.el8.x86_64 2/25 2023-08-27T20:43:05.671 INFO:teuthology.orchestra.run.smithi136.stdout: Upgrading : librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 3/25 2023-08-27T20:43:05.750 INFO:teuthology.orchestra.run.smithi093.stdout: Upgrading : librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 3/25 2023-08-27T20:43:05.754 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 3/25 2023-08-27T20:43:05.845 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 3/25 2023-08-27T20:43:05.979 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 4/25 2023-08-27T20:43:06.066 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 4/25 2023-08-27T20:43:06.154 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 5/25 2023-08-27T20:43:06.233 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 5/25 2023-08-27T20:43:06.532 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 5/25 2023-08-27T20:43:06.630 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 5/25 2023-08-27T20:43:06.712 INFO:teuthology.orchestra.run.smithi136.stdout: Upgrading : librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 6/25 2023-08-27T20:43:06.808 INFO:teuthology.orchestra.run.smithi093.stdout: Upgrading : librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 6/25 2023-08-27T20:43:06.936 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 6/25 2023-08-27T20:43:07.037 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 6/25 2023-08-27T20:43:07.116 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : thrift-0.13.0-2.el8.x86_64 7/25 2023-08-27T20:43:07.229 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : thrift-0.13.0-2.el8.x86_64 7/25 2023-08-27T20:43:07.277 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 8/25 2023-08-27T20:43:07.439 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : librabbitmq-0.9.0-4.el8.x86_64 8/25 2023-08-27T20:43:07.453 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 9/25 2023-08-27T20:43:07.583 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : librdkafka-0.11.4-3.el8.x86_64 9/25 2023-08-27T20:43:08.009 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 9/25 2023-08-27T20:43:08.154 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 10/25 2023-08-27T20:43:08.159 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: librdkafka-0.11.4-3.el8.x86_64 9/25 2023-08-27T20:43:08.249 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 10/25 2023-08-27T20:43:08.325 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 10/25 2023-08-27T20:43:08.389 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 10/25 2023-08-27T20:43:08.415 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-ceph-argparse-2:17.2.6-1016.g4fe39952.el8. 11/25 2023-08-27T20:43:08.552 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-ceph-argparse-2:17.2.6-1016.g4fe39952.el8. 11/25 2023-08-27T20:43:08.594 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 12/25 2023-08-27T20:43:08.781 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 12/25 2023-08-27T20:43:08.797 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 13/25 2023-08-27T20:43:09.017 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 13/25 2023-08-27T20:43:09.023 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 14/25 2023-08-27T20:43:09.186 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_6 15/25 2023-08-27T20:43:09.227 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 14/25 2023-08-27T20:43:09.256 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_6 15/25 2023-08-27T20:43:09.391 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_6 15/25 2023-08-27T20:43:09.454 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-08-27T20:43:09.465 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_6 15/25 2023-08-27T20:43:09.620 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-08-27T20:43:09.700 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : libunwind-1.3.1-3.el8.x86_64 16/25 2023-08-27T20:43:09.832 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-08-27T20:43:09.924 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : gperftools-libs-1:2.7-9.el8.x86_64 17/25 2023-08-27T20:43:10.045 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-ceph-common-2:17.2.6-1016.g4fe39952.el8.x8 19/25 2023-08-27T20:43:10.117 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-prettytable-0.7.2-14.el8.noarch 18/25 2023-08-27T20:43:10.325 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-ceph-common-2:17.2.6-1016.g4fe39952.el8.x8 19/25 2023-08-27T20:43:12.705 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 20/25 2023-08-27T20:43:12.978 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 20/25 2023-08-27T20:43:13.045 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 20/25 2023-08-27T20:43:13.308 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 20/25 2023-08-27T20:43:13.591 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 20/25 2023-08-27T20:43:13.833 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 21/25 2023-08-27T20:43:13.901 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 20/25 2023-08-27T20:43:13.945 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 21/25 2023-08-27T20:43:14.169 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 22/25 2023-08-27T20:43:14.196 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 21/25 2023-08-27T20:43:14.326 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 21/25 2023-08-27T20:43:14.508 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 22/25 2023-08-27T20:43:32.419 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 22/25 2023-08-27T20:43:32.419 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /sys 2023-08-27T20:43:32.419 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /proc 2023-08-27T20:43:32.419 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /dev 2023-08-27T20:43:32.419 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /run 2023-08-27T20:43:32.419 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /mnt 2023-08-27T20:43:32.420 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /var/tmp 2023-08-27T20:43:32.420 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /home 2023-08-27T20:43:32.420 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /tmp 2023-08-27T20:43:32.420 INFO:teuthology.orchestra.run.smithi136.stdout:skipping the directory /dev 2023-08-27T20:43:32.420 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:32.968 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 22/25 2023-08-27T20:43:32.968 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /sys 2023-08-27T20:43:32.968 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /proc 2023-08-27T20:43:32.968 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /dev 2023-08-27T20:43:32.968 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /run 2023-08-27T20:43:32.968 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /mnt 2023-08-27T20:43:32.969 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /var/tmp 2023-08-27T20:43:32.969 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /home 2023-08-27T20:43:32.969 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /tmp 2023-08-27T20:43:32.969 INFO:teuthology.orchestra.run.smithi093.stdout:skipping the directory /dev 2023-08-27T20:43:32.969 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:34.129 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 23/25 2023-08-27T20:43:34.204 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 23/25 2023-08-27T20:43:34.204 INFO:teuthology.orchestra.run.smithi136.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-27T20:43:34.425 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-27T20:43:34.425 INFO:teuthology.orchestra.run.smithi136.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-27T20:43:34.668 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 23/25 2023-08-27T20:43:34.765 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 23/25 2023-08-27T20:43:34.765 INFO:teuthology.orchestra.run.smithi093.stdout: Cleanup : librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-27T20:43:34.973 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: librbd1-1:12.2.7-9.el8.x86_64 24/25 2023-08-27T20:43:34.973 INFO:teuthology.orchestra.run.smithi093.stdout: Cleanup : librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-27T20:43:35.061 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-27T20:43:35.061 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 1/25 2023-08-27T20:43:35.061 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 2/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 3/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 4/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 5/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_6 6/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 7/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1016.g4fe39952.el8. 8/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-ceph-common-2:17.2.6-1016.g4fe39952.el8.x8 9/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 10/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 11/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 12/25 2023-08-27T20:43:35.062 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 13/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 19/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 20/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 21/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 22/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-08-27T20:43:35.063 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 24/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: librados2-1:12.2.7-9.el8.x86_64 25/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 1/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 2/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 3/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 4/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 5/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_6 6/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 7/25 2023-08-27T20:43:35.706 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-ceph-argparse-2:17.2.6-1016.g4fe39952.el8. 8/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-ceph-common-2:17.2.6-1016.g4fe39952.el8.x8 9/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 10/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 11/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 12/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 13/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librdkafka-0.11.4-3.el8.x86_64 14/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : lttng-ust-2.8.1-11.el8.x86_64 15/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-prettytable-0.7.2-14.el8.noarch 16/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librabbitmq-0.9.0-4.el8.x86_64 17/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : gperftools-libs-1:2.7-9.el8.x86_64 18/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : liboath-2.6.2-3.el8.x86_64 19/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : libunwind-1.3.1-3.el8.x86_64 20/25 2023-08-27T20:43:35.707 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : thrift-0.13.0-2.el8.x86_64 21/25 2023-08-27T20:43:35.708 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 22/25 2023-08-27T20:43:35.708 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librados2-1:12.2.7-9.el8.x86_64 23/25 2023-08-27T20:43:35.708 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 24/25 2023-08-27T20:43:36.121 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-08-27T20:43:36.121 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:36.121 INFO:teuthology.orchestra.run.smithi136.stdout:Upgraded: 2023-08-27T20:43:36.121 INFO:teuthology.orchestra.run.smithi136.stdout: librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.121 INFO:teuthology.orchestra.run.smithi136.stdout: librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.121 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: liboath-2.6.2-3.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.122 INFO:teuthology.orchestra.run.smithi136.stdout: libunwind-1.3.1-3.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: python3-ceph-argparse-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: python3-ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: thrift-0.13.0-2.el8.x86_64 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:36.123 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:43:36.439 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-test 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librbd1-1:12.2.7-9.el8.x86_64 25/25 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout:Upgraded: 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout: librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout: librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.937 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-radosgw-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-selinux-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: gperftools-libs-1:2.7-9.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: liboath-2.6.2-3.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: librabbitmq-0.9.0-4.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: libradosstriper1-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: librdkafka-0.11.4-3.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: librgw2-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: libunwind-1.3.1-3.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: lttng-ust-2.8.1-11.el8.x86_64 2023-08-27T20:43:36.938 INFO:teuthology.orchestra.run.smithi093.stdout: python3-ceph-argparse-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: python3-ceph-common-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: python3-prettytable-0.7.2-14.el8.noarch 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: thrift-0.13.0-2.el8.x86_64 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:36.939 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:43:37.291 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:00:45 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:43:37.351 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-test 2023-08-27T20:43:38.195 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:00:45 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:43:38.789 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-test x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 52 M 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-08-27T20:43:38.790 INFO:teuthology.orchestra.run.smithi136.stdout: libcephsqlite x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 174 k 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout:Install 6 Packages 2023-08-27T20:43:38.791 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:38.792 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 53 M 2023-08-27T20:43:38.792 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 233 M 2023-08-27T20:43:38.792 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:43:39.212 INFO:teuthology.orchestra.run.smithi136.stdout:(1/6): libcephsqlite-17.2.6-1016.g4fe39952.el8. 851 kB/s | 174 kB 00:00 2023-08-27T20:43:39.437 INFO:teuthology.orchestra.run.smithi136.stdout:(2/6): jq-1.6-7.el8.x86_64.rpm 471 kB/s | 202 kB 00:00 2023-08-27T20:43:39.630 INFO:teuthology.orchestra.run.smithi136.stdout:(3/6): oniguruma-6.8.2-2.el8.x86_64.rpm 448 kB/s | 187 kB 00:00 2023-08-27T20:43:39.663 INFO:teuthology.orchestra.run.smithi136.stdout:(4/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.4 MB/s | 323 kB 00:00 2023-08-27T20:43:39.689 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:43:39.690 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:43:39.690 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:43:39.690 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:43:39.690 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:43:39.690 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-test x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 52 M 2023-08-27T20:43:39.690 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:43:39.691 INFO:teuthology.orchestra.run.smithi093.stdout: jq x86_64 1.6-7.el8 appstream 202 k 2023-08-27T20:43:39.691 INFO:teuthology.orchestra.run.smithi093.stdout: libcephsqlite x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 174 k 2023-08-27T20:43:39.691 INFO:teuthology.orchestra.run.smithi093.stdout: oniguruma x86_64 6.8.2-2.el8 appstream 187 k 2023-08-27T20:43:39.691 INFO:teuthology.orchestra.run.smithi093.stdout: socat x86_64 1.7.4.1-1.el8 appstream 323 k 2023-08-27T20:43:39.691 INFO:teuthology.orchestra.run.smithi093.stdout: xmlstarlet x86_64 1.6.1-20.el8 appstream 71 k 2023-08-27T20:43:39.691 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:39.691 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:43:39.692 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:43:39.692 INFO:teuthology.orchestra.run.smithi093.stdout:Install 6 Packages 2023-08-27T20:43:39.692 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:39.692 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 53 M 2023-08-27T20:43:39.693 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 233 M 2023-08-27T20:43:39.693 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:43:39.729 INFO:teuthology.orchestra.run.smithi136.stdout:(5/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 710 kB/s | 71 kB 00:00 2023-08-27T20:43:40.238 INFO:teuthology.orchestra.run.smithi093.stdout:(1/6): libcephsqlite-17.2.6-1016.g4fe39952.el8. 496 kB/s | 174 kB 00:00 2023-08-27T20:43:40.288 INFO:teuthology.orchestra.run.smithi093.stdout:(2/6): jq-1.6-7.el8.x86_64.rpm 504 kB/s | 202 kB 00:00 2023-08-27T20:43:40.472 INFO:teuthology.orchestra.run.smithi093.stdout:(3/6): socat-1.7.4.1-1.el8.x86_64.rpm 1.7 MB/s | 323 kB 00:00 2023-08-27T20:43:40.572 INFO:teuthology.orchestra.run.smithi093.stdout:(4/6): xmlstarlet-1.6.1-20.el8.x86_64.rpm 707 kB/s | 71 kB 00:00 2023-08-27T20:43:40.630 INFO:teuthology.orchestra.run.smithi093.stdout:(5/6): oniguruma-6.8.2-2.el8.x86_64.rpm 478 kB/s | 187 kB 00:00 2023-08-27T20:43:42.030 INFO:teuthology.orchestra.run.smithi136.stdout:(6/6): ceph-test-17.2.6-1016.g4fe39952.el8.x86_ 17 MB/s | 52 MB 00:03 2023-08-27T20:43:42.032 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:43:42.032 INFO:teuthology.orchestra.run.smithi136.stdout:Total 16 MB/s | 53 MB 00:03 2023-08-27T20:43:42.076 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:43:42.096 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:43:42.097 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:43:42.381 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:43:42.384 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:43:42.883 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:43:43.090 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-08-27T20:43:43.290 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-08-27T20:43:43.412 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-08-27T20:43:43.493 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-08-27T20:43:43.679 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-08-27T20:43:43.837 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 5/6 2023-08-27T20:43:45.072 INFO:teuthology.orchestra.run.smithi093.stdout:(6/6): ceph-test-17.2.6-1016.g4fe39952.el8.x86_ 10 MB/s | 52 MB 00:05 2023-08-27T20:43:45.074 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:43:45.074 INFO:teuthology.orchestra.run.smithi093.stdout:Total 9.8 MB/s | 53 MB 00:05 2023-08-27T20:43:45.117 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:43:45.138 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:43:45.138 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:43:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:43:45.428 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:43:45.924 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:43:46.174 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : xmlstarlet-1.6.1-20.el8.x86_64 1/6 2023-08-27T20:43:46.396 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : socat-1.7.4.1-1.el8.x86_64 2/6 2023-08-27T20:43:46.578 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : oniguruma-6.8.2-2.el8.x86_64 3/6 2023-08-27T20:43:46.643 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: oniguruma-6.8.2-2.el8.x86_64 3/6 2023-08-27T20:43:46.891 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : jq-1.6-7.el8.x86_64 4/6 2023-08-27T20:43:47.069 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 5/6 2023-08-27T20:43:48.820 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 5/6 2023-08-27T20:43:49.043 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 6/6 2023-08-27T20:43:49.319 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 6/6 2023-08-27T20:43:49.319 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 1/6 2023-08-27T20:43:49.319 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 2/6 2023-08-27T20:43:49.319 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-08-27T20:43:49.319 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-08-27T20:43:49.319 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-08-27T20:43:49.999 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: jq-1.6-7.el8.x86_64 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: socat-1.7.4.1-1.el8.x86_64 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:50.000 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:43:50.194 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph 2023-08-27T20:43:51.044 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:00:59 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:43:52.026 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 5/6 2023-08-27T20:43:52.308 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 6/6 2023-08-27T20:43:52.544 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:43:52.547 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:43:52.547 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:43:52.547 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:43:52.547 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:43:52.547 INFO:teuthology.orchestra.run.smithi136.stdout: ceph x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 6.4 k 2023-08-27T20:43:52.547 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mds x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 2.3 M 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 1.6 M 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 238 k 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mon x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.9 M 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-osd x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 17 M 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-08-27T20:43:52.548 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-08-27T20:43:52.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:52.550 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:43:52.551 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:43:52.551 INFO:teuthology.orchestra.run.smithi136.stdout:Install 32 Packages 2023-08-27T20:43:52.551 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:43:52.551 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 28 M 2023-08-27T20:43:52.551 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 110 M 2023-08-27T20:43:52.552 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:43:52.592 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 6/6 2023-08-27T20:43:52.592 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 1/6 2023-08-27T20:43:52.592 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 2/6 2023-08-27T20:43:52.592 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : jq-1.6-7.el8.x86_64 3/6 2023-08-27T20:43:52.592 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : oniguruma-6.8.2-2.el8.x86_64 4/6 2023-08-27T20:43:52.592 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : socat-1.7.4.1-1.el8.x86_64 5/6 2023-08-27T20:43:52.951 INFO:teuthology.orchestra.run.smithi136.stdout:(1/32): ceph-17.2.6-1016.g4fe39952.el8.x86_64.r 35 kB/s | 6.4 kB 00:00 2023-08-27T20:43:53.093 INFO:teuthology.orchestra.run.smithi136.stdout:(2/32): ceph-mgr-17.2.6-1016.g4fe39952.el8.x86_ 4.8 MB/s | 1.6 MB 00:00 2023-08-27T20:43:53.143 INFO:teuthology.orchestra.run.smithi136.stdout:(3/32): ceph-mds-17.2.6-1016.g4fe39952.el8.x86_ 6.1 MB/s | 2.3 MB 00:00 2023-08-27T20:43:53.219 INFO:teuthology.orchestra.run.smithi136.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1016.g4fe3 3.1 MB/s | 238 kB 00:00 2023-08-27T20:43:53.253 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : xmlstarlet-1.6.1-20.el8.x86_64 6/6 2023-08-27T20:43:53.253 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:53.253 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:43:53.253 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-test-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:53.254 INFO:teuthology.orchestra.run.smithi093.stdout: jq-1.6-7.el8.x86_64 2023-08-27T20:43:53.254 INFO:teuthology.orchestra.run.smithi093.stdout: libcephsqlite-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:43:53.254 INFO:teuthology.orchestra.run.smithi093.stdout: oniguruma-6.8.2-2.el8.x86_64 2023-08-27T20:43:53.254 INFO:teuthology.orchestra.run.smithi093.stdout: socat-1.7.4.1-1.el8.x86_64 2023-08-27T20:43:53.254 INFO:teuthology.orchestra.run.smithi093.stdout: xmlstarlet-1.6.1-20.el8.x86_64 2023-08-27T20:43:53.254 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:53.254 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:43:53.463 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph 2023-08-27T20:43:53.619 INFO:teuthology.orchestra.run.smithi136.stdout:(5/32): python3-mako-1.0.6-14.el8.noarch.rpm 393 kB/s | 157 kB 00:00 2023-08-27T20:43:53.719 INFO:teuthology.orchestra.run.smithi136.stdout:(6/32): python3-markupsafe-0.23-19.el8.x86_64.r 388 kB/s | 39 kB 00:00 2023-08-27T20:43:53.778 INFO:teuthology.orchestra.run.smithi136.stdout:(7/32): ceph-mon-17.2.6-1016.g4fe39952.el8.x86_ 4.8 MB/s | 3.9 MB 00:00 2023-08-27T20:43:53.811 INFO:teuthology.orchestra.run.smithi136.stdout:(8/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 1.1 MB/s | 103 kB 00:00 2023-08-27T20:43:54.020 INFO:teuthology.orchestra.run.smithi136.stdout:(9/32): ceph-osd-17.2.6-1016.g4fe39952.el8.x86_ 18 MB/s | 17 MB 00:00 2023-08-27T20:43:54.195 INFO:teuthology.orchestra.run.smithi136.stdout:(10/32): python3-werkzeug-0.12.2-4.el8.noarch.r 1.2 MB/s | 457 kB 00:00 2023-08-27T20:43:54.271 INFO:teuthology.orchestra.run.smithi136.stdout:(11/32): python3-pytz-2017.2-11.el8.noarch.rpm 116 kB/s | 57 kB 00:00 2023-08-27T20:43:54.316 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:01 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:43:54.546 INFO:teuthology.orchestra.run.smithi136.stdout:(12/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 473 kB/s | 249 kB 00:00 2023-08-27T20:43:54.647 INFO:teuthology.orchestra.run.smithi136.stdout:(13/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 438 kB/s | 44 kB 00:00 2023-08-27T20:43:54.705 INFO:teuthology.orchestra.run.smithi136.stdout:(14/32): python3-pycparser-2.14-14.el8.noarch.r 251 kB/s | 109 kB 00:00 2023-08-27T20:43:54.756 INFO:teuthology.orchestra.run.smithi136.stdout:(15/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 1.7 MB/s | 185 kB 00:00 2023-08-27T20:43:54.863 INFO:teuthology.orchestra.run.smithi136.stdout:(16/32): python3-cherrypy-18.4.0-1.el8.noarch.r 3.5 MB/s | 384 kB 00:00 2023-08-27T20:43:54.889 INFO:teuthology.orchestra.run.smithi136.stdout:(17/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 938 kB/s | 173 kB 00:00 2023-08-27T20:43:54.915 INFO:teuthology.orchestra.run.smithi136.stdout:(18/32): python3-cryptography-3.2.1-6.el8.x86_6 892 kB/s | 641 kB 00:00 2023-08-27T20:43:54.940 INFO:teuthology.orchestra.run.smithi136.stdout:(19/32): python3-jaraco-6.2-6.el8.noarch.rpm 145 kB/s | 11 kB 00:00 2023-08-27T20:43:54.965 INFO:teuthology.orchestra.run.smithi136.stdout:(20/32): python3-jaraco-functools-2.0-4.el8.noa 233 kB/s | 18 kB 00:00 2023-08-27T20:43:54.991 INFO:teuthology.orchestra.run.smithi136.stdout:(21/32): python3-logutils-0.3.5-11.el8.noarch.r 971 kB/s | 49 kB 00:00 2023-08-27T20:43:55.016 INFO:teuthology.orchestra.run.smithi136.stdout:(22/32): python3-pecan-1.3.2-9.el8.noarch.rpm 5.5 MB/s | 283 kB 00:00 2023-08-27T20:43:55.041 INFO:teuthology.orchestra.run.smithi136.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 311 kB/s | 16 kB 00:00 2023-08-27T20:43:55.066 INFO:teuthology.orchestra.run.smithi136.stdout:(24/32): python3-more-itertools-7.2.0-3.el8.noa 466 kB/s | 59 kB 00:00 2023-08-27T20:43:55.092 INFO:teuthology.orchestra.run.smithi136.stdout:(25/32): python3-simplegeneric-0.8.1-17.el8.noa 248 kB/s | 19 kB 00:00 2023-08-27T20:43:55.142 INFO:teuthology.orchestra.run.smithi136.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 238 kB/s | 24 kB 00:00 2023-08-27T20:43:55.167 INFO:teuthology.orchestra.run.smithi136.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 389 kB/s | 29 kB 00:00 2023-08-27T20:43:55.192 INFO:teuthology.orchestra.run.smithi136.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 275 kB/s | 27 kB 00:00 2023-08-27T20:43:55.243 INFO:teuthology.orchestra.run.smithi136.stdout:(29/32): python3-waitress-1.4.3-1.el8.noarch.rp 2.5 MB/s | 255 kB 00:00 2023-08-27T20:43:55.268 INFO:teuthology.orchestra.run.smithi136.stdout:(30/32): python3-webtest-2.0.33-1.el8.noarch.rp 1.1 MB/s | 85 kB 00:00 2023-08-27T20:43:55.293 INFO:teuthology.orchestra.run.smithi136.stdout:(31/32): python3-zc-lockfile-2.0-2.el8.noarch.r 457 kB/s | 23 kB 00:00 2023-08-27T20:43:55.318 INFO:teuthology.orchestra.run.smithi136.stdout:(32/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 2.0 MB/s | 251 kB 00:00 2023-08-27T20:43:55.320 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:43:55.320 INFO:teuthology.orchestra.run.smithi136.stdout:Total 10 MB/s | 28 MB 00:02 2023-08-27T20:43:55.405 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:43:55.442 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:43:55.442 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:43:55.750 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:43:55.754 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:43:55.825 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:43:55.829 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:43:55.829 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:43:55.829 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:43:55.829 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:43:55.829 INFO:teuthology.orchestra.run.smithi093.stdout: ceph x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 6.4 k 2023-08-27T20:43:55.829 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:43:55.829 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mds x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 2.3 M 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 1.6 M 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-modules-core noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 238 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mon x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.9 M 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-osd x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 17 M 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-bcrypt x86_64 3.1.6-2.el8.1 epel 44 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-beautifulsoup4 noarch 4.6.3-2.el8.1 epel 185 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cffi x86_64 1.11.5-6.el8 baseos 249 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cheroot noarch 8.5.2-1.el8 epel 173 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cherrypy noarch 18.4.0-1.el8 epel 384 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cryptography x86_64 3.2.1-6.el8 baseos 641 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jaraco noarch 6.2-6.el8 epel 11 k 2023-08-27T20:43:55.830 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jaraco-functools noarch 2.0-4.el8 epel 18 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-logutils noarch 0.3.5-11.el8 epel 49 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-mako noarch 1.0.6-14.el8 appstream 157 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-markupsafe x86_64 0.23-19.el8 appstream 39 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-more-itertools noarch 7.2.0-3.el8 epel 59 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pecan noarch 1.3.2-9.el8 epel 283 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-portend noarch 2.6-1.el8 epel 16 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyOpenSSL noarch 19.0.0-1.el8 appstream 103 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pycparser noarch 2.14-14.el8 baseos 109 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pytz noarch 2017.2-11.el8 appstream 57 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-simplegeneric noarch 0.8.1-17.el8 epel 19 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-singledispatch noarch 3.4.0.3-18.el8 epel 24 k 2023-08-27T20:43:55.831 INFO:teuthology.orchestra.run.smithi093.stdout: python3-tempora noarch 1.14.1-5.el8 epel 29 k 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: python3-trustme noarch 0.6.0-4.el8 epel 27 k 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: python3-waitress noarch 1.4.3-1.el8 epel 255 k 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: python3-webob noarch 1.8.5-1.el8.1 epel 251 k 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: python3-webtest noarch 2.0.33-1.el8 epel 85 k 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: python3-werkzeug noarch 0.12.2-4.el8 appstream 457 k 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: python3-zc-lockfile noarch 2.0-2.el8 epel 23 k 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout:Install 32 Packages 2023-08-27T20:43:55.832 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:43:55.833 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 28 M 2023-08-27T20:43:55.833 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 110 M 2023-08-27T20:43:55.833 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:43:56.405 INFO:teuthology.orchestra.run.smithi093.stdout:(1/32): ceph-17.2.6-1016.g4fe39952.el8.x86_64.r 51 kB/s | 6.4 kB 00:00 2023-08-27T20:43:56.664 INFO:teuthology.orchestra.run.smithi093.stdout:(2/32): ceph-mds-17.2.6-1016.g4fe39952.el8.x86_ 6.0 MB/s | 2.3 MB 00:00 2023-08-27T20:43:56.706 INFO:teuthology.orchestra.run.smithi093.stdout:(3/32): ceph-mgr-17.2.6-1016.g4fe39952.el8.x86_ 3.7 MB/s | 1.6 MB 00:00 2023-08-27T20:43:56.798 INFO:teuthology.orchestra.run.smithi093.stdout:(4/32): ceph-mgr-modules-core-17.2.6-1016.g4fe3 2.5 MB/s | 238 kB 00:00 2023-08-27T20:43:56.827 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:43:57.032 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-08-27T20:43:57.226 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-08-27T20:43:57.252 INFO:teuthology.orchestra.run.smithi093.stdout:(5/32): python3-mako-1.0.6-14.el8.noarch.rpm 346 kB/s | 157 kB 00:00 2023-08-27T20:43:57.365 INFO:teuthology.orchestra.run.smithi093.stdout:(6/32): python3-markupsafe-0.23-19.el8.x86_64.r 344 kB/s | 39 kB 00:00 2023-08-27T20:43:57.443 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-08-27T20:43:57.473 INFO:teuthology.orchestra.run.smithi093.stdout:(7/32): python3-pyOpenSSL-19.0.0-1.el8.noarch.r 954 kB/s | 103 kB 00:00 2023-08-27T20:43:57.607 INFO:teuthology.orchestra.run.smithi093.stdout:(8/32): ceph-mon-17.2.6-1016.g4fe39952.el8.x86_ 3.3 MB/s | 3.9 MB 00:01 2023-08-27T20:43:57.632 INFO:teuthology.orchestra.run.smithi093.stdout:(9/32): python3-pytz-2017.2-11.el8.noarch.rpm 360 kB/s | 57 kB 00:00 2023-08-27T20:43:57.699 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-08-27T20:43:57.800 INFO:teuthology.orchestra.run.smithi093.stdout:(10/32): ceph-osd-17.2.6-1016.g4fe39952.el8.x86 15 MB/s | 17 MB 00:01 2023-08-27T20:43:57.888 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-08-27T20:43:58.085 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-08-27T20:43:58.258 INFO:teuthology.orchestra.run.smithi093.stdout:(11/32): python3-cffi-1.11.5-6.el8.x86_64.rpm 398 kB/s | 249 kB 00:00 2023-08-27T20:43:58.262 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-08-27T20:43:58.283 INFO:teuthology.orchestra.run.smithi093.stdout:(12/32): python3-werkzeug-0.12.2-4.el8.noarch.r 677 kB/s | 457 kB 00:00 2023-08-27T20:43:58.383 INFO:teuthology.orchestra.run.smithi093.stdout:(13/32): python3-pycparser-2.14-14.el8.noarch.r 871 kB/s | 109 kB 00:00 2023-08-27T20:43:58.434 INFO:teuthology.orchestra.run.smithi093.stdout:(14/32): python3-cryptography-3.2.1-6.el8.x86_6 1.0 MB/s | 641 kB 00:00 2023-08-27T20:43:58.473 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-08-27T20:43:58.689 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-08-27T20:43:58.709 INFO:teuthology.orchestra.run.smithi093.stdout:(15/32): python3-bcrypt-3.1.6-2.el8.1.x86_64.rp 102 kB/s | 44 kB 00:00 2023-08-27T20:43:58.901 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-08-27T20:43:58.926 INFO:teuthology.orchestra.run.smithi093.stdout:(16/32): python3-beautifulsoup4-4.6.3-2.el8.1.n 345 kB/s | 185 kB 00:00 2023-08-27T20:43:58.959 INFO:teuthology.orchestra.run.smithi093.stdout:(17/32): python3-cheroot-8.5.2-1.el8.noarch.rpm 328 kB/s | 173 kB 00:00 2023-08-27T20:43:59.026 INFO:teuthology.orchestra.run.smithi093.stdout:(18/32): python3-jaraco-6.2-6.el8.noarch.rpm 111 kB/s | 11 kB 00:00 2023-08-27T20:43:59.068 INFO:teuthology.orchestra.run.smithi093.stdout:(19/32): python3-jaraco-functools-2.0-4.el8.noa 162 kB/s | 18 kB 00:00 2023-08-27T20:43:59.093 INFO:teuthology.orchestra.run.smithi093.stdout:(20/32): python3-cherrypy-18.4.0-1.el8.noarch.r 1.0 MB/s | 384 kB 00:00 2023-08-27T20:43:59.105 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-08-27T20:43:59.152 INFO:teuthology.orchestra.run.smithi093.stdout:(21/32): python3-logutils-0.3.5-11.el8.noarch.r 388 kB/s | 49 kB 00:00 2023-08-27T20:43:59.177 INFO:teuthology.orchestra.run.smithi093.stdout:(22/32): python3-more-itertools-7.2.0-3.el8.noa 540 kB/s | 59 kB 00:00 2023-08-27T20:43:59.261 INFO:teuthology.orchestra.run.smithi093.stdout:(23/32): python3-portend-2.6-1.el8.noarch.rpm 143 kB/s | 16 kB 00:00 2023-08-27T20:43:59.287 INFO:teuthology.orchestra.run.smithi093.stdout:(24/32): python3-simplegeneric-0.8.1-17.el8.noa 172 kB/s | 19 kB 00:00 2023-08-27T20:43:59.312 INFO:teuthology.orchestra.run.smithi093.stdout:(25/32): python3-pecan-1.3.2-9.el8.noarch.rpm 1.3 MB/s | 283 kB 00:00 2023-08-27T20:43:59.328 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-08-27T20:43:59.370 INFO:teuthology.orchestra.run.smithi093.stdout:(26/32): python3-singledispatch-3.4.0.3-18.el8. 219 kB/s | 24 kB 00:00 2023-08-27T20:43:59.395 INFO:teuthology.orchestra.run.smithi093.stdout:(27/32): python3-tempora-1.14.1-5.el8.noarch.rp 269 kB/s | 29 kB 00:00 2023-08-27T20:43:59.441 INFO:teuthology.orchestra.run.smithi093.stdout:(28/32): python3-trustme-0.6.0-4.el8.noarch.rpm 213 kB/s | 27 kB 00:00 2023-08-27T20:43:59.538 INFO:teuthology.orchestra.run.smithi093.stdout:(29/32): python3-webob-1.8.5-1.el8.1.noarch.rpm 1.7 MB/s | 251 kB 00:00 2023-08-27T20:43:59.563 INFO:teuthology.orchestra.run.smithi093.stdout:(30/32): python3-waitress-1.4.3-1.el8.noarch.rp 1.3 MB/s | 255 kB 00:00 2023-08-27T20:43:59.588 INFO:teuthology.orchestra.run.smithi093.stdout:(31/32): python3-webtest-2.0.33-1.el8.noarch.rp 581 kB/s | 85 kB 00:00 2023-08-27T20:43:59.638 INFO:teuthology.orchestra.run.smithi093.stdout:(32/32): python3-zc-lockfile-2.0-2.el8.noarch.r 229 kB/s | 23 kB 00:00 2023-08-27T20:43:59.643 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:43:59.643 INFO:teuthology.orchestra.run.smithi093.stdout:Total 7.5 MB/s | 28 MB 00:03 2023-08-27T20:43:59.718 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-08-27T20:43:59.728 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:43:59.765 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:43:59.765 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:43:59.970 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-08-27T20:44:00.078 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:44:00.082 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:44:00.167 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-08-27T20:44:00.491 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-08-27T20:44:00.709 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-08-27T20:44:01.026 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-08-27T20:44:01.206 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:44:01.265 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-08-27T20:44:01.435 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-08-27T20:44:01.447 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-more-itertools-7.2.0-3.el8.noarch 1/32 2023-08-27T20:44:01.638 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-08-27T20:44:01.700 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-webob-1.8.5-1.el8.1.noarch 2/32 2023-08-27T20:44:01.876 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-08-27T20:44:01.934 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-zc-lockfile-2.0-2.el8.noarch 3/32 2023-08-27T20:44:02.120 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-08-27T20:44:02.223 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-waitress-1.4.3-1.el8.noarch 4/32 2023-08-27T20:44:02.327 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-08-27T20:44:02.446 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-singledispatch-3.4.0.3-18.el8.noarch 5/32 2023-08-27T20:44:02.647 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-08-27T20:44:02.676 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-simplegeneric-0.8.1-17.el8.noarch 6/32 2023-08-27T20:44:02.878 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-logutils-0.3.5-11.el8.noarch 7/32 2023-08-27T20:44:02.938 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-08-27T20:44:03.105 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-jaraco-6.2-6.el8.noarch 8/32 2023-08-27T20:44:03.310 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1016.g4fe39952.el8. 27/32 2023-08-27T20:44:03.340 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-jaraco-functools-2.0-4.el8.noarch 9/32 2023-08-27T20:44:03.514 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 28/32 2023-08-27T20:44:03.552 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 10/32 2023-08-27T20:44:03.798 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-webtest-2.0.33-1.el8.noarch 11/32 2023-08-27T20:44:04.089 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pycparser-2.14-14.el8.noarch 12/32 2023-08-27T20:44:04.497 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-cffi-1.11.5-6.el8.x86_64 13/32 2023-08-27T20:44:04.811 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-cryptography-3.2.1-6.el8.x86_64 14/32 2023-08-27T20:44:04.993 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 28/32 2023-08-27T20:44:05.041 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pyOpenSSL-19.0.0-1.el8.noarch 15/32 2023-08-27T20:44:05.172 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 29/32 2023-08-27T20:44:05.376 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-trustme-0.6.0-4.el8.noarch 16/32 2023-08-27T20:44:05.566 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 29/32 2023-08-27T20:44:05.617 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-08-27T20:44:05.766 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 30/32 2023-08-27T20:44:05.953 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-bcrypt-3.1.6-2.el8.1.x86_64 18/32 2023-08-27T20:44:05.982 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 30/32 2023-08-27T20:44:06.166 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 31/32 2023-08-27T20:44:06.184 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 31/32 2023-08-27T20:44:06.197 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-werkzeug-0.12.2-4.el8.noarch 19/32 2023-08-27T20:44:06.324 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 32/32 2023-08-27T20:44:06.385 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pytz-2017.2-11.el8.noarch 20/32 2023-08-27T20:44:06.597 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-tempora-1.14.1-5.el8.noarch 21/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 32/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 1/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 2/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 3/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 4/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 5/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1016.g4fe39952.el8. 6/32 2023-08-27T20:44:06.733 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-08-27T20:44:06.734 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-08-27T20:44:06.735 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-08-27T20:44:06.736 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-08-27T20:44:06.865 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-portend-2.6-1.el8.noarch 22/32 2023-08-27T20:44:07.112 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-cherrypy-18.4.0-1.el8.noarch 23/32 2023-08-27T20:44:07.373 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-markupsafe-0.23-19.el8.x86_64 24/32 2023-08-27T20:44:07.728 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-mako-1.0.6-14.el8.noarch 25/32 2023-08-27T20:44:08.405 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-08-27T20:44:08.405 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-modules-core-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-08-27T20:44:08.406 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jaraco-6.2-6.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-mako-1.0.6-14.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-08-27T20:44:08.407 INFO:teuthology.orchestra.run.smithi136.stdout: python3-portend-2.6-1.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pycparser-2.14-14.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pytz-2017.2-11.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-08-27T20:44:08.408 INFO:teuthology.orchestra.run.smithi136.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-08-27T20:44:08.409 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:08.409 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:08.691 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-base 2023-08-27T20:44:08.944 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pecan-1.3.2-9.el8.noarch 26/32 2023-08-27T20:44:09.336 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mgr-modules-core-2:17.2.6-1016.g4fe39952.el8. 27/32 2023-08-27T20:44:09.537 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:17 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:09.604 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 28/32 2023-08-27T20:44:10.869 INFO:teuthology.orchestra.run.smithi136.stdout:Package ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:44:11.048 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:11.049 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:44:11.050 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:11.086 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 28/32 2023-08-27T20:44:11.147 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install cephadm 2023-08-27T20:44:11.271 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 29/32 2023-08-27T20:44:11.662 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 29/32 2023-08-27T20:44:11.857 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 30/32 2023-08-27T20:44:11.992 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:19 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:12.073 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 30/32 2023-08-27T20:44:12.241 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 31/32 2023-08-27T20:44:12.258 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 31/32 2023-08-27T20:44:12.423 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 32/32 2023-08-27T20:44:12.830 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 32/32 2023-08-27T20:44:12.830 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 1/32 2023-08-27T20:44:12.830 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 2/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 3/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 4/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 5/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mgr-modules-core-2:17.2.6-1016.g4fe39952.el8. 6/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-mako-1.0.6-14.el8.noarch 7/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-markupsafe-0.23-19.el8.x86_64 8/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pyOpenSSL-19.0.0-1.el8.noarch 9/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pytz-2017.2-11.el8.noarch 10/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-werkzeug-0.12.2-4.el8.noarch 11/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-cffi-1.11.5-6.el8.x86_64 12/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-cryptography-3.2.1-6.el8.x86_64 13/32 2023-08-27T20:44:12.831 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pycparser-2.14-14.el8.noarch 14/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-bcrypt-3.1.6-2.el8.1.x86_64 15/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-beautifulsoup4-4.6.3-2.el8.1.noarch 16/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-cheroot-8.5.2-1.el8.noarch 17/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-cherrypy-18.4.0-1.el8.noarch 18/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-jaraco-6.2-6.el8.noarch 19/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-jaraco-functools-2.0-4.el8.noarch 20/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-logutils-0.3.5-11.el8.noarch 21/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-more-itertools-7.2.0-3.el8.noarch 22/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pecan-1.3.2-9.el8.noarch 23/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-portend-2.6-1.el8.noarch 24/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-simplegeneric-0.8.1-17.el8.noarch 25/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-singledispatch-3.4.0.3-18.el8.noarch 26/32 2023-08-27T20:44:12.832 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-tempora-1.14.1-5.el8.noarch 27/32 2023-08-27T20:44:12.833 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-trustme-0.6.0-4.el8.noarch 28/32 2023-08-27T20:44:12.833 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-waitress-1.4.3-1.el8.noarch 29/32 2023-08-27T20:44:12.833 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-webob-1.8.5-1.el8.1.noarch 30/32 2023-08-27T20:44:12.833 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-webtest-2.0.33-1.el8.noarch 31/32 2023-08-27T20:44:13.480 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:13.481 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:13.481 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:44:13.481 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:13.481 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:44:13.481 INFO:teuthology.orchestra.run.smithi136.stdout: cephadm noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 80 k 2023-08-27T20:44:13.481 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:13.481 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:44:13.482 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:13.482 INFO:teuthology.orchestra.run.smithi136.stdout:Install 1 Package 2023-08-27T20:44:13.482 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:13.482 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 80 k 2023-08-27T20:44:13.482 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 363 k 2023-08-27T20:44:13.482 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:44:13.728 INFO:teuthology.orchestra.run.smithi136.stdout:cephadm-17.2.6-1016.g4fe39952.el8.noarch.rpm 325 kB/s | 80 kB 00:00 2023-08-27T20:44:13.729 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:13.729 INFO:teuthology.orchestra.run.smithi136.stdout:Total 323 kB/s | 80 kB 00:00 2023-08-27T20:44:13.729 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:44:13.737 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:44:13.737 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:44:13.739 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:44:13.742 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:44:13.786 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:44:13.967 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-zc-lockfile-2.0-2.el8.noarch 32/32 2023-08-27T20:44:13.967 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:13.967 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:44:13.967 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:13.967 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mds-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:13.967 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:13.967 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-modules-core-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mon-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-osd-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-bcrypt-3.1.6-2.el8.1.x86_64 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-beautifulsoup4-4.6.3-2.el8.1.noarch 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cffi-1.11.5-6.el8.x86_64 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cheroot-8.5.2-1.el8.noarch 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cherrypy-18.4.0-1.el8.noarch 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cryptography-3.2.1-6.el8.x86_64 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jaraco-6.2-6.el8.noarch 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jaraco-functools-2.0-4.el8.noarch 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-logutils-0.3.5-11.el8.noarch 2023-08-27T20:44:13.968 INFO:teuthology.orchestra.run.smithi093.stdout: python3-mako-1.0.6-14.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-markupsafe-0.23-19.el8.x86_64 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-more-itertools-7.2.0-3.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pecan-1.3.2-9.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-portend-2.6-1.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyOpenSSL-19.0.0-1.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pycparser-2.14-14.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pytz-2017.2-11.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-simplegeneric-0.8.1-17.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-singledispatch-3.4.0.3-18.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-tempora-1.14.1-5.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-trustme-0.6.0-4.el8.noarch 2023-08-27T20:44:13.969 INFO:teuthology.orchestra.run.smithi093.stdout: python3-waitress-1.4.3-1.el8.noarch 2023-08-27T20:44:13.970 INFO:teuthology.orchestra.run.smithi093.stdout: python3-webob-1.8.5-1.el8.1.noarch 2023-08-27T20:44:13.970 INFO:teuthology.orchestra.run.smithi093.stdout: python3-webtest-2.0.33-1.el8.noarch 2023-08-27T20:44:13.970 INFO:teuthology.orchestra.run.smithi093.stdout: python3-werkzeug-0.12.2-4.el8.noarch 2023-08-27T20:44:13.970 INFO:teuthology.orchestra.run.smithi093.stdout: python3-zc-lockfile-2.0-2.el8.noarch 2023-08-27T20:44:13.970 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:13.970 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:14.211 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-base 2023-08-27T20:44:14.269 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:14.416 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:14.640 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:15.062 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:22 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:44:15.235 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:15.568 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:15.568 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:44:15.569 INFO:teuthology.orchestra.run.smithi136.stdout: cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:15.569 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:15.569 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:15.571 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-immutable-object-cache 2023-08-27T20:44:16.372 INFO:teuthology.orchestra.run.smithi093.stdout:Package ceph-base-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:44:16.432 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:24 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:16.546 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:44:16.547 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:44:16.548 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:16.644 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install cephadm 2023-08-27T20:44:17.485 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:24 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:44:17.942 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:17.942 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repo Size 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 162 k 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout:Install 1 Package 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 162 k 2023-08-27T20:44:17.943 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 490 k 2023-08-27T20:44:17.944 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:44:18.241 INFO:teuthology.orchestra.run.smithi136.stdout:ceph-immutable-object-cache-17.2.6-1016.g4fe399 543 kB/s | 162 kB 00:00 2023-08-27T20:44:18.242 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:18.242 INFO:teuthology.orchestra.run.smithi136.stdout:Total 541 kB/s | 162 kB 00:00 2023-08-27T20:44:18.242 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:44:18.252 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:44:18.253 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:44:18.299 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:44:18.303 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:44:18.426 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:44:18.966 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:44:18.967 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:18.967 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:44:18.967 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:18.967 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:44:18.967 INFO:teuthology.orchestra.run.smithi093.stdout: cephadm noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 80 k 2023-08-27T20:44:18.967 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:18.967 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:44:18.968 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:18.968 INFO:teuthology.orchestra.run.smithi093.stdout:Install 1 Package 2023-08-27T20:44:18.968 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:18.968 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 80 k 2023-08-27T20:44:18.968 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 363 k 2023-08-27T20:44:18.968 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:44:19.644 INFO:teuthology.orchestra.run.smithi093.stdout:cephadm-17.2.6-1016.g4fe39952.el8.noarch.rpm 119 kB/s | 80 kB 00:00 2023-08-27T20:44:19.644 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:19.645 INFO:teuthology.orchestra.run.smithi093.stdout:Total 118 kB/s | 80 kB 00:00 2023-08-27T20:44:19.645 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:44:19.654 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:44:19.654 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:44:19.656 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:44:19.659 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:44:20.302 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:44:20.405 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952. 1/1 2023-08-27T20:44:20.935 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:21.091 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952. 1/1 2023-08-27T20:44:21.124 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:21.365 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:21.694 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952. 1/1 2023-08-27T20:44:21.694 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:21.694 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:44:21.694 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:21.694 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:21.694 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:21.877 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-mgr 2023-08-27T20:44:22.031 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/1 2023-08-27T20:44:22.032 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:22.032 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:44:22.032 INFO:teuthology.orchestra.run.smithi093.stdout: cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:22.032 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:22.032 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:22.254 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-immutable-object-cache 2023-08-27T20:44:22.736 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:30 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:23.117 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:30 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:44:24.064 INFO:teuthology.orchestra.run.smithi136.stdout:Package ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:44:24.239 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:24.241 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:44:24.241 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:24.344 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-mgr-dashboard 2023-08-27T20:44:24.606 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:44:24.607 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:24.607 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repo Size 2023-08-27T20:44:24.607 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:24.607 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:44:24.608 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-immutable-object-cache x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 162 k 2023-08-27T20:44:24.608 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:24.608 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:44:24.608 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:24.608 INFO:teuthology.orchestra.run.smithi093.stdout:Install 1 Package 2023-08-27T20:44:24.608 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:24.609 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 162 k 2023-08-27T20:44:24.609 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 490 k 2023-08-27T20:44:24.609 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:44:24.789 INFO:teuthology.orchestra.run.smithi093.stdout:ceph-immutable-object-cache-17.2.6-1016.g4fe399 896 kB/s | 162 kB 00:00 2023-08-27T20:44:24.790 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:24.790 INFO:teuthology.orchestra.run.smithi093.stdout:Total 889 kB/s | 162 kB 00:00 2023-08-27T20:44:24.790 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:44:24.800 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:44:24.801 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:44:24.848 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:44:24.851 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:44:24.976 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:44:25.197 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952. 1/1 2023-08-27T20:44:25.201 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:33 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:25.606 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952. 1/1 2023-08-27T20:44:26.699 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 1.7 M 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 23 k 2023-08-27T20:44:26.701 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 14 k 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout:Install 6 Packages 2023-08-27T20:44:26.702 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:26.703 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 2.0 M 2023-08-27T20:44:26.703 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 8.0 M 2023-08-27T20:44:26.703 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:44:26.770 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952. 1/1 2023-08-27T20:44:26.771 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:26.771 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:44:26.771 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-immutable-object-cache-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:44:26.771 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:26.771 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:27.000 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-mgr 2023-08-27T20:44:27.053 INFO:teuthology.orchestra.run.smithi136.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1016.g4fe 147 kB/s | 23 kB 00:00 2023-08-27T20:44:27.070 INFO:teuthology.orchestra.run.smithi136.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1016.g4fe3 83 kB/s | 14 kB 00:00 2023-08-27T20:44:27.195 INFO:teuthology.orchestra.run.smithi136.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1016.g4fe39952 5.7 MB/s | 1.7 MB 00:00 2023-08-27T20:44:27.329 INFO:teuthology.orchestra.run.smithi136.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 130 kB/s | 34 kB 00:00 2023-08-27T20:44:27.396 INFO:teuthology.orchestra.run.smithi136.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 124 kB/s | 43 kB 00:00 2023-08-27T20:44:27.496 INFO:teuthology.orchestra.run.smithi136.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 655 kB/s | 196 kB 00:00 2023-08-27T20:44:27.500 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:27.500 INFO:teuthology.orchestra.run.smithi136.stdout:Total 2.5 MB/s | 2.0 MB 00:00 2023-08-27T20:44:27.511 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:44:27.520 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:44:27.520 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:44:27.600 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:44:27.603 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:44:27.857 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:44:27.860 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:34 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:44:28.061 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-08-27T20:44:28.360 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-08-27T20:44:28.564 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-08-27T20:44:28.700 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1016.g4fe39952.el8.n 4/6 2023-08-27T20:44:29.048 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1016.g4fe39952.el8. 5/6 2023-08-27T20:44:29.183 INFO:teuthology.orchestra.run.smithi093.stdout:Package ceph-mgr-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:44:29.291 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarc 6/6 2023-08-27T20:44:29.357 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:44:29.359 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:44:29.359 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:29.456 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-mgr-dashboard 2023-08-27T20:44:29.590 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarc 6/6 2023-08-27T20:44:29.590 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1016.g4fe39952.el8. 1/6 2023-08-27T20:44:29.590 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarc 2/6 2023-08-27T20:44:29.590 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1016.g4fe39952.el8.n 3/6 2023-08-27T20:44:29.590 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-08-27T20:44:29.590 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-08-27T20:44:30.209 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-08-27T20:44:30.209 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:30.209 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:44:30.209 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-grafana-dashboards-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:30.209 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:30.210 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-prometheus-alerts-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:30.210 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-08-27T20:44:30.210 INFO:teuthology.orchestra.run.smithi136.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-08-27T20:44:30.210 INFO:teuthology.orchestra.run.smithi136.stdout: python3-routes-2.4.1-12.el8.noarch 2023-08-27T20:44:30.210 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:30.210 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:30.307 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:37 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:44:30.334 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-mgr-diskprediction-local 2023-08-27T20:44:31.188 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:39 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:31.801 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-dashboard noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 1.7 M 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-grafana-dashboards noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 23 k 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-prometheus-alerts noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 14 k 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jwt noarch 1.6.1-2.el8 baseos 43 k 2023-08-27T20:44:31.802 INFO:teuthology.orchestra.run.smithi093.stdout: python3-repoze-lru noarch 0.7-6.el8 epel 34 k 2023-08-27T20:44:31.803 INFO:teuthology.orchestra.run.smithi093.stdout: python3-routes noarch 2.4.1-12.el8 epel 196 k 2023-08-27T20:44:31.803 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:31.803 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:44:31.803 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:31.803 INFO:teuthology.orchestra.run.smithi093.stdout:Install 6 Packages 2023-08-27T20:44:31.803 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:31.804 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 2.0 M 2023-08-27T20:44:31.804 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 8.0 M 2023-08-27T20:44:31.804 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:44:32.224 INFO:teuthology.orchestra.run.smithi093.stdout:(1/6): ceph-grafana-dashboards-17.2.6-1016.g4fe 156 kB/s | 23 kB 00:00 2023-08-27T20:44:32.275 INFO:teuthology.orchestra.run.smithi093.stdout:(2/6): ceph-prometheus-alerts-17.2.6-1016.g4fe3 73 kB/s | 14 kB 00:00 2023-08-27T20:44:32.417 INFO:teuthology.orchestra.run.smithi093.stdout:(3/6): ceph-mgr-dashboard-17.2.6-1016.g4fe39952 5.0 MB/s | 1.7 MB 00:00 2023-08-27T20:44:32.526 INFO:teuthology.orchestra.run.smithi093.stdout:(4/6): python3-repoze-lru-0.7-6.el8.noarch.rpm 134 kB/s | 34 kB 00:00 2023-08-27T20:44:32.609 INFO:teuthology.orchestra.run.smithi093.stdout:(5/6): python3-jwt-1.6.1-2.el8.noarch.rpm 111 kB/s | 43 kB 00:00 2023-08-27T20:44:32.688 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:32.689 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================================ 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================================ 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 7.4 M 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-08-27T20:44:32.690 INFO:teuthology.orchestra.run.smithi136.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-08-27T20:44:32.691 INFO:teuthology.orchestra.run.smithi136.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-08-27T20:44:32.691 INFO:teuthology.orchestra.run.smithi136.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-08-27T20:44:32.691 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:32.691 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:44:32.691 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================================ 2023-08-27T20:44:32.691 INFO:teuthology.orchestra.run.smithi136.stdout:Install 8 Packages 2023-08-27T20:44:32.691 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:32.692 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 35 M 2023-08-27T20:44:32.692 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 210 M 2023-08-27T20:44:32.692 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:44:32.785 INFO:teuthology.orchestra.run.smithi093.stdout:(6/6): python3-routes-2.4.1-12.el8.noarch.rpm 534 kB/s | 196 kB 00:00 2023-08-27T20:44:32.790 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:32.790 INFO:teuthology.orchestra.run.smithi093.stdout:Total 2.0 MB/s | 2.0 MB 00:00 2023-08-27T20:44:32.801 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:44:32.810 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:44:32.811 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:44:32.892 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:44:32.895 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:44:33.157 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:44:33.296 INFO:teuthology.orchestra.run.smithi136.stdout:(1/8): ceph-mgr-diskprediction-local-17.2.6-101 14 MB/s | 7.4 MB 00:00 2023-08-27T20:44:33.426 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-repoze-lru-0.7-6.el8.noarch 1/6 2023-08-27T20:44:33.721 INFO:teuthology.orchestra.run.smithi136.stdout:(2/8): openblas-0.3.15-4.el8.x86_64.rpm 5.0 MB/s | 4.7 MB 00:00 2023-08-27T20:44:33.729 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-routes-2.4.1-12.el8.noarch 2/6 2023-08-27T20:44:33.789 INFO:teuthology.orchestra.run.smithi136.stdout:(3/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 4.8 MB/s | 4.9 MB 00:01 2023-08-27T20:44:33.830 INFO:teuthology.orchestra.run.smithi136.stdout:(4/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.2 MB/s | 225 kB 00:00 2023-08-27T20:44:33.956 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-jwt-1.6.1-2.el8.noarch 3/6 2023-08-27T20:44:34.077 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-prometheus-alerts-2:17.2.6-1016.g4fe39952.el8.n 4/6 2023-08-27T20:44:34.139 INFO:teuthology.orchestra.run.smithi136.stdout:(5/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 4.4 MB/s | 3.7 MB 00:00 2023-08-27T20:44:34.440 INFO:teuthology.orchestra.run.smithi136.stdout:(6/8): python3-scipy-1.0.0-21.module_el8.5.0+77 21 MB/s | 14 MB 00:00 2023-08-27T20:44:34.461 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-grafana-dashboards-2:17.2.6-1016.g4fe39952.el8. 5/6 2023-08-27T20:44:34.557 INFO:teuthology.orchestra.run.smithi136.stdout:(7/8): libgfortran-8.5.0-20.el8.x86_64.rpm 879 kB/s | 645 kB 00:00 2023-08-27T20:44:34.623 INFO:teuthology.orchestra.run.smithi136.stdout:(8/8): libquadmath-8.5.0-20.el8.x86_64.rpm 356 kB/s | 172 kB 00:00 2023-08-27T20:44:34.624 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:34.624 INFO:teuthology.orchestra.run.smithi136.stdout:Total 18 MB/s | 35 MB 00:01 2023-08-27T20:44:34.700 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarc 6/6 2023-08-27T20:44:34.857 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:44:34.876 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:44:34.876 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:44:35.004 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarc 6/6 2023-08-27T20:44:35.005 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-grafana-dashboards-2:17.2.6-1016.g4fe39952.el8. 1/6 2023-08-27T20:44:35.005 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarc 2/6 2023-08-27T20:44:35.005 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-prometheus-alerts-2:17.2.6-1016.g4fe39952.el8.n 3/6 2023-08-27T20:44:35.005 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-jwt-1.6.1-2.el8.noarch 4/6 2023-08-27T20:44:35.005 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-repoze-lru-0.7-6.el8.noarch 5/6 2023-08-27T20:44:35.249 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:44:35.252 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:44:35.619 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-routes-2.4.1-12.el8.noarch 6/6 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-grafana-dashboards-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-dashboard-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-prometheus-alerts-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jwt-1.6.1-2.el8.noarch 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: python3-repoze-lru-0.7-6.el8.noarch 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: python3-routes-2.4.1-12.el8.noarch 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:35.620 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:35.735 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-mgr-diskprediction-local 2023-08-27T20:44:35.827 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:44:35.976 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-08-27T20:44:36.110 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-08-27T20:44:36.275 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-08-27T20:44:36.587 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:43 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:44:36.881 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-08-27T20:44:37.859 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-08-27T20:44:38.076 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:44:38.077 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================================ 2023-08-27T20:44:38.077 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:44:38.077 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================================ 2023-08-27T20:44:38.077 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:44:38.077 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-diskprediction-local noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 7.4 M 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: libgfortran x86_64 8.5.0-20.el8 baseos 645 k 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: libquadmath x86_64 8.5.0-20.el8 baseos 172 k 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: openblas x86_64 0.3.15-4.el8 appstream 4.7 M 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: openblas-threads x86_64 0.3.15-4.el8 appstream 4.9 M 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: python3-numpy x86_64 1:1.14.3-10.el8 appstream 3.7 M 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: python3-numpy-f2py x86_64 1:1.14.3-10.el8 appstream 225 k 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: python3-scipy x86_64 1.0.0-21.module_el8.5.0+771+e5d9a225 appstream 14 M 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:44:38.078 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================================ 2023-08-27T20:44:38.079 INFO:teuthology.orchestra.run.smithi093.stdout:Install 8 Packages 2023-08-27T20:44:38.079 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:38.079 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 35 M 2023-08-27T20:44:38.079 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 210 M 2023-08-27T20:44:38.079 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:44:38.470 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-08-27T20:44:38.612 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-08-27T20:44:38.858 INFO:teuthology.orchestra.run.smithi093.stdout:(1/8): ceph-mgr-diskprediction-local-17.2.6-101 11 MB/s | 7.4 MB 00:00 2023-08-27T20:44:39.042 INFO:teuthology.orchestra.run.smithi093.stdout:(2/8): openblas-threads-0.3.15-4.el8.x86_64.rpm 5.5 MB/s | 4.9 MB 00:00 2023-08-27T20:44:39.151 INFO:teuthology.orchestra.run.smithi093.stdout:(3/8): python3-numpy-f2py-1.14.3-10.el8.x86_64. 2.0 MB/s | 225 kB 00:00 2023-08-27T20:44:39.259 INFO:teuthology.orchestra.run.smithi093.stdout:(4/8): openblas-0.3.15-4.el8.x86_64.rpm 4.3 MB/s | 4.7 MB 00:01 2023-08-27T20:44:39.286 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-08-27T20:44:39.620 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-08-27T20:44:39.760 INFO:teuthology.orchestra.run.smithi093.stdout:(5/8): python3-scipy-1.0.0-21.module_el8.5.0+77 22 MB/s | 14 MB 00:00 2023-08-27T20:44:39.842 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-27T20:44:39.860 INFO:teuthology.orchestra.run.smithi093.stdout:(6/8): python3-numpy-1.14.3-10.el8.x86_64.rpm 3.7 MB/s | 3.7 MB 00:01 2023-08-27T20:44:40.019 INFO:teuthology.orchestra.run.smithi093.stdout:(7/8): libgfortran-8.5.0-20.el8.x86_64.rpm 850 kB/s | 645 kB 00:00 2023-08-27T20:44:40.277 INFO:teuthology.orchestra.run.smithi093.stdout:(8/8): libquadmath-8.5.0-20.el8.x86_64.rpm 333 kB/s | 172 kB 00:00 2023-08-27T20:44:40.278 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:40.278 INFO:teuthology.orchestra.run.smithi093.stdout:Total 16 MB/s | 35 MB 00:02 2023-08-27T20:44:40.517 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:44:40.535 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:44:40.535 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:44:40.937 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:44:40.940 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:44:41.521 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:44:41.598 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-27T20:44:41.670 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : libquadmath-8.5.0-20.el8.x86_64 1/8 2023-08-27T20:44:41.842 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: libquadmath-8.5.0-20.el8.x86_64 1/8 2023-08-27T20:44:41.977 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : libgfortran-8.5.0-20.el8.x86_64 2/8 2023-08-27T20:44:42.700 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: libgfortran-8.5.0-20.el8.x86_64 2/8 2023-08-27T20:44:42.844 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-08-27T20:44:43.124 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe3995 8/8 2023-08-27T20:44:43.344 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : openblas-0.3.15-4.el8.x86_64 3/8 2023-08-27T20:44:43.448 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe3995 8/8 2023-08-27T20:44:43.448 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe3995 1/8 2023-08-27T20:44:43.448 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-08-27T20:44:43.448 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-08-27T20:44:43.449 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-08-27T20:44:43.449 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-08-27T20:44:43.449 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-08-27T20:44:43.449 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-08-27T20:44:43.998 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: openblas-0.3.15-4.el8.x86_64 3/8 2023-08-27T20:44:44.087 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-08-27T20:44:44.087 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:44.087 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:44:44.087 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:44.087 INFO:teuthology.orchestra.run.smithi136.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout: openblas-0.3.15-4.el8.x86_64 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:44.088 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:44.198 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-08-27T20:44:44.211 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-mgr-rook 2023-08-27T20:44:44.882 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: openblas-threads-0.3.15-4.el8.x86_64 4/8 2023-08-27T20:44:45.070 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:01:53 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:45.248 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-numpy-1:1.14.3-10.el8.x86_64 5/8 2023-08-27T20:44:45.519 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-27T20:44:46.547 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:46.549 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:46.549 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:44:46.549 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:46.549 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:44:46.549 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-rook noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 47 k 2023-08-27T20:44:46.549 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:44:46.549 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-08-27T20:44:46.550 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:46.551 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:44:46.551 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:46.551 INFO:teuthology.orchestra.run.smithi136.stdout:Install 13 Packages 2023-08-27T20:44:46.551 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:46.551 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 1.9 M 2023-08-27T20:44:46.551 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 25 M 2023-08-27T20:44:46.551 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:44:47.151 INFO:teuthology.orchestra.run.smithi136.stdout:(1/13): ceph-mgr-rook-17.2.6-1016.g4fe39952.el8 256 kB/s | 47 kB 00:00 2023-08-27T20:44:47.208 INFO:teuthology.orchestra.run.smithi136.stdout:(2/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 110 kB/s | 27 kB 00:00 2023-08-27T20:44:47.224 INFO:teuthology.orchestra.run.smithi136.stdout:(3/13): python3-jsonpointer-1.10-11.el8.noarch. 77 kB/s | 20 kB 00:00 2023-08-27T20:44:47.270 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 6/8 2023-08-27T20:44:47.383 INFO:teuthology.orchestra.run.smithi136.stdout:(4/13): python3-pyasn1-modules-0.3.7-6.el8.noar 627 kB/s | 110 kB 00:00 2023-08-27T20:44:47.559 INFO:teuthology.orchestra.run.smithi136.stdout:(5/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 310 kB/s | 126 kB 00:00 2023-08-27T20:44:47.692 INFO:teuthology.orchestra.run.smithi136.stdout:(6/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 331 kB/s | 155 kB 00:00 2023-08-27T20:44:47.709 INFO:teuthology.orchestra.run.smithi136.stdout:(7/13): python3-requests-oauthlib-1.0.0-1.el8.n 131 kB/s | 43 kB 00:00 2023-08-27T20:44:47.835 INFO:teuthology.orchestra.run.smithi136.stdout:(8/13): python3-cachetools-3.1.1-4.el8.noarch.r 120 kB/s | 33 kB 00:00 2023-08-27T20:44:47.909 INFO:teuthology.orchestra.run.smithi136.stdout:(9/13): python3-certifi-2018.10.15-7.el8.noarch 75 kB/s | 16 kB 00:00 2023-08-27T20:44:47.985 INFO:teuthology.orchestra.run.smithi136.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 295 kB/s | 81 kB 00:00 2023-08-27T20:44:48.069 INFO:teuthology.orchestra.run.smithi136.stdout:(11/13): python3-rsa-4.9-2.el8.noarch.rpm 405 kB/s | 64 kB 00:00 2023-08-27T20:44:48.107 INFO:teuthology.orchestra.run.smithi136.stdout:(12/13): python3-websocket-client-0.56.0-5.el8. 503 kB/s | 61 kB 00:00 2023-08-27T20:44:48.186 INFO:teuthology.orchestra.run.smithi136.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 3.3 MB/s | 1.2 MB 00:00 2023-08-27T20:44:48.189 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:48.189 INFO:teuthology.orchestra.run.smithi136.stdout:Total 1.2 MB/s | 1.9 MB 00:01 2023-08-27T20:44:48.250 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:44:48.266 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:44:48.266 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:44:48.353 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:44:48.356 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:44:48.536 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 7/8 2023-08-27T20:44:48.743 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe3995 8/8 2023-08-27T20:44:48.906 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:44:49.076 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe3995 8/8 2023-08-27T20:44:49.076 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe3995 1/8 2023-08-27T20:44:49.077 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : openblas-0.3.15-4.el8.x86_64 2/8 2023-08-27T20:44:49.077 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : openblas-threads-0.3.15-4.el8.x86_64 3/8 2023-08-27T20:44:49.077 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-numpy-1:1.14.3-10.el8.x86_64 4/8 2023-08-27T20:44:49.077 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-numpy-f2py-1:1.14.3-10.el8.x86_64 5/8 2023-08-27T20:44:49.077 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x 6/8 2023-08-27T20:44:49.077 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : libgfortran-8.5.0-20.el8.x86_64 7/8 2023-08-27T20:44:49.175 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-08-27T20:44:49.373 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-08-27T20:44:49.600 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-08-27T20:44:49.755 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : libquadmath-8.5.0-20.el8.x86_64 8/8 2023-08-27T20:44:49.755 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:49.755 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:44:49.755 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-diskprediction-local-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:49.755 INFO:teuthology.orchestra.run.smithi093.stdout: libgfortran-8.5.0-20.el8.x86_64 2023-08-27T20:44:49.755 INFO:teuthology.orchestra.run.smithi093.stdout: libquadmath-8.5.0-20.el8.x86_64 2023-08-27T20:44:49.755 INFO:teuthology.orchestra.run.smithi093.stdout: openblas-0.3.15-4.el8.x86_64 2023-08-27T20:44:49.756 INFO:teuthology.orchestra.run.smithi093.stdout: openblas-threads-0.3.15-4.el8.x86_64 2023-08-27T20:44:49.756 INFO:teuthology.orchestra.run.smithi093.stdout: python3-numpy-1:1.14.3-10.el8.x86_64 2023-08-27T20:44:49.756 INFO:teuthology.orchestra.run.smithi093.stdout: python3-numpy-f2py-1:1.14.3-10.el8.x86_64 2023-08-27T20:44:49.756 INFO:teuthology.orchestra.run.smithi093.stdout: python3-scipy-1.0.0-21.module_el8.5.0+771+e5d9a225.x86_64 2023-08-27T20:44:49.756 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:49.756 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:49.863 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-08-27T20:44:49.878 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-mgr-rook 2023-08-27T20:44:50.053 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-08-27T20:44:50.277 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-08-27T20:44:50.536 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-08-27T20:44:50.730 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:01:57 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:44:50.810 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-08-27T20:44:51.854 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-08-27T20:44:52.120 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-08-27T20:44:52.235 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-rook noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 47 k 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cachetools noarch 3.1.1-4.el8 epel 33 k 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout: python3-certifi noarch 2018.10.15-7.el8 epel 16 k 2023-08-27T20:44:52.237 INFO:teuthology.orchestra.run.smithi093.stdout: python3-google-auth noarch 1:1.1.1-10.el8 epel 81 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jsonpatch noarch 1.21-2.el8 appstream 27 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jsonpointer noarch 1.10-11.el8 appstream 20 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-kubernetes noarch 1:11.0.0-6.el8 epel 1.2 M 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-oauthlib noarch 2.1.0-1.el8 baseos 155 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyasn1 noarch 0.3.7-6.el8 appstream 126 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyasn1-modules noarch 0.3.7-6.el8 appstream 110 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-requests-oauthlib noarch 1.0.0-1.el8 baseos 43 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rsa noarch 4.9-2.el8 epel 64 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: python3-websocket-client noarch 0.56.0-5.el8 epel 61 k 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:44:52.238 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:44:52.239 INFO:teuthology.orchestra.run.smithi093.stdout:Install 13 Packages 2023-08-27T20:44:52.239 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:52.239 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 1.9 M 2023-08-27T20:44:52.239 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 25 M 2023-08-27T20:44:52.239 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:44:52.312 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-08-27T20:44:52.489 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-08-27T20:44:52.655 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 13/13 2023-08-27T20:44:52.934 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 13/13 2023-08-27T20:44:52.934 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 1/13 2023-08-27T20:44:52.934 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-08-27T20:44:52.934 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-08-27T20:44:52.935 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-08-27T20:44:52.935 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-08-27T20:44:52.935 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-08-27T20:44:52.935 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-08-27T20:44:52.935 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-08-27T20:44:52.935 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-08-27T20:44:52.935 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-08-27T20:44:52.936 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-08-27T20:44:52.936 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-08-27T20:44:53.017 INFO:teuthology.orchestra.run.smithi093.stdout:(1/13): ceph-mgr-rook-17.2.6-1016.g4fe39952.el8 288 kB/s | 47 kB 00:00 2023-08-27T20:44:53.108 INFO:teuthology.orchestra.run.smithi093.stdout:(2/13): python3-jsonpointer-1.10-11.el8.noarch. 78 kB/s | 20 kB 00:00 2023-08-27T20:44:53.133 INFO:teuthology.orchestra.run.smithi093.stdout:(3/13): python3-jsonpatch-1.21-2.el8.noarch.rpm 94 kB/s | 27 kB 00:00 2023-08-27T20:44:53.309 INFO:teuthology.orchestra.run.smithi093.stdout:(4/13): python3-pyasn1-modules-0.3.7-6.el8.noar 549 kB/s | 110 kB 00:00 2023-08-27T20:44:53.426 INFO:teuthology.orchestra.run.smithi093.stdout:(5/13): python3-pyasn1-0.3.7-6.el8.noarch.rpm 309 kB/s | 126 kB 00:00 2023-08-27T20:44:53.568 INFO:teuthology.orchestra.run.smithi093.stdout:(6/13): python3-cachetools-3.1.1-4.el8.noarch.r 233 kB/s | 33 kB 00:00 2023-08-27T20:44:53.590 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-08-27T20:44:53.590 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:53.590 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:44:53.590 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:53.590 INFO:teuthology.orchestra.run.smithi136.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-08-27T20:44:53.590 INFO:teuthology.orchestra.run.smithi136.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-08-27T20:44:53.591 INFO:teuthology.orchestra.run.smithi136.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-08-27T20:44:53.591 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-08-27T20:44:53.591 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-08-27T20:44:53.591 INFO:teuthology.orchestra.run.smithi136.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-08-27T20:44:53.591 INFO:teuthology.orchestra.run.smithi136.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-08-27T20:44:53.591 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-08-27T20:44:53.592 INFO:teuthology.orchestra.run.smithi136.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-08-27T20:44:53.592 INFO:teuthology.orchestra.run.smithi136.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-08-27T20:44:53.592 INFO:teuthology.orchestra.run.smithi136.stdout: python3-rsa-4.9-2.el8.noarch 2023-08-27T20:44:53.592 INFO:teuthology.orchestra.run.smithi136.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-08-27T20:44:53.592 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:53.592 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:44:53.601 INFO:teuthology.orchestra.run.smithi093.stdout:(7/13): python3-oauthlib-2.1.0-1.el8.noarch.rpm 331 kB/s | 155 kB 00:00 2023-08-27T20:44:53.635 INFO:teuthology.orchestra.run.smithi093.stdout:(8/13): python3-certifi-2018.10.15-7.el8.noarch 243 kB/s | 16 kB 00:00 2023-08-27T20:44:53.661 INFO:teuthology.orchestra.run.smithi093.stdout:(9/13): python3-requests-oauthlib-1.0.0-1.el8.n 122 kB/s | 43 kB 00:00 2023-08-27T20:44:53.702 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-mgr-cephadm 2023-08-27T20:44:53.777 INFO:teuthology.orchestra.run.smithi093.stdout:(10/13): python3-google-auth-1.1.1-10.el8.noarc 463 kB/s | 81 kB 00:00 2023-08-27T20:44:53.844 INFO:teuthology.orchestra.run.smithi093.stdout:(11/13): python3-rsa-4.9-2.el8.noarch.rpm 350 kB/s | 64 kB 00:00 2023-08-27T20:44:53.861 INFO:teuthology.orchestra.run.smithi093.stdout:(12/13): python3-websocket-client-0.56.0-5.el8. 735 kB/s | 61 kB 00:00 2023-08-27T20:44:53.920 INFO:teuthology.orchestra.run.smithi093.stdout:(13/13): python3-kubernetes-11.0.0-6.el8.noarch 4.1 MB/s | 1.2 MB 00:00 2023-08-27T20:44:53.920 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:53.920 INFO:teuthology.orchestra.run.smithi093.stdout:Total 1.1 MB/s | 1.9 MB 00:01 2023-08-27T20:44:53.979 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:44:53.995 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:44:53.995 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:44:54.080 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:44:54.083 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:44:54.554 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:02 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:44:54.683 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:44:54.936 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pyasn1-0.3.7-6.el8.noarch 1/13 2023-08-27T20:44:55.159 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-pyasn1-modules-0.3.7-6.el8.noarch 2/13 2023-08-27T20:44:55.411 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-rsa-4.9-2.el8.noarch 3/13 2023-08-27T20:44:55.665 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-websocket-client-0.56.0-5.el8.noarch 4/13 2023-08-27T20:44:55.897 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-certifi-2018.10.15-7.el8.noarch 5/13 2023-08-27T20:44:56.070 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:44:56.071 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:56.071 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 121 k 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:44:56.072 INFO:teuthology.orchestra.run.smithi136.stdout:Install 5 Packages 2023-08-27T20:44:56.073 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:44:56.073 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 6.0 M 2023-08-27T20:44:56.073 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 26 M 2023-08-27T20:44:56.073 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:44:56.163 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-cachetools-3.1.1-4.el8.noarch 6/13 2023-08-27T20:44:56.463 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-google-auth-1:1.1.1-10.el8.noarch 7/13 2023-08-27T20:44:56.560 INFO:teuthology.orchestra.run.smithi136.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1016.g4fe39952.e 410 kB/s | 121 kB 00:00 2023-08-27T20:44:56.738 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-oauthlib-2.1.0-1.el8.noarch 8/13 2023-08-27T20:44:56.861 INFO:teuthology.orchestra.run.smithi136.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 902 kB/s | 538 kB 00:00 2023-08-27T20:44:57.086 INFO:teuthology.orchestra.run.smithi136.stdout:(3/5): python3-babel-2.5.1-7.el8.noarch.rpm 5.8 MB/s | 4.8 MB 00:00 2023-08-27T20:44:57.186 INFO:teuthology.orchestra.run.smithi136.stdout:(4/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 760 kB/s | 476 kB 00:00 2023-08-27T20:44:57.245 INFO:teuthology.orchestra.run.smithi136.stdout:(5/5): python3-natsort-7.1.1-2.el8.noarch.rpm 157 kB/s | 60 kB 00:00 2023-08-27T20:44:57.247 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:44:57.247 INFO:teuthology.orchestra.run.smithi136.stdout:Total 5.1 MB/s | 6.0 MB 00:01 2023-08-27T20:44:57.298 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:44:57.310 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:44:57.310 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:44:57.412 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:44:57.416 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:44:57.673 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:44:57.765 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-requests-oauthlib-1.0.0-1.el8.noarch 9/13 2023-08-27T20:44:57.985 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 1/5 2023-08-27T20:44:58.040 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-kubernetes-1:11.0.0-6.el8.noarch 10/13 2023-08-27T20:44:58.307 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-jsonpointer-1.10-11.el8.noarch 11/13 2023-08-27T20:44:58.534 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-jsonpatch-1.21-2.el8.noarch 12/13 2023-08-27T20:44:58.734 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 13/13 2023-08-27T20:44:58.931 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 13/13 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 1/13 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-jsonpatch-1.21-2.el8.noarch 2/13 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-jsonpointer-1.10-11.el8.noarch 3/13 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pyasn1-0.3.7-6.el8.noarch 4/13 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-pyasn1-modules-0.3.7-6.el8.noarch 5/13 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-oauthlib-2.1.0-1.el8.noarch 6/13 2023-08-27T20:44:59.025 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-requests-oauthlib-1.0.0-1.el8.noarch 7/13 2023-08-27T20:44:59.026 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-cachetools-3.1.1-4.el8.noarch 8/13 2023-08-27T20:44:59.026 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-certifi-2018.10.15-7.el8.noarch 9/13 2023-08-27T20:44:59.026 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-google-auth-1:1.1.1-10.el8.noarch 10/13 2023-08-27T20:44:59.026 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-kubernetes-1:11.0.0-6.el8.noarch 11/13 2023-08-27T20:44:59.026 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-rsa-4.9-2.el8.noarch 12/13 2023-08-27T20:44:59.304 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 3/5 2023-08-27T20:44:59.572 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-websocket-client-0.56.0-5.el8.noarch 13/13 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-rook-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: python3-cachetools-3.1.1-4.el8.noarch 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: python3-certifi-2018.10.15-7.el8.noarch 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: python3-google-auth-1:1.1.1-10.el8.noarch 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jsonpatch-1.21-2.el8.noarch 2023-08-27T20:44:59.716 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jsonpointer-1.10-11.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: python3-kubernetes-1:11.0.0-6.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: python3-oauthlib-2.1.0-1.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyasn1-0.3.7-6.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: python3-pyasn1-modules-0.3.7-6.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: python3-requests-oauthlib-1.0.0-1.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: python3-rsa-4.9-2.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: python3-websocket-client-0.56.0-5.el8.noarch 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:44:59.717 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:44:59.771 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 5/5 2023-08-27T20:44:59.831 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-mgr-cephadm 2023-08-27T20:45:00.051 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 5/5 2023-08-27T20:45:00.052 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/5 2023-08-27T20:45:00.052 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/5 2023-08-27T20:45:00.052 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/5 2023-08-27T20:45:00.052 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-08-27T20:45:00.647 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 5/5 2023-08-27T20:45:00.647 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:00.647 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:45:00.648 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:45:00.648 INFO:teuthology.orchestra.run.smithi136.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-08-27T20:45:00.648 INFO:teuthology.orchestra.run.smithi136.stdout: python3-babel-2.5.1-7.el8.noarch 2023-08-27T20:45:00.648 INFO:teuthology.orchestra.run.smithi136.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-08-27T20:45:00.648 INFO:teuthology.orchestra.run.smithi136.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-08-27T20:45:00.648 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:00.648 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:00.683 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:07 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:00.786 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install ceph-fuse 2023-08-27T20:45:01.641 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:09 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:02.177 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:02.178 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:02.178 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:45:02.178 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-cephadm noarch 2:17.2.6-1016.g4fe39952.el8 ceph-noarch 121 k 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout: python3-asyncssh noarch 2.7.0-2.el8 epel 476 k 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout: python3-babel noarch 2.5.1-7.el8 appstream 4.8 M 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jinja2 noarch 2.10.1-3.el8 appstream 538 k 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout: python3-natsort noarch 7.1.1-2.el8 epel 60 k 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:02.179 INFO:teuthology.orchestra.run.smithi093.stdout:Install 5 Packages 2023-08-27T20:45:02.180 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:02.180 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 6.0 M 2023-08-27T20:45:02.180 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 26 M 2023-08-27T20:45:02.180 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:45:03.122 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:03.122 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repository Size 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-fuse x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 874 k 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout:Installing dependencies: 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:03.123 INFO:teuthology.orchestra.run.smithi136.stdout:Install 2 Packages 2023-08-27T20:45:03.124 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:03.124 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 957 k 2023-08-27T20:45:03.124 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 3.0 M 2023-08-27T20:45:03.124 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:45:03.420 INFO:teuthology.orchestra.run.smithi136.stdout:(1/2): ceph-fuse-17.2.6-1016.g4fe39952.el8.x86_ 3.3 MB/s | 874 kB 00:00 2023-08-27T20:45:03.662 INFO:teuthology.orchestra.run.smithi136.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 167 kB/s | 83 kB 00:00 2023-08-27T20:45:03.663 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:03.663 INFO:teuthology.orchestra.run.smithi136.stdout:Total 1.7 MB/s | 957 kB 00:00 2023-08-27T20:45:03.674 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:45:03.695 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:45:03.695 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:45:03.782 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:45:03.785 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:45:03.951 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:45:04.362 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-08-27T20:45:04.393 INFO:teuthology.orchestra.run.smithi093.stdout:(1/5): ceph-mgr-cephadm-17.2.6-1016.g4fe39952.e 627 kB/s | 121 kB 00:00 2023-08-27T20:45:04.561 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 2/2 2023-08-27T20:45:04.801 INFO:teuthology.orchestra.run.smithi093.stdout:(2/5): python3-jinja2-2.10.1-3.el8.noarch.rpm 894 kB/s | 538 kB 00:00 2023-08-27T20:45:04.888 INFO:teuthology.orchestra.run.smithi093.stdout:(3/5): python3-asyncssh-2.7.0-2.el8.noarch.rpm 961 kB/s | 476 kB 00:00 2023-08-27T20:45:04.956 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 2/2 2023-08-27T20:45:04.956 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/2 2023-08-27T20:45:05.110 INFO:teuthology.orchestra.run.smithi093.stdout:(4/5): python3-babel-2.5.1-7.el8.noarch.rpm 5.3 MB/s | 4.8 MB 00:00 2023-08-27T20:45:05.135 INFO:teuthology.orchestra.run.smithi093.stdout:(5/5): python3-natsort-7.1.1-2.el8.noarch.rpm 180 kB/s | 60 kB 00:00 2023-08-27T20:45:05.136 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:05.137 INFO:teuthology.orchestra.run.smithi093.stdout:Total 2.0 MB/s | 6.0 MB 00:02 2023-08-27T20:45:05.191 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:45:05.203 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:45:05.203 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:45:05.306 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:45:05.309 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:45:05.568 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-08-27T20:45:05.568 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:05.568 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:45:05.568 INFO:teuthology.orchestra.run.smithi136.stdout: ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-08-27T20:45:05.568 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:05.568 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:05.591 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:45:05.780 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install librados-devel 2023-08-27T20:45:05.922 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-natsort-7.1.1-2.el8.noarch 1/5 2023-08-27T20:45:06.637 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:14 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:06.902 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-asyncssh-2.7.0-2.el8.noarch 2/5 2023-08-27T20:45:07.343 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-babel-2.5.1-7.el8.noarch 3/5 2023-08-27T20:45:07.625 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : python3-jinja2-2.10.1-3.el8.noarch 4/5 2023-08-27T20:45:07.841 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 5/5 2023-08-27T20:45:08.120 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 5/5 2023-08-27T20:45:08.120 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 1/5 2023-08-27T20:45:08.121 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-babel-2.5.1-7.el8.noarch 2/5 2023-08-27T20:45:08.121 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-jinja2-2.10.1-3.el8.noarch 3/5 2023-08-27T20:45:08.121 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-asyncssh-2.7.0-2.el8.noarch 4/5 2023-08-27T20:45:08.136 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:08.136 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:08.136 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repo Size 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout: librados-devel x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 130 k 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout:Install 1 Package 2023-08-27T20:45:08.137 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:08.138 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 130 k 2023-08-27T20:45:08.138 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 464 k 2023-08-27T20:45:08.138 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:45:08.434 INFO:teuthology.orchestra.run.smithi136.stdout:librados-devel-17.2.6-1016.g4fe39952.el8.x86_64 439 kB/s | 130 kB 00:00 2023-08-27T20:45:08.434 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:08.435 INFO:teuthology.orchestra.run.smithi136.stdout:Total 437 kB/s | 130 kB 00:00 2023-08-27T20:45:08.435 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:45:08.440 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:45:08.440 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:45:08.490 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:45:08.493 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:45:08.617 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:45:08.750 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : python3-natsort-7.1.1-2.el8.noarch 5/5 2023-08-27T20:45:08.750 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:08.750 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:45:08.751 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-mgr-cephadm-2:17.2.6-1016.g4fe39952.el8.noarch 2023-08-27T20:45:08.751 INFO:teuthology.orchestra.run.smithi093.stdout: python3-asyncssh-2.7.0-2.el8.noarch 2023-08-27T20:45:08.751 INFO:teuthology.orchestra.run.smithi093.stdout: python3-babel-2.5.1-7.el8.noarch 2023-08-27T20:45:08.751 INFO:teuthology.orchestra.run.smithi093.stdout: python3-jinja2-2.10.1-3.el8.noarch 2023-08-27T20:45:08.751 INFO:teuthology.orchestra.run.smithi093.stdout: python3-natsort-7.1.1-2.el8.noarch 2023-08-27T20:45:08.751 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:08.751 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:08.876 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:08.881 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install ceph-fuse 2023-08-27T20:45:09.158 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:09.734 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:16 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:10.408 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:10.408 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:10.408 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:45:10.408 INFO:teuthology.orchestra.run.smithi136.stdout: librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:10.408 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:10.408 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:10.654 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install libcephfs2 2023-08-27T20:45:11.229 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repository Size 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-fuse x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 874 k 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout:Installing dependencies: 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout: fuse x86_64 2.9.7-17.el8 baseos 83 k 2023-08-27T20:45:11.230 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:11.231 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:45:11.231 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:11.231 INFO:teuthology.orchestra.run.smithi093.stdout:Install 2 Packages 2023-08-27T20:45:11.231 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:11.231 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 957 k 2023-08-27T20:45:11.232 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 3.0 M 2023-08-27T20:45:11.232 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:45:11.531 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:19 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:11.835 INFO:teuthology.orchestra.run.smithi093.stdout:(1/2): ceph-fuse-17.2.6-1016.g4fe39952.el8.x86_ 2.1 MB/s | 874 kB 00:00 2023-08-27T20:45:11.885 INFO:teuthology.orchestra.run.smithi093.stdout:(2/2): fuse-2.9.7-17.el8.x86_64.rpm 179 kB/s | 83 kB 00:00 2023-08-27T20:45:11.886 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:11.886 INFO:teuthology.orchestra.run.smithi093.stdout:Total 1.4 MB/s | 957 kB 00:00 2023-08-27T20:45:11.897 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:45:11.917 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:45:11.918 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:45:12.004 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:45:12.007 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:45:12.167 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:45:12.448 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : fuse-2.9.7-17.el8.x86_64 1/2 2023-08-27T20:45:12.673 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 2/2 2023-08-27T20:45:12.856 INFO:teuthology.orchestra.run.smithi136.stdout:Package libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:13.033 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:13.035 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:45:13.035 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:13.071 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 2/2 2023-08-27T20:45:13.071 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/2 2023-08-27T20:45:13.139 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install libcephfs-devel 2023-08-27T20:45:13.676 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : fuse-2.9.7-17.el8.x86_64 2/2 2023-08-27T20:45:13.676 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:13.676 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:45:13.676 INFO:teuthology.orchestra.run.smithi093.stdout: ceph-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 fuse-2.9.7-17.el8.x86_64 2023-08-27T20:45:13.676 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:13.676 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:13.990 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:21 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:14.651 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install librados-devel 2023-08-27T20:45:15.479 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:15.770 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:22 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:15.772 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:15.773 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repo Size 2023-08-27T20:45:15.773 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:15.773 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:45:15.773 INFO:teuthology.orchestra.run.smithi136.stdout: libcephfs-devel x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 25 k 2023-08-27T20:45:15.773 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:15.773 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:45:15.775 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:15.776 INFO:teuthology.orchestra.run.smithi136.stdout:Install 1 Package 2023-08-27T20:45:15.776 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:15.776 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 25 k 2023-08-27T20:45:15.776 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 109 k 2023-08-27T20:45:15.776 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:45:16.033 INFO:teuthology.orchestra.run.smithi136.stdout:libcephfs-devel-17.2.6-1016.g4fe39952.el8.x86_6 45 kB/s | 25 kB 00:00 2023-08-27T20:45:16.034 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:16.035 INFO:teuthology.orchestra.run.smithi136.stdout:Total 45 kB/s | 25 kB 00:00 2023-08-27T20:45:16.035 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:45:16.039 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:45:16.039 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:45:16.042 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:45:16.045 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:45:16.252 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:45:16.433 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:16.706 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:17.010 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:17.011 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:17.011 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repo Size 2023-08-27T20:45:17.011 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:17.011 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:45:17.011 INFO:teuthology.orchestra.run.smithi093.stdout: librados-devel x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 130 k 2023-08-27T20:45:17.011 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:17.012 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:45:17.012 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:17.012 INFO:teuthology.orchestra.run.smithi093.stdout:Install 1 Package 2023-08-27T20:45:17.012 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:17.012 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 130 k 2023-08-27T20:45:17.012 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 464 k 2023-08-27T20:45:17.012 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:45:17.201 INFO:teuthology.orchestra.run.smithi093.stdout:librados-devel-17.2.6-1016.g4fe39952.el8.x86_64 687 kB/s | 130 kB 00:00 2023-08-27T20:45:17.202 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:17.202 INFO:teuthology.orchestra.run.smithi093.stdout:Total 683 kB/s | 130 kB 00:00 2023-08-27T20:45:17.202 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:45:17.207 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:45:17.207 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:45:17.255 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:45:17.258 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:45:17.302 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:17.302 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:17.302 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:45:17.302 INFO:teuthology.orchestra.run.smithi136.stdout: libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:17.302 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:17.302 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:17.372 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:45:17.411 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install librados2 2023-08-27T20:45:17.651 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:17.950 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:18.271 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:26 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:18.576 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:18.576 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:18.576 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:45:18.576 INFO:teuthology.orchestra.run.smithi093.stdout: librados-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:18.577 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:18.577 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:18.786 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install libcephfs2 2023-08-27T20:45:19.583 INFO:teuthology.orchestra.run.smithi136.stdout:Package librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:19.650 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:26 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:19.757 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:19.759 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:45:19.759 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:19.852 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install librbd1 2023-08-27T20:45:20.710 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:28 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:20.980 INFO:teuthology.orchestra.run.smithi093.stdout:Package libcephfs2-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:21.160 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:21.161 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:45:21.161 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:21.580 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install libcephfs-devel 2023-08-27T20:45:22.035 INFO:teuthology.orchestra.run.smithi136.stdout:Package librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:22.214 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:22.215 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:45:22.216 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:22.306 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install python3-rados 2023-08-27T20:45:22.426 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:29 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:23.201 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:31 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:23.909 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:23.982 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:23.982 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repo Size 2023-08-27T20:45:23.982 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:23.982 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:45:23.982 INFO:teuthology.orchestra.run.smithi093.stdout: libcephfs-devel x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 25 k 2023-08-27T20:45:23.982 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:23.982 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:45:23.983 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:23.983 INFO:teuthology.orchestra.run.smithi093.stdout:Install 1 Package 2023-08-27T20:45:23.983 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:23.983 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 25 k 2023-08-27T20:45:23.983 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 109 k 2023-08-27T20:45:23.984 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:45:24.147 INFO:teuthology.orchestra.run.smithi093.stdout:libcephfs-devel-17.2.6-1016.g4fe39952.el8.x86_6 105 kB/s | 25 kB 00:00 2023-08-27T20:45:24.155 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:24.155 INFO:teuthology.orchestra.run.smithi093.stdout:Total 105 kB/s | 25 kB 00:00 2023-08-27T20:45:24.155 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:45:24.155 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:45:24.155 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:45:24.156 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:45:24.156 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:45:24.202 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:45:24.437 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:24.642 INFO:teuthology.orchestra.run.smithi136.stdout:Package python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:24.692 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:24.693 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:45:24.693 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:24.701 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:24.789 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install python3-rgw 2023-08-27T20:45:25.296 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:25.385 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:25.385 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:45:25.385 INFO:teuthology.orchestra.run.smithi093.stdout: libcephfs-devel-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:25.385 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:25.385 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:25.408 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install librados2 2023-08-27T20:45:25.648 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:33 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:26.259 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:33 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:26.955 INFO:teuthology.orchestra.run.smithi136.stdout:Package python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:27.147 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:27.147 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:45:27.148 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:27.231 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install python3-cephfs 2023-08-27T20:45:27.651 INFO:teuthology.orchestra.run.smithi093.stdout:Package librados2-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:27.824 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:28.127 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:36 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:28.128 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:45:28.128 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:28.128 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install librbd1 2023-08-27T20:45:28.995 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:35 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:29.404 INFO:teuthology.orchestra.run.smithi136.stdout:Package python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:29.578 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:29.580 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:45:29.580 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:29.675 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install python3-rbd 2023-08-27T20:45:30.306 INFO:teuthology.orchestra.run.smithi093.stdout:Package librbd1-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:30.483 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:30.483 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:45:30.483 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:30.531 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:38 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:30.580 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install python3-rados 2023-08-27T20:45:31.432 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:38 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:31.836 INFO:teuthology.orchestra.run.smithi136.stdout:Package python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:32.013 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:32.014 INFO:teuthology.orchestra.run.smithi136.stdout:Nothing to do. 2023-08-27T20:45:32.015 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:32.169 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install rbd-fuse 2023-08-27T20:45:32.748 INFO:teuthology.orchestra.run.smithi093.stdout:Package python3-rados-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:32.923 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:32.924 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:45:32.924 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:33.021 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install python3-rgw 2023-08-27T20:45:33.027 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:41 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:33.875 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:40 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:34.512 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:34.512 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:34.512 INFO:teuthology.orchestra.run.smithi136.stdout: Package Architecture Version Repository Size 2023-08-27T20:45:34.512 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout: rbd-fuse x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 90 k 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout:Install 1 Package 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:34.513 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 90 k 2023-08-27T20:45:34.514 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 264 k 2023-08-27T20:45:34.514 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:45:34.681 INFO:teuthology.orchestra.run.smithi136.stdout:rbd-fuse-17.2.6-1016.g4fe39952.el8.x86_64.rpm 540 kB/s | 90 kB 00:00 2023-08-27T20:45:34.682 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:34.682 INFO:teuthology.orchestra.run.smithi136.stdout:Total 536 kB/s | 90 kB 00:00 2023-08-27T20:45:34.682 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:45:34.691 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:45:34.691 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:45:34.740 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:45:34.743 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:45:34.847 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:45:35.070 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:35.194 INFO:teuthology.orchestra.run.smithi093.stdout:Package python3-rgw-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:35.361 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:35.369 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:35.371 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:45:35.371 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:35.469 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install python3-cephfs 2023-08-27T20:45:35.941 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:35.941 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:35.941 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:45:35.941 INFO:teuthology.orchestra.run.smithi136.stdout: rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:35.942 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:35.942 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:36.165 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install rbd-mirror 2023-08-27T20:45:36.324 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:43 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:37.026 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:45 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:37.625 INFO:teuthology.orchestra.run.smithi093.stdout:Package python3-cephfs-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:37.800 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:37.801 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:45:37.801 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:37.915 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install python3-rbd 2023-08-27T20:45:38.514 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:38.514 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout: Package Arch Version Repo Size 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout: rbd-mirror x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.5 M 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout:Install 1 Package 2023-08-27T20:45:38.515 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:38.516 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 3.5 M 2023-08-27T20:45:38.516 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 16 M 2023-08-27T20:45:38.516 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:45:38.770 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:45 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:39.043 INFO:teuthology.orchestra.run.smithi136.stdout:rbd-mirror-17.2.6-1016.g4fe39952.el8.x86_64.rpm 6.6 MB/s | 3.5 MB 00:00 2023-08-27T20:45:39.043 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:39.043 INFO:teuthology.orchestra.run.smithi136.stdout:Total 6.5 MB/s | 3.5 MB 00:00 2023-08-27T20:45:39.044 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:45:39.055 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:45:39.055 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:45:39.116 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:45:39.119 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:45:39.552 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:45:39.721 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:40.085 INFO:teuthology.orchestra.run.smithi093.stdout:Package python3-rbd-2:17.2.6-1016.g4fe39952.el8.x86_64 is already installed. 2023-08-27T20:45:40.130 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:40.258 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:40.260 INFO:teuthology.orchestra.run.smithi093.stdout:Nothing to do. 2023-08-27T20:45:40.260 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:40.357 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install rbd-fuse 2023-08-27T20:45:40.721 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:40.722 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:40.722 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:45:40.722 INFO:teuthology.orchestra.run.smithi136.stdout: rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:40.722 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:40.722 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:40.922 DEBUG:teuthology.orchestra.run.smithi136:> sudo yum -y install rbd-nbd 2023-08-27T20:45:41.211 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:48 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:41.787 INFO:teuthology.orchestra.run.smithi136.stdout:Last metadata expiration check: 0:02:49 ago on Sun 27 Aug 2023 08:42:52 PM UTC. 2023-08-27T20:45:42.695 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:42.695 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout: Package Architecture Version Repository Size 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout: rbd-fuse x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 90 k 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout:Install 1 Package 2023-08-27T20:45:42.696 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:42.697 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 90 k 2023-08-27T20:45:42.697 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 264 k 2023-08-27T20:45:42.697 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:45:42.944 INFO:teuthology.orchestra.run.smithi093.stdout:rbd-fuse-17.2.6-1016.g4fe39952.el8.x86_64.rpm 366 kB/s | 90 kB 00:00 2023-08-27T20:45:42.945 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:42.945 INFO:teuthology.orchestra.run.smithi093.stdout:Total 364 kB/s | 90 kB 00:00 2023-08-27T20:45:42.945 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:45:42.955 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:45:42.955 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:45:43.004 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:45:43.007 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:45:43.111 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:45:43.338 INFO:teuthology.orchestra.run.smithi136.stdout:Dependencies resolved. 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout: Package Architecture Version Repository Size 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout:Installing: 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout: rbd-nbd x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 173 k 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction Summary 2023-08-27T20:45:43.339 INFO:teuthology.orchestra.run.smithi136.stdout:================================================================================ 2023-08-27T20:45:43.340 INFO:teuthology.orchestra.run.smithi136.stdout:Install 1 Package 2023-08-27T20:45:43.340 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:43.340 INFO:teuthology.orchestra.run.smithi136.stdout:Total download size: 173 k 2023-08-27T20:45:43.340 INFO:teuthology.orchestra.run.smithi136.stdout:Installed size: 552 k 2023-08-27T20:45:43.340 INFO:teuthology.orchestra.run.smithi136.stdout:Downloading Packages: 2023-08-27T20:45:43.343 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:43.529 INFO:teuthology.orchestra.run.smithi136.stdout:rbd-nbd-17.2.6-1016.g4fe39952.el8.x86_64.rpm 916 kB/s | 173 kB 00:00 2023-08-27T20:45:43.529 INFO:teuthology.orchestra.run.smithi136.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:43.530 INFO:teuthology.orchestra.run.smithi136.stdout:Total 910 kB/s | 173 kB 00:00 2023-08-27T20:45:43.530 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction check 2023-08-27T20:45:43.540 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction check succeeded. 2023-08-27T20:45:43.540 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction test 2023-08-27T20:45:43.590 INFO:teuthology.orchestra.run.smithi136.stdout:Transaction test succeeded. 2023-08-27T20:45:43.593 INFO:teuthology.orchestra.run.smithi136.stdout:Running transaction 2023-08-27T20:45:43.632 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:43.713 INFO:teuthology.orchestra.run.smithi136.stdout: Preparing : 1/1 2023-08-27T20:45:43.901 INFO:teuthology.orchestra.run.smithi136.stdout: Installing : rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:44.178 INFO:teuthology.orchestra.run.smithi136.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:44.256 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:44.256 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:44.256 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:45:44.256 INFO:teuthology.orchestra.run.smithi093.stdout: rbd-fuse-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:44.256 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:44.256 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:44.438 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install rbd-mirror 2023-08-27T20:45:44.784 INFO:teuthology.orchestra.run.smithi136.stdout: Verifying : rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:44.784 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:44.784 INFO:teuthology.orchestra.run.smithi136.stdout:Installed: 2023-08-27T20:45:44.784 INFO:teuthology.orchestra.run.smithi136.stdout: rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:44.784 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:45:44.784 INFO:teuthology.orchestra.run.smithi136.stdout:Complete! 2023-08-27T20:45:44.994 DEBUG:teuthology.parallel:result is None 2023-08-27T20:45:45.295 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:52 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:46.779 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout: Package Arch Version Repo Size 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout: rbd-mirror x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 3.5 M 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:45:46.780 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:46.781 INFO:teuthology.orchestra.run.smithi093.stdout:Install 1 Package 2023-08-27T20:45:46.781 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:46.781 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 3.5 M 2023-08-27T20:45:46.781 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 16 M 2023-08-27T20:45:46.781 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:45:47.305 INFO:teuthology.orchestra.run.smithi093.stdout:rbd-mirror-17.2.6-1016.g4fe39952.el8.x86_64.rpm 6.6 MB/s | 3.5 MB 00:00 2023-08-27T20:45:47.305 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:47.306 INFO:teuthology.orchestra.run.smithi093.stdout:Total 6.6 MB/s | 3.5 MB 00:00 2023-08-27T20:45:47.306 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:45:47.317 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:45:47.317 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:45:47.379 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:45:47.382 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:45:47.826 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:45:48.052 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:48.471 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:49.055 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:49.055 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:49.055 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:45:49.055 INFO:teuthology.orchestra.run.smithi093.stdout: rbd-mirror-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:49.055 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:49.055 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:49.298 DEBUG:teuthology.orchestra.run.smithi093:> sudo yum -y install rbd-nbd 2023-08-27T20:45:50.160 INFO:teuthology.orchestra.run.smithi093.stdout:Last metadata expiration check: 0:02:57 ago on Sun 27 Aug 2023 08:42:53 PM UTC. 2023-08-27T20:45:51.662 INFO:teuthology.orchestra.run.smithi093.stdout:Dependencies resolved. 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout: Package Architecture Version Repository Size 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout:Installing: 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout: rbd-nbd x86_64 2:17.2.6-1016.g4fe39952.el8 ceph 173 k 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction Summary 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout:================================================================================ 2023-08-27T20:45:51.663 INFO:teuthology.orchestra.run.smithi093.stdout:Install 1 Package 2023-08-27T20:45:51.664 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:51.664 INFO:teuthology.orchestra.run.smithi093.stdout:Total download size: 173 k 2023-08-27T20:45:51.664 INFO:teuthology.orchestra.run.smithi093.stdout:Installed size: 552 k 2023-08-27T20:45:51.664 INFO:teuthology.orchestra.run.smithi093.stdout:Downloading Packages: 2023-08-27T20:45:51.949 INFO:teuthology.orchestra.run.smithi093.stdout:rbd-nbd-17.2.6-1016.g4fe39952.el8.x86_64.rpm 607 kB/s | 173 kB 00:00 2023-08-27T20:45:51.950 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:45:51.950 INFO:teuthology.orchestra.run.smithi093.stdout:Total 604 kB/s | 173 kB 00:00 2023-08-27T20:45:51.950 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction check 2023-08-27T20:45:51.960 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction check succeeded. 2023-08-27T20:45:51.960 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction test 2023-08-27T20:45:52.011 INFO:teuthology.orchestra.run.smithi093.stdout:Transaction test succeeded. 2023-08-27T20:45:52.014 INFO:teuthology.orchestra.run.smithi093.stdout:Running transaction 2023-08-27T20:45:52.152 INFO:teuthology.orchestra.run.smithi093.stdout: Preparing : 1/1 2023-08-27T20:45:52.342 INFO:teuthology.orchestra.run.smithi093.stdout: Installing : rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:52.645 INFO:teuthology.orchestra.run.smithi093.stdout: Running scriptlet: rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:53.234 INFO:teuthology.orchestra.run.smithi093.stdout: Verifying : rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 1/1 2023-08-27T20:45:53.234 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:53.235 INFO:teuthology.orchestra.run.smithi093.stdout:Installed: 2023-08-27T20:45:53.235 INFO:teuthology.orchestra.run.smithi093.stdout: rbd-nbd-2:17.2.6-1016.g4fe39952.el8.x86_64 2023-08-27T20:45:53.235 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:45:53.235 INFO:teuthology.orchestra.run.smithi093.stdout:Complete! 2023-08-27T20:45:53.458 DEBUG:teuthology.parallel:result is None 2023-08-27T20:45:53.458 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T20:45:53.458 INFO:teuthology.packaging:ref: None 2023-08-27T20:45:53.458 INFO:teuthology.packaging:tag: None 2023-08-27T20:45:53.458 INFO:teuthology.packaging:branch: quincy 2023-08-27T20:45:53.458 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:45:53.459 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-08-27T20:45:53.685 DEBUG:teuthology.orchestra.run.smithi093:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-08-27T20:45:53.708 INFO:teuthology.orchestra.run.smithi093.stdout:17.2.6-1016.g4fe39952.el8 2023-08-27T20:45:53.708 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1016.g4fe39952.el8 2023-08-27T20:45:53.708 INFO:teuthology.task.install:The correct ceph version 17.2.6-1016.g4fe39952 is installed. 2023-08-27T20:45:53.709 WARNING:teuthology.packaging:More than one of ref, tag, branch, or sha1 supplied; using branch 2023-08-27T20:45:53.709 INFO:teuthology.packaging:ref: None 2023-08-27T20:45:53.709 INFO:teuthology.packaging:tag: None 2023-08-27T20:45:53.709 INFO:teuthology.packaging:branch: quincy 2023-08-27T20:45:53.709 INFO:teuthology.packaging:sha1: 9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:45:53.709 DEBUG:teuthology.packaging:Querying https://shaman.ceph.com/api/search?status=ready&project=ceph&flavor=default&distros=centos%2F8%2Fx86_64&ref=quincy 2023-08-27T20:45:53.932 DEBUG:teuthology.orchestra.run.smithi136:> rpm -q ceph --qf '%{VERSION}-%{RELEASE}' 2023-08-27T20:45:53.959 INFO:teuthology.orchestra.run.smithi136.stdout:17.2.6-1016.g4fe39952.el8 2023-08-27T20:45:53.959 INFO:teuthology.packaging:The installed version of ceph is 17.2.6-1016.g4fe39952.el8 2023-08-27T20:45:53.959 INFO:teuthology.task.install:The correct ceph version 17.2.6-1016.g4fe39952 is installed. 2023-08-27T20:45:53.959 INFO:teuthology.task.install.util:Shipping valgrind.supp... 2023-08-27T20:45:53.960 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:45:53.960 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-08-27T20:45:53.988 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:45:53.988 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/home/ubuntu/cephtest/valgrind.supp 2023-08-27T20:45:54.031 INFO:teuthology.task.install.util:Shipping 'daemon-helper'... 2023-08-27T20:45:54.031 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:45:54.031 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/usr/bin/daemon-helper 2023-08-27T20:45:54.058 DEBUG:teuthology.orchestra.run.smithi093:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-08-27T20:45:54.125 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:45:54.125 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/usr/bin/daemon-helper 2023-08-27T20:45:54.155 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod a=rx -- /usr/bin/daemon-helper 2023-08-27T20:45:54.225 INFO:teuthology.task.install.util:Shipping 'adjust-ulimits'... 2023-08-27T20:45:54.225 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:45:54.225 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/usr/bin/adjust-ulimits 2023-08-27T20:45:54.252 DEBUG:teuthology.orchestra.run.smithi093:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-08-27T20:45:54.319 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:45:54.319 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/usr/bin/adjust-ulimits 2023-08-27T20:45:54.348 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod a=rx -- /usr/bin/adjust-ulimits 2023-08-27T20:45:54.417 INFO:teuthology.task.install.util:Shipping 'stdin-killer'... 2023-08-27T20:45:54.417 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:45:54.417 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/usr/bin/stdin-killer 2023-08-27T20:45:54.444 DEBUG:teuthology.orchestra.run.smithi093:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-08-27T20:45:54.510 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:45:54.510 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/usr/bin/stdin-killer 2023-08-27T20:45:54.539 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod a=rx -- /usr/bin/stdin-killer 2023-08-27T20:45:54.606 INFO:teuthology.run_tasks:Running task cephadm... 2023-08-27T20:45:54.686 INFO:tasks.cephadm:Config: {'cephadm_branch': 'quincy', 'cephadm_git_url': 'https://github.com/ceph/ceph', 'conf': {'osd': {'osd_class_default_list': '*', 'osd_class_load_list': '*', 'debug ms': 1, 'debug osd': 20, 'osd shutdown pgref assert': True}, 'global': {'mon election default strategy': 3}, 'mgr': {'debug mgr': 20, 'debug ms': 1}, 'mon': {'debug mon': 20, 'debug ms': 1, 'debug paxos': 20}}, 'image': 'quay.ceph.io/ceph-ci/ceph:quincy', 'create_rbd_pool': True, 'flavor': 'default', 'log-ignorelist': ['\\(MDS_ALL_DOWN\\)', '\\(MDS_UP_LESS_THAN_MAX\\)'], 'sha1': '9cf208207dfe23ffccb33308b22e51570cb45e51'} 2023-08-27T20:45:54.687 INFO:tasks.cephadm:Cluster image is quay.ceph.io/ceph-ci/ceph:quincy 2023-08-27T20:45:54.687 INFO:tasks.cephadm:Cluster fsid is b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:45:54.687 INFO:tasks.cephadm:Choosing monitor IPs and ports... 2023-08-27T20:45:54.687 INFO:tasks.cephadm:Monitor IPs: {'mon.a': '172.21.15.93', 'mon.c': '[v2:172.21.15.93:3301,v1:172.21.15.93:6790]', 'mon.b': '172.21.15.136'} 2023-08-27T20:45:54.688 INFO:tasks.cephadm:First mon is mon.a on smithi093 2023-08-27T20:45:54.688 INFO:tasks.cephadm:First mgr is y 2023-08-27T20:45:54.688 INFO:tasks.cephadm:Normalizing hostnames... 2023-08-27T20:45:54.688 DEBUG:teuthology.orchestra.run.smithi093:> sudo hostname $(hostname -s) 2023-08-27T20:45:54.715 DEBUG:teuthology.orchestra.run.smithi136:> sudo hostname $(hostname -s) 2023-08-27T20:45:54.744 INFO:tasks.cephadm:Downloading cephadm (repo https://github.com/ceph/ceph ref quincy)... 2023-08-27T20:45:54.744 DEBUG:teuthology.orchestra.run.smithi093:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-08-27T20:45:54.909 INFO:teuthology.orchestra.run.smithi093.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Aug 27 20:45 /home/ubuntu/cephtest/cephadm 2023-08-27T20:45:54.909 DEBUG:teuthology.orchestra.run.smithi136:> curl --silent https://raw.githubusercontent.com/ceph/ceph/quincy/src/cephadm/cephadm > /home/ubuntu/cephtest/cephadm && ls -l /home/ubuntu/cephtest/cephadm 2023-08-27T20:45:55.047 INFO:teuthology.orchestra.run.smithi136.stdout:-rw-rw-r--. 1 ubuntu ubuntu 366215 Aug 27 20:45 /home/ubuntu/cephtest/cephadm 2023-08-27T20:45:55.047 DEBUG:teuthology.orchestra.run.smithi093:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-08-27T20:45:55.066 DEBUG:teuthology.orchestra.run.smithi136:> test -s /home/ubuntu/cephtest/cephadm && test $(stat -c%s /home/ubuntu/cephtest/cephadm) -gt 1000 && chmod +x /home/ubuntu/cephtest/cephadm 2023-08-27T20:45:55.096 INFO:tasks.cephadm:Pulling image quay.ceph.io/ceph-ci/ceph:quincy on all hosts... 2023-08-27T20:45:55.096 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-08-27T20:45:55.108 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy pull 2023-08-27T20:45:56.128 INFO:teuthology.orchestra.run.smithi136.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-08-27T20:45:56.181 INFO:teuthology.orchestra.run.smithi093.stderr:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-08-27T20:46:28.877 INFO:teuthology.orchestra.run.smithi093.stdout:{ 2023-08-27T20:46:28.878 INFO:teuthology.orchestra.run.smithi093.stdout: "ceph_version": "ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable)", 2023-08-27T20:46:28.878 INFO:teuthology.orchestra.run.smithi093.stdout: "image_id": "d5e2f73f133dd20b6576b15830ef0b6d2e83d4adcdbe7c32f7624809f186725c", 2023-08-27T20:46:28.878 INFO:teuthology.orchestra.run.smithi093.stdout: "repo_digests": [ 2023-08-27T20:46:28.878 INFO:teuthology.orchestra.run.smithi093.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:9559dd6f9193811bda3cb3a7f08181842d6dcc49053477b47362b0f470b426a9" 2023-08-27T20:46:28.878 INFO:teuthology.orchestra.run.smithi093.stdout: ] 2023-08-27T20:46:28.878 INFO:teuthology.orchestra.run.smithi093.stdout:} 2023-08-27T20:46:34.276 INFO:teuthology.orchestra.run.smithi136.stdout:{ 2023-08-27T20:46:34.276 INFO:teuthology.orchestra.run.smithi136.stdout: "ceph_version": "ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable)", 2023-08-27T20:46:34.277 INFO:teuthology.orchestra.run.smithi136.stdout: "image_id": "d5e2f73f133dd20b6576b15830ef0b6d2e83d4adcdbe7c32f7624809f186725c", 2023-08-27T20:46:34.277 INFO:teuthology.orchestra.run.smithi136.stdout: "repo_digests": [ 2023-08-27T20:46:34.277 INFO:teuthology.orchestra.run.smithi136.stdout: "quay.ceph.io/ceph-ci/ceph@sha256:9559dd6f9193811bda3cb3a7f08181842d6dcc49053477b47362b0f470b426a9" 2023-08-27T20:46:34.277 INFO:teuthology.orchestra.run.smithi136.stdout: ] 2023-08-27T20:46:34.277 INFO:teuthology.orchestra.run.smithi136.stdout:} 2023-08-27T20:46:34.295 DEBUG:teuthology.orchestra.run.smithi093:> sudo mkdir -p /etc/ceph 2023-08-27T20:46:34.334 DEBUG:teuthology.orchestra.run.smithi136:> sudo mkdir -p /etc/ceph 2023-08-27T20:46:34.364 DEBUG:teuthology.orchestra.run.smithi093:> sudo chmod 777 /etc/ceph 2023-08-27T20:46:34.408 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 777 /etc/ceph 2023-08-27T20:46:34.435 INFO:tasks.cephadm:Writing seed config... 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [osd] osd_class_default_list = * 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [osd] osd_class_load_list = * 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [osd] debug ms = 1 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [osd] debug osd = 20 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [osd] osd shutdown pgref assert = True 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [global] mon election default strategy = 3 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [mgr] debug mgr = 20 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [mgr] debug ms = 1 2023-08-27T20:46:34.436 INFO:tasks.cephadm: override: [mon] debug mon = 20 2023-08-27T20:46:34.437 INFO:tasks.cephadm: override: [mon] debug ms = 1 2023-08-27T20:46:34.437 INFO:tasks.cephadm: override: [mon] debug paxos = 20 2023-08-27T20:46:34.437 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:46:34.437 DEBUG:teuthology.orchestra.run.smithi093:> dd of=/home/ubuntu/cephtest/seed.ceph.conf 2023-08-27T20:46:34.471 DEBUG:tasks.cephadm:Final config: [global] # make logging friendly to teuthology log_to_file = true log_to_stderr = false log to journald = false mon cluster log file level = debug mon clock drift allowed = 1.000 # replicate across OSDs, not hosts osd crush chooseleaf type = 0 #osd pool default size = 2 osd pool default erasure code profile = plugin=jerasure technique=reed_sol_van k=2 m=1 crush-failure-domain=osd # enable some debugging auth debug = true ms die on old message = true ms die on bug = true debug asserts on shutdown = true # adjust warnings mon max pg per osd = 10000# >= luminous mon pg warn max object skew = 0 mon osd allow primary affinity = true mon osd allow pg remap = true mon warn on legacy crush tunables = false mon warn on crush straw calc version zero = false mon warn on no sortbitwise = false mon warn on osd down out interval zero = false mon warn on too few osds = false mon_warn_on_pool_pg_num_not_power_of_two = false # disable pg_autoscaler by default for new pools osd_pool_default_pg_autoscale_mode = off # tests delete pools mon allow pool delete = true fsid = b79a52c0-451a-11ee-817b-1b7a5f1ba315 mon election default strategy = 3 [osd] osd scrub load threshold = 5.0 osd scrub max interval = 600 osd mclock profile = high_recovery_ops osd recover clone overlap = true osd recovery max chunk = 1048576 osd deep scrub update digest min age = 30 osd map max advance = 10 osd memory target autotune = true # debugging osd debug shutdown = true osd debug op order = true osd debug verify stray on activate = true osd debug pg log writeout = true osd debug verify cached snaps = true osd debug verify missing on start = true osd debug misdirected ops = true osd op queue = debug_random osd op queue cut off = debug_random osd shutdown pgref assert = True bdev debug aio = true osd sloppy crc = true osd_class_default_list = * osd_class_load_list = * debug ms = 1 debug osd = 20 [mgr] mon reweight min pgs per osd = 4 mon reweight min bytes per osd = 10 mgr/telemetry/nag = false debug mgr = 20 debug ms = 1 [mon] mon data avail warn = 5 mon mgr mkfs grace = 240 mon reweight min pgs per osd = 4 mon osd reporter subtree level = osd mon osd prime pg temp = true mon reweight min bytes per osd = 10 # rotate auth tickets quickly to exercise renewal paths auth mon ticket ttl = 660# 11m auth service ticket ttl = 240# 4m # don't complain about global id reclaim mon_warn_on_insecure_global_id_reclaim = false mon_warn_on_insecure_global_id_reclaim_allowed = false debug mon = 20 debug ms = 1 debug paxos = 20 [client.rgw] rgw cache enabled = true rgw enable ops log = true rgw enable usage log = true 2023-08-27T20:46:34.471 DEBUG:teuthology.orchestra.run.smithi093:mon.a> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.a.service 2023-08-27T20:46:34.513 DEBUG:teuthology.orchestra.run.smithi093:mgr.y> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.y.service 2023-08-27T20:46:34.546 INFO:journalctl@ceph.mon.a.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:46:34.547 INFO:tasks.cephadm:Bootstrapping... 2023-08-27T20:46:34.548 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy -v bootstrap --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 --config /home/ubuntu/cephtest/seed.ceph.conf --output-config /etc/ceph/ceph.conf --output-keyring /etc/ceph/ceph.client.admin.keyring --output-pub-ssh-key /home/ubuntu/cephtest/ceph.pub --mon-id a --mgr-id y --orphan-initial-daemons --skip-monitoring-stack --mon-ip 172.21.15.93 --skip-admin-label && sudo chmod +r /etc/ceph/ceph.client.admin.keyring 2023-08-27T20:46:34.580 INFO:journalctl@ceph.mgr.y.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:46:34.772 INFO:teuthology.orchestra.run.smithi093.stdout:-------------------------------------------------------------------------------- 2023-08-27T20:46:34.773 INFO:teuthology.orchestra.run.smithi093.stdout:cephadm ['--image', 'quay.ceph.io/ceph-ci/ceph:quincy', '-v', 'bootstrap', '--fsid', 'b79a52c0-451a-11ee-817b-1b7a5f1ba315', '--config', '/home/ubuntu/cephtest/seed.ceph.conf', '--output-config', '/etc/ceph/ceph.conf', '--output-keyring', '/etc/ceph/ceph.client.admin.keyring', '--output-pub-ssh-key', '/home/ubuntu/cephtest/ceph.pub', '--mon-id', 'a', '--mgr-id', 'y', '--orphan-initial-daemons', '--skip-monitoring-stack', '--mon-ip', '172.21.15.93', '--skip-admin-label'] 2023-08-27T20:46:34.805 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stdout 4.6.1 2023-08-27T20:46:34.806 INFO:teuthology.orchestra.run.smithi093.stderr:Specifying an fsid for your cluster offers no advantages and may increase the likelihood of fsid conflicts. 2023-08-27T20:46:34.806 INFO:teuthology.orchestra.run.smithi093.stdout:Verifying podman|docker is present... 2023-08-27T20:46:34.832 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stdout 4.6.1 2023-08-27T20:46:34.833 INFO:teuthology.orchestra.run.smithi093.stdout:Verifying lvm2 is present... 2023-08-27T20:46:34.833 INFO:teuthology.orchestra.run.smithi093.stdout:Verifying time synchronization is in place... 2023-08-27T20:46:34.839 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-08-27T20:46:34.840 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-08-27T20:46:34.846 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-08-27T20:46:34.846 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout inactive 2023-08-27T20:46:34.852 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout enabled 2023-08-27T20:46:34.858 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout active 2023-08-27T20:46:34.858 INFO:teuthology.orchestra.run.smithi093.stdout:Unit chronyd.service is enabled and running 2023-08-27T20:46:34.858 INFO:teuthology.orchestra.run.smithi093.stdout:Repeating the final host check... 2023-08-27T20:46:34.884 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stdout 4.6.1 2023-08-27T20:46:34.884 INFO:teuthology.orchestra.run.smithi093.stdout:podman (/bin/podman) version 4.6.1 is present 2023-08-27T20:46:34.884 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl is present 2023-08-27T20:46:34.885 INFO:teuthology.orchestra.run.smithi093.stdout:lvcreate is present 2023-08-27T20:46:34.891 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl is-enabled chrony.service 2023-08-27T20:46:34.891 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Failed to get unit file state for chrony.service: No such file or directory 2023-08-27T20:46:34.896 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 3 from systemctl is-active chrony.service 2023-08-27T20:46:34.896 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout inactive 2023-08-27T20:46:34.902 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout enabled 2023-08-27T20:46:34.908 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout active 2023-08-27T20:46:34.908 INFO:teuthology.orchestra.run.smithi093.stdout:Unit chronyd.service is enabled and running 2023-08-27T20:46:34.909 INFO:teuthology.orchestra.run.smithi093.stdout:Host looks OK 2023-08-27T20:46:34.909 INFO:teuthology.orchestra.run.smithi093.stdout:Cluster fsid: b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:34.909 INFO:teuthology.orchestra.run.smithi093.stdout:Acquiring lock 140495360760296 on /run/cephadm/b79a52c0-451a-11ee-817b-1b7a5f1ba315.lock 2023-08-27T20:46:34.909 INFO:teuthology.orchestra.run.smithi093.stdout:Lock 140495360760296 acquired on /run/cephadm/b79a52c0-451a-11ee-817b-1b7a5f1ba315.lock 2023-08-27T20:46:34.910 INFO:teuthology.orchestra.run.smithi093.stdout:Verifying IP 172.21.15.93 port 3300 ... 2023-08-27T20:46:34.910 INFO:teuthology.orchestra.run.smithi093.stdout:Verifying IP 172.21.15.93 port 6789 ... 2023-08-27T20:46:34.911 INFO:teuthology.orchestra.run.smithi093.stdout:Base mon IP(s) is [172.21.15.93:3300, 172.21.15.93:6789], mon addrv is [v2:172.21.15.93:3300,v1:172.21.15.93:6789] 2023-08-27T20:46:34.913 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout default via 172.21.15.254 dev enp3s0f1 proto dhcp src 172.21.15.93 metric 100 2023-08-27T20:46:34.913 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout 172.21.0.0/20 dev enp3s0f1 proto kernel scope link src 172.21.15.93 metric 100 2023-08-27T20:46:34.916 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout ::1 dev lo proto kernel metric 256 pref medium 2023-08-27T20:46:34.916 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout fe80::/64 dev enp3s0f1 proto kernel metric 256 pref medium 2023-08-27T20:46:34.916 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout default via fe80::327c:5e00:6487:71e0 dev enp3s0f1 proto ra metric 1024 expires 1770sec hoplimit 64 pref medium 2023-08-27T20:46:34.918 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout 1: lo: mtu 65536 state UNKNOWN qlen 1000 2023-08-27T20:46:34.919 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout inet6 ::1/128 scope host 2023-08-27T20:46:34.919 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-08-27T20:46:34.919 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout 5: enp3s0f1: mtu 1500 state UP qlen 1000 2023-08-27T20:46:34.919 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout inet6 fe80::ec4:7aff:fe8f:d635/64 scope link 2023-08-27T20:46:34.919 INFO:teuthology.orchestra.run.smithi093.stdout:/sbin/ip: stdout valid_lft forever preferred_lft forever 2023-08-27T20:46:34.920 INFO:teuthology.orchestra.run.smithi093.stdout:Mon IP `172.21.15.93` is in CIDR network `172.21.0.0/20` 2023-08-27T20:46:34.920 INFO:teuthology.orchestra.run.smithi093.stdout:Mon IP `172.21.15.93` is in CIDR network `172.21.0.0/20` 2023-08-27T20:46:34.920 INFO:teuthology.orchestra.run.smithi093.stdout:Inferred mon public CIDR from local network configuration ['172.21.0.0/20', '172.21.0.0/20'] 2023-08-27T20:46:34.920 INFO:teuthology.orchestra.run.smithi093.stdout:Internal network (--cluster-network) has not been provided, OSD replication will default to the public_network 2023-08-27T20:46:34.921 INFO:teuthology.orchestra.run.smithi093.stdout:Pulling container image quay.ceph.io/ceph-ci/ceph:quincy... 2023-08-27T20:46:35.316 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stdout d5e2f73f133dd20b6576b15830ef0b6d2e83d4adcdbe7c32f7624809f186725c 2023-08-27T20:46:35.317 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stderr Trying to pull quay.ceph.io/ceph-ci/ceph:quincy... 2023-08-27T20:46:35.317 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stderr Getting image source signatures 2023-08-27T20:46:35.317 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stderr Copying blob sha256:057c3839e332631dc98de31c07539ecbea335ba43cb8bb4683c4946c0da0535c 2023-08-27T20:46:35.317 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stderr Copying blob sha256:6a80a02e8363a5635d9205c99753186ff615be95cb0610737aa6f93ac7a9100b 2023-08-27T20:46:35.317 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stderr Copying config sha256:d5e2f73f133dd20b6576b15830ef0b6d2e83d4adcdbe7c32f7624809f186725c 2023-08-27T20:46:35.317 INFO:teuthology.orchestra.run.smithi093.stdout:/bin/podman: stderr Writing manifest to image destination 2023-08-27T20:46:35.918 INFO:teuthology.orchestra.run.smithi093.stdout:ceph: stdout ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable) 2023-08-27T20:46:36.537 INFO:teuthology.orchestra.run.smithi093.stdout:Ceph version: ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable) 2023-08-27T20:46:36.537 INFO:teuthology.orchestra.run.smithi093.stdout:Extracting ceph user uid/gid from container image... 2023-08-27T20:46:36.539 INFO:teuthology.orchestra.run.smithi093.stdout:stat: stdout 167 167 2023-08-27T20:46:36.539 INFO:teuthology.orchestra.run.smithi093.stdout:Creating initial keys... 2023-08-27T20:46:37.050 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph-authtool: stdout AQAstutk9eSvJxAAvtjoqsYFFEqSAUeK4T4/Cw== 2023-08-27T20:46:37.591 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph-authtool: stdout AQAttutkg8iTDhAAMNE1Od/k7/dqRruIEPGIuA== 2023-08-27T20:46:38.114 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph-authtool: stdout AQAttutk7kgcLhAA3d+gVMqrLajiFPdt/9VQ9w== 2023-08-27T20:46:38.345 INFO:teuthology.orchestra.run.smithi093.stdout:Creating initial monmap... 2023-08-27T20:46:38.663 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: monmap file /tmp/monmap 2023-08-27T20:46:38.663 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/monmaptool: stdout setting min_mon_release = octopus 2023-08-27T20:46:38.663 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: set fsid to b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:38.663 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/monmaptool: stdout /usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-08-27T20:46:38.663 INFO:teuthology.orchestra.run.smithi093.stdout:monmaptool for a [v2:172.21.15.93:3300,v1:172.21.15.93:6789] on /usr/bin/monmaptool: monmap file /tmp/monmap 2023-08-27T20:46:38.663 INFO:teuthology.orchestra.run.smithi093.stdout:setting min_mon_release = octopus 2023-08-27T20:46:38.663 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/monmaptool: set fsid to b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:38.664 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/monmaptool: writing epoch 0 to /tmp/monmap (1 monitors) 2023-08-27T20:46:38.664 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:46:38.664 INFO:teuthology.orchestra.run.smithi093.stdout:Creating mon... 2023-08-27T20:46:39.205 INFO:teuthology.orchestra.run.smithi093.stdout:create mon.a on 2023-08-27T20:46:39.301 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Removed /etc/systemd/system/multi-user.target.wants/ceph.target. 2023-08-27T20:46:39.388 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph.target → /etc/systemd/system/ceph.target. 2023-08-27T20:46:39.482 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Created symlink /etc/systemd/system/multi-user.target.wants/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315.target → /etc/systemd/system/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315.target. 2023-08-27T20:46:39.483 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph.target.wants/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315.target → /etc/systemd/system/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315.target. 2023-08-27T20:46:39.606 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.a 2023-08-27T20:46:39.606 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.a.service: Unit ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.a.service not loaded. 2023-08-27T20:46:39.698 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315.target.wants/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.a.service → /etc/systemd/system/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@.service. 2023-08-27T20:46:40.061 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-08-27T20:46:40.062 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout disabled 2023-08-27T20:46:40.069 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-08-27T20:46:40.069 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout inactive 2023-08-27T20:46:40.069 INFO:teuthology.orchestra.run.smithi093.stdout:firewalld.service is not enabled 2023-08-27T20:46:40.069 INFO:teuthology.orchestra.run.smithi093.stdout:Not possible to enable service . firewalld.service is not available 2023-08-27T20:46:40.070 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for mon to start... 2023-08-27T20:46:40.070 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for mon... 2023-08-27T20:46:40.478 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:40 smithi093 ceph-mon[110951]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:46:40.818 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout cluster: 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout id: b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout health: HEALTH_OK 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout services: 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon: 1 daemons, quorum a (age 0.371644s) 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mgr: no daemons active 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout osd: 0 osds: 0 up, 0 in 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout data: 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout pools: 0 pools, 0 pgs 2023-08-27T20:46:40.828 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout objects: 0 objects, 0 B 2023-08-27T20:46:40.829 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout usage: 0 B used, 0 B / 0 B avail 2023-08-27T20:46:40.829 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout pgs: 2023-08-27T20:46:40.829 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:40.829 INFO:teuthology.orchestra.run.smithi093.stdout:mon is available 2023-08-27T20:46:40.829 INFO:teuthology.orchestra.run.smithi093.stdout:Assimilating anything we can from ceph.conf... 2023-08-27T20:46:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:41 smithi093 ceph-mon[110951]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:46:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:41 smithi093 ceph-mon[110951]: monmap e1: 1 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0]} removed_ranks: {} 2023-08-27T20:46:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:41 smithi093 ceph-mon[110951]: fsmap 2023-08-27T20:46:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:41 smithi093 ceph-mon[110951]: osdmap e1: 0 total, 0 up, 0 in 2023-08-27T20:46:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:41 smithi093 ceph-mon[110951]: mgrmap e1: no daemons active 2023-08-27T20:46:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:41 smithi093 ceph-mon[110951]: from='client.? 172.21.15.93:0/2925247725' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-08-27T20:46:41.583 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:41.583 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout [global] 2023-08-27T20:46:41.583 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout fsid = b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon_host = [v2:172.21.15.93:3300,v1:172.21.15.93:6789] 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout [mgr] 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout [osd] 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-08-27T20:46:41.584 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-08-27T20:46:41.585 INFO:teuthology.orchestra.run.smithi093.stdout:Generating new minimal ceph.conf... 2023-08-27T20:46:42.282 INFO:teuthology.orchestra.run.smithi093.stdout:Restarting the monitor... 2023-08-27T20:46:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:42 smithi093 ceph-mon[110951]: from='client.? 172.21.15.93:0/3166997349' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T20:46:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:42 smithi093 ceph-mon[110951]: from='client.? 172.21.15.93:0/3166997349' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-08-27T20:46:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:42 smithi093 ceph-mon[110951]: from='client.? 172.21.15.93:0/927220789' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:46:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:42 smithi093 systemd[1]: Stopping Ceph mon.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:46:42.771 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mon-a[110947]: 2023-08-27T20:46:42.502+0000 7f6336bf2700 -1 received signal: Terminated from /run/podman-init -- /usr/bin/ceph-mon -n mon.a -f --setuser ceph --setgroup ceph --default-log-to-file=false --default-log-to-journald=true --default-log-to-stderr=false --default-mon-cluster-log-to-file=false --default-mon-cluster-log-to-journald=true --default-mon-cluster-log-to-stderr=false (PID: 1) UID: 0 2023-08-27T20:46:42.771 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mon-a[110947]: 2023-08-27T20:46:42.502+0000 7f6336bf2700 -1 mon.a@0(leader) e1 *** Got Signal Terminated *** 2023-08-27T20:46:43.069 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:42 smithi093 bash[111204]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mon-a 2023-08-27T20:46:43.332 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 systemd[1]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.a.service: Succeeded. 2023-08-27T20:46:43.332 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 systemd[1]: Stopped Ceph mon.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:46:43.332 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 systemd[1]: Starting Ceph mon.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:46:43.391 INFO:teuthology.orchestra.run.smithi093.stdout:Setting mon public_network to 172.21.0.0/20 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: set uid:gid to 167:167 (ceph:ceph) 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable), process ceph-mon, pid 2 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: pidfile_write: ignore empty --pid-file 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 bash[111316]: d1926c169165d2a56d6bdbe1bd3352332931b45378ed83077b0bf867c46c5db9 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 systemd[1]: Started Ceph mon.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: load: jerasure load: lrc 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: RocksDB version: 6.15.5 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-27T20:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Compile date Aug 27 2023 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: DB SUMMARY 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: DB Session ID: LOM5P4FKLN9MEXUOT7SV 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: CURRENT file: CURRENT 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: IDENTITY file: IDENTITY 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: MANIFEST file: MANIFEST-000009 size: 131 Bytes 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: SST files in /var/lib/ceph/mon/ceph-a/store.db dir, Total Num: 1, files: 000008.sst 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-a/store.db: 000010.log size: 83283 ; 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.error_if_exists: 0 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.create_if_missing: 0 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.paranoid_checks: 1 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-08-27T20:46:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.env: 0x55eeba79ba80 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.fs: Posix File System 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.info_log: 0x55eebb193da0 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_file_opening_threads: 16 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.statistics: (nil) 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.use_fsync: 0 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_log_file_size: 0 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.log_file_time_to_roll: 0 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.keep_log_file_num: 1000 2023-08-27T20:46:43.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.recycle_log_file_num: 0 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.allow_fallocate: 1 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.allow_mmap_reads: 0 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.allow_mmap_writes: 0 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.use_direct_reads: 0 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.create_missing_column_families: 0 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.db_log_dir: 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-a/store.db 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.table_cache_numshardbits: 6 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.WAL_ttl_seconds: 0 2023-08-27T20:46:43.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.WAL_size_limit_MB: 0 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.is_fd_close_on_exec: 1 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.advise_random_on_open: 1 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.db_write_buffer_size: 0 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.write_buffer_manager: 0x55eebb258930 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.use_adaptive_mutex: 0 2023-08-27T20:46:43.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.rate_limiter: (nil) 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.wal_recovery_mode: 2 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.enable_thread_tracking: 0 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.enable_pipelined_write: 0 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.unordered_write: 0 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.row_cache: None 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.wal_filter: None 2023-08-27T20:46:43.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.allow_ingest_behind: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.preserve_deletes: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.two_write_queues: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.manual_wal_flush: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.atomic_flush: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.persist_stats_to_disk: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.write_dbid_to_manifest: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.log_readahead_size: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.best_efforts_recovery: 0 2023-08-27T20:46:43.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.allow_data_in_errors: 0 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.db_host_id: __hostname__ 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_background_jobs: 2 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_background_compactions: -1 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_subcompactions: 1 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.delayed_write_rate : 16777216 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_total_wal_size: 0 2023-08-27T20:46:43.747 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.stats_dump_period_sec: 600 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.stats_persist_period_sec: 600 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_open_files: -1 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bytes_per_sync: 0 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.wal_bytes_per_sync: 0 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.strict_bytes_per_sync: 0 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_readahead_size: 0 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_background_flushes: -1 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Compression algorithms supported: 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kZSTD supported: 0 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kXpressCompression supported: 0 2023-08-27T20:46:43.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kLZ4HCCompression supported: 1 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kZlibCompression supported: 1 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kSnappyCompression supported: 1 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kLZ4Compression supported: 1 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: kBZip2Compression supported: 0 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Fast CRC32 supported: Supported on x86 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.merge_operator: 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_filter: None 2023-08-27T20:46:43.749 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_filter_factory: None 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.sst_partitioner_factory: None 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.memtable_factory: SkipListFactory 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.table_factory: BlockBasedTable 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55eebb162d30) 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: cache_index_and_filter_blocks: 1 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: pin_top_level_index_and_filter: 1 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: index_type: 0 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: data_block_index_type: 0 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: index_shortening: 1 2023-08-27T20:46:43.750 INFO:journalctl@ceph.mon.a.smithi093.stdout: data_block_hash_table_util_ratio: 0.750000 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: hash_index_allow_collision: 1 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: checksum: 1 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: no_block_cache: 0 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_cache: 0x55eebb21b1f0 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_cache_name: BinnedLRUCache 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_cache_options: 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: capacity : 536870912 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: num_shard_bits : 4 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: strict_capacity_limit : 0 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: high_pri_pool_ratio: 0.000 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_cache_compressed: (nil) 2023-08-27T20:46:43.751 INFO:journalctl@ceph.mon.a.smithi093.stdout: persistent_cache: (nil) 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_size: 4096 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_size_deviation: 10 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_restart_interval: 16 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: index_block_restart_interval: 1 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: metadata_block_size: 4096 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: partition_filters: 0 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: use_delta_encoding: 1 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: whole_key_filtering: 1 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: verify_compression: 0 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: read_amp_bytes_per_bit: 0 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: format_version: 4 2023-08-27T20:46:43.752 INFO:journalctl@ceph.mon.a.smithi093.stdout: enable_index_compression: 1 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout: block_align: 0 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.write_buffer_size: 33554432 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_write_buffer_number: 2 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression: NoCompression 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression: Disabled 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.prefix_extractor: nullptr 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.num_levels: 7 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-27T20:46:43.753 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression_opts.window_bits: -14 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression_opts.level: 32767 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression_opts.strategy: 0 2023-08-27T20:46:43.754 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compression_opts.enabled: false 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.target_file_size_base: 67108864 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.target_file_size_multiplier: 1 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-27T20:46:43.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.arena_block_size: 4194304 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-27T20:46:43.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.disable_auto_compactions: 0 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-27T20:46:43.757 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-27T20:46:43.758 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-27T20:46:43.758 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-27T20:46:43.758 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-27T20:46:43.758 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.table_properties_collectors: 2023-08-27T20:46:43.758 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.inplace_update_support: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.inplace_update_num_locks: 10000 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.memtable_huge_page_size: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.bloom_locality: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.max_successive_merges: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.optimize_filters_for_hits: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.paranoid_file_checks: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.force_consistency_checks: 1 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.report_bg_io_stats: 0 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.ttl: 2592000 2023-08-27T20:46:43.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.periodic_compaction_seconds: 0 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.enable_blob_files: false 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.min_blob_size: 0 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.blob_file_size: 268435456 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.blob_compression_type: NoCompression 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.enable_blob_garbage_collection: false 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-a/store.db/MANIFEST-000009 succeeded,manifest_file_number is 9, next_file_number is 11, last_sequence is 5, log_number is 5,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 5 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/version_set.cc:4082] Creating manifest 13 2023-08-27T20:46:43.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169203393755, "job": 1, "event": "recovery_started", "wal_files": [10]} 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #10 mode 2 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169203394416, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 14, "file_size": 79977, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 78245, "index_size": 220, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 581, "raw_key_size": 10107, "raw_average_key_size": 47, "raw_value_size": 72479, "raw_average_value_size": 343, "num_data_blocks": 10, "num_entries": 211, "num_deletions": 3, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1693169203, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "1777d112-8d13-41cd-a969-3b7d9a2de147", "db_session_id": "LOM5P4FKLN9MEXUOT7SV"}} 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/version_set.cc:4082] Creating manifest 15 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169203394767, "job": 1, "event": "recovery_finished"} 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-a/store.db/000010.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55eebb1af180 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: DB pointer 0x55eebb1ca000 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-08-27T20:46:43.761 INFO:journalctl@ceph.mon.a.smithi093.stdout: ** DB Stats ** 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:46:43.762 INFO:journalctl@ceph.mon.a.smithi093.stdout: L0 2/0 79.86 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: Sum 2/0 79.86 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 174.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T20:46:43.763 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: Cumulative compaction: 0.00 GB write, 24.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: Interval compaction: 0.00 GB write, 24.82 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:46:43.764 INFO:journalctl@ceph.mon.a.smithi093.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: L0 2/0 79.86 KB 0.5 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: Sum 2/0 79.86 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 174.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 174.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T20:46:43.765 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: Cumulative compaction: 0.00 GB write, 24.58 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: starting mon.a rank 0 at public addrs [v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0] at bind addrs [v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0] mon_data /var/lib/ceph/mon/ceph-a fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???) e1 preinit fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???).mds e1 new map 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???).mds e1 print_map 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: e1 2023-08-27T20:46:43.766 INFO:journalctl@ceph.mon.a.smithi093.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout: legacy client fscid: -1 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout: 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout: No filesystems configured 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???).osd e1 crush map has features 3314932999778484224, adjusting msgr requires 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???).osd e1 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a@-1(???).paxosservice(auth 1..2) refresh upgraded, format 0 -> 3 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expand map: {default=false} 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta from 'false' to 'false' 2023-08-27T20:46:43.767 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expanded map: {default=false} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expand map: {default=info} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta from 'info' to 'info' 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expanded map: {default=info} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expand map: {default=daemon} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta from 'daemon' to 'daemon' 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expanded map: {default=daemon} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expand map: {default=debug} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta from 'debug' to 'debug' 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: expand_channel_meta expanded map: {default=debug} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: monmap e1: 1 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0]} removed_ranks: {} 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: fsmap 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: osdmap e1: 0 total, 0 up, 0 in 2023-08-27T20:46:43.768 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:43 smithi093 ceph-mon[111335]: mgrmap e1: no daemons active 2023-08-27T20:46:44.115 INFO:teuthology.orchestra.run.smithi093.stdout:Wrote config to /etc/ceph/ceph.conf 2023-08-27T20:46:44.116 INFO:teuthology.orchestra.run.smithi093.stdout:Wrote keyring to /etc/ceph/ceph.client.admin.keyring 2023-08-27T20:46:44.116 INFO:teuthology.orchestra.run.smithi093.stdout:Creating mgr... 2023-08-27T20:46:44.116 INFO:teuthology.orchestra.run.smithi093.stdout:Verifying port 9283 ... 2023-08-27T20:46:44.239 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl reset-failed ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.y 2023-08-27T20:46:44.239 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Failed to reset failed state of unit ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.y.service: Unit ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.y.service not loaded. 2023-08-27T20:46:44.332 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stderr Created symlink /etc/systemd/system/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315.target.wants/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.y.service → /etc/systemd/system/ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@.service. 2023-08-27T20:46:44.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:44 smithi093 systemd[1]: Starting Ceph mgr.y for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:46:44.652 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-08-27T20:46:44.652 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout disabled 2023-08-27T20:46:44.660 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-08-27T20:46:44.660 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout inactive 2023-08-27T20:46:44.660 INFO:teuthology.orchestra.run.smithi093.stdout:firewalld.service is not enabled 2023-08-27T20:46:44.660 INFO:teuthology.orchestra.run.smithi093.stdout:Not possible to enable service . firewalld.service is not available 2023-08-27T20:46:44.668 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-08-27T20:46:44.668 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout disabled 2023-08-27T20:46:44.676 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-08-27T20:46:44.676 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout inactive 2023-08-27T20:46:44.676 INFO:teuthology.orchestra.run.smithi093.stdout:firewalld.service is not enabled 2023-08-27T20:46:44.676 INFO:teuthology.orchestra.run.smithi093.stdout:Not possible to open ports <[9283]>. firewalld.service is not available 2023-08-27T20:46:44.677 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for mgr to start... 2023-08-27T20:46:44.677 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for mgr... 2023-08-27T20:46:44.944 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3269501682' entity='client.admin' 2023-08-27T20:46:44.944 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:44 smithi093 bash[111545]: d29e0efb4bb5090eca3b68ecd3d70b8fb12c5d087b27954045a4c1c72f39cded 2023-08-27T20:46:44.944 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:44 smithi093 systemd[1]: Started Ceph mgr.y for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:46:44.944 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:44.774+0000 7f7d6840a1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:46:44.944 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:44.943+0000 7f7d6840a1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:46:45.417 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsid": "b79a52c0-451a-11ee-817b-1b7a5f1ba315", 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "health": { 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 0 2023-08-27T20:46:45.418 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "a" 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_age": 1, 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "monmap": { 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:45.419 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-08-27T20:46:45.420 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-08-27T20:46:45.420 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-08-27T20:46:45.420 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-08-27T20:46:45.420 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-08-27T20:46:45.420 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-08-27T20:46:45.423 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:45.423 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-08-27T20:46:45.423 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-08-27T20:46:45.423 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-08-27T20:46:45.423 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-08-27T20:46:45.423 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-08-27T20:46:45.424 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "available": false, 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modules": [ 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "iostat", 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "nfs", 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "restful" 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modified": "2023-08-27T20:46:40.101001+0000", 2023-08-27T20:46:45.425 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:45.426 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:45.426 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-08-27T20:46:45.426 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:46:45.426 INFO:teuthology.orchestra.run.smithi093.stdout:mgr not available, waiting (1/15)... 2023-08-27T20:46:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:45.259+0000 7f7d6840a1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:46:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1025304043' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:46:46.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:45.782+0000 7f7d6840a1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:46:46.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:45.950+0000 7f7d6840a1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:46:46.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:46 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:46.337+0000 7f7d6840a1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:46:47.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:47 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:47.089+0000 7f7d6840a1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:46:47.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:47 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:47.171+0000 7f7d6840a1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:46:47.834 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:47 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:47.784+0000 7f7d6840a1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:46:48.091 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2088105512' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:46:48.091 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:47 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:47.870+0000 7f7d6840a1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:46:48.091 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:47 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:47.959+0000 7f7d6840a1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:46:48.092 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:48 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:48.051+0000 7f7d6840a1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsid": "b79a52c0-451a-11ee-817b-1b7a5f1ba315", 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "health": { 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-08-27T20:46:48.169 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 0 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "a" 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_age": 4, 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "monmap": { 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-08-27T20:46:48.170 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-08-27T20:46:48.171 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "available": false, 2023-08-27T20:46:48.172 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modules": [ 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "iostat", 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "nfs", 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "restful" 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modified": "2023-08-27T20:46:40.101001+0000", 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:48.173 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-08-27T20:46:48.174 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:46:48.174 INFO:teuthology.orchestra.run.smithi093.stdout:mgr not available, waiting (2/15)... 2023-08-27T20:46:48.366 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:48 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:48.278+0000 7f7d6840a1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:46:48.667 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:48 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:48.366+0000 7f7d6840a1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:46:48.667 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:48 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:48.574+0000 7f7d6840a1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:46:48.924 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:48 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:48.667+0000 7f7d6840a1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:46:48.925 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:48 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:48.843+0000 7f7d6840a1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:46:49.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:48 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:48.924+0000 7f7d6840a1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:46:49.990 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:49 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:49.673+0000 7f7d6840a1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:46:49.990 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:49 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:49.857+0000 7f7d6840a1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:46:50.277 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:50.026+0000 7f7d6840a1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:46:50.841 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1770227879' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:46:50.841 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:50.646+0000 7f7d6840a1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:46:50.841 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:50.825+0000 7f7d6840a1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:46:50.927 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:50.927 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:46:50.927 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsid": "b79a52c0-451a-11ee-817b-1b7a5f1ba315", 2023-08-27T20:46:50.927 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "health": { 2023-08-27T20:46:50.927 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-08-27T20:46:50.927 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-08-27T20:46:50.928 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-08-27T20:46:50.928 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:50.928 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-08-27T20:46:50.928 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-08-27T20:46:50.928 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 0 2023-08-27T20:46:50.928 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:50.929 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-08-27T20:46:50.929 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "a" 2023-08-27T20:46:50.929 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:50.929 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_age": 7, 2023-08-27T20:46:50.929 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "monmap": { 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-08-27T20:46:50.930 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-08-27T20:46:50.931 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-08-27T20:46:50.931 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-08-27T20:46:50.931 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-08-27T20:46:50.931 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-08-27T20:46:50.931 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:50.931 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-08-27T20:46:50.931 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-08-27T20:46:50.932 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "available": false, 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modules": [ 2023-08-27T20:46:50.933 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "iostat", 2023-08-27T20:46:50.934 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "nfs", 2023-08-27T20:46:50.934 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "restful" 2023-08-27T20:46:50.934 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:50.934 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:50.934 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:50.934 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-08-27T20:46:50.934 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:50.935 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modified": "2023-08-27T20:46:40.101001+0000", 2023-08-27T20:46:50.935 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:50.935 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:50.935 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-08-27T20:46:50.935 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:46:50.935 INFO:teuthology.orchestra.run.smithi093.stdout:mgr not available, waiting (3/15)... 2023-08-27T20:46:51.117 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:51.005+0000 7f7d6840a1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:46:51.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:51.117+0000 7f7d6840a1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:46:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: Activating manager daemon y 2023-08-27T20:46:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: mgrmap e2: y(active, starting, since 0.00292871s) 2023-08-27T20:46:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:46:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:46:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:46:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:46:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:46:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: Manager daemon y is now available 2023-08-27T20:46:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:46:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:46:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:46:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:46:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:51 smithi093 ceph-mon[111335]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:46:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:53 smithi093 ceph-mon[111335]: mgrmap e3: y(active, since 1.00635s) 2023-08-27T20:46:53.957 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:53.957 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:46:53.957 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsid": "b79a52c0-451a-11ee-817b-1b7a5f1ba315", 2023-08-27T20:46:53.957 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "health": { 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "status": "HEALTH_OK", 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "checks": {}, 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mutes": [] 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "election_epoch": 5, 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum": [ 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 0 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_names": [ 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "a" 2023-08-27T20:46:53.958 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "quorum_age": 10, 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "monmap": { 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "min_mon_release_name": "quincy", 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_mons": 1 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osdmap": { 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_osds": 0, 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_up_osds": 0, 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_up_since": 0, 2023-08-27T20:46:53.959 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_in_osds": 0, 2023-08-27T20:46:53.963 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "osd_in_since": 0, 2023-08-27T20:46:53.963 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_remapped_pgs": 0 2023-08-27T20:46:53.963 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:53.963 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgmap": { 2023-08-27T20:46:53.963 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "pgs_by_state": [], 2023-08-27T20:46:53.963 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pgs": 0, 2023-08-27T20:46:53.963 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_pools": 0, 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_objects": 0, 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "data_bytes": 0, 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_used": 0, 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_avail": 0, 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "bytes_total": 0 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "fsmap": { 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "by_rank": [], 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "up:standby": 0 2023-08-27T20:46:53.964 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mgrmap": { 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "available": true, 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_standbys": 0, 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modules": [ 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "iostat", 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "nfs", 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "restful" 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ], 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "servicemap": { 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 1, 2023-08-27T20:46:53.965 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "modified": "2023-08-27T20:46:40.101001+0000", 2023-08-27T20:46:53.966 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "services": {} 2023-08-27T20:46:53.966 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout }, 2023-08-27T20:46:53.966 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "progress_events": {} 2023-08-27T20:46:53.966 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:46:53.966 INFO:teuthology.orchestra.run.smithi093.stdout:mgr is available 2023-08-27T20:46:54.698 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3675520105' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:46:54.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:54 smithi093 ceph-mon[111335]: mgrmap e4: y(active, since 3s) 2023-08-27T20:46:54.797 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:54.797 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout [global] 2023-08-27T20:46:54.797 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout fsid = b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon_osd_allow_pg_remap = true 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon_osd_allow_primary_affinity = true 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mon_warn_on_no_sortbitwise = false 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout osd_crush_chooseleaf_type = 0 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout [mgr] 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout mgr/telemetry/nag = false 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout [osd] 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout osd_map_max_advance = 10 2023-08-27T20:46:54.798 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout osd_sloppy_crc = true 2023-08-27T20:46:54.799 INFO:teuthology.orchestra.run.smithi093.stdout:Enabling cephadm module... 2023-08-27T20:46:55.726 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2070046131' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T20:46:55.726 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/272688912' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-08-27T20:46:55.729 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setuser ceph since I am not root 2023-08-27T20:46:55.729 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setgroup ceph since I am not root 2023-08-27T20:46:55.729 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:55.572+0000 7f910fb801c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:46:55.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:55.737+0000 7f910fb801c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:46:56.367 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:56 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:56.045+0000 7f910fb801c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:46:56.673 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:56 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:56.596+0000 7f910fb801c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:46:56.674 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/272688912' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-08-27T20:46:56.674 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:56 smithi093 ceph-mon[111335]: mgrmap e5: y(active, since 4s) 2023-08-27T20:46:56.674 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:46:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1859399544' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T20:46:56.721 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:46:56.722 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 5, 2023-08-27T20:46:56.722 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "available": true, 2023-08-27T20:46:56.723 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-08-27T20:46:56.723 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-08-27T20:46:56.723 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:46:56.723 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for the mgr to restart... 2023-08-27T20:46:56.723 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for mgr epoch 5... 2023-08-27T20:46:56.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:56 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:56.773+0000 7f910fb801c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:46:57.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:57.174+0000 7f910fb801c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:46:58.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:57.920+0000 7f910fb801c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:46:58.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:58 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:58.002+0000 7f910fb801c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:46:58.843 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:58 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:58.589+0000 7f910fb801c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:46:58.843 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:58 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:58.670+0000 7f910fb801c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:46:58.844 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:58 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:58.754+0000 7f910fb801c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:46:58.844 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:58 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:58.843+0000 7f910fb801c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:46:59.154 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:59 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:59.065+0000 7f910fb801c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:46:59.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:59 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:59.154+0000 7f910fb801c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:46:59.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:59 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:59.359+0000 7f910fb801c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:46:59.715 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:59 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:59.452+0000 7f910fb801c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:46:59.715 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:59 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:59.631+0000 7f910fb801c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:46:59.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:46:59 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:46:59.715+0000 7f910fb801c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:47:00.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:00.443+0000 7f910fb801c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:47:00.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:00.629+0000 7f910fb801c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:47:01.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:00.801+0000 7f910fb801c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:47:01.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:01.391+0000 7f910fb801c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:47:01.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:01.565+0000 7f910fb801c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:47:02.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:01.740+0000 7f910fb801c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:01.854+0000 7f910fb801c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: Active manager daemon y restarted 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: Activating manager daemon y 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: osdmap e2: 0 total, 0 up, 0 in 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: mgrmap e6: y(active, starting, since 0.00315154s) 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:02.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:02.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: Manager daemon y is now available 2023-08-27T20:47:02.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:02.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:02.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:02.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:02.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:01 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:02 smithi093 ceph-mon[111335]: Found migration_current of "None". Setting to last migration. 2023-08-27T20:47:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:02 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:02 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:02 smithi093 ceph-mon[111335]: mgrmap e7: y(active, since 1.00476s) 2023-08-27T20:47:03.638 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:47:03.638 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 7, 2023-08-27T20:47:03.638 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "initialized": true 2023-08-27T20:47:03.638 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:47:03.638 INFO:teuthology.orchestra.run.smithi093.stdout:mgr epoch 5 is available 2023-08-27T20:47:03.638 INFO:teuthology.orchestra.run.smithi093.stdout:Setting orchestrator backend to cephadm... 2023-08-27T20:47:04.439 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:04 smithi093 ceph-mon[111335]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T20:47:04.439 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:04 smithi093 ceph-mon[111335]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T20:47:04.439 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:04 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:04.439 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:04 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:04.439 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:04 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:04.439 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:04 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:05.337 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout value unchanged 2023-08-27T20:47:05.338 INFO:teuthology.orchestra.run.smithi093.stdout:Generating ssh key... 2023-08-27T20:47:05.498 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:05 smithi093 ceph-mon[111335]: [27/Aug/2023:20:47:03] ENGINE Bus STARTING 2023-08-27T20:47:05.499 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:05 smithi093 ceph-mon[111335]: [27/Aug/2023:20:47:04] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:47:05.499 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:05 smithi093 ceph-mon[111335]: [27/Aug/2023:20:47:04] ENGINE Bus STARTED 2023-08-27T20:47:05.499 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:05 smithi093 ceph-mon[111335]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:05.499 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:05 smithi093 ceph-mon[111335]: mgrmap e8: y(active, since 2s) 2023-08-27T20:47:06.314 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: Generating public/private rsa key pair. 2023-08-27T20:47:06.314 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: Your identification has been saved in /tmp/tmpzze936ij/key. 2023-08-27T20:47:06.314 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: Your public key has been saved in /tmp/tmpzze936ij/key.pub. 2023-08-27T20:47:06.314 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: The key fingerprint is: 2023-08-27T20:47:06.314 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: SHA256:wOECOnFNYOaAdEb/P31Nvk5AwRczS16akCmT+jXi+GA ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:06.314 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: The key's randomart image is: 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: +---[RSA 3072]----+ 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: |=.BB. . .o+ =o| 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: |.Oo.oo . + oo++=| 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: |o . ..+ . o .+o | 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: | . .... . + | 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: | .S+ o o . | 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: | E + = | 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: | . = . . + | 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: | o . . .| 2023-08-27T20:47:06.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: | .o | 2023-08-27T20:47:06.316 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: +----[SHA256]-----+ 2023-08-27T20:47:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-mon[111335]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:06 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:07.187 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhdCBSD16kx48Q7zcNC92PL1kJsHNKW4+n8xdb0opqZBmiheCeHdJbY2tOfojmp++4NSsj/isEhpJx9Ux55SkxIIiQcA3P6F1MVQSy1vAhsiFeTZHp4qAZauED09mC6d21Iy8//rEiQtx0TTdMrlQWzLOJq/18eH3/7QDyj8BDKHYO06nNBpfCFcpAkfF5WLVbo1TIl070Ucqd62w8BOI8UCLxCzGM5MqgPztiteTgZ8VFBdjS3iC8TDrWSL7l7OrEp7oR5dstfemtO7jLCsyoEcvfMLnIG6XZlOw8Pz7M1fJJAF7bp8gB62GCelsrmcnmLHoECR66f+dLaSw2d/Plvd5YMoes5eCjGNLUcwwAM5u8pjSVTlXIDNdXSAxirsLJe5I3o9W0/W8PjSiJjMCtD998c1tqZ3Dl1FVKto0mzyU6bXHoaiRkvqURh8O6j4Bp7hsF0+jja/Hx6ReLEW8RwmRfE8sKiLjLzTRBua26covqtfmJzSMHKnRT2FN0oW0= ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:07.188 INFO:teuthology.orchestra.run.smithi093.stdout:Wrote public SSH key to /home/ubuntu/cephtest/ceph.pub 2023-08-27T20:47:07.188 INFO:teuthology.orchestra.run.smithi093.stdout:Adding key to root@localhost authorized_keys... 2023-08-27T20:47:07.188 INFO:teuthology.orchestra.run.smithi093.stdout:Adding host smithi093... 2023-08-27T20:47:07.676 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:07 smithi093 ceph-mon[111335]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:07.677 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:07 smithi093 ceph-mon[111335]: Generating ssh key... 2023-08-27T20:47:08.457 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:08 smithi093 ceph-mon[111335]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:09.697 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout Added host 'smithi093' with addr '172.21.15.93' 2023-08-27T20:47:09.697 INFO:teuthology.orchestra.run.smithi093.stdout:Deploying unmanaged mon service... 2023-08-27T20:47:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:09 smithi093 ceph-mon[111335]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi093", "addr": "172.21.15.93", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:09 smithi093 ceph-mon[111335]: Deploying cephadm binary to smithi093 2023-08-27T20:47:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:09 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:09 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:10 smithi093 ceph-mon[111335]: Added host smithi093 2023-08-27T20:47:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:10 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:10.668 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout Scheduled mon update... 2023-08-27T20:47:10.668 INFO:teuthology.orchestra.run.smithi093.stdout:Deploying unmanaged mgr service... 2023-08-27T20:47:11.721 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:11 smithi093 ceph-mon[111335]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:11.721 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:11 smithi093 ceph-mon[111335]: Saving service mon spec with placement count:5 2023-08-27T20:47:11.721 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:11 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:11.722 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:11 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:11.722 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:11 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:11.769 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout Scheduled mgr update... 2023-08-27T20:47:12.610 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:12 smithi093 ceph-mon[111335]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:12.610 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:12 smithi093 ceph-mon[111335]: Saving service mgr spec with placement count:2 2023-08-27T20:47:12.610 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1068952983' entity='client.admin' 2023-08-27T20:47:13.525 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:13 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:13.526 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4068444121' entity='client.admin' 2023-08-27T20:47:13.526 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:13 smithi093 ceph-mon[111335]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:13.543 INFO:teuthology.orchestra.run.smithi093.stdout:Enabling the dashboard module... 2023-08-27T20:47:14.658 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3065173678' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-08-27T20:47:14.658 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setuser ceph since I am not root 2023-08-27T20:47:14.658 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setgroup ceph since I am not root 2023-08-27T20:47:14.948 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:14.656+0000 7fd34671b1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:47:14.948 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:14.820+0000 7fd34671b1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:47:15.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:15.142+0000 7fd34671b1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:47:15.668 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3065173678' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-08-27T20:47:15.668 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:15 smithi093 ceph-mon[111335]: mgrmap e9: y(active, since 12s) 2023-08-27T20:47:15.668 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2702055024' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "epoch": 9, 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "available": true, 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "active_name": "y", 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "num_standby": 0 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for the mgr to restart... 2023-08-27T20:47:15.752 INFO:teuthology.orchestra.run.smithi093.stdout:Waiting for mgr epoch 9... 2023-08-27T20:47:15.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:15.667+0000 7fd34671b1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:47:15.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:15.841+0000 7fd34671b1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:47:16.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:16 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:16.243+0000 7fd34671b1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:47:17.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:16 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:16.993+0000 7fd34671b1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:47:17.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:17 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:17.077+0000 7fd34671b1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:47:17.934 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:17 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:17.671+0000 7fd34671b1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:47:17.934 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:17 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:17.754+0000 7fd34671b1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:47:17.934 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:17 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:17.841+0000 7fd34671b1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:47:18.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:17 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:17.933+0000 7fd34671b1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:47:18.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:18.168+0000 7fd34671b1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:47:18.559 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:18.258+0000 7fd34671b1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:47:18.560 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:18.466+0000 7fd34671b1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:47:18.822 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:18.558+0000 7fd34671b1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:47:18.822 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:18.738+0000 7fd34671b1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:47:19.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:18.821+0000 7fd34671b1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:47:19.917 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:19 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:19.556+0000 7fd34671b1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:47:19.917 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:19 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:19.743+0000 7fd34671b1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:47:20.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:19 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:19.916+0000 7fd34671b1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:47:20.858 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:20.506+0000 7fd34671b1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:47:20.858 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:20.681+0000 7fd34671b1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:47:21.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:20.857+0000 7fd34671b1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:47:21.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:47:20.971+0000 7fd34671b1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:47:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: Active manager daemon y restarted 2023-08-27T20:47:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: Activating manager daemon y 2023-08-27T20:47:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: osdmap e3: 0 total, 0 up, 0 in 2023-08-27T20:47:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: mgrmap e10: y(active, starting, since 0.00561952s) 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: Manager daemon y is now available 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:22.351 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout { 2023-08-27T20:47:22.351 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "mgrmap_epoch": 11, 2023-08-27T20:47:22.351 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout "initialized": true 2023-08-27T20:47:22.351 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout } 2023-08-27T20:47:22.351 INFO:teuthology.orchestra.run.smithi093.stdout:mgr epoch 9 is available 2023-08-27T20:47:22.352 INFO:teuthology.orchestra.run.smithi093.stdout:Generating a dashboard self-signed certificate... 2023-08-27T20:47:22.457 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:22 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:22.457 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:22 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:22.457 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:22 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:22.457 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:22 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:22.457 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:22 smithi093 ceph-mon[111335]: mgrmap e11: y(active, since 1.00529s) 2023-08-27T20:47:23.286 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: [27/Aug/2023:20:47:21] ENGINE Bus STARTING 2023-08-27T20:47:23.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: [27/Aug/2023:20:47:21] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:47:23.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: [27/Aug/2023:20:47:21] ENGINE Bus STARTED 2023-08-27T20:47:23.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T20:47:23.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T20:47:23.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:23.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:23.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:23 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:23.326 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout Self-signed certificate created 2023-08-27T20:47:23.326 INFO:teuthology.orchestra.run.smithi093.stdout:Creating initial admin user... 2023-08-27T20:47:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:24 smithi093 ceph-mon[111335]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:24 smithi093 ceph-mon[111335]: mgrmap e12: y(active, since 2s) 2023-08-27T20:47:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:26 smithi093 ceph-mon[111335]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:26 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:26.943 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout {"username": "admin", "password": "$2b$12$e1VIfehjsRA10ZOd8z2QNehDKG.LF1vdMr1k8sC3I.L.VCknjEH9m", "roles": ["administrator"], "name": null, "email": null, "lastUpdate": 1693169245, "enabled": true, "pwdExpirationDate": null, "pwdUpdateRequired": true} 2023-08-27T20:47:26.943 INFO:teuthology.orchestra.run.smithi093.stdout:Fetching dashboard port number... 2023-08-27T20:47:27.685 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2353321583' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-08-27T20:47:27.775 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stdout 8443 2023-08-27T20:47:27.785 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 1 from systemctl is-enabled firewalld.service 2023-08-27T20:47:27.785 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout disabled 2023-08-27T20:47:27.792 INFO:teuthology.orchestra.run.smithi093.stdout:Non-zero exit code 3 from systemctl is-active firewalld.service 2023-08-27T20:47:27.793 INFO:teuthology.orchestra.run.smithi093.stdout:systemctl: stdout inactive 2023-08-27T20:47:27.793 INFO:teuthology.orchestra.run.smithi093.stdout:firewalld.service is not enabled 2023-08-27T20:47:27.793 INFO:teuthology.orchestra.run.smithi093.stdout:Not possible to open ports <[8443]>. firewalld.service is not available 2023-08-27T20:47:27.795 INFO:teuthology.orchestra.run.smithi093.stdout:Ceph Dashboard is now available at: 2023-08-27T20:47:27.795 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:27.795 INFO:teuthology.orchestra.run.smithi093.stdout: URL: https://smithi093.front.sepia.ceph.com:8443/ 2023-08-27T20:47:27.795 INFO:teuthology.orchestra.run.smithi093.stdout: User: admin 2023-08-27T20:47:27.795 INFO:teuthology.orchestra.run.smithi093.stdout: Password: yxxsipbpy0 2023-08-27T20:47:27.795 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:27.796 INFO:teuthology.orchestra.run.smithi093.stdout:Saving cluster configuration to /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config directory 2023-08-27T20:47:27.796 INFO:teuthology.orchestra.run.smithi093.stdout:Enabling autotune for osd_memory_target 2023-08-27T20:47:28.560 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:28 smithi093 ceph-mon[111335]: mgrmap e13: y(active, since 6s) 2023-08-27T20:47:29.550 INFO:teuthology.orchestra.run.smithi093.stdout:/usr/bin/ceph: stderr set mgr/dashboard/cluster/status 2023-08-27T20:47:29.550 INFO:teuthology.orchestra.run.smithi093.stdout:You can access the Ceph CLI as following in case of multi-cluster or non-default config: 2023-08-27T20:47:29.550 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: sudo /home/ubuntu/cephtest/cephadm shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout:Or, if you are only running a single cluster on this host: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: sudo /home/ubuntu/cephtest/cephadm shell 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout:Please consider enabling telemetry to help improve Ceph: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: ceph telemetry on 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.551 INFO:teuthology.orchestra.run.smithi093.stdout:For more information see: 2023-08-27T20:47:29.552 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.552 INFO:teuthology.orchestra.run.smithi093.stdout: https://docs.ceph.com/docs/master/mgr/telemetry/ 2023-08-27T20:47:29.552 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:29.552 INFO:teuthology.orchestra.run.smithi093.stdout:Bootstrap complete. 2023-08-27T20:47:29.584 INFO:tasks.cephadm:Fetching config... 2023-08-27T20:47:29.584 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:47:29.584 DEBUG:teuthology.orchestra.run.smithi093:> dd if=/etc/ceph/ceph.conf of=/dev/stdout 2023-08-27T20:47:29.601 INFO:tasks.cephadm:Fetching client.admin keyring... 2023-08-27T20:47:29.602 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:47:29.602 DEBUG:teuthology.orchestra.run.smithi093:> dd if=/etc/ceph/ceph.client.admin.keyring of=/dev/stdout 2023-08-27T20:47:29.660 INFO:tasks.cephadm:Fetching mon keyring... 2023-08-27T20:47:29.660 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:47:29.660 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd if=/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/keyring of=/dev/stdout 2023-08-27T20:47:29.727 INFO:tasks.cephadm:Fetching pub ssh key... 2023-08-27T20:47:29.727 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:47:29.727 DEBUG:teuthology.orchestra.run.smithi093:> dd if=/home/ubuntu/cephtest/ceph.pub of=/dev/stdout 2023-08-27T20:47:29.786 INFO:tasks.cephadm:Installing pub ssh key for root users... 2023-08-27T20:47:29.786 DEBUG:teuthology.orchestra.run.smithi093:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhdCBSD16kx48Q7zcNC92PL1kJsHNKW4+n8xdb0opqZBmiheCeHdJbY2tOfojmp++4NSsj/isEhpJx9Ux55SkxIIiQcA3P6F1MVQSy1vAhsiFeTZHp4qAZauED09mC6d21Iy8//rEiQtx0TTdMrlQWzLOJq/18eH3/7QDyj8BDKHYO06nNBpfCFcpAkfF5WLVbo1TIl070Ucqd62w8BOI8UCLxCzGM5MqgPztiteTgZ8VFBdjS3iC8TDrWSL7l7OrEp7oR5dstfemtO7jLCsyoEcvfMLnIG6XZlOw8Pz7M1fJJAF7bp8gB62GCelsrmcnmLHoECR66f+dLaSw2d/Plvd5YMoes5eCjGNLUcwwAM5u8pjSVTlXIDNdXSAxirsLJe5I3o9W0/W8PjSiJjMCtD998c1tqZ3Dl1FVKto0mzyU6bXHoaiRkvqURh8O6j4Bp7hsF0+jja/Hx6ReLEW8RwmRfE8sKiLjLzTRBua26covqtfmJzSMHKnRT2FN0oW0= ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-08-27T20:47:29.863 INFO:teuthology.orchestra.run.smithi093.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhdCBSD16kx48Q7zcNC92PL1kJsHNKW4+n8xdb0opqZBmiheCeHdJbY2tOfojmp++4NSsj/isEhpJx9Ux55SkxIIiQcA3P6F1MVQSy1vAhsiFeTZHp4qAZauED09mC6d21Iy8//rEiQtx0TTdMrlQWzLOJq/18eH3/7QDyj8BDKHYO06nNBpfCFcpAkfF5WLVbo1TIl070Ucqd62w8BOI8UCLxCzGM5MqgPztiteTgZ8VFBdjS3iC8TDrWSL7l7OrEp7oR5dstfemtO7jLCsyoEcvfMLnIG6XZlOw8Pz7M1fJJAF7bp8gB62GCelsrmcnmLHoECR66f+dLaSw2d/Plvd5YMoes5eCjGNLUcwwAM5u8pjSVTlXIDNdXSAxirsLJe5I3o9W0/W8PjSiJjMCtD998c1tqZ3Dl1FVKto0mzyU6bXHoaiRkvqURh8O6j4Bp7hsF0+jja/Hx6ReLEW8RwmRfE8sKiLjLzTRBua26covqtfmJzSMHKnRT2FN0oW0= ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:29.874 DEBUG:teuthology.orchestra.run.smithi136:> sudo install -d -m 0700 /root/.ssh && echo 'ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhdCBSD16kx48Q7zcNC92PL1kJsHNKW4+n8xdb0opqZBmiheCeHdJbY2tOfojmp++4NSsj/isEhpJx9Ux55SkxIIiQcA3P6F1MVQSy1vAhsiFeTZHp4qAZauED09mC6d21Iy8//rEiQtx0TTdMrlQWzLOJq/18eH3/7QDyj8BDKHYO06nNBpfCFcpAkfF5WLVbo1TIl070Ucqd62w8BOI8UCLxCzGM5MqgPztiteTgZ8VFBdjS3iC8TDrWSL7l7OrEp7oR5dstfemtO7jLCsyoEcvfMLnIG6XZlOw8Pz7M1fJJAF7bp8gB62GCelsrmcnmLHoECR66f+dLaSw2d/Plvd5YMoes5eCjGNLUcwwAM5u8pjSVTlXIDNdXSAxirsLJe5I3o9W0/W8PjSiJjMCtD998c1tqZ3Dl1FVKto0mzyU6bXHoaiRkvqURh8O6j4Bp7hsF0+jja/Hx6ReLEW8RwmRfE8sKiLjLzTRBua26covqtfmJzSMHKnRT2FN0oW0= ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315' | sudo tee -a /root/.ssh/authorized_keys && sudo chmod 0600 /root/.ssh/authorized_keys 2023-08-27T20:47:29.919 INFO:teuthology.orchestra.run.smithi136.stdout:ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDhdCBSD16kx48Q7zcNC92PL1kJsHNKW4+n8xdb0opqZBmiheCeHdJbY2tOfojmp++4NSsj/isEhpJx9Ux55SkxIIiQcA3P6F1MVQSy1vAhsiFeTZHp4qAZauED09mC6d21Iy8//rEiQtx0TTdMrlQWzLOJq/18eH3/7QDyj8BDKHYO06nNBpfCFcpAkfF5WLVbo1TIl070Ucqd62w8BOI8UCLxCzGM5MqgPztiteTgZ8VFBdjS3iC8TDrWSL7l7OrEp7oR5dstfemtO7jLCsyoEcvfMLnIG6XZlOw8Pz7M1fJJAF7bp8gB62GCelsrmcnmLHoECR66f+dLaSw2d/Plvd5YMoes5eCjGNLUcwwAM5u8pjSVTlXIDNdXSAxirsLJe5I3o9W0/W8PjSiJjMCtD998c1tqZ3Dl1FVKto0mzyU6bXHoaiRkvqURh8O6j4Bp7hsF0+jja/Hx6ReLEW8RwmRfE8sKiLjLzTRBua26covqtfmJzSMHKnRT2FN0oW0= ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:29.931 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph config set mgr mgr/cephadm/allow_ptrace true 2023-08-27T20:47:30.193 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:47:30.478 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1889985893' entity='client.admin' 2023-08-27T20:47:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi093", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:47:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2286130882' entity='client.admin' 2023-08-27T20:47:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:32.253 INFO:tasks.cephadm:Distributing conf and client.admin keyring to all hosts + 0755 2023-08-27T20:47:32.253 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch client-keyring set client.admin '*' --mode 0755 2023-08-27T20:47:32.514 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:47:33.998 INFO:tasks.cephadm:Writing (initial) conf and keyring to smithi136 2023-08-27T20:47:33.998 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:47:33.998 DEBUG:teuthology.orchestra.run.smithi136:> dd of=/etc/ceph/ceph.conf 2023-08-27T20:47:34.023 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:47:34.023 DEBUG:teuthology.orchestra.run.smithi136:> dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:34.084 INFO:tasks.cephadm:Adding host smithi136 to orchestrator... 2023-08-27T20:47:34.085 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch host add smithi136 2023-08-27T20:47:34.356 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:34 smithi093 ceph-mon[111335]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:47:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:35 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:35 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:35 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:35 smithi093 ceph-mon[111335]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi136", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:36 smithi093 ceph-mon[111335]: Deploying cephadm binary to smithi136 2023-08-27T20:47:37.253 INFO:teuthology.orchestra.run.smithi093.stdout:Added host 'smithi136' with addr '172.21.15.136' 2023-08-27T20:47:37.685 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch host ls --format=json 2023-08-27T20:47:37.943 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:47:38.426 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:38.427 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:38 smithi093 ceph-mon[111335]: Added host smithi136 2023-08-27T20:47:38.427 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:38.427 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:38.427 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:39.006 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:47:39.007 INFO:teuthology.orchestra.run.smithi093.stdout:[{"addr": "172.21.15.93", "hostname": "smithi093", "labels": [], "status": ""}, {"addr": "172.21.15.136", "hostname": "smithi136", "labels": [], "status": ""}] 2023-08-27T20:47:39.360 INFO:tasks.cephadm:Setting crush tunables to default 2023-08-27T20:47:39.360 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd crush tunables default 2023-08-27T20:47:39.617 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:47:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:39 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:39 smithi093 ceph-mon[111335]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:47:41.632 INFO:teuthology.orchestra.run.smithi093.stderr:adjusted tunables profile to default 2023-08-27T20:47:41.958 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:41.958 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi136", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3478248900' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:47:41.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:41.960 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:41.960 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:42.026 INFO:tasks.cephadm:Adding mon.a on smithi093 2023-08-27T20:47:42.026 INFO:tasks.cephadm:Adding mon.c on smithi093 2023-08-27T20:47:42.026 INFO:tasks.cephadm:Adding mon.b on smithi136 2023-08-27T20:47:42.026 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch apply mon '3;smithi093:172.21.15.93=a;smithi093:[v2:172.21.15.93:3301,v1:172.21.15.93:6790]=c;smithi136:172.21.15.136=b' 2023-08-27T20:47:42.304 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3478248900' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-08-27T20:47:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:42 smithi093 ceph-mon[111335]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:43.403 INFO:teuthology.orchestra.run.smithi136.stdout:Scheduled mon update... 2023-08-27T20:47:43.817 DEBUG:teuthology.orchestra.run.smithi093:mon.c> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.c.service 2023-08-27T20:47:43.819 DEBUG:teuthology.orchestra.run.smithi136:mon.b> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mon.b.service 2023-08-27T20:47:43.821 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-27T20:47:43.822 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-27T20:47:43.851 INFO:journalctl@ceph.mon.b.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:47:43.857 INFO:journalctl@ceph.mon.c.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:47:44.437 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi093:172.21.15.93=a;smithi093:[v2:172.21.15.93:3301,v1:172.21.15.93:6790]=c;smithi136:172.21.15.136=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: Saving service mon spec with placement smithi093:172.21.15.93=a;smithi093:[v2:172.21.15.93:3301,v1:172.21.15.93:6790]=c;smithi136:172.21.15.136=b;count:3 2023-08-27T20:47:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:47:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:44.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:44 smithi093 ceph-mon[111335]: Deploying daemon mon.b on smithi136 2023-08-27T20:47:45.982 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:47:45.982 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":1,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","modified":"2023-08-27T20:46:38.284854Z","created":"2023-08-27T20:46:38.284854Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:3300","nonce":0},{"type":"v1","addr":"172.21.15.93:6789","nonce":0}]},"addr":"172.21.15.93:6789/0","public_addr":"172.21.15.93:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0]} 2023-08-27T20:47:45.984 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 1 2023-08-27T20:47:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 systemd[1]: Starting Ceph mon.b for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:47:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:46 smithi093 ceph-mon[111335]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/3668061263' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: set uid:gid to 167:167 (ceph:ceph) 2023-08-27T20:47:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable), process ceph-mon, pid 2 2023-08-27T20:47:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: pidfile_write: ignore empty --pid-file 2023-08-27T20:47:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: load: jerasure load: lrc 2023-08-27T20:47:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: RocksDB version: 6.15.5 2023-08-27T20:47:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Compile date Aug 27 2023 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: DB SUMMARY 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: DB Session ID: YQ29KEUVVEAJ6BR8OV2B 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: CURRENT file: CURRENT 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: IDENTITY file: IDENTITY 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: SST files in /var/lib/ceph/mon/ceph-b/store.db dir, Total Num: 0, files: 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-b/store.db: 000004.log size: 511 ; 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.error_if_exists: 0 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.create_if_missing: 0 2023-08-27T20:47:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.paranoid_checks: 1 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.env: 0x55d4c8f76a80 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.fs: Posix File System 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.info_log: 0x55d4cb54dda0 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_file_opening_threads: 16 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.statistics: (nil) 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.use_fsync: 0 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_log_file_size: 0 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.log_file_time_to_roll: 0 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.keep_log_file_num: 1000 2023-08-27T20:47:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.recycle_log_file_num: 0 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.allow_fallocate: 1 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.allow_mmap_reads: 0 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.allow_mmap_writes: 0 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.use_direct_reads: 0 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.create_missing_column_families: 0 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.db_log_dir: 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-b/store.db 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.table_cache_numshardbits: 6 2023-08-27T20:47:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.WAL_ttl_seconds: 0 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.WAL_size_limit_MB: 0 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.is_fd_close_on_exec: 1 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.advise_random_on_open: 1 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.db_write_buffer_size: 0 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.write_buffer_manager: 0x55d4cb612930 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-27T20:47:46.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.use_adaptive_mutex: 0 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.rate_limiter: (nil) 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.wal_recovery_mode: 2 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.enable_thread_tracking: 0 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.enable_pipelined_write: 0 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.unordered_write: 0 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-27T20:47:46.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.row_cache: None 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.wal_filter: None 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.allow_ingest_behind: 0 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.preserve_deletes: 0 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.two_write_queues: 0 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.manual_wal_flush: 0 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.atomic_flush: 0 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-27T20:47:46.947 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.persist_stats_to_disk: 0 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.write_dbid_to_manifest: 0 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.log_readahead_size: 0 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.best_efforts_recovery: 0 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.allow_data_in_errors: 0 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.db_host_id: __hostname__ 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_background_jobs: 2 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_background_compactions: -1 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_subcompactions: 1 2023-08-27T20:47:46.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.delayed_write_rate : 16777216 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_total_wal_size: 0 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.stats_dump_period_sec: 600 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.stats_persist_period_sec: 600 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_open_files: -1 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bytes_per_sync: 0 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.wal_bytes_per_sync: 0 2023-08-27T20:47:46.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.strict_bytes_per_sync: 0 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_readahead_size: 0 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_background_flushes: -1 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Compression algorithms supported: 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kZSTD supported: 0 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kXpressCompression supported: 0 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kLZ4HCCompression supported: 1 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kZlibCompression supported: 1 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kSnappyCompression supported: 1 2023-08-27T20:47:46.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kLZ4Compression supported: 1 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: kBZip2Compression supported: 0 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Fast CRC32 supported: Supported on x86 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 bash[113513]: 833fe184137a9da68d1b394ddb66f7d97ab02e4e3d329ceb2a2cc1c19a63cb9e 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.merge_operator: 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_filter: None 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_filter_factory: None 2023-08-27T20:47:46.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.sst_partitioner_factory: None 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.memtable_factory: SkipListFactory 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.table_factory: BlockBasedTable 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x55d4cb51cd30) 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: cache_index_and_filter_blocks: 1 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: pin_top_level_index_and_filter: 1 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: index_type: 0 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: data_block_index_type: 0 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: index_shortening: 1 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: data_block_hash_table_util_ratio: 0.750000 2023-08-27T20:47:46.952 INFO:journalctl@ceph.mon.b.smithi136.stdout: hash_index_allow_collision: 1 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: checksum: 1 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: no_block_cache: 0 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_cache: 0x55d4cb5d51f0 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_cache_name: BinnedLRUCache 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_cache_options: 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: capacity : 536870912 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: num_shard_bits : 4 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: strict_capacity_limit : 0 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: high_pri_pool_ratio: 0.000 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_cache_compressed: (nil) 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: persistent_cache: (nil) 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_size: 4096 2023-08-27T20:47:46.953 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_size_deviation: 10 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_restart_interval: 16 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: index_block_restart_interval: 1 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: metadata_block_size: 4096 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: partition_filters: 0 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: use_delta_encoding: 1 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: whole_key_filtering: 1 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: verify_compression: 0 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: read_amp_bytes_per_bit: 0 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: format_version: 4 2023-08-27T20:47:46.954 INFO:journalctl@ceph.mon.b.smithi136.stdout: enable_index_compression: 1 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout: block_align: 0 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.write_buffer_size: 33554432 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_write_buffer_number: 2 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression: NoCompression 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression: Disabled 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.prefix_extractor: nullptr 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.num_levels: 7 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-27T20:47:46.955 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression_opts.window_bits: -14 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression_opts.level: 32767 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression_opts.strategy: 0 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-27T20:47:46.956 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compression_opts.enabled: false 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.target_file_size_base: 67108864 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.target_file_size_multiplier: 1 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-27T20:47:46.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.arena_block_size: 4194304 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.disable_auto_compactions: 0 2023-08-27T20:47:46.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.table_properties_collectors: 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.inplace_update_support: 0 2023-08-27T20:47:46.959 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.inplace_update_num_locks: 10000 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.memtable_huge_page_size: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.bloom_locality: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.max_successive_merges: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.optimize_filters_for_hits: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.paranoid_file_checks: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.force_consistency_checks: 1 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.report_bg_io_stats: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.ttl: 2592000 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.periodic_compaction_seconds: 0 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.enable_blob_files: false 2023-08-27T20:47:46.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.min_blob_size: 0 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.blob_file_size: 268435456 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.blob_compression_type: NoCompression 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.enable_blob_garbage_collection: false 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-b/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169266616532, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169266616799, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1545, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 523, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 401, "raw_average_value_size": 80, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1693169266, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "bb288d5a-2147-4a6a-8cc6-d15b09a00c14", "db_session_id": "YQ29KEUVVEAJ6BR8OV2B"}} 2023-08-27T20:47:46.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169266617124, "job": 1, "event": "recovery_finished"} 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-b/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x55d4cb569180 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: DB pointer 0x55d4cb5de000 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout: ** DB Stats ** 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-27T20:47:46.962 INFO:journalctl@ceph.mon.b.smithi136.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:46.963 INFO:journalctl@ceph.mon.b.smithi136.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: Cumulative compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: Interval compaction: 0.00 GB write, 0.57 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T20:47:46.964 INFO:journalctl@ceph.mon.b.smithi136.stdout: 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: L0 1/0 1.51 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: Sum 1/0 1.51 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:46.965 INFO:journalctl@ceph.mon.b.smithi136.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 5.9 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: Cumulative compaction: 0.00 GB write, 0.56 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: 2023-08-27T20:47:46.966 INFO:journalctl@ceph.mon.b.smithi136.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b does not exist in monmap, will attempt to join an existing cluster 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: using public_addr v2:172.21.15.136:0/0 -> [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: starting mon.b rank -1 at public addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] at bind addrs [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] mon_data /var/lib/ceph/mon/ceph-b fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(???) e0 preinit fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 systemd[1]: Started Ceph mon.b for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).mds e1 new map 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).mds e1 print_map 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout: e1 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout: legacy client fscid: -1 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout: 2023-08-27T20:47:46.967 INFO:journalctl@ceph.mon.b.smithi136.stdout: No filesystems configured 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mkfs b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:46.968 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: monmap e1: 1 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0]} removed_ranks: {} 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: fsmap 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: osdmap e1: 0 total, 0 up, 0 in 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e1: no daemons active 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2925247725' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3166997349' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3166997349' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/927220789' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: monmap e1: 1 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0]} removed_ranks: {} 2023-08-27T20:47:46.969 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: fsmap 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: osdmap e1: 0 total, 0 up, 0 in 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e1: no daemons active 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3269501682' entity='client.admin' 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1025304043' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2088105512' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1770227879' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Activating manager daemon y 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e2: y(active, starting, since 0.00292871s) 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:46.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Manager daemon y is now available 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e3: y(active, since 1.00635s) 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3675520105' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e4: y(active, since 3s) 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2070046131' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/272688912' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-08-27T20:47:46.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/272688912' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e5: y(active, since 4s) 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1859399544' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Active manager daemon y restarted 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Activating manager daemon y 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: osdmap e2: 0 total, 0 up, 0 in 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e6: y(active, starting, since 0.00315154s) 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Manager daemon y is now available 2023-08-27T20:47:46.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Found migration_current of "None". Setting to last migration. 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e7: y(active, since 1.00476s) 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: [27/Aug/2023:20:47:03] ENGINE Bus STARTING 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: [27/Aug/2023:20:47:04] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: [27/Aug/2023:20:47:04] ENGINE Bus STARTED 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e8: y(active, since 2s) 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Generating ssh key... 2023-08-27T20:47:46.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi093", "addr": "172.21.15.93", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Deploying cephadm binary to smithi093 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Added host smithi093 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Saving service mon spec with placement count:5 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Saving service mgr spec with placement count:2 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1068952983' entity='client.admin' 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4068444121' entity='client.admin' 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3065173678' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3065173678' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e9: y(active, since 12s) 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2702055024' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Active manager daemon y restarted 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Activating manager daemon y 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: osdmap e3: 0 total, 0 up, 0 in 2023-08-27T20:47:46.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e10: y(active, starting, since 0.00561952s) 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Manager daemon y is now available 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e11: y(active, since 1.00529s) 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: [27/Aug/2023:20:47:21] ENGINE Bus STARTING 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: [27/Aug/2023:20:47:21] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: [27/Aug/2023:20:47:21] ENGINE Bus STARTED 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e12: y(active, since 2s) 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2353321583' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mgrmap e13: y(active, since 6s) 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1889985893' entity='client.admin' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi093", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2286130882' entity='client.admin' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.979 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi136", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Deploying cephadm binary to smithi136 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Added host smithi136 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.981 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi136", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3478248900' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3478248900' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-08-27T20:47:46.982 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi093:172.21.15.93=a;smithi093:[v2:172.21.15.93:3301,v1:172.21.15.93:6790]=c;smithi136:172.21.15.136=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Saving service mon spec with placement smithi093:172.21.15.93=a;smithi093:[v2:172.21.15.93:3301,v1:172.21.15.93:6790]=c;smithi136:172.21.15.136=b;count:3 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: Deploying daemon mon.b on smithi136 2023-08-27T20:47:46.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:46.984 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/3668061263' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:47:46.984 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-08-27T20:47:46.984 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:46 smithi136 ceph-mon[113531]: mon.b@-1(synchronizing) e1 handle_conf_change mon_allow_pool_delete 2023-08-27T20:47:47.328 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-27T20:47:47.329 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-27T20:47:47.590 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:47:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: mon.a calling monitor election 2023-08-27T20:47:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: mon.b calling monitor election 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: monmap e2: 2 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0],b=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0]} removed_ranks: {} 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: fsmap 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: mgrmap e13: y(active, since 30s) 2023-08-27T20:47:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: overall HEALTH_OK 2023-08-27T20:47:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:52.009 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:47:52.009 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":2,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","modified":"2023-08-27T20:47:46.639287Z","created":"2023-08-27T20:46:38.284854Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:3300","nonce":0},{"type":"v1","addr":"172.21.15.93:6789","nonce":0}]},"addr":"172.21.15.93:6789/0","public_addr":"172.21.15.93:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:3300","nonce":0},{"type":"v1","addr":"172.21.15.136:6789","nonce":0}]},"addr":"172.21.15.136:6789/0","public_addr":"172.21.15.136:6789/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1]} 2023-08-27T20:47:52.011 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 2 2023-08-27T20:47:52.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:52.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: mon.a calling monitor election 2023-08-27T20:47:52.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:52.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:52.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:52.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: mon.b calling monitor election 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: monmap e2: 2 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0],b=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0]} removed_ranks: {} 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: fsmap 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: mgrmap e13: y(active, since 30s) 2023-08-27T20:47:52.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: overall HEALTH_OK 2023-08-27T20:47:52.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:52.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:52.879 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:52 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:52.879 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:52 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:47:52.879 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:52 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:52.879 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:52 smithi093 ceph-mon[111335]: Deploying daemon mon.c on smithi093 2023-08-27T20:47:52.879 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/2621177809' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:47:52.880 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:52 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:52 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:52 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:47:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:52 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:52 smithi136 ceph-mon[113531]: Deploying daemon mon.c on smithi093 2023-08-27T20:47:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/2621177809' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:47:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:52 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:53.390 INFO:tasks.cephadm:Waiting for 3 mons in monmap... 2023-08-27T20:47:53.390 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph mon dump -f json 2023-08-27T20:47:53.658 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:47:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 systemd[1]: Starting Ceph mon.c for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:47:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: set uid:gid to 167:167 (ceph:ceph) 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable), process ceph-mon, pid 2 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: pidfile_write: ignore empty --pid-file 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 bash[118883]: 9ec6c73ae721fea49b4142f598237a9d4b1dbcfe15907bc4fab33c365b29046a 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 systemd[1]: Started Ceph mon.c for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: load: jerasure load: lrc 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: RocksDB version: 6.15.5 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Git sha rocksdb_build_git_sha:@0@ 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Compile date Aug 27 2023 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: DB SUMMARY 2023-08-27T20:47:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: DB Session ID: T3CWVMDI9190U5GXUFVD 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: CURRENT file: CURRENT 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: IDENTITY file: IDENTITY 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: MANIFEST file: MANIFEST-000003 size: 57 Bytes 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: SST files in /var/lib/ceph/mon/ceph-c/store.db dir, Total Num: 0, files: 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Write Ahead Log file in /var/lib/ceph/mon/ceph-c/store.db: 000004.log size: 636 ; 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.error_if_exists: 0 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.create_if_missing: 0 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.paranoid_checks: 1 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.track_and_verify_wals_in_manifest: 0 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.env: 0x558d13593a80 2023-08-27T20:47:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.fs: Posix File System 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.info_log: 0x558d14e17da0 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_file_opening_threads: 16 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.statistics: (nil) 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.use_fsync: 0 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_log_file_size: 0 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_manifest_file_size: 1073741824 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.log_file_time_to_roll: 0 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.keep_log_file_num: 1000 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.recycle_log_file_num: 0 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.allow_fallocate: 1 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.allow_mmap_reads: 0 2023-08-27T20:47:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.allow_mmap_writes: 0 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.use_direct_reads: 0 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.use_direct_io_for_flush_and_compaction: 0 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.create_missing_column_families: 0 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.db_log_dir: 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.wal_dir: /var/lib/ceph/mon/ceph-c/store.db 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.table_cache_numshardbits: 6 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.WAL_ttl_seconds: 0 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.WAL_size_limit_MB: 0 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_write_batch_group_size_bytes: 1048576 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.manifest_preallocation_size: 4194304 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.is_fd_close_on_exec: 1 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.advise_random_on_open: 1 2023-08-27T20:47:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.db_write_buffer_size: 0 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.write_buffer_manager: 0x558d14edc900 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.access_hint_on_compaction_start: 1 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.new_table_reader_for_compaction_inputs: 0 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.random_access_max_buffer_size: 1048576 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.use_adaptive_mutex: 0 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.rate_limiter: (nil) 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.sst_file_manager.rate_bytes_per_sec: 0 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.wal_recovery_mode: 2 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.enable_thread_tracking: 0 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.enable_pipelined_write: 0 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.unordered_write: 0 2023-08-27T20:47:54.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.allow_concurrent_memtable_write: 1 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.enable_write_thread_adaptive_yield: 1 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.write_thread_max_yield_usec: 100 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.write_thread_slow_yield_usec: 3 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.row_cache: None 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.wal_filter: None 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.avoid_flush_during_recovery: 0 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.allow_ingest_behind: 0 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.preserve_deletes: 0 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.two_write_queues: 0 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.manual_wal_flush: 0 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.atomic_flush: 0 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.avoid_unnecessary_blocking_io: 0 2023-08-27T20:47:54.245 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.persist_stats_to_disk: 0 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.write_dbid_to_manifest: 0 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.log_readahead_size: 0 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.file_checksum_gen_factory: Unknown 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.best_efforts_recovery: 0 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bgerror_resume_count: 2147483647 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bgerror_resume_retry_interval: 1000000 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.allow_data_in_errors: 0 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.db_host_id: __hostname__ 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_background_jobs: 2 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_background_compactions: -1 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_subcompactions: 1 2023-08-27T20:47:54.246 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.avoid_flush_during_shutdown: 0 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.writable_file_max_buffer_size: 1048576 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.delayed_write_rate : 16777216 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_total_wal_size: 0 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.delete_obsolete_files_period_micros: 21600000000 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.stats_dump_period_sec: 600 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.stats_persist_period_sec: 600 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.stats_history_buffer_size: 1048576 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_open_files: -1 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bytes_per_sync: 0 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.wal_bytes_per_sync: 0 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.strict_bytes_per_sync: 0 2023-08-27T20:47:54.247 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_readahead_size: 0 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_background_flushes: -1 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Compression algorithms supported: 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kZSTDNotFinalCompression supported: 0 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kZSTD supported: 0 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kXpressCompression supported: 0 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kLZ4HCCompression supported: 1 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kZlibCompression supported: 1 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kSnappyCompression supported: 1 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kLZ4Compression supported: 1 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: kBZip2Compression supported: 0 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Fast CRC32 supported: Supported on x86 2023-08-27T20:47:54.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/version_set.cc:4724] Recovering from manifest file: /var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/column_family.cc:595] --------------- Options for column family [default]: 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.comparator: leveldb.BytewiseComparator 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.merge_operator: 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_filter: None 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_filter_factory: None 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.sst_partitioner_factory: None 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.memtable_factory: SkipListFactory 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.table_factory: BlockBasedTable 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: table_factory options: flush_block_policy_factory: FlushBlockBySizePolicyFactory (0x558d14de6d30) 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout: cache_index_and_filter_blocks: 1 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout: cache_index_and_filter_blocks_with_high_priority: 0 2023-08-27T20:47:54.249 INFO:journalctl@ceph.mon.c.smithi093.stdout: pin_l0_filter_and_index_blocks_in_cache: 0 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: pin_top_level_index_and_filter: 1 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: index_type: 0 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: data_block_index_type: 0 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: index_shortening: 1 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: data_block_hash_table_util_ratio: 0.750000 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: hash_index_allow_collision: 1 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: checksum: 1 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: no_block_cache: 0 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_cache: 0x558d14e9f1f0 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_cache_name: BinnedLRUCache 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_cache_options: 2023-08-27T20:47:54.250 INFO:journalctl@ceph.mon.c.smithi093.stdout: capacity : 536870912 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: num_shard_bits : 4 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: strict_capacity_limit : 0 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: high_pri_pool_ratio: 0.000 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_cache_compressed: (nil) 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: persistent_cache: (nil) 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_size: 4096 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_size_deviation: 10 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_restart_interval: 16 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: index_block_restart_interval: 1 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: metadata_block_size: 4096 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: partition_filters: 0 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: use_delta_encoding: 1 2023-08-27T20:47:54.251 INFO:journalctl@ceph.mon.c.smithi093.stdout: filter_policy: rocksdb.BuiltinBloomFilter 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout: whole_key_filtering: 1 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout: verify_compression: 0 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout: read_amp_bytes_per_bit: 0 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout: format_version: 4 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout: enable_index_compression: 1 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout: block_align: 0 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.write_buffer_size: 33554432 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_write_buffer_number: 2 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression: NoCompression 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression: Disabled 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.prefix_extractor: nullptr 2023-08-27T20:47:54.252 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.memtable_insert_with_hint_prefix_extractor: nullptr 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.num_levels: 7 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.min_write_buffer_number_to_merge: 1 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_write_buffer_number_to_maintain: 0 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_write_buffer_size_to_maintain: 0 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression_opts.window_bits: -14 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression_opts.level: 32767 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression_opts.strategy: 0 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression_opts.max_dict_bytes: 0 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression_opts.zstd_max_train_bytes: 0 2023-08-27T20:47:54.253 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression_opts.parallel_threads: 1 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bottommost_compression_opts.enabled: false 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression_opts.window_bits: -14 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression_opts.level: 32767 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression_opts.strategy: 0 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression_opts.max_dict_bytes: 0 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression_opts.zstd_max_train_bytes: 0 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression_opts.parallel_threads: 1 2023-08-27T20:47:54.255 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compression_opts.enabled: false 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.level0_file_num_compaction_trigger: 4 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.level0_slowdown_writes_trigger: 20 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.level0_stop_writes_trigger: 36 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.target_file_size_base: 67108864 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.target_file_size_multiplier: 1 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_base: 268435456 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.level_compaction_dynamic_level_bytes: 1 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier: 10.000000 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[0]: 1 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[1]: 1 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[2]: 1 2023-08-27T20:47:54.256 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[3]: 1 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[4]: 1 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[5]: 1 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_bytes_for_level_multiplier_addtl[6]: 1 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_sequential_skip_in_iterations: 8 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_compaction_bytes: 1677721600 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.arena_block_size: 4194304 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.soft_pending_compaction_bytes_limit: 68719476736 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.hard_pending_compaction_bytes_limit: 274877906944 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.rate_limit_delay_max_milliseconds: 100 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.disable_auto_compactions: 0 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_style: kCompactionStyleLevel 2023-08-27T20:47:54.257 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_pri: kMinOverlappingRatio 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_universal.size_ratio: 1 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_universal.min_merge_width: 2 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_universal.max_merge_width: 4294967295 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_universal.max_size_amplification_percent: 200 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_universal.compression_size_percent: -1 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_universal.stop_style: kCompactionStopStyleTotalSize 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_fifo.max_table_files_size: 1073741824 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.compaction_options_fifo.allow_compaction: 0 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.table_properties_collectors: 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.inplace_update_support: 0 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.inplace_update_num_locks: 10000 2023-08-27T20:47:54.258 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.memtable_prefix_bloom_size_ratio: 0.000000 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.memtable_whole_key_filtering: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.memtable_huge_page_size: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.bloom_locality: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.max_successive_merges: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.optimize_filters_for_hits: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.paranoid_file_checks: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.force_consistency_checks: 1 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.report_bg_io_stats: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.ttl: 2592000 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.periodic_compaction_seconds: 0 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.enable_blob_files: false 2023-08-27T20:47:54.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.min_blob_size: 0 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.blob_file_size: 268435456 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.blob_compression_type: NoCompression 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.enable_blob_garbage_collection: false 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: Options.blob_garbage_collection_age_cutoff: 0.250000 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/version_set.cc:4764] Recovered from manifest file:/var/lib/ceph/mon/ceph-c/store.db/MANIFEST-000003 succeeded,manifest_file_number is 3, next_file_number is 5, last_sequence is 0, log_number is 0,prev_log_number is 0,max_column_family is 0,min_log_number_to_keep is 0 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/version_set.cc:4779] Column family [default] (ID 0), log number is 0 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/version_set.cc:4082] Creating manifest 7 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169273750783, "job": 1, "event": "recovery_started", "wal_files": [4]} 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/db_impl/db_impl_open.cc:845] Recovering log #4 mode 2 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [table/block_based/filter_policy.cc:991] Using legacy Bloom filter with high (20) bits/key. Dramatic filter space and/or accuracy improvement is available with format_version>=5. 2023-08-27T20:47:54.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169273751044, "cf_name": "default", "job": 1, "event": "table_file_creation", "file_number": 8, "file_size": 1670, "file_checksum": "", "file_checksum_func_name": "Unknown", "table_properties": {"data_size": 648, "index_size": 31, "index_partitions": 0, "top_level_index_size": 0, "index_key_is_user_key": 1, "index_value_is_delta_encoded": 1, "filter_size": 69, "raw_key_size": 115, "raw_average_key_size": 23, "raw_value_size": 526, "raw_average_value_size": 105, "num_data_blocks": 1, "num_entries": 5, "num_deletions": 0, "num_merge_operands": 0, "num_range_deletions": 0, "format_version": 0, "fixed_key_len": 0, "filter_policy": "rocksdb.BuiltinBloomFilter", "column_family_name": "default", "column_family_id": 0, "comparator": "leveldb.BytewiseComparator", "merge_operator": "", "prefix_extractor_name": "nullptr", "property_collectors": "[]", "compression": "NoCompression", "compression_options": "window_bits=-14; level=32767; strategy=0; max_dict_bytes=0; zstd_max_train_bytes=0; enabled=0; ", "creation_time": 1693169273, "oldest_key_time": 0, "file_creation_time": 0, "db_id": "4c928c8f-cb53-4f2d-8a57-20c8b7e57533", "db_session_id": "T3CWVMDI9190U5GXUFVD"}} 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/version_set.cc:4082] Creating manifest 9 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: EVENT_LOG_v1 {"time_micros": 1693169273751362, "job": 1, "event": "recovery_finished"} 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [file/delete_scheduler.cc:69] Deleted file /var/lib/ceph/mon/ceph-c/store.db/000004.log immediately, rate_bytes_per_sec 0, total_trash_size 0 max_trash_db_ratio 0.250000 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/db_impl/db_impl_open.cc:1700] SstFileManager instance 0x558d14e33180 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: DB pointer 0x558d14e4e000 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/db_impl/db_impl.cc:901] ------- DUMPING STATS ------- 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: rocksdb: [db/db_impl/db_impl.cc:903] 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout: ** DB Stats ** 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout: Cumulative writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 GB, 0.00 MB/s 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout: Cumulative WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 GB, 0.00 MB/s 2023-08-27T20:47:54.261 INFO:journalctl@ceph.mon.c.smithi093.stdout: Cumulative stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: Interval writes: 0 writes, 0 keys, 0 commit groups, 0.0 writes per commit group, ingest: 0.00 MB, 0.00 MB/s 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: Interval WAL: 0 writes, 0 syncs, 0.00 writes per sync, written: 0.00 MB, 0.00 MB/s 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: Interval stall: 00:00:0.000 H:M:S, 0.0 percent 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.6 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.6 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.6 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:54.262 INFO:journalctl@ceph.mon.c.smithi093.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.6 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: Interval compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T20:47:54.263 INFO:journalctl@ceph.mon.c.smithi093.stdout: 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: Level Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: ---------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: L0 1/0 1.63 KB 0.2 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.6 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: Sum 1/0 1.63 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 1.0 0.0 6.6 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: Int 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.00 0.00 0 0.000 0 0 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: ** Compaction Stats [default] ** 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: Priority Files Size Score Read(GB) Rn(GB) Rnp1(GB) Write(GB) Wnew(GB) Moved(GB) W-Amp Rd(MB/s) Wr(MB/s) Comp(sec) CompMergeCPU(sec) Comp(cnt) Avg(sec) KeyIn KeyDrop 2023-08-27T20:47:54.264 INFO:journalctl@ceph.mon.c.smithi093.stdout: ------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: User 0/0 0.00 KB 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 0.0 6.6 0.00 0.00 1 0.000 0 0 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: Uptime(secs): 0.0 total, 0.0 interval 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: Flush(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(GB): cumulative 0.000, interval 0.000 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(Total Files): cumulative 0, interval 0 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(L0 Files): cumulative 0, interval 0 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: AddFile(Keys): cumulative 0, interval 0 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: Cumulative compaction: 0.00 GB write, 0.60 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: Interval compaction: 0.00 GB write, 0.00 MB/s write, 0.00 GB read, 0.00 MB/s read, 0.0 seconds 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: Stalls(count): 0 level0_slowdown, 0 level0_slowdown_with_compaction, 0 level0_numfiles, 0 level0_numfiles_with_compaction, 0 stop for pending_compaction_bytes, 0 slowdown for pending_compaction_bytes, 0 memtable_compaction, 0 memtable_slowdown, interval 0 total count 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: 2023-08-27T20:47:54.265 INFO:journalctl@ceph.mon.c.smithi093.stdout: ** File Read Latency Histogram By Level [default] ** 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c does not exist in monmap, will attempt to join an existing cluster 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: using public_addrv [v2:172.21.15.93:3301/0,v1:172.21.15.93:6790/0] 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: starting mon.c rank -1 at public addrs [v2:172.21.15.93:3301/0,v1:172.21.15.93:6790/0] at bind addrs [v2:172.21.15.93:3301/0,v1:172.21.15.93:6790/0] mon_data /var/lib/ceph/mon/ceph-c fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(???) e0 preinit fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).mds e1 new map 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).mds e1 print_map 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout: e1 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout: enable_multiple, ever_enabled_multiple: 1,1 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout: default compat: compat={},rocompat={},incompat={1=base v0.20,2=client writeable ranges,3=default file layouts on dirs,4=dir inode in separate object,5=mds uses versioned encoding,6=dirfrag is stored in omap,8=no anchor table,9=file layout v2,10=snaprealm v2} 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout: legacy client fscid: -1 2023-08-27T20:47:54.266 INFO:journalctl@ceph.mon.c.smithi093.stdout: 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout: No filesystems configured 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e0 _set_cache_ratios kv ratio 0.25 inc ratio 0.375 full ratio 0.375 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e0 register_cache_with_pcm pcm target: 2147483648 pcm max: 1020054732 pcm min: 134217728 inc_osd_cache size: 1 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e1 e1: 0 total, 0 up, 0 in 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e2 e2: 0 total, 0 up, 0 in 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e3 e3: 0 total, 0 up, 0 in 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e4 e4: 0 total, 0 up, 0 in 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e4 crush map has features 3314932999778484224, adjusting msgr requires 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).osd e4 crush map has features 288514050185494528, adjusting msgr requires 2023-08-27T20:47:54.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mkfs b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: monmap e1: 1 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0]} removed_ranks: {} 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: fsmap 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: osdmap e1: 0 total, 0 up, 0 in 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e1: no daemons active 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2925247725' entity='client.admin' cmd=[{"prefix": "status"}]: dispatch 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3166997349' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3166997349' entity='client.admin' cmd='[{"prefix": "config assimilate-conf"}]': finished 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/927220789' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.a is new leader, mons a in quorum (ranks 0) 2023-08-27T20:47:54.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: monmap e1: 1 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0]} removed_ranks: {} 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: fsmap 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: osdmap e1: 0 total, 0 up, 0 in 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e1: no daemons active 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3269501682' entity='client.admin' 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1025304043' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2088105512' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1770227879' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Activating manager daemon y 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e2: y(active, starting, since 0.00292871s) 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:54.269 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Manager daemon y is now available 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14100 172.21.15.93:0/4100246569' entity='mgr.y' 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e3: y(active, since 1.00635s) 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3675520105' entity='client.admin' cmd=[{"prefix": "status", "format": "json-pretty"}]: dispatch 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e4: y(active, since 3s) 2023-08-27T20:47:54.270 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2070046131' entity='client.admin' cmd=[{"prefix": "config assimilate-conf"}]: dispatch 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/272688912' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "cephadm"}]: dispatch 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/272688912' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "cephadm"}]': finished 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e5: y(active, since 4s) 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1859399544' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Active manager daemon y restarted 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Activating manager daemon y 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: osdmap e2: 0 total, 0 up, 0 in 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e6: y(active, starting, since 0.00315154s) 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:54.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Manager daemon y is now available 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Found migration_current of "None". Setting to last migration. 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e7: y(active, since 1.00476s) 2023-08-27T20:47:54.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14124 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: [27/Aug/2023:20:47:03] ENGINE Bus STARTING 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: [27/Aug/2023:20:47:04] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: [27/Aug/2023:20:47:04] ENGINE Bus STARTED 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14132 -' entity='client.admin' cmd=[{"prefix": "orch set backend", "module_name": "cephadm", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e8: y(active, since 2s) 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14134 -' entity='client.admin' cmd=[{"prefix": "cephadm set-user", "user": "root", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14136 -' entity='client.admin' cmd=[{"prefix": "cephadm generate-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Generating ssh key... 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14138 -' entity='client.admin' cmd=[{"prefix": "cephadm get-pub-key", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14140 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi093", "addr": "172.21.15.93", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Deploying cephadm binary to smithi093 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Added host smithi093 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14142 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Saving service mon spec with placement count:5 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14144 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "unmanaged": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Saving service mgr spec with placement count:2 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1068952983' entity='client.admin' 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4068444121' entity='client.admin' 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14120 172.21.15.93:0/653478021' entity='mgr.y' 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3065173678' entity='client.admin' cmd=[{"prefix": "mgr module enable", "module": "dashboard"}]: dispatch 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3065173678' entity='client.admin' cmd='[{"prefix": "mgr module enable", "module": "dashboard"}]': finished 2023-08-27T20:47:54.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e9: y(active, since 12s) 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2702055024' entity='client.admin' cmd=[{"prefix": "mgr stat"}]: dispatch 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Active manager daemon y restarted 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Activating manager daemon y 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: osdmap e3: 0 total, 0 up, 0 in 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e10: y(active, starting, since 0.00561952s) 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Manager daemon y is now available 2023-08-27T20:47:54.276 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e11: y(active, since 1.00529s) 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: [27/Aug/2023:20:47:21] ENGINE Bus STARTING 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: [27/Aug/2023:20:47:21] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: [27/Aug/2023:20:47:21] ENGINE Bus STARTED 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "get_command_descriptions"}]: dispatch 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14156 -' entity='client.admin' cmd=[{"prefix": "mgr_status"}]: dispatch 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.277 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14164 -' entity='client.admin' cmd=[{"prefix": "dashboard create-self-signed-cert", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e12: y(active, since 2s) 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14166 -' entity='client.admin' cmd=[{"prefix": "dashboard ac-user-create", "username": "admin", "rolename": "administrator", "force_password": true, "pwd_update_required": true, "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2353321583' entity='client.admin' cmd=[{"prefix": "config get", "who": "mgr", "key": "mgr/dashboard/ssl_server_port"}]: dispatch 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e13: y(active, since 6s) 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1889985893' entity='client.admin' 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi093", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:47:54.278 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2286130882' entity='client.admin' 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14176 -' entity='client.admin' cmd=[{"prefix": "orch client-keyring set", "entity": "client.admin", "placement": "*", "mode": "0755", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:47:54.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14178 -' entity='client.admin' cmd=[{"prefix": "orch host add", "hostname": "smithi136", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Deploying cephadm binary to smithi136 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Added host smithi136 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14180 -' entity='client.admin' cmd=[{"prefix": "orch host ls", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd/host:smithi136", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3478248900' entity='client.admin' cmd=[{"prefix": "osd crush tunables", "profile": "default"}]: dispatch 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:47:54.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: pgmap v3: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3478248900' entity='client.admin' cmd='[{"prefix": "osd crush tunables", "profile": "default"}]': finished 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: pgmap v5: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.14184 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mon", "placement": "3;smithi093:172.21.15.93=a;smithi093:[v2:172.21.15.93:3301,v1:172.21.15.93:6790]=c;smithi136:172.21.15.136=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Saving service mon spec with placement smithi093:172.21.15.93=a;smithi093:[v2:172.21.15.93:3301,v1:172.21.15.93:6790]=c;smithi136:172.21.15.136=b;count:3 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.282 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Deploying daemon mon.b on smithi136 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: pgmap v6: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/3668061263' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.a calling monitor election 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:54.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: pgmap v7: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.b calling monitor election 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: pgmap v8: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: pgmap v9: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.a is new leader, mons a,b in quorum (ranks 0,1) 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: monmap e2: 2 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0],b=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0]} removed_ranks: {} 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: fsmap 2023-08-27T20:47:54.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mgrmap e13: y(active, since 30s) 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: overall HEALTH_OK 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: Deploying daemon mon.c on smithi093 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/2621177809' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing).paxosservice(auth 1..3) refresh upgraded, format 0 -> 3 2023-08-27T20:47:54.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:53 smithi093 ceph-mon[118902]: mon.c@-1(synchronizing) e2 handle_conf_change mon_allow_pool_delete 2023-08-27T20:47:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: mon.a calling monitor election 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: mon.b calling monitor election 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: mon.c calling monitor election 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: monmap e3: 3 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0],b=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],c=[v2:172.21.15.93:3301/0,v1:172.21.15.93:6790/0]} removed_ranks: {} 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: fsmap 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: mgrmap e13: y(active, since 37s) 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: overall HEALTH_OK 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:59.218 INFO:teuthology.orchestra.run.smithi136.stdout: 2023-08-27T20:47:59.218 INFO:teuthology.orchestra.run.smithi136.stdout:{"epoch":3,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","modified":"2023-08-27T20:47:53.773078Z","created":"2023-08-27T20:46:38.284854Z","min_mon_release":17,"min_mon_release_name":"quincy","election_strategy":1,"disallowed_leaders: ":"","stretch_mode":false,"tiebreaker_mon":"","removed_ranks: ":"","features":{"persistent":["kraken","luminous","mimic","osdmap-prune","nautilus","octopus","pacific","elector-pinging","quincy"],"optional":[]},"mons":[{"rank":0,"name":"a","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:3300","nonce":0},{"type":"v1","addr":"172.21.15.93:6789","nonce":0}]},"addr":"172.21.15.93:6789/0","public_addr":"172.21.15.93:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":1,"name":"b","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:3300","nonce":0},{"type":"v1","addr":"172.21.15.136:6789","nonce":0}]},"addr":"172.21.15.136:6789/0","public_addr":"172.21.15.136:6789/0","priority":0,"weight":0,"crush_location":"{}"},{"rank":2,"name":"c","public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:3301","nonce":0},{"type":"v1","addr":"172.21.15.93:6790","nonce":0}]},"addr":"172.21.15.93:6790/0","public_addr":"172.21.15.93:6790/0","priority":0,"weight":0,"crush_location":"{}"}],"quorum":[0,1,2]} 2023-08-27T20:47:59.220 INFO:teuthology.orchestra.run.smithi136.stderr:dumped monmap epoch 3 2023-08-27T20:47:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: mon.a calling monitor election 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: mon.b calling monitor election 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: mon.c calling monitor election 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: monmap e3: 3 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0],b=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],c=[v2:172.21.15.93:3301/0,v1:172.21.15.93:6790/0]} removed_ranks: {} 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: fsmap 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: mgrmap e13: y(active, since 37s) 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: overall HEALTH_OK 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: mon.a calling monitor election 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: mon.b calling monitor election 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: pgmap v11: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: mon.c calling monitor election 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: pgmap v12: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: mon.a is new leader, mons a,b,c in quorum (ranks 0,1,2) 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: monmap e3: 3 mons at {a=[v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0],b=[v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0],c=[v2:172.21.15.93:3301/0,v1:172.21.15.93:6790/0]} removed_ranks: {} 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: fsmap 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: osdmap e4: 0 total, 0 up, 0 in 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: mgrmap e13: y(active, since 37s) 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: overall HEALTH_OK 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:59.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:47:59.541 INFO:tasks.cephadm:Generating final ceph.conf file... 2023-08-27T20:47:59.542 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph config generate-minimal-conf 2023-08-27T20:47:59.806 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:59 smithi136 ceph-mon[113531]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/250071642' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:48:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:47:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:48:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[111335]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/250071642' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:48:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:48:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[118902]: pgmap v13: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/250071642' entity='client.admin' cmd=[{"prefix": "mon dump", "format": "json"}]: dispatch 2023-08-27T20:48:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:47:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:48:00.984 INFO:teuthology.orchestra.run.smithi093.stdout:# minimal ceph.conf for b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:48:00.984 INFO:teuthology.orchestra.run.smithi093.stdout:[global] 2023-08-27T20:48:00.985 INFO:teuthology.orchestra.run.smithi093.stdout: fsid = b79a52c0-451a-11ee-817b-1b7a5f1ba315 2023-08-27T20:48:00.985 INFO:teuthology.orchestra.run.smithi093.stdout: mon_host = [v2:172.21.15.93:3300/0,v1:172.21.15.93:6789/0] [v2:172.21.15.136:3300/0,v1:172.21.15.136:6789/0] [v2:172.21.15.93:3301/0,v1:172.21.15.93:6790/0] 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:48:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:48:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:01.763 INFO:tasks.cephadm:Distributing (final) config and client.admin keyring... 2023-08-27T20:48:01.763 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:48:01.763 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/etc/ceph/ceph.conf 2023-08-27T20:48:01.794 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:48:01.794 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:48:01.862 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:48:01.862 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.conf 2023-08-27T20:48:01.899 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:48:01.899 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:48:01.969 INFO:tasks.cephadm:Adding mgr.y on smithi093 2023-08-27T20:48:01.969 INFO:tasks.cephadm:Adding mgr.x on smithi136 2023-08-27T20:48:01.969 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch apply mgr '2;smithi093=y;smithi136=x' 2023-08-27T20:48:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: Reconfiguring mon.a (unknown last config time)... 2023-08-27T20:48:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: Reconfiguring daemon mon.a on smithi093 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/902448352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: Reconfiguring mon.a (unknown last config time)... 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: Reconfiguring daemon mon.a on smithi093 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/902448352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: Reconfiguring mon.a (unknown last config time)... 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: Reconfiguring daemon mon.a on smithi093 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/902448352' entity='client.admin' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: pgmap v14: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:02.263 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:48:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:02 smithi136 ceph-mon[113531]: Reconfiguring mon.c (monmap changed)... 2023-08-27T20:48:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:02 smithi136 ceph-mon[113531]: Reconfiguring daemon mon.c on smithi093 2023-08-27T20:48:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[118902]: Reconfiguring mon.c (monmap changed)... 2023-08-27T20:48:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[118902]: Reconfiguring daemon mon.c on smithi093 2023-08-27T20:48:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[111335]: Reconfiguring mon.c (monmap changed)... 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[111335]: Reconfiguring daemon mon.c on smithi093 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "mon."}]: dispatch 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config get", "who": "mon", "key": "public_network"}]: dispatch 2023-08-27T20:48:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:03.439 INFO:teuthology.orchestra.run.smithi136.stdout:Scheduled mgr update... 2023-08-27T20:48:03.995 DEBUG:teuthology.orchestra.run.smithi136:mgr.x> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.x.service 2023-08-27T20:48:03.997 INFO:tasks.cephadm:Deploying OSDs... 2023-08-27T20:48:03.997 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:48:03.998 DEBUG:teuthology.orchestra.run.smithi093:> dd if=/scratch_devs of=/dev/stdout 2023-08-27T20:48:04.020 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-08-27T20:48:04.021 DEBUG:teuthology.orchestra.run.smithi093:> stat /dev/vg_nvme/lv_1 2023-08-27T20:48:04.026 INFO:journalctl@ceph.mgr.x.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout:Device: 6h/6d Inode: 392689 Links: 1 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout:Access: 2023-08-27 20:47:53.391947890 +0000 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout:Modify: 2023-08-27 20:41:31.549526022 +0000 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout:Change: 2023-08-27 20:41:31.549526022 +0000 2023-08-27T20:48:04.083 INFO:teuthology.orchestra.run.smithi093.stdout: Birth: - 2023-08-27T20:48:04.084 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-08-27T20:48:04.156 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records in 2023-08-27T20:48:04.156 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records out 2023-08-27T20:48:04.156 INFO:teuthology.orchestra.run.smithi093.stderr:512 bytes copied, 0.000269443 s, 1.9 MB/s 2023-08-27T20:48:04.157 DEBUG:teuthology.orchestra.run.smithi093:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-08-27T20:48:04.221 DEBUG:teuthology.orchestra.run.smithi093:> stat /dev/vg_nvme/lv_2 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout:Device: 6h/6d Inode: 391560 Links: 1 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout:Access: 2023-08-27 20:47:53.391947890 +0000 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout:Modify: 2023-08-27 20:41:32.055515183 +0000 2023-08-27T20:48:04.284 INFO:teuthology.orchestra.run.smithi093.stdout:Change: 2023-08-27 20:41:32.055515183 +0000 2023-08-27T20:48:04.285 INFO:teuthology.orchestra.run.smithi093.stdout: Birth: - 2023-08-27T20:48:04.285 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-08-27T20:48:04.356 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records in 2023-08-27T20:48:04.357 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records out 2023-08-27T20:48:04.357 INFO:teuthology.orchestra.run.smithi093.stderr:512 bytes copied, 0.000231972 s, 2.2 MB/s 2023-08-27T20:48:04.358 DEBUG:teuthology.orchestra.run.smithi093:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-08-27T20:48:04.421 DEBUG:teuthology.orchestra.run.smithi093:> stat /dev/vg_nvme/lv_3 2023-08-27T20:48:04.482 INFO:teuthology.orchestra.run.smithi093.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-08-27T20:48:04.482 INFO:teuthology.orchestra.run.smithi093.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:04.482 INFO:teuthology.orchestra.run.smithi093.stdout:Device: 6h/6d Inode: 391711 Links: 1 2023-08-27T20:48:04.482 INFO:teuthology.orchestra.run.smithi093.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:04.483 INFO:teuthology.orchestra.run.smithi093.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:04.483 INFO:teuthology.orchestra.run.smithi093.stdout:Access: 2023-08-27 20:47:53.391947890 +0000 2023-08-27T20:48:04.483 INFO:teuthology.orchestra.run.smithi093.stdout:Modify: 2023-08-27 20:41:32.583503873 +0000 2023-08-27T20:48:04.483 INFO:teuthology.orchestra.run.smithi093.stdout:Change: 2023-08-27 20:41:32.583503873 +0000 2023-08-27T20:48:04.483 INFO:teuthology.orchestra.run.smithi093.stdout: Birth: - 2023-08-27T20:48:04.483 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-08-27T20:48:04.553 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records in 2023-08-27T20:48:04.553 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records out 2023-08-27T20:48:04.553 INFO:teuthology.orchestra.run.smithi093.stderr:512 bytes copied, 0.000235608 s, 2.2 MB/s 2023-08-27T20:48:04.554 DEBUG:teuthology.orchestra.run.smithi093:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: Reconfiguring mon.b (monmap changed)... 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: Reconfiguring daemon mon.b on smithi136 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi093=y;smithi136=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: Saving service mgr spec with placement smithi093=y;smithi136=x;count:2 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:04.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:04.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:04.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:48:04.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-27T20:48:04.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:48:04.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:04.616 DEBUG:teuthology.orchestra.run.smithi093:> stat /dev/vg_nvme/lv_4 2023-08-27T20:48:04.677 INFO:teuthology.orchestra.run.smithi093.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-08-27T20:48:04.677 INFO:teuthology.orchestra.run.smithi093.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:04.678 INFO:teuthology.orchestra.run.smithi093.stdout:Device: 6h/6d Inode: 393563 Links: 1 2023-08-27T20:48:04.678 INFO:teuthology.orchestra.run.smithi093.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:04.678 INFO:teuthology.orchestra.run.smithi093.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:04.678 INFO:teuthology.orchestra.run.smithi093.stdout:Access: 2023-08-27 20:47:53.391947890 +0000 2023-08-27T20:48:04.678 INFO:teuthology.orchestra.run.smithi093.stdout:Modify: 2023-08-27 20:41:33.096492885 +0000 2023-08-27T20:48:04.678 INFO:teuthology.orchestra.run.smithi093.stdout:Change: 2023-08-27 20:41:33.096492885 +0000 2023-08-27T20:48:04.678 INFO:teuthology.orchestra.run.smithi093.stdout: Birth: - 2023-08-27T20:48:04.678 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-08-27T20:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: Reconfiguring mon.b (monmap changed)... 2023-08-27T20:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: Reconfiguring daemon mon.b on smithi136 2023-08-27T20:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi093=y;smithi136=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: Saving service mgr spec with placement smithi093=y;smithi136=x;count:2 2023-08-27T20:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: Reconfiguring mon.b (monmap changed)... 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: Reconfiguring daemon mon.b on smithi136 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: pgmap v15: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='client.24100 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "mgr", "placement": "2;smithi093=y;smithi136=x", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: Saving service mgr spec with placement smithi093=y;smithi136=x;count:2 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:48:04.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]': finished 2023-08-27T20:48:04.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:48:04.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:04.747 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records in 2023-08-27T20:48:04.747 INFO:teuthology.orchestra.run.smithi093.stderr:1+0 records out 2023-08-27T20:48:04.747 INFO:teuthology.orchestra.run.smithi093.stderr:512 bytes copied, 0.000227813 s, 2.2 MB/s 2023-08-27T20:48:04.748 DEBUG:teuthology.orchestra.run.smithi093:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-08-27T20:48:04.810 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:48:04.810 DEBUG:teuthology.orchestra.run.smithi136:> dd if=/scratch_devs of=/dev/stdout 2023-08-27T20:48:04.830 DEBUG:teuthology.misc:devs=['/dev/vg_nvme/lv_1', '/dev/vg_nvme/lv_2', '/dev/vg_nvme/lv_3', '/dev/vg_nvme/lv_4'] 2023-08-27T20:48:04.831 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_1 2023-08-27T20:48:04.890 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_1 -> ../dm-0 2023-08-27T20:48:04.890 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:04.890 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 6h/6d Inode: 386813 Links: 1 2023-08-27T20:48:04.890 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:04.890 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:04.890 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2023-08-27 20:48:04.874996653 +0000 2023-08-27T20:48:04.891 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2023-08-27 20:41:31.539434913 +0000 2023-08-27T20:48:04.891 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2023-08-27 20:41:31.539434913 +0000 2023-08-27T20:48:04.891 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2023-08-27T20:48:04.891 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_1 of=/dev/null count=1 2023-08-27T20:48:04.960 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2023-08-27T20:48:04.961 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2023-08-27T20:48:04.961 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000232653 s, 2.2 MB/s 2023-08-27T20:48:04.962 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_1 2023-08-27T20:48:05.021 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_2 2023-08-27T20:48:05.079 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_2 -> ../dm-1 2023-08-27T20:48:05.080 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:05.080 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 6h/6d Inode: 386916 Links: 1 2023-08-27T20:48:05.080 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:05.081 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:05.081 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2023-08-27 20:48:04.874996653 +0000 2023-08-27T20:48:05.081 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2023-08-27 20:41:32.053423723 +0000 2023-08-27T20:48:05.081 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2023-08-27 20:41:32.053423723 +0000 2023-08-27T20:48:05.081 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2023-08-27T20:48:05.081 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_2 of=/dev/null count=1 2023-08-27T20:48:05.146 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2023-08-27T20:48:05.146 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2023-08-27T20:48:05.146 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000204247 s, 2.5 MB/s 2023-08-27T20:48:05.147 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_2 2023-08-27T20:48:05.176 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:04 smithi136 systemd[1]: Starting Ceph mgr.x for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:48:05.194 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_3 2023-08-27T20:48:05.254 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_3 -> ../dm-2 2023-08-27T20:48:05.254 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:05.255 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 6h/6d Inode: 386981 Links: 1 2023-08-27T20:48:05.255 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:05.255 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:05.255 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2023-08-27 20:48:04.874996653 +0000 2023-08-27T20:48:05.255 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2023-08-27 20:41:32.580412248 +0000 2023-08-27T20:48:05.255 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2023-08-27 20:41:32.580412248 +0000 2023-08-27T20:48:05.255 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2023-08-27T20:48:05.255 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_3 of=/dev/null count=1 2023-08-27T20:48:05.322 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2023-08-27T20:48:05.323 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2023-08-27T20:48:05.323 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.00024325 s, 2.1 MB/s 2023-08-27T20:48:05.324 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_3 2023-08-27T20:48:05.384 DEBUG:teuthology.orchestra.run.smithi136:> stat /dev/vg_nvme/lv_4 2023-08-27T20:48:05.443 INFO:teuthology.orchestra.run.smithi136.stdout: File: /dev/vg_nvme/lv_4 -> ../dm-3 2023-08-27T20:48:05.443 INFO:teuthology.orchestra.run.smithi136.stdout: Size: 7 Blocks: 0 IO Block: 4096 symbolic link 2023-08-27T20:48:05.444 INFO:teuthology.orchestra.run.smithi136.stdout:Device: 6h/6d Inode: 391764 Links: 1 2023-08-27T20:48:05.444 INFO:teuthology.orchestra.run.smithi136.stdout:Access: (0777/lrwxrwxrwx) Uid: ( 0/ root) Gid: ( 0/ root) 2023-08-27T20:48:05.444 INFO:teuthology.orchestra.run.smithi136.stdout:Context: system_u:object_r:device_t:s0 2023-08-27T20:48:05.444 INFO:teuthology.orchestra.run.smithi136.stdout:Access: 2023-08-27 20:48:04.874996653 +0000 2023-08-27T20:48:05.444 INFO:teuthology.orchestra.run.smithi136.stdout:Modify: 2023-08-27 20:41:33.100400926 +0000 2023-08-27T20:48:05.444 INFO:teuthology.orchestra.run.smithi136.stdout:Change: 2023-08-27 20:41:33.100400926 +0000 2023-08-27T20:48:05.444 INFO:teuthology.orchestra.run.smithi136.stdout: Birth: - 2023-08-27T20:48:05.444 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd if=/dev/vg_nvme/lv_4 of=/dev/null count=1 2023-08-27T20:48:05.521 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:05 smithi136 bash[115381]: 03135b918d8047cc94f6b64db01ea3a1cbf361f476678f0fd6d81a6f1aa30156 2023-08-27T20:48:05.521 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:05 smithi136 systemd[1]: Started Ceph mgr.x for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:48:05.521 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-mgr[115473]: -- 172.21.15.136:0/2162116043 <== mon.1 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55c71d4849c0 con 0x55c71d45b000 2023-08-27T20:48:05.521 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:05.378+0000 7f74e26c91c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:48:05.526 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records in 2023-08-27T20:48:05.526 INFO:teuthology.orchestra.run.smithi136.stderr:1+0 records out 2023-08-27T20:48:05.526 INFO:teuthology.orchestra.run.smithi136.stderr:512 bytes copied, 0.000329607 s, 1.6 MB/s 2023-08-27T20:48:05.526 DEBUG:teuthology.orchestra.run.smithi136:> ! mount | grep -v devtmpfs | grep -q /dev/vg_nvme/lv_4 2023-08-27T20:48:05.526 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-mon[113531]: Deploying daemon mgr.x on smithi136 2023-08-27T20:48:05.527 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.527 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.527 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.527 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.586 INFO:tasks.cephadm:Deploying osd.0 on smithi093 with /dev/vg_nvme/lv_4... 2023-08-27T20:48:05.586 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-08-27T20:48:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[111335]: Deploying daemon mgr.x on smithi136 2023-08-27T20:48:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[118902]: Deploying daemon mgr.x on smithi136 2023-08-27T20:48:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:05 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:05.856 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:05.901 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:05.550+0000 7f74e26c91c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:48:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:05.900+0000 7f74e26c91c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:48:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:06.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:06.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:06.465+0000 7f74e26c91c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:48:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:06.634+0000 7f74e26c91c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: pgmap v16: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:07.083 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-08-27T20:48:07.083 INFO:teuthology.orchestra.run.smithi093.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-08-27T20:48:07.083 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 10+0 records in 2023-08-27T20:48:07.083 INFO:teuthology.orchestra.run.smithi093.stderr:10+0 records out 2023-08-27T20:48:07.083 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0111254 s, 943 MB/s 2023-08-27T20:48:07.083 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping successful for: 2023-08-27T20:48:07.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:07.024+0000 7f74e26c91c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:48:07.598 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi093:vg_nvme/lv_4 2023-08-27T20:48:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[111335]: Reconfiguring mgr.y (unknown last config time)... 2023-08-27T20:48:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:48:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:48:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[111335]: Reconfiguring daemon mgr.y on smithi093 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[118902]: Reconfiguring mgr.y (unknown last config time)... 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[118902]: Reconfiguring daemon mgr.y on smithi093 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:07 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:07.790 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-mon[113531]: Reconfiguring mgr.y (unknown last config time)... 2023-08-27T20:48:07.790 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.y", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:48:07.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:48:07.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:07.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-mon[113531]: Reconfiguring daemon mgr.y on smithi093 2023-08-27T20:48:07.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:07.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:07.856 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:07.790+0000 7f74e26c91c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:48:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:07.873+0000 7f74e26c91c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:48:08.587 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[118902]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:08.587 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:08.587 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:08.587 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:08.587 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:08.587 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[111335]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:08.588 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:08.588 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:08.588 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:08.588 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:08 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:08.726 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:08.468+0000 7f74e26c91c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:48:08.726 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:08.550+0000 7f74e26c91c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:48:08.726 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:08.634+0000 7f74e26c91c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:48:08.726 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-mon[113531]: pgmap v17: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:08.726 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:08.726 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:08.727 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:08.727 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:09.042 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:08.725+0000 7f74e26c91c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:48:09.042 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:08.951+0000 7f74e26c91c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:48:09.340 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:09.041+0000 7f74e26c91c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:48:09.340 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:09.247+0000 7f74e26c91c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:48:09.601 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:09.340+0000 7f74e26c91c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:48:09.601 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:09.519+0000 7f74e26c91c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:48:09.602 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:09.602 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:09.602 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:09 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:09 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:09 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:09 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:09 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:09 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:09.601+0000 7f74e26c91c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:48:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:10.336+0000 7f74e26c91c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:48:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:10.519+0000 7f74e26c91c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:48:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:10 smithi136 ceph-mon[113531]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:10 smithi136 ceph-mon[113531]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:10 smithi093 ceph-mon[111335]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:10 smithi093 ceph-mon[111335]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:10 smithi093 ceph-mon[118902]: from='client.14220 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:10 smithi093 ceph-mon[118902]: pgmap v18: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:10.690+0000 7f74e26c91c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:48:11.634 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:11.285+0000 7f74e26c91c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:48:11.634 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:11.458+0000 7f74e26c91c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:48:11.635 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/592772041' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8288f9e-5794-4532-9495-65379f66b902"}]: dispatch 2023-08-27T20:48:11.635 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/592772041' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8288f9e-5794-4532-9495-65379f66b902"}]': finished 2023-08-27T20:48:11.635 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-mon[113531]: osdmap e5: 1 total, 0 up, 1 in 2023-08-27T20:48:11.635 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:11.635 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1472385682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/592772041' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8288f9e-5794-4532-9495-65379f66b902"}]: dispatch 2023-08-27T20:48:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/592772041' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8288f9e-5794-4532-9495-65379f66b902"}]': finished 2023-08-27T20:48:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[111335]: osdmap e5: 1 total, 0 up, 1 in 2023-08-27T20:48:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1472385682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/592772041' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "f8288f9e-5794-4532-9495-65379f66b902"}]: dispatch 2023-08-27T20:48:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/592772041' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "f8288f9e-5794-4532-9495-65379f66b902"}]': finished 2023-08-27T20:48:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[118902]: osdmap e5: 1 total, 0 up, 1 in 2023-08-27T20:48:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1472385682' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:11.634+0000 7f74e26c91c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:48:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:48:11.747+0000 7f74e26c91c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:48:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[111335]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[111335]: Standby manager daemon x started 2023-08-27T20:48:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:48:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:48:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:48:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:48:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[118902]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[118902]: Standby manager daemon x started 2023-08-27T20:48:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:48:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:48:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:48:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:48:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:12 smithi136 ceph-mon[113531]: pgmap v20: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:12 smithi136 ceph-mon[113531]: Standby manager daemon x started 2023-08-27T20:48:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:48:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:48:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:48:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/2206001184' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:48:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:13 smithi093 ceph-mon[111335]: mgrmap e14: y(active, since 51s), standbys: x 2023-08-27T20:48:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:48:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:13 smithi093 ceph-mon[118902]: mgrmap e14: y(active, since 51s), standbys: x 2023-08-27T20:48:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:48:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:13 smithi136 ceph-mon[113531]: mgrmap e14: y(active, since 51s), standbys: x 2023-08-27T20:48:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:48:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:14 smithi093 ceph-mon[111335]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:14 smithi093 ceph-mon[118902]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:14 smithi136 ceph-mon[113531]: pgmap v21: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:16 smithi093 ceph-mon[111335]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:16 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-27T20:48:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:16 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:16 smithi093 ceph-mon[118902]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:16 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-27T20:48:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:16 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:16 smithi136 ceph-mon[113531]: pgmap v22: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:16 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.0"}]: dispatch 2023-08-27T20:48:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:16 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:17.720 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:17 smithi093 ceph-mon[111335]: Deploying daemon osd.0 on smithi093 2023-08-27T20:48:17.720 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:17 smithi093 ceph-mon[118902]: Deploying daemon osd.0 on smithi093 2023-08-27T20:48:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:17 smithi136 ceph-mon[113531]: Deploying daemon osd.0 on smithi093 2023-08-27T20:48:18.813 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:18 smithi093 ceph-mon[111335]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:18.813 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:18 smithi093 ceph-mon[118902]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:18 smithi136 ceph-mon[113531]: pgmap v23: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:20.210 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[111335]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:20.210 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[118902]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:20 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:20 smithi136 ceph-mon[113531]: pgmap v24: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:20 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:20 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:20 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:21.180 INFO:teuthology.orchestra.run.smithi093.stdout:Created osd(s) 0 on host 'smithi093' 2023-08-27T20:48:21.611 DEBUG:teuthology.orchestra.run.smithi093:osd.0> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.0.service 2023-08-27T20:48:21.613 INFO:tasks.cephadm:Deploying osd.1 on smithi093 with /dev/vg_nvme/lv_3... 2023-08-27T20:48:21.613 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-08-27T20:48:21.644 INFO:journalctl@ceph.osd.0.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:48:21.801 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.801 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.801 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:21.801 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:21.802 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.803 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:21.803 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.803 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.803 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: pgmap v25: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:22.111 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:22.706 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:22 smithi093 ceph-mon[111335]: from='osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T20:48:22.707 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:22 smithi093 ceph-mon[111335]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T20:48:22.707 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:22 smithi093 ceph-mon[118902]: from='osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T20:48:22.707 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:22 smithi093 ceph-mon[118902]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T20:48:22.707 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-0[123164]: 2023-08-27T20:48:22.415+0000 7fc82576d540 -1 osd.0 0 log_to_monitors true 2023-08-27T20:48:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:22 smithi136 ceph-mon[113531]: from='osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600]' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T20:48:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:22 smithi136 ceph-mon[113531]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]: dispatch 2023-08-27T20:48:23.427 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-08-27T20:48:23.427 INFO:teuthology.orchestra.run.smithi093.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-08-27T20:48:23.427 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 10+0 records in 2023-08-27T20:48:23.427 INFO:teuthology.orchestra.run.smithi093.stderr:10+0 records out 2023-08-27T20:48:23.427 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0116284 s, 902 MB/s 2023-08-27T20:48:23.427 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping successful for: 2023-08-27T20:48:23.886 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[111335]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-27T20:48:23.886 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[111335]: osdmap e6: 1 total, 0 up, 1 in 2023-08-27T20:48:23.886 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:23.886 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[111335]: from='osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:23.887 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[111335]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:23.887 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[111335]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:23.887 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[118902]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-27T20:48:23.887 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[118902]: osdmap e6: 1 total, 0 up, 1 in 2023-08-27T20:48:23.887 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:23.887 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[118902]: from='osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:23.888 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[118902]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:23.888 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-mon[118902]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:23.888 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-0[123164]: 2023-08-27T20:48:23.613+0000 7fc81bf52700 -1 osd.0 0 waiting for initial osdmap 2023-08-27T20:48:23.888 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:48:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-0[123164]: 2023-08-27T20:48:23.615+0000 7fc81654c700 -1 osd.0 7 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:48:23.925 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi093:vg_nvme/lv_3 2023-08-27T20:48:23.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:23 smithi136 ceph-mon[113531]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["0"]}]': finished 2023-08-27T20:48:23.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:23 smithi136 ceph-mon[113531]: osdmap e6: 1 total, 0 up, 1 in 2023-08-27T20:48:23.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:23 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:23.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:23 smithi136 ceph-mon[113531]: from='osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600]' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:23.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:23 smithi136 ceph-mon[113531]: from='osd.0 ' entity='osd.0' cmd=[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:23.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:23 smithi136 ceph-mon[113531]: pgmap v27: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:24.239 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: osdmap e7: 1 total, 0 up, 1 in 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi093 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 16089M 2023-08-27T20:48:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:24 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.981 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: osdmap e7: 1 total, 0 up, 1 in 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi093 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 16089M 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='osd.0 ' entity='osd.0' cmd='[{"prefix": "osd crush create-or-move", "id": 0, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: osdmap e7: 1 total, 0 up, 1 in 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi093 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:24.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 16089M 2023-08-27T20:48:24.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:24.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:24.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:24.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:24 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:48:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:48:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600] boot 2023-08-27T20:48:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: osdmap e8: 1 total, 1 up, 1 in 2023-08-27T20:48:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600] boot 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: osdmap e8: 1 total, 1 up, 1 in 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: osd.0 [v2:172.21.15.93:6802/468012600,v1:172.21.15.93:6803/468012600] boot 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: osdmap e8: 1 total, 1 up, 1 in 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: pgmap v30: 0 pgs: ; 0 B data, 0 B used, 0 B / 0 B avail 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: from='client.14247 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:25.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:25.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:26 smithi136 ceph-mon[113531]: osdmap e9: 1 total, 1 up, 1 in 2023-08-27T20:48:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:26 smithi093 ceph-mon[111335]: osdmap e9: 1 total, 1 up, 1 in 2023-08-27T20:48:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:26 smithi093 ceph-mon[118902]: osdmap e9: 1 total, 1 up, 1 in 2023-08-27T20:48:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:27 smithi136 ceph-mon[113531]: pgmap v32: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2812405329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "703653bb-1bc5-418f-bf13-4e71cb82a4da"}]: dispatch 2023-08-27T20:48:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2812405329' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "703653bb-1bc5-418f-bf13-4e71cb82a4da"}]': finished 2023-08-27T20:48:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:27 smithi136 ceph-mon[113531]: osdmap e10: 2 total, 1 up, 2 in 2023-08-27T20:48:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:27 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[111335]: pgmap v32: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2812405329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "703653bb-1bc5-418f-bf13-4e71cb82a4da"}]: dispatch 2023-08-27T20:48:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2812405329' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "703653bb-1bc5-418f-bf13-4e71cb82a4da"}]': finished 2023-08-27T20:48:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[111335]: osdmap e10: 2 total, 1 up, 2 in 2023-08-27T20:48:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[118902]: pgmap v32: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2812405329' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "703653bb-1bc5-418f-bf13-4e71cb82a4da"}]: dispatch 2023-08-27T20:48:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2812405329' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "703653bb-1bc5-418f-bf13-4e71cb82a4da"}]': finished 2023-08-27T20:48:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[118902]: osdmap e10: 2 total, 1 up, 2 in 2023-08-27T20:48:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:27 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1500520115' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1500520115' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1500520115' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:29 smithi136 ceph-mon[113531]: pgmap v34: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:29.976 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:29 smithi093 ceph-mon[111335]: pgmap v34: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:29.976 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:29 smithi093 ceph-mon[118902]: pgmap v34: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:32 smithi136 ceph-mon[113531]: pgmap v35: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:32 smithi093 ceph-mon[111335]: pgmap v35: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:32 smithi093 ceph-mon[118902]: pgmap v35: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:33.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:33 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-27T20:48:33.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:33 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:33.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:33 smithi093 ceph-mon[118902]: Deploying daemon osd.1 on smithi093 2023-08-27T20:48:33.176 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:33 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-27T20:48:33.176 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:33 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:33.176 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:33 smithi093 ceph-mon[111335]: Deploying daemon osd.1 on smithi093 2023-08-27T20:48:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:33 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.1"}]: dispatch 2023-08-27T20:48:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:33 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:33 smithi136 ceph-mon[113531]: Deploying daemon osd.1 on smithi093 2023-08-27T20:48:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:34 smithi136 ceph-mon[113531]: pgmap v36: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:34 smithi093 ceph-mon[111335]: pgmap v36: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:34 smithi093 ceph-mon[118902]: pgmap v36: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[111335]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[118902]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:36 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:36 smithi136 ceph-mon[113531]: pgmap v37: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:36 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:36 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:36 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:37.634 INFO:teuthology.orchestra.run.smithi093.stdout:Created osd(s) 1 on host 'smithi093' 2023-08-27T20:48:38.014 DEBUG:teuthology.orchestra.run.smithi093:osd.1> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.1.service 2023-08-27T20:48:38.016 INFO:tasks.cephadm:Deploying osd.2 on smithi093 with /dev/vg_nvme/lv_2... 2023-08-27T20:48:38.016 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-08-27T20:48:38.050 INFO:journalctl@ceph.osd.1.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: pgmap v38: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:38.568 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:38.846 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:48:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-1[127261]: 2023-08-27T20:48:38.668+0000 7f89f45f4540 -1 osd.1 0 log_to_monitors true 2023-08-27T20:48:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:39 smithi136 ceph-mon[113531]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-27T20:48:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:39 smithi093 ceph-mon[111335]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-27T20:48:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:39 smithi093 ceph-mon[118902]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]: dispatch 2023-08-27T20:48:39.995 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-08-27T20:48:39.995 INFO:teuthology.orchestra.run.smithi093.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-08-27T20:48:39.995 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 10+0 records in 2023-08-27T20:48:39.995 INFO:teuthology.orchestra.run.smithi093.stderr:10+0 records out 2023-08-27T20:48:39.996 INFO:teuthology.orchestra.run.smithi093.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0114152 s, 919 MB/s 2023-08-27T20:48:39.996 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 2023-08-27T20:48:39.996 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping successful for: 2023-08-27T20:48:40.687 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi093:vg_nvme/lv_2 2023-08-27T20:48:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: osdmap e11: 2 total, 1 up, 2 in 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:40 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.714 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-1[127261]: 2023-08-27T20:48:40.439+0000 7f89eadd9700 -1 osd.1 0 waiting for initial osdmap 2023-08-27T20:48:40.714 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-1[127261]: 2023-08-27T20:48:40.443+0000 7f89e4bd2700 -1 osd.1 12 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:48:40.714 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: osdmap e11: 2 total, 1 up, 2 in 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:40.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: pgmap v39: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["1"]}]': finished 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: osdmap e11: 2 total, 1 up, 2 in 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd=[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:40.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:40.717 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.717 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:40.717 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:40.717 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:40 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:40.957 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:41.705 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi093 2023-08-27T20:48:41.706 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 8044M 2023-08-27T20:48:41.706 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[111335]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:41.706 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[111335]: osdmap e12: 2 total, 1 up, 2 in 2023-08-27T20:48:41.706 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:41.706 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:41.706 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi093 2023-08-27T20:48:41.706 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 8044M 2023-08-27T20:48:41.707 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[118902]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:41.707 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[118902]: osdmap e12: 2 total, 1 up, 2 in 2023-08-27T20:48:41.707 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:41.707 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:41 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:41 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi093 2023-08-27T20:48:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:41 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 8044M 2023-08-27T20:48:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:41 smithi136 ceph-mon[113531]: from='osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391]' entity='osd.1' cmd='[{"prefix": "osd crush create-or-move", "id": 1, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:41 smithi136 ceph-mon[113531]: osdmap e12: 2 total, 1 up, 2 in 2023-08-27T20:48:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:41 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:41 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391] boot 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: osdmap e13: 2 total, 2 up, 2 in 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391] boot 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: osdmap e13: 2 total, 2 up, 2 in 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:42 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:48:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: pgmap v42: 0 pgs: ; 0 B data, 4.9 MiB used, 89 GiB / 89 GiB avail 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: osd.1 [v2:172.21.15.93:6810/1906627391,v1:172.21.15.93:6811/1906627391] boot 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: osdmap e13: 2 total, 2 up, 2 in 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:42 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:43 smithi093 ceph-mon[111335]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:43 smithi093 ceph-mon[111335]: osdmap e14: 2 total, 2 up, 2 in 2023-08-27T20:48:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:43 smithi093 ceph-mon[118902]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:43 smithi093 ceph-mon[118902]: osdmap e14: 2 total, 2 up, 2 in 2023-08-27T20:48:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:43 smithi136 ceph-mon[113531]: from='client.14274 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:48:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:43 smithi136 ceph-mon[113531]: osdmap e14: 2 total, 2 up, 2 in 2023-08-27T20:48:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[111335]: pgmap v45: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2948683981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca3d4d17-b4ba-402a-999d-496be3cda08d"}]: dispatch 2023-08-27T20:48:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2948683981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca3d4d17-b4ba-402a-999d-496be3cda08d"}]': finished 2023-08-27T20:48:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[111335]: osdmap e15: 3 total, 2 up, 3 in 2023-08-27T20:48:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/476852005' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[118902]: pgmap v45: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2948683981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca3d4d17-b4ba-402a-999d-496be3cda08d"}]: dispatch 2023-08-27T20:48:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2948683981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca3d4d17-b4ba-402a-999d-496be3cda08d"}]': finished 2023-08-27T20:48:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[118902]: osdmap e15: 3 total, 2 up, 3 in 2023-08-27T20:48:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/476852005' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:44 smithi136 ceph-mon[113531]: pgmap v45: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2948683981' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "ca3d4d17-b4ba-402a-999d-496be3cda08d"}]: dispatch 2023-08-27T20:48:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2948683981' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "ca3d4d17-b4ba-402a-999d-496be3cda08d"}]': finished 2023-08-27T20:48:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:44 smithi136 ceph-mon[113531]: osdmap e15: 3 total, 2 up, 3 in 2023-08-27T20:48:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:44 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/476852005' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:48:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:46 smithi093 ceph-mon[111335]: pgmap v47: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:46 smithi093 ceph-mon[118902]: pgmap v47: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:46 smithi136 ceph-mon[113531]: pgmap v47: 0 pgs: ; 0 B data, 9.8 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:48.658 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:48 smithi093 ceph-mon[111335]: pgmap v48: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:48.658 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:48 smithi093 ceph-mon[118902]: pgmap v48: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:48.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:48 smithi136 ceph-mon[113531]: pgmap v48: 0 pgs: ; 0 B data, 9.9 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:49 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-27T20:48:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:49 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:49 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-27T20:48:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:49 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:49 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.2"}]: dispatch 2023-08-27T20:48:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:49 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:50 smithi093 ceph-mon[111335]: Deploying daemon osd.2 on smithi093 2023-08-27T20:48:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:50 smithi093 ceph-mon[111335]: pgmap v49: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:50 smithi093 ceph-mon[118902]: Deploying daemon osd.2 on smithi093 2023-08-27T20:48:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:50 smithi093 ceph-mon[118902]: pgmap v49: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:50 smithi136 ceph-mon[113531]: Deploying daemon osd.2 on smithi093 2023-08-27T20:48:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:50 smithi136 ceph-mon[113531]: pgmap v49: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:52.727 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[111335]: pgmap v50: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:52.727 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:52.727 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:52.727 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:52.727 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[118902]: pgmap v50: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:52.727 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:52.727 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:52.728 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:52 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:52 smithi136 ceph-mon[113531]: pgmap v50: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:52 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:52 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:52 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:54.376 INFO:teuthology.orchestra.run.smithi093.stdout:Created osd(s) 2 on host 'smithi093' 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: pgmap v51: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: pgmap v51: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:54.810 DEBUG:teuthology.orchestra.run.smithi093:osd.2> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.2.service 2023-08-27T20:48:54.853 INFO:tasks.cephadm:Deploying osd.3 on smithi093 with /dev/vg_nvme/lv_1... 2023-08-27T20:48:54.853 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-08-27T20:48:54.885 INFO:journalctl@ceph.osd.2.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:48:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: pgmap v51: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:48:55.355 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:55.628 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:55 smithi093 ceph-mon[111335]: from='osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T20:48:55.628 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:55 smithi093 ceph-mon[111335]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T20:48:55.628 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:55 smithi093 ceph-mon[118902]: from='osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T20:48:55.628 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:55 smithi093 ceph-mon[118902]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T20:48:55.629 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:48:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-2[131523]: 2023-08-27T20:48:55.429+0000 7fb010228540 -1 osd.2 0 log_to_monitors true 2023-08-27T20:48:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:55 smithi136 ceph-mon[113531]: from='osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168]' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T20:48:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:55 smithi136 ceph-mon[113531]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]: dispatch 2023-08-27T20:48:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[118902]: pgmap v52: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[118902]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-27T20:48:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[118902]: osdmap e16: 3 total, 2 up, 3 in 2023-08-27T20:48:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[118902]: from='osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[118902]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:56.739 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-2[131523]: 2023-08-27T20:48:56.488+0000 7fb008210700 -1 osd.2 0 waiting for initial osdmap 2023-08-27T20:48:56.739 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-2[131523]: 2023-08-27T20:48:56.492+0000 7fb000005700 -1 osd.2 17 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:48:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[111335]: pgmap v52: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[111335]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-27T20:48:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[111335]: osdmap e16: 3 total, 2 up, 3 in 2023-08-27T20:48:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[111335]: from='osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:56 smithi093 ceph-mon[111335]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:56.907 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-08-27T20:48:56.907 INFO:teuthology.orchestra.run.smithi093.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-08-27T20:48:56.908 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 10+0 records in 2023-08-27T20:48:56.908 INFO:teuthology.orchestra.run.smithi093.stderr:10+0 records out 2023-08-27T20:48:56.908 INFO:teuthology.orchestra.run.smithi093.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.0134605 s, 779 MB/s 2023-08-27T20:48:56.908 INFO:teuthology.orchestra.run.smithi093.stderr:--> Zapping successful for: 2023-08-27T20:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:56 smithi136 ceph-mon[113531]: pgmap v52: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:56 smithi136 ceph-mon[113531]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["2"]}]': finished 2023-08-27T20:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:56 smithi136 ceph-mon[113531]: osdmap e16: 3 total, 2 up, 3 in 2023-08-27T20:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:56 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:56 smithi136 ceph-mon[113531]: from='osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168]' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:56 smithi136 ceph-mon[113531]: from='osd.2 ' entity='osd.2' cmd=[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:48:57.267 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi093:vg_nvme/lv_1 2023-08-27T20:48:57.533 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:48:57.799 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:57.799 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: osdmap e17: 3 total, 2 up, 3 in 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:57.800 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: osdmap e17: 3 total, 2 up, 3 in 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.801 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:57.802 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:57.802 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='osd.2 ' entity='osd.2' cmd='[{"prefix": "osd crush create-or-move", "id": 2, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: osdmap e17: 3 total, 2 up, 3 in 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:48:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi093 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 5363M 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: pgmap v55: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168] boot 2023-08-27T20:48:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: osdmap e18: 3 total, 3 up, 3 in 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi093 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 5363M 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: pgmap v55: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168] boot 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: osdmap e18: 3 total, 3 up, 3 in 2023-08-27T20:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:58 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi093 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 5363M 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: pgmap v55: 0 pgs: ; 0 B data, 10 MiB used, 179 GiB / 179 GiB avail 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: osd.2 [v2:172.21.15.93:6818/3915054168,v1:172.21.15.93:6819/3915054168] boot 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: osdmap e18: 3 total, 3 up, 3 in 2023-08-27T20:48:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:58 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:48:59.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:59.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:59.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:59.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T20:48:59.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-27T20:48:59.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[111335]: osdmap e19: 3 total, 3 up, 3 in 2023-08-27T20:48:59.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:48:59.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:59.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:59.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:59.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T20:48:59.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-27T20:48:59.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[118902]: osdmap e19: 3 total, 3 up, 3 in 2023-08-27T20:48:59.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:48:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:48:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:48:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:48:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:48:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]: dispatch 2023-08-27T20:48:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "osd pool create", "format": "json", "pool": ".mgr", "pg_num": 1, "pg_num_min": 1, "pg_num_max": 32}]': finished 2023-08-27T20:48:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:59 smithi136 ceph-mon[113531]: osdmap e19: 3 total, 3 up, 3 in 2023-08-27T20:48:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:48:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:49:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:00 smithi136 ceph-mon[113531]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:00 smithi136 ceph-mon[113531]: pgmap v57: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:00 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:49:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:00 smithi136 ceph-mon[113531]: osdmap e20: 3 total, 3 up, 3 in 2023-08-27T20:49:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[111335]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[111335]: pgmap v57: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:49:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[111335]: osdmap e20: 3 total, 3 up, 3 in 2023-08-27T20:49:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[118902]: from='client.24191 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi093:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[118902]: pgmap v57: 0 pgs: ; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "osd pool application enable", "format": "json", "pool": ".mgr", "app": "mgr", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:49:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:00 smithi093 ceph-mon[118902]: osdmap e20: 3 total, 3 up, 3 in 2023-08-27T20:49:01.931 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/170749039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]: dispatch 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]: dispatch 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]': finished 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[118902]: osdmap e21: 4 total, 3 up, 4 in 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/479659908' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/170749039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]: dispatch 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]: dispatch 2023-08-27T20:49:01.932 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]': finished 2023-08-27T20:49:01.933 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[111335]: osdmap e21: 4 total, 3 up, 4 in 2023-08-27T20:49:01.933 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:01.933 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/479659908' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:01.933 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134086]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-27T20:49:01.933 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134086]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:01.933 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134086]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:01.933 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134086]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/170749039' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]: dispatch 2023-08-27T20:49:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:01 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]: dispatch 2023-08-27T20:49:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:01 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "04f9b593-2f0e-4b0e-b5ba-2778d688e66d"}]': finished 2023-08-27T20:49:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:01 smithi136 ceph-mon[113531]: osdmap e21: 4 total, 3 up, 4 in 2023-08-27T20:49:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/479659908' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:02.192 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134275]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-27T20:49:02.192 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134275]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:02.192 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134275]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:02.192 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 27 20:49:01 smithi093 sudo[134275]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134463]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134463]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134463]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134463]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134468]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134468]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134468]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:02.489 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134468]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:02.936 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[111335]: pgmap v61: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:02.936 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[111335]: osdmap e22: 4 total, 3 up, 4 in 2023-08-27T20:49:02.936 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:02.936 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:02.937 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[118902]: pgmap v61: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:02.937 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[118902]: osdmap e22: 4 total, 3 up, 4 in 2023-08-27T20:49:02.937 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:02.937 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:02.937 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134658]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-27T20:49:02.937 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134658]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:02.937 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134658]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:02.938 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134658]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:02 smithi136 ceph-mon[113531]: pgmap v61: 1 pgs: 1 unknown; 0 B data, 15 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:02 smithi136 ceph-mon[113531]: osdmap e22: 4 total, 3 up, 4 in 2023-08-27T20:49:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:03.189 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134857]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-27T20:49:03.189 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134857]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:03.189 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134857]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:03.189 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134857]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:03.190 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134852]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-27T20:49:03.190 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134852]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:03.190 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134852]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:03.190 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 27 20:49:02 smithi093 sudo[134852]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134980]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-27T20:49:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134980]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134980]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 sudo[134980]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:03.931 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116363]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-27T20:49:03.931 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116363]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:03.931 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116363]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:03.931 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116363]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:03.931 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116366]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-27T20:49:03.932 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116366]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:03.932 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116366]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:03.932 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 sudo[116366]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: mgrmap e15: y(active, since 101s), standbys: x 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:04.194 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: mgrmap e15: y(active, since 101s), standbys: x 2023-08-27T20:49:04.194 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:04.194 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:04.194 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:04.194 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: mgrmap e15: y(active, since 101s), standbys: x 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:04.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:04.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:04.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:04.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:04.450 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135132]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-27T20:49:04.451 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135132]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:04.451 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135132]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:04.451 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135132]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135152]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-27T20:49:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135152]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-27T20:49:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135152]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-27T20:49:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 sudo[135152]: pam_unix(sudo:session): session closed for user root 2023-08-27T20:49:05.174 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: pgmap v63: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:05.174 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:05.174 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: pgmap v63: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:05.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:05.176 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:05.176 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:05.176 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:05.176 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:05.176 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:04 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: pgmap v63: 1 pgs: 1 active+clean; 577 KiB data, 16 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-27T20:49:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:49:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:49:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:49:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:04 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-27T20:49:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:05 smithi136 ceph-mon[113531]: pgmap v64: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:05 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-27T20:49:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:05 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:05 smithi093 ceph-mon[111335]: pgmap v64: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:05 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-27T20:49:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:05 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:05 smithi093 ceph-mon[118902]: pgmap v64: 1 pgs: 1 active+clean; 577 KiB data, 17 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:05 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.3"}]: dispatch 2023-08-27T20:49:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:05 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:06 smithi136 ceph-mon[113531]: Deploying daemon osd.3 on smithi093 2023-08-27T20:49:07.221 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:06 smithi093 ceph-mon[111335]: Deploying daemon osd.3 on smithi093 2023-08-27T20:49:07.221 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:06 smithi093 ceph-mon[118902]: Deploying daemon osd.3 on smithi093 2023-08-27T20:49:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:07 smithi093 ceph-mon[111335]: pgmap v65: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:07 smithi093 ceph-mon[118902]: pgmap v65: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:07 smithi136 ceph-mon[113531]: pgmap v65: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail 2023-08-27T20:49:10.154 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:10.155 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:10.155 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[111335]: pgmap v66: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-08-27T20:49:10.155 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:10.155 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:10.155 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:10.155 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[118902]: pgmap v66: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-08-27T20:49:10.155 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:10 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:10 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:10 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:10 smithi136 ceph-mon[113531]: pgmap v66: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 3.0 KiB/s rd, 131 KiB/s wr, 9 op/s 2023-08-27T20:49:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:10 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:11.165 INFO:teuthology.orchestra.run.smithi093.stdout:Created osd(s) 3 on host 'smithi093' 2023-08-27T20:49:11.675 DEBUG:teuthology.orchestra.run.smithi093:osd.3> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.3.service 2023-08-27T20:49:11.679 INFO:tasks.cephadm:Deploying osd.4 on smithi136 with /dev/vg_nvme/lv_4... 2023-08-27T20:49:11.680 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_4 2023-08-27T20:49:11.711 INFO:journalctl@ceph.osd.3.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:49:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-08-27T20:49:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-08-27T20:49:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:11 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: pgmap v67: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 2.2 KiB/s rd, 98 KiB/s wr, 6 op/s 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:11 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:11.944 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:49:12.300 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 27 20:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-3[135787]: 2023-08-27T20:49:12.224+0000 7f278c82c540 -1 osd.3 0 log_to_monitors true 2023-08-27T20:49:12.743 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:12 smithi136 ceph-mon[113531]: from='osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T20:49:12.744 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:12 smithi136 ceph-mon[113531]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T20:49:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:12 smithi093 ceph-mon[111335]: from='osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T20:49:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:12 smithi093 ceph-mon[111335]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T20:49:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:12 smithi093 ceph-mon[118902]: from='osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389]' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T20:49:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:12 smithi093 ceph-mon[118902]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]: dispatch 2023-08-27T20:49:13.166 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_4 2023-08-27T20:49:13.166 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_4 bs=1M count=10 conv=fsync 2023-08-27T20:49:13.167 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2023-08-27T20:49:13.167 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2023-08-27T20:49:13.167 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00812609 s, 1.3 GB/s 2023-08-27T20:49:13.167 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2023-08-27T20:49:13.570 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi136:vg_nvme/lv_4 2023-08-27T20:49:13.829 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:49:13.975 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: osdmap e23: 4 total, 3 up, 4 in 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi093 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 4022M 2023-08-27T20:49:13.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:13.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:13.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:13 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-27T20:49:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: osdmap e23: 4 total, 3 up, 4 in 2023-08-27T20:49:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi093 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 4022M 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["3"]}]': finished 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: osdmap e23: 4 total, 3 up, 4 in 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389]' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:49:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='osd.3 ' entity='osd.3' cmd=[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]: dispatch 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: pgmap v69: 1 pgs: 1 active+clean; 577 KiB data, 19 MiB used, 268 GiB / 268 GiB avail; 1.8 KiB/s rd, 78 KiB/s wr, 5 op/s 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi093 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 4022M 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:13.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:13.993 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:13.993 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-3[135787]: 2023-08-27T20:49:13.718+0000 7f2783011700 -1 osd.3 0 waiting for initial osdmap 2023-08-27T20:49:13.993 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 27 20:49:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-3[135787]: 2023-08-27T20:49:13.722+0000 7f277de0c700 -1 osd.3 24 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:49:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:14 smithi136 ceph-mon[113531]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:49:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:14 smithi136 ceph-mon[113531]: osdmap e24: 4 total, 3 up, 4 in 2023-08-27T20:49:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:14 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:14 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[111335]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:49:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[111335]: osdmap e24: 4 total, 3 up, 4 in 2023-08-27T20:49:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[118902]: from='osd.3 ' entity='osd.3' cmd='[{"prefix": "osd crush create-or-move", "id": 3, "weight":0.0873, "args": ["host=smithi093", "root=default"]}]': finished 2023-08-27T20:49:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[118902]: osdmap e24: 4 total, 3 up, 4 in 2023-08-27T20:49:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:14 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:15.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:49:15.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:49:15.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389] boot 2023-08-27T20:49:15.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: osdmap e25: 4 total, 4 up, 4 in 2023-08-27T20:49:15.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:15.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:15.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:15.893 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:15.893 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:15.893 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:15 smithi136 ceph-mon[113531]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:49:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:49:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389] boot 2023-08-27T20:49:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: osdmap e25: 4 total, 4 up, 4 in 2023-08-27T20:49:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[111335]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: osd.3 [v2:172.21.15.93:6826/4177593389,v1:172.21.15.93:6827/4177593389] boot 2023-08-27T20:49:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: osdmap e25: 4 total, 4 up, 4 in 2023-08-27T20:49:15.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:49:15.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: from='client.24199 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_4", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:15.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:15.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:15.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:15.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:15 smithi093 ceph-mon[118902]: pgmap v72: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[111335]: osdmap e26: 4 total, 4 up, 4 in 2023-08-27T20:49:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/1783638059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]: dispatch 2023-08-27T20:49:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]: dispatch 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]': finished 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[111335]: osdmap e27: 5 total, 4 up, 5 in 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[118902]: osdmap e26: 4 total, 4 up, 4 in 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/1783638059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]: dispatch 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]: dispatch 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]': finished 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[118902]: osdmap e27: 5 total, 4 up, 5 in 2023-08-27T20:49:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:16 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:16 smithi136 ceph-mon[113531]: osdmap e26: 4 total, 4 up, 4 in 2023-08-27T20:49:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/1783638059' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]: dispatch 2023-08-27T20:49:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:16 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]: dispatch 2023-08-27T20:49:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:16 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "2ca1eb6c-000f-42cc-a188-e0044386e9fd"}]': finished 2023-08-27T20:49:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:16 smithi136 ceph-mon[113531]: osdmap e27: 5 total, 4 up, 5 in 2023-08-27T20:49:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:16 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:17 smithi093 ceph-mon[111335]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/2711014615' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:17 smithi093 ceph-mon[118902]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/2711014615' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:17 smithi136 ceph-mon[113531]: pgmap v75: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/2711014615' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:20.705 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:20 smithi136 ceph-mon[113531]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:20 smithi093 ceph-mon[111335]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:20 smithi093 ceph-mon[118902]: pgmap v76: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-27T20:49:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:21 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-27T20:49:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:21 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:21.821 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.4"}]: dispatch 2023-08-27T20:49:21.821 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:21 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:22 smithi093 ceph-mon[111335]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:22 smithi093 ceph-mon[111335]: Deploying daemon osd.4 on smithi136 2023-08-27T20:49:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:22 smithi093 ceph-mon[118902]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:22 smithi093 ceph-mon[118902]: Deploying daemon osd.4 on smithi136 2023-08-27T20:49:22.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:22 smithi136 ceph-mon[113531]: pgmap v77: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:22.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:22 smithi136 ceph-mon[113531]: Deploying daemon osd.4 on smithi136 2023-08-27T20:49:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:24 smithi093 ceph-mon[111335]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:24 smithi093 ceph-mon[118902]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:24 smithi136 ceph-mon[113531]: pgmap v78: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:25.725 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:25.725 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:25.725 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[111335]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[118902]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:26 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.747 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:26 smithi136 ceph-mon[113531]: pgmap v79: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:26.747 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:26 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.747 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:26 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.747 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:26 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:26.747 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:26 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:26.747 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:26 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:26.873 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 4 on host 'smithi136' 2023-08-27T20:49:27.255 DEBUG:teuthology.orchestra.run.smithi136:osd.4> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.4.service 2023-08-27T20:49:27.257 INFO:tasks.cephadm:Deploying osd.5 on smithi136 with /dev/vg_nvme/lv_3... 2023-08-27T20:49:27.257 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_3 2023-08-27T20:49:27.288 INFO:journalctl@ceph.osd.4.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:49:27.752 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:49:27.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:27 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:27.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:27 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:27.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:27 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:27.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:27 smithi136 ceph-mon[113531]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:27.953 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 27 20:49:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-4[118838]: 2023-08-27T20:49:27.904+0000 7f1b166a4540 -1 osd.4 0 log_to_monitors true 2023-08-27T20:49:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[118902]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:27 smithi093 ceph-mon[111335]: pgmap v80: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:28 smithi136 ceph-mon[113531]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T20:49:29.174 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_3 2023-08-27T20:49:29.175 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_3 bs=1M count=10 conv=fsync 2023-08-27T20:49:29.175 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2023-08-27T20:49:29.175 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2023-08-27T20:49:29.175 INFO:teuthology.orchestra.run.smithi136.stderr:10485760 bytes (10 MB, 10 MiB) copied, 0.0083279 s, 1.3 GB/s 2023-08-27T20:49:29.175 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 2023-08-27T20:49:29.175 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2023-08-27T20:49:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:28 smithi093 ceph-mon[111335]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T20:49:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:28 smithi093 ceph-mon[118902]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]: dispatch 2023-08-27T20:49:29.501 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi136:vg_nvme/lv_3 2023-08-27T20:49:29.760 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:49:30.188 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-4[118838]: 2023-08-27T20:49:29.867+0000 7f1b0ce89700 -1 osd.4 0 waiting for initial osdmap 2023-08-27T20:49:30.189 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-4[118838]: 2023-08-27T20:49:29.875+0000 7f1b06c82700 -1 osd.4 29 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:49:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-27T20:49:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: osdmap e28: 5 total, 4 up, 5 in 2023-08-27T20:49:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi136 2023-08-27T20:49:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi136 to 17113M 2023-08-27T20:49:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:29 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-27T20:49:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: osdmap e28: 5 total, 4 up, 5 in 2023-08-27T20:49:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi136 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi136 to 17113M 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["4"]}]': finished 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: osdmap e28: 5 total, 4 up, 5 in 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd=[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi136 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: pgmap v82: 1 pgs: 1 active+clean; 577 KiB data, 24 MiB used, 358 GiB / 358 GiB avail 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi136 to 17113M 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:30.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:30.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:29 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:30 smithi136 ceph-mon[113531]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:49:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:30 smithi136 ceph-mon[113531]: osdmap e29: 5 total, 4 up, 5 in 2023-08-27T20:49:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:30 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:30 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:30 smithi136 ceph-mon[113531]: osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894] boot 2023-08-27T20:49:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:30 smithi136 ceph-mon[113531]: osdmap e30: 5 total, 5 up, 5 in 2023-08-27T20:49:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[111335]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:49:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[111335]: osdmap e29: 5 total, 4 up, 5 in 2023-08-27T20:49:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[111335]: osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894] boot 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[111335]: osdmap e30: 5 total, 5 up, 5 in 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[118902]: from='osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894]' entity='osd.4' cmd='[{"prefix": "osd crush create-or-move", "id": 4, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[118902]: osdmap e29: 5 total, 4 up, 5 in 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[118902]: osd.4 [v2:172.21.15.136:6800/4129055894,v1:172.21.15.136:6801/4129055894] boot 2023-08-27T20:49:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:30 smithi093 ceph-mon[118902]: osdmap e30: 5 total, 5 up, 5 in 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:31 smithi136 ceph-mon[113531]: osdmap e31: 5 total, 5 up, 5 in 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[118902]: osdmap e31: 5 total, 5 up, 5 in 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: from='client.24226 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_3", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: pgmap v85: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:31 smithi093 ceph-mon[111335]: osdmap e31: 5 total, 5 up, 5 in 2023-08-27T20:49:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/2495462633' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]: dispatch 2023-08-27T20:49:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:32 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]: dispatch 2023-08-27T20:49:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:32 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]': finished 2023-08-27T20:49:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:32 smithi136 ceph-mon[113531]: osdmap e32: 6 total, 5 up, 6 in 2023-08-27T20:49:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:32 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/2495462633' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]: dispatch 2023-08-27T20:49:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]: dispatch 2023-08-27T20:49:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]': finished 2023-08-27T20:49:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[111335]: osdmap e32: 6 total, 5 up, 6 in 2023-08-27T20:49:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/2495462633' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]: dispatch 2023-08-27T20:49:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]: dispatch 2023-08-27T20:49:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "eb791c51-36bd-4562-9715-7081bdb9b50b"}]': finished 2023-08-27T20:49:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[118902]: osdmap e32: 6 total, 5 up, 6 in 2023-08-27T20:49:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:32 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:33 smithi136 ceph-mon[113531]: pgmap v88: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/1893213426' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:33 smithi136 ceph-mon[113531]: osdmap e33: 6 total, 5 up, 6 in 2023-08-27T20:49:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:33 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[111335]: pgmap v88: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/1893213426' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[111335]: osdmap e33: 6 total, 5 up, 6 in 2023-08-27T20:49:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[118902]: pgmap v88: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/1893213426' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[118902]: osdmap e33: 6 total, 5 up, 6 in 2023-08-27T20:49:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:33 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:35 smithi093 ceph-mon[111335]: osdmap e34: 6 total, 5 up, 6 in 2023-08-27T20:49:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:35 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:35 smithi093 ceph-mon[118902]: osdmap e34: 6 total, 5 up, 6 in 2023-08-27T20:49:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:35 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:35 smithi136 ceph-mon[113531]: osdmap e34: 6 total, 5 up, 6 in 2023-08-27T20:49:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:35 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:36 smithi093 ceph-mon[111335]: pgmap v91: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:36 smithi093 ceph-mon[118902]: pgmap v91: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:36.777 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:36 smithi136 ceph-mon[113531]: pgmap v91: 1 pgs: 1 remapped+peering; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:38.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:38 smithi136 ceph-mon[113531]: pgmap v92: 1 pgs: 1 remapped+peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:38.619 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-27T20:49:38.619 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:38 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:38 smithi093 ceph-mon[111335]: pgmap v92: 1 pgs: 1 remapped+peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-27T20:49:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:38 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:38 smithi093 ceph-mon[118902]: pgmap v92: 1 pgs: 1 remapped+peering; 577 KiB data, 30 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.5"}]: dispatch 2023-08-27T20:49:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:38 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:39 smithi093 ceph-mon[111335]: Deploying daemon osd.5 on smithi136 2023-08-27T20:49:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:39 smithi093 ceph-mon[118902]: Deploying daemon osd.5 on smithi136 2023-08-27T20:49:39.801 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:39 smithi136 ceph-mon[113531]: Deploying daemon osd.5 on smithi136 2023-08-27T20:49:40.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:40 smithi136 ceph-mon[113531]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-08-27T20:49:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:40 smithi093 ceph-mon[111335]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-08-27T20:49:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:40 smithi093 ceph-mon[118902]: pgmap v93: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 89 KiB/s, 0 objects/s recovering 2023-08-27T20:49:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:41 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:41 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:41 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:41 smithi136 ceph-mon[113531]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-08-27T20:49:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[111335]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-08-27T20:49:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:41 smithi093 ceph-mon[118902]: pgmap v94: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-08-27T20:49:42.823 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 5 on host 'smithi136' 2023-08-27T20:49:43.197 DEBUG:teuthology.orchestra.run.smithi136:osd.5> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.5.service 2023-08-27T20:49:43.200 INFO:tasks.cephadm:Deploying osd.6 on smithi136 with /dev/vg_nvme/lv_2... 2023-08-27T20:49:43.200 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_2 2023-08-27T20:49:43.234 INFO:journalctl@ceph.osd.5.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:49:43.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:43.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:43.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:43 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:43.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:43 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:43.746 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:49:44.430 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 27 20:49:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-5[122959]: 2023-08-27T20:49:44.134+0000 7f67871e7540 -1 osd.5 0 log_to_monitors true 2023-08-27T20:49:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:44 smithi136 ceph-mon[113531]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-08-27T20:49:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:44 smithi136 ceph-mon[113531]: from='osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T20:49:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:44 smithi136 ceph-mon[113531]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T20:49:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:44 smithi093 ceph-mon[111335]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-08-27T20:49:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:44 smithi093 ceph-mon[111335]: from='osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T20:49:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:44 smithi093 ceph-mon[111335]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T20:49:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:44 smithi093 ceph-mon[118902]: pgmap v95: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 60 KiB/s, 0 objects/s recovering 2023-08-27T20:49:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:44 smithi093 ceph-mon[118902]: from='osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808]' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T20:49:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:44 smithi093 ceph-mon[118902]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]: dispatch 2023-08-27T20:49:45.243 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_2 2023-08-27T20:49:45.244 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_2 bs=1M count=10 conv=fsync 2023-08-27T20:49:45.244 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2023-08-27T20:49:45.244 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2023-08-27T20:49:45.244 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00966353 s, 1.1 GB/s 2023-08-27T20:49:45.244 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2023-08-27T20:49:45.573 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi136:vg_nvme/lv_2 2023-08-27T20:49:45.835 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:49:46.188 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-5[122959]: 2023-08-27T20:49:45.816+0000 7f677d9cc700 -1 osd.5 0 waiting for initial osdmap 2023-08-27T20:49:46.189 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-5[122959]: 2023-08-27T20:49:45.821+0000 7f677a7cb700 -1 osd.5 36 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-27T20:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: osdmap e35: 6 total, 5 up, 6 in 2023-08-27T20:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi136 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi136 to 8556M 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:45 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-27T20:49:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: osdmap e35: 6 total, 5 up, 6 in 2023-08-27T20:49:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi136 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi136 to 8556M 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["5"]}]': finished 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: osdmap e35: 6 total, 5 up, 6 in 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808]' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='osd.5 ' entity='osd.5' cmd=[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: pgmap v97: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi136 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:49:46.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi136 to 8556M 2023-08-27T20:49:46.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:46.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:46.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:45 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:46 smithi136 ceph-mon[113531]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:49:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:46 smithi136 ceph-mon[113531]: osdmap e36: 6 total, 5 up, 6 in 2023-08-27T20:49:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:46 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[111335]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:49:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[111335]: osdmap e36: 6 total, 5 up, 6 in 2023-08-27T20:49:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[118902]: from='osd.5 ' entity='osd.5' cmd='[{"prefix": "osd crush create-or-move", "id": 5, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:49:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[118902]: osdmap e36: 6 total, 5 up, 6 in 2023-08-27T20:49:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:46 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:49:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808] boot 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: osdmap e37: 6 total, 6 up, 6 in 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:47 smithi136 ceph-mon[113531]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:49:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:49:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808] boot 2023-08-27T20:49:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: osdmap e37: 6 total, 6 up, 6 in 2023-08-27T20:49:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[111335]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: osd.5 [v2:172.21.15.136:6808/1032556808,v1:172.21.15.136:6809/1032556808] boot 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: osdmap e37: 6 total, 6 up, 6 in 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:49:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: from='client.24253 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_2", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:49:48.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:49:48.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:49:48.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:48.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:47 smithi093 ceph-mon[118902]: pgmap v100: 1 pgs: 1 active+clean; 577 KiB data, 29 MiB used, 447 GiB / 447 GiB avail 2023-08-27T20:49:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:48 smithi136 ceph-mon[113531]: osdmap e38: 6 total, 6 up, 6 in 2023-08-27T20:49:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/1004478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]: dispatch 2023-08-27T20:49:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:48 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]: dispatch 2023-08-27T20:49:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:48 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]': finished 2023-08-27T20:49:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:48 smithi136 ceph-mon[113531]: osdmap e39: 7 total, 6 up, 7 in 2023-08-27T20:49:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:48 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[111335]: osdmap e38: 6 total, 6 up, 6 in 2023-08-27T20:49:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/1004478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]: dispatch 2023-08-27T20:49:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]: dispatch 2023-08-27T20:49:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]': finished 2023-08-27T20:49:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[111335]: osdmap e39: 7 total, 6 up, 7 in 2023-08-27T20:49:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[118902]: osdmap e38: 6 total, 6 up, 6 in 2023-08-27T20:49:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/1004478' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]: dispatch 2023-08-27T20:49:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]: dispatch 2023-08-27T20:49:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "9d1b03b4-2611-4877-83ac-05b34cb091ce"}]': finished 2023-08-27T20:49:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[118902]: osdmap e39: 7 total, 6 up, 7 in 2023-08-27T20:49:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:48 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:49 smithi136 ceph-mon[113531]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/2544321783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:49 smithi136 ceph-mon[113531]: osdmap e40: 7 total, 6 up, 7 in 2023-08-27T20:49:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:49 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[111335]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/2544321783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[111335]: osdmap e40: 7 total, 6 up, 7 in 2023-08-27T20:49:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[118902]: pgmap v103: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/2544321783' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:49:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[118902]: osdmap e40: 7 total, 6 up, 7 in 2023-08-27T20:49:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:49 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:51 smithi136 ceph-mon[113531]: osdmap e41: 7 total, 6 up, 7 in 2023-08-27T20:49:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:51 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:51 smithi136 ceph-mon[113531]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:51 smithi093 ceph-mon[111335]: osdmap e41: 7 total, 6 up, 7 in 2023-08-27T20:49:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:51 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:51 smithi093 ceph-mon[111335]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:51 smithi093 ceph-mon[118902]: osdmap e41: 7 total, 6 up, 7 in 2023-08-27T20:49:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:51 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:49:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:51 smithi093 ceph-mon[118902]: pgmap v106: 1 pgs: 1 active+clean; 577 KiB data, 34 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:54.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:54 smithi136 ceph-mon[113531]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:54.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-27T20:49:54.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:54 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:54 smithi093 ceph-mon[111335]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-27T20:49:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:54 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:54 smithi093 ceph-mon[118902]: pgmap v107: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.6"}]: dispatch 2023-08-27T20:49:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:54 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:55.543 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:55 smithi136 ceph-mon[113531]: Deploying daemon osd.6 on smithi136 2023-08-27T20:49:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:55 smithi093 ceph-mon[118902]: Deploying daemon osd.6 on smithi136 2023-08-27T20:49:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:55 smithi093 ceph-mon[111335]: Deploying daemon osd.6 on smithi136 2023-08-27T20:49:56.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:56 smithi136 ceph-mon[113531]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:56 smithi093 ceph-mon[111335]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:56 smithi093 ceph-mon[118902]: pgmap v108: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[118902]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:57 smithi093 ceph-mon[111335]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:58.298 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:58.298 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:58.298 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:57 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:58.298 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:57 smithi136 ceph-mon[113531]: pgmap v109: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:49:59.090 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 6 on host 'smithi136' 2023-08-27T20:49:59.489 DEBUG:teuthology.orchestra.run.smithi136:osd.6> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.6.service 2023-08-27T20:49:59.491 INFO:tasks.cephadm:Deploying osd.7 on smithi136 with /dev/vg_nvme/lv_1... 2023-08-27T20:49:59.491 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph-volume lvm zap /dev/vg_nvme/lv_1 2023-08-27T20:49:59.523 INFO:journalctl@ceph.osd.6.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:49:59.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:59.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:59.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:49:59 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:49:59.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:49:59 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:00.040 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:50:00.390 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 27 20:50:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-6[127162]: 2023-08-27T20:50:00.160+0000 7f1c5a8a1540 -1 osd.6 0 log_to_monitors true 2023-08-27T20:50:00.645 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:00 smithi136 ceph-mon[113531]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:00.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:00 smithi136 ceph-mon[113531]: overall HEALTH_OK 2023-08-27T20:50:00.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:00 smithi136 ceph-mon[113531]: from='osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T20:50:00.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:00 smithi136 ceph-mon[113531]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T20:50:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[111335]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[111335]: overall HEALTH_OK 2023-08-27T20:50:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[111335]: from='osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T20:50:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[111335]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T20:50:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[118902]: pgmap v110: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[118902]: overall HEALTH_OK 2023-08-27T20:50:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[118902]: from='osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467]' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T20:50:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:00 smithi093 ceph-mon[118902]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]: dispatch 2023-08-27T20:50:01.304 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping: /dev/vg_nvme/lv_1 2023-08-27T20:50:01.305 INFO:teuthology.orchestra.run.smithi136.stderr:Running command: /usr/bin/dd if=/dev/zero of=/dev/vg_nvme/lv_1 bs=1M count=10 conv=fsync 2023-08-27T20:50:01.305 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10+0 records in 2023-08-27T20:50:01.306 INFO:teuthology.orchestra.run.smithi136.stderr:10+0 records out 2023-08-27T20:50:01.306 INFO:teuthology.orchestra.run.smithi136.stderr: stderr: 10485760 bytes (10 MB, 10 MiB) copied, 0.00768944 s, 1.4 GB/s 2023-08-27T20:50:01.306 INFO:teuthology.orchestra.run.smithi136.stderr:--> Zapping successful for: 2023-08-27T20:50:01.818 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch daemon add osd smithi136:vg_nvme/lv_1 2023-08-27T20:50:02.080 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:50:02.188 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-6[127162]: 2023-08-27T20:50:01.833+0000 7f1c51086700 -1 osd.6 0 waiting for initial osdmap 2023-08-27T20:50:02.188 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-6[127162]: 2023-08-27T20:50:01.840+0000 7f1c4b680700 -1 osd.6 43 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: osdmap e42: 7 total, 6 up, 7 in 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:01 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-27T20:50:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: osdmap e42: 7 total, 6 up, 7 in 2023-08-27T20:50:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["6"]}]': finished 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: osdmap e42: 7 total, 6 up, 7 in 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467]' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='osd.6 ' entity='osd.6' cmd=[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: pgmap v112: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:02.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:02.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:02.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:02.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:01 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:02 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi136 2023-08-27T20:50:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:02 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi136 to 5704M 2023-08-27T20:50:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:02 smithi136 ceph-mon[113531]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:50:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:02 smithi136 ceph-mon[113531]: osdmap e43: 7 total, 6 up, 7 in 2023-08-27T20:50:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:02 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi136 2023-08-27T20:50:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi136 to 5704M 2023-08-27T20:50:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[111335]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:50:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[111335]: osdmap e43: 7 total, 6 up, 7 in 2023-08-27T20:50:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi136 2023-08-27T20:50:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi136 to 5704M 2023-08-27T20:50:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[118902]: from='osd.6 ' entity='osd.6' cmd='[{"prefix": "osd crush create-or-move", "id": 6, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:50:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[118902]: osdmap e43: 7 total, 6 up, 7 in 2023-08-27T20:50:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:02 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:04.162 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:50:04.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:50:04.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:04.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467] boot 2023-08-27T20:50:04.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: osdmap e44: 7 total, 7 up, 7 in 2023-08-27T20:50:04.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:04.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:04.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:04.164 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:50:04.164 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:50:04.164 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:04.164 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:03 smithi136 ceph-mon[113531]: osdmap e45: 7 total, 7 up, 7 in 2023-08-27T20:50:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:50:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:50:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467] boot 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: osdmap e44: 7 total, 7 up, 7 in 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[111335]: osdmap e45: 7 total, 7 up, 7 in 2023-08-27T20:50:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:50:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:50:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: osd.6 [v2:172.21.15.136:6816/1621616467,v1:172.21.15.136:6817/1621616467] boot 2023-08-27T20:50:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: osdmap e44: 7 total, 7 up, 7 in 2023-08-27T20:50:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: pgmap v115: 1 pgs: 1 active+clean; 577 KiB data, 35 MiB used, 536 GiB / 536 GiB avail 2023-08-27T20:50:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: from='client.24280 -' entity='client.admin' cmd=[{"prefix": "orch daemon add osd", "svc_arg": "smithi136:vg_nvme/lv_1", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd tree", "states": ["destroyed"], "format": "json"}]: dispatch 2023-08-27T20:50:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.bootstrap-osd"}]: dispatch 2023-08-27T20:50:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:03 smithi093 ceph-mon[118902]: osdmap e45: 7 total, 7 up, 7 in 2023-08-27T20:50:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[111335]: osdmap e46: 7 total, 7 up, 7 in 2023-08-27T20:50:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/4198641749' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]: dispatch 2023-08-27T20:50:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]: dispatch 2023-08-27T20:50:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[111335]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]': finished 2023-08-27T20:50:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[111335]: osdmap e47: 8 total, 7 up, 8 in 2023-08-27T20:50:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[118902]: osdmap e46: 7 total, 7 up, 7 in 2023-08-27T20:50:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/4198641749' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]: dispatch 2023-08-27T20:50:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]: dispatch 2023-08-27T20:50:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[118902]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]': finished 2023-08-27T20:50:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[118902]: osdmap e47: 8 total, 7 up, 8 in 2023-08-27T20:50:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:05 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:05 smithi136 ceph-mon[113531]: osdmap e46: 7 total, 7 up, 7 in 2023-08-27T20:50:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/4198641749' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]: dispatch 2023-08-27T20:50:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:05 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd=[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]: dispatch 2023-08-27T20:50:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:05 smithi136 ceph-mon[113531]: from='client.? ' entity='client.bootstrap-osd' cmd='[{"prefix": "osd new", "uuid": "500cecba-b803-4827-9e93-025f93e19a50"}]': finished 2023-08-27T20:50:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:05 smithi136 ceph-mon[113531]: osdmap e47: 8 total, 7 up, 8 in 2023-08-27T20:50:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:05 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[118902]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-08-27T20:50:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/2204152356' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:50:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[118902]: osdmap e48: 8 total, 7 up, 8 in 2023-08-27T20:50:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[111335]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-08-27T20:50:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/2204152356' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:50:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[111335]: osdmap e48: 8 total, 7 up, 8 in 2023-08-27T20:50:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:06 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:06.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:06 smithi136 ceph-mon[113531]: pgmap v119: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-08-27T20:50:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/2204152356' entity='client.bootstrap-osd' cmd=[{"prefix": "mon getmap"}]: dispatch 2023-08-27T20:50:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:06 smithi136 ceph-mon[113531]: osdmap e48: 8 total, 7 up, 8 in 2023-08-27T20:50:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:06 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:08 smithi093 ceph-mon[111335]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-08-27T20:50:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:08 smithi093 ceph-mon[118902]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-08-27T20:50:08.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:08 smithi136 ceph-mon[113531]: pgmap v121: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail 2023-08-27T20:50:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:10 smithi093 ceph-mon[111335]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-08-27T20:50:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:10 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-27T20:50:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:10 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:10 smithi093 ceph-mon[118902]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-08-27T20:50:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:10 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-27T20:50:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:10 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:10.761 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:10 smithi136 ceph-mon[113531]: pgmap v122: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-08-27T20:50:10.761 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:10 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "osd.7"}]: dispatch 2023-08-27T20:50:10.761 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:10 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:11.659 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:11 smithi136 ceph-mon[113531]: Deploying daemon osd.7 on smithi136 2023-08-27T20:50:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:11 smithi093 ceph-mon[118902]: Deploying daemon osd.7 on smithi136 2023-08-27T20:50:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:11 smithi093 ceph-mon[111335]: Deploying daemon osd.7 on smithi136 2023-08-27T20:50:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:12 smithi136 ceph-mon[113531]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-08-27T20:50:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:12 smithi093 ceph-mon[118902]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-08-27T20:50:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:12 smithi093 ceph-mon[111335]: pgmap v123: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-08-27T20:50:14.089 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:14 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:14.090 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:14 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:14.090 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:14 smithi136 ceph-mon[113531]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2023-08-27T20:50:14.090 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:14 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[111335]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2023-08-27T20:50:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[118902]: pgmap v124: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 71 KiB/s, 0 objects/s recovering 2023-08-27T20:50:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:14 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:14.910 INFO:teuthology.orchestra.run.smithi136.stdout:Created osd(s) 7 on host 'smithi136' 2023-08-27T20:50:15.282 DEBUG:teuthology.orchestra.run.smithi136:osd.7> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@osd.7.service 2023-08-27T20:50:15.284 INFO:tasks.cephadm:Waiting for 8 OSDs to come up... 2023-08-27T20:50:15.284 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd stat -f json 2023-08-27T20:50:15.318 INFO:journalctl@ceph.osd.7.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:50:15.565 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.700 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:15 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:15.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:15 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:16.502 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 27 20:50:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-7[131506]: 2023-08-27T20:50:16.218+0000 7fe066092540 -1 osd.7 0 log_to_monitors true 2023-08-27T20:50:16.726 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:50:16.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:16 smithi136 ceph-mon[113531]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:50:16.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:16 smithi136 ceph-mon[113531]: from='osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T20:50:16.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:16 smithi136 ceph-mon[113531]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T20:50:16.883 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:16 smithi093 ceph-mon[111335]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:50:16.883 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:16 smithi093 ceph-mon[111335]: from='osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T20:50:16.883 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:16 smithi093 ceph-mon[111335]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T20:50:16.884 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:16 smithi093 ceph-mon[118902]: pgmap v125: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:50:16.884 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:16 smithi093 ceph-mon[118902]: from='osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217]' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T20:50:16.884 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:16 smithi093 ceph-mon[118902]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]: dispatch 2023-08-27T20:50:17.047 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":48,"num_osds":8,"num_up_osds":7,"osd_up_since":1693169402,"num_in_osds":8,"osd_in_since":1693169404,"num_remapped_pgs":0} 2023-08-27T20:50:17.914 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3301153499' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T20:50:17.914 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.914 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.914 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.914 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: osdmap e49: 8 total, 7 up, 8 in 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:17.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-mon[113531]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3301153499' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T20:50:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: osdmap e49: 8 total, 7 up, 8 in 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[111335]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3301153499' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:17.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:17.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush set-device-class", "class": "ssd", "ids": ["7"]}]': finished 2023-08-27T20:50:17.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: osdmap e49: 8 total, 7 up, 8 in 2023-08-27T20:50:17.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:17.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217]' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:17.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:17 smithi093 ceph-mon[118902]: from='osd.7 ' entity='osd.7' cmd=[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]: dispatch 2023-08-27T20:50:18.049 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd stat -f json 2023-08-27T20:50:18.188 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-7[131506]: 2023-08-27T20:50:17.912+0000 7fe05c877700 -1 osd.7 0 waiting for initial osdmap 2023-08-27T20:50:18.189 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 27 20:50:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-osd-7[131506]: 2023-08-27T20:50:17.921+0000 7fe059676700 -1 osd.7 50 set_numa_affinity unable to identify public interface '' numa node: (2) No such file or directory 2023-08-27T20:50:18.323 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: Detected new or changed devices on smithi136 2023-08-27T20:50:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi136 to 4278M 2023-08-27T20:50:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:50:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:50:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: osdmap e50: 8 total, 7 up, 8 in 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217] boot 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: osdmap e51: 8 total, 8 up, 8 in 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: Detected new or changed devices on smithi136 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi136 to 4278M 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: osdmap e50: 8 total, 7 up, 8 in 2023-08-27T20:50:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217] boot 2023-08-27T20:50:18.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: osdmap e51: 8 total, 8 up, 8 in 2023-08-27T20:50:18.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:18 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: Detected new or changed devices on smithi136 2023-08-27T20:50:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi136 to 4278M 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: pgmap v127: 1 pgs: 1 active+clean; 577 KiB data, 41 MiB used, 626 GiB / 626 GiB avail; 58 KiB/s, 0 objects/s recovering 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: from='osd.7 ' entity='osd.7' cmd='[{"prefix": "osd crush create-or-move", "id": 7, "weight":0.0873, "args": ["host=smithi136", "root=default"]}]': finished 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: osdmap e50: 8 total, 7 up, 8 in 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: osd.7 [v2:172.21.15.136:6824/4170532217,v1:172.21.15.136:6825/4170532217] boot 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: osdmap e51: 8 total, 8 up, 8 in 2023-08-27T20:50:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:18 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:19.400 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:50:19.735 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[111335]: purged_snaps scrub starts 2023-08-27T20:50:19.735 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[111335]: purged_snaps scrub ok 2023-08-27T20:50:19.736 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/698400897' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T20:50:19.736 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[111335]: osdmap e52: 8 total, 8 up, 8 in 2023-08-27T20:50:19.736 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[118902]: purged_snaps scrub starts 2023-08-27T20:50:19.736 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[118902]: purged_snaps scrub ok 2023-08-27T20:50:19.736 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/698400897' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T20:50:19.736 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:19 smithi093 ceph-mon[118902]: osdmap e52: 8 total, 8 up, 8 in 2023-08-27T20:50:19.811 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":51,"num_osds":8,"num_up_osds":8,"osd_up_since":1693169418,"num_in_osds":8,"osd_in_since":1693169404,"num_remapped_pgs":1} 2023-08-27T20:50:19.812 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T20:50:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:19 smithi136 ceph-mon[113531]: purged_snaps scrub starts 2023-08-27T20:50:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:19 smithi136 ceph-mon[113531]: purged_snaps scrub ok 2023-08-27T20:50:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/698400897' entity='client.admin' cmd=[{"prefix": "osd stat", "format": "json"}]: dispatch 2023-08-27T20:50:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:19 smithi136 ceph-mon[113531]: osdmap e52: 8 total, 8 up, 8 in 2023-08-27T20:50:20.071 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:20 smithi093 ceph-mon[111335]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:20 smithi093 ceph-mon[111335]: osdmap e53: 8 total, 8 up, 8 in 2023-08-27T20:50:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:20 smithi093 ceph-mon[118902]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:20 smithi093 ceph-mon[118902]: osdmap e53: 8 total, 8 up, 8 in 2023-08-27T20:50:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:20 smithi136 ceph-mon[113531]: pgmap v130: 1 pgs: 1 active+clean; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:20 smithi136 ceph-mon[113531]: osdmap e53: 8 total, 8 up, 8 in 2023-08-27T20:50:21.156 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:50:21.156 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":53,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","created":"2023-08-27T20:46:40.100445+0000","modified":"2023-08-27T20:50:20.463420+0000","last_up_change":"2023-08-27T20:50:18.452654+0000","last_in_change":"2023-08-27T20:50:04.867005+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-27T20:48:59.294309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"f8288f9e-5794-4532-9495-65379f66b902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":52,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6803","nonce":468012600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6805","nonce":468012600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6809","nonce":468012600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6807","nonce":468012600}]},"public_addr":"172.21.15.93:6803/468012600","cluster_addr":"172.21.15.93:6805/468012600","heartbeat_back_addr":"172.21.15.93:6809/468012600","heartbeat_front_addr":"172.21.15.93:6807/468012600","state":["exists","up"]},{"osd":1,"uuid":"703653bb-1bc5-418f-bf13-4e71cb82a4da","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6811","nonce":1906627391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6813","nonce":1906627391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6817","nonce":1906627391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6815","nonce":1906627391}]},"public_addr":"172.21.15.93:6811/1906627391","cluster_addr":"172.21.15.93:6813/1906627391","heartbeat_back_addr":"172.21.15.93:6817/1906627391","heartbeat_front_addr":"172.21.15.93:6815/1906627391","state":["exists","up"]},{"osd":2,"uuid":"ca3d4d17-b4ba-402a-999d-496be3cda08d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6819","nonce":3915054168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6820","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6821","nonce":3915054168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6825","nonce":3915054168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6822","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6823","nonce":3915054168}]},"public_addr":"172.21.15.93:6819/3915054168","cluster_addr":"172.21.15.93:6821/3915054168","heartbeat_back_addr":"172.21.15.93:6825/3915054168","heartbeat_front_addr":"172.21.15.93:6823/3915054168","state":["exists","up"]},{"osd":3,"uuid":"04f9b593-2f0e-4b0e-b5ba-2778d688e66d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6826","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6827","nonce":4177593389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6829","nonce":4177593389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6832","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6833","nonce":4177593389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6831","nonce":4177593389}]},"public_addr":"172.21.15.93:6827/4177593389","cluster_addr":"172.21.15.93:6829/4177593389","heartbeat_back_addr":"172.21.15.93:6833/4177593389","heartbeat_front_addr":"172.21.15.93:6831/4177593389","state":["exists","up"]},{"osd":4,"uuid":"2ca1eb6c-000f-42cc-a188-e0044386e9fd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6801","nonce":4129055894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6803","nonce":4129055894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6807","nonce":4129055894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6805","nonce":4129055894}]},"public_addr":"172.21.15.136:6801/4129055894","cluster_addr":"172.21.15.136:6803/4129055894","heartbeat_back_addr":"172.21.15.136:6807/4129055894","heartbeat_front_addr":"172.21.15.136:6805/4129055894","state":["exists","up"]},{"osd":5,"uuid":"eb791c51-36bd-4562-9715-7081bdb9b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6809","nonce":1032556808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6811","nonce":1032556808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6815","nonce":1032556808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6813","nonce":1032556808}]},"public_addr":"172.21.15.136:6809/1032556808","cluster_addr":"172.21.15.136:6811/1032556808","heartbeat_back_addr":"172.21.15.136:6815/1032556808","heartbeat_front_addr":"172.21.15.136:6813/1032556808","state":["exists","up"]},{"osd":6,"uuid":"9d1b03b4-2611-4877-83ac-05b34cb091ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":47,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6817","nonce":1621616467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6819","nonce":1621616467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6823","nonce":1621616467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6821","nonce":1621616467}]},"public_addr":"172.21.15.136:6817/1621616467","cluster_addr":"172.21.15.136:6819/1621616467","heartbeat_back_addr":"172.21.15.136:6823/1621616467","heartbeat_front_addr":"172.21.15.136:6821/1621616467","state":["exists","up"]},{"osd":7,"uuid":"500cecba-b803-4827-9e93-025f93e19a50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6825","nonce":4170532217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6827","nonce":4170532217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6831","nonce":4170532217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6829","nonce":4170532217}]},"public_addr":"172.21.15.136:6825/4170532217","cluster_addr":"172.21.15.136:6827/4170532217","heartbeat_back_addr":"172.21.15.136:6831/4170532217","heartbeat_front_addr":"172.21.15.136:6829/4170532217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:23.430450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:39.677984+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:56.383706+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:13.215635+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:28.875590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:45.109289+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:01.198559+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:17.193536+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[{"pgid":"1.0","osds":[0,6]}],"primary_temp":[],"blocklist":{"172.21.15.93:6801/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1009313451":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/4182115085":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/1072616791":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/1575139232":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/966563482":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/2843438722":"2023-08-28T20:47:20.986783+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T20:50:21.522 INFO:tasks.cephadm.ceph_manager.ceph:[{'pool': 1, 'pool_name': '.mgr', 'create_time': '2023-08-27T20:48:59.294309+0000', 'flags': 1, 'flags_names': 'hashpspool', 'type': 1, 'size': 3, 'min_size': 2, 'crush_rule': 0, 'peering_crush_bucket_count': 0, 'peering_crush_bucket_target': 0, 'peering_crush_bucket_barrier': 0, 'peering_crush_bucket_mandatory_member': 2147483647, 'object_hash': 2, 'pg_autoscale_mode': 'off', 'pg_num': 1, 'pg_placement_num': 1, 'pg_placement_num_target': 1, 'pg_num_target': 1, 'pg_num_pending': 1, 'last_pg_merge_meta': {'source_pgid': '0.0', 'ready_epoch': 0, 'last_epoch_started': 0, 'last_epoch_clean': 0, 'source_version': "0'0", 'target_version': "0'0"}, 'last_change': '22', 'last_force_op_resend': '0', 'last_force_op_resend_prenautilus': '0', 'last_force_op_resend_preluminous': '0', 'auid': 0, 'snap_mode': 'selfmanaged', 'snap_seq': 0, 'snap_epoch': 0, 'pool_snaps': [], 'removed_snaps': '[]', 'quota_max_bytes': 0, 'quota_max_objects': 0, 'tiers': [], 'tier_of': -1, 'read_tier': -1, 'write_tier': -1, 'cache_mode': 'none', 'target_max_bytes': 0, 'target_max_objects': 0, 'cache_target_dirty_ratio_micro': 400000, 'cache_target_dirty_high_ratio_micro': 600000, 'cache_target_full_ratio_micro': 800000, 'cache_min_flush_age': 0, 'cache_min_evict_age': 0, 'erasure_code_profile': '', 'hit_set_params': {'type': 'none'}, 'hit_set_period': 0, 'hit_set_count': 0, 'use_gmt_hitset': True, 'min_read_recency_for_promote': 0, 'min_write_recency_for_promote': 0, 'hit_set_grade_decay_rate': 0, 'hit_set_search_last_n': 0, 'grade_table': [], 'stripe_width': 0, 'expected_num_objects': 0, 'fast_read': False, 'options': {'pg_num_max': 32, 'pg_num_min': 1}, 'application_metadata': {'mgr': {}}}] 2023-08-27T20:50:21.522 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd pool get .mgr pg_num 2023-08-27T20:50:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1391497061' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1391497061' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:21.779 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1391497061' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:22 smithi093 ceph-mon[111335]: pgmap v133: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2023-08-27T20:50:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:22 smithi093 ceph-mon[111335]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-08-27T20:50:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:22 smithi093 ceph-mon[111335]: osdmap e54: 8 total, 8 up, 8 in 2023-08-27T20:50:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:22 smithi093 ceph-mon[118902]: pgmap v133: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2023-08-27T20:50:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:22 smithi093 ceph-mon[118902]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-08-27T20:50:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:22 smithi093 ceph-mon[118902]: osdmap e54: 8 total, 8 up, 8 in 2023-08-27T20:50:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:22 smithi136 ceph-mon[113531]: pgmap v133: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2023-08-27T20:50:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:22 smithi136 ceph-mon[113531]: Health check failed: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded (PG_DEGRADED) 2023-08-27T20:50:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:22 smithi136 ceph-mon[113531]: osdmap e54: 8 total, 8 up, 8 in 2023-08-27T20:50:22.947 INFO:teuthology.orchestra.run.smithi093.stdout:pg_num: 1 2023-08-27T20:50:23.336 INFO:tasks.cephadm:Adding prometheus.a on smithi136 2023-08-27T20:50:23.336 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch apply prometheus '1;smithi136=a' 2023-08-27T20:50:23.612 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:50:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:23 smithi136 ceph-mon[113531]: osdmap e55: 8 total, 8 up, 8 in 2023-08-27T20:50:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3206904185' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-08-27T20:50:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:23 smithi093 ceph-mon[111335]: osdmap e55: 8 total, 8 up, 8 in 2023-08-27T20:50:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3206904185' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-08-27T20:50:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:23 smithi093 ceph-mon[118902]: osdmap e55: 8 total, 8 up, 8 in 2023-08-27T20:50:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3206904185' entity='client.admin' cmd=[{"prefix": "osd pool get", "pool": ".mgr", "var": "pg_num"}]: dispatch 2023-08-27T20:50:24.641 INFO:teuthology.orchestra.run.smithi136.stdout:Scheduled prometheus update... 2023-08-27T20:50:24.809 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:24 smithi136 ceph-mon[113531]: pgmap v136: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2023-08-27T20:50:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:24 smithi093 ceph-mon[111335]: pgmap v136: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2023-08-27T20:50:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:24 smithi093 ceph-mon[118902]: pgmap v136: 1 pgs: 1 activating+undersized+degraded+remapped; 577 KiB data, 47 MiB used, 715 GiB / 715 GiB avail; 2/6 objects degraded (33.333%) 2023-08-27T20:50:25.010 DEBUG:teuthology.orchestra.run.smithi136:prometheus.a> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@prometheus.a.service 2023-08-27T20:50:25.012 INFO:tasks.cephadm:Adding node-exporter.a on smithi093 2023-08-27T20:50:25.012 INFO:tasks.cephadm:Adding node-exporter.b on smithi136 2023-08-27T20:50:25.013 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch apply node-exporter '2;smithi093=a;smithi136=b' 2023-08-27T20:50:25.042 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:50:25.300 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:50:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi136=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: Saving service prometheus spec with placement smithi136=a;count:1 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: Deploying daemon prometheus.a on smithi136 2023-08-27T20:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:25 smithi136 ceph-mon[113531]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-08-27T20:50:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi136=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: Saving service prometheus spec with placement smithi136=a;count:1 2023-08-27T20:50:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: Deploying daemon prometheus.a on smithi136 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[111335]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: from='client.24322 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "prometheus", "placement": "1;smithi136=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: Saving service prometheus spec with placement smithi136=a;count:1 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:25.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:25.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: Deploying daemon prometheus.a on smithi136 2023-08-27T20:50:25.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:25 smithi093 ceph-mon[118902]: pgmap v137: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 104 KiB/s, 0 objects/s recovering 2023-08-27T20:50:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:26 smithi136 ceph-mon[113531]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-08-27T20:50:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:26 smithi136 ceph-mon[113531]: Cluster is now healthy 2023-08-27T20:50:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:26 smithi093 ceph-mon[111335]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-08-27T20:50:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:26 smithi093 ceph-mon[111335]: Cluster is now healthy 2023-08-27T20:50:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:26 smithi093 ceph-mon[118902]: Health check cleared: PG_DEGRADED (was: Degraded data redundancy: 2/6 objects degraded (33.333%), 1 pg degraded) 2023-08-27T20:50:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:26 smithi093 ceph-mon[118902]: Cluster is now healthy 2023-08-27T20:50:27.078 INFO:teuthology.orchestra.run.smithi136.stdout:Scheduled node-exporter update... 2023-08-27T20:50:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:28 smithi136 ceph-mon[113531]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-08-27T20:50:28.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:28 smithi136 ceph-mon[113531]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi093=a;smithi136=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:28.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:28 smithi136 ceph-mon[113531]: Saving service node-exporter spec with placement smithi093=a;smithi136=b;count:2 2023-08-27T20:50:28.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:28 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:28.584 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[118902]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-08-27T20:50:28.584 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[118902]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi093=a;smithi136=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:28.584 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[118902]: Saving service node-exporter spec with placement smithi093=a;smithi136=b;count:2 2023-08-27T20:50:28.584 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:28.585 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[111335]: pgmap v138: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 88 KiB/s, 0 objects/s recovering 2023-08-27T20:50:28.585 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[111335]: from='client.24328 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "node-exporter", "placement": "2;smithi093=a;smithi136=b", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:28.585 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[111335]: Saving service node-exporter spec with placement smithi093=a;smithi136=b;count:2 2023-08-27T20:50:28.586 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:28 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:28.941 DEBUG:teuthology.orchestra.run.smithi093:node-exporter.a> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@node-exporter.a.service 2023-08-27T20:50:28.944 DEBUG:teuthology.orchestra.run.smithi136:node-exporter.b> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@node-exporter.b.service 2023-08-27T20:50:28.947 INFO:tasks.cephadm:Adding alertmanager.a on smithi093 2023-08-27T20:50:28.947 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch apply alertmanager '1;smithi093=a' 2023-08-27T20:50:28.979 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:50:28.981 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:50:29.801 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:50:30.725 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:30 smithi136 ceph-mon[113531]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-08-27T20:50:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:30 smithi093 ceph-mon[111335]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-08-27T20:50:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:30 smithi093 ceph-mon[118902]: pgmap v139: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 72 KiB/s, 0 objects/s recovering 2023-08-27T20:50:31.292 INFO:teuthology.orchestra.run.smithi136.stdout:Scheduled alertmanager update... 2023-08-27T20:50:31.861 DEBUG:teuthology.orchestra.run.smithi093:alertmanager.a> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@alertmanager.a.service 2023-08-27T20:50:31.864 INFO:tasks.cephadm:Adding grafana.a on smithi136 2023-08-27T20:50:31.865 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph orch apply grafana '1;smithi136=a' 2023-08-27T20:50:31.898 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:-- Logs begin at Sun 2023-08-27 20:34:28 UTC. -- 2023-08-27T20:50:32.141 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:50:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:32 smithi136 ceph-mon[113531]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-08-27T20:50:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:32 smithi136 ceph-mon[113531]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi093=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:32 smithi136 ceph-mon[113531]: Saving service alertmanager spec with placement smithi093=a;count:1 2023-08-27T20:50:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:32 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[118902]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-08-27T20:50:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[118902]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi093=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[118902]: Saving service alertmanager spec with placement smithi093=a;count:1 2023-08-27T20:50:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[111335]: pgmap v140: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 61 KiB/s, 0 objects/s recovering 2023-08-27T20:50:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[111335]: from='client.24334 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "alertmanager", "placement": "1;smithi093=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[111335]: Saving service alertmanager spec with placement smithi093=a;count:1 2023-08-27T20:50:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:32 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:33.188 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:50:33 smithi136 bash[134342]: d768ffb45e04a6337aff87fcfa21029075fc89aaafed760aafb0b23660a73c2e 2023-08-27T20:50:33.188 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:50:33 smithi136 systemd[1]: Started Ceph prometheus.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:50:33.332 INFO:teuthology.orchestra.run.smithi136.stdout:Scheduled grafana update... 2023-08-27T20:50:33.695 DEBUG:teuthology.orchestra.run.smithi136:grafana.a> sudo journalctl -f -n 0 -u ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@grafana.a.service 2023-08-27T20:50:33.697 INFO:tasks.cephadm:Setting up client nodes... 2023-08-27T20:50:33.697 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph auth get-or-create client.0 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-08-27T20:50:33.726 INFO:journalctl@ceph.grafana.a.smithi136.stdout:-- Logs begin at Sun 2023-08-27 20:34:30 UTC. -- 2023-08-27T20:50:33.970 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:34.392 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setuser ceph since I am not root 2023-08-27T20:50:34.392 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setgroup ceph since I am not root 2023-08-27T20:50:34.392 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mgr[111564]: -- 172.21.15.93:0/1636487351 <== mon.0 v2:172.21.15.93:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x560f6d36c9c0 con 0x560f6d342800 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:34.392+0000 7f19423831c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi136=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: Saving service grafana spec with placement smithi136=a;count:1 2023-08-27T20:50:34.393 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi136=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: Saving service grafana spec with placement smithi136=a;count:1 2023-08-27T20:50:34.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: pgmap v141: 1 pgs: 1 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail; 55 KiB/s, 0 objects/s recovering 2023-08-27T20:50:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd=[{"prefix": "mgr module enable", "module": "prometheus"}]: dispatch 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: from='client.24340 -' entity='client.admin' cmd=[{"prefix": "orch apply", "service_type": "grafana", "placement": "1;smithi136=a", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: Saving service grafana spec with placement smithi136=a;count:1 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ignoring --setuser ceph since I am not root 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ignoring --setgroup ceph since I am not root 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-mgr[115473]: -- 172.21.15.136:0/3952255788 <== mon.2 v2:172.21.15.93:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55ff061989c0 con 0x55ff0616ec00 2023-08-27T20:50:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:34.391+0000 7ffa6ee801c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:50:34.725 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:34.556+0000 7f19423831c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:50:34.844 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:34.552+0000 7ffa6ee801c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:50:34.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:34.870+0000 7f19423831c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:50:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:34.843+0000 7ffa6ee801c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:50:35.214 INFO:teuthology.orchestra.run.smithi093.stdout:[client.0] 2023-08-27T20:50:35.215 INFO:teuthology.orchestra.run.smithi093.stdout: key = AQAbt+tkYPF+DBAAVekBcMryVyeWmcvpWKan5A== 2023-08-27T20:50:35.396 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[111335]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-27T20:50:35.396 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[111335]: mgrmap e16: y(active, since 3m), standbys: x 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3952028975' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[118902]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[118902]: mgrmap e16: y(active, since 3m), standbys: x 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3952028975' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:35.397 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T20:50:35.518 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:35.352+0000 7ffa6ee801c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:50:35.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-mon[113531]: from='mgr.14152 172.21.15.93:0/1668409922' entity='mgr.y' cmd='[{"prefix": "mgr module enable", "module": "prometheus"}]': finished 2023-08-27T20:50:35.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-mon[113531]: mgrmap e16: y(active, since 3m), standbys: x 2023-08-27T20:50:35.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3952028975' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:35.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:35.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.0", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T20:50:35.594 DEBUG:teuthology.orchestra.run.smithi093:> set -ex 2023-08-27T20:50:35.594 DEBUG:teuthology.orchestra.run.smithi093:> sudo dd of=/etc/ceph/ceph.client.0.keyring 2023-08-27T20:50:35.594 DEBUG:teuthology.orchestra.run.smithi093:> sudo chmod 0644 /etc/ceph/ceph.client.0.keyring 2023-08-27T20:50:35.637 DEBUG:teuthology.orchestra.run.smithi136:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph auth get-or-create client.1 mon 'allow *' osd 'allow *' mds 'allow *' mgr 'allow *' 2023-08-27T20:50:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:35.413+0000 7f19423831c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:50:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:35.585+0000 7f19423831c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:50:35.906 INFO:teuthology.orchestra.run.smithi136.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.b/config 2023-08-27T20:50:35.935 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:35.517+0000 7ffa6ee801c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:50:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:35.935+0000 7ffa6ee801c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:50:36.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:35.985+0000 7f19423831c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:50:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:36.741+0000 7ffa6ee801c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:50:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:36.831+0000 7ffa6ee801c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:50:37.097 INFO:teuthology.orchestra.run.smithi136.stdout:[client.1] 2023-08-27T20:50:37.098 INFO:teuthology.orchestra.run.smithi136.stdout: key = AQAdt+tkBj+VBRAAq1zrS5p0DDIgP/clS2AFfQ== 2023-08-27T20:50:37.147 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:36 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:36.742+0000 7f19423831c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:50:37.147 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:36 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:36.827+0000 7f19423831c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:50:37.273 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.136:0/3044478707' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:37.273 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:37.273 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T20:50:37.422 DEBUG:teuthology.orchestra.run.smithi136:> set -ex 2023-08-27T20:50:37.422 DEBUG:teuthology.orchestra.run.smithi136:> sudo dd of=/etc/ceph/ceph.client.1.keyring 2023-08-27T20:50:37.422 DEBUG:teuthology.orchestra.run.smithi136:> sudo chmod 0644 /etc/ceph/ceph.client.1.keyring 2023-08-27T20:50:37.424 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.136:0/3044478707' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:37.424 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:37.424 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T20:50:37.424 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.136:0/3044478707' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:37.425 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd=[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]: dispatch 2023-08-27T20:50:37.425 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd='[{"prefix": "auth get-or-create", "entity": "client.1", "caps": ["mon", "allow *", "osd", "allow *", "mds", "allow *", "mgr", "allow *"]}]': finished 2023-08-27T20:50:37.463 INFO:tasks.cephadm:Waiting for OSDs to come up 2023-08-27T20:50:37.464 DEBUG:teuthology.orchestra.run.smithi093:> adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph --cluster ceph osd dump --format=json 2023-08-27T20:50:37.560 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:37.476+0000 7ffa6ee801c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:50:37.696 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:37.424+0000 7f19423831c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:50:37.696 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:37.510+0000 7f19423831c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:50:37.696 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:37.602+0000 7f19423831c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:50:37.696 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:37.695+0000 7f19423831c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:50:37.778 INFO:teuthology.misc.health.smithi093.stdout: 2023-08-27T20:50:37.779 INFO:teuthology.misc.health.smithi093.stdout:{"epoch":55,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","created":"2023-08-27T20:46:40.100445+0000","modified":"2023-08-27T20:50:22.518946+0000","last_up_change":"2023-08-27T20:50:18.452654+0000","last_in_change":"2023-08-27T20:50:04.867005+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":1,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-27T20:48:59.294309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}}],"osds":[{"osd":0,"uuid":"f8288f9e-5794-4532-9495-65379f66b902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6803","nonce":468012600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6805","nonce":468012600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6809","nonce":468012600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6807","nonce":468012600}]},"public_addr":"172.21.15.93:6803/468012600","cluster_addr":"172.21.15.93:6805/468012600","heartbeat_back_addr":"172.21.15.93:6809/468012600","heartbeat_front_addr":"172.21.15.93:6807/468012600","state":["exists","up"]},{"osd":1,"uuid":"703653bb-1bc5-418f-bf13-4e71cb82a4da","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":33,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6811","nonce":1906627391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6813","nonce":1906627391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6817","nonce":1906627391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6815","nonce":1906627391}]},"public_addr":"172.21.15.93:6811/1906627391","cluster_addr":"172.21.15.93:6813/1906627391","heartbeat_back_addr":"172.21.15.93:6817/1906627391","heartbeat_front_addr":"172.21.15.93:6815/1906627391","state":["exists","up"]},{"osd":2,"uuid":"ca3d4d17-b4ba-402a-999d-496be3cda08d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6819","nonce":3915054168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6820","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6821","nonce":3915054168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6825","nonce":3915054168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6822","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6823","nonce":3915054168}]},"public_addr":"172.21.15.93:6819/3915054168","cluster_addr":"172.21.15.93:6821/3915054168","heartbeat_back_addr":"172.21.15.93:6825/3915054168","heartbeat_front_addr":"172.21.15.93:6823/3915054168","state":["exists","up"]},{"osd":3,"uuid":"04f9b593-2f0e-4b0e-b5ba-2778d688e66d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6826","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6827","nonce":4177593389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6829","nonce":4177593389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6832","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6833","nonce":4177593389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6831","nonce":4177593389}]},"public_addr":"172.21.15.93:6827/4177593389","cluster_addr":"172.21.15.93:6829/4177593389","heartbeat_back_addr":"172.21.15.93:6833/4177593389","heartbeat_front_addr":"172.21.15.93:6831/4177593389","state":["exists","up"]},{"osd":4,"uuid":"2ca1eb6c-000f-42cc-a188-e0044386e9fd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6801","nonce":4129055894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6803","nonce":4129055894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6807","nonce":4129055894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6805","nonce":4129055894}]},"public_addr":"172.21.15.136:6801/4129055894","cluster_addr":"172.21.15.136:6803/4129055894","heartbeat_back_addr":"172.21.15.136:6807/4129055894","heartbeat_front_addr":"172.21.15.136:6805/4129055894","state":["exists","up"]},{"osd":5,"uuid":"eb791c51-36bd-4562-9715-7081bdb9b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":40,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6809","nonce":1032556808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6811","nonce":1032556808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6815","nonce":1032556808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6813","nonce":1032556808}]},"public_addr":"172.21.15.136:6809/1032556808","cluster_addr":"172.21.15.136:6811/1032556808","heartbeat_back_addr":"172.21.15.136:6815/1032556808","heartbeat_front_addr":"172.21.15.136:6813/1032556808","state":["exists","up"]},{"osd":6,"uuid":"9d1b03b4-2611-4877-83ac-05b34cb091ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":47,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6817","nonce":1621616467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6819","nonce":1621616467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6823","nonce":1621616467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6821","nonce":1621616467}]},"public_addr":"172.21.15.136:6817/1621616467","cluster_addr":"172.21.15.136:6819/1621616467","heartbeat_back_addr":"172.21.15.136:6823/1621616467","heartbeat_front_addr":"172.21.15.136:6821/1621616467","state":["exists","up"]},{"osd":7,"uuid":"500cecba-b803-4827-9e93-025f93e19a50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":54,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6825","nonce":4170532217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6827","nonce":4170532217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6831","nonce":4170532217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6829","nonce":4170532217}]},"public_addr":"172.21.15.136:6825/4170532217","cluster_addr":"172.21.15.136:6827/4170532217","heartbeat_back_addr":"172.21.15.136:6831/4170532217","heartbeat_front_addr":"172.21.15.136:6829/4170532217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:23.430450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:39.677984+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:56.383706+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:13.215635+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:28.875590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:45.109289+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:01.198559+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:17.193536+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.93:6801/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1009313451":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/4182115085":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/1072616791":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/1575139232":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/966563482":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/2843438722":"2023-08-28T20:47:20.986783+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T20:50:37.792 DEBUG:teuthology.misc:8 of 8 OSDs are up 2023-08-27T20:50:37.792 INFO:tasks.cephadm:Creating RBD pool 2023-08-27T20:50:37.792 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- sudo ceph --cluster ceph osd pool create rbd 8 2023-08-27T20:50:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:37.560+0000 7ffa6ee801c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:50:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:37.648+0000 7ffa6ee801c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:50:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:37.739+0000 7ffa6ee801c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:50:37.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:37 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:37.927+0000 7f19423831c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:50:38.070 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:38.278 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2848470295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:38.278 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:37.976+0000 7ffa6ee801c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:50:38.279 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:38.067+0000 7ffa6ee801c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:50:38.370 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2848470295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:38.370 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2848470295' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:38.370 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:38.034+0000 7f19423831c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:50:38.370 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:38.270+0000 7f19423831c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:50:38.371 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:38.369+0000 7f19423831c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:50:38.552 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:38.278+0000 7ffa6ee801c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:50:38.552 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:38.372+0000 7ffa6ee801c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:50:38.645 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:38.557+0000 7f19423831c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:50:38.899 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:38.645+0000 7f19423831c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:50:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:38.551+0000 7ffa6ee801c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:50:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:38.635+0000 7ffa6ee801c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:50:39.471 INFO:teuthology.orchestra.run.smithi093.stderr:pool 'rbd' created 2023-08-27T20:50:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2118831512' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T20:50:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:39 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T20:50:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:39.373+0000 7ffa6ee801c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:50:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:39.564+0000 7ffa6ee801c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:50:39.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2118831512' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T20:50:39.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:39 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T20:50:39.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2118831512' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T20:50:39.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:39 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]: dispatch 2023-08-27T20:50:39.696 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:39 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:39.415+0000 7f19423831c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:50:39.697 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:39 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:39.616+0000 7f19423831c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:50:39.900 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- sudo ceph --cluster ceph osd pool application enable rbd rbd --yes-i-really-mean-it 2023-08-27T20:50:39.989 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:39 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:39.797+0000 7f19423831c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:50:40.169 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:39.738+0000 7ffa6ee801c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:50:40.623 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-27T20:50:40.624 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-mon[111335]: osdmap e56: 8 total, 8 up, 8 in 2023-08-27T20:50:40.624 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-27T20:50:40.624 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-mon[118902]: osdmap e56: 8 total, 8 up, 8 in 2023-08-27T20:50:40.624 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:40.434+0000 7f19423831c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:50:40.624 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:40.623+0000 7f19423831c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:50:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "rbd", "pg_num": 8}]': finished 2023-08-27T20:50:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-mon[113531]: osdmap e56: 8 total, 8 up, 8 in 2023-08-27T20:50:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:40.339+0000 7ffa6ee801c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:50:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:40.521+0000 7ffa6ee801c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:50:40.888 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:40.815+0000 7f19423831c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:40.697+0000 7ffa6ee801c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:40.811+0000 7ffa6ee801c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: [27/Aug/2023:20:50:40] ENGINE Bus STARTING 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: CherryPy Checker: 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: The Application mounted at '' has an empty config. 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: [27/Aug/2023:20:50:40] ENGINE Serving on http://:::9283 2023-08-27T20:50:41.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: [27/Aug/2023:20:50:40] ENGINE Bus STARTED 2023-08-27T20:50:41.142 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:40.936+0000 7f19423831c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:50:41.142 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:41] ENGINE Bus STARTING 2023-08-27T20:50:41.477 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: osdmap e57: 8 total, 8 up, 8 in 2023-08-27T20:50:41.478 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: CherryPy Checker: 2023-08-27T20:50:41.478 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: The Application mounted at '' has an empty config. 2023-08-27T20:50:41.478 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:50:41.478 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:41] ENGINE Serving on http://:::9283 2023-08-27T20:50:41.478 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:41] ENGINE Bus STARTED 2023-08-27T20:50:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: Standby manager daemon x restarted 2023-08-27T20:50:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: Standby manager daemon x started 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: Active manager daemon y restarted 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: Activating manager daemon y 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: osdmap e58: 8 total, 8 up, 8 in 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: mgrmap e17: y(active, starting, since 0.00916644s), standbys: x 2023-08-27T20:50:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:41.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: Manager daemon y is now available 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: osdmap e57: 8 total, 8 up, 8 in 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: Standby manager daemon x restarted 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: Standby manager daemon x started 2023-08-27T20:50:41.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: Active manager daemon y restarted 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: Activating manager daemon y 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: osdmap e58: 8 total, 8 up, 8 in 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: mgrmap e17: y(active, starting, since 0.00916644s), standbys: x 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:50:41.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:50:41.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: Manager daemon y is now available 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:50:41.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3184639238' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:50:41.746 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:50:41.747 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:50:41.747 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3184639238' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:50:41.747 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:41.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: osdmap e57: 8 total, 8 up, 8 in 2023-08-27T20:50:41.854 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: Standby manager daemon x restarted 2023-08-27T20:50:41.855 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: Standby manager daemon x started 2023-08-27T20:50:41.855 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:50:41.855 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:50:41.855 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:50:41.855 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/3388115658' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:50:41.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: Active manager daemon y restarted 2023-08-27T20:50:41.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: Activating manager daemon y 2023-08-27T20:50:41.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: osdmap e58: 8 total, 8 up, 8 in 2023-08-27T20:50:41.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: mgrmap e17: y(active, starting, since 0.00916644s), standbys: x 2023-08-27T20:50:41.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:50:41.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:50:41.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:50:41.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:50:41.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:50:41.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:50:41.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:50:41.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:50:41.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:50:41.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:50:41.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:50:41.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:50:41.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: Manager daemon y is now available 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:50:41.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:50:41.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:50:41.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3184639238' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:50:41.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:41.980 INFO:teuthology.orchestra.run.smithi093.stderr:enabled application 'rbd' on pool 'rbd' 2023-08-27T20:50:42.380 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-08-27T20:50:42.380 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-08-27T20:50:42.381 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph mgr dump --format=json 2023-08-27T20:50:42.674 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:43.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: [27/Aug/2023:20:50:41] ENGINE Bus STARTING 2023-08-27T20:50:43.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: [27/Aug/2023:20:50:41] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:50:43.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: [27/Aug/2023:20:50:41] ENGINE Bus STARTED 2023-08-27T20:50:43.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: mgrmap e18: y(active, since 1.0127s), standbys: x 2023-08-27T20:50:43.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3184639238' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: osdmap e59: 8 total, 8 up, 8 in 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: [27/Aug/2023:20:50:41] ENGINE Bus STARTING 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: [27/Aug/2023:20:50:41] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: [27/Aug/2023:20:50:41] ENGINE Bus STARTED 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: mgrmap e18: y(active, since 1.0127s), standbys: x 2023-08-27T20:50:43.200 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:43.201 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3184639238' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:50:43.201 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: osdmap e59: 8 total, 8 up, 8 in 2023-08-27T20:50:43.201 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.201 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.201 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.201 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: [27/Aug/2023:20:50:41] ENGINE Bus STARTING 2023-08-27T20:50:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: [27/Aug/2023:20:50:41] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:50:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: [27/Aug/2023:20:50:41] ENGINE Bus STARTED 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: mgrmap e18: y(active, since 1.0127s), standbys: x 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: pgmap v4: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3184639238' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "rbd", "app": "rbd", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: osdmap e59: 8 total, 8 up, 8 in 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:43.907 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:50:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3617348593' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:44.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3617348593' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T20:50:44.301 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":18,"active_gid":14460,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":4025849521},{"type":"v1","addr":"172.21.15.93:6801","nonce":4025849521}]},"active_addr":"172.21.15.93:6801/4025849521","active_change":"2023-08-27T20:50:40.950772+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":14466,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":58,"active_clients":[{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.93:0","nonce":713101306}]}]} 2023-08-27T20:50:44.305 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-08-27T20:50:44.305 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-08-27T20:50:44.305 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T20:50:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: pgmap v5: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:50:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:50:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:50:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3617348593' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T20:50:44.583 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi136 to 4022M 2023-08-27T20:50:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:50:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:50:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:50:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:50:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:50:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:50:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: mgrmap e19: y(active, since 3s), standbys: x 2023-08-27T20:50:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:50:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:50:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi136 to 4022M 2023-08-27T20:50:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:50:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:50:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:50:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: mgrmap e19: y(active, since 3s), standbys: x 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:44 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi136 to 4022M 2023-08-27T20:50:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: mgrmap e19: y(active, since 3s), standbys: x 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:44 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:45.743 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:50:45.743 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":59,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","created":"2023-08-27T20:46:40.100445+0000","modified":"2023-08-27T20:50:41.956425+0000","last_up_change":"2023-08-27T20:50:18.452654+0000","last_in_change":"2023-08-27T20:50:04.867005+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-27T20:48:59.294309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T20:50:39.429736+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f8288f9e-5794-4532-9495-65379f66b902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6803","nonce":468012600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6805","nonce":468012600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6809","nonce":468012600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6807","nonce":468012600}]},"public_addr":"172.21.15.93:6803/468012600","cluster_addr":"172.21.15.93:6805/468012600","heartbeat_back_addr":"172.21.15.93:6809/468012600","heartbeat_front_addr":"172.21.15.93:6807/468012600","state":["exists","up"]},{"osd":1,"uuid":"703653bb-1bc5-418f-bf13-4e71cb82a4da","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6811","nonce":1906627391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6813","nonce":1906627391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6817","nonce":1906627391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6815","nonce":1906627391}]},"public_addr":"172.21.15.93:6811/1906627391","cluster_addr":"172.21.15.93:6813/1906627391","heartbeat_back_addr":"172.21.15.93:6817/1906627391","heartbeat_front_addr":"172.21.15.93:6815/1906627391","state":["exists","up"]},{"osd":2,"uuid":"ca3d4d17-b4ba-402a-999d-496be3cda08d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6819","nonce":3915054168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6820","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6821","nonce":3915054168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6825","nonce":3915054168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6822","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6823","nonce":3915054168}]},"public_addr":"172.21.15.93:6819/3915054168","cluster_addr":"172.21.15.93:6821/3915054168","heartbeat_back_addr":"172.21.15.93:6825/3915054168","heartbeat_front_addr":"172.21.15.93:6823/3915054168","state":["exists","up"]},{"osd":3,"uuid":"04f9b593-2f0e-4b0e-b5ba-2778d688e66d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6826","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6827","nonce":4177593389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6829","nonce":4177593389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6832","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6833","nonce":4177593389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6831","nonce":4177593389}]},"public_addr":"172.21.15.93:6827/4177593389","cluster_addr":"172.21.15.93:6829/4177593389","heartbeat_back_addr":"172.21.15.93:6833/4177593389","heartbeat_front_addr":"172.21.15.93:6831/4177593389","state":["exists","up"]},{"osd":4,"uuid":"2ca1eb6c-000f-42cc-a188-e0044386e9fd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6801","nonce":4129055894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6803","nonce":4129055894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6807","nonce":4129055894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6805","nonce":4129055894}]},"public_addr":"172.21.15.136:6801/4129055894","cluster_addr":"172.21.15.136:6803/4129055894","heartbeat_back_addr":"172.21.15.136:6807/4129055894","heartbeat_front_addr":"172.21.15.136:6805/4129055894","state":["exists","up"]},{"osd":5,"uuid":"eb791c51-36bd-4562-9715-7081bdb9b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6809","nonce":1032556808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6811","nonce":1032556808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6815","nonce":1032556808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6813","nonce":1032556808}]},"public_addr":"172.21.15.136:6809/1032556808","cluster_addr":"172.21.15.136:6811/1032556808","heartbeat_back_addr":"172.21.15.136:6815/1032556808","heartbeat_front_addr":"172.21.15.136:6813/1032556808","state":["exists","up"]},{"osd":6,"uuid":"9d1b03b4-2611-4877-83ac-05b34cb091ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6817","nonce":1621616467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6819","nonce":1621616467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6823","nonce":1621616467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6821","nonce":1621616467}]},"public_addr":"172.21.15.136:6817/1621616467","cluster_addr":"172.21.15.136:6819/1621616467","heartbeat_back_addr":"172.21.15.136:6823/1621616467","heartbeat_front_addr":"172.21.15.136:6821/1621616467","state":["exists","up"]},{"osd":7,"uuid":"500cecba-b803-4827-9e93-025f93e19a50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6825","nonce":4170532217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6827","nonce":4170532217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6831","nonce":4170532217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6829","nonce":4170532217}]},"public_addr":"172.21.15.136:6825/4170532217","cluster_addr":"172.21.15.136:6827/4170532217","heartbeat_back_addr":"172.21.15.136:6831/4170532217","heartbeat_front_addr":"172.21.15.136:6829/4170532217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:23.430450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:39.677984+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:56.383706+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:13.215635+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:28.875590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:45.109289+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:01.198559+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:17.193536+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.93:0/2819309041":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1735542319":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6801/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6801/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1009313451":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/4182115085":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/804147002":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1072616791":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/750179315":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1575139232":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/966563482":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/2843438722":"2023-08-28T20:47:20.986783+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T20:50:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:45 smithi093 ceph-mon[111335]: Deploying daemon alertmanager.a on smithi093 2023-08-27T20:50:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:45 smithi093 ceph-mon[111335]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3461438644' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:45 smithi093 ceph-mon[118902]: Deploying daemon alertmanager.a on smithi093 2023-08-27T20:50:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:45 smithi093 ceph-mon[118902]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3461438644' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:45 smithi136 ceph-mon[113531]: Deploying daemon alertmanager.a on smithi093 2023-08-27T20:50:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:45 smithi136 ceph-mon[113531]: pgmap v6: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3461438644' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:46.959 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-08-27T20:50:46.960 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T20:50:47.586 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:47 smithi093 ceph-mon[111335]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:47 smithi093 ceph-mon[118902]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:47 smithi136 ceph-mon[113531]: pgmap v7: 9 pgs: 9 active+clean; 577 KiB data, 48 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:49.372 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:50:49.372 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":59,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","created":"2023-08-27T20:46:40.100445+0000","modified":"2023-08-27T20:50:41.956425+0000","last_up_change":"2023-08-27T20:50:18.452654+0000","last_in_change":"2023-08-27T20:50:04.867005+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":2,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-27T20:48:59.294309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T20:50:39.429736+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}}],"osds":[{"osd":0,"uuid":"f8288f9e-5794-4532-9495-65379f66b902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":53,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6803","nonce":468012600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6805","nonce":468012600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6809","nonce":468012600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6807","nonce":468012600}]},"public_addr":"172.21.15.93:6803/468012600","cluster_addr":"172.21.15.93:6805/468012600","heartbeat_back_addr":"172.21.15.93:6809/468012600","heartbeat_front_addr":"172.21.15.93:6807/468012600","state":["exists","up"]},{"osd":1,"uuid":"703653bb-1bc5-418f-bf13-4e71cb82a4da","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6811","nonce":1906627391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6813","nonce":1906627391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6817","nonce":1906627391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6815","nonce":1906627391}]},"public_addr":"172.21.15.93:6811/1906627391","cluster_addr":"172.21.15.93:6813/1906627391","heartbeat_back_addr":"172.21.15.93:6817/1906627391","heartbeat_front_addr":"172.21.15.93:6815/1906627391","state":["exists","up"]},{"osd":2,"uuid":"ca3d4d17-b4ba-402a-999d-496be3cda08d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6819","nonce":3915054168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6820","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6821","nonce":3915054168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6825","nonce":3915054168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6822","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6823","nonce":3915054168}]},"public_addr":"172.21.15.93:6819/3915054168","cluster_addr":"172.21.15.93:6821/3915054168","heartbeat_back_addr":"172.21.15.93:6825/3915054168","heartbeat_front_addr":"172.21.15.93:6823/3915054168","state":["exists","up"]},{"osd":3,"uuid":"04f9b593-2f0e-4b0e-b5ba-2778d688e66d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6826","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6827","nonce":4177593389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6829","nonce":4177593389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6832","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6833","nonce":4177593389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6831","nonce":4177593389}]},"public_addr":"172.21.15.93:6827/4177593389","cluster_addr":"172.21.15.93:6829/4177593389","heartbeat_back_addr":"172.21.15.93:6833/4177593389","heartbeat_front_addr":"172.21.15.93:6831/4177593389","state":["exists","up"]},{"osd":4,"uuid":"2ca1eb6c-000f-42cc-a188-e0044386e9fd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":0,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6801","nonce":4129055894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6803","nonce":4129055894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6807","nonce":4129055894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6805","nonce":4129055894}]},"public_addr":"172.21.15.136:6801/4129055894","cluster_addr":"172.21.15.136:6803/4129055894","heartbeat_back_addr":"172.21.15.136:6807/4129055894","heartbeat_front_addr":"172.21.15.136:6805/4129055894","state":["exists","up"]},{"osd":5,"uuid":"eb791c51-36bd-4562-9715-7081bdb9b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6809","nonce":1032556808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6811","nonce":1032556808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6815","nonce":1032556808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6813","nonce":1032556808}]},"public_addr":"172.21.15.136:6809/1032556808","cluster_addr":"172.21.15.136:6811/1032556808","heartbeat_back_addr":"172.21.15.136:6815/1032556808","heartbeat_front_addr":"172.21.15.136:6813/1032556808","state":["exists","up"]},{"osd":6,"uuid":"9d1b03b4-2611-4877-83ac-05b34cb091ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6817","nonce":1621616467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6819","nonce":1621616467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6823","nonce":1621616467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6821","nonce":1621616467}]},"public_addr":"172.21.15.136:6817/1621616467","cluster_addr":"172.21.15.136:6819/1621616467","heartbeat_back_addr":"172.21.15.136:6823/1621616467","heartbeat_front_addr":"172.21.15.136:6821/1621616467","state":["exists","up"]},{"osd":7,"uuid":"500cecba-b803-4827-9e93-025f93e19a50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":56,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6825","nonce":4170532217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6827","nonce":4170532217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6831","nonce":4170532217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6829","nonce":4170532217}]},"public_addr":"172.21.15.136:6825/4170532217","cluster_addr":"172.21.15.136:6827/4170532217","heartbeat_back_addr":"172.21.15.136:6831/4170532217","heartbeat_front_addr":"172.21.15.136:6829/4170532217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:23.430450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:39.677984+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:56.383706+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:13.215635+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:28.875590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:45.109289+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:01.198559+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:17.193536+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.93:0/2819309041":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1735542319":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6801/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6801/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1009313451":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/4182115085":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/804147002":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1072616791":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/750179315":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1575139232":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/966563482":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/2843438722":"2023-08-28T20:47:20.986783+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T20:50:49.695 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:49 smithi093 ceph-mon[118902]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:49.695 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/751656180' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:49.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:49 smithi093 ceph-mon[111335]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:49.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/751656180' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:49.901 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.0 flush_pg_stats 2023-08-27T20:50:49.901 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.1 flush_pg_stats 2023-08-27T20:50:49.902 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.2 flush_pg_stats 2023-08-27T20:50:49.902 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.3 flush_pg_stats 2023-08-27T20:50:49.903 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.4 flush_pg_stats 2023-08-27T20:50:49.903 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.5 flush_pg_stats 2023-08-27T20:50:49.904 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.6 flush_pg_stats 2023-08-27T20:50:49.904 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.7 flush_pg_stats 2023-08-27T20:50:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:49 smithi136 ceph-mon[113531]: pgmap v8: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/751656180' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:50:50.501 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:50:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:50:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:50:50.962 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:51.209 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:51.561 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:51.613 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:51.916 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:51.972 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:50:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[142406]: ts=2023-08-27T20:50:51.869Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-08-27T20:50:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:50:51 smithi093 systemd[1]: Started Ceph alertmanager.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:50:52.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[111335]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:52.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.314 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[118902]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:52.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:52] ENGINE Bus STOPPING 2023-08-27T20:50:52.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:52] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T20:50:52.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:52] ENGINE Bus STOPPED 2023-08-27T20:50:52.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:52] ENGINE Bus STARTING 2023-08-27T20:50:52.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:52] ENGINE Serving on http://:::9283 2023-08-27T20:50:52.315 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:50:52] ENGINE Bus STARTED 2023-08-27T20:50:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:52 smithi136 ceph-mon[113531]: pgmap v9: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:52 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:52 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:52 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:52 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.174 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:53 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:53 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:53 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T20:50:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:53 smithi136 ceph-mon[113531]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T20:50:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:53 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:53 smithi136 ceph-mon[113531]: Deploying daemon grafana.a on smithi136 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[111335]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[111335]: Deploying daemon grafana.a on smithi136 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T20:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[118902]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-ssl-verify", "value": "false"}]: dispatch 2023-08-27T20:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-mon[118902]: Deploying daemon grafana.a on smithi136 2023-08-27T20:50:54.097 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:50:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[142406]: ts=2023-08-27T20:50:53.843Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000198115s 2023-08-27T20:50:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:54 smithi136 ceph-mon[113531]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:54 smithi093 ceph-mon[111335]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:54 smithi093 ceph-mon[118902]: pgmap v10: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:56.100 INFO:teuthology.orchestra.run.smithi093.stdout:219043332105 2023-08-27T20:50:56.100 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.7 2023-08-27T20:50:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:56 smithi093 ceph-mon[111335]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:56 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:56 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:50:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:56 smithi093 ceph-mon[118902]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:56 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:56 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:50:56.414 INFO:teuthology.orchestra.run.smithi093.stdout:107374182421 2023-08-27T20:50:56.415 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.3 2023-08-27T20:50:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:56 smithi136 ceph-mon[113531]: pgmap v11: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:56 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:50:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:56 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:50:56.731 INFO:teuthology.orchestra.run.smithi093.stdout:34359738400 2023-08-27T20:50:56.731 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.0 2023-08-27T20:50:57.615 INFO:teuthology.orchestra.run.smithi093.stdout:55834574878 2023-08-27T20:50:57.615 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.1 2023-08-27T20:50:58.102 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:58 smithi093 ceph-mon[118902]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:58.281 INFO:teuthology.orchestra.run.smithi093.stdout:128849018899 2023-08-27T20:50:58.281 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.4 2023-08-27T20:50:58.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:58 smithi136 ceph-mon[113531]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:58 smithi093 ceph-mon[111335]: pgmap v12: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:58.847 INFO:teuthology.orchestra.run.smithi093.stdout:158913789968 2023-08-27T20:50:58.848 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.5 2023-08-27T20:50:59.055 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:59.085 INFO:teuthology.orchestra.run.smithi093.stdout:77309411355 2023-08-27T20:50:59.085 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.2 2023-08-27T20:50:59.371 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:50:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:50:59 smithi093 ceph-mon[111335]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:50:59 smithi093 ceph-mon[118902]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:50:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:50:59 smithi136 ceph-mon[113531]: pgmap v13: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:00.007 INFO:teuthology.orchestra.run.smithi093.stdout:188978561038 2023-08-27T20:51:00.007 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.6 2023-08-27T20:51:00.508 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:51:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:51:01.340 INFO:teuthology.orchestra.run.smithi093.stdout:34359738401 2023-08-27T20:51:01.362 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:01.391 INFO:teuthology.orchestra.run.smithi093.stdout:55834574878 2023-08-27T20:51:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[142406]: ts=2023-08-27T20:51:01.845Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.001485309s 2023-08-27T20:51:02.040 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574878 got 55834574878 for osd.1 2023-08-27T20:51:02.040 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:02.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:02 smithi136 ceph-mon[113531]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:02.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3984637057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T20:51:02.273 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/897820539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T20:51:02.321 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:02.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:02 smithi093 ceph-mon[111335]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:02.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3984637057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T20:51:02.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/897820539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T20:51:02.438 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:02 smithi093 ceph-mon[118902]: pgmap v14: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:02.438 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3984637057' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T20:51:02.439 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/897820539' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T20:51:02.487 INFO:teuthology.orchestra.run.smithi093.stdout:107374182422 2023-08-27T20:51:02.576 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:02.616 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738400 got 34359738401 for osd.0 2023-08-27T20:51:02.616 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:03.143 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/682835827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T20:51:03.304 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/682835827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T20:51:03.305 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/682835827' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T20:51:03.325 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:03.408 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 systemd[1]: Starting Ceph grafana.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:51:03.926 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 bash[136599]: cf2ec1682289c3adb6ffc7a3e248dff97f9eb35ec9fa5af52e0b6ac8b4ea03cf 2023-08-27T20:51:03.926 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 systemd[1]: Started Ceph grafana.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807594532Z level=info msg="Starting Grafana" version=9.4.7 commit=4add91f03d branch=HEAD compiled=2023-03-16T23:56:52Z 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807784471Z level=warn msg="\"sentry\" frontend logging provider is deprecated and will be removed in the next major version. Use \"grafana\" provider instead." 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807802199Z level=info msg="Config loaded from" file=/usr/share/grafana/conf/defaults.ini 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807810193Z level=info msg="Config loaded from" file=/etc/grafana/grafana.ini 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.80781734Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_DATA=/var/lib/grafana" 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807823412Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_LOGS=/var/log/grafana" 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807829164Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PLUGINS=/var/lib/grafana/plugins" 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807834962Z level=info msg="Config overridden from Environment variable" var="GF_PATHS_PROVISIONING=/etc/grafana/provisioning" 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807840908Z level=info msg="Path Home" path=/usr/share/grafana 2023-08-27T20:51:03.927 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807846562Z level=info msg="Path Data" path=/var/lib/grafana 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807852009Z level=info msg="Path Logs" path=/var/log/grafana 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807858152Z level=info msg="Path Plugins" path=/var/lib/grafana/plugins 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.807863731Z level=info msg="Path Provisioning" path=/etc/grafana/provisioning 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=settings t=2023-08-27T20:51:03.80786943Z level=info msg="App mode production" 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=sqlstore t=2023-08-27T20:51:03.807914163Z level=info msg="Connecting to DB" dbtype=sqlite3 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=sqlstore t=2023-08-27T20:51:03.807929501Z level=warn msg="SQLite database file has broader permissions than it should" path=/var/lib/grafana/grafana.db mode=-rw-r--r-- expected=-rw-r----- 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.808814311Z level=info msg="Starting DB migrations" 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.80888747Z level=info msg="Executing migration" id="create migration_log table" 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.809449511Z level=info msg="Executing migration" id="create user table" 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.80989944Z level=info msg="Executing migration" id="add unique index user.login" 2023-08-27T20:51:03.928 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.812428908Z level=info msg="Executing migration" id="add unique index user.email" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.814937084Z level=info msg="Executing migration" id="drop index UQE_user_login - v1" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.817531424Z level=info msg="Executing migration" id="drop index UQE_user_email - v1" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.818087261Z level=info msg="Executing migration" id="Rename table user to user_v1 - v1" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.820687958Z level=info msg="Executing migration" id="create user table v2" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.821130855Z level=info msg="Executing migration" id="create index UQE_user_login - v2" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.823304842Z level=info msg="Executing migration" id="create index UQE_user_email - v2" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.82391818Z level=info msg="Executing migration" id="copy data_source v1 to v2" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.825752477Z level=info msg="Executing migration" id="Drop old table user_v1" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.82614122Z level=info msg="Executing migration" id="Add column help_flags1 to user table" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.828243472Z level=info msg="Executing migration" id="Update user table charset" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.828546165Z level=info msg="Executing migration" id="Add last_seen_at column to user" 2023-08-27T20:51:03.929 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.830907577Z level=info msg="Executing migration" id="Add missing user data" 2023-08-27T20:51:03.930 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.831469569Z level=info msg="Executing migration" id="Add is_disabled column to user" 2023-08-27T20:51:03.930 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.833420445Z level=info msg="Executing migration" id="Add index user.login/user.email" 2023-08-27T20:51:03.930 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.833859913Z level=info msg="Executing migration" id="Add is_service_account column to user" 2023-08-27T20:51:03.930 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.834314631Z level=info msg="Executing migration" id="Update is_service_account column to nullable" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.835444366Z level=info msg="Executing migration" id="create temp user table v1-7" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.836614946Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v1-7" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.837460489Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v1-7" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.837859447Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v1-7" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.838847981Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v1-7" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.839991036Z level=info msg="Executing migration" id="Update temp_user table charset" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.84092857Z level=info msg="Executing migration" id="drop index IDX_temp_user_email - v1" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.841828651Z level=info msg="Executing migration" id="drop index IDX_temp_user_org_id - v1" 2023-08-27T20:51:03.931 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.842826144Z level=info msg="Executing migration" id="drop index IDX_temp_user_code - v1" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.843234902Z level=info msg="Executing migration" id="drop index IDX_temp_user_status - v1" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.844311586Z level=info msg="Executing migration" id="Rename table temp_user to temp_user_tmp_qwerty - v1" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.845165422Z level=info msg="Executing migration" id="create temp_user v2" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.846366385Z level=info msg="Executing migration" id="create index IDX_temp_user_email - v2" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.846818128Z level=info msg="Executing migration" id="create index IDX_temp_user_org_id - v2" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.847820987Z level=info msg="Executing migration" id="create index IDX_temp_user_code - v2" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.848781976Z level=info msg="Executing migration" id="create index IDX_temp_user_status - v2" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.849774555Z level=info msg="Executing migration" id="copy temp_user v1 to v2" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.850117428Z level=info msg="Executing migration" id="drop temp_user_tmp_qwerty" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.851212613Z level=info msg="Executing migration" id="Set created for temp users that will otherwise prematurely expire" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.852065497Z level=info msg="Executing migration" id="create star table" 2023-08-27T20:51:03.932 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.852982265Z level=info msg="Executing migration" id="add unique index star.user_id_dashboard_id" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.853415194Z level=info msg="Executing migration" id="create org table v1" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.853942815Z level=info msg="Executing migration" id="create index UQE_org_name - v1" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.854929323Z level=info msg="Executing migration" id="create org_user table v1" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.856067836Z level=info msg="Executing migration" id="create index IDX_org_user_org_id - v1" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.856823466Z level=info msg="Executing migration" id="create index UQE_org_user_org_id_user_id - v1" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.857903227Z level=info msg="Executing migration" id="create index IDX_org_user_user_id - v1" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.85872449Z level=info msg="Executing migration" id="Update org table charset" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.859835958Z level=info msg="Executing migration" id="Update org_user table charset" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.86070066Z level=info msg="Executing migration" id="Migrate all Read Only Viewers to Viewers" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.861584545Z level=info msg="Executing migration" id="create dashboard table" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.862649139Z level=info msg="Executing migration" id="add index dashboard.account_id" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.863501651Z level=info msg="Executing migration" id="add unique index dashboard_account_id_slug" 2023-08-27T20:51:03.933 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.864447719Z level=info msg="Executing migration" id="create dashboard_tag table" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.865513027Z level=info msg="Executing migration" id="add unique index dashboard_tag.dasboard_id_term" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.866323162Z level=info msg="Executing migration" id="drop index UQE_dashboard_tag_dashboard_id_term - v1" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.867273943Z level=info msg="Executing migration" id="Rename table dashboard to dashboard_v1 - v1" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.868187003Z level=info msg="Executing migration" id="create dashboard v2" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.869082462Z level=info msg="Executing migration" id="create index IDX_dashboard_org_id - v2" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.869499054Z level=info msg="Executing migration" id="create index UQE_dashboard_org_id_slug - v2" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.870010946Z level=info msg="Executing migration" id="copy dashboard v1 to v2" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.871066809Z level=info msg="Executing migration" id="drop table dashboard_v1" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.872013262Z level=info msg="Executing migration" id="alter dashboard.data to mediumtext v1" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.872787879Z level=info msg="Executing migration" id="Add column updated_by in dashboard - v2" 2023-08-27T20:51:03.934 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.873313322Z level=info msg="Executing migration" id="Add column created_by in dashboard - v2" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.873783766Z level=info msg="Executing migration" id="Add column gnetId in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.874753311Z level=info msg="Executing migration" id="Add index for gnetId in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.875676904Z level=info msg="Executing migration" id="Add column plugin_id in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.876585445Z level=info msg="Executing migration" id="Add index for plugin_id in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.877647309Z level=info msg="Executing migration" id="Add index for dashboard_id in dashboard_tag" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.878611136Z level=info msg="Executing migration" id="Update dashboard table charset" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.879464977Z level=info msg="Executing migration" id="Update dashboard_tag table charset" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.880438399Z level=info msg="Executing migration" id="Add column folder_id in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.881245621Z level=info msg="Executing migration" id="Add column isFolder in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.882150673Z level=info msg="Executing migration" id="Add column has_acl in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.883249019Z level=info msg="Executing migration" id="Add column uid in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.883962711Z level=info msg="Executing migration" id="Update uid column values in dashboard" 2023-08-27T20:51:03.935 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.884355614Z level=info msg="Executing migration" id="Add unique index dashboard_org_id_uid" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.884895293Z level=info msg="Executing migration" id="Remove unique index org_id_slug" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.885933991Z level=info msg="Executing migration" id="Update dashboard title length" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.886671131Z level=info msg="Executing migration" id="Add unique index for dashboard_org_id_title_folder_id" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.887630909Z level=info msg="Executing migration" id="create dashboard_provisioning" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.888132128Z level=info msg="Executing migration" id="Rename table dashboard_provisioning to dashboard_provisioning_tmp_qwerty - v1" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.889043788Z level=info msg="Executing migration" id="create dashboard_provisioning v2" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.890143524Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id - v2" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.890971876Z level=info msg="Executing migration" id="create index IDX_dashboard_provisioning_dashboard_id_name - v2" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.891727477Z level=info msg="Executing migration" id="copy dashboard_provisioning v1 to v2" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.892760094Z level=info msg="Executing migration" id="drop dashboard_provisioning_tmp_qwerty" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.893135873Z level=info msg="Executing migration" id="Add check_sum column" 2023-08-27T20:51:03.936 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.895491591Z level=info msg="Executing migration" id="Add index for dashboard_title" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.898026084Z level=info msg="Executing migration" id="delete tags for deleted dashboards" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.900643811Z level=info msg="Executing migration" id="delete stars for deleted dashboards" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.90318895Z level=info msg="Executing migration" id="Add index for dashboard_is_folder" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.905853692Z level=info msg="Executing migration" id="Add isPublic for dashboard" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.908663486Z level=info msg="Executing migration" id="create data_source table" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.911262388Z level=info msg="Executing migration" id="add index data_source.account_id" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.913757886Z level=info msg="Executing migration" id="add unique index data_source.account_id_name" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.91635296Z level=info msg="Executing migration" id="drop index IDX_data_source_account_id - v1" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.918932802Z level=info msg="Executing migration" id="drop index UQE_data_source_account_id_name - v1" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.921756063Z level=info msg="Executing migration" id="Rename table data_source to data_source_v1 - v1" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.924199311Z level=info msg="Executing migration" id="create data_source table v2" 2023-08-27T20:51:03.937 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.926862417Z level=info msg="Executing migration" id="create index IDX_data_source_org_id - v2" 2023-08-27T20:51:04.053 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182421 got 107374182422 for osd.3 2023-08-27T20:51:04.054 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:04.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-mon[113531]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:04.179 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.179 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.179 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.179 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.180 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.929413823Z level=info msg="Executing migration" id="create index UQE_data_source_org_id_name - v2" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.932047024Z level=info msg="Executing migration" id="Drop old table data_source_v1 #2" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.934589183Z level=info msg="Executing migration" id="Add column with_credentials" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.937321465Z level=info msg="Executing migration" id="Add secure json data column" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.938042923Z level=info msg="Executing migration" id="Update data_source table charset" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.939014163Z level=info msg="Executing migration" id="Update initial version to 1" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.939346439Z level=info msg="Executing migration" id="Add read_only data column" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.94007907Z level=info msg="Executing migration" id="Migrate logging ds to loki ds" 2023-08-27T20:51:04.181 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.940408892Z level=info msg="Executing migration" id="Update json_data with nulls" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.941003834Z level=info msg="Executing migration" id="Add uid column" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.941590576Z level=info msg="Executing migration" id="Update uid value" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.94197367Z level=info msg="Executing migration" id="Add unique index datasource_org_id_uid" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.942531954Z level=info msg="Executing migration" id="add unique index datasource_org_id_is_default" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.943015241Z level=info msg="Executing migration" id="create api_key table" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.943977062Z level=info msg="Executing migration" id="add index api_key.account_id" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.944962949Z level=info msg="Executing migration" id="add index api_key.key" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.946016705Z level=info msg="Executing migration" id="add index api_key.account_id_name" 2023-08-27T20:51:04.182 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.947095439Z level=info msg="Executing migration" id="drop index IDX_api_key_account_id - v1" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.94785248Z level=info msg="Executing migration" id="drop index UQE_api_key_key - v1" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.948311884Z level=info msg="Executing migration" id="drop index UQE_api_key_account_id_name - v1" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.948750732Z level=info msg="Executing migration" id="Rename table api_key to api_key_v1 - v1" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.949821182Z level=info msg="Executing migration" id="create api_key table v2" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.950912414Z level=info msg="Executing migration" id="create index IDX_api_key_org_id - v2" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.951761778Z level=info msg="Executing migration" id="create index UQE_api_key_key - v2" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.952199446Z level=info msg="Executing migration" id="create index UQE_api_key_org_id_name - v2" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.953220412Z level=info msg="Executing migration" id="copy api_key v1 to v2" 2023-08-27T20:51:04.183 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.954087099Z level=info msg="Executing migration" id="Drop old table api_key_v1" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.954437087Z level=info msg="Executing migration" id="Update api_key table charset" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.954723346Z level=info msg="Executing migration" id="Add expires to api_key table" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.955279085Z level=info msg="Executing migration" id="Add service account foreign key" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.955851745Z level=info msg="Executing migration" id="set service account foreign key to nil if 0" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.956807837Z level=info msg="Executing migration" id="Add last_used_at to api_key table" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.957774837Z level=info msg="Executing migration" id="Add is_revoked column to api_key table" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.958796731Z level=info msg="Executing migration" id="create dashboard_snapshot table v4" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.959211734Z level=info msg="Executing migration" id="drop table dashboard_snapshot_v4 #1" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.960318359Z level=info msg="Executing migration" id="create dashboard_snapshot table v5 #2" 2023-08-27T20:51:04.184 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.961213836Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_key - v5" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.962252805Z level=info msg="Executing migration" id="create index UQE_dashboard_snapshot_delete_key - v5" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.962835172Z level=info msg="Executing migration" id="create index IDX_dashboard_snapshot_user_id - v5" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.963789871Z level=info msg="Executing migration" id="alter dashboard_snapshot to mediumtext v2" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.964744358Z level=info msg="Executing migration" id="Update dashboard_snapshot table charset" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.965823758Z level=info msg="Executing migration" id="Add column external_delete_url to dashboard_snapshots table" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.966463036Z level=info msg="Executing migration" id="Add encrypted dashboard json column" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.967417354Z level=info msg="Executing migration" id="Change dashboard_encrypted column to MEDIUMBLOB" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.968362747Z level=info msg="Executing migration" id="create quota table v1" 2023-08-27T20:51:04.185 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.969331763Z level=info msg="Executing migration" id="create index UQE_quota_org_id_user_id_target - v1" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.969839311Z level=info msg="Executing migration" id="Update quota table charset" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.97012312Z level=info msg="Executing migration" id="create plugin_setting table" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.970763938Z level=info msg="Executing migration" id="create index UQE_plugin_setting_org_id_plugin_id - v1" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.971717584Z level=info msg="Executing migration" id="Add column plugin_version to plugin_settings" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.972739054Z level=info msg="Executing migration" id="Update plugin_setting table charset" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.973197322Z level=info msg="Executing migration" id="create session table" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.973657235Z level=info msg="Executing migration" id="Drop old table playlist table" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.973957008Z level=info msg="Executing migration" id="Drop old table playlist_item table" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.974498561Z level=info msg="Executing migration" id="create playlist table v2" 2023-08-27T20:51:04.186 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.974923409Z level=info msg="Executing migration" id="create playlist item table v2" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.975333488Z level=info msg="Executing migration" id="Update playlist table charset" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.976171805Z level=info msg="Executing migration" id="Update playlist_item table charset" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.976455471Z level=info msg="Executing migration" id="drop preferences table v2" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.976729231Z level=info msg="Executing migration" id="drop preferences table v3" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.976988699Z level=info msg="Executing migration" id="create preferences table v3" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.977402741Z level=info msg="Executing migration" id="Update preferences table charset" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.977663723Z level=info msg="Executing migration" id="Add column team_id in preferences" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.978289997Z level=info msg="Executing migration" id="Update team_id column values in preferences" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.979112684Z level=info msg="Executing migration" id="Add column week_start in preferences" 2023-08-27T20:51:04.187 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.979687747Z level=info msg="Executing migration" id="Add column preferences.json_data" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.980297267Z level=info msg="Executing migration" id="alter preferences.json_data to mediumtext v1" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.980588958Z level=info msg="Executing migration" id="Add preferences index org_id" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.981018202Z level=info msg="Executing migration" id="Add preferences index user_id" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.982069524Z level=info msg="Executing migration" id="create alert table v1" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.982883923Z level=info msg="Executing migration" id="add index alert org_id & id " 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.983876818Z level=info msg="Executing migration" id="add index alert state" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.984318578Z level=info msg="Executing migration" id="add index alert dashboard_id" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.984727164Z level=info msg="Executing migration" id="Create alert_rule_tag table v1" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.985087601Z level=info msg="Executing migration" id="Add unique index alert_rule_tag.alert_id_tag_id" 2023-08-27T20:51:04.188 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.985528658Z level=info msg="Executing migration" id="drop index UQE_alert_rule_tag_alert_id_tag_id - v1" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.985893413Z level=info msg="Executing migration" id="Rename table alert_rule_tag to alert_rule_tag_v1 - v1" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.987331564Z level=info msg="Executing migration" id="Create alert_rule_tag table v2" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.987703952Z level=info msg="Executing migration" id="create index UQE_alert_rule_tag_alert_id_tag_id - Add unique index alert_rule_tag.alert_id_tag_id V2" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.988130764Z level=info msg="Executing migration" id="copy alert_rule_tag v1 to v2" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.988444347Z level=info msg="Executing migration" id="drop table alert_rule_tag_v1" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.988768785Z level=info msg="Executing migration" id="create alert_notification table v1" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.989113968Z level=info msg="Executing migration" id="Add column is_default" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.989714981Z level=info msg="Executing migration" id="Add column frequency" 2023-08-27T20:51:04.189 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.990355823Z level=info msg="Executing migration" id="Add column send_reminder" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.990981108Z level=info msg="Executing migration" id="Add column disable_resolve_message" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.991642489Z level=info msg="Executing migration" id="add index alert_notification org_id & name" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.99206512Z level=info msg="Executing migration" id="Update alert table charset" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.992350815Z level=info msg="Executing migration" id="Update alert_notification table charset" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.992615159Z level=info msg="Executing migration" id="create notification_journal table v1" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.993006348Z level=info msg="Executing migration" id="add index notification_journal org_id & alert_id & notifier_id" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.993452007Z level=info msg="Executing migration" id="drop alert_notification_journal" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.993795561Z level=info msg="Executing migration" id="create alert_notification_state table v1" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.994203008Z level=info msg="Executing migration" id="add index alert_notification_state org_id & alert_id & notifier_id" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.994615064Z level=info msg="Executing migration" id="Add for to alert table" 2023-08-27T20:51:04.190 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.995267957Z level=info msg="Executing migration" id="Add column uid in alert_notification" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.995921471Z level=info msg="Executing migration" id="Update uid column values in alert_notification" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.996235907Z level=info msg="Executing migration" id="Add unique index alert_notification_org_id_uid" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.996666918Z level=info msg="Executing migration" id="Remove unique index org_id_name" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.997043033Z level=info msg="Executing migration" id="Add column secure_settings in alert_notification" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.997696236Z level=info msg="Executing migration" id="alter alert.settings to mediumtext" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.997960329Z level=info msg="Executing migration" id="Add non-unique index alert_notification_state_alert_id" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.998389644Z level=info msg="Executing migration" id="Add non-unique index alert_rule_tag_alert_id" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.998802123Z level=info msg="Executing migration" id="Drop old annotation table v4" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.99908488Z level=info msg="Executing migration" id="create annotation table v5" 2023-08-27T20:51:04.191 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:03.999545206Z level=info msg="Executing migration" id="add index annotation 0 v3" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.00225817Z level=info msg="Executing migration" id="add index annotation 1 v3" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.002751632Z level=info msg="Executing migration" id="add index annotation 2 v3" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.003157663Z level=info msg="Executing migration" id="add index annotation 3 v3" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.003609995Z level=info msg="Executing migration" id="add index annotation 4 v3" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.006359918Z level=info msg="Executing migration" id="Update annotation table charset" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.006648812Z level=info msg="Executing migration" id="Add column region_id to annotation table" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.009006784Z level=info msg="Executing migration" id="Drop category_id index" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.009495213Z level=info msg="Executing migration" id="Add column tags to annotation table" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.011604055Z level=info msg="Executing migration" id="Create annotation_tag table v2" 2023-08-27T20:51:04.192 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.012028484Z level=info msg="Executing migration" id="Add unique index annotation_tag.annotation_id_tag_id" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.01434769Z level=info msg="Executing migration" id="drop index UQE_annotation_tag_annotation_id_tag_id - v2" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.017158481Z level=info msg="Executing migration" id="Rename table annotation_tag to annotation_tag_v2 - v2" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.018848306Z level=info msg="Executing migration" id="Create annotation_tag table v3" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.021572663Z level=info msg="Executing migration" id="create index UQE_annotation_tag_annotation_id_tag_id - Add unique index annotation_tag.annotation_id_tag_id V3" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.024179746Z level=info msg="Executing migration" id="copy annotation_tag v2 to v3" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.026882896Z level=info msg="Executing migration" id="drop table annotation_tag_v2" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.027272478Z level=info msg="Executing migration" id="Update alert annotations and set TEXT to empty" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.030102739Z level=info msg="Executing migration" id="Add created time to annotation table" 2023-08-27T20:51:04.193 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.031025534Z level=info msg="Executing migration" id="Add updated time to annotation table" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.032651873Z level=info msg="Executing migration" id="Add index for created in annotation table" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.035358334Z level=info msg="Executing migration" id="Add index for updated in annotation table" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.038215611Z level=info msg="Executing migration" id="Convert existing annotations from seconds to milliseconds" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.040562991Z level=info msg="Executing migration" id="Add epoch_end column" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.043194486Z level=info msg="Executing migration" id="Add index for epoch_end" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.045822549Z level=info msg="Executing migration" id="Make epoch_end the same as epoch" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.04837859Z level=info msg="Executing migration" id="Move region to single row" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.051156987Z level=info msg="Executing migration" id="Remove index org_id_epoch from annotation table" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.053803746Z level=info msg="Executing migration" id="Remove index org_id_dashboard_id_panel_id_epoch from annotation table" 2023-08-27T20:51:04.194 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.056280822Z level=info msg="Executing migration" id="Add index for org_id_dashboard_id_epoch_end_epoch on annotation table" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.059005666Z level=info msg="Executing migration" id="Add index for org_id_epoch_end_epoch on annotation table" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.061743582Z level=info msg="Executing migration" id="Remove index org_id_epoch_epoch_end from annotation table" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.064406558Z level=info msg="Executing migration" id="Add index for alert_id on annotation table" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.066997838Z level=info msg="Executing migration" id="Increase tags column to length 4096" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.070903534Z level=info msg="Executing migration" id="create test_data table" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.073184461Z level=info msg="Executing migration" id="create dashboard_version table v1" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.073638257Z level=info msg="Executing migration" id="add index dashboard_version.dashboard_id" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.076219478Z level=info msg="Executing migration" id="add unique index dashboard_version.dashboard_id and dashboard_version.version" 2023-08-27T20:51:04.195 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.079155291Z level=info msg="Executing migration" id="Set dashboard version to 1 where 0" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.081893536Z level=info msg="Executing migration" id="save existing dashboard data in dashboard_version table v1" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.084382172Z level=info msg="Executing migration" id="alter dashboard_version.data to mediumtext v1" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.086925688Z level=info msg="Executing migration" id="create team table" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.089316701Z level=info msg="Executing migration" id="add index team.org_id" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.09010789Z level=info msg="Executing migration" id="add unique index team_org_id_name" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.092736135Z level=info msg="Executing migration" id="create team member table" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.095482684Z level=info msg="Executing migration" id="add index team_member.org_id" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.098001622Z level=info msg="Executing migration" id="add unique index team_member_org_id_team_id_user_id" 2023-08-27T20:51:04.196 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.100508337Z level=info msg="Executing migration" id="add index team_member.team_id" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.103134589Z level=info msg="Executing migration" id="Add column email to team table" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.105727106Z level=info msg="Executing migration" id="Add column external to team_member table" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.108381677Z level=info msg="Executing migration" id="Add column permission to team_member table" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.111004878Z level=info msg="Executing migration" id="create dashboard acl table" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.113715051Z level=info msg="Executing migration" id="add index dashboard_acl_dashboard_id" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.11616356Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_user_id" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.118765437Z level=info msg="Executing migration" id="add unique index dashboard_acl_dashboard_id_team_id" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.121526531Z level=info msg="Executing migration" id="add index dashboard_acl_user_id" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.124111246Z level=info msg="Executing migration" id="add index dashboard_acl_team_id" 2023-08-27T20:51:04.197 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.126755175Z level=info msg="Executing migration" id="add index dashboard_acl_org_id_role" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.127198317Z level=info msg="Executing migration" id="add index dashboard_permission" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.129561688Z level=info msg="Executing migration" id="save default acl rules in dashboard_acl table" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.132183909Z level=info msg="Executing migration" id="delete acl rules for deleted dashboards and folders" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.134776427Z level=info msg="Executing migration" id="create tag table" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.135364226Z level=info msg="Executing migration" id="add index tag.key_value" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.137867565Z level=info msg="Executing migration" id="create login attempt table" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.140468776Z level=info msg="Executing migration" id="add index login_attempt.username" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.14289731Z level=info msg="Executing migration" id="drop index IDX_login_attempt_username - v1" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.143330146Z level=info msg="Executing migration" id="Rename table login_attempt to login_attempt_tmp_qwerty - v1" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.146020912Z level=info msg="Executing migration" id="create login_attempt v2" 2023-08-27T20:51:04.198 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.148633029Z level=info msg="Executing migration" id="create index IDX_login_attempt_username - v2" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.151138268Z level=info msg="Executing migration" id="copy login_attempt v1 to v2" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.151478541Z level=info msg="Executing migration" id="drop login_attempt_tmp_qwerty" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.1540695Z level=info msg="Executing migration" id="create user auth table" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.154574435Z level=info msg="Executing migration" id="create index IDX_user_auth_auth_module_auth_id - v1" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.156523542Z level=info msg="Executing migration" id="alter user_auth.auth_id to length 190" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.159174399Z level=info msg="Executing migration" id="Add OAuth access token to user_auth" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.159931694Z level=info msg="Executing migration" id="Add OAuth refresh token to user_auth" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.162429093Z level=info msg="Executing migration" id="Add OAuth token type to user_auth" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.163214801Z level=info msg="Executing migration" id="Add OAuth expiry to user_auth" 2023-08-27T20:51:04.199 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.165127757Z level=info msg="Executing migration" id="Add index to user_id column in user_auth" 2023-08-27T20:51:04.200 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.167681904Z level=info msg="Executing migration" id="Add OAuth ID token to user_auth" 2023-08-27T20:51:04.200 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.168530816Z level=info msg="Executing migration" id="create server_lock table" 2023-08-27T20:51:04.200 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.170986795Z level=info msg="Executing migration" id="add index server_lock.operation_uid" 2023-08-27T20:51:04.200 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.173633092Z level=info msg="Executing migration" id="create user auth token table" 2023-08-27T20:51:04.200 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.175994843Z level=info msg="Executing migration" id="add unique index user_auth_token.auth_token" 2023-08-27T20:51:04.200 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.176582387Z level=info msg="Executing migration" id="add unique index user_auth_token.prev_auth_token" 2023-08-27T20:51:04.200 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.179316141Z level=info msg="Executing migration" id="add index user_auth_token.user_id" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.181694232Z level=info msg="Executing migration" id="Add revoked_at to the user auth token" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.184129347Z level=info msg="Executing migration" id="create cache_data table" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.184686001Z level=info msg="Executing migration" id="add unique index cache_data.cache_key" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.18726756Z level=info msg="Executing migration" id="create short_url table v1" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.189770189Z level=info msg="Executing migration" id="add index short_url.org_id-uid" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.192371224Z level=info msg="Executing migration" id="alter table short_url alter column created_by type to bigint" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.192685465Z level=info msg="Executing migration" id="delete alert_definition table" 2023-08-27T20:51:04.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.195190566Z level=info msg="Executing migration" id="recreate alert_definition table" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.195650176Z level=info msg="Executing migration" id="add index in alert_definition on org_id and title columns" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.19623788Z level=info msg="Executing migration" id="add index in alert_definition on org_id and uid columns" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.197225314Z level=info msg="Executing migration" id="alter alert_definition table data column to mediumtext in mysql" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.197528016Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and title columns" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.19864368Z level=info msg="Executing migration" id="drop index in alert_definition on org_id and uid columns" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.199130393Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and title columns" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.199537536Z level=info msg="Executing migration" id="add unique index in alert_definition on org_id and uid columns" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.199987846Z level=info msg="Executing migration" id="Add column paused in alert_definition" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.20078964Z level=info msg="Executing migration" id="drop alert_definition table" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.201251997Z level=info msg="Executing migration" id="delete alert_definition_version table" 2023-08-27T20:51:04.441 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.202350249Z level=info msg="Executing migration" id="recreate alert_definition_version table" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.203163279Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_id and version columns" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.20425571Z level=info msg="Executing migration" id="add index in alert_definition_version table on alert_definition_uid and version columns" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.204797977Z level=info msg="Executing migration" id="alter alert_definition_version table data column to mediumtext in mysql" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.205718522Z level=info msg="Executing migration" id="drop alert_definition_version table" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.206137369Z level=info msg="Executing migration" id="create alert_instance table" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.206560207Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, def_uid and current_state columns" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.206990748Z level=info msg="Executing migration" id="add index in alert_instance table on def_org_id, current_state columns" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.207455469Z level=info msg="Executing migration" id="add column current_state_end to alert_instance" 2023-08-27T20:51:04.442 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.208272486Z level=info msg="Executing migration" id="remove index def_org_id, def_uid, current_state on alert_instance" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.208703365Z level=info msg="Executing migration" id="remove index def_org_id, current_state on alert_instance" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.209670738Z level=info msg="Executing migration" id="rename def_org_id to rule_org_id in alert_instance" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.213094251Z level=info msg="Executing migration" id="rename def_uid to rule_uid in alert_instance" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.216437434Z level=info msg="Executing migration" id="add index rule_org_id, rule_uid, current_state on alert_instance" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.217408207Z level=info msg="Executing migration" id="add index rule_org_id, current_state on alert_instance" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.217871591Z level=info msg="Executing migration" id="add current_reason column related to current_state" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.218842202Z level=info msg="Executing migration" id="create alert_rule table" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.219926511Z level=info msg="Executing migration" id="add index in alert_rule on org_id and title columns" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.220904125Z level=info msg="Executing migration" id="add index in alert_rule on org_id and uid columns" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.221465718Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespace_uid, group_uid columns" 2023-08-27T20:51:04.443 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.222388458Z level=info msg="Executing migration" id="alter alert_rule table data column to mediumtext in mysql" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.223451266Z level=info msg="Executing migration" id="add column for to alert_rule" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.22450928Z level=info msg="Executing migration" id="add column annotations to alert_rule" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.225358538Z level=info msg="Executing migration" id="add column labels to alert_rule" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.226339835Z level=info msg="Executing migration" id="remove unique index from alert_rule on org_id, title columns" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.227349381Z level=info msg="Executing migration" id="add index in alert_rule on org_id, namespase_uid and title columns" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.227885647Z level=info msg="Executing migration" id="add dashboard_uid column to alert_rule" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.228687226Z level=info msg="Executing migration" id="add panel_id column to alert_rule" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.229518603Z level=info msg="Executing migration" id="add index in alert_rule on org_id, dashboard_uid and panel_id columns" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.229982896Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.230855861Z level=info msg="Executing migration" id="add is_paused column to alert_rule table" 2023-08-27T20:51:04.444 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.231752641Z level=info msg="Executing migration" id="fix is_paused column for alert_rule table" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.232742297Z level=info msg="Executing migration" id="create alert_rule_version table" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.233221726Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_uid and version columns" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.234178745Z level=info msg="Executing migration" id="add index in alert_rule_version table on rule_org_id, rule_namespace_uid and rule_group columns" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.235199716Z level=info msg="Executing migration" id="alter alert_rule_version table data column to mediumtext in mysql" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.235946352Z level=info msg="Executing migration" id="add column for to alert_rule_version" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.23678157Z level=info msg="Executing migration" id="add column annotations to alert_rule_version" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.237692324Z level=info msg="Executing migration" id="add column labels to alert_rule_version" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.238557005Z level=info msg="Executing migration" id="add rule_group_idx column to alert_rule_version" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.239499878Z level=info msg="Executing migration" id="add is_paused column to alert_rule_versions table" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.240391062Z level=info msg="Executing migration" id="fix is_paused column for alert_rule_version table" 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.24072176Z level=info msg="Executing migration" id=create_alert_configuration_table 2023-08-27T20:51:04.445 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.241784794Z level=info msg="Executing migration" id="Add column default in alert_configuration" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.242645573Z level=info msg="Executing migration" id="alert alert_configuration alertmanager_configuration column from TEXT to MEDIUMTEXT if mysql" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.243528936Z level=info msg="Executing migration" id="add column org_id in alert_configuration" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.244446599Z level=info msg="Executing migration" id="add index in alert_configuration table on org_id column" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.245341694Z level=info msg="Executing migration" id="add configuration_hash column to alert_configuration" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.246279276Z level=info msg="Executing migration" id=create_ngalert_configuration_table 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.247167149Z level=info msg="Executing migration" id="add index in ngalert_configuration on org_id column" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.247615811Z level=info msg="Executing migration" id="add column send_alerts_to in ngalert_configuration" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.248520417Z level=info msg="Executing migration" id="create provenance_type table" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.24943824Z level=info msg="Executing migration" id="add index to uniquify (record_key, record_type, org_id) columns" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.250276754Z level=info msg="Executing migration" id="create alert_image table" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.250722683Z level=info msg="Executing migration" id="add unique index on token to alert_image table" 2023-08-27T20:51:04.446 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.251585657Z level=info msg="Executing migration" id="support longer URLs in alert_image table" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.252542807Z level=info msg="Executing migration" id=create_alert_configuration_history_table 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.25297994Z level=info msg="Executing migration" id="drop non-unique orgID index on alert_configuration" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.253426717Z level=info msg="Executing migration" id="drop unique orgID index on alert_configuration if exists" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.253509325Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop unique orgID index on alert_configuration if exists" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.253759692Z level=info msg="Executing migration" id="extract alertmanager configuration history to separate table" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.254111202Z level=info msg="Executing migration" id="add unique index on orgID to alert_configuration" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.254554935Z level=info msg="Executing migration" id="clear migration entry \"remove unified alerting data\"" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.25467265Z level=info msg="Executing migration" id="move dashboard alerts to unified alerting" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.254782123Z level=info msg="alerts found to migrate" alerts=0 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.25512639Z level=info msg="Executing migration" id="create library_element table v1" 2023-08-27T20:51:04.447 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.25575282Z level=info msg="Executing migration" id="add index library_element org_id-folder_id-name-kind" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.257932361Z level=info msg="Executing migration" id="create library_element_connection table v1" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.260512188Z level=info msg="Executing migration" id="add index library_element_connection element_id-kind-connection_id" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.261144814Z level=info msg="Executing migration" id="add unique index library_element org_id_uid" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.263755757Z level=info msg="Executing migration" id="increase max description length to 2048" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.266422067Z level=info msg="Executing migration" id="clone move dashboard alerts to unified alerting" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.269127444Z level=info msg="Executing migration" id="create data_keys table" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.269553047Z level=info msg="Executing migration" id="create secrets table" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.272169008Z level=info msg="Executing migration" id="rename data_keys name column to id" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.276460808Z level=info msg="Executing migration" id="add name column into data_keys" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.279112388Z level=info msg="Executing migration" id="copy data_keys id column values into name" 2023-08-27T20:51:04.448 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.279441465Z level=info msg="Executing migration" id="rename data_keys name column to label" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.283782519Z level=info msg="Executing migration" id="rename data_keys id column back to name" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.28804417Z level=info msg="Executing migration" id="create kv_store table v1" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.290634287Z level=info msg="Executing migration" id="add index kv_store.org_id-namespace-key" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.291112718Z level=info msg="Executing migration" id="update dashboard_uid and panel_id from existing annotations" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.293792111Z level=info msg="Executing migration" id="create permission table" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.29424242Z level=info msg="Executing migration" id="add unique index permission.role_id" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.296543811Z level=info msg="Executing migration" id="add unique index role_id_action_scope" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.299149769Z level=info msg="Executing migration" id="create role table" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.29958127Z level=info msg="Executing migration" id="add column display_name" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.300572979Z level=info msg="Executing migration" id="add column group_name" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.301540724Z level=info msg="Executing migration" id="add index role.org_id" 2023-08-27T20:51:04.449 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.302657004Z level=info msg="Executing migration" id="add unique index role_org_id_name" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.303453584Z level=info msg="Executing migration" id="add index role_org_id_uid" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.304385674Z level=info msg="Executing migration" id="create team role table" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.305344277Z level=info msg="Executing migration" id="add index team_role.org_id" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.306436069Z level=info msg="Executing migration" id="add unique index team_role_org_id_team_id_role_id" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.306968851Z level=info msg="Executing migration" id="add index team_role.team_id" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.30980737Z level=info msg="Executing migration" id="create user role table" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.312453824Z level=info msg="Executing migration" id="add index user_role.org_id" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.315081739Z level=info msg="Executing migration" id="add unique index user_role_org_id_user_id_role_id" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.315591822Z level=info msg="Executing migration" id="add index user_role.user_id" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.318352185Z level=info msg="Executing migration" id="create builtin role table" 2023-08-27T20:51:04.450 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.321172043Z level=info msg="Executing migration" id="add index builtin_role.role_id" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.323790509Z level=info msg="Executing migration" id="add index builtin_role.name" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.324258632Z level=info msg="Executing migration" id="Add column org_id to builtin_role table" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.326864205Z level=info msg="Executing migration" id="add index builtin_role.org_id" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.327383205Z level=info msg="Executing migration" id="add unique index builtin_role_org_id_role_id_role" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.327843476Z level=info msg="Executing migration" id="Remove unique index role_org_id_uid" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.328259268Z level=info msg="Executing migration" id="add unique index role.uid" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.329992138Z level=info msg="Executing migration" id="create seed assignment table" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.33268305Z level=info msg="Executing migration" id="add unique index builtin_role_role_name" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.333174872Z level=info msg="Executing migration" id="add column hidden to role table" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.334097136Z level=info msg="Executing migration" id="create query_history table v1" 2023-08-27T20:51:04.451 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.334498985Z level=info msg="Executing migration" id="add index query_history.org_id-created_by-datasource_uid" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.334928701Z level=info msg="Executing migration" id="alter table query_history alter column created_by type to bigint" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.33518886Z level=info msg="Executing migration" id="teams permissions migration" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.335553682Z level=info msg="Executing migration" id="dashboard permissions" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.33596583Z level=info msg="Executing migration" id="dashboard permissions uid scopes" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.336338893Z level=info msg="Executing migration" id="drop managed folder create actions" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.33664165Z level=info msg="Executing migration" id="alerting notification permissions" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.336953528Z level=info msg="Executing migration" id="create query_history_star table v1" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.337349202Z level=info msg="Executing migration" id="add index query_history.user_id-query_uid" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.337771731Z level=info msg="Executing migration" id="add column org_id in query_history_star" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.338735613Z level=info msg="Executing migration" id="alter table query_history_star_mig column user_id type to bigint" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.339029385Z level=info msg="Executing migration" id="create correlation table v1" 2023-08-27T20:51:04.452 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.339444816Z level=info msg="Executing migration" id="add index correlations.uid" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.33985893Z level=info msg="Executing migration" id="add index correlations.source_uid" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.3402913Z level=info msg="Executing migration" id="add correlation config column" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.341286444Z level=info msg="Executing migration" id="create entity_events table" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.341709195Z level=info msg="Executing migration" id="create dashboard public config v1" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.344338022Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v1" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.3444424Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index UQE_dashboard_public_config_uid - v1" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.344725846Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.344801117Z level=warn msg="Skipping migration: Already executed, but not recorded in migration log" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.345046162Z level=info msg="Executing migration" id="Drop old dashboard public config table" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.345422965Z level=info msg="Executing migration" id="recreate dashboard public config v1" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.345790083Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v1" 2023-08-27T20:51:04.453 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.346217941Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v1" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.346640977Z level=info msg="Executing migration" id="drop index UQE_dashboard_public_config_uid - v2" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.347008897Z level=info msg="Executing migration" id="drop index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.347423359Z level=info msg="Executing migration" id="Drop public config table" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.347803298Z level=info msg="Executing migration" id="Recreate dashboard public config v2" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.348222096Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_uid - v2" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.348641147Z level=info msg="Executing migration" id="create index IDX_dashboard_public_config_org_id_dashboard_uid - v2" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.349073868Z level=info msg="Executing migration" id="create index UQE_dashboard_public_config_access_token - v2" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.349501925Z level=info msg="Executing migration" id="Rename table dashboard_public_config to dashboard_public - v2" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.352528994Z level=info msg="Executing migration" id="add annotations_enabled column" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.353560644Z level=info msg="Executing migration" id="add time_selection_enabled column" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.354586481Z level=info msg="Executing migration" id="delete orphaned public dashboards" 2023-08-27T20:51:04.454 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.354894825Z level=info msg="Executing migration" id="add share column" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.355881808Z level=info msg="Executing migration" id="create default alerting folders" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.356325077Z level=info msg="Executing migration" id="create file table" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.356744366Z level=info msg="Executing migration" id="file table idx: path natural pk" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.357165144Z level=info msg="Executing migration" id="file table idx: parent_folder_path_hash fast folder retrieval" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.359503407Z level=info msg="Executing migration" id="create file_meta table" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.362271245Z level=info msg="Executing migration" id="file table idx: path key" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.364831552Z level=info msg="Executing migration" id="set path collation in file table" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.367514472Z level=info msg="Executing migration" id="managed permissions migration" 2023-08-27T20:51:04.455 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.367921251Z level=info msg="Executing migration" id="managed folder permissions alert actions migration" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.370464223Z level=info msg="Executing migration" id="RBAC action name migrator" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.370971833Z level=info msg="Executing migration" id="Add UID column to playlist" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.373124076Z level=info msg="Executing migration" id="Update uid column values in playlist" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.373467822Z level=info msg="Executing migration" id="Add index for uid in playlist" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.375840181Z level=info msg="Executing migration" id="update group index for alert rules" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.376268872Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated migration" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.379016025Z level=info msg="Executing migration" id="admin only folder/dashboard permission" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.381582663Z level=info msg="Executing migration" id="add action column to seed_assignment" 2023-08-27T20:51:04.456 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.384193788Z level=info msg="Executing migration" id="add scope column to seed_assignment" 2023-08-27T20:51:04.457 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.385232334Z level=info msg="Executing migration" id="remove unique index builtin_role_role_name before nullable update" 2023-08-27T20:51:04.457 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.387939836Z level=info msg="Executing migration" id="update seed_assignment role_name column to nullable" 2023-08-27T20:51:04.457 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.397796409Z level=info msg="Executing migration" id="add unique index builtin_role_name back" 2023-08-27T20:51:04.457 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.398280857Z level=info msg="Executing migration" id="add unique index builtin_role_action_scope" 2023-08-27T20:51:04.457 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.400905288Z level=info msg="Executing migration" id="add primary key to seed_assigment" 2023-08-27T20:51:04.457 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.40439745Z level=info msg="Executing migration" id="managed folder permissions alert actions repeated fixed migration" 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.407051154Z level=info msg="Executing migration" id="migrate external alertmanagers to datsourcse" 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.409588093Z level=info msg="Executing migration" id="create folder table" 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.412174761Z level=info msg="Executing migration" id="Add index for parent_uid" 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.412752132Z level=info msg="Executing migration" id="Add unique index for folder.uid and folder.org_id" 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.415451556Z level=info msg="Executing migration" id="Update folder title length" 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.41793204Z level=info msg="Executing migration" id="Add unique index for folder.title and folder.parent_uid" 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=migrator t=2023-08-27T20:51:04.420565064Z level=info msg="migrations completed" performed=485 skipped=0 duration=611.694413ms 2023-08-27T20:51:04.458 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=sqlstore t=2023-08-27T20:51:04.421177641Z level=info msg="Created default organization" 2023-08-27T20:51:04.459 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[111335]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:04.459 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.459 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.459 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.459 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.460 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[118902]: pgmap v15: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:04.460 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.460 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.460 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.460 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:04 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:04.707 INFO:teuthology.orchestra.run.smithi093.stdout:219043332107 2023-08-27T20:51:04.938 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=plugin.loader t=2023-08-27T20:51:04.480193308Z level=info msg="Plugin registered" pluginID=input 2023-08-27T20:51:04.938 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=plugin.loader t=2023-08-27T20:51:04.506983796Z level=info msg="Plugin registered" pluginID=grafana-piechart-panel 2023-08-27T20:51:04.939 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=plugin.loader t=2023-08-27T20:51:04.507004095Z level=info msg="Plugin registered" pluginID=vonage-status-panel 2023-08-27T20:51:04.939 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=secrets t=2023-08-27T20:51:04.507079221Z level=info msg="Envelope encryption state" enabled=true currentprovider=secretKey.v1 2023-08-27T20:51:04.939 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=query_data t=2023-08-27T20:51:04.508809836Z level=info msg="Query Service initialization" 2023-08-27T20:51:04.939 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=live.push_http t=2023-08-27T20:51:04.511124198Z level=info msg="Live Push Gateway initialization" 2023-08-27T20:51:05.137 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:05.291 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:05 smithi093 ceph-mon[111335]: Deploying daemon node-exporter.a on smithi093 2023-08-27T20:51:05.291 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4156222489' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T20:51:05.291 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:05 smithi093 ceph-mon[118902]: Deploying daemon node-exporter.a on smithi093 2023-08-27T20:51:05.291 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4156222489' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T20:51:05.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-mon[113531]: Deploying daemon node-exporter.a on smithi093 2023-08-27T20:51:05.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4156222489' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T20:51:05.304 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=infra.usagestats.collector t=2023-08-27T20:51:05.218134335Z level=info msg="registering usage stat providers" usageStatsProvidersLen=2 2023-08-27T20:51:05.304 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=provisioning.datasources t=2023-08-27T20:51:05.218629612Z level=warn msg="[Deprecated] the datasource provisioning config is outdated. please upgrade" filename=/etc/grafana/provisioning/datasources/ceph-dashboard.yml 2023-08-27T20:51:05.304 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=provisioning.datasources t=2023-08-27T20:51:05.251302087Z level=info msg="inserting datasource from configuration " name=Dashboard1 uid=P43CA22E17D0F9596 2023-08-27T20:51:05.304 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=provisioning.datasources t=2023-08-27T20:51:05.28259028Z level=info msg="inserting datasource from configuration " name=Loki uid=P8E80F9AEF21F6940 2023-08-27T20:51:05.304 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=provisioning.alerting t=2023-08-27T20:51:05.304078696Z level=info msg="starting to provision alerting" 2023-08-27T20:51:05.304 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=provisioning.alerting t=2023-08-27T20:51:05.304097501Z level=info msg="finished to provision alerting" 2023-08-27T20:51:05.358 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332105 got 219043332107 for osd.7 2023-08-27T20:51:05.359 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:05.688 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=grafanaStorageLogger t=2023-08-27T20:51:05.304343351Z level=info msg="storage starting" 2023-08-27T20:51:05.688 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=ngalert.state.manager t=2023-08-27T20:51:05.304343698Z level=info msg="Warming state cache for startup" 2023-08-27T20:51:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=http.server t=2023-08-27T20:51:05.306018501Z level=info msg="HTTP Server Listen" address=[::]:3000 protocol=https subUrl= socket= 2023-08-27T20:51:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=sqlstore.transactions t=2023-08-27T20:51:05.315313242Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-08-27T20:51:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=ngalert.state.manager t=2023-08-27T20:51:05.320402065Z level=info msg="State cache has been initialized" states=0 duration=16.054359ms 2023-08-27T20:51:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=ticker t=2023-08-27T20:51:05.320540455Z level=info msg=starting first_tick=2023-08-27T20:51:10Z 2023-08-27T20:51:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=ngalert.multiorg.alertmanager t=2023-08-27T20:51:05.320568403Z level=info msg="starting MultiOrg Alertmanager" 2023-08-27T20:51:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 20:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=sqlstore.transactions t=2023-08-27T20:51:05.386333362Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-08-27T20:51:05.974 INFO:teuthology.orchestra.run.smithi093.stdout:158913789970 2023-08-27T20:51:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:06 smithi093 ceph-mon[118902]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1918918217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T20:51:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:06 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:06 smithi093 ceph-mon[111335]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1918918217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T20:51:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:06 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:06.396 INFO:teuthology.orchestra.run.smithi093.stdout:77309411356 2023-08-27T20:51:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:06 smithi136 ceph-mon[113531]: pgmap v16: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1918918217' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T20:51:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:06 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:06.505 INFO:teuthology.orchestra.run.smithi093.stdout:128849018900 2023-08-27T20:51:06.734 INFO:teuthology.orchestra.run.smithi093.stdout:188978561039 2023-08-27T20:51:07.036 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:06 smithi093 bash[144707]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-08-27T20:51:07.198 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789968 got 158913789970 for osd.5 2023-08-27T20:51:07.198 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4176992247' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T20:51:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4033185724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T20:51:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2478312192' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T20:51:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4176992247' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T20:51:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4033185724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T20:51:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2478312192' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T20:51:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4176992247' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T20:51:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4033185724' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T20:51:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2478312192' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T20:51:07.490 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:07 smithi093 bash[144707]: Getting image source signatures 2023-08-27T20:51:07.490 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:07 smithi093 bash[144707]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-08-27T20:51:07.490 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:07 smithi093 bash[144707]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-08-27T20:51:07.490 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:07 smithi093 bash[144707]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-08-27T20:51:07.770 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018899 got 128849018900 for osd.4 2023-08-27T20:51:07.770 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:08.333 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-mon[111335]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:08.333 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-mon[118902]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:08.354 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411355 got 77309411356 for osd.2 2023-08-27T20:51:08.355 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:08.355 INFO:tasks.cephadm.ceph_manager.ceph:need seq 188978561038 got 188978561039 for osd.6 2023-08-27T20:51:08.355 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:08.355 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-27T20:51:08.356 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T20:51:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:08 smithi136 ceph-mon[113531]: pgmap v17: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:08.739 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 bash[144707]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-08-27T20:51:08.739 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 bash[144707]: Writing manifest to image destination 2023-08-27T20:51:08.793 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:09.232 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.964Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.964Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.964Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=arp 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=bcache 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=bonding 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=btrfs 2023-08-27T20:51:09.233 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=conntrack 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=cpu 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=diskstats 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=dmi 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=edac 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=entropy 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=filefd 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=filesystem 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=hwmon 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=infiniband 2023-08-27T20:51:09.234 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=ipvs 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=loadavg 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=mdadm 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=meminfo 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=netclass 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.965Z caller=node_exporter.go:117 level=info collector=netdev 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=netstat 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=nfs 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=nfsd 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=nvme 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=os 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=pressure 2023-08-27T20:51:09.235 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=rapl 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=schedstat 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=selinux 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=sockstat 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=softnet 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=stat 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=tapestats 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=textfile 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=time 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=uname 2023-08-27T20:51:09.236 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=vmstat 2023-08-27T20:51:09.237 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=xfs 2023-08-27T20:51:09.237 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=node_exporter.go:117 level=info collector=zfs 2023-08-27T20:51:09.237 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-08-27T20:51:09.237 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-a[144862]: ts=2023-08-27T20:51:08.966Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-08-27T20:51:09.237 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 bash[144707]: 977eb54f6fca8cda5931aa86c0203b5ca6c0780aa0e7fd1a82b4ba1c7f70a3d1 2023-08-27T20:51:09.237 INFO:journalctl@ceph.node-exporter.a.smithi093.stdout:Aug 27 20:51:08 smithi093 systemd[1]: Started Ceph node-exporter.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:51:09.656 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:09 smithi136 systemd[1]: Starting Ceph node-exporter.b for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:51:10.157 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:10 smithi136 ceph-mon[113531]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:10.157 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:10 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.157 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:10 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.157 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:10 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.157 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:10 smithi136 ceph-mon[113531]: Deploying daemon node-exporter.b on smithi136 2023-08-27T20:51:10.157 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:09 smithi136 bash[136871]: Trying to pull quay.io/prometheus/node-exporter:v1.5.0... 2023-08-27T20:51:10.183 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:10.185 INFO:teuthology.orchestra.run.smithi093.stderr:dumped all 2023-08-27T20:51:10.369 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[111335]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:10.369 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.369 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.370 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.370 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[111335]: Deploying daemon node-exporter.b on smithi136 2023-08-27T20:51:10.370 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[118902]: pgmap v18: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:10.370 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.370 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.370 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:10.370 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:10 smithi093 ceph-mon[118902]: Deploying daemon node-exporter.b on smithi136 2023-08-27T20:51:10.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:51:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:51:10.439 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:10 smithi136 bash[136871]: Getting image source signatures 2023-08-27T20:51:10.439 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:10 smithi136 bash[136871]: Copying blob sha256:c0e87333d3805a7b7e1c25783dd25f27da0fe70a8e262eee7ed190c8dc843fa7 2023-08-27T20:51:10.439 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:10 smithi136 bash[136871]: Copying blob sha256:5c12815fee558b157dee7f7509dedbaba0a8379098858a65ec869e1f1526ea0c 2023-08-27T20:51:10.439 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:10 smithi136 bash[136871]: Copying blob sha256:22b70bddd3acadc892fca4c2af4260629bfda5dfd11ebc106a93ce24e752b5ed 2023-08-27T20:51:10.614 INFO:teuthology.orchestra.run.smithi093.stdout:{"pg_ready":true,"pg_map":{"version":18,"stamp":"2023-08-27T20:51:08.969184+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":49756,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":43392,"kb_avail":749911460,"statfs":{"total":767960285184,"available":767909335040,"internally_reserved":0,"allocated":6451200,"data_stored":3717960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44433408},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002358"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:40.957860+0000","last_change":"2023-08-27T20:50:40.472176+0000","last_active":"2023-08-27T20:50:40.957860+0000","last_peered":"2023-08-27T20:50:40.957860+0000","last_clean":"2023-08-27T20:50:40.957860+0000","last_became_active":"2023-08-27T20:50:40.471979+0000","last_became_peered":"2023-08-27T20:50:40.471979+0000","last_unstale":"2023-08-27T20:50:40.957860+0000","last_undegraded":"2023-08-27T20:50:40.957860+0000","last_fullsized":"2023-08-27T20:50:40.957860+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:16:03.929633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.003535+0000","last_change":"2023-08-27T20:50:40.474630+0000","last_active":"2023-08-27T20:50:41.003535+0000","last_peered":"2023-08-27T20:50:41.003535+0000","last_clean":"2023-08-27T20:50:41.003535+0000","last_became_active":"2023-08-27T20:50:40.474345+0000","last_became_peered":"2023-08-27T20:50:40.474345+0000","last_unstale":"2023-08-27T20:50:41.003535+0000","last_undegraded":"2023-08-27T20:50:41.003535+0000","last_fullsized":"2023-08-27T20:50:41.003535+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:35:03.409338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004308+0000","last_change":"2023-08-27T20:50:41.004048+0000","last_active":"2023-08-27T20:50:41.004308+0000","last_peered":"2023-08-27T20:50:41.004308+0000","last_clean":"2023-08-27T20:50:41.004308+0000","last_became_active":"2023-08-27T20:50:41.003731+0000","last_became_peered":"2023-08-27T20:50:41.003731+0000","last_unstale":"2023-08-27T20:50:41.004308+0000","last_undegraded":"2023-08-27T20:50:41.004308+0000","last_fullsized":"2023-08-27T20:50:41.004308+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:58.339578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004423+0000","last_change":"2023-08-27T20:50:41.004114+0000","last_active":"2023-08-27T20:50:41.004423+0000","last_peered":"2023-08-27T20:50:41.004423+0000","last_clean":"2023-08-27T20:50:41.004423+0000","last_became_active":"2023-08-27T20:50:41.003717+0000","last_became_peered":"2023-08-27T20:50:41.003717+0000","last_unstale":"2023-08-27T20:50:41.004423+0000","last_undegraded":"2023-08-27T20:50:41.004423+0000","last_fullsized":"2023-08-27T20:50:41.004423+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:24:46.799377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"58'128","reported_seq":270,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.070128+0000","last_change":"2023-08-27T20:50:22.534100+0000","last_active":"2023-08-27T20:50:41.070128+0000","last_peered":"2023-08-27T20:50:41.070128+0000","last_clean":"2023-08-27T20:50:41.070128+0000","last_became_active":"2023-08-27T20:50:22.533604+0000","last_became_peered":"2023-08-27T20:50:22.533604+0000","last_unstale":"2023-08-27T20:50:41.070128+0000","last_undegraded":"2023-08-27T20:50:41.070128+0000","last_fullsized":"2023-08-27T20:50:41.070128+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_clean_scrub_stamp":"2023-08-27T20:48:59.426866+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:50:03.653850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.725693+0000","last_change":"2023-08-27T20:50:40.472409+0000","last_active":"2023-08-27T20:50:41.725693+0000","last_peered":"2023-08-27T20:50:41.725693+0000","last_clean":"2023-08-27T20:50:41.725693+0000","last_became_active":"2023-08-27T20:50:40.472101+0000","last_became_peered":"2023-08-27T20:50:40.472101+0000","last_unstale":"2023-08-27T20:50:41.725693+0000","last_undegraded":"2023-08-27T20:50:41.725693+0000","last_fullsized":"2023-08-27T20:50:41.725693+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:37:42.899620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004529+0000","last_change":"2023-08-27T20:50:41.003113+0000","last_active":"2023-08-27T20:50:41.004529+0000","last_peered":"2023-08-27T20:50:41.004529+0000","last_clean":"2023-08-27T20:50:41.004529+0000","last_became_active":"2023-08-27T20:50:41.002820+0000","last_became_peered":"2023-08-27T20:50:41.002820+0000","last_unstale":"2023-08-27T20:50:41.004529+0000","last_undegraded":"2023-08-27T20:50:41.004529+0000","last_fullsized":"2023-08-27T20:50:41.004529+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:03:10.585918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.725708+0000","last_change":"2023-08-27T20:50:40.475080+0000","last_active":"2023-08-27T20:50:41.725708+0000","last_peered":"2023-08-27T20:50:41.725708+0000","last_clean":"2023-08-27T20:50:41.725708+0000","last_became_active":"2023-08-27T20:50:40.474840+0000","last_became_peered":"2023-08-27T20:50:40.474840+0000","last_unstale":"2023-08-27T20:50:41.725708+0000","last_undegraded":"2023-08-27T20:50:41.725708+0000","last_fullsized":"2023-08-27T20:50:41.725708+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:35:31.367035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004429+0000","last_change":"2023-08-27T20:50:41.004078+0000","last_active":"2023-08-27T20:50:41.004429+0000","last_peered":"2023-08-27T20:50:41.004429+0000","last_clean":"2023-08-27T20:50:41.004429+0000","last_became_active":"2023-08-27T20:50:41.003803+0000","last_became_peered":"2023-08-27T20:50:41.003803+0000","last_unstale":"2023-08-27T20:50:41.004429+0000","last_undegraded":"2023-08-27T20:50:41.004429+0000","last_fullsized":"2023-08-27T20:50:41.004429+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:14.109789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":3,"up_from":25,"seq":107374182424,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5764,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739388,"statfs":{"total":95995035648,"available":95989133312,"internally_reserved":0,"allocated":585728,"data_stored":243504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.42799999999999999}]},{"osd":1,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.46400000000000002}]},{"osd":2,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.28999999999999998}]},{"osd":4,"last update":"Sun Aug 27 20:50:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.49199999999999999}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.65100000000000002}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.623}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67500000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411357,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":243504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.40600000000000003,"15min":0.41199999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.52500000000000002,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.26200000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.34300000000000003}]},{"osd":3,"last update":"Sun Aug 27 20:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.373}]},{"osd":4,"last update":"Sun Aug 27 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.41799999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.438}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574880,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6404,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738748,"statfs":{"total":95995035648,"available":95988477952,"internally_reserved":0,"allocated":585728,"data_stored":243504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:43 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.35499999999999998,"15min":0.35299999999999998},"min":{"1min":0.23999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.377,"15min":0.36499999999999999},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38900000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:01 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.59799999999999998,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.45300000000000001,"15min":0.46300000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54400000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.50600000000000001}]},{"osd":3,"last update":"Sun Aug 27 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.44},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.40899999999999997}]},{"osd":4,"last update":"Sun Aug 27 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.61399999999999999}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.54600000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.621}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66900000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738403,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7048,"kb_used_data":1152,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738104,"statfs":{"total":95995035648,"available":95987818496,"internally_reserved":0,"allocated":1179648,"data_stored":833872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 20:50:43 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35599999999999998,"15min":0.35699999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.49099999999999999,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.252},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.39900000000000002,"15min":0.39800000000000002},"min":{"1min":0.25700000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.39000000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.41599999999999998,"15min":0.42399999999999999},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40999999999999998,"15min":0.41699999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.27500000000000002}]},{"osd":3,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.318}]},{"osd":4,"last update":"Sun Aug 27 20:50:34 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.372},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.54600000000000004}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.46600000000000003}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.505}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]}]},{"osd":7,"up_from":51,"seq":219043332108,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6276,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":1175552,"data_stored":833549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.372}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.442}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018901,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5824,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739328,"statfs":{"total":95995035648,"available":95989071872,"internally_reserved":0,"allocated":581632,"data_stored":243239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.38},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42999999999999999}]},{"osd":1,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.498}]},{"osd":2,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.44700000000000001}]},{"osd":3,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46100000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.28100000000000003}]},{"osd":6,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.35099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47399999999999998}]}]},{"osd":5,"up_from":37,"seq":158913789971,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":243239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54700000000000004}]},{"osd":1,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.502}]},{"osd":2,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.40400000000000003}]},{"osd":3,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.51800000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.373}]},{"osd":6,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.48599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58999999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561040,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6340,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":1175552,"data_stored":833549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.41199999999999998}]},{"osd":1,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.583,"5min":1.583,"15min":1.583},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.438}]},{"osd":2,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.613,"5min":3.613,"15min":3.613},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.9009999999999998,"5min":5.9009999999999998,"15min":5.9009999999999998},"last":0.42899999999999999}]},{"osd":3,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":3.7389999999999999,"5min":3.7389999999999999,"15min":3.7389999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.34000000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.28599999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.45700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T20:51:10.616 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T20:51:10.879 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:11.403 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:11 smithi093 ceph-mon[111335]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:11.404 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:11 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:11.404 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:11 smithi093 ceph-mon[118902]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:11.404 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:11 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-mon[113531]: from='client.24515 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:11.439 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 bash[136871]: Copying config sha256:0da6a335fe1356545476b749c68f022c897de3a2139e8f0054f6937349ee2b83 2023-08-27T20:51:11.439 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 bash[136871]: Writing manifest to image destination 2023-08-27T20:51:11.939 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.741Z caller=node_exporter.go:180 level=info msg="Starting node_exporter" version="(version=1.5.0, branch=HEAD, revision=1b48970ffcf5630534fb00bb0687d73c66d1c959)" 2023-08-27T20:51:11.939 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.741Z caller=node_exporter.go:181 level=info msg="Build context" build_context="(go=go1.19.3, user=root@6e7732a7b81b, date=20221129-18:59:09)" 2023-08-27T20:51:11.939 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.741Z caller=filesystem_common.go:111 level=info collector=filesystem msg="Parsed flag --collector.filesystem.mount-points-exclude" flag=^/(dev|proc|run/credentials/.+|sys|var/lib/docker/.+|var/lib/containers/storage/.+)($|/) 2023-08-27T20:51:11.939 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.741Z caller=filesystem_common.go:113 level=info collector=filesystem msg="Parsed flag --collector.filesystem.fs-types-exclude" flag=^(autofs|binfmt_misc|bpf|cgroup2?|configfs|debugfs|devpts|devtmpfs|fusectl|hugetlbfs|iso9660|mqueue|nsfs|overlay|proc|procfs|pstore|rpc_pipefs|securityfs|selinuxfs|squashfs|sysfs|tracefs)$ 2023-08-27T20:51:11.939 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.741Z caller=diskstats_common.go:111 level=info collector=diskstats msg="Parsed flag --collector.diskstats.device-exclude" flag=^(ram|loop|fd|(h|s|v|xv)d[a-z]|nvme\d+n\d+p)\d+$ 2023-08-27T20:51:11.939 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.741Z caller=diskstats_linux.go:264 level=error collector=diskstats msg="Failed to open directory, disabling udev device properties" path=/run/udev/data 2023-08-27T20:51:11.939 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:110 level=info msg="Enabled collectors" 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=arp 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=bcache 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=bonding 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=btrfs 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=conntrack 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=cpu 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=cpufreq 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=diskstats 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=dmi 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=edac 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=entropy 2023-08-27T20:51:11.940 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=fibrechannel 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=filefd 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=filesystem 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=hwmon 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=infiniband 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=ipvs 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=loadavg 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=mdadm 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=meminfo 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=netclass 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=netdev 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=netstat 2023-08-27T20:51:11.941 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=nfs 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=nfsd 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=nvme 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=os 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=powersupplyclass 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=pressure 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=rapl 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=schedstat 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=selinux 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=sockstat 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=softnet 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=stat 2023-08-27T20:51:11.942 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=tapestats 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=textfile 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=thermal_zone 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=time 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=udp_queues 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=uname 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=vmstat 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=xfs 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=node_exporter.go:117 level=info collector=zfs 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9100 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-node-exporter-b[136922]: ts=2023-08-27T20:51:11.742Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9100 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 bash[136871]: ae167925824d5ac6d1fb518f6e18f3fc3124181e832b9fe1d067dad8adf28e21 2023-08-27T20:51:11.943 INFO:journalctl@ceph.node-exporter.b.smithi136.stdout:Aug 27 20:51:11 smithi136 systemd[1]: Started Ceph node-exporter.b for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:51:12.005 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:12.007 INFO:teuthology.orchestra.run.smithi093.stderr:dumped all 2023-08-27T20:51:12.183 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[111335]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:12.183 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[118902]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.184 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.185 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.185 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.185 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:12 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:12.389 INFO:teuthology.orchestra.run.smithi093.stdout:{"pg_ready":true,"pg_map":{"version":19,"stamp":"2023-08-27T20:51:10.969534+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":27,"acting":27,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":27,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":49756,"kb_used_data":6300,"kb_used_omap":0,"kb_used_meta":43392,"kb_avail":749911460,"statfs":{"total":767960285184,"available":767909335040,"internally_reserved":0,"allocated":6451200,"data_stored":3717960,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":44433408},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002335"},"pg_stats":[{"pgid":"2.7","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:40.957860+0000","last_change":"2023-08-27T20:50:40.472176+0000","last_active":"2023-08-27T20:50:40.957860+0000","last_peered":"2023-08-27T20:50:40.957860+0000","last_clean":"2023-08-27T20:50:40.957860+0000","last_became_active":"2023-08-27T20:50:40.471979+0000","last_became_peered":"2023-08-27T20:50:40.471979+0000","last_unstale":"2023-08-27T20:50:40.957860+0000","last_undegraded":"2023-08-27T20:50:40.957860+0000","last_fullsized":"2023-08-27T20:50:40.957860+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:16:03.929633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.003535+0000","last_change":"2023-08-27T20:50:40.474630+0000","last_active":"2023-08-27T20:50:41.003535+0000","last_peered":"2023-08-27T20:50:41.003535+0000","last_clean":"2023-08-27T20:50:41.003535+0000","last_became_active":"2023-08-27T20:50:40.474345+0000","last_became_peered":"2023-08-27T20:50:40.474345+0000","last_unstale":"2023-08-27T20:50:41.003535+0000","last_undegraded":"2023-08-27T20:50:41.003535+0000","last_fullsized":"2023-08-27T20:50:41.003535+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:35:03.409338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004308+0000","last_change":"2023-08-27T20:50:41.004048+0000","last_active":"2023-08-27T20:50:41.004308+0000","last_peered":"2023-08-27T20:50:41.004308+0000","last_clean":"2023-08-27T20:50:41.004308+0000","last_became_active":"2023-08-27T20:50:41.003731+0000","last_became_peered":"2023-08-27T20:50:41.003731+0000","last_unstale":"2023-08-27T20:50:41.004308+0000","last_undegraded":"2023-08-27T20:50:41.004308+0000","last_fullsized":"2023-08-27T20:50:41.004308+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:58.339578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004423+0000","last_change":"2023-08-27T20:50:41.004114+0000","last_active":"2023-08-27T20:50:41.004423+0000","last_peered":"2023-08-27T20:50:41.004423+0000","last_clean":"2023-08-27T20:50:41.004423+0000","last_became_active":"2023-08-27T20:50:41.003717+0000","last_became_peered":"2023-08-27T20:50:41.003717+0000","last_unstale":"2023-08-27T20:50:41.004423+0000","last_undegraded":"2023-08-27T20:50:41.004423+0000","last_fullsized":"2023-08-27T20:50:41.004423+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:24:46.799377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"1.0","version":"58'128","reported_seq":270,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.070128+0000","last_change":"2023-08-27T20:50:22.534100+0000","last_active":"2023-08-27T20:50:41.070128+0000","last_peered":"2023-08-27T20:50:41.070128+0000","last_clean":"2023-08-27T20:50:41.070128+0000","last_became_active":"2023-08-27T20:50:22.533604+0000","last_became_peered":"2023-08-27T20:50:22.533604+0000","last_unstale":"2023-08-27T20:50:41.070128+0000","last_undegraded":"2023-08-27T20:50:41.070128+0000","last_fullsized":"2023-08-27T20:50:41.070128+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_clean_scrub_stamp":"2023-08-27T20:48:59.426866+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:50:03.653850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.725693+0000","last_change":"2023-08-27T20:50:40.472409+0000","last_active":"2023-08-27T20:50:41.725693+0000","last_peered":"2023-08-27T20:50:41.725693+0000","last_clean":"2023-08-27T20:50:41.725693+0000","last_became_active":"2023-08-27T20:50:40.472101+0000","last_became_peered":"2023-08-27T20:50:40.472101+0000","last_unstale":"2023-08-27T20:50:41.725693+0000","last_undegraded":"2023-08-27T20:50:41.725693+0000","last_fullsized":"2023-08-27T20:50:41.725693+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:37:42.899620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004529+0000","last_change":"2023-08-27T20:50:41.003113+0000","last_active":"2023-08-27T20:50:41.004529+0000","last_peered":"2023-08-27T20:50:41.004529+0000","last_clean":"2023-08-27T20:50:41.004529+0000","last_became_active":"2023-08-27T20:50:41.002820+0000","last_became_peered":"2023-08-27T20:50:41.002820+0000","last_unstale":"2023-08-27T20:50:41.004529+0000","last_undegraded":"2023-08-27T20:50:41.004529+0000","last_fullsized":"2023-08-27T20:50:41.004529+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:03:10.585918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.725708+0000","last_change":"2023-08-27T20:50:40.475080+0000","last_active":"2023-08-27T20:50:41.725708+0000","last_peered":"2023-08-27T20:50:41.725708+0000","last_clean":"2023-08-27T20:50:41.725708+0000","last_became_active":"2023-08-27T20:50:40.474840+0000","last_became_peered":"2023-08-27T20:50:40.474840+0000","last_unstale":"2023-08-27T20:50:41.725708+0000","last_undegraded":"2023-08-27T20:50:41.725708+0000","last_fullsized":"2023-08-27T20:50:41.725708+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:35:31.367035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":11,"reported_epoch":58,"state":"active+clean","last_fresh":"2023-08-27T20:50:41.004429+0000","last_change":"2023-08-27T20:50:41.004078+0000","last_active":"2023-08-27T20:50:41.004429+0000","last_peered":"2023-08-27T20:50:41.004429+0000","last_clean":"2023-08-27T20:50:41.004429+0000","last_became_active":"2023-08-27T20:50:41.003803+0000","last_became_peered":"2023-08-27T20:50:41.003803+0000","last_unstale":"2023-08-27T20:50:41.004429+0000","last_undegraded":"2023-08-27T20:50:41.004429+0000","last_fullsized":"2023-08-27T20:50:41.004429+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:14.109789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]}],"pool_stats":[{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3}],"osd_stats":[{"osd":3,"up_from":25,"seq":107374182424,"num_pgs":1,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5764,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739388,"statfs":{"total":95995035648,"available":95989133312,"internally_reserved":0,"allocated":585728,"data_stored":243504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.372,"5min":0.372,"15min":0.372},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"last":0.32400000000000001},{"interface":"front","average":{"1min":0.45700000000000002,"5min":0.45700000000000002,"15min":0.45700000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.628,"5min":0.628,"15min":0.628},"last":0.42799999999999999}]},{"osd":1,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.30399999999999999,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.38100000000000001},{"interface":"front","average":{"1min":0.44500000000000001,"5min":0.44500000000000001,"15min":0.44500000000000001},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.66800000000000004,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.46400000000000002}]},{"osd":2,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.495,"5min":0.495,"15min":0.495},"min":{"1min":0.35399999999999998,"5min":0.35399999999999998,"15min":0.35399999999999998},"max":{"1min":0.64600000000000002,"5min":0.64600000000000002,"15min":0.64600000000000002},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.314,"5min":0.314,"15min":0.314},"max":{"1min":0.59399999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.28999999999999998}]},{"osd":4,"last update":"Sun Aug 27 20:50:31 2023","interfaces":[{"interface":"back","average":{"1min":0.57699999999999996,"5min":0.57699999999999996,"15min":0.57699999999999996},"min":{"1min":0.43099999999999999,"5min":0.43099999999999999,"15min":0.43099999999999999},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.58999999999999997,"5min":0.58999999999999997,"15min":0.58999999999999997},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.78300000000000003,"5min":0.78300000000000003,"15min":0.78300000000000003},"last":0.49199999999999999}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.66100000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.65100000000000002}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.52600000000000002},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.623}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.67500000000000004}]}]},{"osd":2,"up_from":18,"seq":77309411357,"num_pgs":3,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6340,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5760,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":585728,"data_stored":243504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5898240},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.29999999999999999},{"interface":"front","average":{"1min":0.372,"5min":0.40600000000000003,"15min":0.41199999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.52500000000000002,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.26200000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.32100000000000001},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.34300000000000003}]},{"osd":3,"last update":"Sun Aug 27 20:50:16 2023","interfaces":[{"interface":"back","average":{"1min":0.47599999999999998,"5min":0.47599999999999998,"15min":0.47599999999999998},"min":{"1min":0.28399999999999997,"5min":0.28399999999999997,"15min":0.28399999999999997},"max":{"1min":0.67400000000000004,"5min":0.67400000000000004,"15min":0.67400000000000004},"last":0.38300000000000001},{"interface":"front","average":{"1min":0.498,"5min":0.498,"15min":0.498},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.59099999999999997,"5min":0.59099999999999997,"15min":0.59099999999999997},"last":0.373}]},{"osd":4,"last update":"Sun Aug 27 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.89600000000000002,"5min":0.89600000000000002,"15min":0.89600000000000002},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.61199999999999999,"5min":0.61199999999999999,"15min":0.61199999999999999},"min":{"1min":0.433,"5min":0.433,"15min":0.433},"max":{"1min":0.91800000000000004,"5min":0.91800000000000004,"15min":0.91800000000000004},"last":0.41799999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.438}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.40500000000000003}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.50600000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574880,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6404,"kb_used_data":572,"kb_used_omap":0,"kb_used_meta":5824,"kb_avail":93738748,"statfs":{"total":95995035648,"available":95988477952,"internally_reserved":0,"allocated":585728,"data_stored":243504,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5963776},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:43 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.35499999999999998,"15min":0.35299999999999998},"min":{"1min":0.23999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.377,"15min":0.36499999999999999},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38900000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:01 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.59799999999999998,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35099999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.45300000000000001,"15min":0.46300000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54400000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.50600000000000001}]},{"osd":3,"last update":"Sun Aug 27 20:50:17 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.67800000000000005,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.44},{"interface":"front","average":{"1min":0.45800000000000002,"5min":0.45800000000000002,"15min":0.45800000000000002},"min":{"1min":0.312,"5min":0.312,"15min":0.312},"max":{"1min":0.63400000000000001,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.40899999999999997}]},{"osd":4,"last update":"Sun Aug 27 20:50:33 2023","interfaces":[{"interface":"back","average":{"1min":0.59599999999999997,"5min":0.59599999999999997,"15min":0.59599999999999997},"min":{"1min":0.40200000000000002,"5min":0.40200000000000002,"15min":0.40200000000000002},"max":{"1min":0.78600000000000003,"5min":0.78600000000000003,"15min":0.78600000000000003},"last":0.64700000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.45300000000000001,"5min":0.45300000000000001,"15min":0.45300000000000001},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.61399999999999999}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.60299999999999998},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.54600000000000004}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.621}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.66900000000000004}]}]},{"osd":0,"up_from":8,"seq":34359738403,"num_pgs":5,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7048,"kb_used_data":1152,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738104,"statfs":{"total":95995035648,"available":95987818496,"internally_reserved":0,"allocated":1179648,"data_stored":833872,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 20:50:43 2023","interfaces":[{"interface":"back","average":{"1min":0.35199999999999998,"5min":0.35599999999999998,"15min":0.35699999999999998},"min":{"1min":0.218,"5min":0.218,"15min":0.218},"max":{"1min":0.49099999999999999,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.252},{"interface":"front","average":{"1min":0.40400000000000003,"5min":0.39900000000000002,"15min":0.39800000000000002},"min":{"1min":0.25700000000000001,"5min":0.224,"15min":0.224},"max":{"1min":0.63,"5min":0.63,"15min":0.63},"last":0.39000000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.41599999999999998,"15min":0.42399999999999999},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.34000000000000002},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40999999999999998,"15min":0.41699999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.27500000000000002}]},{"osd":3,"last update":"Sun Aug 27 20:50:15 2023","interfaces":[{"interface":"back","average":{"1min":0.48099999999999998,"5min":0.48099999999999998,"15min":0.48099999999999998},"min":{"1min":0.33800000000000002,"5min":0.33800000000000002,"15min":0.33800000000000002},"max":{"1min":0.65500000000000003,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.46200000000000002,"15min":0.46200000000000002},"min":{"1min":0.32000000000000001,"5min":0.32000000000000001,"15min":0.32000000000000001},"max":{"1min":0.57299999999999995,"5min":0.57299999999999995,"15min":0.57299999999999995},"last":0.318}]},{"osd":4,"last update":"Sun Aug 27 20:50:34 2023","interfaces":[{"interface":"back","average":{"1min":0.57499999999999996,"5min":0.57499999999999996,"15min":0.57499999999999996},"min":{"1min":0.42799999999999999,"5min":0.42799999999999999,"15min":0.42799999999999999},"max":{"1min":0.69299999999999995,"5min":0.69299999999999995,"15min":0.69299999999999995},"last":0.372},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.58599999999999997,"15min":0.58599999999999997},"min":{"1min":0.442,"5min":0.442,"15min":0.442},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.54600000000000004}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.46600000000000003}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.505}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47799999999999998}]}]},{"osd":7,"up_from":51,"seq":219043332108,"num_pgs":6,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6276,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":5120,"kb_avail":93738876,"statfs":{"total":95995035648,"available":95988609024,"internally_reserved":0,"allocated":1175552,"data_stored":833549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5242880},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.55900000000000005}]},{"osd":1,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.626}]},{"osd":2,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47899999999999998}]},{"osd":3,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.53200000000000003}]},{"osd":4,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.372}]},{"osd":5,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.442}]},{"osd":6,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.46100000000000002}]}]},{"osd":4,"up_from":30,"seq":128849018901,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5824,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5248,"kb_avail":93739328,"statfs":{"total":95995035648,"available":95989071872,"internally_reserved":0,"allocated":581632,"data_stored":243239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5373952},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.56200000000000006,"5min":0.56200000000000006,"15min":0.56200000000000006},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.997,"5min":0.997,"15min":0.997},"last":0.38},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53500000000000003,"15min":0.53500000000000003},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.76000000000000001,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.42999999999999999}]},{"osd":1,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.41699999999999998,"5min":0.41699999999999998,"15min":0.41699999999999998},"max":{"1min":0.96299999999999997,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.42199999999999999},{"interface":"front","average":{"1min":0.54700000000000004,"5min":0.54700000000000004,"15min":0.54700000000000004},"min":{"1min":0.38900000000000001,"5min":0.38900000000000001,"15min":0.38900000000000001},"max":{"1min":0.71999999999999997,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.498}]},{"osd":2,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.54800000000000004,"15min":0.54800000000000004},"min":{"1min":0.42599999999999999,"5min":0.42599999999999999,"15min":0.42599999999999999},"max":{"1min":0.72299999999999998,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.39000000000000001},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.54100000000000004,"15min":0.54100000000000004},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.44700000000000001}]},{"osd":3,"last update":"Sun Aug 27 20:50:35 2023","interfaces":[{"interface":"back","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.36499999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.0089999999999999,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.38700000000000001,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.83399999999999996,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46100000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.26800000000000002},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.28100000000000003}]},{"osd":6,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.40600000000000003},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.35099999999999998}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.47399999999999998}]}]},{"osd":5,"up_from":37,"seq":158913789971,"num_pgs":2,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":5760,"kb_used_data":568,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93739392,"statfs":{"total":95995035648,"available":95989137408,"internally_reserved":0,"allocated":581632,"data_stored":243239,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.41599999999999998},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54700000000000004}]},{"osd":1,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.45600000000000002},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.502}]},{"osd":2,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.53400000000000003},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.40400000000000003}]},{"osd":3,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.51800000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.42999999999999999},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.373}]},{"osd":6,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.57999999999999996},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.48599999999999999}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.58999999999999997}]}]},{"osd":6,"up_from":44,"seq":188978561040,"num_pgs":4,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6340,"kb_used_data":1148,"kb_used_omap":0,"kb_used_meta":5184,"kb_avail":93738812,"statfs":{"total":95995035648,"available":95988543488,"internally_reserved":0,"allocated":1175552,"data_stored":833549,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5308416},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.41199999999999998}]},{"osd":1,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.583,"5min":1.583,"15min":1.583},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.438}]},{"osd":2,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.613,"5min":3.613,"15min":3.613},"last":0.40300000000000002},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.9009999999999998,"5min":5.9009999999999998,"15min":5.9009999999999998},"last":0.42899999999999999}]},{"osd":3,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":3.7389999999999999,"5min":3.7389999999999999,"15min":3.7389999999999999},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.34000000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.28599999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.31900000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.45700000000000002}]},{"osd":7,"last update":"Thu Jan 1 00:00:00 1970","interfaces":[{"interface":"back","average":{"1min":0,"5min":0,"15min":0},"min":{"1min":0,"5min":0,"15min":0},"max":{"1min":0,"5min":0,"15min":0},"last":0.42099999999999999}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T20:51:12.391 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-27T20:51:12.391 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-08-27T20:51:12.391 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-08-27T20:51:12.392 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph health --format=json 2023-08-27T20:51:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:12 smithi136 ceph-mon[113531]: pgmap v19: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:12 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:12 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:12 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:12 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:12 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:12 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:12.719 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:13 smithi136 ceph-mon[113531]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:13 smithi093 ceph-mon[111335]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:13 smithi093 ceph-mon[118902]: from='client.24521 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:14.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:14 smithi136 ceph-mon[113531]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:14.280 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:14.280 INFO:teuthology.orchestra.run.smithi093.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-08-27T20:51:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:14 smithi093 ceph-mon[111335]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:14 smithi093 ceph-mon[118902]: pgmap v20: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:14.755 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-08-27T20:51:14.756 INFO:tasks.cephadm:Setup complete, yielding 2023-08-27T20:51:14.756 INFO:teuthology.run_tasks:Running task cephadm.shell... 2023-08-27T20:51:14.764 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi093.front.sepia.ceph.com 2023-08-27T20:51:14.765 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- bash -c 'ceph fs volume create foo' 2023-08-27T20:51:15.086 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:15 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:15 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/870836900' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T20:51:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:15 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:15 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/870836900' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T20:51:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:15 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:15 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/870836900' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: Reconfiguring alertmanager.a (dependencies changed)... 2023-08-27T20:51:16.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: Reconfiguring daemon alertmanager.a on smithi093 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: Reconfiguring alertmanager.a (dependencies changed)... 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: Reconfiguring daemon alertmanager.a on smithi093 2023-08-27T20:51:16.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:16 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: pgmap v21: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: Reconfiguring alertmanager.a (dependencies changed)... 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: Reconfiguring daemon alertmanager.a on smithi093 2023-08-27T20:51:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:16 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:17 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T20:51:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:17 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T20:51:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:17 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T20:51:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:17 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T20:51:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:17 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T20:51:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:17 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]: dispatch 2023-08-27T20:51:17.649 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:17 smithi093 systemd[1]: Stopping Ceph alertmanager.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:51:17.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:17 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[142406]: ts=2023-08-27T20:51:17.649Z caller=main.go:583 level=info msg="Received SIGTERM, exiting gracefully..." 2023-08-27T20:51:17.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:17 smithi093 bash[146168]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a 2023-08-27T20:51:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:18 smithi136 ceph-mon[113531]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:51:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:18 smithi136 ceph-mon[113531]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:18 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-27T20:51:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:18 smithi136 ceph-mon[113531]: osdmap e60: 8 total, 8 up, 8 in 2023-08-27T20:51:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:18 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:18 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:18.461 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[111335]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:51:18.461 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[111335]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[111335]: osdmap e60: 8 total, 8 up, 8 in 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mon-a[111331]: 2023-08-27T20:51:18.109+0000 7f169df4e700 -1 log_channel(cluster) log [ERR] : Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[118902]: from='client.24533 -' entity='client.admin' cmd=[{"prefix": "fs volume create", "name": "foo", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[118902]: pgmap v22: 9 pgs: 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.meta"}]': finished 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[118902]: osdmap e60: 8 total, 8 up, 8 in 2023-08-27T20:51:18.462 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:18.463 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:18.483 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim false --force' 2023-08-27T20:51:18.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 systemd[1]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@alertmanager.a.service: Succeeded. 2023-08-27T20:51:18.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 systemd[1]: Stopped Ceph alertmanager.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:51:18.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 systemd[1]: Starting Ceph alertmanager.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:51:18.969 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:19.135 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.875Z caller=main.go:240 level=info msg="Starting Alertmanager" version="(version=0.25.0, branch=HEAD, revision=258fab7cdd551f2cf251ed0348f0ad7289aee789)" 2023-08-27T20:51:19.135 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.875Z caller=main.go:241 level=info build_context="(go=go1.19.4, user=root@abe866dd5717, date=20221222-14:51:36)" 2023-08-27T20:51:19.135 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.876Z caller=cluster.go:185 level=info component=cluster msg="setting advertise address explicitly" addr=172.21.15.93 port=9094 2023-08-27T20:51:19.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.877Z caller=cluster.go:681 level=info component=cluster msg="Waiting for gossip to settle..." interval=2s 2023-08-27T20:51:19.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.901Z caller=coordinator.go:113 level=info component=configuration msg="Loading configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-27T20:51:19.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.901Z caller=coordinator.go:126 level=info component=configuration msg="Completed loading of configuration file" file=/etc/alertmanager/alertmanager.yml 2023-08-27T20:51:19.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.903Z caller=tls_config.go:232 level=info msg="Listening on" address=[::]:9093 2023-08-27T20:51:19.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:18.903Z caller=tls_config.go:235 level=info msg="TLS is disabled." http2=false address=[::]:9093 2023-08-27T20:51:19.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 bash[146318]: 09806d7189457e037893651f6ef6fa489e4b3d9581ffef6eddff219847f011a8 2023-08-27T20:51:19.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:18 smithi093 systemd[1]: Started Ceph alertmanager.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:51:19.136 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-27T20:51:19.136 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: osdmap e61: 8 total, 8 up, 8 in 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: osdmap e62: 8 total, 8 up, 8 in 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: fsmap foo:0 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: Saving service mds.foo spec with placement count:2 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: osdmap e61: 8 total, 8 up, 8 in 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: osdmap e62: 8 total, 8 up, 8 in 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: fsmap foo:0 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: Saving service mds.foo spec with placement count:2 2023-08-27T20:51:19.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.139 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.139 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:19 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pool create", "pool": "cephfs.foo.data"}]': finished 2023-08-27T20:51:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: osdmap e61: 8 total, 8 up, 8 in 2023-08-27T20:51:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]: dispatch 2023-08-27T20:51:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: Health check failed: 1 filesystem is offline (MDS_ALL_DOWN) 2023-08-27T20:51:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: Health check failed: 1 filesystem is online with fewer MDS than max_mds (MDS_UP_LESS_THAN_MAX) 2023-08-27T20:51:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: osdmap e62: 8 total, 8 up, 8 in 2023-08-27T20:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "fs new", "fs_name": "foo", "metadata": "cephfs.foo.meta", "data": "cephfs.foo.data"}]': finished 2023-08-27T20:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: fsmap foo:0 2023-08-27T20:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: Saving service mds.foo spec with placement count:2 2023-08-27T20:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:19 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:20.358 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-mon[113531]: pgmap v26: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:20.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-mon[113531]: Reconfiguring prometheus.a (dependencies changed)... 2023-08-27T20:51:20.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-mon[113531]: Reconfiguring daemon prometheus.a on smithi136 2023-08-27T20:51:20.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-mon[113531]: osdmap e63: 8 total, 8 up, 8 in 2023-08-27T20:51:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[111335]: pgmap v26: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[111335]: Reconfiguring prometheus.a (dependencies changed)... 2023-08-27T20:51:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[111335]: Reconfiguring daemon prometheus.a on smithi136 2023-08-27T20:51:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[111335]: osdmap e63: 8 total, 8 up, 8 in 2023-08-27T20:51:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[118902]: pgmap v26: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[118902]: Reconfiguring prometheus.a (dependencies changed)... 2023-08-27T20:51:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[118902]: Reconfiguring daemon prometheus.a on smithi136 2023-08-27T20:51:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-mon[118902]: osdmap e63: 8 total, 8 up, 8 in 2023-08-27T20:51:20.688 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- bash -c 'ceph config set mon mon_warn_on_insecure_global_id_reclaim_allowed false --force' 2023-08-27T20:51:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:51:20.689 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 systemd[1]: Stopping Ceph prometheus.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:51:20.952 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:20.878Z caller=cluster.go:706 level=info component=cluster msg="gossip not settled" polls=0 before=0 now=1 elapsed=2.000542367s 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=main.go:840 level=warn msg="Received SIGTERM, exiting gracefully..." 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=main.go:864 level=info msg="Stopping scrape discovery manager..." 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=main.go:878 level=info msg="Stopping notify discovery manager..." 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=manager.go:988 level=info component="rule manager" msg="Stopping rule manager..." 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=main.go:860 level=info msg="Scrape discovery manager stopped" 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=main.go:874 level=info msg="Notify discovery manager stopped" 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=manager.go:998 level=info component="rule manager" msg="Rule manager stopped" 2023-08-27T20:51:21.184 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=main.go:915 level=info msg="Stopping scrape manager..." 2023-08-27T20:51:21.185 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.736Z caller=main.go:907 level=info msg="Scrape manager stopped" 2023-08-27T20:51:21.185 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.737Z caller=notifier.go:606 level=info component=notifier msg="Stopping notification manager..." 2023-08-27T20:51:21.185 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.737Z caller=main.go:1135 level=info msg="Notifier manager stopped" 2023-08-27T20:51:21.185 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[134377]: ts=2023-08-27T20:51:20.737Z caller=main.go:1147 level=info msg="See you next time!" 2023-08-27T20:51:21.185 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:20 smithi136 bash[137708]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a 2023-08-27T20:51:21.438 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 systemd[1]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@prometheus.a.service: Succeeded. 2023-08-27T20:51:21.438 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 systemd[1]: Stopped Ceph prometheus.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:51:21.439 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 systemd[1]: Starting Ceph prometheus.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:51:21.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STOPPING 2023-08-27T20:51:21.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T20:51:21.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STOPPED 2023-08-27T20:51:21.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STARTING 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Serving on http://:::9283 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STARTED 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STOPPING 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STOPPED 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STARTING 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Serving on http://:::9283 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STARTED 2023-08-27T20:51:21.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STOPPING 2023-08-27T20:51:21.939 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.453Z caller=main.go:564 level=info msg="Starting Prometheus Server" mode=server version="(version=2.43.0, branch=HEAD, revision=edfc3bcd025dd6fe296c167a14a216cab1e552ee)" 2023-08-27T20:51:21.939 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.453Z caller=main.go:569 level=info build_context="(go=go1.19.7, platform=linux/amd64, user=root@8a0ee342e522, date=20230321-12:56:07, tags=netgo,builtinassets)" 2023-08-27T20:51:21.939 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.453Z caller=main.go:570 level=info host_details="(Linux 4.18.0-512.el8.x86_64 #1 SMP Tue Aug 22 16:03:20 UTC 2023 x86_64 smithi136 (none))" 2023-08-27T20:51:21.939 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.453Z caller=main.go:571 level=info fd_limits="(soft=1048576, hard=1048576)" 2023-08-27T20:51:21.939 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.453Z caller=main.go:572 level=info vm_limits="(soft=unlimited, hard=unlimited)" 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.454Z caller=web.go:561 level=info component=web msg="Start listening for connections" address=:9095 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.455Z caller=main.go:1005 level=info msg="Starting TSDB ..." 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 bash[137817]: cf83aa0a04c6befc74240041e9fb089d07c1e7c244cb5eb48b40a9c9ca446271 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.457Z caller=tls_config.go:232 level=info component=web msg="Listening on" address=[::]:9095 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.457Z caller=tls_config.go:235 level=info component=web msg="TLS is disabled." http2=false address=[::]:9095 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 systemd[1]: Started Ceph prometheus.a for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.459Z caller=head.go:587 level=info component=tsdb msg="Replaying on-disk memory mappable chunks if any" 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.459Z caller=head.go:658 level=info component=tsdb msg="On-disk memory mappable chunks replay completed" duration=2.317µs 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.459Z caller=head.go:664 level=info component=tsdb msg="Replaying WAL, this may take a while" 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.459Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=0 maxSegment=1 2023-08-27T20:51:21.940 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.460Z caller=head.go:735 level=info component=tsdb msg="WAL segment loaded" segment=1 maxSegment=1 2023-08-27T20:51:21.941 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.460Z caller=head.go:772 level=info component=tsdb msg="WAL replay completed" checkpoint_replay_duration=30.329µs wal_replay_duration=938.643µs wbl_replay_duration=445ns total_replay_duration=1.014248ms 2023-08-27T20:51:21.941 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.461Z caller=main.go:1026 level=info fs_type=XFS_SUPER_MAGIC 2023-08-27T20:51:21.941 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.461Z caller=main.go:1029 level=info msg="TSDB started" 2023-08-27T20:51:21.941 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.461Z caller=main.go:1209 level=info msg="Loading configuration file" filename=/etc/prometheus/prometheus.yml 2023-08-27T20:51:21.941 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.476Z caller=main.go:1246 level=info msg="Completed loading of configuration file" filename=/etc/prometheus/prometheus.yml totalDuration=15.78683ms db_storage=1.18µs remote_storage=1.289µs web_handler=442ns query_engine=563ns scrape=499.299µs scrape_sd=23.085µs notify=27.626µs notify_sd=7.1µs rules=15.015918ms tracing=7.464µs 2023-08-27T20:51:21.941 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.477Z caller=main.go:990 level=info msg="Server is ready to receive web requests." 2023-08-27T20:51:21.941 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 20:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T20:51:21.477Z caller=manager.go:974 level=info component="rule manager" msg="Starting rule manager..." 2023-08-27T20:51:22.119 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T20:51:22.119 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STOPPED 2023-08-27T20:51:22.119 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STARTING 2023-08-27T20:51:22.119 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Serving on http://:::9283 2023-08-27T20:51:22.119 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:21] ENGINE Bus STARTED 2023-08-27T20:51:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: pgmap v28: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi093.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi136.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi136.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:22 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: pgmap v28: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T20:51:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi093.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T20:51:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T20:51:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi136.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T20:51:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T20:51:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi136.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T20:51:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: pgmap v28: 73 pgs: 64 unknown, 9 active+clean; 577 KiB data, 49 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T20:51:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi093.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T20:51:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T20:51:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi136.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T20:51:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T20:51:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi136.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T20:51:22.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:22.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:22 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:22.622 INFO:teuthology.run_tasks:Running task ceph.healthy... 2023-08-27T20:51:22.631 INFO:tasks.ceph:Waiting until ceph daemons up and pgs clean... 2023-08-27T20:51:22.631 INFO:tasks.cephadm.ceph_manager.ceph:waiting for mgr available 2023-08-27T20:51:22.632 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph mgr dump --format=json 2023-08-27T20:51:22.921 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:23.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:23 smithi136 ceph-mon[113531]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T20:51:23.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:23 smithi136 ceph-mon[113531]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi093.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T20:51:23.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:23 smithi136 ceph-mon[113531]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T20:51:23.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:23 smithi136 ceph-mon[113531]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi136.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T20:51:23.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:23 smithi136 ceph-mon[113531]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T20:51:23.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:23 smithi136 ceph-mon[113531]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi136.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T20:51:23.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[111335]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T20:51:23.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[111335]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi093.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T20:51:23.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[111335]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T20:51:23.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[111335]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi136.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T20:51:23.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[111335]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T20:51:23.313 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[111335]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi136.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T20:51:23.313 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[118902]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-alertmanager-api-host"}]: dispatch 2023-08-27T20:51:23.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[118902]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-alertmanager-api-host", "value": "http://smithi093.front.sepia.ceph.com:9093"}]: dispatch 2023-08-27T20:51:23.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[118902]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-prometheus-api-host"}]: dispatch 2023-08-27T20:51:23.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[118902]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-prometheus-api-host", "value": "http://smithi136.front.sepia.ceph.com:9095"}]: dispatch 2023-08-27T20:51:23.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[118902]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard get-grafana-api-url"}]: dispatch 2023-08-27T20:51:23.314 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:23 smithi093 ceph-mon[118902]: from='mon.? -' entity='mon.' cmd=[{"prefix": "dashboard set-grafana-api-url", "value": "https://smithi136.front.sepia.ceph.com:3000"}]: dispatch 2023-08-27T20:51:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:24 smithi136 ceph-mon[113531]: pgmap v29: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:24 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:24 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:24 smithi093 ceph-mon[111335]: pgmap v29: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:24 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:24 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:24 smithi093 ceph-mon[118902]: pgmap v29: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:24 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:24 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:24.550 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:24.962 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":19,"active_gid":14460,"active_name":"y","active_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6800","nonce":4025849521},{"type":"v1","addr":"172.21.15.93:6801","nonce":4025849521}]},"active_addr":"172.21.15.93:6801/4025849521","active_change":"2023-08-27T20:50:40.950772+0000","active_mgr_features":4540138320759226367,"available":true,"standbys":[{"gid":14466,"name":"x","mgr_features":4540138320759226367,"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}]}],"modules":["cephadm","dashboard","iostat","nfs","prometheus","restful"],"available_modules":[{"name":"alerts","can_run":true,"error_string":"","module_options":{"interval":{"name":"interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"How frequently to reexamine health status","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"smtp_destination":{"name":"smtp_destination","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Email address to send alerts to","long_desc":"","tags":[],"see_also":[]},"smtp_from_name":{"name":"smtp_from_name","type":"str","level":"advanced","flags":1,"default_value":"Ceph","min":"","max":"","enum_allowed":[],"desc":"Email From: name","long_desc":"","tags":[],"see_also":[]},"smtp_host":{"name":"smtp_host","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_password":{"name":"smtp_password","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Password to authenticate with","long_desc":"","tags":[],"see_also":[]},"smtp_port":{"name":"smtp_port","type":"int","level":"advanced","flags":1,"default_value":"465","min":"","max":"","enum_allowed":[],"desc":"SMTP port","long_desc":"","tags":[],"see_also":[]},"smtp_sender":{"name":"smtp_sender","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"SMTP envelope sender","long_desc":"","tags":[],"see_also":[]},"smtp_ssl":{"name":"smtp_ssl","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Use SSL to connect to SMTP server","long_desc":"","tags":[],"see_also":[]},"smtp_user":{"name":"smtp_user","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"User to authenticate as","long_desc":"","tags":[],"see_also":[]}}},{"name":"balancer","can_run":true,"error_string":"","module_options":{"active":{"name":"active","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"automatically balance PGs across cluster","long_desc":"","tags":[],"see_also":[]},"begin_time":{"name":"begin_time","type":"str","level":"advanced","flags":1,"default_value":"0000","min":"","max":"","enum_allowed":[],"desc":"beginning time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"begin_weekday":{"name":"begin_weekday","type":"uint","level":"advanced","flags":1,"default_value":"0","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to this day of the week or later","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"crush_compat_max_iterations":{"name":"crush_compat_max_iterations","type":"uint","level":"advanced","flags":1,"default_value":"25","min":"1","max":"250","enum_allowed":[],"desc":"maximum number of iterations to attempt optimization","long_desc":"","tags":[],"see_also":[]},"crush_compat_metrics":{"name":"crush_compat_metrics","type":"str","level":"advanced","flags":1,"default_value":"pgs,objects,bytes","min":"","max":"","enum_allowed":[],"desc":"metrics with which to calculate OSD utilization","long_desc":"Value is a list of one or more of \"pgs\", \"objects\", or \"bytes\", and indicates which metrics to use to balance utilization.","tags":[],"see_also":[]},"crush_compat_step":{"name":"crush_compat_step","type":"float","level":"advanced","flags":1,"default_value":"0.5","min":"0.001","max":"0.999","enum_allowed":[],"desc":"aggressiveness of optimization","long_desc":".99 is very aggressive, .01 is less aggressive","tags":[],"see_also":[]},"end_time":{"name":"end_time","type":"str","level":"advanced","flags":1,"default_value":"2400","min":"","max":"","enum_allowed":[],"desc":"ending time of day to automatically balance","long_desc":"This is a time of day in the format HHMM.","tags":[],"see_also":[]},"end_weekday":{"name":"end_weekday","type":"uint","level":"advanced","flags":1,"default_value":"7","min":"0","max":"7","enum_allowed":[],"desc":"Restrict automatic balancing to days of the week earlier than this","long_desc":"0 or 7 = Sunday, 1 = Monday, etc.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_score":{"name":"min_score","type":"float","level":"advanced","flags":1,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"minimum score, below which no optimization is attempted","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":1,"default_value":"upmap","min":"","max":"","enum_allowed":["crush-compat","none","upmap"],"desc":"Balancer mode","long_desc":"","tags":[],"see_also":[]},"pool_ids":{"name":"pool_ids","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"pools which the automatic balancing will be limited to","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and attempt optimization","long_desc":"","tags":[],"see_also":[]},"upmap_max_deviation":{"name":"upmap_max_deviation","type":"int","level":"advanced","flags":1,"default_value":"5","min":"1","max":"","enum_allowed":[],"desc":"deviation below which no optimization is attempted","long_desc":"If the number of PGs are within this count then no optimization is attempted","tags":[],"see_also":[]},"upmap_max_optimizations":{"name":"upmap_max_optimizations","type":"uint","level":"advanced","flags":1,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"maximum upmap optimizations to make per attempt","long_desc":"","tags":[],"see_also":[]}}},{"name":"cephadm","can_run":true,"error_string":"","module_options":{"agent_down_multiplier":{"name":"agent_down_multiplier","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"","max":"","enum_allowed":[],"desc":"Multiplied by agent refresh rate to calculate how long agent must not report before being marked down","long_desc":"","tags":[],"see_also":[]},"agent_refresh_rate":{"name":"agent_refresh_rate","type":"secs","level":"advanced","flags":0,"default_value":"20","min":"","max":"","enum_allowed":[],"desc":"How often agent on each host will try to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"agent_starting_port":{"name":"agent_starting_port","type":"int","level":"advanced","flags":0,"default_value":"4721","min":"","max":"","enum_allowed":[],"desc":"First port agent will try to bind to (will also try up to next 1000 subsequent ports if blocked)","long_desc":"","tags":[],"see_also":[]},"allow_ptrace":{"name":"allow_ptrace","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow SYS_PTRACE capability on ceph containers","long_desc":"The SYS_PTRACE capability is needed to attach to a process with gdb or strace. Enabling this options can allow debugging daemons that encounter problems at runtime.","tags":[],"see_also":[]},"autotune_interval":{"name":"autotune_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to autotune daemon memory","long_desc":"","tags":[],"see_also":[]},"autotune_memory_target_ratio":{"name":"autotune_memory_target_ratio","type":"float","level":"advanced","flags":0,"default_value":"0.7","min":"","max":"","enum_allowed":[],"desc":"ratio of total system memory to divide amongst autotuned daemons","long_desc":"","tags":[],"see_also":[]},"cgroups_split":{"name":"cgroups_split","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Pass --cgroups=split when cephadm creates containers (currently podman only)","long_desc":"","tags":[],"see_also":[]},"config_checks_enabled":{"name":"config_checks_enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable or disable the cephadm configuration analysis","long_desc":"","tags":[],"see_also":[]},"config_dashboard":{"name":"config_dashboard","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"manage configs like API endpoints in Dashboard.","long_desc":"","tags":[],"see_also":[]},"container_image_alertmanager":{"name":"container_image_alertmanager","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/alertmanager:v0.25.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_base":{"name":"container_image_base","type":"str","level":"advanced","flags":1,"default_value":"quay.io/ceph/ceph","min":"","max":"","enum_allowed":[],"desc":"Container image name, without the tag","long_desc":"","tags":[],"see_also":[]},"container_image_grafana":{"name":"container_image_grafana","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/ceph-grafana:9.4.7","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_haproxy":{"name":"container_image_haproxy","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/haproxy:2.3","min":"","max":"","enum_allowed":[],"desc":"HAproxy container image","long_desc":"","tags":[],"see_also":[]},"container_image_keepalived":{"name":"container_image_keepalived","type":"str","level":"advanced","flags":0,"default_value":"quay.io/ceph/keepalived:2.1.5","min":"","max":"","enum_allowed":[],"desc":"Keepalived container image","long_desc":"","tags":[],"see_also":[]},"container_image_loki":{"name":"container_image_loki","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/loki:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Loki container image","long_desc":"","tags":[],"see_also":[]},"container_image_node_exporter":{"name":"container_image_node_exporter","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/node-exporter:v1.5.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_prometheus":{"name":"container_image_prometheus","type":"str","level":"advanced","flags":0,"default_value":"quay.io/prometheus/prometheus:v2.43.0","min":"","max":"","enum_allowed":[],"desc":"Prometheus container image","long_desc":"","tags":[],"see_also":[]},"container_image_promtail":{"name":"container_image_promtail","type":"str","level":"advanced","flags":0,"default_value":"docker.io/grafana/promtail:2.4.0","min":"","max":"","enum_allowed":[],"desc":"Promtail container image","long_desc":"","tags":[],"see_also":[]},"container_image_snmp_gateway":{"name":"container_image_snmp_gateway","type":"str","level":"advanced","flags":0,"default_value":"docker.io/maxwo/snmp-notifier:v1.2.1","min":"","max":"","enum_allowed":[],"desc":"SNMP Gateway container image","long_desc":"","tags":[],"see_also":[]},"container_init":{"name":"container_init","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Run podman/docker with `--init`","long_desc":"","tags":[],"see_also":[]},"daemon_cache_timeout":{"name":"daemon_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"seconds to cache service (daemon) inventory","long_desc":"","tags":[],"see_also":[]},"default_cephadm_command_timeout":{"name":"default_cephadm_command_timeout","type":"secs","level":"advanced","flags":0,"default_value":"900","min":"","max":"","enum_allowed":[],"desc":"Default timeout applied to cephadm commands run directly on the host (in seconds)","long_desc":"","tags":[],"see_also":[]},"default_registry":{"name":"default_registry","type":"str","level":"advanced","flags":0,"default_value":"docker.io","min":"","max":"","enum_allowed":[],"desc":"Search-registry to which we should normalize unqualified image names. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"device_cache_timeout":{"name":"device_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"1800","min":"","max":"","enum_allowed":[],"desc":"seconds to cache device inventory","long_desc":"","tags":[],"see_also":[]},"device_enhanced_scan":{"name":"device_enhanced_scan","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use libstoragemgmt during device scans","long_desc":"","tags":[],"see_also":[]},"facts_cache_timeout":{"name":"facts_cache_timeout","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"seconds to cache host facts data","long_desc":"","tags":[],"see_also":[]},"host_check_interval":{"name":"host_check_interval","type":"secs","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to perform a host check","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_refresh_metadata":{"name":"log_refresh_metadata","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Log all refresh metadata. Includes daemon, device, and host info collected regularly. Only has effect if logging at debug level","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"log to the \"cephadm\" cluster log channel\"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf":{"name":"manage_etc_ceph_ceph_conf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Manage and own /etc/ceph/ceph.conf on the hosts.","long_desc":"","tags":[],"see_also":[]},"manage_etc_ceph_ceph_conf_hosts":{"name":"manage_etc_ceph_ceph_conf_hosts","type":"str","level":"advanced","flags":0,"default_value":"*","min":"","max":"","enum_allowed":[],"desc":"PlacementSpec describing on which hosts to manage /etc/ceph/ceph.conf","long_desc":"","tags":[],"see_also":[]},"max_count_per_host":{"name":"max_count_per_host","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of daemons per service per host","long_desc":"","tags":[],"see_also":[]},"max_osd_draining_count":{"name":"max_osd_draining_count","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"max number of osds that will be drained simultaneously when osds are removed","long_desc":"","tags":[],"see_also":[]},"migration_current":{"name":"migration_current","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"internal - do not modify","long_desc":"","tags":[],"see_also":[]},"mode":{"name":"mode","type":"str","level":"advanced","flags":0,"default_value":"root","min":"","max":"","enum_allowed":["cephadm-package","root"],"desc":"mode for remote execution of cephadm","long_desc":"","tags":[],"see_also":[]},"prometheus_alerts_path":{"name":"prometheus_alerts_path","type":"str","level":"advanced","flags":0,"default_value":"/etc/prometheus/ceph/ceph_default_alerts.yml","min":"","max":"","enum_allowed":[],"desc":"location of alerts to include in prometheus deployments","long_desc":"","tags":[],"see_also":[]},"registry_insecure":{"name":"registry_insecure","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Registry is to be considered insecure (no TLS available). Only for development purposes.","long_desc":"","tags":[],"see_also":[]},"registry_password":{"name":"registry_password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository password. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"registry_url":{"name":"registry_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Registry url for login purposes. This is not the default registry","long_desc":"","tags":[],"see_also":[]},"registry_username":{"name":"registry_username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"Custom repository username. Only used for logging into a registry.","long_desc":"","tags":[],"see_also":[]},"ssh_config_file":{"name":"ssh_config_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"customized SSH config file to connect to managed hosts","long_desc":"","tags":[],"see_also":[]},"use_agent":{"name":"use_agent","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Use cephadm agent on each host to gather and send metadata","long_desc":"","tags":[],"see_also":[]},"use_repo_digest":{"name":"use_repo_digest","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Automatically convert image tags to image digest. Make sure all daemons use the same image","long_desc":"","tags":[],"see_also":[]},"warn_on_failed_host_check":{"name":"warn_on_failed_host_check","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if the host check fails","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_daemons":{"name":"warn_on_stray_daemons","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected that are not managed by cephadm","long_desc":"","tags":[],"see_also":[]},"warn_on_stray_hosts":{"name":"warn_on_stray_hosts","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"raise a health warning if daemons are detected on a host that is not managed by cephadm","long_desc":"","tags":[],"see_also":[]}}},{"name":"crash","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"retain_interval":{"name":"retain_interval","type":"secs","level":"advanced","flags":1,"default_value":"31536000","min":"","max":"","enum_allowed":[],"desc":"how long to retain crashes before pruning them","long_desc":"","tags":[],"see_also":[]},"warn_recent_interval":{"name":"warn_recent_interval","type":"secs","level":"advanced","flags":1,"default_value":"1209600","min":"","max":"","enum_allowed":[],"desc":"time interval in which to warn about recent crashes","long_desc":"","tags":[],"see_also":[]}}},{"name":"dashboard","can_run":true,"error_string":"","module_options":{"ACCOUNT_LOCKOUT_ATTEMPTS":{"name":"ACCOUNT_LOCKOUT_ATTEMPTS","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_HOST":{"name":"ALERTMANAGER_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ALERTMANAGER_API_SSL_VERIFY":{"name":"ALERTMANAGER_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_ENABLED":{"name":"AUDIT_API_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"AUDIT_API_LOG_PAYLOAD":{"name":"AUDIT_API_LOG_PAYLOAD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ENABLE_BROWSABLE_API":{"name":"ENABLE_BROWSABLE_API","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_CEPHFS":{"name":"FEATURE_TOGGLE_CEPHFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_DASHBOARD":{"name":"FEATURE_TOGGLE_DASHBOARD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_ISCSI":{"name":"FEATURE_TOGGLE_ISCSI","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_MIRRORING":{"name":"FEATURE_TOGGLE_MIRRORING","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_NFS":{"name":"FEATURE_TOGGLE_NFS","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RBD":{"name":"FEATURE_TOGGLE_RBD","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"FEATURE_TOGGLE_RGW":{"name":"FEATURE_TOGGLE_RGW","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE":{"name":"GANESHA_CLUSTERS_RADOS_POOL_NAMESPACE","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_PASSWORD":{"name":"GRAFANA_API_PASSWORD","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_SSL_VERIFY":{"name":"GRAFANA_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_URL":{"name":"GRAFANA_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_API_USERNAME":{"name":"GRAFANA_API_USERNAME","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_FRONTEND_API_URL":{"name":"GRAFANA_FRONTEND_API_URL","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"GRAFANA_UPDATE_DASHBOARDS":{"name":"GRAFANA_UPDATE_DASHBOARDS","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISCSI_API_SSL_VERIFICATION":{"name":"ISCSI_API_SSL_VERIFICATION","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ISSUE_TRACKER_API_KEY":{"name":"ISSUE_TRACKER_API_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_HOST":{"name":"PROMETHEUS_API_HOST","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PROMETHEUS_API_SSL_VERIFY":{"name":"PROMETHEUS_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_COMPLEXITY_ENABLED":{"name":"PWD_POLICY_CHECK_COMPLEXITY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED":{"name":"PWD_POLICY_CHECK_EXCLUSION_LIST_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_LENGTH_ENABLED":{"name":"PWD_POLICY_CHECK_LENGTH_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_OLDPWD_ENABLED":{"name":"PWD_POLICY_CHECK_OLDPWD_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_REPETITIVE_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED":{"name":"PWD_POLICY_CHECK_SEQUENTIAL_CHARS_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_CHECK_USERNAME_ENABLED":{"name":"PWD_POLICY_CHECK_USERNAME_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_ENABLED":{"name":"PWD_POLICY_ENABLED","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_EXCLUSION_LIST":{"name":"PWD_POLICY_EXCLUSION_LIST","type":"str","level":"advanced","flags":0,"default_value":"osd,host,dashboard,pool,block,nfs,ceph,monitors,gateway,logs,crush,maps","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_COMPLEXITY":{"name":"PWD_POLICY_MIN_COMPLEXITY","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"PWD_POLICY_MIN_LENGTH":{"name":"PWD_POLICY_MIN_LENGTH","type":"int","level":"advanced","flags":0,"default_value":"8","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"REST_REQUESTS_TIMEOUT":{"name":"REST_REQUESTS_TIMEOUT","type":"int","level":"advanced","flags":0,"default_value":"45","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ACCESS_KEY":{"name":"RGW_API_ACCESS_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_ADMIN_RESOURCE":{"name":"RGW_API_ADMIN_RESOURCE","type":"str","level":"advanced","flags":0,"default_value":"admin","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SECRET_KEY":{"name":"RGW_API_SECRET_KEY","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"RGW_API_SSL_VERIFY":{"name":"RGW_API_SSL_VERIFY","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_SPAN":{"name":"USER_PWD_EXPIRATION_SPAN","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_1":{"name":"USER_PWD_EXPIRATION_WARNING_1","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"USER_PWD_EXPIRATION_WARNING_2":{"name":"USER_PWD_EXPIRATION_WARNING_2","type":"int","level":"advanced","flags":0,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"cross_origin_url":{"name":"cross_origin_url","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"crt_file":{"name":"crt_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"debug":{"name":"debug","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Enable/disable debug options","long_desc":"","tags":[],"see_also":[]},"jwt_token_ttl":{"name":"jwt_token_ttl","type":"int","level":"advanced","flags":0,"default_value":"28800","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"motd":{"name":"motd","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"The message of the day","long_desc":"","tags":[],"see_also":[]},"redirect_resolve_ip_addr":{"name":"redirect_resolve_ip_addr","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":0,"default_value":"8080","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"ssl_server_port":{"name":"ssl_server_port","type":"int","level":"advanced","flags":0,"default_value":"8443","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":0,"default_value":"redirect","min":"","max":"","enum_allowed":["error","redirect"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":0,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url_prefix":{"name":"url_prefix","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"devicehealth","can_run":true,"error_string":"","module_options":{"enable_monitoring":{"name":"enable_monitoring","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"monitor device health metrics","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mark_out_threshold":{"name":"mark_out_threshold","type":"secs","level":"advanced","flags":1,"default_value":"2419200","min":"","max":"","enum_allowed":[],"desc":"automatically mark OSD if it may fail before this long","long_desc":"","tags":[],"see_also":[]},"pool_name":{"name":"pool_name","type":"str","level":"advanced","flags":1,"default_value":"device_health_metrics","min":"","max":"","enum_allowed":[],"desc":"name of pool in which to store device health metrics","long_desc":"","tags":[],"see_also":[]},"retention_period":{"name":"retention_period","type":"secs","level":"advanced","flags":1,"default_value":"15552000","min":"","max":"","enum_allowed":[],"desc":"how long to retain device health metrics","long_desc":"","tags":[],"see_also":[]},"scrape_frequency":{"name":"scrape_frequency","type":"secs","level":"advanced","flags":1,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"how frequently to scrape device health metrics","long_desc":"","tags":[],"see_also":[]},"self_heal":{"name":"self_heal","type":"bool","level":"advanced","flags":1,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"preemptively heal cluster around devices that may fail","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"how frequently to wake up and check device health","long_desc":"","tags":[],"see_also":[]},"warn_threshold":{"name":"warn_threshold","type":"secs","level":"advanced","flags":1,"default_value":"7257600","min":"","max":"","enum_allowed":[],"desc":"raise health warning if OSD may fail before this long","long_desc":"","tags":[],"see_also":[]}}},{"name":"diskprediction_local","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predict_interval":{"name":"predict_interval","type":"str","level":"advanced","flags":0,"default_value":"86400","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"predictor_model":{"name":"predictor_model","type":"str","level":"advanced","flags":0,"default_value":"prophetstor","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"str","level":"advanced","flags":0,"default_value":"600","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"influx","can_run":false,"error_string":"influxdb python module not found","module_options":{"batch_size":{"name":"batch_size","type":"int","level":"advanced","flags":0,"default_value":"5000","min":"","max":"","enum_allowed":[],"desc":"How big batches of data points should be when sending to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"database":{"name":"database","type":"str","level":"advanced","flags":0,"default_value":"ceph","min":"","max":"","enum_allowed":[],"desc":"InfluxDB database name. You will need to create this database and grant write privileges to the configured username or the username must have admin privileges to create it.","long_desc":"","tags":[],"see_also":[]},"hostname":{"name":"hostname","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server hostname","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"30","min":"5","max":"","enum_allowed":[],"desc":"Time between reports to InfluxDB. Default 30 seconds.","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"password":{"name":"password","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"password of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"port":{"name":"port","type":"int","level":"advanced","flags":0,"default_value":"8086","min":"","max":"","enum_allowed":[],"desc":"InfluxDB server port","long_desc":"","tags":[],"see_also":[]},"ssl":{"name":"ssl","type":"str","level":"advanced","flags":0,"default_value":"false","min":"","max":"","enum_allowed":[],"desc":"Use https connection for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]},"threads":{"name":"threads","type":"int","level":"advanced","flags":0,"default_value":"5","min":"1","max":"32","enum_allowed":[],"desc":"How many worker threads should be spawned for sending data to InfluxDB.","long_desc":"","tags":[],"see_also":[]},"username":{"name":"username","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"username of InfluxDB server user","long_desc":"","tags":[],"see_also":[]},"verify_ssl":{"name":"verify_ssl","type":"str","level":"advanced","flags":0,"default_value":"true","min":"","max":"","enum_allowed":[],"desc":"Verify https cert for InfluxDB server. Use \"true\" or \"false\".","long_desc":"","tags":[],"see_also":[]}}},{"name":"insights","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"iostat","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"k8sevents","can_run":true,"error_string":"","module_options":{"ceph_event_retention_days":{"name":"ceph_event_retention_days","type":"int","level":"advanced","flags":0,"default_value":"7","min":"","max":"","enum_allowed":[],"desc":"Days to hold ceph event information within local cache","long_desc":"","tags":[],"see_also":[]},"config_check_secs":{"name":"config_check_secs","type":"int","level":"advanced","flags":0,"default_value":"10","min":"10","max":"","enum_allowed":[],"desc":"interval (secs) to check for cluster configuration changes","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"localpool","can_run":true,"error_string":"","module_options":{"failure_domain":{"name":"failure_domain","type":"str","level":"advanced","flags":1,"default_value":"host","min":"","max":"","enum_allowed":[],"desc":"failure domain for any created local pool","long_desc":"what failure domain we should separate data replicas across.","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"min_size":{"name":"min_size","type":"int","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"default min_size for any created local pool","long_desc":"value to set min_size to (unchanged from Ceph's default if this option is not set)","tags":[],"see_also":[]},"num_rep":{"name":"num_rep","type":"int","level":"advanced","flags":1,"default_value":"3","min":"","max":"","enum_allowed":[],"desc":"default replica count for any created local pool","long_desc":"","tags":[],"see_also":[]},"pg_num":{"name":"pg_num","type":"int","level":"advanced","flags":1,"default_value":"128","min":"","max":"","enum_allowed":[],"desc":"default pg_num for any created local pool","long_desc":"","tags":[],"see_also":[]},"prefix":{"name":"prefix","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"name prefix for any created local pool","long_desc":"","tags":[],"see_also":[]},"subtree":{"name":"subtree","type":"str","level":"advanced","flags":1,"default_value":"rack","min":"","max":"","enum_allowed":[],"desc":"CRUSH level for which to create a local pool","long_desc":"which CRUSH subtree type the module should create a pool for.","tags":[],"see_also":[]}}},{"name":"mds_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"mirroring","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"nfs","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"orchestrator":{"name":"orchestrator","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["cephadm","rook","test_orchestrator"],"desc":"Orchestrator backend","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_perf_query","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"osd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"pg_autoscaler","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"noautoscale":{"name":"noautoscale","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"global autoscale flag","long_desc":"Option to turn on/off the autoscaler for all pools","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"threshold":{"name":"threshold","type":"float","level":"advanced","flags":0,"default_value":"3.0","min":"1.0","max":"","enum_allowed":[],"desc":"scaling threshold","long_desc":"The factor by which the `NEW PG_NUM` must vary from the current`PG_NUM` before being accepted. Cannot be less than 1.0","tags":[],"see_also":[]}}},{"name":"progress","can_run":true,"error_string":"","module_options":{"allow_pg_recovery_event":{"name":"allow_pg_recovery_event","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow the module to show pg recovery progress","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_completed_events":{"name":"max_completed_events","type":"int","level":"advanced","flags":1,"default_value":"50","min":"","max":"","enum_allowed":[],"desc":"number of past completed events to remember","long_desc":"","tags":[],"see_also":[]},"sleep_interval":{"name":"sleep_interval","type":"secs","level":"advanced","flags":1,"default_value":"5","min":"","max":"","enum_allowed":[],"desc":"how long the module is going to sleep","long_desc":"","tags":[],"see_also":[]}}},{"name":"prometheus","can_run":true,"error_string":"","module_options":{"cache":{"name":"cache","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools":{"name":"rbd_stats_pools","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rbd_stats_pools_refresh_interval":{"name":"rbd_stats_pools_refresh_interval","type":"int","level":"advanced","flags":0,"default_value":"300","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"scrape_interval":{"name":"scrape_interval","type":"float","level":"advanced","flags":0,"default_value":"15.0","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"::","min":"","max":"","enum_allowed":[],"desc":"the IPv4 or IPv6 address on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"int","level":"advanced","flags":1,"default_value":"9283","min":"","max":"","enum_allowed":[],"desc":"the port on which the module listens for HTTP requests","long_desc":"","tags":[],"see_also":[]},"stale_cache_strategy":{"name":"stale_cache_strategy","type":"str","level":"advanced","flags":0,"default_value":"log","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_behaviour":{"name":"standby_behaviour","type":"str","level":"advanced","flags":1,"default_value":"default","min":"","max":"","enum_allowed":["default","error"],"desc":"","long_desc":"","tags":[],"see_also":[]},"standby_error_status_code":{"name":"standby_error_status_code","type":"int","level":"advanced","flags":1,"default_value":"500","min":"400","max":"599","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rbd_support","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_snap_create":{"name":"max_concurrent_snap_create","type":"int","level":"advanced","flags":0,"default_value":"10","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"mirror_snapshot_schedule":{"name":"mirror_snapshot_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"trash_purge_schedule":{"name":"trash_purge_schedule","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"restful","can_run":true,"error_string":"","module_options":{"enable_auth":{"name":"enable_auth","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"key_file":{"name":"key_file","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_addr":{"name":"server_addr","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"server_port":{"name":"server_port","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rgw","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"rook","can_run":true,"error_string":"","module_options":{"drive_group_interval":{"name":"drive_group_interval","type":"float","level":"advanced","flags":0,"default_value":"300.0","min":"","max":"","enum_allowed":[],"desc":"interval in seconds between re-application of applied drive_groups","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"storage_class":{"name":"storage_class","type":"str","level":"advanced","flags":0,"default_value":"local","min":"","max":"","enum_allowed":[],"desc":"storage class name for LSO-discovered PVs","long_desc":"","tags":[],"see_also":[]}}},{"name":"selftest","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption1":{"name":"roption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"roption2":{"name":"roption2","type":"str","level":"advanced","flags":0,"default_value":"xyz","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption1":{"name":"rwoption1","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption2":{"name":"rwoption2","type":"int","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption3":{"name":"rwoption3","type":"float","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption4":{"name":"rwoption4","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption5":{"name":"rwoption5","type":"bool","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption6":{"name":"rwoption6","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"rwoption7":{"name":"rwoption7","type":"int","level":"advanced","flags":0,"default_value":"","min":"1","max":"42","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testkey":{"name":"testkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testlkey":{"name":"testlkey","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"testnewline":{"name":"testnewline","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"snap_schedule","can_run":true,"error_string":"","module_options":{"allow_m_granularity":{"name":"allow_m_granularity","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"allow minute scheduled snapshots","long_desc":"","tags":[],"see_also":[]},"dump_on_update":{"name":"dump_on_update","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"dump database to debug log on update","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"stats","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"status","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telegraf","can_run":true,"error_string":"","module_options":{"address":{"name":"address","type":"str","level":"advanced","flags":0,"default_value":"unixgram:///tmp/telegraf.sock","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"15","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"telemetry","can_run":true,"error_string":"","module_options":{"channel_basic":{"name":"channel_basic","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share basic cluster information (size, version)","long_desc":"","tags":[],"see_also":[]},"channel_crash":{"name":"channel_crash","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share metadata about Ceph daemon crashes (version, stack straces, etc)","long_desc":"","tags":[],"see_also":[]},"channel_device":{"name":"channel_device","type":"bool","level":"advanced","flags":0,"default_value":"True","min":"","max":"","enum_allowed":[],"desc":"Share device health metrics (e.g., SMART data, minus potentially identifying info like serial numbers)","long_desc":"","tags":[],"see_also":[]},"channel_ident":{"name":"channel_ident","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share a user-provided description and/or contact email for the cluster","long_desc":"","tags":[],"see_also":[]},"channel_perf":{"name":"channel_perf","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"Share various performance metrics of a cluster","long_desc":"","tags":[],"see_also":[]},"contact":{"name":"contact","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"description":{"name":"description","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"device_url":{"name":"device_url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/device","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"enabled":{"name":"enabled","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"int","level":"advanced","flags":0,"default_value":"24","min":"8","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"last_opt_revision":{"name":"last_opt_revision","type":"int","level":"advanced","flags":0,"default_value":"1","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"leaderboard":{"name":"leaderboard","type":"bool","level":"advanced","flags":0,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"organization":{"name":"organization","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"proxy":{"name":"proxy","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"url":{"name":"url","type":"str","level":"advanced","flags":0,"default_value":"https://telemetry.ceph.com/report","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"test_orchestrator","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}},{"name":"volumes","can_run":true,"error_string":"","module_options":{"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"max_concurrent_clones":{"name":"max_concurrent_clones","type":"int","level":"advanced","flags":0,"default_value":"4","min":"","max":"","enum_allowed":[],"desc":"Number of asynchronous cloner threads","long_desc":"","tags":[],"see_also":[]},"snapshot_clone_delay":{"name":"snapshot_clone_delay","type":"int","level":"advanced","flags":0,"default_value":"0","min":"","max":"","enum_allowed":[],"desc":"Delay clone begin operation by snapshot_clone_delay seconds","long_desc":"","tags":[],"see_also":[]}}},{"name":"zabbix","can_run":true,"error_string":"","module_options":{"discovery_interval":{"name":"discovery_interval","type":"uint","level":"advanced","flags":0,"default_value":"100","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"identifier":{"name":"identifier","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"interval":{"name":"interval","type":"secs","level":"advanced","flags":0,"default_value":"60","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_level":{"name":"log_level","type":"str","level":"advanced","flags":1,"default_value":"","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster":{"name":"log_to_cluster","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_cluster_level":{"name":"log_to_cluster_level","type":"str","level":"advanced","flags":1,"default_value":"info","min":"","max":"","enum_allowed":["","critical","debug","error","info","warning"],"desc":"","long_desc":"","tags":[],"see_also":[]},"log_to_file":{"name":"log_to_file","type":"bool","level":"advanced","flags":1,"default_value":"False","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_host":{"name":"zabbix_host","type":"str","level":"advanced","flags":0,"default_value":"","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_port":{"name":"zabbix_port","type":"int","level":"advanced","flags":0,"default_value":"10051","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]},"zabbix_sender":{"name":"zabbix_sender","type":"str","level":"advanced","flags":0,"default_value":"/usr/bin/zabbix_sender","min":"","max":"","enum_allowed":[],"desc":"","long_desc":"","tags":[],"see_also":[]}}}],"services":{"dashboard":"https://172.21.15.93:8443/","prometheus":"http://172.21.15.93:9283/"},"always_on_modules":{"octopus":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"pacific":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"],"quincy":["balancer","crash","devicehealth","orchestrator","pg_autoscaler","progress","rbd_support","status","telemetry","volumes"]},"last_failure_osd_epoch":58,"active_clients":[{"name":"devicehealth","addrvec":[{"type":"v2","addr":"172.21.15.93:0","nonce":3005830643}]},{"name":"libcephsqlite","addrvec":[{"type":"v2","addr":"172.21.15.93:0","nonce":713101306}]},{"name":"rbd_support","addrvec":[{"type":"v2","addr":"172.21.15.93:0","nonce":421269516}]},{"name":"volumes","addrvec":[{"type":"v2","addr":"172.21.15.93:0","nonce":3173026838}]}]} 2023-08-27T20:51:24.967 INFO:tasks.cephadm.ceph_manager.ceph:mgr available! 2023-08-27T20:51:24.967 INFO:tasks.cephadm.ceph_manager.ceph:waiting for all up 2023-08-27T20:51:24.967 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T20:51:25.392 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4138039283' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T20:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:25 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4138039283' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T20:51:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4138039283' entity='client.admin' cmd=[{"prefix": "mgr dump", "format": "json"}]: dispatch 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:25.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:25.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi093.szcdyz", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T20:51:25.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:25 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[111335]: Deploying daemon mds.foo.smithi093.szcdyz on smithi093 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[111335]: pgmap v30: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[118902]: Deploying daemon mds.foo.smithi093.szcdyz on smithi093 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[118902]: pgmap v30: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:26.416 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:26 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:26 smithi136 ceph-mon[113531]: Deploying daemon mds.foo.smithi093.szcdyz on smithi093 2023-08-27T20:51:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:26 smithi136 ceph-mon[113531]: pgmap v30: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:26 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:26 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:26.898 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:26.898 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":63,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","created":"2023-08-27T20:46:40.100445+0000","modified":"2023-08-27T20:51:19.112450+0000","last_up_change":"2023-08-27T20:50:18.452654+0000","last_in_change":"2023-08-27T20:50:04.867005+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-27T20:48:59.294309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T20:50:39.429736+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-27T20:51:16.824303+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-27T20:51:17.324126+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"f8288f9e-5794-4532-9495-65379f66b902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6803","nonce":468012600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6805","nonce":468012600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6809","nonce":468012600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6807","nonce":468012600}]},"public_addr":"172.21.15.93:6803/468012600","cluster_addr":"172.21.15.93:6805/468012600","heartbeat_back_addr":"172.21.15.93:6809/468012600","heartbeat_front_addr":"172.21.15.93:6807/468012600","state":["exists","up"]},{"osd":1,"uuid":"703653bb-1bc5-418f-bf13-4e71cb82a4da","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6811","nonce":1906627391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6813","nonce":1906627391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6817","nonce":1906627391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6815","nonce":1906627391}]},"public_addr":"172.21.15.93:6811/1906627391","cluster_addr":"172.21.15.93:6813/1906627391","heartbeat_back_addr":"172.21.15.93:6817/1906627391","heartbeat_front_addr":"172.21.15.93:6815/1906627391","state":["exists","up"]},{"osd":2,"uuid":"ca3d4d17-b4ba-402a-999d-496be3cda08d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6819","nonce":3915054168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6820","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6821","nonce":3915054168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6825","nonce":3915054168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6822","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6823","nonce":3915054168}]},"public_addr":"172.21.15.93:6819/3915054168","cluster_addr":"172.21.15.93:6821/3915054168","heartbeat_back_addr":"172.21.15.93:6825/3915054168","heartbeat_front_addr":"172.21.15.93:6823/3915054168","state":["exists","up"]},{"osd":3,"uuid":"04f9b593-2f0e-4b0e-b5ba-2778d688e66d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6826","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6827","nonce":4177593389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6829","nonce":4177593389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6832","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6833","nonce":4177593389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6831","nonce":4177593389}]},"public_addr":"172.21.15.93:6827/4177593389","cluster_addr":"172.21.15.93:6829/4177593389","heartbeat_back_addr":"172.21.15.93:6833/4177593389","heartbeat_front_addr":"172.21.15.93:6831/4177593389","state":["exists","up"]},{"osd":4,"uuid":"2ca1eb6c-000f-42cc-a188-e0044386e9fd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6801","nonce":4129055894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6803","nonce":4129055894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6807","nonce":4129055894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6805","nonce":4129055894}]},"public_addr":"172.21.15.136:6801/4129055894","cluster_addr":"172.21.15.136:6803/4129055894","heartbeat_back_addr":"172.21.15.136:6807/4129055894","heartbeat_front_addr":"172.21.15.136:6805/4129055894","state":["exists","up"]},{"osd":5,"uuid":"eb791c51-36bd-4562-9715-7081bdb9b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6809","nonce":1032556808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6811","nonce":1032556808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6815","nonce":1032556808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6813","nonce":1032556808}]},"public_addr":"172.21.15.136:6809/1032556808","cluster_addr":"172.21.15.136:6811/1032556808","heartbeat_back_addr":"172.21.15.136:6815/1032556808","heartbeat_front_addr":"172.21.15.136:6813/1032556808","state":["exists","up"]},{"osd":6,"uuid":"9d1b03b4-2611-4877-83ac-05b34cb091ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6817","nonce":1621616467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6819","nonce":1621616467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6823","nonce":1621616467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6821","nonce":1621616467}]},"public_addr":"172.21.15.136:6817/1621616467","cluster_addr":"172.21.15.136:6819/1621616467","heartbeat_back_addr":"172.21.15.136:6823/1621616467","heartbeat_front_addr":"172.21.15.136:6821/1621616467","state":["exists","up"]},{"osd":7,"uuid":"500cecba-b803-4827-9e93-025f93e19a50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6825","nonce":4170532217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6827","nonce":4170532217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6831","nonce":4170532217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6829","nonce":4170532217}]},"public_addr":"172.21.15.136:6825/4170532217","cluster_addr":"172.21.15.136:6827/4170532217","heartbeat_back_addr":"172.21.15.136:6831/4170532217","heartbeat_front_addr":"172.21.15.136:6829/4170532217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:23.430450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:39.677984+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:56.383706+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:13.215635+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:28.875590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:45.109289+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:01.198559+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:17.193536+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.93:0/2819309041":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1735542319":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6801/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6801/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1009313451":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/4182115085":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/804147002":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1072616791":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/750179315":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1575139232":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/966563482":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/2843438722":"2023-08-28T20:47:20.986783+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T20:51:27.296 INFO:tasks.cephadm.ceph_manager.ceph:all up! 2023-08-27T20:51:27.296 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T20:51:27.560 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: Deploying daemon mds.foo.smithi136.hhkxbr on smithi136 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/41811368' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:boot 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: daemon mds.foo.smithi093.szcdyz assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: Cluster is now healthy 2023-08-27T20:51:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: fsmap foo:0 1 up:standby 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:creating} 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[111335]: daemon mds.foo.smithi093.szcdyz is now active in filesystem foo as rank 0 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: Deploying daemon mds.foo.smithi136.hhkxbr on smithi136 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/41811368' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:boot 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: daemon mds.foo.smithi093.szcdyz assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: Cluster is now healthy 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: fsmap foo:0 1 up:standby 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:creating} 2023-08-27T20:51:27.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:27 smithi093 ceph-mon[118902]: daemon mds.foo.smithi093.szcdyz is now active in filesystem foo as rank 0 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]: dispatch 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "auth get-or-create", "entity": "mds.foo.smithi136.hhkxbr", "caps": ["mon", "profile mds", "osd", "allow rw tag cephfs *=*", "mds", "allow"]}]': finished 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: Deploying daemon mds.foo.smithi136.hhkxbr on smithi136 2023-08-27T20:51:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/41811368' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: pgmap v31: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:boot 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: daemon mds.foo.smithi093.szcdyz assigned to filesystem foo as rank 0 (now has 1 ranks) 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: Health check cleared: MDS_ALL_DOWN (was: 1 filesystem is offline) 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: Health check cleared: MDS_UP_LESS_THAN_MAX (was: 1 filesystem is online with fewer MDS than max_mds) 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: Cluster is now healthy 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: fsmap foo:0 1 up:standby 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:creating} 2023-08-27T20:51:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:27 smithi136 ceph-mon[113531]: daemon mds.foo.smithi093.szcdyz is now active in filesystem foo as rank 0 2023-08-27T20:51:28.736 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:28.737 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":63,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","created":"2023-08-27T20:46:40.100445+0000","modified":"2023-08-27T20:51:19.112450+0000","last_up_change":"2023-08-27T20:50:18.452654+0000","last_in_change":"2023-08-27T20:50:04.867005+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":4,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"jewel","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-27T20:48:59.294309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T20:50:39.429736+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-27T20:51:16.824303+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-27T20:51:17.324126+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}}],"osds":[{"osd":0,"uuid":"f8288f9e-5794-4532-9495-65379f66b902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6803","nonce":468012600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6805","nonce":468012600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6809","nonce":468012600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6807","nonce":468012600}]},"public_addr":"172.21.15.93:6803/468012600","cluster_addr":"172.21.15.93:6805/468012600","heartbeat_back_addr":"172.21.15.93:6809/468012600","heartbeat_front_addr":"172.21.15.93:6807/468012600","state":["exists","up"]},{"osd":1,"uuid":"703653bb-1bc5-418f-bf13-4e71cb82a4da","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6811","nonce":1906627391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6813","nonce":1906627391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6817","nonce":1906627391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6815","nonce":1906627391}]},"public_addr":"172.21.15.93:6811/1906627391","cluster_addr":"172.21.15.93:6813/1906627391","heartbeat_back_addr":"172.21.15.93:6817/1906627391","heartbeat_front_addr":"172.21.15.93:6815/1906627391","state":["exists","up"]},{"osd":2,"uuid":"ca3d4d17-b4ba-402a-999d-496be3cda08d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6819","nonce":3915054168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6820","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6821","nonce":3915054168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6825","nonce":3915054168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6822","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6823","nonce":3915054168}]},"public_addr":"172.21.15.93:6819/3915054168","cluster_addr":"172.21.15.93:6821/3915054168","heartbeat_back_addr":"172.21.15.93:6825/3915054168","heartbeat_front_addr":"172.21.15.93:6823/3915054168","state":["exists","up"]},{"osd":3,"uuid":"04f9b593-2f0e-4b0e-b5ba-2778d688e66d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6826","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6827","nonce":4177593389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6829","nonce":4177593389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6832","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6833","nonce":4177593389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6831","nonce":4177593389}]},"public_addr":"172.21.15.93:6827/4177593389","cluster_addr":"172.21.15.93:6829/4177593389","heartbeat_back_addr":"172.21.15.93:6833/4177593389","heartbeat_front_addr":"172.21.15.93:6831/4177593389","state":["exists","up"]},{"osd":4,"uuid":"2ca1eb6c-000f-42cc-a188-e0044386e9fd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6801","nonce":4129055894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6803","nonce":4129055894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6807","nonce":4129055894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6805","nonce":4129055894}]},"public_addr":"172.21.15.136:6801/4129055894","cluster_addr":"172.21.15.136:6803/4129055894","heartbeat_back_addr":"172.21.15.136:6807/4129055894","heartbeat_front_addr":"172.21.15.136:6805/4129055894","state":["exists","up"]},{"osd":5,"uuid":"eb791c51-36bd-4562-9715-7081bdb9b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6809","nonce":1032556808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6811","nonce":1032556808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6815","nonce":1032556808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6813","nonce":1032556808}]},"public_addr":"172.21.15.136:6809/1032556808","cluster_addr":"172.21.15.136:6811/1032556808","heartbeat_back_addr":"172.21.15.136:6815/1032556808","heartbeat_front_addr":"172.21.15.136:6813/1032556808","state":["exists","up"]},{"osd":6,"uuid":"9d1b03b4-2611-4877-83ac-05b34cb091ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":61,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6817","nonce":1621616467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6819","nonce":1621616467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6823","nonce":1621616467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6821","nonce":1621616467}]},"public_addr":"172.21.15.136:6817/1621616467","cluster_addr":"172.21.15.136:6819/1621616467","heartbeat_back_addr":"172.21.15.136:6823/1621616467","heartbeat_front_addr":"172.21.15.136:6821/1621616467","state":["exists","up"]},{"osd":7,"uuid":"500cecba-b803-4827-9e93-025f93e19a50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":60,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6825","nonce":4170532217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6827","nonce":4170532217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6831","nonce":4170532217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6829","nonce":4170532217}]},"public_addr":"172.21.15.136:6825/4170532217","cluster_addr":"172.21.15.136:6827/4170532217","heartbeat_back_addr":"172.21.15.136:6831/4170532217","heartbeat_front_addr":"172.21.15.136:6829/4170532217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:23.430450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:39.677984+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:56.383706+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:13.215635+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:28.875590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:45.109289+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:01.198559+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:17.193536+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.93:0/2819309041":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1735542319":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6801/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6801/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1009313451":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/4182115085":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/804147002":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1072616791":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/750179315":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/1575139232":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/966563482":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/2843438722":"2023-08-28T20:47:20.986783+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T20:51:29.075 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:51:28 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:51:28.880Z caller=cluster.go:698 level=info component=cluster msg="gossip settled; proceeding" elapsed=10.003073042s 2023-08-27T20:51:29.136 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.0 flush_pg_stats 2023-08-27T20:51:29.136 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.1 flush_pg_stats 2023-08-27T20:51:29.137 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.2 flush_pg_stats 2023-08-27T20:51:29.137 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.3 flush_pg_stats 2023-08-27T20:51:29.138 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.4 flush_pg_stats 2023-08-27T20:51:29.138 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.5 flush_pg_stats 2023-08-27T20:51:29.138 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.6 flush_pg_stats 2023-08-27T20:51:29.139 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph tell osd.7 flush_pg_stats 2023-08-27T20:51:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:29 smithi136 ceph-mon[113531]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:active 2023-08-27T20:51:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:29 smithi136 ceph-mon[113531]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 2023-08-27T20:51:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1989377807' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:51:29.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:29 smithi093 ceph-mon[118902]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:active 2023-08-27T20:51:29.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:29 smithi093 ceph-mon[118902]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 2023-08-27T20:51:29.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1989377807' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:51:29.495 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:29 smithi093 ceph-mon[111335]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:active 2023-08-27T20:51:29.495 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:29 smithi093 ceph-mon[111335]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 2023-08-27T20:51:29.495 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1989377807' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:51:29.948 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:29.999 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:30.150 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:30.284 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:30.358 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:30 smithi136 ceph-mon[113531]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:30.394 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:30 smithi093 ceph-mon[111335]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:30.394 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:30 smithi093 ceph-mon[118902]: pgmap v32: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:51:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:51:31.207 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:31.408 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: mds.? [v2:172.21.15.136:6832/1240601924,v1:172.21.15.136:6833/1240601924] up:boot 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: mds.? [v2:172.21.15.136:6832/1240601924,v1:172.21.15.136:6833/1240601924] up:boot 2023-08-27T20:51:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:51:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:31 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.567 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.567 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: mds.? [v2:172.21.15.136:6832/1240601924,v1:172.21.15.136:6833/1240601924] up:boot 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:31.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:31 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:31.761 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:32.404 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:32 smithi093 ceph-mon[111335]: pgmap v33: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:32.404 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:32 smithi093 ceph-mon[111335]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:active 2023-08-27T20:51:32.404 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:32 smithi093 ceph-mon[111335]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:32.404 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:32 smithi093 ceph-mon[118902]: pgmap v33: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:32.404 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:32 smithi093 ceph-mon[118902]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:active 2023-08-27T20:51:32.404 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:32 smithi093 ceph-mon[118902]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:32 smithi136 ceph-mon[113531]: pgmap v33: 73 pgs: 73 active+clean; 577 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:32 smithi136 ceph-mon[113531]: mds.? [v2:172.21.15.93:6834/1638024913,v1:172.21.15.93:6835/1638024913] up:active 2023-08-27T20:51:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:32 smithi136 ceph-mon[113531]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:32.767 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:34 smithi093 ceph-mon[111335]: pgmap v34: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:34 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:34 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:34 smithi093 ceph-mon[118902]: pgmap v34: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:34 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:34 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:34.547 INFO:teuthology.orchestra.run.smithi093.stdout:77309411363 2023-08-27T20:51:34.547 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.2 2023-08-27T20:51:34.585 INFO:teuthology.orchestra.run.smithi093.stdout:128849018907 2023-08-27T20:51:34.586 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.4 2023-08-27T20:51:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:34 smithi136 ceph-mon[113531]: pgmap v34: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:34 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:34 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:35.434 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:35] "GET /metrics HTTP/1.1" 200 208021 "" "Prometheus/2.43.0" 2023-08-27T20:51:35.859 INFO:teuthology.orchestra.run.smithi093.stdout:55834574886 2023-08-27T20:51:35.860 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.1 2023-08-27T20:51:36.099 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:35 smithi093 ceph-mon[111335]: mds.? [v2:172.21.15.136:6832/1240601924,v1:172.21.15.136:6833/1240601924] up:standby 2023-08-27T20:51:36.099 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:35 smithi093 ceph-mon[111335]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:36.099 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:35 smithi093 ceph-mon[111335]: pgmap v35: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:36.099 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:35 smithi093 ceph-mon[118902]: mds.? [v2:172.21.15.136:6832/1240601924,v1:172.21.15.136:6833/1240601924] up:standby 2023-08-27T20:51:36.100 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:35 smithi093 ceph-mon[118902]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:36.100 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:35 smithi093 ceph-mon[118902]: pgmap v35: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:35 smithi136 ceph-mon[113531]: mds.? [v2:172.21.15.136:6832/1240601924,v1:172.21.15.136:6833/1240601924] up:standby 2023-08-27T20:51:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:35 smithi136 ceph-mon[113531]: fsmap foo:1 {0=foo.smithi093.szcdyz=up:active} 1 up:standby 2023-08-27T20:51:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:35 smithi136 ceph-mon[113531]: pgmap v35: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:36.200 INFO:teuthology.orchestra.run.smithi093.stdout:158913789977 2023-08-27T20:51:36.200 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.5 2023-08-27T20:51:36.726 INFO:teuthology.orchestra.run.smithi093.stdout:219043332114 2023-08-27T20:51:36.726 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.7 2023-08-27T20:51:38.093 INFO:teuthology.orchestra.run.smithi093.stdout:34359738409 2023-08-27T20:51:38.094 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.0 2023-08-27T20:51:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:38 smithi093 ceph-mon[111335]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:38 smithi093 ceph-mon[118902]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:38 smithi136 ceph-mon[113531]: pgmap v36: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:38.584 INFO:teuthology.orchestra.run.smithi093.stdout:107374182431 2023-08-27T20:51:38.584 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.3 2023-08-27T20:51:38.618 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:38.950 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:38.973 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:38.974 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:38.975 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:39.551 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:39 smithi093 ceph-mon[111335]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:39.552 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:39 smithi093 ceph-mon[118902]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:39 smithi136 ceph-mon[113531]: pgmap v37: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:40.424 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:40.646 INFO:teuthology.orchestra.run.smithi093.stdout:77309411364 2023-08-27T20:51:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:51:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:51:40.792 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2414532399' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T20:51:40.792 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2414532399' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T20:51:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2414532399' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 2}]: dispatch 2023-08-27T20:51:41.891 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:41.891 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2023-08-27T20:51:41.891 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.891 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2023-08-27T20:51:41.891 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.891 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-08-27T20:51:41.893 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:41.893 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:41 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: pgmap v38: 73 pgs: 73 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail; 1.1 KiB/s wr, 3 op/s 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]: dispatch 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]: dispatch 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]: dispatch 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:41 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:42.311 INFO:tasks.cephadm.ceph_manager.ceph:need seq 77309411363 got 77309411364 for osd.2 2023-08-27T20:51:42.312 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:42.409 INFO:teuthology.orchestra.run.smithi093.stdout:219043332115 2023-08-27T20:51:42.607 INFO:teuthology.orchestra.run.smithi093.stdout:188978561047 2023-08-27T20:51:42.608 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd last-stat-seq osd.6 2023-08-27T20:51:42.852 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2023-08-27T20:51:42.852 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-08-27T20:51:42.852 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-08-27T20:51:42.852 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: osdmap e64: 8 total, 8 up, 8 in 2023-08-27T20:51:42.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:42.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:42.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:42.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:42.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:42.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:42.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/911246673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: osdmap e64: 8 total, 8 up, 8 in 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:42.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:42.855 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:42.855 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/911246673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T20:51:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.9", "id": [1, 5]}]': finished 2023-08-27T20:51:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.e", "id": [1, 2]}]': finished 2023-08-27T20:51:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "osd pg-upmap-items", "format": "json", "pgid": "4.f", "id": [1, 2]}]': finished 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: osdmap e64: 8 total, 8 up, 8 in 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/911246673' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 7}]: dispatch 2023-08-27T20:51:43.384 INFO:tasks.cephadm.ceph_manager.ceph:need seq 219043332114 got 219043332115 for osd.7 2023-08-27T20:51:43.384 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:43.840 INFO:teuthology.orchestra.run.smithi093.stdout:55834574888 2023-08-27T20:51:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:43 smithi093 ceph-mon[118902]: osdmap e65: 8 total, 8 up, 8 in 2023-08-27T20:51:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:43 smithi093 ceph-mon[118902]: pgmap v41: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:43 smithi093 ceph-mon[111335]: osdmap e65: 8 total, 8 up, 8 in 2023-08-27T20:51:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:43 smithi093 ceph-mon[111335]: pgmap v41: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:44.062 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:43 smithi136 ceph-mon[113531]: osdmap e65: 8 total, 8 up, 8 in 2023-08-27T20:51:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:43 smithi136 ceph-mon[113531]: pgmap v41: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:44.633 INFO:teuthology.orchestra.run.smithi093.stdout:128849018909 2023-08-27T20:51:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1937443430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T20:51:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4035886730' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T20:51:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1937443430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T20:51:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4035886730' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T20:51:44.766 INFO:tasks.cephadm.ceph_manager.ceph:need seq 55834574886 got 55834574888 for osd.1 2023-08-27T20:51:44.766 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1937443430' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 1}]: dispatch 2023-08-27T20:51:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:44 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:44 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4035886730' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 4}]: dispatch 2023-08-27T20:51:45.632 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:45] "GET /metrics HTTP/1.1" 200 238722 "" "Prometheus/2.43.0" 2023-08-27T20:51:45.734 INFO:teuthology.orchestra.run.smithi093.stdout:158913789979 2023-08-27T20:51:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:45 smithi093 ceph-mon[111335]: pgmap v42: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:45 smithi093 ceph-mon[118902]: pgmap v42: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:46.034 INFO:teuthology.orchestra.run.smithi093.stdout:34359738411 2023-08-27T20:51:46.112 INFO:tasks.cephadm.ceph_manager.ceph:need seq 128849018907 got 128849018909 for osd.4 2023-08-27T20:51:46.112 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:45 smithi136 ceph-mon[113531]: pgmap v42: 73 pgs: 3 peering, 70 active+clean; 579 KiB data, 50 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:46.448 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:46.703 INFO:tasks.cephadm.ceph_manager.ceph:need seq 34359738409 got 34359738411 for osd.0 2023-08-27T20:51:46.703 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/415550828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T20:51:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1586485428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T20:51:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/415550828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T20:51:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1586485428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T20:51:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/415550828' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 5}]: dispatch 2023-08-27T20:51:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1586485428' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 0}]: dispatch 2023-08-27T20:51:47.418 INFO:teuthology.orchestra.run.smithi093.stdout:107374182432 2023-08-27T20:51:47.824 INFO:tasks.cephadm.ceph_manager.ceph:need seq 158913789977 got 158913789979 for osd.5 2023-08-27T20:51:47.825 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:47 smithi093 ceph-mon[111335]: pgmap v43: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1961880100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T20:51:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:47 smithi093 ceph-mon[118902]: pgmap v43: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1961880100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T20:51:48.083 INFO:tasks.cephadm.ceph_manager.ceph:need seq 107374182431 got 107374182432 for osd.3 2023-08-27T20:51:48.083 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:47 smithi136 ceph-mon[113531]: pgmap v43: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1961880100' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 3}]: dispatch 2023-08-27T20:51:48.626 INFO:teuthology.orchestra.run.smithi093.stdout:188978561049 2023-08-27T20:51:49.208 INFO:tasks.cephadm.ceph_manager.ceph:need seq 188978561047 got 188978561049 for osd.6 2023-08-27T20:51:49.208 DEBUG:teuthology.parallel:result is None 2023-08-27T20:51:49.208 INFO:tasks.cephadm.ceph_manager.ceph:waiting for clean 2023-08-27T20:51:49.208 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T20:51:49.686 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/615233356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T20:51:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:49 smithi093 ceph-mon[111335]: pgmap v44: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/615233356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T20:51:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:49 smithi093 ceph-mon[118902]: pgmap v44: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/615233356' entity='client.admin' cmd=[{"prefix": "osd last-stat-seq", "id": 6}]: dispatch 2023-08-27T20:51:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:49 smithi136 ceph-mon[113531]: pgmap v44: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:51:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:51:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:50 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:51.016 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:51.018 INFO:teuthology.orchestra.run.smithi093.stderr:dumped all 2023-08-27T20:51:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:50 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:50 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:50 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:50 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:50 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:51.406 INFO:teuthology.orchestra.run.smithi093.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2023-08-27T20:51:50.977353+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52028,"kb_used_data":6972,"kb_used_omap":0,"kb_used_meta":44992,"kb_avail":749909188,"statfs":{"total":767960285184,"available":767907008512,"internally_reserved":0,"allocated":7139328,"data_stored":4072998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46071808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.272468"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707376+0000","last_change":"2023-08-27T20:51:18.131159+0000","last_active":"2023-08-27T20:51:41.707376+0000","last_peered":"2023-08-27T20:51:41.707376+0000","last_clean":"2023-08-27T20:51:41.707376+0000","last_became_active":"2023-08-27T20:51:18.130806+0000","last_became_peered":"2023-08-27T20:51:18.130806+0000","last_unstale":"2023-08-27T20:51:41.707376+0000","last_undegraded":"2023-08-27T20:51:41.707376+0000","last_fullsized":"2023-08-27T20:51:41.707376+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:26:50.146935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740465+0000","last_change":"2023-08-27T20:51:18.097797+0000","last_active":"2023-08-27T20:51:41.740465+0000","last_peered":"2023-08-27T20:51:41.740465+0000","last_clean":"2023-08-27T20:51:41.740465+0000","last_became_active":"2023-08-27T20:51:18.097523+0000","last_became_peered":"2023-08-27T20:51:18.097523+0000","last_unstale":"2023-08-27T20:51:41.740465+0000","last_undegraded":"2023-08-27T20:51:41.740465+0000","last_fullsized":"2023-08-27T20:51:41.740465+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:10:27.963795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709263+0000","last_change":"2023-08-27T20:51:18.133000+0000","last_active":"2023-08-27T20:51:41.709263+0000","last_peered":"2023-08-27T20:51:41.709263+0000","last_clean":"2023-08-27T20:51:41.709263+0000","last_became_active":"2023-08-27T20:51:18.132719+0000","last_became_peered":"2023-08-27T20:51:18.132719+0000","last_unstale":"2023-08-27T20:51:41.709263+0000","last_undegraded":"2023-08-27T20:51:41.709263+0000","last_fullsized":"2023-08-27T20:51:41.709263+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:00:48.680834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710063+0000","last_change":"2023-08-27T20:51:18.102113+0000","last_active":"2023-08-27T20:51:41.710063+0000","last_peered":"2023-08-27T20:51:41.710063+0000","last_clean":"2023-08-27T20:51:41.710063+0000","last_became_active":"2023-08-27T20:51:18.101848+0000","last_became_peered":"2023-08-27T20:51:18.101848+0000","last_unstale":"2023-08-27T20:51:41.710063+0000","last_undegraded":"2023-08-27T20:51:41.710063+0000","last_fullsized":"2023-08-27T20:51:41.710063+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:49:21.087984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707664+0000","last_change":"2023-08-27T20:51:18.129808+0000","last_active":"2023-08-27T20:51:41.707664+0000","last_peered":"2023-08-27T20:51:41.707664+0000","last_clean":"2023-08-27T20:51:41.707664+0000","last_became_active":"2023-08-27T20:51:18.129515+0000","last_became_peered":"2023-08-27T20:51:18.129515+0000","last_unstale":"2023-08-27T20:51:41.707664+0000","last_undegraded":"2023-08-27T20:51:41.707664+0000","last_fullsized":"2023-08-27T20:51:41.707664+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:45:42.761867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706520+0000","last_change":"2023-08-27T20:51:18.106356+0000","last_active":"2023-08-27T20:51:41.706520+0000","last_peered":"2023-08-27T20:51:41.706520+0000","last_clean":"2023-08-27T20:51:41.706520+0000","last_became_active":"2023-08-27T20:51:18.105473+0000","last_became_peered":"2023-08-27T20:51:18.105473+0000","last_unstale":"2023-08-27T20:51:41.706520+0000","last_undegraded":"2023-08-27T20:51:41.706520+0000","last_fullsized":"2023-08-27T20:51:41.706520+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:47:12.781410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707787+0000","last_change":"2023-08-27T20:51:18.129448+0000","last_active":"2023-08-27T20:51:41.707787+0000","last_peered":"2023-08-27T20:51:41.707787+0000","last_clean":"2023-08-27T20:51:41.707787+0000","last_became_active":"2023-08-27T20:51:18.129152+0000","last_became_peered":"2023-08-27T20:51:18.129152+0000","last_unstale":"2023-08-27T20:51:41.707787+0000","last_undegraded":"2023-08-27T20:51:41.707787+0000","last_fullsized":"2023-08-27T20:51:41.707787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:52:03.853769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706327+0000","last_change":"2023-08-27T20:51:18.096858+0000","last_active":"2023-08-27T20:51:41.706327+0000","last_peered":"2023-08-27T20:51:41.706327+0000","last_clean":"2023-08-27T20:51:41.706327+0000","last_became_active":"2023-08-27T20:51:18.095892+0000","last_became_peered":"2023-08-27T20:51:18.095892+0000","last_unstale":"2023-08-27T20:51:41.706327+0000","last_undegraded":"2023-08-27T20:51:41.706327+0000","last_fullsized":"2023-08-27T20:51:41.706327+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:27:14.781996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705301+0000","last_change":"2023-08-27T20:51:18.131530+0000","last_active":"2023-08-27T20:51:41.705301+0000","last_peered":"2023-08-27T20:51:41.705301+0000","last_clean":"2023-08-27T20:51:41.705301+0000","last_became_active":"2023-08-27T20:51:18.131263+0000","last_became_peered":"2023-08-27T20:51:18.131263+0000","last_unstale":"2023-08-27T20:51:41.705301+0000","last_undegraded":"2023-08-27T20:51:41.705301+0000","last_fullsized":"2023-08-27T20:51:41.705301+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:46.350911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740303+0000","last_change":"2023-08-27T20:51:18.098125+0000","last_active":"2023-08-27T20:51:41.740303+0000","last_peered":"2023-08-27T20:51:41.740303+0000","last_clean":"2023-08-27T20:51:41.740303+0000","last_became_active":"2023-08-27T20:51:18.097870+0000","last_became_peered":"2023-08-27T20:51:18.097870+0000","last_unstale":"2023-08-27T20:51:41.740303+0000","last_undegraded":"2023-08-27T20:51:41.740303+0000","last_fullsized":"2023-08-27T20:51:41.740303+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:32:32.322197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709177+0000","last_change":"2023-08-27T20:51:18.124571+0000","last_active":"2023-08-27T20:51:41.709177+0000","last_peered":"2023-08-27T20:51:41.709177+0000","last_clean":"2023-08-27T20:51:41.709177+0000","last_became_active":"2023-08-27T20:51:18.124278+0000","last_became_peered":"2023-08-27T20:51:18.124278+0000","last_unstale":"2023-08-27T20:51:41.709177+0000","last_undegraded":"2023-08-27T20:51:41.709177+0000","last_fullsized":"2023-08-27T20:51:41.709177+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:27:56.736173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740429+0000","last_change":"2023-08-27T20:51:18.097033+0000","last_active":"2023-08-27T20:51:41.740429+0000","last_peered":"2023-08-27T20:51:41.740429+0000","last_clean":"2023-08-27T20:51:41.740429+0000","last_became_active":"2023-08-27T20:51:18.096760+0000","last_became_peered":"2023-08-27T20:51:18.096760+0000","last_unstale":"2023-08-27T20:51:41.740429+0000","last_undegraded":"2023-08-27T20:51:41.740429+0000","last_fullsized":"2023-08-27T20:51:41.740429+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:22:34.415568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710480+0000","last_change":"2023-08-27T20:50:40.472176+0000","last_active":"2023-08-27T20:51:41.710480+0000","last_peered":"2023-08-27T20:51:41.710480+0000","last_clean":"2023-08-27T20:51:41.710480+0000","last_became_active":"2023-08-27T20:50:40.471979+0000","last_became_peered":"2023-08-27T20:50:40.471979+0000","last_unstale":"2023-08-27T20:51:41.710480+0000","last_undegraded":"2023-08-27T20:51:41.710480+0000","last_fullsized":"2023-08-27T20:51:41.710480+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:16:03.929633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708629+0000","last_change":"2023-08-27T20:51:18.175070+0000","last_active":"2023-08-27T20:51:41.708629+0000","last_peered":"2023-08-27T20:51:41.708629+0000","last_clean":"2023-08-27T20:51:41.708629+0000","last_became_active":"2023-08-27T20:51:18.174799+0000","last_became_peered":"2023-08-27T20:51:18.174799+0000","last_unstale":"2023-08-27T20:51:41.708629+0000","last_undegraded":"2023-08-27T20:51:41.708629+0000","last_fullsized":"2023-08-27T20:51:41.708629+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:04:36.156817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706214+0000","last_change":"2023-08-27T20:50:40.474630+0000","last_active":"2023-08-27T20:51:41.706214+0000","last_peered":"2023-08-27T20:51:41.706214+0000","last_clean":"2023-08-27T20:51:41.706214+0000","last_became_active":"2023-08-27T20:50:40.474345+0000","last_became_peered":"2023-08-27T20:50:40.474345+0000","last_unstale":"2023-08-27T20:51:41.706214+0000","last_undegraded":"2023-08-27T20:51:41.706214+0000","last_fullsized":"2023-08-27T20:51:41.706214+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:35:03.409338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710133+0000","last_change":"2023-08-27T20:51:18.103183+0000","last_active":"2023-08-27T20:51:41.710133+0000","last_peered":"2023-08-27T20:51:41.710133+0000","last_clean":"2023-08-27T20:51:41.710133+0000","last_became_active":"2023-08-27T20:51:18.102921+0000","last_became_peered":"2023-08-27T20:51:18.102921+0000","last_unstale":"2023-08-27T20:51:41.710133+0000","last_undegraded":"2023-08-27T20:51:41.710133+0000","last_fullsized":"2023-08-27T20:51:41.710133+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:47:12.209253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740223+0000","last_change":"2023-08-27T20:51:18.175452+0000","last_active":"2023-08-27T20:51:41.740223+0000","last_peered":"2023-08-27T20:51:41.740223+0000","last_clean":"2023-08-27T20:51:41.740223+0000","last_became_active":"2023-08-27T20:51:18.174003+0000","last_became_peered":"2023-08-27T20:51:18.174003+0000","last_unstale":"2023-08-27T20:51:41.740223+0000","last_undegraded":"2023-08-27T20:51:41.740223+0000","last_fullsized":"2023-08-27T20:51:41.740223+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:26:59.035718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705919+0000","last_change":"2023-08-27T20:51:18.101498+0000","last_active":"2023-08-27T20:51:41.705919+0000","last_peered":"2023-08-27T20:51:41.705919+0000","last_clean":"2023-08-27T20:51:41.705919+0000","last_became_active":"2023-08-27T20:51:18.101220+0000","last_became_peered":"2023-08-27T20:51:18.101220+0000","last_unstale":"2023-08-27T20:51:41.705919+0000","last_undegraded":"2023-08-27T20:51:41.705919+0000","last_fullsized":"2023-08-27T20:51:41.705919+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:27:28.524484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936758+0000","last_change":"2023-08-27T20:50:41.004048+0000","last_active":"2023-08-27T20:51:41.936758+0000","last_peered":"2023-08-27T20:51:41.936758+0000","last_clean":"2023-08-27T20:51:41.936758+0000","last_became_active":"2023-08-27T20:50:41.003731+0000","last_became_peered":"2023-08-27T20:50:41.003731+0000","last_unstale":"2023-08-27T20:51:41.936758+0000","last_undegraded":"2023-08-27T20:51:41.936758+0000","last_fullsized":"2023-08-27T20:51:41.936758+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:58.339578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740432+0000","last_change":"2023-08-27T20:51:18.132942+0000","last_active":"2023-08-27T20:51:41.740432+0000","last_peered":"2023-08-27T20:51:41.740432+0000","last_clean":"2023-08-27T20:51:41.740432+0000","last_became_active":"2023-08-27T20:51:18.132641+0000","last_became_peered":"2023-08-27T20:51:18.132641+0000","last_unstale":"2023-08-27T20:51:41.740432+0000","last_undegraded":"2023-08-27T20:51:41.740432+0000","last_fullsized":"2023-08-27T20:51:41.740432+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:16:24.405549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740170+0000","last_change":"2023-08-27T20:51:18.096016+0000","last_active":"2023-08-27T20:51:41.740170+0000","last_peered":"2023-08-27T20:51:41.740170+0000","last_clean":"2023-08-27T20:51:41.740170+0000","last_became_active":"2023-08-27T20:51:18.095736+0000","last_became_peered":"2023-08-27T20:51:18.095736+0000","last_unstale":"2023-08-27T20:51:41.740170+0000","last_undegraded":"2023-08-27T20:51:41.740170+0000","last_fullsized":"2023-08-27T20:51:41.740170+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:22:49.189530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936564+0000","last_change":"2023-08-27T20:50:41.004114+0000","last_active":"2023-08-27T20:51:41.936564+0000","last_peered":"2023-08-27T20:51:41.936564+0000","last_clean":"2023-08-27T20:51:41.936564+0000","last_became_active":"2023-08-27T20:50:41.003717+0000","last_became_peered":"2023-08-27T20:50:41.003717+0000","last_unstale":"2023-08-27T20:51:41.936564+0000","last_undegraded":"2023-08-27T20:51:41.936564+0000","last_fullsized":"2023-08-27T20:51:41.936564+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:24:46.799377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710310+0000","last_change":"2023-08-27T20:51:18.174265+0000","last_active":"2023-08-27T20:51:41.710310+0000","last_peered":"2023-08-27T20:51:41.710310+0000","last_clean":"2023-08-27T20:51:41.710310+0000","last_became_active":"2023-08-27T20:51:18.173954+0000","last_became_peered":"2023-08-27T20:51:18.173954+0000","last_unstale":"2023-08-27T20:51:41.710310+0000","last_undegraded":"2023-08-27T20:51:41.710310+0000","last_fullsized":"2023-08-27T20:51:41.710310+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:07:21.475116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709718+0000","last_change":"2023-08-27T20:51:18.105105+0000","last_active":"2023-08-27T20:51:41.709718+0000","last_peered":"2023-08-27T20:51:41.709718+0000","last_clean":"2023-08-27T20:51:41.709718+0000","last_became_active":"2023-08-27T20:51:18.104814+0000","last_became_peered":"2023-08-27T20:51:18.104814+0000","last_unstale":"2023-08-27T20:51:41.709718+0000","last_undegraded":"2023-08-27T20:51:41.709718+0000","last_fullsized":"2023-08-27T20:51:41.709718+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:48:25.708690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"58'128","reported_seq":276,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936456+0000","last_change":"2023-08-27T20:50:22.534100+0000","last_active":"2023-08-27T20:51:41.936456+0000","last_peered":"2023-08-27T20:51:41.936456+0000","last_clean":"2023-08-27T20:51:41.936456+0000","last_became_active":"2023-08-27T20:50:22.533604+0000","last_became_peered":"2023-08-27T20:50:22.533604+0000","last_unstale":"2023-08-27T20:51:41.936456+0000","last_undegraded":"2023-08-27T20:51:41.936456+0000","last_fullsized":"2023-08-27T20:51:41.936456+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_clean_scrub_stamp":"2023-08-27T20:48:59.426866+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:50:03.653850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706260+0000","last_change":"2023-08-27T20:50:40.472409+0000","last_active":"2023-08-27T20:51:41.706260+0000","last_peered":"2023-08-27T20:51:41.706260+0000","last_clean":"2023-08-27T20:51:41.706260+0000","last_became_active":"2023-08-27T20:50:40.472101+0000","last_became_peered":"2023-08-27T20:50:40.472101+0000","last_unstale":"2023-08-27T20:51:41.706260+0000","last_undegraded":"2023-08-27T20:51:41.706260+0000","last_fullsized":"2023-08-27T20:51:41.706260+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:37:42.899620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707058+0000","last_change":"2023-08-27T20:51:18.128769+0000","last_active":"2023-08-27T20:51:41.707058+0000","last_peered":"2023-08-27T20:51:41.707058+0000","last_clean":"2023-08-27T20:51:41.707058+0000","last_became_active":"2023-08-27T20:51:18.128121+0000","last_became_peered":"2023-08-27T20:51:18.128121+0000","last_unstale":"2023-08-27T20:51:41.707058+0000","last_undegraded":"2023-08-27T20:51:41.707058+0000","last_fullsized":"2023-08-27T20:51:41.707058+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:54:21.500010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705110+0000","last_change":"2023-08-27T20:51:18.102189+0000","last_active":"2023-08-27T20:51:41.705110+0000","last_peered":"2023-08-27T20:51:41.705110+0000","last_clean":"2023-08-27T20:51:41.705110+0000","last_became_active":"2023-08-27T20:51:18.101928+0000","last_became_peered":"2023-08-27T20:51:18.101928+0000","last_unstale":"2023-08-27T20:51:41.705110+0000","last_undegraded":"2023-08-27T20:51:41.705110+0000","last_fullsized":"2023-08-27T20:51:41.705110+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:39:54.277133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705393+0000","last_change":"2023-08-27T20:50:41.003113+0000","last_active":"2023-08-27T20:51:41.705393+0000","last_peered":"2023-08-27T20:51:41.705393+0000","last_clean":"2023-08-27T20:51:41.705393+0000","last_became_active":"2023-08-27T20:50:41.002820+0000","last_became_peered":"2023-08-27T20:50:41.002820+0000","last_unstale":"2023-08-27T20:51:41.705393+0000","last_undegraded":"2023-08-27T20:51:41.705393+0000","last_fullsized":"2023-08-27T20:51:41.705393+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:03:10.585918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705840+0000","last_change":"2023-08-27T20:51:18.133061+0000","last_active":"2023-08-27T20:51:41.705840+0000","last_peered":"2023-08-27T20:51:41.705840+0000","last_clean":"2023-08-27T20:51:41.705840+0000","last_became_active":"2023-08-27T20:51:18.132737+0000","last_became_peered":"2023-08-27T20:51:18.132737+0000","last_unstale":"2023-08-27T20:51:41.705840+0000","last_undegraded":"2023-08-27T20:51:41.705840+0000","last_fullsized":"2023-08-27T20:51:41.705840+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:38:31.433133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707681+0000","last_change":"2023-08-27T20:51:18.102990+0000","last_active":"2023-08-27T20:51:41.707681+0000","last_peered":"2023-08-27T20:51:41.707681+0000","last_clean":"2023-08-27T20:51:41.707681+0000","last_became_active":"2023-08-27T20:51:18.102475+0000","last_became_peered":"2023-08-27T20:51:18.102475+0000","last_unstale":"2023-08-27T20:51:41.707681+0000","last_undegraded":"2023-08-27T20:51:41.707681+0000","last_fullsized":"2023-08-27T20:51:41.707681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T20:52:23.666385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706322+0000","last_change":"2023-08-27T20:50:40.475080+0000","last_active":"2023-08-27T20:51:41.706322+0000","last_peered":"2023-08-27T20:51:41.706322+0000","last_clean":"2023-08-27T20:51:41.706322+0000","last_became_active":"2023-08-27T20:50:40.474840+0000","last_became_peered":"2023-08-27T20:50:40.474840+0000","last_unstale":"2023-08-27T20:51:41.706322+0000","last_undegraded":"2023-08-27T20:51:41.706322+0000","last_fullsized":"2023-08-27T20:51:41.706322+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:35:31.367035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705486+0000","last_change":"2023-08-27T20:51:18.125800+0000","last_active":"2023-08-27T20:51:41.705486+0000","last_peered":"2023-08-27T20:51:41.705486+0000","last_clean":"2023-08-27T20:51:41.705486+0000","last_became_active":"2023-08-27T20:51:18.125533+0000","last_became_peered":"2023-08-27T20:51:18.125533+0000","last_unstale":"2023-08-27T20:51:41.705486+0000","last_undegraded":"2023-08-27T20:51:41.705486+0000","last_fullsized":"2023-08-27T20:51:41.705486+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:58:23.165148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706417+0000","last_change":"2023-08-27T20:51:18.098948+0000","last_active":"2023-08-27T20:51:41.706417+0000","last_peered":"2023-08-27T20:51:41.706417+0000","last_clean":"2023-08-27T20:51:41.706417+0000","last_became_active":"2023-08-27T20:51:18.098648+0000","last_became_peered":"2023-08-27T20:51:18.098648+0000","last_unstale":"2023-08-27T20:51:41.706417+0000","last_undegraded":"2023-08-27T20:51:41.706417+0000","last_fullsized":"2023-08-27T20:51:41.706417+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:57:40.046678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705976+0000","last_change":"2023-08-27T20:50:41.004078+0000","last_active":"2023-08-27T20:51:41.705976+0000","last_peered":"2023-08-27T20:51:41.705976+0000","last_clean":"2023-08-27T20:51:41.705976+0000","last_became_active":"2023-08-27T20:50:41.003803+0000","last_became_peered":"2023-08-27T20:50:41.003803+0000","last_unstale":"2023-08-27T20:51:41.705976+0000","last_undegraded":"2023-08-27T20:51:41.705976+0000","last_fullsized":"2023-08-27T20:51:41.705976+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:14.109789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-08-27T20:51:42.784026+0000","last_change":"2023-08-27T20:51:42.784026+0000","last_active":"2023-08-27T20:51:42.784026+0000","last_peered":"2023-08-27T20:51:42.784026+0000","last_clean":"2023-08-27T20:51:42.784026+0000","last_became_active":"2023-08-27T20:51:42.783779+0000","last_became_peered":"2023-08-27T20:51:42.783779+0000","last_unstale":"2023-08-27T20:51:42.784026+0000","last_undegraded":"2023-08-27T20:51:42.784026+0000","last_fullsized":"2023-08-27T20:51:42.784026+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:11:19.094709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708652+0000","last_change":"2023-08-27T20:51:18.101235+0000","last_active":"2023-08-27T20:51:41.708652+0000","last_peered":"2023-08-27T20:51:41.708652+0000","last_clean":"2023-08-27T20:51:41.708652+0000","last_became_active":"2023-08-27T20:51:18.100949+0000","last_became_peered":"2023-08-27T20:51:18.100949+0000","last_unstale":"2023-08-27T20:51:41.708652+0000","last_undegraded":"2023-08-27T20:51:41.708652+0000","last_fullsized":"2023-08-27T20:51:41.708652+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:22:26.784792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-08-27T20:51:42.749070+0000","last_change":"2023-08-27T20:51:42.749070+0000","last_active":"2023-08-27T20:51:42.749070+0000","last_peered":"2023-08-27T20:51:42.749070+0000","last_clean":"2023-08-27T20:51:42.749070+0000","last_became_active":"2023-08-27T20:51:42.748736+0000","last_became_peered":"2023-08-27T20:51:42.748736+0000","last_unstale":"2023-08-27T20:51:42.749070+0000","last_undegraded":"2023-08-27T20:51:42.749070+0000","last_fullsized":"2023-08-27T20:51:42.749070+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:58:27.930976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709293+0000","last_change":"2023-08-27T20:51:18.104763+0000","last_active":"2023-08-27T20:51:41.709293+0000","last_peered":"2023-08-27T20:51:41.709293+0000","last_clean":"2023-08-27T20:51:41.709293+0000","last_became_active":"2023-08-27T20:51:18.104258+0000","last_became_peered":"2023-08-27T20:51:18.104258+0000","last_unstale":"2023-08-27T20:51:41.709293+0000","last_undegraded":"2023-08-27T20:51:41.709293+0000","last_fullsized":"2023-08-27T20:51:41.709293+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:23:37.009396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708140+0000","last_change":"2023-08-27T20:51:18.127756+0000","last_active":"2023-08-27T20:51:41.708140+0000","last_peered":"2023-08-27T20:51:41.708140+0000","last_clean":"2023-08-27T20:51:41.708140+0000","last_became_active":"2023-08-27T20:51:18.127374+0000","last_became_peered":"2023-08-27T20:51:18.127374+0000","last_unstale":"2023-08-27T20:51:41.708140+0000","last_undegraded":"2023-08-27T20:51:41.708140+0000","last_fullsized":"2023-08-27T20:51:41.708140+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:56:07.099266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710335+0000","last_change":"2023-08-27T20:51:18.108839+0000","last_active":"2023-08-27T20:51:41.710335+0000","last_peered":"2023-08-27T20:51:41.710335+0000","last_clean":"2023-08-27T20:51:41.710335+0000","last_became_active":"2023-08-27T20:51:18.108560+0000","last_became_peered":"2023-08-27T20:51:18.108560+0000","last_unstale":"2023-08-27T20:51:41.710335+0000","last_undegraded":"2023-08-27T20:51:41.710335+0000","last_fullsized":"2023-08-27T20:51:41.710335+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:36:37.649085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707720+0000","last_change":"2023-08-27T20:51:18.129095+0000","last_active":"2023-08-27T20:51:41.707720+0000","last_peered":"2023-08-27T20:51:41.707720+0000","last_clean":"2023-08-27T20:51:41.707720+0000","last_became_active":"2023-08-27T20:51:18.128667+0000","last_became_peered":"2023-08-27T20:51:18.128667+0000","last_unstale":"2023-08-27T20:51:41.707720+0000","last_undegraded":"2023-08-27T20:51:41.707720+0000","last_fullsized":"2023-08-27T20:51:41.707720+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:14:50.669396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709781+0000","last_change":"2023-08-27T20:51:18.096242+0000","last_active":"2023-08-27T20:51:41.709781+0000","last_peered":"2023-08-27T20:51:41.709781+0000","last_clean":"2023-08-27T20:51:41.709781+0000","last_became_active":"2023-08-27T20:51:18.096012+0000","last_became_peered":"2023-08-27T20:51:18.096012+0000","last_unstale":"2023-08-27T20:51:41.709781+0000","last_undegraded":"2023-08-27T20:51:41.709781+0000","last_fullsized":"2023-08-27T20:51:41.709781+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:07:12.658963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740366+0000","last_change":"2023-08-27T20:51:18.132475+0000","last_active":"2023-08-27T20:51:41.740366+0000","last_peered":"2023-08-27T20:51:41.740366+0000","last_clean":"2023-08-27T20:51:41.740366+0000","last_became_active":"2023-08-27T20:51:18.132202+0000","last_became_peered":"2023-08-27T20:51:18.132202+0000","last_unstale":"2023-08-27T20:51:41.740366+0000","last_undegraded":"2023-08-27T20:51:41.740366+0000","last_fullsized":"2023-08-27T20:51:41.740366+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:59:04.704593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706343+0000","last_change":"2023-08-27T20:51:18.102046+0000","last_active":"2023-08-27T20:51:41.706343+0000","last_peered":"2023-08-27T20:51:41.706343+0000","last_clean":"2023-08-27T20:51:41.706343+0000","last_became_active":"2023-08-27T20:51:18.101748+0000","last_became_peered":"2023-08-27T20:51:18.101748+0000","last_unstale":"2023-08-27T20:51:41.706343+0000","last_undegraded":"2023-08-27T20:51:41.706343+0000","last_fullsized":"2023-08-27T20:51:41.706343+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:23:02.841371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710127+0000","last_change":"2023-08-27T20:51:18.174360+0000","last_active":"2023-08-27T20:51:41.710127+0000","last_peered":"2023-08-27T20:51:41.710127+0000","last_clean":"2023-08-27T20:51:41.710127+0000","last_became_active":"2023-08-27T20:51:18.174027+0000","last_became_peered":"2023-08-27T20:51:18.174027+0000","last_unstale":"2023-08-27T20:51:41.710127+0000","last_undegraded":"2023-08-27T20:51:41.710127+0000","last_fullsized":"2023-08-27T20:51:41.710127+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:51.281103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936830+0000","last_change":"2023-08-27T20:51:18.102162+0000","last_active":"2023-08-27T20:51:41.936830+0000","last_peered":"2023-08-27T20:51:41.936830+0000","last_clean":"2023-08-27T20:51:41.936830+0000","last_became_active":"2023-08-27T20:51:18.101864+0000","last_became_peered":"2023-08-27T20:51:18.101864+0000","last_unstale":"2023-08-27T20:51:41.936830+0000","last_undegraded":"2023-08-27T20:51:41.936830+0000","last_fullsized":"2023-08-27T20:51:41.936830+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:31.686394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-08-27T20:51:42.784195+0000","last_change":"2023-08-27T20:51:42.784195+0000","last_active":"2023-08-27T20:51:42.784195+0000","last_peered":"2023-08-27T20:51:42.784195+0000","last_clean":"2023-08-27T20:51:42.784195+0000","last_became_active":"2023-08-27T20:51:42.783951+0000","last_became_peered":"2023-08-27T20:51:42.783951+0000","last_unstale":"2023-08-27T20:51:42.784195+0000","last_undegraded":"2023-08-27T20:51:42.784195+0000","last_fullsized":"2023-08-27T20:51:42.784195+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:45.897182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936811+0000","last_change":"2023-08-27T20:51:18.104712+0000","last_active":"2023-08-27T20:51:41.936811+0000","last_peered":"2023-08-27T20:51:41.936811+0000","last_clean":"2023-08-27T20:51:41.936811+0000","last_became_active":"2023-08-27T20:51:18.104420+0000","last_became_peered":"2023-08-27T20:51:18.104420+0000","last_unstale":"2023-08-27T20:51:41.936811+0000","last_undegraded":"2023-08-27T20:51:41.936811+0000","last_fullsized":"2023-08-27T20:51:41.936811+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:26:27.616548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706156+0000","last_change":"2023-08-27T20:51:18.174616+0000","last_active":"2023-08-27T20:51:41.706156+0000","last_peered":"2023-08-27T20:51:41.706156+0000","last_clean":"2023-08-27T20:51:41.706156+0000","last_became_active":"2023-08-27T20:51:18.173970+0000","last_became_peered":"2023-08-27T20:51:18.173970+0000","last_unstale":"2023-08-27T20:51:41.706156+0000","last_undegraded":"2023-08-27T20:51:41.706156+0000","last_fullsized":"2023-08-27T20:51:41.706156+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:17:07.500477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936914+0000","last_change":"2023-08-27T20:51:18.100527+0000","last_active":"2023-08-27T20:51:41.936914+0000","last_peered":"2023-08-27T20:51:41.936914+0000","last_clean":"2023-08-27T20:51:41.936914+0000","last_became_active":"2023-08-27T20:51:18.100150+0000","last_became_peered":"2023-08-27T20:51:18.100150+0000","last_unstale":"2023-08-27T20:51:41.936914+0000","last_undegraded":"2023-08-27T20:51:41.936914+0000","last_fullsized":"2023-08-27T20:51:41.936914+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:31:30.303986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710029+0000","last_change":"2023-08-27T20:51:18.174292+0000","last_active":"2023-08-27T20:51:41.710029+0000","last_peered":"2023-08-27T20:51:41.710029+0000","last_clean":"2023-08-27T20:51:41.710029+0000","last_became_active":"2023-08-27T20:51:18.173994+0000","last_became_peered":"2023-08-27T20:51:18.173994+0000","last_unstale":"2023-08-27T20:51:41.710029+0000","last_undegraded":"2023-08-27T20:51:41.710029+0000","last_fullsized":"2023-08-27T20:51:41.710029+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:16:13.606362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":21,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710086+0000","last_change":"2023-08-27T20:51:18.105714+0000","last_active":"2023-08-27T20:51:41.710086+0000","last_peered":"2023-08-27T20:51:41.710086+0000","last_clean":"2023-08-27T20:51:41.710086+0000","last_became_active":"2023-08-27T20:51:18.105364+0000","last_became_peered":"2023-08-27T20:51:18.105364+0000","last_unstale":"2023-08-27T20:51:41.710086+0000","last_undegraded":"2023-08-27T20:51:41.710086+0000","last_fullsized":"2023-08-27T20:51:41.710086+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:47.202084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740323+0000","last_change":"2023-08-27T20:51:18.176875+0000","last_active":"2023-08-27T20:51:41.740323+0000","last_peered":"2023-08-27T20:51:41.740323+0000","last_clean":"2023-08-27T20:51:41.740323+0000","last_became_active":"2023-08-27T20:51:18.176547+0000","last_became_peered":"2023-08-27T20:51:18.176547+0000","last_unstale":"2023-08-27T20:51:41.740323+0000","last_undegraded":"2023-08-27T20:51:41.740323+0000","last_fullsized":"2023-08-27T20:51:41.740323+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:13:16.094024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936591+0000","last_change":"2023-08-27T20:51:18.098165+0000","last_active":"2023-08-27T20:51:41.936591+0000","last_peered":"2023-08-27T20:51:41.936591+0000","last_clean":"2023-08-27T20:51:41.936591+0000","last_became_active":"2023-08-27T20:51:18.097846+0000","last_became_peered":"2023-08-27T20:51:18.097846+0000","last_unstale":"2023-08-27T20:51:41.936591+0000","last_undegraded":"2023-08-27T20:51:41.936591+0000","last_fullsized":"2023-08-27T20:51:41.936591+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:43:32.430484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706324+0000","last_change":"2023-08-27T20:51:18.174803+0000","last_active":"2023-08-27T20:51:41.706324+0000","last_peered":"2023-08-27T20:51:41.706324+0000","last_clean":"2023-08-27T20:51:41.706324+0000","last_became_active":"2023-08-27T20:51:18.174313+0000","last_became_peered":"2023-08-27T20:51:18.174313+0000","last_unstale":"2023-08-27T20:51:41.706324+0000","last_undegraded":"2023-08-27T20:51:41.706324+0000","last_fullsized":"2023-08-27T20:51:41.706324+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:10:32.503697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740200+0000","last_change":"2023-08-27T20:51:18.097273+0000","last_active":"2023-08-27T20:51:41.740200+0000","last_peered":"2023-08-27T20:51:41.740200+0000","last_clean":"2023-08-27T20:51:41.740200+0000","last_became_active":"2023-08-27T20:51:18.096940+0000","last_became_peered":"2023-08-27T20:51:18.096940+0000","last_unstale":"2023-08-27T20:51:41.740200+0000","last_undegraded":"2023-08-27T20:51:41.740200+0000","last_fullsized":"2023-08-27T20:51:41.740200+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:01:47.450662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709885+0000","last_change":"2023-08-27T20:51:18.174487+0000","last_active":"2023-08-27T20:51:41.709885+0000","last_peered":"2023-08-27T20:51:41.709885+0000","last_clean":"2023-08-27T20:51:41.709885+0000","last_became_active":"2023-08-27T20:51:18.174142+0000","last_became_peered":"2023-08-27T20:51:18.174142+0000","last_unstale":"2023-08-27T20:51:41.709885+0000","last_undegraded":"2023-08-27T20:51:41.709885+0000","last_fullsized":"2023-08-27T20:51:41.709885+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:21:00.888579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936595+0000","last_change":"2023-08-27T20:51:18.096913+0000","last_active":"2023-08-27T20:51:41.936595+0000","last_peered":"2023-08-27T20:51:41.936595+0000","last_clean":"2023-08-27T20:51:41.936595+0000","last_became_active":"2023-08-27T20:51:18.096600+0000","last_became_peered":"2023-08-27T20:51:18.096600+0000","last_unstale":"2023-08-27T20:51:41.936595+0000","last_undegraded":"2023-08-27T20:51:41.936595+0000","last_fullsized":"2023-08-27T20:51:41.936595+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:41:23.301928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707895+0000","last_change":"2023-08-27T20:51:18.131165+0000","last_active":"2023-08-27T20:51:41.707895+0000","last_peered":"2023-08-27T20:51:41.707895+0000","last_clean":"2023-08-27T20:51:41.707895+0000","last_became_active":"2023-08-27T20:51:18.130812+0000","last_became_peered":"2023-08-27T20:51:18.130812+0000","last_unstale":"2023-08-27T20:51:41.707895+0000","last_undegraded":"2023-08-27T20:51:41.707895+0000","last_fullsized":"2023-08-27T20:51:41.707895+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:17:11.521086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"63'6","reported_seq":19,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707844+0000","last_change":"2023-08-27T20:51:18.106623+0000","last_active":"2023-08-27T20:51:41.707844+0000","last_peered":"2023-08-27T20:51:41.707844+0000","last_clean":"2023-08-27T20:51:41.707844+0000","last_became_active":"2023-08-27T20:51:18.106343+0000","last_became_peered":"2023-08-27T20:51:18.106343+0000","last_unstale":"2023-08-27T20:51:41.707844+0000","last_undegraded":"2023-08-27T20:51:41.707844+0000","last_fullsized":"2023-08-27T20:51:41.707844+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:43:08.157700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705485+0000","last_change":"2023-08-27T20:51:18.126515+0000","last_active":"2023-08-27T20:51:41.705485+0000","last_peered":"2023-08-27T20:51:41.705485+0000","last_clean":"2023-08-27T20:51:41.705485+0000","last_became_active":"2023-08-27T20:51:18.123514+0000","last_became_peered":"2023-08-27T20:51:18.123514+0000","last_unstale":"2023-08-27T20:51:41.705485+0000","last_undegraded":"2023-08-27T20:51:41.705485+0000","last_fullsized":"2023-08-27T20:51:41.705485+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:00.412812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936738+0000","last_change":"2023-08-27T20:51:18.102959+0000","last_active":"2023-08-27T20:51:41.936738+0000","last_peered":"2023-08-27T20:51:41.936738+0000","last_clean":"2023-08-27T20:51:41.936738+0000","last_became_active":"2023-08-27T20:51:18.102698+0000","last_became_peered":"2023-08-27T20:51:18.102698+0000","last_unstale":"2023-08-27T20:51:41.936738+0000","last_undegraded":"2023-08-27T20:51:41.936738+0000","last_fullsized":"2023-08-27T20:51:41.936738+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:00:59.591758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709306+0000","last_change":"2023-08-27T20:51:18.174261+0000","last_active":"2023-08-27T20:51:41.709306+0000","last_peered":"2023-08-27T20:51:41.709306+0000","last_clean":"2023-08-27T20:51:41.709306+0000","last_became_active":"2023-08-27T20:51:18.174020+0000","last_became_peered":"2023-08-27T20:51:18.174020+0000","last_unstale":"2023-08-27T20:51:41.709306+0000","last_undegraded":"2023-08-27T20:51:41.709306+0000","last_fullsized":"2023-08-27T20:51:41.709306+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:06:33.961645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706383+0000","last_change":"2023-08-27T20:51:18.097098+0000","last_active":"2023-08-27T20:51:41.706383+0000","last_peered":"2023-08-27T20:51:41.706383+0000","last_clean":"2023-08-27T20:51:41.706383+0000","last_became_active":"2023-08-27T20:51:18.096745+0000","last_became_peered":"2023-08-27T20:51:18.096745+0000","last_unstale":"2023-08-27T20:51:41.706383+0000","last_undegraded":"2023-08-27T20:51:41.706383+0000","last_fullsized":"2023-08-27T20:51:41.706383+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:42:05.626352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709158+0000","last_change":"2023-08-27T20:51:18.129959+0000","last_active":"2023-08-27T20:51:41.709158+0000","last_peered":"2023-08-27T20:51:41.709158+0000","last_clean":"2023-08-27T20:51:41.709158+0000","last_became_active":"2023-08-27T20:51:18.129690+0000","last_became_peered":"2023-08-27T20:51:18.129690+0000","last_unstale":"2023-08-27T20:51:41.709158+0000","last_undegraded":"2023-08-27T20:51:41.709158+0000","last_fullsized":"2023-08-27T20:51:41.709158+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:35:08.487865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740426+0000","last_change":"2023-08-27T20:51:18.097550+0000","last_active":"2023-08-27T20:51:41.740426+0000","last_peered":"2023-08-27T20:51:41.740426+0000","last_clean":"2023-08-27T20:51:41.740426+0000","last_became_active":"2023-08-27T20:51:18.097145+0000","last_became_peered":"2023-08-27T20:51:18.097145+0000","last_unstale":"2023-08-27T20:51:41.740426+0000","last_undegraded":"2023-08-27T20:51:41.740426+0000","last_fullsized":"2023-08-27T20:51:41.740426+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:36:26.771185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710455+0000","last_change":"2023-08-27T20:51:18.128634+0000","last_active":"2023-08-27T20:51:41.710455+0000","last_peered":"2023-08-27T20:51:41.710455+0000","last_clean":"2023-08-27T20:51:41.710455+0000","last_became_active":"2023-08-27T20:51:18.128367+0000","last_became_peered":"2023-08-27T20:51:18.128367+0000","last_unstale":"2023-08-27T20:51:41.710455+0000","last_undegraded":"2023-08-27T20:51:41.710455+0000","last_fullsized":"2023-08-27T20:51:41.710455+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:51:25.311153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708754+0000","last_change":"2023-08-27T20:51:18.094635+0000","last_active":"2023-08-27T20:51:41.708754+0000","last_peered":"2023-08-27T20:51:41.708754+0000","last_clean":"2023-08-27T20:51:41.708754+0000","last_became_active":"2023-08-27T20:51:18.094323+0000","last_became_peered":"2023-08-27T20:51:18.094323+0000","last_unstale":"2023-08-27T20:51:41.708754+0000","last_undegraded":"2023-08-27T20:51:41.708754+0000","last_fullsized":"2023-08-27T20:51:41.708754+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:29:06.163259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740325+0000","last_change":"2023-08-27T20:51:18.174303+0000","last_active":"2023-08-27T20:51:41.740325+0000","last_peered":"2023-08-27T20:51:41.740325+0000","last_clean":"2023-08-27T20:51:41.740325+0000","last_became_active":"2023-08-27T20:51:18.173979+0000","last_became_peered":"2023-08-27T20:51:18.173979+0000","last_unstale":"2023-08-27T20:51:41.740325+0000","last_undegraded":"2023-08-27T20:51:41.740325+0000","last_fullsized":"2023-08-27T20:51:41.740325+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T20:58:16.258007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705161+0000","last_change":"2023-08-27T20:51:18.100237+0000","last_active":"2023-08-27T20:51:41.705161+0000","last_peered":"2023-08-27T20:51:41.705161+0000","last_clean":"2023-08-27T20:51:41.705161+0000","last_became_active":"2023-08-27T20:51:18.099976+0000","last_became_peered":"2023-08-27T20:51:18.099976+0000","last_unstale":"2023-08-27T20:51:41.705161+0000","last_undegraded":"2023-08-27T20:51:41.705161+0000","last_fullsized":"2023-08-27T20:51:41.705161+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:46:15.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710048+0000","last_change":"2023-08-27T20:51:18.129248+0000","last_active":"2023-08-27T20:51:41.710048+0000","last_peered":"2023-08-27T20:51:41.710048+0000","last_clean":"2023-08-27T20:51:41.710048+0000","last_became_active":"2023-08-27T20:51:18.128963+0000","last_became_peered":"2023-08-27T20:51:18.128963+0000","last_unstale":"2023-08-27T20:51:41.710048+0000","last_undegraded":"2023-08-27T20:51:41.710048+0000","last_fullsized":"2023-08-27T20:51:41.710048+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:31:19.516025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707525+0000","last_change":"2023-08-27T20:51:18.103078+0000","last_active":"2023-08-27T20:51:41.707525+0000","last_peered":"2023-08-27T20:51:41.707525+0000","last_clean":"2023-08-27T20:51:41.707525+0000","last_became_active":"2023-08-27T20:51:18.102289+0000","last_became_peered":"2023-08-27T20:51:18.102289+0000","last_unstale":"2023-08-27T20:51:41.707525+0000","last_undegraded":"2023-08-27T20:51:41.707525+0000","last_fullsized":"2023-08-27T20:51:41.707525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:46:25.395790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":3,"up_from":25,"seq":107374182433,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":648,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":663552,"data_stored":287109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:16 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.36199999999999999,"15min":0.36799999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"last":0.307},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.442,"15min":0.45200000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.57199999999999995,"5min":0.628,"15min":0.628},"last":0.34100000000000003}]},{"osd":1,"last update":"Sun Aug 27 20:51:16 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.44600000000000001,"15min":0.45800000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.48399999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.432,"15min":0.44},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57399999999999995,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.32200000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:16 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.47699999999999998,"15min":0.48899999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.59099999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.36599999999999999}]},{"osd":4,"last update":"Sun Aug 27 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.56299999999999994,"15min":0.57199999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.59899999999999998,"15min":0.59299999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.907,"5min":2.907,"15min":2.907},"last":0.76100000000000001}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.80800000000000005}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.71699999999999997}]},{"osd":7,"last update":"Sun Aug 27 20:51:20 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.69399999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411366,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6548,"kb_used_data":652,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738604,"statfs":{"total":95995035648,"available":95988330496,"internally_reserved":0,"allocated":667648,"data_stored":287607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.40600000000000003,"15min":0.41199999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.52500000000000002,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.34899999999999998}]},{"osd":1,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.38200000000000001}]},{"osd":3,"last update":"Sun Aug 27 20:51:18 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.45900000000000002,"15min":0.46999999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.48199999999999998,"15min":0.49199999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.433}]},{"osd":4,"last update":"Sun Aug 27 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.60699999999999998,"15min":0.60999999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.60999999999999999}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.60199999999999998}]},{"osd":6,"last update":"Sun Aug 27 20:51:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.64800000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.68400000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6732,"kb_used_data":644,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738420,"statfs":{"total":95995035648,"available":95988142080,"internally_reserved":0,"allocated":659456,"data_stored":287019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:43 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.35499999999999998,"15min":0.35299999999999998},"min":{"1min":0.23999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.377,"15min":0.36499999999999999},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38200000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:01 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.59799999999999998,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.45300000000000001,"15min":0.46300000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54400000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.28100000000000003}]},{"osd":3,"last update":"Sun Aug 27 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.46200000000000002,"15min":0.46999999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.48199999999999998,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.44700000000000001,"15min":0.45400000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.61099999999999999,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.33700000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.495}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.48399999999999999}]},{"osd":6,"last update":"Sun Aug 27 20:51:07 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.44},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57999999999999996}]},{"osd":7,"last update":"Sun Aug 27 20:51:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.52100000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738412,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7320,"kb_used_data":1232,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93737832,"statfs":{"total":95995035648,"available":95987539968,"internally_reserved":0,"allocated":1261568,"data_stored":877963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 20:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":0.34499999999999997,"15min":0.35299999999999998},"min":{"1min":0.23100000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.41599999999999998,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.33200000000000002,"5min":0.38600000000000001,"15min":0.39400000000000002},"min":{"1min":0.249,"5min":0.224,"15min":0.224},"max":{"1min":0.44900000000000001,"5min":0.63,"15min":0.63},"last":0.312}]},{"osd":2,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.41599999999999998,"15min":0.42399999999999999},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40999999999999998,"15min":0.41699999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.51800000000000002}]},{"osd":3,"last update":"Sun Aug 27 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.45900000000000002,"15min":0.47299999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58099999999999996,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.44800000000000001,"15min":0.45700000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.34100000000000003}]},{"osd":4,"last update":"Sun Aug 27 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56399999999999995,"15min":0.57099999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.748},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.57399999999999995,"15min":0.58199999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.873}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.67000000000000004}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.77100000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:51:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.64700000000000002}]}]},{"osd":7,"up_from":51,"seq":219043332117,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6568,"kb_used_data":1248,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738584,"statfs":{"total":95995035648,"available":95988310016,"internally_reserved":0,"allocated":1277952,"data_stored":878816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.55800000000000005}]},{"osd":1,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.40799999999999997}]},{"osd":2,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48399999999999999}]},{"osd":3,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.44700000000000001}]},{"osd":4,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.59599999999999997}]},{"osd":5,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.65200000000000002}]},{"osd":6,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.63700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018910,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":679936,"data_stored":288436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.55100000000000005,"15min":0.55800000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.63100000000000001,"5min":0.997,"15min":0.997},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.497,"5min":0.52700000000000002,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.627,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.44500000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.623,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.62,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.504}]},{"osd":2,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70099999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.5,"5min":0.53200000000000003,"15min":0.53800000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.66800000000000004,"5min":0.871,"15min":0.871},"last":0.47799999999999998}]},{"osd":3,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.68899999999999995,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.65900000000000003,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46200000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.36299999999999999}]},{"osd":6,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.33300000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:51:21 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.42099999999999999}]}]},{"osd":5,"up_from":37,"seq":158913789980,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6032,"kb_used_data":648,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739120,"statfs":{"total":95995035648,"available":95988858880,"internally_reserved":0,"allocated":663552,"data_stored":287378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54300000000000004}]},{"osd":1,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.47799999999999998}]},{"osd":2,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.5}]},{"osd":3,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.45300000000000001}]},{"osd":4,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.435},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.59299999999999997}]},{"osd":6,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.57199999999999995}]},{"osd":7,"last update":"Sun Aug 27 20:51:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.64800000000000002}]}]},{"osd":6,"up_from":44,"seq":188978561049,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6556,"kb_used_data":1236,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738596,"statfs":{"total":95995035648,"available":95988322304,"internally_reserved":0,"allocated":1265664,"data_stored":878670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.76300000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.583,"5min":1.583,"15min":1.583},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.64900000000000002}]},{"osd":2,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.613,"5min":3.613,"15min":3.613},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.9009999999999998,"5min":5.9009999999999998,"15min":5.9009999999999998},"last":0.71199999999999997}]},{"osd":3,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":3.7389999999999999,"5min":3.7389999999999999,"15min":3.7389999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.77800000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.47499999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.42499999999999999}]},{"osd":7,"last update":"Sun Aug 27 20:51:18 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.58199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T20:51:51.413 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T20:51:51.676 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:51 smithi093 ceph-mon[111335]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:51 smithi093 ceph-mon[111335]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:51.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:51 smithi093 ceph-mon[118902]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:51.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:51 smithi093 ceph-mon[118902]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:51 smithi136 ceph-mon[113531]: pgmap v45: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:51 smithi136 ceph-mon[113531]: from='client.24674 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:52.808 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:52.810 INFO:teuthology.orchestra.run.smithi093.stderr:dumped all 2023-08-27T20:51:53.182 INFO:teuthology.orchestra.run.smithi093.stdout:{"pg_ready":true,"pg_map":{"version":45,"stamp":"2023-08-27T20:51:50.977353+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":592674,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":224,"num_write_kb":2429,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":167,"ondisk_log_size":167,"up":219,"acting":219,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":222,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":52028,"kb_used_data":6972,"kb_used_omap":0,"kb_used_meta":44992,"kb_avail":749909188,"statfs":{"total":767960285184,"available":767907008512,"internally_reserved":0,"allocated":7139328,"data_stored":4072998,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":46071808},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"9.272468"},"pg_stats":[{"pgid":"4.18","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707376+0000","last_change":"2023-08-27T20:51:18.131159+0000","last_active":"2023-08-27T20:51:41.707376+0000","last_peered":"2023-08-27T20:51:41.707376+0000","last_clean":"2023-08-27T20:51:41.707376+0000","last_became_active":"2023-08-27T20:51:18.130806+0000","last_became_peered":"2023-08-27T20:51:18.130806+0000","last_unstale":"2023-08-27T20:51:41.707376+0000","last_undegraded":"2023-08-27T20:51:41.707376+0000","last_fullsized":"2023-08-27T20:51:41.707376+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:26:50.146935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740465+0000","last_change":"2023-08-27T20:51:18.097797+0000","last_active":"2023-08-27T20:51:41.740465+0000","last_peered":"2023-08-27T20:51:41.740465+0000","last_clean":"2023-08-27T20:51:41.740465+0000","last_became_active":"2023-08-27T20:51:18.097523+0000","last_became_peered":"2023-08-27T20:51:18.097523+0000","last_unstale":"2023-08-27T20:51:41.740465+0000","last_undegraded":"2023-08-27T20:51:41.740465+0000","last_fullsized":"2023-08-27T20:51:41.740465+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:10:27.963795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709263+0000","last_change":"2023-08-27T20:51:18.133000+0000","last_active":"2023-08-27T20:51:41.709263+0000","last_peered":"2023-08-27T20:51:41.709263+0000","last_clean":"2023-08-27T20:51:41.709263+0000","last_became_active":"2023-08-27T20:51:18.132719+0000","last_became_peered":"2023-08-27T20:51:18.132719+0000","last_unstale":"2023-08-27T20:51:41.709263+0000","last_undegraded":"2023-08-27T20:51:41.709263+0000","last_fullsized":"2023-08-27T20:51:41.709263+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:00:48.680834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710063+0000","last_change":"2023-08-27T20:51:18.102113+0000","last_active":"2023-08-27T20:51:41.710063+0000","last_peered":"2023-08-27T20:51:41.710063+0000","last_clean":"2023-08-27T20:51:41.710063+0000","last_became_active":"2023-08-27T20:51:18.101848+0000","last_became_peered":"2023-08-27T20:51:18.101848+0000","last_unstale":"2023-08-27T20:51:41.710063+0000","last_undegraded":"2023-08-27T20:51:41.710063+0000","last_fullsized":"2023-08-27T20:51:41.710063+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:49:21.087984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707664+0000","last_change":"2023-08-27T20:51:18.129808+0000","last_active":"2023-08-27T20:51:41.707664+0000","last_peered":"2023-08-27T20:51:41.707664+0000","last_clean":"2023-08-27T20:51:41.707664+0000","last_became_active":"2023-08-27T20:51:18.129515+0000","last_became_peered":"2023-08-27T20:51:18.129515+0000","last_unstale":"2023-08-27T20:51:41.707664+0000","last_undegraded":"2023-08-27T20:51:41.707664+0000","last_fullsized":"2023-08-27T20:51:41.707664+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:45:42.761867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706520+0000","last_change":"2023-08-27T20:51:18.106356+0000","last_active":"2023-08-27T20:51:41.706520+0000","last_peered":"2023-08-27T20:51:41.706520+0000","last_clean":"2023-08-27T20:51:41.706520+0000","last_became_active":"2023-08-27T20:51:18.105473+0000","last_became_peered":"2023-08-27T20:51:18.105473+0000","last_unstale":"2023-08-27T20:51:41.706520+0000","last_undegraded":"2023-08-27T20:51:41.706520+0000","last_fullsized":"2023-08-27T20:51:41.706520+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:47:12.781410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707787+0000","last_change":"2023-08-27T20:51:18.129448+0000","last_active":"2023-08-27T20:51:41.707787+0000","last_peered":"2023-08-27T20:51:41.707787+0000","last_clean":"2023-08-27T20:51:41.707787+0000","last_became_active":"2023-08-27T20:51:18.129152+0000","last_became_peered":"2023-08-27T20:51:18.129152+0000","last_unstale":"2023-08-27T20:51:41.707787+0000","last_undegraded":"2023-08-27T20:51:41.707787+0000","last_fullsized":"2023-08-27T20:51:41.707787+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:52:03.853769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706327+0000","last_change":"2023-08-27T20:51:18.096858+0000","last_active":"2023-08-27T20:51:41.706327+0000","last_peered":"2023-08-27T20:51:41.706327+0000","last_clean":"2023-08-27T20:51:41.706327+0000","last_became_active":"2023-08-27T20:51:18.095892+0000","last_became_peered":"2023-08-27T20:51:18.095892+0000","last_unstale":"2023-08-27T20:51:41.706327+0000","last_undegraded":"2023-08-27T20:51:41.706327+0000","last_fullsized":"2023-08-27T20:51:41.706327+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:27:14.781996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705301+0000","last_change":"2023-08-27T20:51:18.131530+0000","last_active":"2023-08-27T20:51:41.705301+0000","last_peered":"2023-08-27T20:51:41.705301+0000","last_clean":"2023-08-27T20:51:41.705301+0000","last_became_active":"2023-08-27T20:51:18.131263+0000","last_became_peered":"2023-08-27T20:51:18.131263+0000","last_unstale":"2023-08-27T20:51:41.705301+0000","last_undegraded":"2023-08-27T20:51:41.705301+0000","last_fullsized":"2023-08-27T20:51:41.705301+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:46.350911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740303+0000","last_change":"2023-08-27T20:51:18.098125+0000","last_active":"2023-08-27T20:51:41.740303+0000","last_peered":"2023-08-27T20:51:41.740303+0000","last_clean":"2023-08-27T20:51:41.740303+0000","last_became_active":"2023-08-27T20:51:18.097870+0000","last_became_peered":"2023-08-27T20:51:18.097870+0000","last_unstale":"2023-08-27T20:51:41.740303+0000","last_undegraded":"2023-08-27T20:51:41.740303+0000","last_fullsized":"2023-08-27T20:51:41.740303+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:32:32.322197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709177+0000","last_change":"2023-08-27T20:51:18.124571+0000","last_active":"2023-08-27T20:51:41.709177+0000","last_peered":"2023-08-27T20:51:41.709177+0000","last_clean":"2023-08-27T20:51:41.709177+0000","last_became_active":"2023-08-27T20:51:18.124278+0000","last_became_peered":"2023-08-27T20:51:18.124278+0000","last_unstale":"2023-08-27T20:51:41.709177+0000","last_undegraded":"2023-08-27T20:51:41.709177+0000","last_fullsized":"2023-08-27T20:51:41.709177+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:27:56.736173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740429+0000","last_change":"2023-08-27T20:51:18.097033+0000","last_active":"2023-08-27T20:51:41.740429+0000","last_peered":"2023-08-27T20:51:41.740429+0000","last_clean":"2023-08-27T20:51:41.740429+0000","last_became_active":"2023-08-27T20:51:18.096760+0000","last_became_peered":"2023-08-27T20:51:18.096760+0000","last_unstale":"2023-08-27T20:51:41.740429+0000","last_undegraded":"2023-08-27T20:51:41.740429+0000","last_fullsized":"2023-08-27T20:51:41.740429+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:22:34.415568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710480+0000","last_change":"2023-08-27T20:50:40.472176+0000","last_active":"2023-08-27T20:51:41.710480+0000","last_peered":"2023-08-27T20:51:41.710480+0000","last_clean":"2023-08-27T20:51:41.710480+0000","last_became_active":"2023-08-27T20:50:40.471979+0000","last_became_peered":"2023-08-27T20:50:40.471979+0000","last_unstale":"2023-08-27T20:51:41.710480+0000","last_undegraded":"2023-08-27T20:51:41.710480+0000","last_fullsized":"2023-08-27T20:51:41.710480+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:16:03.929633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708629+0000","last_change":"2023-08-27T20:51:18.175070+0000","last_active":"2023-08-27T20:51:41.708629+0000","last_peered":"2023-08-27T20:51:41.708629+0000","last_clean":"2023-08-27T20:51:41.708629+0000","last_became_active":"2023-08-27T20:51:18.174799+0000","last_became_peered":"2023-08-27T20:51:18.174799+0000","last_unstale":"2023-08-27T20:51:41.708629+0000","last_undegraded":"2023-08-27T20:51:41.708629+0000","last_fullsized":"2023-08-27T20:51:41.708629+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:04:36.156817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706214+0000","last_change":"2023-08-27T20:50:40.474630+0000","last_active":"2023-08-27T20:51:41.706214+0000","last_peered":"2023-08-27T20:51:41.706214+0000","last_clean":"2023-08-27T20:51:41.706214+0000","last_became_active":"2023-08-27T20:50:40.474345+0000","last_became_peered":"2023-08-27T20:50:40.474345+0000","last_unstale":"2023-08-27T20:51:41.706214+0000","last_undegraded":"2023-08-27T20:51:41.706214+0000","last_fullsized":"2023-08-27T20:51:41.706214+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:35:03.409338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710133+0000","last_change":"2023-08-27T20:51:18.103183+0000","last_active":"2023-08-27T20:51:41.710133+0000","last_peered":"2023-08-27T20:51:41.710133+0000","last_clean":"2023-08-27T20:51:41.710133+0000","last_became_active":"2023-08-27T20:51:18.102921+0000","last_became_peered":"2023-08-27T20:51:18.102921+0000","last_unstale":"2023-08-27T20:51:41.710133+0000","last_undegraded":"2023-08-27T20:51:41.710133+0000","last_fullsized":"2023-08-27T20:51:41.710133+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:47:12.209253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740223+0000","last_change":"2023-08-27T20:51:18.175452+0000","last_active":"2023-08-27T20:51:41.740223+0000","last_peered":"2023-08-27T20:51:41.740223+0000","last_clean":"2023-08-27T20:51:41.740223+0000","last_became_active":"2023-08-27T20:51:18.174003+0000","last_became_peered":"2023-08-27T20:51:18.174003+0000","last_unstale":"2023-08-27T20:51:41.740223+0000","last_undegraded":"2023-08-27T20:51:41.740223+0000","last_fullsized":"2023-08-27T20:51:41.740223+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:26:59.035718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705919+0000","last_change":"2023-08-27T20:51:18.101498+0000","last_active":"2023-08-27T20:51:41.705919+0000","last_peered":"2023-08-27T20:51:41.705919+0000","last_clean":"2023-08-27T20:51:41.705919+0000","last_became_active":"2023-08-27T20:51:18.101220+0000","last_became_peered":"2023-08-27T20:51:18.101220+0000","last_unstale":"2023-08-27T20:51:41.705919+0000","last_undegraded":"2023-08-27T20:51:41.705919+0000","last_fullsized":"2023-08-27T20:51:41.705919+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:27:28.524484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936758+0000","last_change":"2023-08-27T20:50:41.004048+0000","last_active":"2023-08-27T20:51:41.936758+0000","last_peered":"2023-08-27T20:51:41.936758+0000","last_clean":"2023-08-27T20:51:41.936758+0000","last_became_active":"2023-08-27T20:50:41.003731+0000","last_became_peered":"2023-08-27T20:50:41.003731+0000","last_unstale":"2023-08-27T20:51:41.936758+0000","last_undegraded":"2023-08-27T20:51:41.936758+0000","last_fullsized":"2023-08-27T20:51:41.936758+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:58.339578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740432+0000","last_change":"2023-08-27T20:51:18.132942+0000","last_active":"2023-08-27T20:51:41.740432+0000","last_peered":"2023-08-27T20:51:41.740432+0000","last_clean":"2023-08-27T20:51:41.740432+0000","last_became_active":"2023-08-27T20:51:18.132641+0000","last_became_peered":"2023-08-27T20:51:18.132641+0000","last_unstale":"2023-08-27T20:51:41.740432+0000","last_undegraded":"2023-08-27T20:51:41.740432+0000","last_fullsized":"2023-08-27T20:51:41.740432+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:16:24.405549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740170+0000","last_change":"2023-08-27T20:51:18.096016+0000","last_active":"2023-08-27T20:51:41.740170+0000","last_peered":"2023-08-27T20:51:41.740170+0000","last_clean":"2023-08-27T20:51:41.740170+0000","last_became_active":"2023-08-27T20:51:18.095736+0000","last_became_peered":"2023-08-27T20:51:18.095736+0000","last_unstale":"2023-08-27T20:51:41.740170+0000","last_undegraded":"2023-08-27T20:51:41.740170+0000","last_fullsized":"2023-08-27T20:51:41.740170+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:22:49.189530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936564+0000","last_change":"2023-08-27T20:50:41.004114+0000","last_active":"2023-08-27T20:51:41.936564+0000","last_peered":"2023-08-27T20:51:41.936564+0000","last_clean":"2023-08-27T20:51:41.936564+0000","last_became_active":"2023-08-27T20:50:41.003717+0000","last_became_peered":"2023-08-27T20:50:41.003717+0000","last_unstale":"2023-08-27T20:51:41.936564+0000","last_undegraded":"2023-08-27T20:51:41.936564+0000","last_fullsized":"2023-08-27T20:51:41.936564+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:24:46.799377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710310+0000","last_change":"2023-08-27T20:51:18.174265+0000","last_active":"2023-08-27T20:51:41.710310+0000","last_peered":"2023-08-27T20:51:41.710310+0000","last_clean":"2023-08-27T20:51:41.710310+0000","last_became_active":"2023-08-27T20:51:18.173954+0000","last_became_peered":"2023-08-27T20:51:18.173954+0000","last_unstale":"2023-08-27T20:51:41.710310+0000","last_undegraded":"2023-08-27T20:51:41.710310+0000","last_fullsized":"2023-08-27T20:51:41.710310+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:07:21.475116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709718+0000","last_change":"2023-08-27T20:51:18.105105+0000","last_active":"2023-08-27T20:51:41.709718+0000","last_peered":"2023-08-27T20:51:41.709718+0000","last_clean":"2023-08-27T20:51:41.709718+0000","last_became_active":"2023-08-27T20:51:18.104814+0000","last_became_peered":"2023-08-27T20:51:18.104814+0000","last_unstale":"2023-08-27T20:51:41.709718+0000","last_undegraded":"2023-08-27T20:51:41.709718+0000","last_fullsized":"2023-08-27T20:51:41.709718+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:48:25.708690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"1.0","version":"58'128","reported_seq":276,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936456+0000","last_change":"2023-08-27T20:50:22.534100+0000","last_active":"2023-08-27T20:51:41.936456+0000","last_peered":"2023-08-27T20:51:41.936456+0000","last_clean":"2023-08-27T20:51:41.936456+0000","last_became_active":"2023-08-27T20:50:22.533604+0000","last_became_peered":"2023-08-27T20:50:22.533604+0000","last_unstale":"2023-08-27T20:51:41.936456+0000","last_undegraded":"2023-08-27T20:51:41.936456+0000","last_fullsized":"2023-08-27T20:51:41.936456+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_clean_scrub_stamp":"2023-08-27T20:48:59.426866+0000","objects_scrubbed":0,"log_size":128,"ondisk_log_size":128,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:50:03.653850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706260+0000","last_change":"2023-08-27T20:50:40.472409+0000","last_active":"2023-08-27T20:51:41.706260+0000","last_peered":"2023-08-27T20:51:41.706260+0000","last_clean":"2023-08-27T20:51:41.706260+0000","last_became_active":"2023-08-27T20:50:40.472101+0000","last_became_peered":"2023-08-27T20:50:40.472101+0000","last_unstale":"2023-08-27T20:51:41.706260+0000","last_undegraded":"2023-08-27T20:51:41.706260+0000","last_fullsized":"2023-08-27T20:51:41.706260+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:37:42.899620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707058+0000","last_change":"2023-08-27T20:51:18.128769+0000","last_active":"2023-08-27T20:51:41.707058+0000","last_peered":"2023-08-27T20:51:41.707058+0000","last_clean":"2023-08-27T20:51:41.707058+0000","last_became_active":"2023-08-27T20:51:18.128121+0000","last_became_peered":"2023-08-27T20:51:18.128121+0000","last_unstale":"2023-08-27T20:51:41.707058+0000","last_undegraded":"2023-08-27T20:51:41.707058+0000","last_fullsized":"2023-08-27T20:51:41.707058+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:54:21.500010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705110+0000","last_change":"2023-08-27T20:51:18.102189+0000","last_active":"2023-08-27T20:51:41.705110+0000","last_peered":"2023-08-27T20:51:41.705110+0000","last_clean":"2023-08-27T20:51:41.705110+0000","last_became_active":"2023-08-27T20:51:18.101928+0000","last_became_peered":"2023-08-27T20:51:18.101928+0000","last_unstale":"2023-08-27T20:51:41.705110+0000","last_undegraded":"2023-08-27T20:51:41.705110+0000","last_fullsized":"2023-08-27T20:51:41.705110+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:39:54.277133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705393+0000","last_change":"2023-08-27T20:50:41.003113+0000","last_active":"2023-08-27T20:51:41.705393+0000","last_peered":"2023-08-27T20:51:41.705393+0000","last_clean":"2023-08-27T20:51:41.705393+0000","last_became_active":"2023-08-27T20:50:41.002820+0000","last_became_peered":"2023-08-27T20:50:41.002820+0000","last_unstale":"2023-08-27T20:51:41.705393+0000","last_undegraded":"2023-08-27T20:51:41.705393+0000","last_fullsized":"2023-08-27T20:51:41.705393+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:03:10.585918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705840+0000","last_change":"2023-08-27T20:51:18.133061+0000","last_active":"2023-08-27T20:51:41.705840+0000","last_peered":"2023-08-27T20:51:41.705840+0000","last_clean":"2023-08-27T20:51:41.705840+0000","last_became_active":"2023-08-27T20:51:18.132737+0000","last_became_peered":"2023-08-27T20:51:18.132737+0000","last_unstale":"2023-08-27T20:51:41.705840+0000","last_undegraded":"2023-08-27T20:51:41.705840+0000","last_fullsized":"2023-08-27T20:51:41.705840+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:38:31.433133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707681+0000","last_change":"2023-08-27T20:51:18.102990+0000","last_active":"2023-08-27T20:51:41.707681+0000","last_peered":"2023-08-27T20:51:41.707681+0000","last_clean":"2023-08-27T20:51:41.707681+0000","last_became_active":"2023-08-27T20:51:18.102475+0000","last_became_peered":"2023-08-27T20:51:18.102475+0000","last_unstale":"2023-08-27T20:51:41.707681+0000","last_undegraded":"2023-08-27T20:51:41.707681+0000","last_fullsized":"2023-08-27T20:51:41.707681+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T20:52:23.666385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706322+0000","last_change":"2023-08-27T20:50:40.475080+0000","last_active":"2023-08-27T20:51:41.706322+0000","last_peered":"2023-08-27T20:51:41.706322+0000","last_clean":"2023-08-27T20:51:41.706322+0000","last_became_active":"2023-08-27T20:50:40.474840+0000","last_became_peered":"2023-08-27T20:50:40.474840+0000","last_unstale":"2023-08-27T20:51:41.706322+0000","last_undegraded":"2023-08-27T20:51:41.706322+0000","last_fullsized":"2023-08-27T20:51:41.706322+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:35:31.367035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705486+0000","last_change":"2023-08-27T20:51:18.125800+0000","last_active":"2023-08-27T20:51:41.705486+0000","last_peered":"2023-08-27T20:51:41.705486+0000","last_clean":"2023-08-27T20:51:41.705486+0000","last_became_active":"2023-08-27T20:51:18.125533+0000","last_became_peered":"2023-08-27T20:51:18.125533+0000","last_unstale":"2023-08-27T20:51:41.705486+0000","last_undegraded":"2023-08-27T20:51:41.705486+0000","last_fullsized":"2023-08-27T20:51:41.705486+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:58:23.165148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706417+0000","last_change":"2023-08-27T20:51:18.098948+0000","last_active":"2023-08-27T20:51:41.706417+0000","last_peered":"2023-08-27T20:51:41.706417+0000","last_clean":"2023-08-27T20:51:41.706417+0000","last_became_active":"2023-08-27T20:51:18.098648+0000","last_became_peered":"2023-08-27T20:51:18.098648+0000","last_unstale":"2023-08-27T20:51:41.706417+0000","last_undegraded":"2023-08-27T20:51:41.706417+0000","last_fullsized":"2023-08-27T20:51:41.706417+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:57:40.046678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":17,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705976+0000","last_change":"2023-08-27T20:50:41.004078+0000","last_active":"2023-08-27T20:51:41.705976+0000","last_peered":"2023-08-27T20:51:41.705976+0000","last_clean":"2023-08-27T20:51:41.705976+0000","last_became_active":"2023-08-27T20:50:41.003803+0000","last_became_peered":"2023-08-27T20:50:41.003803+0000","last_unstale":"2023-08-27T20:51:41.705976+0000","last_undegraded":"2023-08-27T20:51:41.705976+0000","last_fullsized":"2023-08-27T20:51:41.705976+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:14.109789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":10,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-08-27T20:51:42.784026+0000","last_change":"2023-08-27T20:51:42.784026+0000","last_active":"2023-08-27T20:51:42.784026+0000","last_peered":"2023-08-27T20:51:42.784026+0000","last_clean":"2023-08-27T20:51:42.784026+0000","last_became_active":"2023-08-27T20:51:42.783779+0000","last_became_peered":"2023-08-27T20:51:42.783779+0000","last_unstale":"2023-08-27T20:51:42.784026+0000","last_undegraded":"2023-08-27T20:51:42.784026+0000","last_fullsized":"2023-08-27T20:51:42.784026+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:11:19.094709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708652+0000","last_change":"2023-08-27T20:51:18.101235+0000","last_active":"2023-08-27T20:51:41.708652+0000","last_peered":"2023-08-27T20:51:41.708652+0000","last_clean":"2023-08-27T20:51:41.708652+0000","last_became_active":"2023-08-27T20:51:18.100949+0000","last_became_peered":"2023-08-27T20:51:18.100949+0000","last_unstale":"2023-08-27T20:51:41.708652+0000","last_undegraded":"2023-08-27T20:51:41.708652+0000","last_fullsized":"2023-08-27T20:51:41.708652+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:22:26.784792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-08-27T20:51:42.749070+0000","last_change":"2023-08-27T20:51:42.749070+0000","last_active":"2023-08-27T20:51:42.749070+0000","last_peered":"2023-08-27T20:51:42.749070+0000","last_clean":"2023-08-27T20:51:42.749070+0000","last_became_active":"2023-08-27T20:51:42.748736+0000","last_became_peered":"2023-08-27T20:51:42.748736+0000","last_unstale":"2023-08-27T20:51:42.749070+0000","last_undegraded":"2023-08-27T20:51:42.749070+0000","last_fullsized":"2023-08-27T20:51:42.749070+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:58:27.930976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709293+0000","last_change":"2023-08-27T20:51:18.104763+0000","last_active":"2023-08-27T20:51:41.709293+0000","last_peered":"2023-08-27T20:51:41.709293+0000","last_clean":"2023-08-27T20:51:41.709293+0000","last_became_active":"2023-08-27T20:51:18.104258+0000","last_became_peered":"2023-08-27T20:51:18.104258+0000","last_unstale":"2023-08-27T20:51:41.709293+0000","last_undegraded":"2023-08-27T20:51:41.709293+0000","last_fullsized":"2023-08-27T20:51:41.709293+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:23:37.009396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708140+0000","last_change":"2023-08-27T20:51:18.127756+0000","last_active":"2023-08-27T20:51:41.708140+0000","last_peered":"2023-08-27T20:51:41.708140+0000","last_clean":"2023-08-27T20:51:41.708140+0000","last_became_active":"2023-08-27T20:51:18.127374+0000","last_became_peered":"2023-08-27T20:51:18.127374+0000","last_unstale":"2023-08-27T20:51:41.708140+0000","last_undegraded":"2023-08-27T20:51:41.708140+0000","last_fullsized":"2023-08-27T20:51:41.708140+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:56:07.099266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710335+0000","last_change":"2023-08-27T20:51:18.108839+0000","last_active":"2023-08-27T20:51:41.710335+0000","last_peered":"2023-08-27T20:51:41.710335+0000","last_clean":"2023-08-27T20:51:41.710335+0000","last_became_active":"2023-08-27T20:51:18.108560+0000","last_became_peered":"2023-08-27T20:51:18.108560+0000","last_unstale":"2023-08-27T20:51:41.710335+0000","last_undegraded":"2023-08-27T20:51:41.710335+0000","last_fullsized":"2023-08-27T20:51:41.710335+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:36:37.649085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707720+0000","last_change":"2023-08-27T20:51:18.129095+0000","last_active":"2023-08-27T20:51:41.707720+0000","last_peered":"2023-08-27T20:51:41.707720+0000","last_clean":"2023-08-27T20:51:41.707720+0000","last_became_active":"2023-08-27T20:51:18.128667+0000","last_became_peered":"2023-08-27T20:51:18.128667+0000","last_unstale":"2023-08-27T20:51:41.707720+0000","last_undegraded":"2023-08-27T20:51:41.707720+0000","last_fullsized":"2023-08-27T20:51:41.707720+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:14:50.669396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709781+0000","last_change":"2023-08-27T20:51:18.096242+0000","last_active":"2023-08-27T20:51:41.709781+0000","last_peered":"2023-08-27T20:51:41.709781+0000","last_clean":"2023-08-27T20:51:41.709781+0000","last_became_active":"2023-08-27T20:51:18.096012+0000","last_became_peered":"2023-08-27T20:51:18.096012+0000","last_unstale":"2023-08-27T20:51:41.709781+0000","last_undegraded":"2023-08-27T20:51:41.709781+0000","last_fullsized":"2023-08-27T20:51:41.709781+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:07:12.658963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740366+0000","last_change":"2023-08-27T20:51:18.132475+0000","last_active":"2023-08-27T20:51:41.740366+0000","last_peered":"2023-08-27T20:51:41.740366+0000","last_clean":"2023-08-27T20:51:41.740366+0000","last_became_active":"2023-08-27T20:51:18.132202+0000","last_became_peered":"2023-08-27T20:51:18.132202+0000","last_unstale":"2023-08-27T20:51:41.740366+0000","last_undegraded":"2023-08-27T20:51:41.740366+0000","last_fullsized":"2023-08-27T20:51:41.740366+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:59:04.704593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706343+0000","last_change":"2023-08-27T20:51:18.102046+0000","last_active":"2023-08-27T20:51:41.706343+0000","last_peered":"2023-08-27T20:51:41.706343+0000","last_clean":"2023-08-27T20:51:41.706343+0000","last_became_active":"2023-08-27T20:51:18.101748+0000","last_became_peered":"2023-08-27T20:51:18.101748+0000","last_unstale":"2023-08-27T20:51:41.706343+0000","last_undegraded":"2023-08-27T20:51:41.706343+0000","last_fullsized":"2023-08-27T20:51:41.706343+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:23:02.841371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710127+0000","last_change":"2023-08-27T20:51:18.174360+0000","last_active":"2023-08-27T20:51:41.710127+0000","last_peered":"2023-08-27T20:51:41.710127+0000","last_clean":"2023-08-27T20:51:41.710127+0000","last_became_active":"2023-08-27T20:51:18.174027+0000","last_became_peered":"2023-08-27T20:51:18.174027+0000","last_unstale":"2023-08-27T20:51:41.710127+0000","last_undegraded":"2023-08-27T20:51:41.710127+0000","last_fullsized":"2023-08-27T20:51:41.710127+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:51.281103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936830+0000","last_change":"2023-08-27T20:51:18.102162+0000","last_active":"2023-08-27T20:51:41.936830+0000","last_peered":"2023-08-27T20:51:41.936830+0000","last_clean":"2023-08-27T20:51:41.936830+0000","last_became_active":"2023-08-27T20:51:18.101864+0000","last_became_peered":"2023-08-27T20:51:18.101864+0000","last_unstale":"2023-08-27T20:51:41.936830+0000","last_undegraded":"2023-08-27T20:51:41.936830+0000","last_fullsized":"2023-08-27T20:51:41.936830+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:31.686394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":22,"reported_epoch":65,"state":"active+clean","last_fresh":"2023-08-27T20:51:42.784195+0000","last_change":"2023-08-27T20:51:42.784195+0000","last_active":"2023-08-27T20:51:42.784195+0000","last_peered":"2023-08-27T20:51:42.784195+0000","last_clean":"2023-08-27T20:51:42.784195+0000","last_became_active":"2023-08-27T20:51:42.783951+0000","last_became_peered":"2023-08-27T20:51:42.783951+0000","last_unstale":"2023-08-27T20:51:42.784195+0000","last_undegraded":"2023-08-27T20:51:42.784195+0000","last_fullsized":"2023-08-27T20:51:42.784195+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:45.897182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936811+0000","last_change":"2023-08-27T20:51:18.104712+0000","last_active":"2023-08-27T20:51:41.936811+0000","last_peered":"2023-08-27T20:51:41.936811+0000","last_clean":"2023-08-27T20:51:41.936811+0000","last_became_active":"2023-08-27T20:51:18.104420+0000","last_became_peered":"2023-08-27T20:51:18.104420+0000","last_unstale":"2023-08-27T20:51:41.936811+0000","last_undegraded":"2023-08-27T20:51:41.936811+0000","last_fullsized":"2023-08-27T20:51:41.936811+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:26:27.616548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706156+0000","last_change":"2023-08-27T20:51:18.174616+0000","last_active":"2023-08-27T20:51:41.706156+0000","last_peered":"2023-08-27T20:51:41.706156+0000","last_clean":"2023-08-27T20:51:41.706156+0000","last_became_active":"2023-08-27T20:51:18.173970+0000","last_became_peered":"2023-08-27T20:51:18.173970+0000","last_unstale":"2023-08-27T20:51:41.706156+0000","last_undegraded":"2023-08-27T20:51:41.706156+0000","last_fullsized":"2023-08-27T20:51:41.706156+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:17:07.500477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936914+0000","last_change":"2023-08-27T20:51:18.100527+0000","last_active":"2023-08-27T20:51:41.936914+0000","last_peered":"2023-08-27T20:51:41.936914+0000","last_clean":"2023-08-27T20:51:41.936914+0000","last_became_active":"2023-08-27T20:51:18.100150+0000","last_became_peered":"2023-08-27T20:51:18.100150+0000","last_unstale":"2023-08-27T20:51:41.936914+0000","last_undegraded":"2023-08-27T20:51:41.936914+0000","last_fullsized":"2023-08-27T20:51:41.936914+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:31:30.303986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710029+0000","last_change":"2023-08-27T20:51:18.174292+0000","last_active":"2023-08-27T20:51:41.710029+0000","last_peered":"2023-08-27T20:51:41.710029+0000","last_clean":"2023-08-27T20:51:41.710029+0000","last_became_active":"2023-08-27T20:51:18.173994+0000","last_became_peered":"2023-08-27T20:51:18.173994+0000","last_unstale":"2023-08-27T20:51:41.710029+0000","last_undegraded":"2023-08-27T20:51:41.710029+0000","last_fullsized":"2023-08-27T20:51:41.710029+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:16:13.606362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":21,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710086+0000","last_change":"2023-08-27T20:51:18.105714+0000","last_active":"2023-08-27T20:51:41.710086+0000","last_peered":"2023-08-27T20:51:41.710086+0000","last_clean":"2023-08-27T20:51:41.710086+0000","last_became_active":"2023-08-27T20:51:18.105364+0000","last_became_peered":"2023-08-27T20:51:18.105364+0000","last_unstale":"2023-08-27T20:51:41.710086+0000","last_undegraded":"2023-08-27T20:51:41.710086+0000","last_fullsized":"2023-08-27T20:51:41.710086+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:47.202084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740323+0000","last_change":"2023-08-27T20:51:18.176875+0000","last_active":"2023-08-27T20:51:41.740323+0000","last_peered":"2023-08-27T20:51:41.740323+0000","last_clean":"2023-08-27T20:51:41.740323+0000","last_became_active":"2023-08-27T20:51:18.176547+0000","last_became_peered":"2023-08-27T20:51:18.176547+0000","last_unstale":"2023-08-27T20:51:41.740323+0000","last_undegraded":"2023-08-27T20:51:41.740323+0000","last_fullsized":"2023-08-27T20:51:41.740323+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:13:16.094024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936591+0000","last_change":"2023-08-27T20:51:18.098165+0000","last_active":"2023-08-27T20:51:41.936591+0000","last_peered":"2023-08-27T20:51:41.936591+0000","last_clean":"2023-08-27T20:51:41.936591+0000","last_became_active":"2023-08-27T20:51:18.097846+0000","last_became_peered":"2023-08-27T20:51:18.097846+0000","last_unstale":"2023-08-27T20:51:41.936591+0000","last_undegraded":"2023-08-27T20:51:41.936591+0000","last_fullsized":"2023-08-27T20:51:41.936591+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:43:32.430484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706324+0000","last_change":"2023-08-27T20:51:18.174803+0000","last_active":"2023-08-27T20:51:41.706324+0000","last_peered":"2023-08-27T20:51:41.706324+0000","last_clean":"2023-08-27T20:51:41.706324+0000","last_became_active":"2023-08-27T20:51:18.174313+0000","last_became_peered":"2023-08-27T20:51:18.174313+0000","last_unstale":"2023-08-27T20:51:41.706324+0000","last_undegraded":"2023-08-27T20:51:41.706324+0000","last_fullsized":"2023-08-27T20:51:41.706324+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:10:32.503697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740200+0000","last_change":"2023-08-27T20:51:18.097273+0000","last_active":"2023-08-27T20:51:41.740200+0000","last_peered":"2023-08-27T20:51:41.740200+0000","last_clean":"2023-08-27T20:51:41.740200+0000","last_became_active":"2023-08-27T20:51:18.096940+0000","last_became_peered":"2023-08-27T20:51:18.096940+0000","last_unstale":"2023-08-27T20:51:41.740200+0000","last_undegraded":"2023-08-27T20:51:41.740200+0000","last_fullsized":"2023-08-27T20:51:41.740200+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:01:47.450662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709885+0000","last_change":"2023-08-27T20:51:18.174487+0000","last_active":"2023-08-27T20:51:41.709885+0000","last_peered":"2023-08-27T20:51:41.709885+0000","last_clean":"2023-08-27T20:51:41.709885+0000","last_became_active":"2023-08-27T20:51:18.174142+0000","last_became_peered":"2023-08-27T20:51:18.174142+0000","last_unstale":"2023-08-27T20:51:41.709885+0000","last_undegraded":"2023-08-27T20:51:41.709885+0000","last_fullsized":"2023-08-27T20:51:41.709885+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:21:00.888579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":15,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936595+0000","last_change":"2023-08-27T20:51:18.096913+0000","last_active":"2023-08-27T20:51:41.936595+0000","last_peered":"2023-08-27T20:51:41.936595+0000","last_clean":"2023-08-27T20:51:41.936595+0000","last_became_active":"2023-08-27T20:51:18.096600+0000","last_became_peered":"2023-08-27T20:51:18.096600+0000","last_unstale":"2023-08-27T20:51:41.936595+0000","last_undegraded":"2023-08-27T20:51:41.936595+0000","last_fullsized":"2023-08-27T20:51:41.936595+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:41:23.301928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707895+0000","last_change":"2023-08-27T20:51:18.131165+0000","last_active":"2023-08-27T20:51:41.707895+0000","last_peered":"2023-08-27T20:51:41.707895+0000","last_clean":"2023-08-27T20:51:41.707895+0000","last_became_active":"2023-08-27T20:51:18.130812+0000","last_became_peered":"2023-08-27T20:51:18.130812+0000","last_unstale":"2023-08-27T20:51:41.707895+0000","last_undegraded":"2023-08-27T20:51:41.707895+0000","last_fullsized":"2023-08-27T20:51:41.707895+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:17:11.521086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.14","version":"63'6","reported_seq":19,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707844+0000","last_change":"2023-08-27T20:51:18.106623+0000","last_active":"2023-08-27T20:51:41.707844+0000","last_peered":"2023-08-27T20:51:41.707844+0000","last_clean":"2023-08-27T20:51:41.707844+0000","last_became_active":"2023-08-27T20:51:18.106343+0000","last_became_peered":"2023-08-27T20:51:18.106343+0000","last_unstale":"2023-08-27T20:51:41.707844+0000","last_undegraded":"2023-08-27T20:51:41.707844+0000","last_fullsized":"2023-08-27T20:51:41.707844+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":6,"ondisk_log_size":6,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:43:08.157700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":1536,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":6,"num_write_kb":3,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705485+0000","last_change":"2023-08-27T20:51:18.126515+0000","last_active":"2023-08-27T20:51:41.705485+0000","last_peered":"2023-08-27T20:51:41.705485+0000","last_clean":"2023-08-27T20:51:41.705485+0000","last_became_active":"2023-08-27T20:51:18.123514+0000","last_became_peered":"2023-08-27T20:51:18.123514+0000","last_unstale":"2023-08-27T20:51:41.705485+0000","last_undegraded":"2023-08-27T20:51:41.705485+0000","last_fullsized":"2023-08-27T20:51:41.705485+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:00.412812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":16,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.936738+0000","last_change":"2023-08-27T20:51:18.102959+0000","last_active":"2023-08-27T20:51:41.936738+0000","last_peered":"2023-08-27T20:51:41.936738+0000","last_clean":"2023-08-27T20:51:41.936738+0000","last_became_active":"2023-08-27T20:51:18.102698+0000","last_became_peered":"2023-08-27T20:51:18.102698+0000","last_unstale":"2023-08-27T20:51:41.936738+0000","last_undegraded":"2023-08-27T20:51:41.936738+0000","last_fullsized":"2023-08-27T20:51:41.936738+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:00:59.591758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709306+0000","last_change":"2023-08-27T20:51:18.174261+0000","last_active":"2023-08-27T20:51:41.709306+0000","last_peered":"2023-08-27T20:51:41.709306+0000","last_clean":"2023-08-27T20:51:41.709306+0000","last_became_active":"2023-08-27T20:51:18.174020+0000","last_became_peered":"2023-08-27T20:51:18.174020+0000","last_unstale":"2023-08-27T20:51:41.709306+0000","last_undegraded":"2023-08-27T20:51:41.709306+0000","last_fullsized":"2023-08-27T20:51:41.709306+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:06:33.961645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.706383+0000","last_change":"2023-08-27T20:51:18.097098+0000","last_active":"2023-08-27T20:51:41.706383+0000","last_peered":"2023-08-27T20:51:41.706383+0000","last_clean":"2023-08-27T20:51:41.706383+0000","last_became_active":"2023-08-27T20:51:18.096745+0000","last_became_peered":"2023-08-27T20:51:18.096745+0000","last_unstale":"2023-08-27T20:51:41.706383+0000","last_undegraded":"2023-08-27T20:51:41.706383+0000","last_fullsized":"2023-08-27T20:51:41.706383+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:42:05.626352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.709158+0000","last_change":"2023-08-27T20:51:18.129959+0000","last_active":"2023-08-27T20:51:41.709158+0000","last_peered":"2023-08-27T20:51:41.709158+0000","last_clean":"2023-08-27T20:51:41.709158+0000","last_became_active":"2023-08-27T20:51:18.129690+0000","last_became_peered":"2023-08-27T20:51:18.129690+0000","last_unstale":"2023-08-27T20:51:41.709158+0000","last_undegraded":"2023-08-27T20:51:41.709158+0000","last_fullsized":"2023-08-27T20:51:41.709158+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:35:08.487865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740426+0000","last_change":"2023-08-27T20:51:18.097550+0000","last_active":"2023-08-27T20:51:41.740426+0000","last_peered":"2023-08-27T20:51:41.740426+0000","last_clean":"2023-08-27T20:51:41.740426+0000","last_became_active":"2023-08-27T20:51:18.097145+0000","last_became_peered":"2023-08-27T20:51:18.097145+0000","last_unstale":"2023-08-27T20:51:41.740426+0000","last_undegraded":"2023-08-27T20:51:41.740426+0000","last_fullsized":"2023-08-27T20:51:41.740426+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:36:26.771185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710455+0000","last_change":"2023-08-27T20:51:18.128634+0000","last_active":"2023-08-27T20:51:41.710455+0000","last_peered":"2023-08-27T20:51:41.710455+0000","last_clean":"2023-08-27T20:51:41.710455+0000","last_became_active":"2023-08-27T20:51:18.128367+0000","last_became_peered":"2023-08-27T20:51:18.128367+0000","last_unstale":"2023-08-27T20:51:41.710455+0000","last_undegraded":"2023-08-27T20:51:41.710455+0000","last_fullsized":"2023-08-27T20:51:41.710455+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:51:25.311153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.708754+0000","last_change":"2023-08-27T20:51:18.094635+0000","last_active":"2023-08-27T20:51:41.708754+0000","last_peered":"2023-08-27T20:51:41.708754+0000","last_clean":"2023-08-27T20:51:41.708754+0000","last_became_active":"2023-08-27T20:51:18.094323+0000","last_became_peered":"2023-08-27T20:51:18.094323+0000","last_unstale":"2023-08-27T20:51:41.708754+0000","last_undegraded":"2023-08-27T20:51:41.708754+0000","last_fullsized":"2023-08-27T20:51:41.708754+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:29:06.163259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.740325+0000","last_change":"2023-08-27T20:51:18.174303+0000","last_active":"2023-08-27T20:51:41.740325+0000","last_peered":"2023-08-27T20:51:41.740325+0000","last_clean":"2023-08-27T20:51:41.740325+0000","last_became_active":"2023-08-27T20:51:18.173979+0000","last_became_peered":"2023-08-27T20:51:18.173979+0000","last_unstale":"2023-08-27T20:51:41.740325+0000","last_undegraded":"2023-08-27T20:51:41.740325+0000","last_fullsized":"2023-08-27T20:51:41.740325+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T20:58:16.258007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":13,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.705161+0000","last_change":"2023-08-27T20:51:18.100237+0000","last_active":"2023-08-27T20:51:41.705161+0000","last_peered":"2023-08-27T20:51:41.705161+0000","last_clean":"2023-08-27T20:51:41.705161+0000","last_became_active":"2023-08-27T20:51:18.099976+0000","last_became_peered":"2023-08-27T20:51:18.099976+0000","last_unstale":"2023-08-27T20:51:41.705161+0000","last_undegraded":"2023-08-27T20:51:41.705161+0000","last_fullsized":"2023-08-27T20:51:41.705161+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:46:15.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":12,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.710048+0000","last_change":"2023-08-27T20:51:18.129248+0000","last_active":"2023-08-27T20:51:41.710048+0000","last_peered":"2023-08-27T20:51:41.710048+0000","last_clean":"2023-08-27T20:51:41.710048+0000","last_became_active":"2023-08-27T20:51:18.128963+0000","last_became_peered":"2023-08-27T20:51:18.128963+0000","last_unstale":"2023-08-27T20:51:41.710048+0000","last_undegraded":"2023-08-27T20:51:41.710048+0000","last_fullsized":"2023-08-27T20:51:41.710048+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:31:19.516025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1a","version":"63'1","reported_seq":14,"reported_epoch":64,"state":"active+clean","last_fresh":"2023-08-27T20:51:41.707525+0000","last_change":"2023-08-27T20:51:18.103078+0000","last_active":"2023-08-27T20:51:41.707525+0000","last_peered":"2023-08-27T20:51:41.707525+0000","last_clean":"2023-08-27T20:51:41.707525+0000","last_became_active":"2023-08-27T20:51:18.102289+0000","last_became_peered":"2023-08-27T20:51:18.102289+0000","last_unstale":"2023-08-27T20:51:41.707525+0000","last_undegraded":"2023-08-27T20:51:41.707525+0000","last_fullsized":"2023-08-27T20:51:41.707525+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:46:25.395790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":2306,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":45,"num_write_kb":13,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":6918,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":39,"ondisk_log_size":39,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":212,"num_read_kb":306,"num_write":179,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":128,"ondisk_log_size":128,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":3,"up_from":25,"seq":107374182433,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6096,"kb_used_data":648,"kb_used_omap":0,"kb_used_meta":5440,"kb_avail":93739056,"statfs":{"total":95995035648,"available":95988793344,"internally_reserved":0,"allocated":663552,"data_stored":287109,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5570560},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:16 2023","interfaces":[{"interface":"back","average":{"1min":0.32200000000000001,"5min":0.36199999999999999,"15min":0.36799999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"last":0.307},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.442,"15min":0.45200000000000001},"min":{"1min":0.23400000000000001,"5min":0.23400000000000001,"15min":0.23400000000000001},"max":{"1min":0.57199999999999995,"5min":0.628,"15min":0.628},"last":0.34100000000000003}]},{"osd":1,"last update":"Sun Aug 27 20:51:16 2023","interfaces":[{"interface":"back","average":{"1min":0.377,"5min":0.44600000000000001,"15min":0.45800000000000002},"min":{"1min":0.25800000000000001,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.48399999999999999,"5min":0.58599999999999997,"15min":0.58599999999999997},"last":0.29199999999999998},{"interface":"front","average":{"1min":0.38300000000000001,"5min":0.432,"15min":0.44},"min":{"1min":0.27800000000000002,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.57399999999999995,"5min":0.66800000000000004,"15min":0.66800000000000004},"last":0.32200000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:16 2023","interfaces":[{"interface":"back","average":{"1min":0.40500000000000003,"5min":0.47699999999999998,"15min":0.48899999999999999},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.66100000000000003,"5min":0.66100000000000003,"15min":0.66100000000000003},"last":0.35599999999999998},{"interface":"front","average":{"1min":0.41299999999999998,"5min":0.44900000000000001,"15min":0.45500000000000002},"min":{"1min":0.28999999999999998,"5min":0.28999999999999998,"15min":0.28999999999999998},"max":{"1min":0.59099999999999997,"5min":0.59399999999999997,"15min":0.59399999999999997},"last":0.36599999999999999}]},{"osd":4,"last update":"Sun Aug 27 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.51000000000000001,"5min":0.56299999999999994,"15min":0.57199999999999995},"min":{"1min":0.375,"5min":0.375,"15min":0.375},"max":{"1min":0.92700000000000005,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.74099999999999999},{"interface":"front","average":{"1min":0.63900000000000001,"5min":0.59899999999999998,"15min":0.59299999999999997},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":2.907,"5min":2.907,"15min":2.907},"last":0.76100000000000001}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61099999999999999,"5min":0.61099999999999999,"15min":0.61099999999999999},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.82899999999999996,"5min":0.82899999999999996,"15min":0.82899999999999996},"last":0.65800000000000003},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.84399999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.80800000000000005}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.60099999999999998,"5min":0.60099999999999998,"15min":0.60099999999999998},"min":{"1min":0.39300000000000002,"5min":0.39300000000000002,"15min":0.39300000000000002},"max":{"1min":0.88300000000000001,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.58499999999999996},{"interface":"front","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.46100000000000002,"5min":0.46100000000000002,"15min":0.46100000000000002},"max":{"1min":0.77200000000000002,"5min":0.77200000000000002,"15min":0.77200000000000002},"last":0.71699999999999997}]},{"osd":7,"last update":"Sun Aug 27 20:51:20 2023","interfaces":[{"interface":"back","average":{"1min":0.59899999999999998,"5min":0.59899999999999998,"15min":0.59899999999999998},"min":{"1min":0.501,"5min":0.501,"15min":0.501},"max":{"1min":0.874,"5min":0.874,"15min":0.874},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.60399999999999998,"15min":0.60399999999999998},"min":{"1min":0.42999999999999999,"5min":0.42999999999999999,"15min":0.42999999999999999},"max":{"1min":0.80700000000000005,"5min":0.80700000000000005,"15min":0.80700000000000005},"last":0.69399999999999995}]}]},{"osd":2,"up_from":18,"seq":77309411366,"num_pgs":19,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6548,"kb_used_data":652,"kb_used_omap":0,"kb_used_meta":5888,"kb_avail":93738604,"statfs":{"total":95995035648,"available":95988330496,"internally_reserved":0,"allocated":667648,"data_stored":287607,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6029312},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.374,"5min":0.41999999999999998,"15min":0.42799999999999999},"min":{"1min":0.23100000000000001,"5min":0.23100000000000001,"15min":0.23100000000000001},"max":{"1min":0.73799999999999999,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.45800000000000002},{"interface":"front","average":{"1min":0.372,"5min":0.40600000000000003,"15min":0.41199999999999998},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.52500000000000002,"5min":0.58199999999999996,"15min":0.58199999999999996},"last":0.34899999999999998}]},{"osd":1,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.39300000000000002,"5min":0.40899999999999997,"15min":0.41099999999999998},"min":{"1min":0.252,"5min":0.252,"15min":0.252},"max":{"1min":0.89900000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.41799999999999998},{"interface":"front","average":{"1min":0.41699999999999998,"5min":0.44,"15min":0.44400000000000001},"min":{"1min":0.28799999999999998,"5min":0.28799999999999998,"15min":0.28799999999999998},"max":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"last":0.38200000000000001}]},{"osd":3,"last update":"Sun Aug 27 20:51:18 2023","interfaces":[{"interface":"back","average":{"1min":0.39500000000000002,"5min":0.45900000000000002,"15min":0.46999999999999997},"min":{"1min":0.23300000000000001,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.44500000000000001},{"interface":"front","average":{"1min":0.42199999999999999,"5min":0.48199999999999998,"15min":0.49199999999999999},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.64900000000000002,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.433}]},{"osd":4,"last update":"Sun Aug 27 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.60599999999999998,"5min":0.59799999999999998,"15min":0.59599999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":1.377,"5min":1.377,"15min":1.377},"last":0.52400000000000002},{"interface":"front","average":{"1min":0.58699999999999997,"5min":0.60699999999999998,"15min":0.60999999999999999},"min":{"1min":0.39700000000000002,"5min":0.39700000000000002,"15min":0.39700000000000002},"max":{"1min":0.95899999999999996,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.60999999999999999}]},{"osd":5,"last update":"Sun Aug 27 20:50:47 2023","interfaces":[{"interface":"back","average":{"1min":0.61299999999999999,"5min":0.61299999999999999,"15min":0.61299999999999999},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":1.321,"5min":1.321,"15min":1.321},"last":0.66800000000000004},{"interface":"front","average":{"1min":0.60699999999999998,"5min":0.60699999999999998,"15min":0.60699999999999998},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.82499999999999996,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.60199999999999998}]},{"osd":6,"last update":"Sun Aug 27 20:51:08 2023","interfaces":[{"interface":"back","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.90500000000000003,"5min":0.90500000000000003,"15min":0.90500000000000003},"last":0.76400000000000001},{"interface":"front","average":{"1min":0.64200000000000002,"5min":0.64200000000000002,"15min":0.64200000000000002},"min":{"1min":0.38400000000000001,"5min":0.38400000000000001,"15min":0.38400000000000001},"max":{"1min":0.98299999999999998,"5min":0.98299999999999998,"15min":0.98299999999999998},"last":0.64800000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.61899999999999999,"5min":0.61899999999999999,"15min":0.61899999999999999},"min":{"1min":0.46500000000000002,"5min":0.46500000000000002,"15min":0.46500000000000002},"max":{"1min":1.353,"5min":1.353,"15min":1.353},"last":0.73499999999999999},{"interface":"front","average":{"1min":0.623,"5min":0.623,"15min":0.623},"min":{"1min":0.44700000000000001,"5min":0.44700000000000001,"15min":0.44700000000000001},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.68400000000000005}]}]},{"osd":1,"up_from":13,"seq":55834574889,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6732,"kb_used_data":644,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93738420,"statfs":{"total":95995035648,"available":95988142080,"internally_reserved":0,"allocated":659456,"data_stored":287019,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:43 2023","interfaces":[{"interface":"back","average":{"1min":0.37,"5min":0.35499999999999998,"15min":0.35299999999999998},"min":{"1min":0.23999999999999999,"5min":0.23899999999999999,"15min":0.23899999999999999},"max":{"1min":0.51600000000000001,"5min":0.51600000000000001,"15min":0.51600000000000001},"last":0.30099999999999999},{"interface":"front","average":{"1min":0.45000000000000001,"5min":0.377,"15min":0.36499999999999999},"min":{"1min":0.27300000000000002,"5min":0.253,"15min":0.253},"max":{"1min":0.79800000000000004,"5min":0.79800000000000004,"15min":0.79800000000000004},"last":0.38200000000000001}]},{"osd":2,"last update":"Sun Aug 27 20:51:01 2023","interfaces":[{"interface":"back","average":{"1min":0.41399999999999998,"5min":0.42999999999999999,"15min":0.433},"min":{"1min":0.26500000000000001,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.59799999999999998,"5min":0.68300000000000005,"15min":0.68300000000000005},"last":0.35199999999999998},{"interface":"front","average":{"1min":0.39300000000000002,"5min":0.45300000000000001,"15min":0.46300000000000002},"min":{"1min":0.29299999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":0.54400000000000004,"5min":0.64900000000000002,"15min":0.64900000000000002},"last":0.28100000000000003}]},{"osd":3,"last update":"Sun Aug 27 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.41099999999999998,"5min":0.46200000000000002,"15min":0.46999999999999997},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.48199999999999998,"5min":0.67800000000000005,"15min":0.67800000000000005},"last":0.36699999999999999},{"interface":"front","average":{"1min":0.40600000000000003,"5min":0.44700000000000001,"15min":0.45400000000000001},"min":{"1min":0.29099999999999998,"5min":0.29099999999999998,"15min":0.29099999999999998},"max":{"1min":0.61099999999999999,"5min":0.63400000000000001,"15min":0.63400000000000001},"last":0.33700000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.59499999999999997,"15min":0.59499999999999997},"min":{"1min":0.33700000000000002,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.84199999999999997,"5min":0.84199999999999997,"15min":0.84199999999999997},"last":0.46600000000000003},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.58699999999999997,"15min":0.58699999999999997},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.89700000000000002,"5min":0.89700000000000002,"15min":0.89700000000000002},"last":0.495}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.56999999999999995,"15min":0.56999999999999995},"min":{"1min":0.42199999999999999,"5min":0.42199999999999999,"15min":0.42199999999999999},"max":{"1min":0.80200000000000005,"5min":0.80200000000000005,"15min":0.80200000000000005},"last":0.54300000000000004},{"interface":"front","average":{"1min":0.61499999999999999,"5min":0.61499999999999999,"15min":0.61499999999999999},"min":{"1min":0.39500000000000002,"5min":0.39500000000000002,"15min":0.39500000000000002},"max":{"1min":0.871,"5min":0.871,"15min":0.871},"last":0.48399999999999999}]},{"osd":6,"last update":"Sun Aug 27 20:51:07 2023","interfaces":[{"interface":"back","average":{"1min":0.621,"5min":0.621,"15min":0.621},"min":{"1min":0.439,"5min":0.439,"15min":0.439},"max":{"1min":0.94099999999999995,"5min":0.94099999999999995,"15min":0.94099999999999995},"last":0.44},{"interface":"front","average":{"1min":0.622,"5min":0.622,"15min":0.622},"min":{"1min":0.45400000000000001,"5min":0.45400000000000001,"15min":0.45400000000000001},"max":{"1min":0.85999999999999999,"5min":0.85999999999999999,"15min":0.85999999999999999},"last":0.57999999999999996}]},{"osd":7,"last update":"Sun Aug 27 20:51:21 2023","interfaces":[{"interface":"back","average":{"1min":0.61799999999999999,"5min":0.61799999999999999,"15min":0.61799999999999999},"min":{"1min":0.41999999999999998,"5min":0.41999999999999998,"15min":0.41999999999999998},"max":{"1min":0.91100000000000003,"5min":0.91100000000000003,"15min":0.91100000000000003},"last":0.42399999999999999},{"interface":"front","average":{"1min":0.60799999999999998,"5min":0.60799999999999998,"15min":0.60799999999999998},"min":{"1min":0.40500000000000003,"5min":0.40500000000000003,"15min":0.40500000000000003},"max":{"1min":0.93000000000000005,"5min":0.93000000000000005,"15min":0.93000000000000005},"last":0.52100000000000002}]}]},{"osd":0,"up_from":8,"seq":34359738412,"num_pgs":26,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":7320,"kb_used_data":1232,"kb_used_omap":0,"kb_used_meta":6080,"kb_avail":93737832,"statfs":{"total":95995035648,"available":95987539968,"internally_reserved":0,"allocated":1261568,"data_stored":877963,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":6225920},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 20:51:45 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":0.34499999999999997,"15min":0.35299999999999998},"min":{"1min":0.23100000000000001,"5min":0.218,"15min":0.218},"max":{"1min":0.41599999999999998,"5min":0.57699999999999996,"15min":0.57699999999999996},"last":0.27100000000000002},{"interface":"front","average":{"1min":0.33200000000000002,"5min":0.38600000000000001,"15min":0.39400000000000002},"min":{"1min":0.249,"5min":0.224,"15min":0.224},"max":{"1min":0.44900000000000001,"5min":0.63,"15min":0.63},"last":0.312}]},{"osd":2,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.36899999999999999,"5min":0.41599999999999998,"15min":0.42399999999999999},"min":{"1min":0.27400000000000002,"5min":0.26500000000000001,"15min":0.26500000000000001},"max":{"1min":0.55200000000000005,"5min":0.55200000000000005,"15min":0.55200000000000005},"last":0.35699999999999998},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40999999999999998,"15min":0.41699999999999998},"min":{"1min":0.245,"5min":0.245,"15min":0.245},"max":{"1min":0.66200000000000003,"5min":0.66200000000000003,"15min":0.66200000000000003},"last":0.51800000000000002}]},{"osd":3,"last update":"Sun Aug 27 20:51:17 2023","interfaces":[{"interface":"back","average":{"1min":0.375,"5min":0.45900000000000002,"15min":0.47299999999999998},"min":{"1min":0.26800000000000002,"5min":0.26800000000000002,"15min":0.26800000000000002},"max":{"1min":0.58099999999999996,"5min":0.65500000000000003,"15min":0.65500000000000003},"last":0.55200000000000005},{"interface":"front","average":{"1min":0.39200000000000002,"5min":0.44800000000000001,"15min":0.45700000000000002},"min":{"1min":0.28499999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.70199999999999996,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.34100000000000003}]},{"osd":4,"last update":"Sun Aug 27 20:51:35 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.56399999999999995,"15min":0.57099999999999995},"min":{"1min":0.372,"5min":0.372,"15min":0.372},"max":{"1min":0.88600000000000001,"5min":0.88600000000000001,"15min":0.88600000000000001},"last":0.748},{"interface":"front","average":{"1min":0.52600000000000002,"5min":0.57399999999999995,"15min":0.58199999999999996},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.86699999999999999,"5min":0.86699999999999999,"15min":0.86699999999999999},"last":0.873}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.55600000000000005,"5min":0.55600000000000005,"15min":0.55600000000000005},"min":{"1min":0.38100000000000001,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.77800000000000002},{"interface":"front","average":{"1min":0.54900000000000004,"5min":0.54900000000000004,"15min":0.54900000000000004},"min":{"1min":0.40000000000000002,"5min":0.40000000000000002,"15min":0.40000000000000002},"max":{"1min":0.754,"5min":0.754,"15min":0.754},"last":0.67000000000000004}]},{"osd":6,"last update":"Sun Aug 27 20:51:03 2023","interfaces":[{"interface":"back","average":{"1min":0.53800000000000003,"5min":0.53800000000000003,"15min":0.53800000000000003},"min":{"1min":0.36699999999999999,"5min":0.36699999999999999,"15min":0.36699999999999999},"max":{"1min":0.76200000000000001,"5min":0.76200000000000001,"15min":0.76200000000000001},"last":0.85599999999999998},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55700000000000005,"15min":0.55700000000000005},"min":{"1min":0.38300000000000001,"5min":0.38300000000000001,"15min":0.38300000000000001},"max":{"1min":0.747,"5min":0.747,"15min":0.747},"last":0.77100000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:51:23 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34799999999999998,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.79200000000000004,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.80800000000000005},{"interface":"front","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.436,"5min":0.436,"15min":0.436},"max":{"1min":0.77300000000000002,"5min":0.77300000000000002,"15min":0.77300000000000002},"last":0.64700000000000002}]}]},{"osd":7,"up_from":51,"seq":219043332117,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6568,"kb_used_data":1248,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738584,"statfs":{"total":95995035648,"available":95988310016,"internally_reserved":0,"allocated":1277952,"data_stored":878816,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.58899999999999997,"15min":0.58899999999999997},"min":{"1min":0.35799999999999998,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.82099999999999995,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.38800000000000001},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.55800000000000005,"15min":0.55800000000000005},"min":{"1min":0.39100000000000001,"5min":0.39100000000000001,"15min":0.39100000000000001},"max":{"1min":0.69399999999999995,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.55800000000000005}]},{"osd":1,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.40899999999999997,"5min":0.40899999999999997,"15min":0.40899999999999997},"max":{"1min":0.72999999999999998,"5min":0.72999999999999998,"15min":0.72999999999999998},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.34899999999999998,"5min":0.34899999999999998,"15min":0.34899999999999998},"max":{"1min":0.83599999999999997,"5min":0.83599999999999997,"15min":0.83599999999999997},"last":0.40799999999999997}]},{"osd":2,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.56000000000000005,"15min":0.56000000000000005},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80600000000000005,"5min":0.80600000000000005,"15min":0.80600000000000005},"last":0.42499999999999999},{"interface":"front","average":{"1min":0.59699999999999998,"5min":0.59699999999999998,"15min":0.59699999999999998},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.76700000000000002,"5min":0.76700000000000002,"15min":0.76700000000000002},"last":0.48399999999999999}]},{"osd":3,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.58399999999999996,"5min":0.58399999999999996,"15min":0.58399999999999996},"min":{"1min":0.40999999999999998,"5min":0.40999999999999998,"15min":0.40999999999999998},"max":{"1min":0.78000000000000003,"5min":0.78000000000000003,"15min":0.78000000000000003},"last":0.52000000000000002},{"interface":"front","average":{"1min":0.59299999999999997,"5min":0.59299999999999997,"15min":0.59299999999999997},"min":{"1min":0.44900000000000001,"5min":0.44900000000000001,"15min":0.44900000000000001},"max":{"1min":0.85699999999999998,"5min":0.85699999999999998,"15min":0.85699999999999998},"last":0.44700000000000001}]},{"osd":4,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.29999999999999999,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.64700000000000002,"5min":0.64700000000000002,"15min":0.64700000000000002},"last":0.46800000000000003},{"interface":"front","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.316,"5min":0.316,"15min":0.316},"max":{"1min":0.64100000000000001,"5min":0.64100000000000001,"15min":0.64100000000000001},"last":0.59599999999999997}]},{"osd":5,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.47299999999999998,"5min":0.47299999999999998,"15min":0.47299999999999998},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.73999999999999999,"5min":0.73999999999999999,"15min":0.73999999999999999},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.47699999999999998,"5min":0.47699999999999998,"15min":0.47699999999999998},"min":{"1min":0.33400000000000002,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.77900000000000003,"5min":0.77900000000000003,"15min":0.77900000000000003},"last":0.65200000000000002}]},{"osd":6,"last update":"Sun Aug 27 20:51:19 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33900000000000002,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.71699999999999997,"5min":0.71699999999999997,"15min":0.71699999999999997},"last":0.61399999999999999},{"interface":"front","average":{"1min":0.48699999999999999,"5min":0.48699999999999999,"15min":0.48699999999999999},"min":{"1min":0.317,"5min":0.317,"15min":0.317},"max":{"1min":0.70499999999999996,"5min":0.70499999999999996,"15min":0.70499999999999996},"last":0.63700000000000001}]}]},{"osd":4,"up_from":30,"seq":128849018910,"num_pgs":33,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6176,"kb_used_data":664,"kb_used_omap":0,"kb_used_meta":5504,"kb_avail":93738976,"statfs":{"total":95995035648,"available":95988711424,"internally_reserved":0,"allocated":679936,"data_stored":288436,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5636096},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.50800000000000001,"5min":0.55100000000000005,"15min":0.55800000000000005},"min":{"1min":0.37,"5min":0.37,"15min":0.37},"max":{"1min":0.63100000000000001,"5min":0.997,"15min":0.997},"last":0.54400000000000004},{"interface":"front","average":{"1min":0.497,"5min":0.52700000000000002,"15min":0.53200000000000003},"min":{"1min":0.35699999999999998,"5min":0.35699999999999998,"15min":0.35699999999999998},"max":{"1min":0.627,"5min":0.76000000000000001,"15min":0.76000000000000001},"last":0.44500000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.503,"5min":0.53900000000000003,"15min":0.54500000000000004},"min":{"1min":0.371,"5min":0.371,"15min":0.371},"max":{"1min":0.623,"5min":0.96299999999999997,"15min":0.96299999999999997},"last":0.49199999999999999},{"interface":"front","average":{"1min":0.51700000000000002,"5min":0.54100000000000004,"15min":0.54500000000000004},"min":{"1min":0.32500000000000001,"5min":0.32500000000000001,"15min":0.32500000000000001},"max":{"1min":0.62,"5min":0.71999999999999997,"15min":0.71999999999999997},"last":0.504}]},{"osd":2,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.54500000000000004,"15min":0.54700000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.70099999999999996,"5min":0.72299999999999998,"15min":0.72299999999999998},"last":0.47099999999999997},{"interface":"front","average":{"1min":0.5,"5min":0.53200000000000003,"15min":0.53800000000000003},"min":{"1min":0.38,"5min":0.38,"15min":0.38},"max":{"1min":0.66800000000000004,"5min":0.871,"15min":0.871},"last":0.47799999999999998}]},{"osd":3,"last update":"Sun Aug 27 20:51:37 2023","interfaces":[{"interface":"back","average":{"1min":0.52100000000000002,"5min":0.54800000000000004,"15min":0.55200000000000005},"min":{"1min":0.36399999999999999,"5min":0.36399999999999999,"15min":0.36399999999999999},"max":{"1min":0.68899999999999995,"5min":1.0089999999999999,"15min":1.0089999999999999},"last":0.52800000000000002},{"interface":"front","average":{"1min":0.53500000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.34300000000000003,"5min":0.34300000000000003,"15min":0.34300000000000003},"max":{"1min":0.65900000000000003,"5min":0.83399999999999996,"15min":0.83399999999999996},"last":0.46200000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:50:48 2023","interfaces":[{"interface":"back","average":{"1min":0.41499999999999998,"5min":0.41499999999999998,"15min":0.41499999999999998},"min":{"1min":0.29799999999999999,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"last":0.34899999999999998},{"interface":"front","average":{"1min":0.436,"5min":0.436,"15min":0.436},"min":{"1min":0.32200000000000001,"5min":0.32200000000000001,"15min":0.32200000000000001},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.36299999999999999}]},{"osd":6,"last update":"Sun Aug 27 20:51:02 2023","interfaces":[{"interface":"back","average":{"1min":0.47499999999999998,"5min":0.47499999999999998,"15min":0.47499999999999998},"min":{"1min":0.36899999999999999,"5min":0.36899999999999999,"15min":0.36899999999999999},"max":{"1min":0.71599999999999997,"5min":0.71599999999999997,"15min":0.71599999999999997},"last":0.56399999999999995},{"interface":"front","average":{"1min":0.46600000000000003,"5min":0.46600000000000003,"15min":0.46600000000000003},"min":{"1min":0.30599999999999999,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.60999999999999999,"5min":0.60999999999999999,"15min":0.60999999999999999},"last":0.33300000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:51:21 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.496,"15min":0.496},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.67000000000000004,"5min":0.67000000000000004,"15min":0.67000000000000004},"last":0.40799999999999997},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.48199999999999998,"15min":0.48199999999999998},"min":{"1min":0.30299999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.73099999999999998,"5min":0.73099999999999998,"15min":0.73099999999999998},"last":0.42099999999999999}]}]},{"osd":5,"up_from":37,"seq":158913789980,"num_pgs":21,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6032,"kb_used_data":648,"kb_used_omap":0,"kb_used_meta":5376,"kb_avail":93739120,"statfs":{"total":95995035648,"available":95988858880,"internally_reserved":0,"allocated":663552,"data_stored":287378,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5505024},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.33100000000000002,"5min":0.33100000000000002,"15min":0.33100000000000002},"max":{"1min":0.86299999999999999,"5min":0.86299999999999999,"15min":0.86299999999999999},"last":0.63300000000000001},{"interface":"front","average":{"1min":0.55100000000000005,"5min":0.55100000000000005,"15min":0.55100000000000005},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.73499999999999999,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.54300000000000004}]},{"osd":1,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54000000000000004,"5min":0.54000000000000004,"15min":0.54000000000000004},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.73899999999999999,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.53000000000000003},{"interface":"front","average":{"1min":0.54300000000000004,"5min":0.54300000000000004,"15min":0.54300000000000004},"min":{"1min":0.40100000000000002,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.752,"5min":0.752,"15min":0.752},"last":0.47799999999999998}]},{"osd":2,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.377,"5min":0.377,"15min":0.377},"max":{"1min":0.90000000000000002,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.51100000000000001},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.56799999999999995,"15min":0.56799999999999995},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.81399999999999995,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.5}]},{"osd":3,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.53700000000000003,"5min":0.53700000000000003,"15min":0.53700000000000003},"min":{"1min":0.40400000000000003,"5min":0.40400000000000003,"15min":0.40400000000000003},"max":{"1min":0.81200000000000006,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.61599999999999999},{"interface":"front","average":{"1min":0.55900000000000005,"5min":0.55900000000000005,"15min":0.55900000000000005},"min":{"1min":0.38800000000000001,"5min":0.38800000000000001,"15min":0.38800000000000001},"max":{"1min":0.72799999999999998,"5min":0.72799999999999998,"15min":0.72799999999999998},"last":0.45300000000000001}]},{"osd":4,"last update":"Sun Aug 27 20:50:50 2023","interfaces":[{"interface":"back","average":{"1min":0.44600000000000001,"5min":0.44600000000000001,"15min":0.44600000000000001},"min":{"1min":0.29399999999999998,"5min":0.29399999999999998,"15min":0.29399999999999998},"max":{"1min":0.88,"5min":0.88,"15min":0.88},"last":0.435},{"interface":"front","average":{"1min":0.46999999999999997,"5min":0.46999999999999997,"15min":0.46999999999999997},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.97799999999999998,"5min":0.97799999999999998,"15min":0.97799999999999998},"last":0.59299999999999997}]},{"osd":6,"last update":"Sun Aug 27 20:51:06 2023","interfaces":[{"interface":"back","average":{"1min":0.52000000000000002,"5min":0.52000000000000002,"15min":0.52000000000000002},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.92800000000000005,"5min":0.92800000000000005,"15min":0.92800000000000005},"last":0.55800000000000005},{"interface":"front","average":{"1min":0.53400000000000003,"5min":0.53400000000000003,"15min":0.53400000000000003},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.77100000000000002,"5min":0.77100000000000002,"15min":0.77100000000000002},"last":0.57199999999999995}]},{"osd":7,"last update":"Sun Aug 27 20:51:21 2023","interfaces":[{"interface":"back","average":{"1min":0.58299999999999996,"5min":0.58299999999999996,"15min":0.58299999999999996},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.79300000000000004,"5min":0.79300000000000004,"15min":0.79300000000000004},"last":0.58399999999999996},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.58499999999999996,"15min":0.58499999999999996},"min":{"1min":0.36199999999999999,"5min":0.36199999999999999,"15min":0.36199999999999999},"max":{"1min":0.95299999999999996,"5min":0.95299999999999996,"15min":0.95299999999999996},"last":0.64800000000000002}]}]},{"osd":6,"up_from":44,"seq":188978561049,"num_pgs":27,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":6556,"kb_used_data":1236,"kb_used_omap":0,"kb_used_meta":5312,"kb_avail":93738596,"statfs":{"total":95995035648,"available":95988322304,"internally_reserved":0,"allocated":1265664,"data_stored":878670,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":5439488},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51800000000000002,"15min":0.51800000000000002},"min":{"1min":0.36599999999999999,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.64300000000000002,"5min":0.64300000000000002,"15min":0.64300000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.55400000000000005,"5min":0.55400000000000005,"15min":0.55400000000000005},"min":{"1min":0.28899999999999998,"5min":0.28899999999999998,"15min":0.28899999999999998},"max":{"1min":1.524,"5min":1.524,"15min":1.524},"last":0.76300000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.56399999999999995,"5min":0.56399999999999995,"15min":0.56399999999999995},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":1.583,"5min":1.583,"15min":1.583},"last":0.57599999999999996},{"interface":"front","average":{"1min":0.54200000000000004,"5min":0.54200000000000004,"15min":0.54200000000000004},"min":{"1min":0.39000000000000001,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.72699999999999998,"5min":0.72699999999999998,"15min":0.72699999999999998},"last":0.64900000000000002}]},{"osd":2,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.68899999999999995,"5min":0.68899999999999995,"15min":0.68899999999999995},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":3.613,"5min":3.613,"15min":3.613},"last":0.81999999999999995},{"interface":"front","average":{"1min":0.81499999999999995,"5min":0.81499999999999995,"15min":0.81499999999999995},"min":{"1min":0.38600000000000001,"5min":0.38600000000000001,"15min":0.38600000000000001},"max":{"1min":5.9009999999999998,"5min":5.9009999999999998,"15min":5.9009999999999998},"last":0.71199999999999997}]},{"osd":3,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.69099999999999995,"5min":0.69099999999999995,"15min":0.69099999999999995},"min":{"1min":0.41199999999999998,"5min":0.41199999999999998,"15min":0.41199999999999998},"max":{"1min":3.7389999999999999,"5min":3.7389999999999999,"15min":3.7389999999999999},"last":0.61299999999999999},{"interface":"front","average":{"1min":0.71399999999999997,"5min":0.71399999999999997,"15min":0.71399999999999997},"min":{"1min":0.34000000000000002,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":3.4100000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.77800000000000002}]},{"osd":4,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.435,"5min":0.435,"15min":0.435},"min":{"1min":0.32400000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.753,"5min":0.753,"15min":0.753},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.46400000000000002,"5min":0.46400000000000002,"15min":0.46400000000000002},"min":{"1min":0.27300000000000002,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.47499999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:51:04 2023","interfaces":[{"interface":"back","average":{"1min":0.433,"5min":0.433,"15min":0.433},"min":{"1min":0.28699999999999998,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.79700000000000004,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.45100000000000001},{"interface":"front","average":{"1min":0.503,"5min":0.503,"15min":0.503},"min":{"1min":0.33000000000000002,"5min":0.33000000000000002,"15min":0.33000000000000002},"max":{"1min":0.82299999999999995,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.42499999999999999}]},{"osd":7,"last update":"Sun Aug 27 20:51:18 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.53200000000000003,"15min":0.53200000000000003},"min":{"1min":0.309,"5min":0.309,"15min":0.309},"max":{"1min":0.86499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.53600000000000003},{"interface":"front","average":{"1min":0.55500000000000005,"5min":0.55500000000000005,"15min":0.55500000000000005},"min":{"1min":0.34499999999999997,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.94399999999999995,"5min":0.94399999999999995,"15min":0.94399999999999995},"last":0.58199999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":1682,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":1606,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":1752,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T20:51:53.187 INFO:tasks.cephadm.ceph_manager.ceph:clean! 2023-08-27T20:51:53.187 INFO:tasks.ceph:Waiting until ceph cluster ceph is healthy... 2023-08-27T20:51:53.187 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy 2023-08-27T20:51:53.187 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph health --format=json 2023-08-27T20:51:53.451 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:54 smithi136 ceph-mon[113531]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:54 smithi136 ceph-mon[113531]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:54 smithi093 ceph-mon[111335]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:54 smithi093 ceph-mon[111335]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:54 smithi093 ceph-mon[118902]: from='client.14805 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:51:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:54 smithi093 ceph-mon[118902]: pgmap v46: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:54.639 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:51:54.640 INFO:teuthology.orchestra.run.smithi093.stdout:{"status":"HEALTH_OK","checks":{},"mutes":[]} 2023-08-27T20:51:55.076 INFO:tasks.cephadm.ceph_manager.ceph:wait_until_healthy done 2023-08-27T20:51:55.076 INFO:teuthology.run_tasks:Running task print... 2023-08-27T20:51:55.089 INFO:teuthology.task.print:**** upgrading first half of cluster, with stress **** 2023-08-27T20:51:55.089 INFO:teuthology.run_tasks:Running task parallel... 2023-08-27T20:51:55.102 INFO:teuthology.task.parallel:starting parallel... 2023-08-27T20:51:55.102 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-08-27T20:51:55.103 INFO:teuthology.task.sequential:In sequential, running task full_sequential... 2023-08-27T20:51:55.103 INFO:teuthology.task.full_sequential:In full_sequential, running task rados... 2023-08-27T20:51:55.106 INFO:tasks.rados:Beginning rados... 2023-08-27T20:51:55.106 INFO:tasks.rados:joining rados 2023-08-27T20:51:55.106 INFO:teuthology.task.parallel:In parallel, running task sequential... 2023-08-27T20:51:55.106 INFO:teuthology.task.sequential:In sequential, running task cephadm.shell... 2023-08-27T20:51:55.107 INFO:tasks.cephadm:Running commands on role mon.a host ubuntu@smithi093.front.sepia.ceph.com 2023-08-27T20:51:55.107 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph config set mgr mgr/cephadm/daemon_cache_timeout 60' 2023-08-27T20:51:55.108 INFO:tasks.rados:clients are ['client.0', 'client.1'] 2023-08-27T20:51:55.108 INFO:tasks.rados:starting run 0 out of 1 2023-08-27T20:51:55.108 INFO:tasks.cephadm.ceph_manager.ceph:creating pool_name unique_pool_0 2023-08-27T20:51:55.108 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd pool create unique_pool_0 16 2023-08-27T20:51:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4139373601' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T20:51:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4139373601' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T20:51:55.381 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4139373601' entity='client.admin' cmd=[{"prefix": "health", "format": "json"}]: dispatch 2023-08-27T20:51:55.658 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:51:55] "GET /metrics HTTP/1.1" 200 238722 "" "Prometheus/2.43.0" 2023-08-27T20:51:55.703 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:56.323 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:56 smithi093 ceph-mon[111335]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:56.323 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:56 smithi093 ceph-mon[118902]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:56 smithi136 ceph-mon[113531]: pgmap v47: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:57 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:51:57.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:57] ENGINE Bus STOPPING 2023-08-27T20:51:57.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:57] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T20:51:57.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:57] ENGINE Bus STOPPED 2023-08-27T20:51:57.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:57] ENGINE Bus STARTING 2023-08-27T20:51:57.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:57] ENGINE Serving on http://:::9283 2023-08-27T20:51:57.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:51:57 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:51:57] ENGINE Bus STARTED 2023-08-27T20:51:57.840 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph config set global log_to_journald false --force' 2023-08-27T20:51:58.060 INFO:teuthology.orchestra.run.smithi093.stderr:pool 'unique_pool_0' created 2023-08-27T20:51:58.087 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[111335]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1994457188' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[118902]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1994457188' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' 2023-08-27T20:51:58.088 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:58.089 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:58.089 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:58.089 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:58 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:58.108 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:58.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:58 smithi136 ceph-mon[113531]: pgmap v48: 73 pgs: 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:51:58.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1994457188' entity='client.admin' cmd=[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]: dispatch 2023-08-27T20:51:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:58 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' 2023-08-27T20:51:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:58 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:51:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:58 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:51:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:58 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:51:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:58 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:51:58.548 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd pool application enable unique_pool_0 rados --yes-i-really-mean-it || true 2023-08-27T20:51:59.092 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:51:59.310 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1994457188' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-27T20:51:59.310 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:51:59 smithi093 ceph-mon[111335]: osdmap e66: 8 total, 8 up, 8 in 2023-08-27T20:51:59.310 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1994457188' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-27T20:51:59.310 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:51:59 smithi093 ceph-mon[118902]: osdmap e66: 8 total, 8 up, 8 in 2023-08-27T20:51:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1994457188' entity='client.admin' cmd='[{"prefix": "osd pool create", "pool": "unique_pool_0", "pg_num": 16}]': finished 2023-08-27T20:51:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:51:59 smithi136 ceph-mon[113531]: osdmap e66: 8 total, 8 up, 8 in 2023-08-27T20:52:00.184 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch upgrade start --image quay.ceph.io/ceph-ci/ceph:$sha1' 2023-08-27T20:52:00.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:00 smithi136 ceph-mon[113531]: pgmap v50: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:00.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:00 smithi136 ceph-mon[113531]: osdmap e67: 8 total, 8 up, 8 in 2023-08-27T20:52:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:00 smithi093 ceph-mon[111335]: pgmap v50: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:00 smithi093 ceph-mon[111335]: osdmap e67: 8 total, 8 up, 8 in 2023-08-27T20:52:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:00 smithi093 ceph-mon[118902]: pgmap v50: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:00 smithi093 ceph-mon[118902]: osdmap e67: 8 total, 8 up, 8 in 2023-08-27T20:52:00.512 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:52:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:00] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:52:01.085 INFO:teuthology.orchestra.run.smithi093.stderr:enabled application 'rados' on pool 'unique_pool_0' 2023-08-27T20:52:01.413 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:01 smithi093 ceph-mon[111335]: osdmap e68: 8 total, 8 up, 8 in 2023-08-27T20:52:01.414 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3622394424' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:52:01.414 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:01 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:52:01.414 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:01 smithi093 ceph-mon[118902]: osdmap e68: 8 total, 8 up, 8 in 2023-08-27T20:52:01.414 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3622394424' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:52:01.414 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:01 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:52:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:01 smithi136 ceph-mon[113531]: osdmap e68: 8 total, 8 up, 8 in 2023-08-27T20:52:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3622394424' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:52:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:01 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd=[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]: dispatch 2023-08-27T20:52:02.127 INFO:teuthology.orchestra.run.smithi093.stdout:Initiating upgrade to quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:52:02.351 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:02 smithi093 ceph-mon[111335]: pgmap v53: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:02.351 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:02 smithi093 ceph-mon[111335]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:52:02.351 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:02 smithi093 ceph-mon[111335]: osdmap e69: 8 total, 8 up, 8 in 2023-08-27T20:52:02.351 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:02 smithi093 ceph-mon[118902]: pgmap v53: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:02.351 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:02 smithi093 ceph-mon[118902]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:52:02.351 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:02 smithi093 ceph-mon[118902]: osdmap e69: 8 total, 8 up, 8 in 2023-08-27T20:52:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:02 smithi136 ceph-mon[113531]: pgmap v53: 89 pgs: 16 unknown, 73 active+clean; 579 KiB data, 51 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:02 smithi136 ceph-mon[113531]: from='client.? ' entity='client.admin' cmd='[{"prefix": "osd pool application enable", "pool": "unique_pool_0", "app": "rados", "yes_i_really_mean_it": true}]': finished 2023-08-27T20:52:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:02 smithi136 ceph-mon[113531]: osdmap e69: 8 total, 8 up, 8 in 2023-08-27T20:52:02.565 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'ceph orch ps' 2023-08-27T20:52:02.639 DEBUG:teuthology.orchestra.run.smithi093:> CEPH_CLIENT_ID=0 adjust-ulimits ceph-coverage /home/ubuntu/cephtest/archive/coverage ceph_test_rados --max-ops 4000 --objects 500 --max-in-flight 16 --size 4000000 --min-stride-size 400000 --max-stride-size 800000 --max-seconds 0 --op read 45 --op write 45 --op delete 10 --pool unique_pool_0 2023-08-27T20:52:02.667 INFO:tasks.rados.rados.0.smithi093.stdout:adding op weight read -> 45 2023-08-27T20:52:02.667 INFO:tasks.rados.rados.0.smithi093.stdout:adding op weight write -> 45 2023-08-27T20:52:02.667 INFO:tasks.rados.rados.0.smithi093.stdout:adding op weight delete -> 10 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout:ceph version 17.2.6-1016-g4fe39952 (4fe39952b691f82409c13a09ef1217f15a8c4218) quincy (stable) 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout:Configuration: 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout: Number of operations: 4000 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout: Number of objects: 500 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout: Max in flight operations: 16 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout: Object size (in bytes): 4000000 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout: Write stride min: 400000 2023-08-27T20:52:02.668 INFO:tasks.rados.rados.0.smithi093.stdout: Write stride max: 800000 2023-08-27T20:52:02.841 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:52:03.158 INFO:tasks.rados.rados.0.smithi093.stdout:1: write initial oid 1 2023-08-27T20:52:03.158 INFO:tasks.rados.rados.0.smithi093.stdout:1: seq_num 0 ranges {416807=475249,1542129=474678} 2023-08-27T20:52:03.161 INFO:tasks.rados.rados.0.smithi093.stdout:1: writing smithi093153278-1 from 416807 to 892056 tid 1 2023-08-27T20:52:03.168 INFO:tasks.rados.rados.0.smithi093.stdout:1: writing smithi093153278-1 from 1542129 to 2016807 tid 2 2023-08-27T20:52:03.169 INFO:tasks.rados.rados.0.smithi093.stdout:2: write initial oid 2 2023-08-27T20:52:03.169 INFO:tasks.rados.rados.0.smithi093.stdout:2: seq_num 1 ranges {416807=475249,1542129=474678} 2023-08-27T20:52:03.174 INFO:tasks.rados.rados.0.smithi093.stdout:2: writing smithi093153278-2 from 416807 to 892056 tid 1 2023-08-27T20:52:03.182 INFO:tasks.rados.rados.0.smithi093.stdout:2: writing smithi093153278-2 from 1542129 to 2016807 tid 2 2023-08-27T20:52:03.182 INFO:tasks.rados.rados.0.smithi093.stdout:3: write initial oid 3 2023-08-27T20:52:03.182 INFO:tasks.rados.rados.0.smithi093.stdout:3: seq_num 2 ranges {433614=550498,1600611=433003} 2023-08-27T20:52:03.188 INFO:tasks.rados.rados.0.smithi093.stdout:3: writing smithi093153278-3 from 433614 to 984112 tid 1 2023-08-27T20:52:03.195 INFO:tasks.rados.rados.0.smithi093.stdout:3: writing smithi093153278-3 from 1600611 to 2033614 tid 2 2023-08-27T20:52:03.195 INFO:tasks.rados.rados.0.smithi093.stdout:4: write initial oid 4 2023-08-27T20:52:03.195 INFO:tasks.rados.rados.0.smithi093.stdout:1: finishing write tid 1 to smithi093153278-1 2023-08-27T20:52:03.195 INFO:tasks.rados.rados.0.smithi093.stdout:1: finishing write tid 2 to smithi093153278-1 2023-08-27T20:52:03.195 INFO:tasks.rados.rados.0.smithi093.stdout:1: finishing write tid 3 to smithi093153278-1 2023-08-27T20:52:03.195 INFO:tasks.rados.rados.0.smithi093.stdout:4: seq_num 3 ranges {450421=625747,1659093=391328} 2023-08-27T20:52:03.202 INFO:tasks.rados.rados.0.smithi093.stdout:4: writing smithi093153278-4 from 450421 to 1076168 tid 1 2023-08-27T20:52:03.209 INFO:tasks.rados.rados.0.smithi093.stdout:4: writing smithi093153278-4 from 1659093 to 2050421 tid 2 2023-08-27T20:52:03.209 INFO:tasks.rados.rados.0.smithi093.stdout:5: write initial oid 5 2023-08-27T20:52:03.209 INFO:tasks.rados.rados.0.smithi093.stdout:5: seq_num 4 ranges {467228=700996,1717575=349653} 2023-08-27T20:52:03.216 INFO:tasks.rados.rados.0.smithi093.stdout:5: writing smithi093153278-5 from 467228 to 1168224 tid 1 2023-08-27T20:52:03.222 INFO:tasks.rados.rados.0.smithi093.stdout:5: writing smithi093153278-5 from 1717575 to 2067228 tid 2 2023-08-27T20:52:03.222 INFO:tasks.rados.rados.0.smithi093.stdout:6: write initial oid 6 2023-08-27T20:52:03.222 INFO:tasks.rados.rados.0.smithi093.stdout:6: seq_num 5 ranges {484035=776245,2059704=24331} 2023-08-27T20:52:03.230 INFO:tasks.rados.rados.0.smithi093.stdout:6: writing smithi093153278-6 from 484035 to 1260280 tid 1 2023-08-27T20:52:03.235 INFO:tasks.rados.rados.0.smithi093.stdout:6: writing smithi093153278-6 from 2059704 to 2084035 tid 2 2023-08-27T20:52:03.235 INFO:tasks.rados.rados.0.smithi093.stdout:7: write initial oid 7 2023-08-27T20:52:03.235 INFO:tasks.rados.rados.0.smithi093.stdout:7: seq_num 6 ranges {500842=451494,1718186=382656} 2023-08-27T20:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/666083728' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[118902]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:52:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:52:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:52:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/666083728' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-08-27T20:52:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:03 smithi093 ceph-mon[111335]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-08-27T20:52:03.241 INFO:tasks.rados.rados.0.smithi093.stdout:7: writing smithi093153278-7 from 500842 to 952336 tid 1 2023-08-27T20:52:03.248 INFO:tasks.rados.rados.0.smithi093.stdout:7: writing smithi093153278-7 from 1718186 to 2100842 tid 2 2023-08-27T20:52:03.248 INFO:tasks.rados.rados.0.smithi093.stdout:8: write initial oid 8 2023-08-27T20:52:03.249 INFO:tasks.rados.rados.0.smithi093.stdout:8: seq_num 7 ranges {517649=526743,1776668=340981} 2023-08-27T20:52:03.255 INFO:tasks.rados.rados.0.smithi093.stdout:8: writing smithi093153278-8 from 517649 to 1044392 tid 1 2023-08-27T20:52:03.262 INFO:tasks.rados.rados.0.smithi093.stdout:8: writing smithi093153278-8 from 1776668 to 2117649 tid 2 2023-08-27T20:52:03.262 INFO:tasks.rados.rados.0.smithi093.stdout:9: write initial oid 9 2023-08-27T20:52:03.262 INFO:tasks.rados.rados.0.smithi093.stdout:9: seq_num 8 ranges {534456=718345,1951503=182953} 2023-08-27T20:52:03.270 INFO:tasks.rados.rados.0.smithi093.stdout:9: writing smithi093153278-9 from 534456 to 1252801 tid 1 2023-08-27T20:52:03.276 INFO:tasks.rados.rados.0.smithi093.stdout:9: writing smithi093153278-9 from 1951503 to 2134456 tid 2 2023-08-27T20:52:03.276 INFO:tasks.rados.rados.0.smithi093.stdout:10: write initial oid 10 2023-08-27T20:52:03.276 INFO:tasks.rados.rados.0.smithi093.stdout:10: seq_num 9 ranges {551263=793594,1893632=257631} 2023-08-27T20:52:03.284 INFO:tasks.rados.rados.0.smithi093.stdout:10: writing smithi093153278-10 from 551263 to 1344857 tid 1 2023-08-27T20:52:03.289 INFO:tasks.rados.rados.0.smithi093.stdout:10: writing smithi093153278-10 from 1893632 to 2151263 tid 2 2023-08-27T20:52:03.290 INFO:tasks.rados.rados.0.smithi093.stdout:11: write initial oid 11 2023-08-27T20:52:03.290 INFO:tasks.rados.rados.0.smithi093.stdout:11: seq_num 10 ranges {568070=468843,1552114=615956} 2023-08-27T20:52:03.296 INFO:tasks.rados.rados.0.smithi093.stdout:11: writing smithi093153278-11 from 568070 to 1036913 tid 1 2023-08-27T20:52:03.303 INFO:tasks.rados.rados.0.smithi093.stdout:11: writing smithi093153278-11 from 1552114 to 2168070 tid 2 2023-08-27T20:52:03.303 INFO:tasks.rados.rados.0.smithi093.stdout:12: write initial oid 12 2023-08-27T20:52:03.303 INFO:tasks.rados.rados.0.smithi093.stdout:1: finishing write tid 4 to smithi093153278-1 2023-08-27T20:52:03.304 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 1 v 3 (ObjNum 0 snap 0 seq_num 0) dirty exists 2023-08-27T20:52:03.304 INFO:tasks.rados.rados.0.smithi093.stdout:1: left oid 1 (ObjNum 0 snap 0 seq_num 0) 2023-08-27T20:52:03.304 INFO:tasks.rados.rados.0.smithi093.stdout:2: finishing write tid 1 to smithi093153278-2 2023-08-27T20:52:03.304 INFO:tasks.rados.rados.0.smithi093.stdout:12: seq_num 11 ranges {584877=544092,1610596=562003} 2023-08-27T20:52:03.310 INFO:tasks.rados.rados.0.smithi093.stdout:12: writing smithi093153278-12 from 584877 to 1128969 tid 1 2023-08-27T20:52:03.317 INFO:tasks.rados.rados.0.smithi093.stdout:12: writing smithi093153278-12 from 1610596 to 2172599 tid 2 2023-08-27T20:52:03.317 INFO:tasks.rados.rados.0.smithi093.stdout:1: done (11 left) 2023-08-27T20:52:03.317 INFO:tasks.rados.rados.0.smithi093.stdout:13: write initial oid 13 2023-08-27T20:52:03.317 INFO:tasks.rados.rados.0.smithi093.stdout:13: seq_num 12 ranges {601684=619341,1669078=532606} 2023-08-27T20:52:03.324 INFO:tasks.rados.rados.0.smithi093.stdout:13: writing smithi093153278-13 from 601684 to 1221025 tid 1 2023-08-27T20:52:03.331 INFO:tasks.rados.rados.0.smithi093.stdout:13: writing smithi093153278-13 from 1669078 to 2201684 tid 2 2023-08-27T20:52:03.331 INFO:tasks.rados.rados.0.smithi093.stdout:14: write initial oid 14 2023-08-27T20:52:03.331 INFO:tasks.rados.rados.0.smithi093.stdout:14: seq_num 13 ranges {618491=694590,2011207=207284} 2023-08-27T20:52:03.339 INFO:tasks.rados.rados.0.smithi093.stdout:14: writing smithi093153278-14 from 618491 to 1313081 tid 1 2023-08-27T20:52:03.344 INFO:tasks.rados.rados.0.smithi093.stdout:14: writing smithi093153278-14 from 2011207 to 2218491 tid 2 2023-08-27T20:52:03.345 INFO:tasks.rados.rados.0.smithi093.stdout:15: write initial oid 15 2023-08-27T20:52:03.345 INFO:tasks.rados.rados.0.smithi093.stdout:15: seq_num 14 ranges {635298=769839,2069689=165609} 2023-08-27T20:52:03.353 INFO:tasks.rados.rados.0.smithi093.stdout:15: writing smithi093153278-15 from 635298 to 1405137 tid 1 2023-08-27T20:52:03.359 INFO:tasks.rados.rados.0.smithi093.stdout:15: writing smithi093153278-15 from 2069689 to 2235298 tid 2 2023-08-27T20:52:03.359 INFO:tasks.rados.rados.0.smithi093.stdout:16: write initial oid 16 2023-08-27T20:52:03.359 INFO:tasks.rados.rados.0.smithi093.stdout:2: finishing write tid 2 to smithi093153278-2 2023-08-27T20:52:03.359 INFO:tasks.rados.rados.0.smithi093.stdout:2: finishing write tid 3 to smithi093153278-2 2023-08-27T20:52:03.359 INFO:tasks.rados.rados.0.smithi093.stdout:16: seq_num 15 ranges {652105=445088,1728171=452988} 2023-08-27T20:52:03.365 INFO:tasks.rados.rados.0.smithi093.stdout:16: writing smithi093153278-16 from 652105 to 1097193 tid 1 2023-08-27T20:52:03.372 INFO:tasks.rados.rados.0.smithi093.stdout:16: writing smithi093153278-16 from 1728171 to 2181159 tid 2 2023-08-27T20:52:03.372 INFO:tasks.rados.rados.0.smithi093.stdout:17: write initial oid 17 2023-08-27T20:52:03.372 INFO:tasks.rados.rados.0.smithi093.stdout:17: seq_num 16 ranges {668912=636690,1903006=365906} 2023-08-27T20:52:03.380 INFO:tasks.rados.rados.0.smithi093.stdout:17: writing smithi093153278-17 from 668912 to 1305602 tid 1 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:17: writing smithi093153278-17 from 1903006 to 2268912 tid 2 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:2: finishing write tid 4 to smithi093153278-2 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 2 v 3 (ObjNum 1 snap 0 seq_num 1) dirty exists 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:2: left oid 2 (ObjNum 1 snap 0 seq_num 1) 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:3: finishing write tid 1 to smithi093153278-3 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:2: done (15 left) 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:18: write initial oid 18 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:3: finishing write tid 2 to smithi093153278-3 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:3: finishing write tid 3 to smithi093153278-3 2023-08-27T20:52:03.387 INFO:tasks.rados.rados.0.smithi093.stdout:3: finishing write tid 4 to smithi093153278-3 2023-08-27T20:52:03.388 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 3 v 3 (ObjNum 2 snap 0 seq_num 2) dirty exists 2023-08-27T20:52:03.388 INFO:tasks.rados.rados.0.smithi093.stdout:3: left oid 3 (ObjNum 2 snap 0 seq_num 2) 2023-08-27T20:52:03.388 INFO:tasks.rados.rados.0.smithi093.stdout:4: finishing write tid 1 to smithi093153278-4 2023-08-27T20:52:03.388 INFO:tasks.rados.rados.0.smithi093.stdout:4: finishing write tid 2 to smithi093153278-4 2023-08-27T20:52:03.388 INFO:tasks.rados.rados.0.smithi093.stdout:18: seq_num 17 ranges {685719=711939,1845135=440584} 2023-08-27T20:52:03.395 INFO:tasks.rados.rados.0.smithi093.stdout:18: writing smithi093153278-18 from 685719 to 1397658 tid 1 2023-08-27T20:52:03.401 INFO:tasks.rados.rados.0.smithi093.stdout:18: writing smithi093153278-18 from 1845135 to 2285719 tid 2 2023-08-27T20:52:03.401 INFO:tasks.rados.rados.0.smithi093.stdout:3: done (15 left) 2023-08-27T20:52:03.401 INFO:tasks.rados.rados.0.smithi093.stdout:19: write initial oid 19 2023-08-27T20:52:03.401 INFO:tasks.rados.rados.0.smithi093.stdout:19: seq_num 18 ranges {702526=787188,1903617=398909} 2023-08-27T20:52:03.410 INFO:tasks.rados.rados.0.smithi093.stdout:19: writing smithi093153278-19 from 702526 to 1489714 tid 1 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:19: writing smithi093153278-19 from 1903617 to 2302526 tid 2 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:4: finishing write tid 3 to smithi093153278-4 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:4: finishing write tid 4 to smithi093153278-4 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 4 v 3 (ObjNum 3 snap 0 seq_num 3) dirty exists 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:4: left oid 4 (ObjNum 3 snap 0 seq_num 3) 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:5: finishing write tid 1 to smithi093153278-5 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:5: finishing write tid 2 to smithi093153278-5 2023-08-27T20:52:03.416 INFO:tasks.rados.rados.0.smithi093.stdout:5: finishing write tid 3 to smithi093153278-5 2023-08-27T20:52:03.417 INFO:tasks.rados.rados.0.smithi093.stdout:4: done (15 left) 2023-08-27T20:52:03.417 INFO:tasks.rados.rados.0.smithi093.stdout:20: write initial oid 20 2023-08-27T20:52:03.417 INFO:tasks.rados.rados.0.smithi093.stdout:20: seq_num 19 ranges {719333=462437,1962099=357234} 2023-08-27T20:52:03.423 INFO:tasks.rados.rados.0.smithi093.stdout:20: writing smithi093153278-20 from 719333 to 1181770 tid 1 2023-08-27T20:52:03.430 INFO:tasks.rados.rados.0.smithi093.stdout:20: writing smithi093153278-20 from 1962099 to 2319333 tid 2 2023-08-27T20:52:03.430 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.430 INFO:tasks.rados.rados.0.smithi093.stdout:5: finishing write tid 4 to smithi093153278-5 2023-08-27T20:52:03.430 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 5 v 3 (ObjNum 4 snap 0 seq_num 4) dirty exists 2023-08-27T20:52:03.431 INFO:tasks.rados.rados.0.smithi093.stdout:5: left oid 5 (ObjNum 4 snap 0 seq_num 4) 2023-08-27T20:52:03.431 INFO:tasks.rados.rados.0.smithi093.stdout:6: finishing write tid 1 to smithi093153278-6 2023-08-27T20:52:03.431 INFO:tasks.rados.rados.0.smithi093.stdout:6: finishing write tid 2 to smithi093153278-6 2023-08-27T20:52:03.431 INFO:tasks.rados.rados.0.smithi093.stdout:5: done (15 left) 2023-08-27T20:52:03.431 INFO:tasks.rados.rados.0.smithi093.stdout:21: write initial oid 21 2023-08-27T20:52:03.431 INFO:tasks.rados.rados.0.smithi093.stdout:21: seq_num 20 ranges {736140=537686,2020581=315559} 2023-08-27T20:52:03.438 INFO:tasks.rados.rados.0.smithi093.stdout:21: writing smithi093153278-21 from 736140 to 1273826 tid 1 2023-08-27T20:52:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='client.14841 -' entity='client.admin' cmd=[{"prefix": "orch upgrade start", "image": "quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:52:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: Upgrade: Started with target quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:52:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:52:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:52:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:52:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: Upgrade: First pull of quay.ceph.io/ceph-ci/ceph:9cf208207dfe23ffccb33308b22e51570cb45e51 2023-08-27T20:52:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/666083728' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-08-27T20:52:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:03 smithi136 ceph-mon[113531]: from='client.? ' entity='client.0' cmd=[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]: dispatch 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:21: writing smithi093153278-21 from 2020581 to 2336140 tid 2 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:6: finishing write tid 3 to smithi093153278-6 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:6: finishing write tid 4 to smithi093153278-6 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 6 v 6 (ObjNum 5 snap 0 seq_num 5) dirty exists 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:6: left oid 6 (ObjNum 5 snap 0 seq_num 5) 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:7: finishing write tid 1 to smithi093153278-7 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:7: finishing write tid 2 to smithi093153278-7 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:7: finishing write tid 3 to smithi093153278-7 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:6: done (15 left) 2023-08-27T20:52:03.445 INFO:tasks.rados.rados.0.smithi093.stdout:22: write initial oid 22 2023-08-27T20:52:03.446 INFO:tasks.rados.rados.0.smithi093.stdout:22: seq_num 21 ranges {752947=612935,1962710=390237} 2023-08-27T20:52:03.453 INFO:tasks.rados.rados.0.smithi093.stdout:22: writing smithi093153278-22 from 752947 to 1365882 tid 1 2023-08-27T20:52:03.459 INFO:tasks.rados.rados.0.smithi093.stdout:22: writing smithi093153278-22 from 1962710 to 2352947 tid 2 2023-08-27T20:52:03.459 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:7: finishing write tid 4 to smithi093153278-7 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 7 v 3 (ObjNum 6 snap 0 seq_num 6) dirty exists 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:7: left oid 7 (ObjNum 6 snap 0 seq_num 6) 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:8: finishing write tid 1 to smithi093153278-8 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:8: finishing write tid 2 to smithi093153278-8 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:8: finishing write tid 3 to smithi093153278-8 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:7: done (15 left) 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:23: write initial oid 23 2023-08-27T20:52:03.460 INFO:tasks.rados.rados.0.smithi093.stdout:23: seq_num 22 ranges {769754=688184,2021192=348562} 2023-08-27T20:52:03.468 INFO:tasks.rados.rados.0.smithi093.stdout:23: writing smithi093153278-23 from 769754 to 1457938 tid 1 2023-08-27T20:52:03.474 INFO:tasks.rados.rados.0.smithi093.stdout:23: writing smithi093153278-23 from 2021192 to 2369754 tid 2 2023-08-27T20:52:03.474 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:8: finishing write tid 4 to smithi093153278-8 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 8 v 3 (ObjNum 7 snap 0 seq_num 7) dirty exists 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:8: left oid 8 (ObjNum 7 snap 0 seq_num 7) 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:9: finishing write tid 1 to smithi093153278-9 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:9: finishing write tid 2 to smithi093153278-9 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:8: done (15 left) 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:24: write initial oid 24 2023-08-27T20:52:03.475 INFO:tasks.rados.rados.0.smithi093.stdout:24: seq_num 23 ranges {786561=479786,1796027=467664} 2023-08-27T20:52:03.482 INFO:tasks.rados.rados.0.smithi093.stdout:24: writing smithi093153278-24 from 786561 to 1266347 tid 1 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:24: writing smithi093153278-24 from 1796027 to 2263691 tid 2 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:9: finishing write tid 3 to smithi093153278-9 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:9: finishing write tid 4 to smithi093153278-9 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 9 v 6 (ObjNum 8 snap 0 seq_num 8) dirty exists 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:9: left oid 9 (ObjNum 8 snap 0 seq_num 8) 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:10: finishing write tid 1 to smithi093153278-10 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:10: finishing write tid 2 to smithi093153278-10 2023-08-27T20:52:03.489 INFO:tasks.rados.rados.0.smithi093.stdout:10: finishing write tid 3 to smithi093153278-10 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:10: finishing write tid 4 to smithi093153278-10 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 10 v 3 (ObjNum 9 snap 0 seq_num 9) dirty exists 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:10: left oid 10 (ObjNum 9 snap 0 seq_num 9) 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:11: finishing write tid 1 to smithi093153278-11 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:9: done (15 left) 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:10: done (14 left) 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:25: write initial oid 25 2023-08-27T20:52:03.490 INFO:tasks.rados.rados.0.smithi093.stdout:25: seq_num 24 ranges {403368=555035,1454509=727675} 2023-08-27T20:52:03.494 INFO:tasks.rados.rados.0.smithi093.stdout:25: writing smithi093153278-25 from 403368 to 958403 tid 1 2023-08-27T20:52:03.502 INFO:tasks.rados.rados.0.smithi093.stdout:25: writing smithi093153278-25 from 1454509 to 2182184 tid 2 2023-08-27T20:52:03.503 INFO:tasks.rados.rados.0.smithi093.stdout:26: write initial oid 26 2023-08-27T20:52:03.503 INFO:tasks.rados.rados.0.smithi093.stdout:26: seq_num 25 ranges {420175=630284,1796638=471333} 2023-08-27T20:52:03.510 INFO:tasks.rados.rados.0.smithi093.stdout:26: writing smithi093153278-26 from 420175 to 1050459 tid 1 2023-08-27T20:52:03.517 INFO:tasks.rados.rados.0.smithi093.stdout:26: writing smithi093153278-26 from 1796638 to 2267971 tid 2 2023-08-27T20:52:03.518 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.518 INFO:tasks.rados.rados.0.smithi093.stdout:11: finishing write tid 2 to smithi093153278-11 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:11: finishing write tid 3 to smithi093153278-11 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:11: finishing write tid 4 to smithi093153278-11 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 11 v 6 (ObjNum 10 snap 0 seq_num 10) dirty exists 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:11: left oid 11 (ObjNum 10 snap 0 seq_num 10) 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:12: finishing write tid 1 to smithi093153278-12 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:12: finishing write tid 2 to smithi093153278-12 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:12: finishing write tid 3 to smithi093153278-12 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:12: finishing write tid 4 to smithi093153278-12 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 12 v 6 (ObjNum 11 snap 0 seq_num 11) dirty exists 2023-08-27T20:52:03.519 INFO:tasks.rados.rados.0.smithi093.stdout:12: left oid 12 (ObjNum 11 snap 0 seq_num 11) 2023-08-27T20:52:03.520 INFO:tasks.rados.rados.0.smithi093.stdout:13: finishing write tid 1 to smithi093153278-13 2023-08-27T20:52:03.520 INFO:tasks.rados.rados.0.smithi093.stdout:11: done (15 left) 2023-08-27T20:52:03.520 INFO:tasks.rados.rados.0.smithi093.stdout:12: done (14 left) 2023-08-27T20:52:03.520 INFO:tasks.rados.rados.0.smithi093.stdout:27: write initial oid 27 2023-08-27T20:52:03.520 INFO:tasks.rados.rados.0.smithi093.stdout:27: seq_num 26 ranges {436982=705533,1855120=581862} 2023-08-27T20:52:03.526 INFO:tasks.rados.rados.0.smithi093.stdout:27: writing smithi093153278-27 from 436982 to 1142515 tid 1 2023-08-27T20:52:03.536 INFO:tasks.rados.rados.0.smithi093.stdout:27: writing smithi093153278-27 from 1855120 to 2436982 tid 2 2023-08-27T20:52:03.536 INFO:tasks.rados.rados.0.smithi093.stdout:28: write initial oid 28 2023-08-27T20:52:03.536 INFO:tasks.rados.rados.0.smithi093.stdout:28: seq_num 27 ranges {453789=780782,1913602=475002} 2023-08-27T20:52:03.545 INFO:tasks.rados.rados.0.smithi093.stdout:28: writing smithi093153278-28 from 453789 to 1234571 tid 1 2023-08-27T20:52:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:28: writing smithi093153278-28 from 1913602 to 2388604 tid 2 2023-08-27T20:52:03.551 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:13: finishing write tid 2 to smithi093153278-13 2023-08-27T20:52:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:13: finishing write tid 3 to smithi093153278-13 2023-08-27T20:52:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:13: finishing write tid 4 to smithi093153278-13 2023-08-27T20:52:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 13 v 9 (ObjNum 12 snap 0 seq_num 12) dirty exists 2023-08-27T20:52:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:13: left oid 13 (ObjNum 12 snap 0 seq_num 12) 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:14: finishing write tid 1 to smithi093153278-14 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:14: finishing write tid 2 to smithi093153278-14 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:14: finishing write tid 3 to smithi093153278-14 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:14: finishing write tid 4 to smithi093153278-14 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 14 v 3 (ObjNum 13 snap 0 seq_num 13) dirty exists 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:14: left oid 14 (ObjNum 13 snap 0 seq_num 13) 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:15: finishing write tid 1 to smithi093153278-15 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:15: finishing write tid 2 to smithi093153278-15 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:15: finishing write tid 3 to smithi093153278-15 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:15: finishing write tid 4 to smithi093153278-15 2023-08-27T20:52:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 15 v 6 (ObjNum 14 snap 0 seq_num 14) dirty exists 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:15: left oid 15 (ObjNum 14 snap 0 seq_num 14) 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:16: finishing write tid 1 to smithi093153278-16 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:16: finishing write tid 2 to smithi093153278-16 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:16: finishing write tid 3 to smithi093153278-16 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:16: finishing write tid 4 to smithi093153278-16 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 16 v 9 (ObjNum 15 snap 0 seq_num 15) dirty exists 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:16: left oid 16 (ObjNum 15 snap 0 seq_num 15) 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:17: finishing write tid 1 to smithi093153278-17 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:13: done (15 left) 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:14: done (14 left) 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:15: done (13 left) 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:16: done (12 left) 2023-08-27T20:52:03.553 INFO:tasks.rados.rados.0.smithi093.stdout:29: write initial oid 29 2023-08-27T20:52:03.554 INFO:tasks.rados.rados.0.smithi093.stdout:29: seq_num 28 ranges {470596=456031,1572084=618660} 2023-08-27T20:52:03.555 INFO:tasks.rados.rados.0.smithi093.stdout:29: writing smithi093153278-29 from 470596 to 926627 tid 1 2023-08-27T20:52:03.562 INFO:tasks.rados.rados.0.smithi093.stdout:29: writing smithi093153278-29 from 1572084 to 2190744 tid 2 2023-08-27T20:52:03.563 INFO:tasks.rados.rados.0.smithi093.stdout:30: write initial oid 30 2023-08-27T20:52:03.563 INFO:tasks.rados.rados.0.smithi093.stdout:30: seq_num 29 ranges {487403=531280,1514213=478671} 2023-08-27T20:52:03.568 INFO:tasks.rados.rados.0.smithi093.stdout:30: writing smithi093153278-30 from 487403 to 1018683 tid 1 2023-08-27T20:52:03.574 INFO:tasks.rados.rados.0.smithi093.stdout:30: writing smithi093153278-30 from 1514213 to 1992884 tid 2 2023-08-27T20:52:03.575 INFO:tasks.rados.rados.0.smithi093.stdout:31: write initial oid 31 2023-08-27T20:52:03.575 INFO:tasks.rados.rados.0.smithi093.stdout:31: seq_num 30 ranges {504210=606529,1572695=622329} 2023-08-27T20:52:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:31: writing smithi093153278-31 from 504210 to 1110739 tid 1 2023-08-27T20:52:03.589 INFO:tasks.rados.rados.0.smithi093.stdout:31: writing smithi093153278-31 from 1572695 to 2195024 tid 2 2023-08-27T20:52:03.589 INFO:tasks.rados.rados.0.smithi093.stdout:32: write initial oid 32 2023-08-27T20:52:03.589 INFO:tasks.rados.rados.0.smithi093.stdout:32: seq_num 31 ranges {521017=798131,1747530=482340} 2023-08-27T20:52:03.597 INFO:tasks.rados.rados.0.smithi093.stdout:32: writing smithi093153278-32 from 521017 to 1319148 tid 1 2023-08-27T20:52:03.604 INFO:tasks.rados.rados.0.smithi093.stdout:32: writing smithi093153278-32 from 1747530 to 2229870 tid 2 2023-08-27T20:52:03.604 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.604 INFO:tasks.rados.rados.0.smithi093.stdout:17: finishing write tid 2 to smithi093153278-17 2023-08-27T20:52:03.604 INFO:tasks.rados.rados.0.smithi093.stdout:17: finishing write tid 3 to smithi093153278-17 2023-08-27T20:52:03.604 INFO:tasks.rados.rados.0.smithi093.stdout:17: finishing write tid 4 to smithi093153278-17 2023-08-27T20:52:03.604 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 17 v 6 (ObjNum 16 snap 0 seq_num 16) dirty exists 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:17: left oid 17 (ObjNum 16 snap 0 seq_num 16) 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:18: finishing write tid 1 to smithi093153278-18 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:18: finishing write tid 2 to smithi093153278-18 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:18: finishing write tid 3 to smithi093153278-18 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:18: finishing write tid 4 to smithi093153278-18 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 18 v 9 (ObjNum 17 snap 0 seq_num 17) dirty exists 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:18: left oid 18 (ObjNum 17 snap 0 seq_num 17) 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:17: done (15 left) 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:18: done (14 left) 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:33: write initial oid 33 2023-08-27T20:52:03.605 INFO:tasks.rados.rados.0.smithi093.stdout:33: seq_num 32 ranges {537824=473380,1806012=625998} 2023-08-27T20:52:03.610 INFO:tasks.rados.rados.0.smithi093.stdout:33: writing smithi093153278-33 from 537824 to 1011204 tid 1 2023-08-27T20:52:03.621 INFO:tasks.rados.rados.0.smithi093.stdout:33: writing smithi093153278-33 from 1806012 to 2432010 tid 2 2023-08-27T20:52:03.621 INFO:tasks.rados.rados.0.smithi093.stdout:34: write initial oid 34 2023-08-27T20:52:03.621 INFO:tasks.rados.rados.0.smithi093.stdout:34: seq_num 33 ranges {554631=548629,1748141=486009} 2023-08-27T20:52:03.630 INFO:tasks.rados.rados.0.smithi093.stdout:34: writing smithi093153278-34 from 554631 to 1103260 tid 1 2023-08-27T20:52:03.635 INFO:tasks.rados.rados.0.smithi093.stdout:34: writing smithi093153278-34 from 1748141 to 2234150 tid 2 2023-08-27T20:52:03.635 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.635 INFO:tasks.rados.rados.0.smithi093.stdout:19: finishing write tid 1 to smithi093153278-19 2023-08-27T20:52:03.635 INFO:tasks.rados.rados.0.smithi093.stdout:19: finishing write tid 2 to smithi093153278-19 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:19: finishing write tid 3 to smithi093153278-19 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:19: finishing write tid 4 to smithi093153278-19 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 19 v 3 (ObjNum 18 snap 0 seq_num 18) dirty exists 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:19: left oid 19 (ObjNum 18 snap 0 seq_num 18) 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:20: finishing write tid 1 to smithi093153278-20 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:20: finishing write tid 2 to smithi093153278-20 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:20: finishing write tid 3 to smithi093153278-20 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:19: done (15 left) 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:35: write initial oid 35 2023-08-27T20:52:03.636 INFO:tasks.rados.rados.0.smithi093.stdout:35: seq_num 34 ranges {571438=623878,1806623=629667} 2023-08-27T20:52:03.643 INFO:tasks.rados.rados.0.smithi093.stdout:35: writing smithi093153278-35 from 571438 to 1195316 tid 1 2023-08-27T20:52:03.653 INFO:tasks.rados.rados.0.smithi093.stdout:35: writing smithi093153278-35 from 1806623 to 2436290 tid 2 2023-08-27T20:52:03.653 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.653 INFO:tasks.rados.rados.0.smithi093.stdout:20: finishing write tid 4 to smithi093153278-20 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 3 (ObjNum 19 snap 0 seq_num 19) dirty exists 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:20: left oid 20 (ObjNum 19 snap 0 seq_num 19) 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:21: finishing write tid 1 to smithi093153278-21 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:21: finishing write tid 2 to smithi093153278-21 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:21: finishing write tid 3 to smithi093153278-21 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:21: finishing write tid 4 to smithi093153278-21 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 21 v 12 (ObjNum 20 snap 0 seq_num 20) dirty exists 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:21: left oid 21 (ObjNum 20 snap 0 seq_num 20) 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:22: finishing write tid 1 to smithi093153278-22 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:22: finishing write tid 2 to smithi093153278-22 2023-08-27T20:52:03.654 INFO:tasks.rados.rados.0.smithi093.stdout:22: finishing write tid 3 to smithi093153278-22 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:22: finishing write tid 4 to smithi093153278-22 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 22 v 9 (ObjNum 21 snap 0 seq_num 21) dirty exists 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:22: left oid 22 (ObjNum 21 snap 0 seq_num 21) 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:23: finishing write tid 1 to smithi093153278-23 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:23: finishing write tid 2 to smithi093153278-23 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:23: finishing write tid 3 to smithi093153278-23 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:23: finishing write tid 4 to smithi093153278-23 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 23 v 6 (ObjNum 22 snap 0 seq_num 22) dirty exists 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:23: left oid 23 (ObjNum 22 snap 0 seq_num 22) 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:24: finishing write tid 1 to smithi093153278-24 2023-08-27T20:52:03.655 INFO:tasks.rados.rados.0.smithi093.stdout:24: finishing write tid 2 to smithi093153278-24 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:24: finishing write tid 3 to smithi093153278-24 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:24: finishing write tid 4 to smithi093153278-24 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 24 v 12 (ObjNum 23 snap 0 seq_num 23) dirty exists 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:24: left oid 24 (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:25: finishing write tid 1 to smithi093153278-25 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:25: finishing write tid 2 to smithi093153278-25 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:25: finishing write tid 3 to smithi093153278-25 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:25: finishing write tid 4 to smithi093153278-25 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 25 v 6 (ObjNum 24 snap 0 seq_num 24) dirty exists 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:25: left oid 25 (ObjNum 24 snap 0 seq_num 24) 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:26: finishing write tid 1 to smithi093153278-26 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:26: finishing write tid 2 to smithi093153278-26 2023-08-27T20:52:03.656 INFO:tasks.rados.rados.0.smithi093.stdout:26: finishing write tid 3 to smithi093153278-26 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:26: finishing write tid 4 to smithi093153278-26 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 26 v 12 (ObjNum 25 snap 0 seq_num 25) dirty exists 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:26: left oid 26 (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:28: finishing write tid 1 to smithi093153278-28 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:27: finishing write tid 1 to smithi093153278-27 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:28: finishing write tid 2 to smithi093153278-28 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:28: finishing write tid 3 to smithi093153278-28 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:28: finishing write tid 4 to smithi093153278-28 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 28 v 15 (ObjNum 27 snap 0 seq_num 27) dirty exists 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:28: left oid 28 (ObjNum 27 snap 0 seq_num 27) 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:27: finishing write tid 2 to smithi093153278-27 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:27: finishing write tid 3 to smithi093153278-27 2023-08-27T20:52:03.657 INFO:tasks.rados.rados.0.smithi093.stdout:27: finishing write tid 4 to smithi093153278-27 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 27 v 3 (ObjNum 26 snap 0 seq_num 26) dirty exists 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:27: left oid 27 (ObjNum 26 snap 0 seq_num 26) 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:29: finishing write tid 1 to smithi093153278-29 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:29: finishing write tid 2 to smithi093153278-29 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:29: finishing write tid 3 to smithi093153278-29 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:29: finishing write tid 4 to smithi093153278-29 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 29 v 9 (ObjNum 28 snap 0 seq_num 28) dirty exists 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:29: left oid 29 (ObjNum 28 snap 0 seq_num 28) 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:30: finishing write tid 1 to smithi093153278-30 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:30: finishing write tid 2 to smithi093153278-30 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:30: finishing write tid 3 to smithi093153278-30 2023-08-27T20:52:03.658 INFO:tasks.rados.rados.0.smithi093.stdout:30: finishing write tid 4 to smithi093153278-30 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 30 v 12 (ObjNum 29 snap 0 seq_num 29) dirty exists 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:30: left oid 30 (ObjNum 29 snap 0 seq_num 29) 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:31: finishing write tid 1 to smithi093153278-31 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:31: finishing write tid 2 to smithi093153278-31 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:31: finishing write tid 3 to smithi093153278-31 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:31: finishing write tid 4 to smithi093153278-31 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 31 v 9 (ObjNum 30 snap 0 seq_num 30) dirty exists 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:31: left oid 31 (ObjNum 30 snap 0 seq_num 30) 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:32: finishing write tid 1 to smithi093153278-32 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:32: finishing write tid 2 to smithi093153278-32 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:32: finishing write tid 3 to smithi093153278-32 2023-08-27T20:52:03.659 INFO:tasks.rados.rados.0.smithi093.stdout:32: finishing write tid 4 to smithi093153278-32 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 32 v 15 (ObjNum 31 snap 0 seq_num 31) dirty exists 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:32: left oid 32 (ObjNum 31 snap 0 seq_num 31) 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:33: finishing write tid 1 to smithi093153278-33 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:33: finishing write tid 2 to smithi093153278-33 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:33: finishing write tid 3 to smithi093153278-33 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:33: finishing write tid 4 to smithi093153278-33 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 33 v 3 (ObjNum 32 snap 0 seq_num 32) dirty exists 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:33: left oid 33 (ObjNum 32 snap 0 seq_num 32) 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:34: finishing write tid 1 to smithi093153278-34 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:34: finishing write tid 2 to smithi093153278-34 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:34: finishing write tid 3 to smithi093153278-34 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:34: finishing write tid 4 to smithi093153278-34 2023-08-27T20:52:03.660 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 34 v 6 (ObjNum 33 snap 0 seq_num 33) dirty exists 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:34: left oid 34 (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:35: finishing write tid 1 to smithi093153278-35 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:20: done (15 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:21: done (14 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:22: done (13 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:23: done (12 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:24: done (11 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:25: done (10 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:26: done (9 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:27: done (8 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:28: done (7 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:29: done (6 left) 2023-08-27T20:52:03.661 INFO:tasks.rados.rados.0.smithi093.stdout:30: done (5 left) 2023-08-27T20:52:03.662 INFO:tasks.rados.rados.0.smithi093.stdout:31: done (4 left) 2023-08-27T20:52:03.662 INFO:tasks.rados.rados.0.smithi093.stdout:32: done (3 left) 2023-08-27T20:52:03.662 INFO:tasks.rados.rados.0.smithi093.stdout:33: done (2 left) 2023-08-27T20:52:03.662 INFO:tasks.rados.rados.0.smithi093.stdout:34: done (1 left) 2023-08-27T20:52:03.662 INFO:tasks.rados.rados.0.smithi093.stdout:36: write initial oid 36 2023-08-27T20:52:03.662 INFO:tasks.rados.rados.0.smithi093.stdout:36: seq_num 35 ranges {588245=699127,1865105=489678} 2023-08-27T20:52:03.662 INFO:tasks.rados.rados.0.smithi093.stdout:36: writing smithi093153278-36 from 588245 to 1287372 tid 1 2023-08-27T20:52:03.666 INFO:tasks.rados.rados.0.smithi093.stdout:36: writing smithi093153278-36 from 1865105 to 2354783 tid 2 2023-08-27T20:52:03.666 INFO:tasks.rados.rados.0.smithi093.stdout:37: write initial oid 37 2023-08-27T20:52:03.666 INFO:tasks.rados.rados.0.smithi093.stdout:37: seq_num 36 ranges {605052=774376,1923587=633336} 2023-08-27T20:52:03.675 INFO:tasks.rados.rados.0.smithi093.stdout:37: writing smithi093153278-37 from 605052 to 1379428 tid 1 2023-08-27T20:52:03.683 INFO:tasks.rados.rados.0.smithi093.stdout:37: writing smithi093153278-37 from 1923587 to 2556923 tid 2 2023-08-27T20:52:03.684 INFO:tasks.rados.rados.0.smithi093.stdout:38: write initial oid 38 2023-08-27T20:52:03.684 INFO:tasks.rados.rados.0.smithi093.stdout:35: finishing write tid 2 to smithi093153278-35 2023-08-27T20:52:03.684 INFO:tasks.rados.rados.0.smithi093.stdout:38: seq_num 37 ranges {621859=449625,1865716=756143} 2023-08-27T20:52:03.690 INFO:tasks.rados.rados.0.smithi093.stdout:38: writing smithi093153278-38 from 621859 to 1071484 tid 1 2023-08-27T20:52:03.700 INFO:tasks.rados.rados.0.smithi093.stdout:38: writing smithi093153278-38 from 1865716 to 2621859 tid 2 2023-08-27T20:52:03.700 INFO:tasks.rados.rados.0.smithi093.stdout:39: write initial oid 39 2023-08-27T20:52:03.700 INFO:tasks.rados.rados.0.smithi093.stdout:39: seq_num 38 ranges {638666=524874,1924198=637005} 2023-08-27T20:52:03.707 INFO:tasks.rados.rados.0.smithi093.stdout:39: writing smithi093153278-39 from 638666 to 1163540 tid 1 2023-08-27T20:52:03.716 INFO:tasks.rados.rados.0.smithi093.stdout:39: writing smithi093153278-39 from 1924198 to 2561203 tid 2 2023-08-27T20:52:03.716 INFO:tasks.rados.rados.0.smithi093.stdout:40: write initial oid 40 2023-08-27T20:52:03.716 INFO:tasks.rados.rados.0.smithi093.stdout:40: seq_num 39 ranges {655473=716476,2099033=556440} 2023-08-27T20:52:03.724 INFO:tasks.rados.rados.0.smithi093.stdout:40: writing smithi093153278-40 from 655473 to 1371949 tid 1 2023-08-27T20:52:03.733 INFO:tasks.rados.rados.0.smithi093.stdout:40: writing smithi093153278-40 from 2099033 to 2655473 tid 2 2023-08-27T20:52:03.733 INFO:tasks.rados.rados.0.smithi093.stdout:41: write initial oid 41 2023-08-27T20:52:03.733 INFO:tasks.rados.rados.0.smithi093.stdout:41: seq_num 40 ranges {672280=791725,2157515=514765} 2023-08-27T20:52:03.742 INFO:tasks.rados.rados.0.smithi093.stdout:41: writing smithi093153278-41 from 672280 to 1464005 tid 1 2023-08-27T20:52:03.750 INFO:tasks.rados.rados.0.smithi093.stdout:41: writing smithi093153278-41 from 2157515 to 2672280 tid 2 2023-08-27T20:52:03.750 INFO:tasks.rados.rados.0.smithi093.stdout:42: write initial oid 42 2023-08-27T20:52:03.750 INFO:tasks.rados.rados.0.smithi093.stdout:42: seq_num 41 ranges {689087=466974,1699644=784332} 2023-08-27T20:52:03.756 INFO:tasks.rados.rados.0.smithi093.stdout:42: writing smithi093153278-42 from 689087 to 1156061 tid 1 2023-08-27T20:52:03.765 INFO:tasks.rados.rados.0.smithi093.stdout:42: writing smithi093153278-42 from 1699644 to 2483976 tid 2 2023-08-27T20:52:03.766 INFO:tasks.rados.rados.0.smithi093.stdout:43: write initial oid 43 2023-08-27T20:52:03.766 INFO:tasks.rados.rados.0.smithi093.stdout:43: seq_num 42 ranges {705894=542223,1758126=644343} 2023-08-27T20:52:03.773 INFO:tasks.rados.rados.0.smithi093.stdout:43: writing smithi093153278-43 from 705894 to 1248117 tid 1 2023-08-27T20:52:03.781 INFO:tasks.rados.rados.0.smithi093.stdout:43: writing smithi093153278-43 from 1758126 to 2402469 tid 2 2023-08-27T20:52:03.781 INFO:tasks.rados.rados.0.smithi093.stdout:44: write initial oid 44 2023-08-27T20:52:03.781 INFO:tasks.rados.rados.0.smithi093.stdout:44: seq_num 43 ranges {722701=617472,1816608=788001} 2023-08-27T20:52:03.790 INFO:tasks.rados.rados.0.smithi093.stdout:44: writing smithi093153278-44 from 722701 to 1340173 tid 1 2023-08-27T20:52:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:44: writing smithi093153278-44 from 1816608 to 2604609 tid 2 2023-08-27T20:52:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:45: write initial oid 45 2023-08-27T20:52:03.799 INFO:tasks.rados.rados.0.smithi093.stdout:45: seq_num 44 ranges {739508=692721,1875090=648012} 2023-08-27T20:52:03.807 INFO:tasks.rados.rados.0.smithi093.stdout:45: writing smithi093153278-45 from 739508 to 1432229 tid 1 2023-08-27T20:52:03.815 INFO:tasks.rados.rados.0.smithi093.stdout:45: writing smithi093153278-45 from 1875090 to 2523102 tid 2 2023-08-27T20:52:03.815 INFO:tasks.rados.rados.0.smithi093.stdout:46: write initial oid 46 2023-08-27T20:52:03.815 INFO:tasks.rados.rados.0.smithi093.stdout:46: seq_num 45 ranges {756315=767970,1933572=791670} 2023-08-27T20:52:03.824 INFO:tasks.rados.rados.0.smithi093.stdout:46: writing smithi093153278-46 from 756315 to 1524285 tid 1 2023-08-27T20:52:03.833 INFO:tasks.rados.rados.0.smithi093.stdout:46: writing smithi093153278-46 from 1933572 to 2725242 tid 2 2023-08-27T20:52:03.833 INFO:tasks.rados.rados.0.smithi093.stdout:47: write initial oid 47 2023-08-27T20:52:03.833 INFO:tasks.rados.rados.0.smithi093.stdout:47: seq_num 46 ranges {773122=559572,1992054=651681} 2023-08-27T20:52:03.840 INFO:tasks.rados.rados.0.smithi093.stdout:47: writing smithi093153278-47 from 773122 to 1332694 tid 1 2023-08-27T20:52:03.850 INFO:tasks.rados.rados.0.smithi093.stdout:47: writing smithi093153278-47 from 1992054 to 2643735 tid 2 2023-08-27T20:52:03.850 INFO:tasks.rados.rados.0.smithi093.stdout:48: write initial oid 48 2023-08-27T20:52:03.850 INFO:tasks.rados.rados.0.smithi093.stdout:48: seq_num 47 ranges {789929=634821,2050536=739393} 2023-08-27T20:52:03.858 INFO:tasks.rados.rados.0.smithi093.stdout:48: writing smithi093153278-48 from 789929 to 1424750 tid 1 2023-08-27T20:52:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:48: writing smithi093153278-48 from 2050536 to 2789929 tid 2 2023-08-27T20:52:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:49: write initial oid 49 2023-08-27T20:52:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:49: seq_num 48 ranges {406736=710070,1709018=655350} 2023-08-27T20:52:03.873 INFO:tasks.rados.rados.0.smithi093.stdout:49: writing smithi093153278-49 from 406736 to 1116806 tid 1 2023-08-27T20:52:03.881 INFO:tasks.rados.rados.0.smithi093.stdout:49: writing smithi093153278-49 from 1709018 to 2364368 tid 2 2023-08-27T20:52:03.881 INFO:tasks.rados.rados.0.smithi093.stdout:50: write initial oid 50 2023-08-27T20:52:03.881 INFO:tasks.rados.rados.0.smithi093.stdout:50: seq_num 49 ranges {423543=785319,1767500=799008} 2023-08-27T20:52:03.889 INFO:tasks.rados.rados.0.smithi093.stdout:50: writing smithi093153278-50 from 423543 to 1208862 tid 1 2023-08-27T20:52:03.898 INFO:tasks.rados.rados.0.smithi093.stdout:50: writing smithi093153278-50 from 1767500 to 2566508 tid 2 2023-08-27T20:52:03.898 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.898 INFO:tasks.rados.rados.0.smithi093.stdout:35: finishing write tid 3 to smithi093153278-35 2023-08-27T20:52:03.898 INFO:tasks.rados.rados.0.smithi093.stdout:35: finishing write tid 4 to smithi093153278-35 2023-08-27T20:52:03.898 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 35 v 6 (ObjNum 34 snap 0 seq_num 34) dirty exists 2023-08-27T20:52:03.898 INFO:tasks.rados.rados.0.smithi093.stdout:35: left oid 35 (ObjNum 34 snap 0 seq_num 34) 2023-08-27T20:52:03.898 INFO:tasks.rados.rados.0.smithi093.stdout:36: finishing write tid 1 to smithi093153278-36 2023-08-27T20:52:03.899 INFO:tasks.rados.rados.0.smithi093.stdout:35: done (15 left) 2023-08-27T20:52:03.899 INFO:tasks.rados.rados.0.smithi093.stdout:51: write initial oid 51 2023-08-27T20:52:03.899 INFO:tasks.rados.rados.0.smithi093.stdout:51: seq_num 50 ranges {440350=460568,1309629=542666,2323973=516377} 2023-08-27T20:52:03.903 INFO:tasks.rados.rados.0.smithi093.stdout:51: writing smithi093153278-51 from 440350 to 900918 tid 1 2023-08-27T20:52:03.909 INFO:tasks.rados.rados.0.smithi093.stdout:51: writing smithi093153278-51 from 1309629 to 1852295 tid 2 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:51: writing smithi093153278-51 from 2323973 to 2840350 tid 3 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:36: finishing write tid 2 to smithi093153278-36 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:36: finishing write tid 3 to smithi093153278-36 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:36: finishing write tid 4 to smithi093153278-36 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 9 (ObjNum 35 snap 0 seq_num 35) dirty exists 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:36: left oid 36 (ObjNum 35 snap 0 seq_num 35) 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:37: finishing write tid 1 to smithi093153278-37 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:36: done (15 left) 2023-08-27T20:52:03.916 INFO:tasks.rados.rados.0.smithi093.stdout:52: write initial oid 52 2023-08-27T20:52:03.917 INFO:tasks.rados.rados.0.smithi093.stdout:52: seq_num 51 ranges {457157=535817,1768111=402677} 2023-08-27T20:52:03.921 INFO:tasks.rados.rados.0.smithi093.stdout:52: writing smithi093153278-52 from 457157 to 992974 tid 1 2023-08-27T20:52:03.929 INFO:tasks.rados.rados.0.smithi093.stdout:52: writing smithi093153278-52 from 1768111 to 2170788 tid 2 2023-08-27T20:52:03.929 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.929 INFO:tasks.rados.rados.0.smithi093.stdout:37: finishing write tid 2 to smithi093153278-37 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:37: finishing write tid 3 to smithi093153278-37 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:37: finishing write tid 4 to smithi093153278-37 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 37 v 3 (ObjNum 36 snap 0 seq_num 36) dirty exists 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:37: left oid 37 (ObjNum 36 snap 0 seq_num 36) 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:38: finishing write tid 1 to smithi093153278-38 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:38: finishing write tid 2 to smithi093153278-38 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:37: done (15 left) 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:53: write initial oid 53 2023-08-27T20:52:03.930 INFO:tasks.rados.rados.0.smithi093.stdout:53: seq_num 52 ranges {473964=611066,1826593=546335,2778819=95145} 2023-08-27T20:52:03.936 INFO:tasks.rados.rados.0.smithi093.stdout:53: writing smithi093153278-53 from 473964 to 1085030 tid 1 2023-08-27T20:52:03.944 INFO:tasks.rados.rados.0.smithi093.stdout:53: writing smithi093153278-53 from 1826593 to 2372928 tid 2 2023-08-27T20:52:03.947 INFO:tasks.rados.rados.0.smithi093.stdout:53: writing smithi093153278-53 from 2778819 to 2873964 tid 3 2023-08-27T20:52:03.948 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.948 INFO:tasks.rados.rados.0.smithi093.stdout:38: finishing write tid 3 to smithi093153278-38 2023-08-27T20:52:03.948 INFO:tasks.rados.rados.0.smithi093.stdout:38: finishing write tid 4 to smithi093153278-38 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 38 v 9 (ObjNum 37 snap 0 seq_num 37) dirty exists 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:38: left oid 38 (ObjNum 37 snap 0 seq_num 37) 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:39: finishing write tid 1 to smithi093153278-39 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:39: finishing write tid 2 to smithi093153278-39 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:39: finishing write tid 3 to smithi093153278-39 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:38: done (15 left) 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:54: write initial oid 54 2023-08-27T20:52:03.949 INFO:tasks.rados.rados.0.smithi093.stdout:54: seq_num 53 ranges {490771=686315,1885075=406346} 2023-08-27T20:52:03.955 INFO:tasks.rados.rados.0.smithi093.stdout:54: writing smithi093153278-54 from 490771 to 1177086 tid 1 2023-08-27T20:52:03.963 INFO:tasks.rados.rados.0.smithi093.stdout:54: writing smithi093153278-54 from 1885075 to 2291421 tid 2 2023-08-27T20:52:03.963 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.963 INFO:tasks.rados.rados.0.smithi093.stdout:39: finishing write tid 4 to smithi093153278-39 2023-08-27T20:52:03.963 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 39 v 12 (ObjNum 38 snap 0 seq_num 38) dirty exists 2023-08-27T20:52:03.963 INFO:tasks.rados.rados.0.smithi093.stdout:39: left oid 39 (ObjNum 38 snap 0 seq_num 38) 2023-08-27T20:52:03.964 INFO:tasks.rados.rados.0.smithi093.stdout:40: finishing write tid 1 to smithi093153278-40 2023-08-27T20:52:03.964 INFO:tasks.rados.rados.0.smithi093.stdout:39: done (15 left) 2023-08-27T20:52:03.964 INFO:tasks.rados.rados.0.smithi093.stdout:55: write initial oid 55 2023-08-27T20:52:03.964 INFO:tasks.rados.rados.0.smithi093.stdout:55: seq_num 54 ranges {507578=477917,1543557=550004,2833665=73913} 2023-08-27T20:52:03.968 INFO:tasks.rados.rados.0.smithi093.stdout:55: writing smithi093153278-55 from 507578 to 985495 tid 1 2023-08-27T20:52:03.974 INFO:tasks.rados.rados.0.smithi093.stdout:55: writing smithi093153278-55 from 1543557 to 2093561 tid 2 2023-08-27T20:52:03.980 INFO:tasks.rados.rados.0.smithi093.stdout:55: writing smithi093153278-55 from 2833665 to 2907578 tid 3 2023-08-27T20:52:03.980 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.980 INFO:tasks.rados.rados.0.smithi093.stdout:40: finishing write tid 2 to smithi093153278-40 2023-08-27T20:52:03.980 INFO:tasks.rados.rados.0.smithi093.stdout:40: finishing write tid 3 to smithi093153278-40 2023-08-27T20:52:03.981 INFO:tasks.rados.rados.0.smithi093.stdout:40: finishing write tid 4 to smithi093153278-40 2023-08-27T20:52:03.981 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 40 v 9 (ObjNum 39 snap 0 seq_num 39) dirty exists 2023-08-27T20:52:03.981 INFO:tasks.rados.rados.0.smithi093.stdout:40: left oid 40 (ObjNum 39 snap 0 seq_num 39) 2023-08-27T20:52:03.981 INFO:tasks.rados.rados.0.smithi093.stdout:41: finishing write tid 1 to smithi093153278-41 2023-08-27T20:52:03.981 INFO:tasks.rados.rados.0.smithi093.stdout:41: finishing write tid 2 to smithi093153278-41 2023-08-27T20:52:03.981 INFO:tasks.rados.rados.0.smithi093.stdout:40: done (15 left) 2023-08-27T20:52:03.981 INFO:tasks.rados.rados.0.smithi093.stdout:56: write initial oid 56 2023-08-27T20:52:03.982 INFO:tasks.rados.rados.0.smithi093.stdout:56: seq_num 55 ranges {524385=553166,1602039=410015,2577441=346944} 2023-08-27T20:52:03.985 INFO:tasks.rados.rados.0.smithi093.stdout:56: writing smithi093153278-56 from 524385 to 1077551 tid 1 2023-08-27T20:52:03.990 INFO:tasks.rados.rados.0.smithi093.stdout:56: writing smithi093153278-56 from 1602039 to 2012054 tid 2 2023-08-27T20:52:03.997 INFO:tasks.rados.rados.0.smithi093.stdout:56: writing smithi093153278-56 from 2577441 to 2924385 tid 3 2023-08-27T20:52:03.997 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:03.997 INFO:tasks.rados.rados.0.smithi093.stdout:41: finishing write tid 3 to smithi093153278-41 2023-08-27T20:52:03.997 INFO:tasks.rados.rados.0.smithi093.stdout:41: finishing write tid 4 to smithi093153278-41 2023-08-27T20:52:03.997 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 41 v 6 (ObjNum 40 snap 0 seq_num 40) dirty exists 2023-08-27T20:52:03.997 INFO:tasks.rados.rados.0.smithi093.stdout:41: left oid 41 (ObjNum 40 snap 0 seq_num 40) 2023-08-27T20:52:03.998 INFO:tasks.rados.rados.0.smithi093.stdout:42: finishing write tid 1 to smithi093153278-42 2023-08-27T20:52:03.998 INFO:tasks.rados.rados.0.smithi093.stdout:43: finishing write tid 1 to smithi093153278-43 2023-08-27T20:52:03.998 INFO:tasks.rados.rados.0.smithi093.stdout:42: finishing write tid 2 to smithi093153278-42 2023-08-27T20:52:03.998 INFO:tasks.rados.rados.0.smithi093.stdout:42: finishing write tid 3 to smithi093153278-42 2023-08-27T20:52:03.998 INFO:tasks.rados.rados.0.smithi093.stdout:42: finishing write tid 4 to smithi093153278-42 2023-08-27T20:52:03.998 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 42 v 9 (ObjNum 41 snap 0 seq_num 41) dirty exists 2023-08-27T20:52:03.998 INFO:tasks.rados.rados.0.smithi093.stdout:42: left oid 42 (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:52:03.999 INFO:tasks.rados.rados.0.smithi093.stdout:43: finishing write tid 2 to smithi093153278-43 2023-08-27T20:52:03.999 INFO:tasks.rados.rados.0.smithi093.stdout:41: done (15 left) 2023-08-27T20:52:03.999 INFO:tasks.rados.rados.0.smithi093.stdout:42: done (14 left) 2023-08-27T20:52:03.999 INFO:tasks.rados.rados.0.smithi093.stdout:57: write initial oid 57 2023-08-27T20:52:03.999 INFO:tasks.rados.rados.0.smithi093.stdout:57: seq_num 56 ranges {541192=628415,1660521=553673,2888511=52681} 2023-08-27T20:52:04.003 INFO:tasks.rados.rados.0.smithi093.stdout:57: writing smithi093153278-57 from 541192 to 1169607 tid 1 2023-08-27T20:52:04.010 INFO:tasks.rados.rados.0.smithi093.stdout:57: writing smithi093153278-57 from 1660521 to 2214194 tid 2 2023-08-27T20:52:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:57: writing smithi093153278-57 from 2888511 to 2941192 tid 3 2023-08-27T20:52:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:58: write initial oid 58 2023-08-27T20:52:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:58: seq_num 57 ranges {557999=703664,1719003=413684,2632287=325712} 2023-08-27T20:52:04.022 INFO:tasks.rados.rados.0.smithi093.stdout:58: writing smithi093153278-58 from 557999 to 1261663 tid 1 2023-08-27T20:52:04.027 INFO:tasks.rados.rados.0.smithi093.stdout:58: writing smithi093153278-58 from 1719003 to 2132687 tid 2 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:58: writing smithi093153278-58 from 2632287 to 2957999 tid 3 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:43: finishing write tid 3 to smithi093153278-43 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:43: finishing write tid 4 to smithi093153278-43 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 43 v 18 (ObjNum 42 snap 0 seq_num 42) dirty exists 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:43: left oid 43 (ObjNum 42 snap 0 seq_num 42) 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:44: finishing write tid 1 to smithi093153278-44 2023-08-27T20:52:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:43: done (15 left) 2023-08-27T20:52:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:59: write initial oid 59 2023-08-27T20:52:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:59: seq_num 58 ranges {574806=778913,2061132=557342} 2023-08-27T20:52:04.040 INFO:tasks.rados.rados.0.smithi093.stdout:59: writing smithi093153278-59 from 574806 to 1353719 tid 1 2023-08-27T20:52:04.049 INFO:tasks.rados.rados.0.smithi093.stdout:59: writing smithi093153278-59 from 2061132 to 2618474 tid 2 2023-08-27T20:52:04.049 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.049 INFO:tasks.rados.rados.0.smithi093.stdout:44: finishing write tid 2 to smithi093153278-44 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:44: finishing write tid 3 to smithi093153278-44 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:44: finishing write tid 4 to smithi093153278-44 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 44 v 12 (ObjNum 43 snap 0 seq_num 43) dirty exists 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:44: left oid 44 (ObjNum 43 snap 0 seq_num 43) 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:45: finishing write tid 1 to smithi093153278-45 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:45: finishing write tid 2 to smithi093153278-45 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:45: finishing write tid 3 to smithi093153278-45 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:45: finishing write tid 4 to smithi093153278-45 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 45 v 6 (ObjNum 44 snap 0 seq_num 44) dirty exists 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:45: left oid 45 (ObjNum 44 snap 0 seq_num 44) 2023-08-27T20:52:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:46: finishing write tid 1 to smithi093153278-46 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:46: finishing write tid 2 to smithi093153278-46 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:46: finishing write tid 3 to smithi093153278-46 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:46: finishing write tid 4 to smithi093153278-46 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 46 v 9 (ObjNum 45 snap 0 seq_num 45) dirty exists 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:46: left oid 46 (ObjNum 45 snap 0 seq_num 45) 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:47: finishing write tid 1 to smithi093153278-47 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:47: finishing write tid 2 to smithi093153278-47 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:47: finishing write tid 3 to smithi093153278-47 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:47: finishing write tid 4 to smithi093153278-47 2023-08-27T20:52:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 47 v 6 (ObjNum 46 snap 0 seq_num 46) dirty exists 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:47: left oid 47 (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:48: finishing write tid 1 to smithi093153278-48 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:48: finishing write tid 2 to smithi093153278-48 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:48: finishing write tid 3 to smithi093153278-48 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:48: finishing write tid 4 to smithi093153278-48 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 48 v 15 (ObjNum 47 snap 0 seq_num 47) dirty exists 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:48: left oid 48 (ObjNum 47 snap 0 seq_num 47) 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:49: finishing write tid 1 to smithi093153278-49 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:49: finishing write tid 2 to smithi093153278-49 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:49: finishing write tid 3 to smithi093153278-49 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:44: done (15 left) 2023-08-27T20:52:04.052 INFO:tasks.rados.rados.0.smithi093.stdout:45: done (14 left) 2023-08-27T20:52:04.053 INFO:tasks.rados.rados.0.smithi093.stdout:46: done (13 left) 2023-08-27T20:52:04.053 INFO:tasks.rados.rados.0.smithi093.stdout:47: done (12 left) 2023-08-27T20:52:04.053 INFO:tasks.rados.rados.0.smithi093.stdout:48: done (11 left) 2023-08-27T20:52:04.053 INFO:tasks.rados.rados.0.smithi093.stdout:60: write initial oid 60 2023-08-27T20:52:04.053 INFO:tasks.rados.rados.0.smithi093.stdout:60: seq_num 59 ranges {591613=454162,1719614=417353,2570780=420833} 2023-08-27T20:52:04.054 INFO:tasks.rados.rados.0.smithi093.stdout:60: writing smithi093153278-60 from 591613 to 1045775 tid 1 2023-08-27T20:52:04.061 INFO:tasks.rados.rados.0.smithi093.stdout:60: writing smithi093153278-60 from 1719614 to 2136967 tid 2 2023-08-27T20:52:04.067 INFO:tasks.rados.rados.0.smithi093.stdout:60: writing smithi093153278-60 from 2570780 to 2991613 tid 3 2023-08-27T20:52:04.067 INFO:tasks.rados.rados.0.smithi093.stdout:61: write initial oid 61 2023-08-27T20:52:04.067 INFO:tasks.rados.rados.0.smithi093.stdout:61: seq_num 60 ranges {608420=529411,1778096=561011,2881850=126570} 2023-08-27T20:52:04.075 INFO:tasks.rados.rados.0.smithi093.stdout:61: writing smithi093153278-61 from 608420 to 1137831 tid 1 2023-08-27T20:52:04.082 INFO:tasks.rados.rados.0.smithi093.stdout:61: writing smithi093153278-61 from 1778096 to 2339107 tid 2 2023-08-27T20:52:04.087 INFO:tasks.rados.rados.0.smithi093.stdout:61: writing smithi093153278-61 from 2881850 to 3008420 tid 3 2023-08-27T20:52:04.087 INFO:tasks.rados.rados.0.smithi093.stdout:62: write initial oid 62 2023-08-27T20:52:04.087 INFO:tasks.rados.rados.0.smithi093.stdout:62: seq_num 61 ranges {625227=721013,1952931=704669} 2023-08-27T20:52:04.095 INFO:tasks.rados.rados.0.smithi093.stdout:62: writing smithi093153278-62 from 625227 to 1346240 tid 1 2023-08-27T20:52:04.104 INFO:tasks.rados.rados.0.smithi093.stdout:62: writing smithi093153278-62 from 1952931 to 2657600 tid 2 2023-08-27T20:52:04.104 INFO:tasks.rados.rados.0.smithi093.stdout:63: write initial oid 63 2023-08-27T20:52:04.104 INFO:tasks.rados.rados.0.smithi093.stdout:63: seq_num 62 ranges {642034=796262,1895060=564680} 2023-08-27T20:52:04.113 INFO:tasks.rados.rados.0.smithi093.stdout:63: writing smithi093153278-63 from 642034 to 1438296 tid 1 2023-08-27T20:52:04.120 INFO:tasks.rados.rados.0.smithi093.stdout:63: writing smithi093153278-63 from 1895060 to 2459740 tid 2 2023-08-27T20:52:04.120 INFO:tasks.rados.rados.0.smithi093.stdout:64: write initial oid 64 2023-08-27T20:52:04.120 INFO:tasks.rados.rados.0.smithi093.stdout:64: seq_num 63 ranges {658841=471511,1553542=708338,2964119=94722} 2023-08-27T20:52:04.127 INFO:tasks.rados.rados.0.smithi093.stdout:64: writing smithi093153278-64 from 658841 to 1130352 tid 1 2023-08-27T20:52:04.135 INFO:tasks.rados.rados.0.smithi093.stdout:64: writing smithi093153278-64 from 1553542 to 2261880 tid 2 2023-08-27T20:52:04.140 INFO:tasks.rados.rados.0.smithi093.stdout:64: writing smithi093153278-64 from 2964119 to 3058841 tid 3 2023-08-27T20:52:04.140 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.140 INFO:tasks.rados.rados.0.smithi093.stdout:49: finishing write tid 4 to smithi093153278-49 2023-08-27T20:52:04.140 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 49 v 3 (ObjNum 48 snap 0 seq_num 48) dirty exists 2023-08-27T20:52:04.141 INFO:tasks.rados.rados.0.smithi093.stdout:49: left oid 49 (ObjNum 48 snap 0 seq_num 48) 2023-08-27T20:52:04.141 INFO:tasks.rados.rados.0.smithi093.stdout:50: finishing write tid 1 to smithi093153278-50 2023-08-27T20:52:04.141 INFO:tasks.rados.rados.0.smithi093.stdout:49: done (15 left) 2023-08-27T20:52:04.141 INFO:tasks.rados.rados.0.smithi093.stdout:65: write initial oid 65 2023-08-27T20:52:04.141 INFO:tasks.rados.rados.0.smithi093.stdout:65: seq_num 64 ranges {675648=546760,2012024=568349} 2023-08-27T20:52:04.147 INFO:tasks.rados.rados.0.smithi093.stdout:65: writing smithi093153278-65 from 675648 to 1222408 tid 1 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:65: writing smithi093153278-65 from 2012024 to 2580373 tid 2 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:50: finishing write tid 2 to smithi093153278-50 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:50: finishing write tid 3 to smithi093153278-50 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:50: finishing write tid 4 to smithi093153278-50 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 50 v 12 (ObjNum 49 snap 0 seq_num 49) dirty exists 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:50: left oid 50 (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:51: finishing write tid 1 to smithi093153278-51 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:50: done (15 left) 2023-08-27T20:52:04.157 INFO:tasks.rados.rados.0.smithi093.stdout:66: write initial oid 66 2023-08-27T20:52:04.158 INFO:tasks.rados.rados.0.smithi093.stdout:66: seq_num 65 ranges {692455=622009,2070506=712007} 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:NAME HOST PORTS STATUS REFRESHED AGE MEM USE MEM LIM VERSION IMAGE ID CONTAINER ID 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:alertmanager.a smithi093 *:9093,9094 running (45s) 14s ago 72s 16.5M - 0.25.0 c8568f914cd2 09806d718945 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:grafana.a smithi136 *:3000 running (60s) 20s ago 60s 87.1M - 9.4.7 2c41d148cca3 cf2ec1682289 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:mds.foo.smithi093.szcdyz smithi093 running (37s) 14s ago 37s 14.9M - 17.2.6-1016-g4fe39952 d5e2f73f133d ef030cd27492 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:mds.foo.smithi136.hhkxbr smithi136 running (35s) 20s ago 33s 14.6M - 17.2.6-1016-g4fe39952 d5e2f73f133d 37f9c9781423 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:mgr.x smithi136 *:8443 running (3m) 20s ago 3m 438M - 17.2.6-1016-g4fe39952 d5e2f73f133d 03135b918d80 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:mgr.y smithi093 *:9283 running (5m) 14s ago 5m 492M - 17.2.6-1016-g4fe39952 d5e2f73f133d d29e0efb4bb5 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:mon.a smithi093 running (5m) 14s ago 5m 82.6M 2048M 17.2.6-1016-g4fe39952 d5e2f73f133d d1926c169165 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:mon.b smithi136 running (4m) 20s ago 4m 67.8M 2048M 17.2.6-1016-g4fe39952 d5e2f73f133d 833fe184137a 2023-08-27T20:52:04.165 INFO:teuthology.orchestra.run.smithi093.stdout:mon.c smithi093 running (4m) 14s ago 4m 67.1M 2048M 17.2.6-1016-g4fe39952 d5e2f73f133d 9ec6c73ae721 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:node-exporter.a smithi093 *:9100 running (55s) 14s ago 55s 9210k - 1.5.0 0da6a335fe13 977eb54f6fca 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:node-exporter.b smithi136 *:9100 running (52s) 20s ago 52s 9244k - 1.5.0 0da6a335fe13 ae167925824d 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.0 smithi093 running (3m) 14s ago 3m 55.0M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d 814af38c0887 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.1 smithi093 running (3m) 14s ago 3m 59.5M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d f3a41f39a30f 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.2 smithi093 running (3m) 14s ago 3m 49.8M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d 3fb7f35fefcf 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.3 smithi093 running (2m) 14s ago 2m 48.7M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d 4593ade24d22 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.4 smithi136 running (2m) 20s ago 2m 50.5M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d 59c23f051519 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.5 smithi136 running (2m) 20s ago 2m 47.8M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d b85e75d531b4 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.6 smithi136 running (2m) 20s ago 2m 50.3M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d 7ac4ed3ed339 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:osd.7 smithi136 running (111s) 20s ago 111s 48.3M 4022M 17.2.6-1016-g4fe39952 d5e2f73f133d a05bd12665f2 2023-08-27T20:52:04.166 INFO:teuthology.orchestra.run.smithi093.stdout:prometheus.a smithi136 *:9095 running (42s) 20s ago 91s 34.0M - 2.43.0 a07b618ecd1d cf83aa0a04c6 2023-08-27T20:52:04.167 INFO:tasks.rados.rados.0.smithi093.stdout:66: writing smithi093153278-66 from 692455 to 1314464 tid 1 2023-08-27T20:52:04.175 INFO:tasks.rados.rados.0.smithi093.stdout:66: writing smithi093153278-66 from 2070506 to 2782513 tid 2 2023-08-27T20:52:04.175 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.175 INFO:tasks.rados.rados.0.smithi093.stdout:51: finishing write tid 2 to smithi093153278-51 2023-08-27T20:52:04.175 INFO:tasks.rados.rados.0.smithi093.stdout:51: finishing write tid 3 to smithi093153278-51 2023-08-27T20:52:04.175 INFO:tasks.rados.rados.0.smithi093.stdout:51: finishing write tid 4 to smithi093153278-51 2023-08-27T20:52:04.175 INFO:tasks.rados.rados.0.smithi093.stdout:51: finishing write tid 5 to smithi093153278-51 2023-08-27T20:52:04.175 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 51 v 10 (ObjNum 50 snap 0 seq_num 50) dirty exists 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:51: left oid 51 (ObjNum 50 snap 0 seq_num 50) 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:52: finishing write tid 1 to smithi093153278-52 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:52: finishing write tid 2 to smithi093153278-52 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:51: done (15 left) 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:67: write initial oid 67 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:52: finishing write tid 3 to smithi093153278-52 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:52: finishing write tid 4 to smithi093153278-52 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 52 v 12 (ObjNum 51 snap 0 seq_num 51) dirty exists 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:52: left oid 52 (ObjNum 51 snap 0 seq_num 51) 2023-08-27T20:52:04.176 INFO:tasks.rados.rados.0.smithi093.stdout:53: finishing write tid 1 to smithi093153278-53 2023-08-27T20:52:04.177 INFO:tasks.rados.rados.0.smithi093.stdout:53: finishing write tid 2 to smithi093153278-53 2023-08-27T20:52:04.177 INFO:tasks.rados.rados.0.smithi093.stdout:53: finishing write tid 3 to smithi093153278-53 2023-08-27T20:52:04.177 INFO:tasks.rados.rados.0.smithi093.stdout:53: finishing write tid 4 to smithi093153278-53 2023-08-27T20:52:04.177 INFO:tasks.rados.rados.0.smithi093.stdout:53: finishing write tid 5 to smithi093153278-53 2023-08-27T20:52:04.177 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 53 v 13 (ObjNum 52 snap 0 seq_num 52) dirty exists 2023-08-27T20:52:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:53: left oid 53 (ObjNum 52 snap 0 seq_num 52) 2023-08-27T20:52:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:54: finishing write tid 1 to smithi093153278-54 2023-08-27T20:52:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:54: finishing write tid 2 to smithi093153278-54 2023-08-27T20:52:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:54: finishing write tid 3 to smithi093153278-54 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:54: finishing write tid 4 to smithi093153278-54 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 54 v 15 (ObjNum 53 snap 0 seq_num 53) dirty exists 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:54: left oid 54 (ObjNum 53 snap 0 seq_num 53) 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:55: finishing write tid 1 to smithi093153278-55 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:55: finishing write tid 2 to smithi093153278-55 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:55: finishing write tid 3 to smithi093153278-55 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:55: finishing write tid 4 to smithi093153278-55 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:55: finishing write tid 5 to smithi093153278-55 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 55 v 13 (ObjNum 54 snap 0 seq_num 54) dirty exists 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:55: left oid 55 (ObjNum 54 snap 0 seq_num 54) 2023-08-27T20:52:04.179 INFO:tasks.rados.rados.0.smithi093.stdout:56: finishing write tid 1 to smithi093153278-56 2023-08-27T20:52:04.180 INFO:tasks.rados.rados.0.smithi093.stdout:56: finishing write tid 2 to smithi093153278-56 2023-08-27T20:52:04.180 INFO:tasks.rados.rados.0.smithi093.stdout:56: finishing write tid 3 to smithi093153278-56 2023-08-27T20:52:04.180 INFO:tasks.rados.rados.0.smithi093.stdout:56: finishing write tid 4 to smithi093153278-56 2023-08-27T20:52:04.180 INFO:tasks.rados.rados.0.smithi093.stdout:67: seq_num 66 ranges {709262=697258,2012635=572018,3046388=62874} 2023-08-27T20:52:04.182 INFO:tasks.rados.rados.0.smithi093.stdout:67: writing smithi093153278-67 from 709262 to 1406520 tid 1 2023-08-27T20:52:04.190 INFO:tasks.rados.rados.0.smithi093.stdout:67: writing smithi093153278-67 from 2012635 to 2584653 tid 2 2023-08-27T20:52:04.193 INFO:tasks.rados.rados.0.smithi093.stdout:67: writing smithi093153278-67 from 3046388 to 3109262 tid 3 2023-08-27T20:52:04.193 INFO:tasks.rados.rados.0.smithi093.stdout:52: done (15 left) 2023-08-27T20:52:04.193 INFO:tasks.rados.rados.0.smithi093.stdout:53: done (14 left) 2023-08-27T20:52:04.193 INFO:tasks.rados.rados.0.smithi093.stdout:54: done (13 left) 2023-08-27T20:52:04.193 INFO:tasks.rados.rados.0.smithi093.stdout:55: done (12 left) 2023-08-27T20:52:04.193 INFO:tasks.rados.rados.0.smithi093.stdout:68: write initial oid 68 2023-08-27T20:52:04.194 INFO:tasks.rados.rados.0.smithi093.stdout:68: seq_num 67 ranges {726069=772507,2071117=715676} 2023-08-27T20:52:04.202 INFO:tasks.rados.rados.0.smithi093.stdout:68: writing smithi093153278-68 from 726069 to 1498576 tid 1 2023-08-27T20:52:04.211 INFO:tasks.rados.rados.0.smithi093.stdout:68: writing smithi093153278-68 from 2071117 to 2786793 tid 2 2023-08-27T20:52:04.211 INFO:tasks.rados.rados.0.smithi093.stdout:69: write initial oid 69 2023-08-27T20:52:04.211 INFO:tasks.rados.rados.0.smithi093.stdout:69: seq_num 68 ranges {742876=447756,1729599=575687,3101234=41642} 2023-08-27T20:52:04.218 INFO:tasks.rados.rados.0.smithi093.stdout:69: writing smithi093153278-69 from 742876 to 1190632 tid 1 2023-08-27T20:52:04.225 INFO:tasks.rados.rados.0.smithi093.stdout:69: writing smithi093153278-69 from 1729599 to 2305286 tid 2 2023-08-27T20:52:04.230 INFO:tasks.rados.rados.0.smithi093.stdout:69: writing smithi093153278-69 from 3101234 to 3142876 tid 3 2023-08-27T20:52:04.230 INFO:tasks.rados.rados.0.smithi093.stdout:70: write initial oid 70 2023-08-27T20:52:04.230 INFO:tasks.rados.rados.0.smithi093.stdout:70: seq_num 69 ranges {759683=639358,1904434=719345,3128657=31026} 2023-08-27T20:52:04.238 INFO:tasks.rados.rados.0.smithi093.stdout:70: writing smithi093153278-70 from 759683 to 1399041 tid 1 2023-08-27T20:52:04.247 INFO:tasks.rados.rados.0.smithi093.stdout:70: writing smithi093153278-70 from 1904434 to 2623779 tid 2 2023-08-27T20:52:04.250 INFO:tasks.rados.rados.0.smithi093.stdout:70: writing smithi093153278-70 from 3128657 to 3159683 tid 3 2023-08-27T20:52:04.250 INFO:tasks.rados.rados.0.smithi093.stdout:71: write initial oid 71 2023-08-27T20:52:04.250 INFO:tasks.rados.rados.0.smithi093.stdout:56: finishing write tid 5 to smithi093153278-56 2023-08-27T20:52:04.250 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 56 v 19 (ObjNum 55 snap 0 seq_num 55) dirty exists 2023-08-27T20:52:04.250 INFO:tasks.rados.rados.0.smithi093.stdout:56: left oid 56 (ObjNum 55 snap 0 seq_num 55) 2023-08-27T20:52:04.251 INFO:tasks.rados.rados.0.smithi093.stdout:57: finishing write tid 1 to smithi093153278-57 2023-08-27T20:52:04.251 INFO:tasks.rados.rados.0.smithi093.stdout:71: seq_num 70 ranges {776490=714607,2246563=579356} 2023-08-27T20:52:04.259 INFO:tasks.rados.rados.0.smithi093.stdout:71: writing smithi093153278-71 from 776490 to 1491097 tid 1 2023-08-27T20:52:04.268 INFO:tasks.rados.rados.0.smithi093.stdout:71: writing smithi093153278-71 from 2246563 to 2825919 tid 2 2023-08-27T20:52:04.268 INFO:tasks.rados.rados.0.smithi093.stdout:56: done (15 left) 2023-08-27T20:52:04.268 INFO:tasks.rados.rados.0.smithi093.stdout:72: write initial oid 72 2023-08-27T20:52:04.268 INFO:tasks.rados.rados.0.smithi093.stdout:72: seq_num 71 ranges {793297=789856,2305045=723014} 2023-08-27T20:52:04.277 INFO:tasks.rados.rados.0.smithi093.stdout:72: writing smithi093153278-72 from 793297 to 1583153 tid 1 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:72: writing smithi093153278-72 from 2305045 to 3028059 tid 2 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:57: finishing write tid 2 to smithi093153278-57 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:57: finishing write tid 3 to smithi093153278-57 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:57: finishing write tid 4 to smithi093153278-57 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:57: finishing write tid 5 to smithi093153278-57 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 57 v 4 (ObjNum 56 snap 0 seq_num 56) dirty exists 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:57: left oid 57 (ObjNum 56 snap 0 seq_num 56) 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:58: finishing write tid 1 to smithi093153278-58 2023-08-27T20:52:04.287 INFO:tasks.rados.rados.0.smithi093.stdout:57: done (15 left) 2023-08-27T20:52:04.288 INFO:tasks.rados.rados.0.smithi093.stdout:73: write initial oid 73 2023-08-27T20:52:04.288 INFO:tasks.rados.rados.0.smithi093.stdout:73: seq_num 72 ranges {410104=465105,1563527=583025,2810926=399178} 2023-08-27T20:52:04.292 INFO:tasks.rados.rados.0.smithi093.stdout:73: writing smithi093153278-73 from 410104 to 875209 tid 1 2023-08-27T20:52:04.300 INFO:tasks.rados.rados.0.smithi093.stdout:73: writing smithi093153278-73 from 1563527 to 2146552 tid 2 2023-08-27T20:52:04.308 INFO:tasks.rados.rados.0.smithi093.stdout:73: writing smithi093153278-73 from 2810926 to 3210104 tid 3 2023-08-27T20:52:04.308 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.308 INFO:tasks.rados.rados.0.smithi093.stdout:58: finishing write tid 2 to smithi093153278-58 2023-08-27T20:52:04.308 INFO:tasks.rados.rados.0.smithi093.stdout:58: finishing write tid 3 to smithi093153278-58 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:58: finishing write tid 4 to smithi093153278-58 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:58: finishing write tid 5 to smithi093153278-58 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 58 v 16 (ObjNum 57 snap 0 seq_num 57) dirty exists 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:58: left oid 58 (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:59: finishing write tid 1 to smithi093153278-59 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:59: finishing write tid 2 to smithi093153278-59 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:59: finishing write tid 3 to smithi093153278-59 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:59: finishing write tid 4 to smithi093153278-59 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 59 v 13 (ObjNum 58 snap 0 seq_num 58) dirty exists 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:59: left oid 59 (ObjNum 58 snap 0 seq_num 58) 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:60: finishing write tid 1 to smithi093153278-60 2023-08-27T20:52:04.309 INFO:tasks.rados.rados.0.smithi093.stdout:60: finishing write tid 2 to smithi093153278-60 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:60: finishing write tid 3 to smithi093153278-60 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:60: finishing write tid 4 to smithi093153278-60 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:60: finishing write tid 5 to smithi093153278-60 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 60 v 17 (ObjNum 59 snap 0 seq_num 59) dirty exists 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:60: left oid 60 (ObjNum 59 snap 0 seq_num 59) 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:61: finishing write tid 1 to smithi093153278-61 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:61: finishing write tid 2 to smithi093153278-61 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:61: finishing write tid 3 to smithi093153278-61 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:61: finishing write tid 4 to smithi093153278-61 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:61: finishing write tid 5 to smithi093153278-61 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 61 v 10 (ObjNum 60 snap 0 seq_num 60) dirty exists 2023-08-27T20:52:04.310 INFO:tasks.rados.rados.0.smithi093.stdout:61: left oid 61 (ObjNum 60 snap 0 seq_num 60) 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:62: finishing write tid 1 to smithi093153278-62 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:62: finishing write tid 2 to smithi093153278-62 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:58: done (15 left) 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:59: done (14 left) 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:60: done (13 left) 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:61: done (12 left) 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:74: write initial oid 74 2023-08-27T20:52:04.311 INFO:tasks.rados.rados.0.smithi093.stdout:74: seq_num 73 ranges {426911=540354,1622009=726683,3121996=104915} 2023-08-27T20:52:04.313 INFO:tasks.rados.rados.0.smithi093.stdout:74: writing smithi093153278-74 from 426911 to 967265 tid 1 2023-08-27T20:52:04.325 INFO:tasks.rados.rados.0.smithi093.stdout:74: writing smithi093153278-74 from 1622009 to 2348692 tid 2 2023-08-27T20:52:04.326 INFO:tasks.rados.rados.0.smithi093.stdout:74: writing smithi093153278-74 from 3121996 to 3226911 tid 3 2023-08-27T20:52:04.326 INFO:tasks.rados.rados.0.smithi093.stdout:75: write initial oid 75 2023-08-27T20:52:04.327 INFO:tasks.rados.rados.0.smithi093.stdout:75: seq_num 74 ranges {443718=615603,1564138=470341,2633066=610652} 2023-08-27T20:52:04.332 INFO:tasks.rados.rados.0.smithi093.stdout:75: writing smithi093153278-75 from 443718 to 1059321 tid 1 2023-08-27T20:52:04.338 INFO:tasks.rados.rados.0.smithi093.stdout:75: writing smithi093153278-75 from 1564138 to 2034479 tid 2 2023-08-27T20:52:04.345 INFO:tasks.rados.rados.0.smithi093.stdout:75: writing smithi093153278-75 from 2633066 to 3243718 tid 3 2023-08-27T20:52:04.345 INFO:tasks.rados.rados.0.smithi093.stdout:76: write initial oid 76 2023-08-27T20:52:04.346 INFO:tasks.rados.rados.0.smithi093.stdout:76: seq_num 75 ranges {460525=690852,1622620=730352,3060489=200036} 2023-08-27T20:52:04.352 INFO:tasks.rados.rados.0.smithi093.stdout:76: writing smithi093153278-76 from 460525 to 1151377 tid 1 2023-08-27T20:52:04.359 INFO:tasks.rados.rados.0.smithi093.stdout:76: writing smithi093153278-76 from 1622620 to 2352972 tid 2 2023-08-27T20:52:04.365 INFO:tasks.rados.rados.0.smithi093.stdout:76: writing smithi093153278-76 from 3060489 to 3260525 tid 3 2023-08-27T20:52:04.365 INFO:tasks.rados.rados.0.smithi093.stdout:77: write initial oid 77 2023-08-27T20:52:04.365 INFO:tasks.rados.rados.0.smithi093.stdout:77: seq_num 76 ranges {477332=766101,1681102=474010,2687912=589420} 2023-08-27T20:52:04.373 INFO:tasks.rados.rados.0.smithi093.stdout:77: writing smithi093153278-77 from 477332 to 1243433 tid 1 2023-08-27T20:52:04.378 INFO:tasks.rados.rados.0.smithi093.stdout:77: writing smithi093153278-77 from 1681102 to 2155112 tid 2 2023-08-27T20:52:04.386 INFO:tasks.rados.rados.0.smithi093.stdout:77: writing smithi093153278-77 from 2687912 to 3277332 tid 3 2023-08-27T20:52:04.386 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.386 INFO:tasks.rados.rados.0.smithi093.stdout:62: finishing write tid 3 to smithi093153278-62 2023-08-27T20:52:04.386 INFO:tasks.rados.rados.0.smithi093.stdout:62: finishing write tid 4 to smithi093153278-62 2023-08-27T20:52:04.386 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 62 v 15 (ObjNum 61 snap 0 seq_num 61) dirty exists 2023-08-27T20:52:04.386 INFO:tasks.rados.rados.0.smithi093.stdout:62: left oid 62 (ObjNum 61 snap 0 seq_num 61) 2023-08-27T20:52:04.387 INFO:tasks.rados.rados.0.smithi093.stdout:63: finishing write tid 1 to smithi093153278-63 2023-08-27T20:52:04.387 INFO:tasks.rados.rados.0.smithi093.stdout:63: finishing write tid 2 to smithi093153278-63 2023-08-27T20:52:04.387 INFO:tasks.rados.rados.0.smithi093.stdout:62: done (15 left) 2023-08-27T20:52:04.387 INFO:tasks.rados.rados.0.smithi093.stdout:78: write initial oid 78 2023-08-27T20:52:04.387 INFO:tasks.rados.rados.0.smithi093.stdout:78: seq_num 77 ranges {494139=557703,1455937=734021,2948041=346098} 2023-08-27T20:52:04.392 INFO:tasks.rados.rados.0.smithi093.stdout:78: writing smithi093153278-78 from 494139 to 1051842 tid 1 2023-08-27T20:52:04.400 INFO:tasks.rados.rados.0.smithi093.stdout:78: writing smithi093153278-78 from 1455937 to 2189958 tid 2 2023-08-27T20:52:04.407 INFO:tasks.rados.rados.0.smithi093.stdout:78: writing smithi093153278-78 from 2948041 to 3294139 tid 3 2023-08-27T20:52:04.407 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.407 INFO:tasks.rados.rados.0.smithi093.stdout:63: finishing write tid 3 to smithi093153278-63 2023-08-27T20:52:04.407 INFO:tasks.rados.rados.0.smithi093.stdout:63: finishing write tid 4 to smithi093153278-63 2023-08-27T20:52:04.407 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 63 v 20 (ObjNum 62 snap 0 seq_num 62) dirty exists 2023-08-27T20:52:04.408 INFO:tasks.rados.rados.0.smithi093.stdout:63: left oid 63 (ObjNum 62 snap 0 seq_num 62) 2023-08-27T20:52:04.408 INFO:tasks.rados.rados.0.smithi093.stdout:64: finishing write tid 1 to smithi093153278-64 2023-08-27T20:52:04.408 INFO:tasks.rados.rados.0.smithi093.stdout:64: finishing write tid 2 to smithi093153278-64 2023-08-27T20:52:04.408 INFO:tasks.rados.rados.0.smithi093.stdout:63: done (15 left) 2023-08-27T20:52:04.408 INFO:tasks.rados.rados.0.smithi093.stdout:79: write initial oid 79 2023-08-27T20:52:04.408 INFO:tasks.rados.rados.0.smithi093.stdout:79: seq_num 78 ranges {510946=632952,1798066=477679,2742758=457217} 2023-08-27T20:52:04.414 INFO:tasks.rados.rados.0.smithi093.stdout:79: writing smithi093153278-79 from 510946 to 1143898 tid 1 2023-08-27T20:52:04.421 INFO:tasks.rados.rados.0.smithi093.stdout:79: writing smithi093153278-79 from 1798066 to 2275745 tid 2 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:79: writing smithi093153278-79 from 2742758 to 3199975 tid 3 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:64: finishing write tid 3 to smithi093153278-64 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:64: finishing write tid 4 to smithi093153278-64 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:64: finishing write tid 5 to smithi093153278-64 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 64 v 7 (ObjNum 63 snap 0 seq_num 63) dirty exists 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:64: left oid 64 (ObjNum 63 snap 0 seq_num 63) 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:65: finishing write tid 1 to smithi093153278-65 2023-08-27T20:52:04.428 INFO:tasks.rados.rados.0.smithi093.stdout:65: finishing write tid 2 to smithi093153278-65 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:65: finishing write tid 3 to smithi093153278-65 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:65: finishing write tid 4 to smithi093153278-65 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 65 v 16 (ObjNum 64 snap 0 seq_num 64) dirty exists 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:65: left oid 65 (ObjNum 64 snap 0 seq_num 64) 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:66: finishing write tid 1 to smithi093153278-66 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:64: done (15 left) 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:65: done (14 left) 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:80: write initial oid 80 2023-08-27T20:52:04.429 INFO:tasks.rados.rados.0.smithi093.stdout:80: seq_num 79 ranges {527753=708201,1856548=737690,3286534=41219} 2023-08-27T20:52:04.435 INFO:tasks.rados.rados.0.smithi093.stdout:80: writing smithi093153278-80 from 527753 to 1235954 tid 1 2023-08-27T20:52:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:04 smithi136 ceph-mon[113531]: pgmap v55: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:04 smithi136 ceph-mon[113531]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2023-08-27T20:52:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:04 smithi136 ceph-mon[113531]: osdmap e70: 8 total, 8 up, 8 in 2023-08-27T20:52:04.443 INFO:tasks.rados.rados.0.smithi093.stdout:80: writing smithi093153278-80 from 1856548 to 2594238 tid 2 2023-08-27T20:52:04.448 INFO:tasks.rados.rados.0.smithi093.stdout:80: writing smithi093153278-80 from 3286534 to 3327753 tid 3 2023-08-27T20:52:04.448 INFO:tasks.rados.rados.0.smithi093.stdout:81: write initial oid 81 2023-08-27T20:52:04.448 INFO:tasks.rados.rados.0.smithi093.stdout:81: seq_num 80 ranges {544560=783450,1915030=481348,2797604=479761} 2023-08-27T20:52:04.456 INFO:tasks.rados.rados.0.smithi093.stdout:81: writing smithi093153278-81 from 544560 to 1328010 tid 1 2023-08-27T20:52:04.463 INFO:tasks.rados.rados.0.smithi093.stdout:81: writing smithi093153278-81 from 1915030 to 2396378 tid 2 2023-08-27T20:52:04.469 INFO:tasks.rados.rados.0.smithi093.stdout:81: writing smithi093153278-81 from 2797604 to 3277365 tid 3 2023-08-27T20:52:04.469 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.469 INFO:tasks.rados.rados.0.smithi093.stdout:66: finishing write tid 2 to smithi093153278-66 2023-08-27T20:52:04.469 INFO:tasks.rados.rados.0.smithi093.stdout:66: finishing write tid 3 to smithi093153278-66 2023-08-27T20:52:04.469 INFO:tasks.rados.rados.0.smithi093.stdout:66: finishing write tid 4 to smithi093153278-66 2023-08-27T20:52:04.469 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 66 v 16 (ObjNum 65 snap 0 seq_num 65) dirty exists 2023-08-27T20:52:04.469 INFO:tasks.rados.rados.0.smithi093.stdout:66: left oid 66 (ObjNum 65 snap 0 seq_num 65) 2023-08-27T20:52:04.470 INFO:tasks.rados.rados.0.smithi093.stdout:67: finishing write tid 1 to smithi093153278-67 2023-08-27T20:52:04.470 INFO:tasks.rados.rados.0.smithi093.stdout:66: done (15 left) 2023-08-27T20:52:04.470 INFO:tasks.rados.rados.0.smithi093.stdout:82: write initial oid 82 2023-08-27T20:52:04.470 INFO:tasks.rados.rados.0.smithi093.stdout:82: seq_num 81 ranges {561367=458699,1573512=741359,2941380=419987} 2023-08-27T20:52:04.475 INFO:tasks.rados.rados.0.smithi093.stdout:82: writing smithi093153278-82 from 561367 to 1020066 tid 1 2023-08-27T20:52:04.483 INFO:tasks.rados.rados.0.smithi093.stdout:82: writing smithi093153278-82 from 1573512 to 2314871 tid 2 2023-08-27T20:52:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:04 smithi093 ceph-mon[111335]: pgmap v55: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:04 smithi093 ceph-mon[111335]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2023-08-27T20:52:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:04 smithi093 ceph-mon[111335]: osdmap e70: 8 total, 8 up, 8 in 2023-08-27T20:52:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:04 smithi093 ceph-mon[118902]: pgmap v55: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:04 smithi093 ceph-mon[118902]: from='client.? ' entity='client.0' cmd='[{"prefix": "osd pool set", "pool": "unique_pool_0", "var": "write_fadvise_dontneed", "val": "false"}]': finished 2023-08-27T20:52:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:04 smithi093 ceph-mon[118902]: osdmap e70: 8 total, 8 up, 8 in 2023-08-27T20:52:04.490 INFO:tasks.rados.rados.0.smithi093.stdout:82: writing smithi093153278-82 from 2941380 to 3361367 tid 3 2023-08-27T20:52:04.490 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:67: finishing write tid 2 to smithi093153278-67 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:67: finishing write tid 3 to smithi093153278-67 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:67: finishing write tid 4 to smithi093153278-67 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:67: finishing write tid 5 to smithi093153278-67 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 67 v 23 (ObjNum 66 snap 0 seq_num 66) dirty exists 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:67: left oid 67 (ObjNum 66 snap 0 seq_num 66) 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:68: finishing write tid 1 to smithi093153278-68 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:68: finishing write tid 2 to smithi093153278-68 2023-08-27T20:52:04.491 INFO:tasks.rados.rados.0.smithi093.stdout:68: finishing write tid 3 to smithi093153278-68 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:68: finishing write tid 4 to smithi093153278-68 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 68 v 6 (ObjNum 67 snap 0 seq_num 67) dirty exists 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:68: left oid 68 (ObjNum 67 snap 0 seq_num 67) 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:69: finishing write tid 1 to smithi093153278-69 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:67: done (15 left) 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:68: done (14 left) 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:83: write initial oid 83 2023-08-27T20:52:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:83: seq_num 82 ranges {578174=533948,1515641=485017,2736097=502305} 2023-08-27T20:52:04.497 INFO:tasks.rados.rados.0.smithi093.stdout:83: writing smithi093153278-83 from 578174 to 1112122 tid 1 2023-08-27T20:52:04.503 INFO:tasks.rados.rados.0.smithi093.stdout:83: writing smithi093153278-83 from 1515641 to 2000658 tid 2 2023-08-27T20:52:04.511 INFO:tasks.rados.rados.0.smithi093.stdout:83: writing smithi093153278-83 from 2736097 to 3238402 tid 3 2023-08-27T20:52:04.511 INFO:tasks.rados.rados.0.smithi093.stdout:84: write initial oid 84 2023-08-27T20:52:04.511 INFO:tasks.rados.rados.0.smithi093.stdout:84: seq_num 83 ranges {594981=609197,1974123=745028,3279873=115108} 2023-08-27T20:52:04.518 INFO:tasks.rados.rados.0.smithi093.stdout:84: writing smithi093153278-84 from 594981 to 1204178 tid 1 2023-08-27T20:52:04.528 INFO:tasks.rados.rados.0.smithi093.stdout:84: writing smithi093153278-84 from 1974123 to 2719151 tid 2 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:84: writing smithi093153278-84 from 3279873 to 3394981 tid 3 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:69: finishing write tid 2 to smithi093153278-69 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:69: finishing write tid 3 to smithi093153278-69 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:69: finishing write tid 4 to smithi093153278-69 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:69: finishing write tid 5 to smithi093153278-69 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 69 v 16 (ObjNum 68 snap 0 seq_num 68) dirty exists 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:69: left oid 69 (ObjNum 68 snap 0 seq_num 68) 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:70: finishing write tid 1 to smithi093153278-70 2023-08-27T20:52:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:70: finishing write tid 2 to smithi093153278-70 2023-08-27T20:52:04.534 INFO:tasks.rados.rados.0.smithi093.stdout:69: done (15 left) 2023-08-27T20:52:04.534 INFO:tasks.rados.rados.0.smithi093.stdout:85: write initial oid 85 2023-08-27T20:52:04.534 INFO:tasks.rados.rados.0.smithi093.stdout:85: seq_num 84 ranges {611788=400799,1748958=488686,2907296=504492} 2023-08-27T20:52:04.539 INFO:tasks.rados.rados.0.smithi093.stdout:85: writing smithi093153278-85 from 611788 to 1012587 tid 1 2023-08-27T20:52:04.546 INFO:tasks.rados.rados.0.smithi093.stdout:85: writing smithi093153278-85 from 1748958 to 2237644 tid 2 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:85: writing smithi093153278-85 from 2907296 to 3411788 tid 3 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:70: finishing write tid 3 to smithi093153278-70 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:70: finishing write tid 4 to smithi093153278-70 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:70: finishing write tid 5 to smithi093153278-70 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 70 v 20 (ObjNum 69 snap 0 seq_num 69) dirty exists 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:70: left oid 70 (ObjNum 69 snap 0 seq_num 69) 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:71: finishing write tid 1 to smithi093153278-71 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:71: finishing write tid 2 to smithi093153278-71 2023-08-27T20:52:04.556 INFO:tasks.rados.rados.0.smithi093.stdout:71: finishing write tid 3 to smithi093153278-71 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:71: finishing write tid 4 to smithi093153278-71 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 71 v 15 (ObjNum 70 snap 0 seq_num 70) dirty exists 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:71: left oid 71 (ObjNum 70 snap 0 seq_num 70) 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:72: finishing write tid 1 to smithi093153278-72 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:72: finishing write tid 2 to smithi093153278-72 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:72: finishing write tid 3 to smithi093153278-72 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:72: finishing write tid 4 to smithi093153278-72 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 72 v 13 (ObjNum 71 snap 0 seq_num 71) dirty exists 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:72: left oid 72 (ObjNum 71 snap 0 seq_num 71) 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:73: finishing write tid 1 to smithi093153278-73 2023-08-27T20:52:04.557 INFO:tasks.rados.rados.0.smithi093.stdout:73: finishing write tid 2 to smithi093153278-73 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:73: finishing write tid 3 to smithi093153278-73 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:73: finishing write tid 4 to smithi093153278-73 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:73: finishing write tid 5 to smithi093153278-73 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 73 v 17 (ObjNum 72 snap 0 seq_num 72) dirty exists 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:73: left oid 73 (ObjNum 72 snap 0 seq_num 72) 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:74: finishing write tid 1 to smithi093153278-74 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:74: finishing write tid 2 to smithi093153278-74 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:74: finishing write tid 3 to smithi093153278-74 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:74: finishing write tid 4 to smithi093153278-74 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:74: finishing write tid 5 to smithi093153278-74 2023-08-27T20:52:04.558 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 74 v 13 (ObjNum 73 snap 0 seq_num 73) dirty exists 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:74: left oid 74 (ObjNum 73 snap 0 seq_num 73) 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:75: finishing write tid 1 to smithi093153278-75 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:75: finishing write tid 2 to smithi093153278-75 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:75: finishing write tid 3 to smithi093153278-75 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:75: finishing write tid 4 to smithi093153278-75 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:75: finishing write tid 5 to smithi093153278-75 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 75 v 17 (ObjNum 74 snap 0 seq_num 74) dirty exists 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:75: left oid 75 (ObjNum 74 snap 0 seq_num 74) 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:76: finishing write tid 1 to smithi093153278-76 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:76: finishing write tid 2 to smithi093153278-76 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:76: finishing write tid 3 to smithi093153278-76 2023-08-27T20:52:04.559 INFO:tasks.rados.rados.0.smithi093.stdout:76: finishing write tid 4 to smithi093153278-76 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:76: finishing write tid 5 to smithi093153278-76 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 76 v 21 (ObjNum 75 snap 0 seq_num 75) dirty exists 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:76: left oid 76 (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:77: finishing write tid 1 to smithi093153278-77 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:77: finishing write tid 2 to smithi093153278-77 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:77: finishing write tid 3 to smithi093153278-77 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:77: finishing write tid 4 to smithi093153278-77 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:77: finishing write tid 5 to smithi093153278-77 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 77 v 8 (ObjNum 76 snap 0 seq_num 76) dirty exists 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:77: left oid 77 (ObjNum 76 snap 0 seq_num 76) 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:78: finishing write tid 1 to smithi093153278-78 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:78: finishing write tid 2 to smithi093153278-78 2023-08-27T20:52:04.560 INFO:tasks.rados.rados.0.smithi093.stdout:78: finishing write tid 3 to smithi093153278-78 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:78: finishing write tid 4 to smithi093153278-78 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:78: finishing write tid 5 to smithi093153278-78 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 24 (ObjNum 77 snap 0 seq_num 77) dirty exists 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:78: left oid 78 (ObjNum 77 snap 0 seq_num 77) 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:79: finishing write tid 1 to smithi093153278-79 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:79: finishing write tid 2 to smithi093153278-79 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:79: finishing write tid 3 to smithi093153278-79 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:79: finishing write tid 4 to smithi093153278-79 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:79: finishing write tid 5 to smithi093153278-79 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 79 v 21 (ObjNum 78 snap 0 seq_num 78) dirty exists 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:79: left oid 79 (ObjNum 78 snap 0 seq_num 78) 2023-08-27T20:52:04.561 INFO:tasks.rados.rados.0.smithi093.stdout:80: finishing write tid 1 to smithi093153278-80 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:80: finishing write tid 2 to smithi093153278-80 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:80: finishing write tid 3 to smithi093153278-80 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:80: finishing write tid 4 to smithi093153278-80 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:80: finishing write tid 5 to smithi093153278-80 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 80 v 19 (ObjNum 79 snap 0 seq_num 79) dirty exists 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:80: left oid 80 (ObjNum 79 snap 0 seq_num 79) 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:81: finishing write tid 1 to smithi093153278-81 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:81: finishing write tid 2 to smithi093153278-81 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:81: finishing write tid 3 to smithi093153278-81 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:81: finishing write tid 4 to smithi093153278-81 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:81: finishing write tid 5 to smithi093153278-81 2023-08-27T20:52:04.562 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 25 (ObjNum 80 snap 0 seq_num 80) dirty exists 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:81: left oid 81 (ObjNum 80 snap 0 seq_num 80) 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:82: finishing write tid 1 to smithi093153278-82 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:82: finishing write tid 2 to smithi093153278-82 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:82: finishing write tid 3 to smithi093153278-82 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:82: finishing write tid 4 to smithi093153278-82 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:82: finishing write tid 5 to smithi093153278-82 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 82 v 19 (ObjNum 81 snap 0 seq_num 81) dirty exists 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:82: left oid 82 (ObjNum 81 snap 0 seq_num 81) 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:83: finishing write tid 1 to smithi093153278-83 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:83: finishing write tid 2 to smithi093153278-83 2023-08-27T20:52:04.563 INFO:tasks.rados.rados.0.smithi093.stdout:83: finishing write tid 3 to smithi093153278-83 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:83: finishing write tid 4 to smithi093153278-83 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:83: finishing write tid 5 to smithi093153278-83 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 83 v 19 (ObjNum 82 snap 0 seq_num 82) dirty exists 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:83: left oid 83 (ObjNum 82 snap 0 seq_num 82) 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:84: finishing write tid 1 to smithi093153278-84 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:84: finishing write tid 2 to smithi093153278-84 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:84: finishing write tid 3 to smithi093153278-84 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:84: finishing write tid 4 to smithi093153278-84 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:84: finishing write tid 5 to smithi093153278-84 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 84 v 25 (ObjNum 83 snap 0 seq_num 83) dirty exists 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:84: left oid 84 (ObjNum 83 snap 0 seq_num 83) 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:85: finishing write tid 1 to smithi093153278-85 2023-08-27T20:52:04.564 INFO:tasks.rados.rados.0.smithi093.stdout:85: finishing write tid 2 to smithi093153278-85 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:70: done (15 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:71: done (14 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:72: done (13 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:73: done (12 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:74: done (11 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:75: done (10 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:76: done (9 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:77: done (8 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:78: done (7 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:79: done (6 left) 2023-08-27T20:52:04.565 INFO:tasks.rados.rados.0.smithi093.stdout:80: done (5 left) 2023-08-27T20:52:04.566 INFO:tasks.rados.rados.0.smithi093.stdout:81: done (4 left) 2023-08-27T20:52:04.566 INFO:tasks.rados.rados.0.smithi093.stdout:82: done (3 left) 2023-08-27T20:52:04.566 INFO:tasks.rados.rados.0.smithi093.stdout:83: done (2 left) 2023-08-27T20:52:04.566 INFO:tasks.rados.rados.0.smithi093.stdout:84: done (1 left) 2023-08-27T20:52:04.566 INFO:tasks.rados.rados.0.smithi093.stdout:86: write initial oid 86 2023-08-27T20:52:04.566 INFO:tasks.rados.rados.0.smithi093.stdout:86: seq_num 85 ranges {628595=476048,1807440=632344,2934719=452474} 2023-08-27T20:52:04.566 INFO:tasks.rados.rados.0.smithi093.stdout:86: writing smithi093153278-86 from 628595 to 1104643 tid 1 2023-08-27T20:52:04.569 INFO:tasks.rados.rados.0.smithi093.stdout:86: writing smithi093153278-86 from 1807440 to 2439784 tid 2 2023-08-27T20:52:04.573 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'echo wait for minority of mons to upgrade' 2023-08-27T20:52:04.575 INFO:tasks.rados.rados.0.smithi093.stdout:86: writing smithi093153278-86 from 2934719 to 3387193 tid 3 2023-08-27T20:52:04.576 INFO:tasks.rados.rados.0.smithi093.stdout:87: write initial oid 87 2023-08-27T20:52:04.576 INFO:tasks.rados.rados.0.smithi093.stdout:87: seq_num 86 ranges {645402=551297,1749569=492355,2845789=599613} 2023-08-27T20:52:04.583 INFO:tasks.rados.rados.0.smithi093.stdout:87: writing smithi093153278-87 from 645402 to 1196699 tid 1 2023-08-27T20:52:04.590 INFO:tasks.rados.rados.0.smithi093.stdout:87: writing smithi093153278-87 from 1749569 to 2241924 tid 2 2023-08-27T20:52:04.597 INFO:tasks.rados.rados.0.smithi093.stdout:87: writing smithi093153278-87 from 2845789 to 3445402 tid 3 2023-08-27T20:52:04.597 INFO:tasks.rados.rados.0.smithi093.stdout:88: write initial oid 88 2023-08-27T20:52:04.597 INFO:tasks.rados.rados.0.smithi093.stdout:88: seq_num 87 ranges {662209=626546,1808051=636013,2873212=588997} 2023-08-27T20:52:04.605 INFO:tasks.rados.rados.0.smithi093.stdout:88: writing smithi093153278-88 from 662209 to 1288755 tid 1 2023-08-27T20:52:04.616 INFO:tasks.rados.rados.0.smithi093.stdout:88: writing smithi093153278-88 from 1808051 to 2444064 tid 2 2023-08-27T20:52:04.620 INFO:tasks.rados.rados.0.smithi093.stdout:88: writing smithi093153278-88 from 2873212 to 3462209 tid 3 2023-08-27T20:52:04.620 INFO:tasks.rados.rados.0.smithi093.stdout:89: write initial oid 89 2023-08-27T20:52:04.620 INFO:tasks.rados.rados.0.smithi093.stdout:89: seq_num 88 ranges {679016=701795,1866533=496024,2900635=402643} 2023-08-27T20:52:04.628 INFO:tasks.rados.rados.0.smithi093.stdout:89: writing smithi093153278-89 from 679016 to 1380811 tid 1 2023-08-27T20:52:04.634 INFO:tasks.rados.rados.0.smithi093.stdout:89: writing smithi093153278-89 from 1866533 to 2362557 tid 2 2023-08-27T20:52:04.641 INFO:tasks.rados.rados.0.smithi093.stdout:89: writing smithi093153278-89 from 2900635 to 3303278 tid 3 2023-08-27T20:52:04.641 INFO:tasks.rados.rados.0.smithi093.stdout:90: write initial oid 90 2023-08-27T20:52:04.641 INFO:tasks.rados.rados.0.smithi093.stdout:90: seq_num 89 ranges {695823=777044,1925015=639682,3328058=167765} 2023-08-27T20:52:04.649 INFO:tasks.rados.rados.0.smithi093.stdout:90: writing smithi093153278-90 from 695823 to 1472867 tid 1 2023-08-27T20:52:04.657 INFO:tasks.rados.rados.0.smithi093.stdout:90: writing smithi093153278-90 from 1925015 to 2564697 tid 2 2023-08-27T20:52:04.662 INFO:tasks.rados.rados.0.smithi093.stdout:90: writing smithi093153278-90 from 3328058 to 3495823 tid 3 2023-08-27T20:52:04.662 INFO:tasks.rados.rados.0.smithi093.stdout:91: write initial oid 91 2023-08-27T20:52:04.663 INFO:tasks.rados.rados.0.smithi093.stdout:85: finishing write tid 3 to smithi093153278-85 2023-08-27T20:52:04.663 INFO:tasks.rados.rados.0.smithi093.stdout:91: seq_num 90 ranges {712630=452293,1583497=499693,2555481=425187} 2023-08-27T20:52:04.669 INFO:tasks.rados.rados.0.smithi093.stdout:91: writing smithi093153278-91 from 712630 to 1164923 tid 1 2023-08-27T20:52:04.675 INFO:tasks.rados.rados.0.smithi093.stdout:91: writing smithi093153278-91 from 1583497 to 2083190 tid 2 2023-08-27T20:52:04.681 INFO:tasks.rados.rados.0.smithi093.stdout:91: writing smithi093153278-91 from 2555481 to 2980668 tid 3 2023-08-27T20:52:04.681 INFO:tasks.rados.rados.0.smithi093.stdout:92: write initial oid 92 2023-08-27T20:52:04.681 INFO:tasks.rados.rados.0.smithi093.stdout:92: seq_num 91 ranges {729437=527542,1925626=643351,3266551=262886} 2023-08-27T20:52:04.688 INFO:tasks.rados.rados.0.smithi093.stdout:92: writing smithi093153278-92 from 729437 to 1256979 tid 1 2023-08-27T20:52:04.696 INFO:tasks.rados.rados.0.smithi093.stdout:92: writing smithi093153278-92 from 1925626 to 2568977 tid 2 2023-08-27T20:52:04.702 INFO:tasks.rados.rados.0.smithi093.stdout:92: writing smithi093153278-92 from 3266551 to 3529437 tid 3 2023-08-27T20:52:04.702 INFO:tasks.rados.rados.0.smithi093.stdout:93: write initial oid 93 2023-08-27T20:52:04.703 INFO:tasks.rados.rados.0.smithi093.stdout:93: seq_num 92 ranges {746244=719144,2100461=503362,3126680=419564} 2023-08-27T20:52:04.711 INFO:tasks.rados.rados.0.smithi093.stdout:93: writing smithi093153278-93 from 746244 to 1465388 tid 1 2023-08-27T20:52:04.718 INFO:tasks.rados.rados.0.smithi093.stdout:93: writing smithi093153278-93 from 2100461 to 2603823 tid 2 2023-08-27T20:52:04.724 INFO:tasks.rados.rados.0.smithi093.stdout:93: writing smithi093153278-93 from 3126680 to 3546244 tid 3 2023-08-27T20:52:04.724 INFO:tasks.rados.rados.0.smithi093.stdout:94: write initial oid 94 2023-08-27T20:52:04.724 INFO:tasks.rados.rados.0.smithi093.stdout:94: seq_num 93 ranges {763051=794393,2158943=647020,3437750=125301} 2023-08-27T20:52:04.734 INFO:tasks.rados.rados.0.smithi093.stdout:94: writing smithi093153278-94 from 763051 to 1557444 tid 1 2023-08-27T20:52:04.742 INFO:tasks.rados.rados.0.smithi093.stdout:94: writing smithi093153278-94 from 2158943 to 2805963 tid 2 2023-08-27T20:52:04.747 INFO:tasks.rados.rados.0.smithi093.stdout:94: writing smithi093153278-94 from 3437750 to 3563051 tid 3 2023-08-27T20:52:04.747 INFO:tasks.rados.rados.0.smithi093.stdout:95: write initial oid 95 2023-08-27T20:52:04.747 INFO:tasks.rados.rados.0.smithi093.stdout:95: seq_num 94 ranges {779858=469642,1817425=507031,2781526=586628} 2023-08-27T20:52:04.754 INFO:tasks.rados.rados.0.smithi093.stdout:95: writing smithi093153278-95 from 779858 to 1249500 tid 1 2023-08-27T20:52:04.761 INFO:tasks.rados.rados.0.smithi093.stdout:95: writing smithi093153278-95 from 1817425 to 2324456 tid 2 2023-08-27T20:52:04.768 INFO:tasks.rados.rados.0.smithi093.stdout:95: writing smithi093153278-95 from 2781526 to 3368154 tid 3 2023-08-27T20:52:04.768 INFO:tasks.rados.rados.0.smithi093.stdout:96: write initial oid 96 2023-08-27T20:52:04.768 INFO:tasks.rados.rados.0.smithi093.stdout:96: seq_num 95 ranges {796665=544891,1759554=650689,2976243=620422} 2023-08-27T20:52:04.776 INFO:tasks.rados.rados.0.smithi093.stdout:96: writing smithi093153278-96 from 796665 to 1341556 tid 1 2023-08-27T20:52:04.783 INFO:tasks.rados.rados.0.smithi093.stdout:96: writing smithi093153278-96 from 1759554 to 2410243 tid 2 2023-08-27T20:52:04.790 INFO:tasks.rados.rados.0.smithi093.stdout:96: writing smithi093153278-96 from 2976243 to 3596665 tid 3 2023-08-27T20:52:04.790 INFO:tasks.rados.rados.0.smithi093.stdout:97: write initial oid 97 2023-08-27T20:52:04.790 INFO:tasks.rados.rados.0.smithi093.stdout:97: seq_num 96 ranges {413472=620140,1818036=510700,3120019=493453} 2023-08-27T20:52:04.796 INFO:tasks.rados.rados.0.smithi093.stdout:97: writing smithi093153278-97 from 413472 to 1033612 tid 1 2023-08-27T20:52:04.804 INFO:tasks.rados.rados.0.smithi093.stdout:97: writing smithi093153278-97 from 1818036 to 2328736 tid 2 2023-08-27T20:52:04.812 INFO:tasks.rados.rados.0.smithi093.stdout:97: writing smithi093153278-97 from 3120019 to 3613472 tid 3 2023-08-27T20:52:04.812 INFO:tasks.rados.rados.0.smithi093.stdout:98: write initial oid 98 2023-08-27T20:52:04.813 INFO:tasks.rados.rados.0.smithi093.stdout:98: seq_num 97 ranges {430279=695389,1876518=654358,3031089=536797} 2023-08-27T20:52:04.819 INFO:tasks.rados.rados.0.smithi093.stdout:98: writing smithi093153278-98 from 430279 to 1125668 tid 1 2023-08-27T20:52:04.829 INFO:tasks.rados.rados.0.smithi093.stdout:98: writing smithi093153278-98 from 1876518 to 2530876 tid 2 2023-08-27T20:52:04.835 INFO:tasks.rados.rados.0.smithi093.stdout:98: writing smithi093153278-98 from 3031089 to 3567886 tid 3 2023-08-27T20:52:04.836 INFO:tasks.rados.rados.0.smithi093.stdout:99: write initial oid 99 2023-08-27T20:52:04.836 INFO:tasks.rados.rados.0.smithi093.stdout:99: seq_num 98 ranges {447086=770638,1935000=798016,3458512=188574} 2023-08-27T20:52:04.844 INFO:tasks.rados.rados.0.smithi093.stdout:99: writing smithi093153278-99 from 447086 to 1217724 tid 1 2023-08-27T20:52:04.854 INFO:tasks.rados.rados.0.smithi093.stdout:99: writing smithi093153278-99 from 1935000 to 2733016 tid 2 2023-08-27T20:52:04.859 INFO:tasks.rados.rados.0.smithi093.stdout:99: writing smithi093153278-99 from 3458512 to 3647086 tid 3 2023-08-27T20:52:04.860 INFO:tasks.rados.rados.0.smithi093.stdout:100: write initial oid 100 2023-08-27T20:52:04.860 INFO:tasks.rados.rados.0.smithi093.stdout:100: seq_num 99 ranges {463893=562240,1593482=658027,2685935=559341} 2023-08-27T20:52:04.869 INFO:tasks.rados.rados.0.smithi093.stdout:100: writing smithi093153278-100 from 463893 to 1026133 tid 1 2023-08-27T20:52:04.876 INFO:tasks.rados.rados.0.smithi093.stdout:100: writing smithi093153278-100 from 1593482 to 2251509 tid 2 2023-08-27T20:52:04.883 INFO:tasks.rados.rados.0.smithi093.stdout:100: writing smithi093153278-100 from 2685935 to 3245276 tid 3 2023-08-27T20:52:04.883 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.884 INFO:tasks.rados.rados.0.smithi093.stdout:85: finishing write tid 4 to smithi093153278-85 2023-08-27T20:52:04.884 INFO:tasks.rados.rados.0.smithi093.stdout:85: finishing write tid 5 to smithi093153278-85 2023-08-27T20:52:04.884 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 85 v 22 (ObjNum 84 snap 0 seq_num 84) dirty exists 2023-08-27T20:52:04.884 INFO:tasks.rados.rados.0.smithi093.stdout:85: left oid 85 (ObjNum 84 snap 0 seq_num 84) 2023-08-27T20:52:04.884 INFO:tasks.rados.rados.0.smithi093.stdout:86: finishing write tid 1 to smithi093153278-86 2023-08-27T20:52:04.884 INFO:tasks.rados.rados.0.smithi093.stdout:85: done (15 left) 2023-08-27T20:52:04.885 INFO:tasks.rados.rados.0.smithi093.stdout:101: write initial oid 101 2023-08-27T20:52:04.885 INFO:tasks.rados.rados.0.smithi093.stdout:101: seq_num 100 ranges {480700=637489,1651964=401685,2713358=770613} 2023-08-27T20:52:04.890 INFO:tasks.rados.rados.0.smithi093.stdout:101: writing smithi093153278-101 from 480700 to 1118189 tid 1 2023-08-27T20:52:04.896 INFO:tasks.rados.rados.0.smithi093.stdout:101: writing smithi093153278-101 from 1651964 to 2053649 tid 2 2023-08-27T20:52:04.906 INFO:tasks.rados.rados.0.smithi093.stdout:101: writing smithi093153278-101 from 2713358 to 3483971 tid 3 2023-08-27T20:52:04.906 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:86: finishing write tid 2 to smithi093153278-86 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:86: finishing write tid 3 to smithi093153278-86 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:86: finishing write tid 4 to smithi093153278-86 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:86: finishing write tid 5 to smithi093153278-86 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 86 v 19 (ObjNum 85 snap 0 seq_num 85) dirty exists 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:86: left oid 86 (ObjNum 85 snap 0 seq_num 85) 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:87: finishing write tid 1 to smithi093153278-87 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:86: done (15 left) 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:102: write initial oid 102 2023-08-27T20:52:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:102: seq_num 101 ranges {497507=712738,1710446=661696,3140781=556726} 2023-08-27T20:52:04.915 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:52:04.916 INFO:tasks.rados.rados.0.smithi093.stdout:102: writing smithi093153278-102 from 497507 to 1210245 tid 1 2023-08-27T20:52:04.923 INFO:tasks.rados.rados.0.smithi093.stdout:102: writing smithi093153278-102 from 1710446 to 2372142 tid 2 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:102: writing smithi093153278-102 from 3140781 to 3697507 tid 3 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:87: finishing write tid 2 to smithi093153278-87 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:87: finishing write tid 3 to smithi093153278-87 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:87: finishing write tid 4 to smithi093153278-87 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:87: finishing write tid 5 to smithi093153278-87 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 28 (ObjNum 86 snap 0 seq_num 86) dirty exists 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:87: left oid 87 (ObjNum 86 snap 0 seq_num 86) 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:88: finishing write tid 1 to smithi093153278-88 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:88: finishing write tid 2 to smithi093153278-88 2023-08-27T20:52:04.933 INFO:tasks.rados.rados.0.smithi093.stdout:88: finishing write tid 3 to smithi093153278-88 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:88: finishing write tid 4 to smithi093153278-88 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:88: finishing write tid 5 to smithi093153278-88 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 26 (ObjNum 87 snap 0 seq_num 87) dirty exists 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:88: left oid 88 (ObjNum 87 snap 0 seq_num 87) 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:89: finishing write tid 1 to smithi093153278-89 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:89: finishing write tid 2 to smithi093153278-89 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:89: finishing write tid 3 to smithi093153278-89 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:89: finishing write tid 4 to smithi093153278-89 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:89: finishing write tid 5 to smithi093153278-89 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 89 v 12 (ObjNum 88 snap 0 seq_num 88) dirty exists 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:89: left oid 89 (ObjNum 88 snap 0 seq_num 88) 2023-08-27T20:52:04.934 INFO:tasks.rados.rados.0.smithi093.stdout:90: finishing write tid 1 to smithi093153278-90 2023-08-27T20:52:04.935 INFO:tasks.rados.rados.0.smithi093.stdout:90: finishing write tid 2 to smithi093153278-90 2023-08-27T20:52:04.935 INFO:tasks.rados.rados.0.smithi093.stdout:87: done (15 left) 2023-08-27T20:52:04.935 INFO:tasks.rados.rados.0.smithi093.stdout:88: done (14 left) 2023-08-27T20:52:04.935 INFO:tasks.rados.rados.0.smithi093.stdout:89: done (13 left) 2023-08-27T20:52:04.935 INFO:tasks.rados.rados.0.smithi093.stdout:103: write initial oid 103 2023-08-27T20:52:04.935 INFO:tasks.rados.rados.0.smithi093.stdout:103: seq_num 102 ranges {514314=787987,1768928=405354,2768204=509510} 2023-08-27T20:52:04.941 INFO:tasks.rados.rados.0.smithi093.stdout:103: writing smithi093153278-103 from 514314 to 1302301 tid 1 2023-08-27T20:52:04.946 INFO:tasks.rados.rados.0.smithi093.stdout:103: writing smithi093153278-103 from 1768928 to 2174282 tid 2 2023-08-27T20:52:04.954 INFO:tasks.rados.rados.0.smithi093.stdout:103: writing smithi093153278-103 from 2768204 to 3277714 tid 3 2023-08-27T20:52:04.954 INFO:tasks.rados.rados.0.smithi093.stdout:104: write initial oid 104 2023-08-27T20:52:04.954 INFO:tasks.rados.rados.0.smithi093.stdout:104: seq_num 103 ranges {531121=463236,1711057=665365,3079274=651847} 2023-08-27T20:52:04.960 INFO:tasks.rados.rados.0.smithi093.stdout:104: writing smithi093153278-104 from 531121 to 994357 tid 1 2023-08-27T20:52:04.968 INFO:tasks.rados.rados.0.smithi093.stdout:104: writing smithi093153278-104 from 1711057 to 2376422 tid 2 2023-08-27T20:52:04.978 INFO:tasks.rados.rados.0.smithi093.stdout:104: writing smithi093153278-104 from 3079274 to 3731121 tid 3 2023-08-27T20:52:04.978 INFO:tasks.rados.rados.0.smithi093.stdout:90: finishing write tid 3 to smithi093153278-90 2023-08-27T20:52:04.978 INFO:tasks.rados.rados.0.smithi093.stdout:90: finishing write tid 4 to smithi093153278-90 2023-08-27T20:52:04.978 INFO:tasks.rados.rados.0.smithi093.stdout:90: finishing write tid 5 to smithi093153278-90 2023-08-27T20:52:04.978 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 90 v 20 (ObjNum 89 snap 0 seq_num 89) dirty exists 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:90: left oid 90 (ObjNum 89 snap 0 seq_num 89) 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:91: finishing write tid 1 to smithi093153278-91 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:91: finishing write tid 2 to smithi093153278-91 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:91: finishing write tid 3 to smithi093153278-91 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:91: finishing write tid 4 to smithi093153278-91 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:91: finishing write tid 5 to smithi093153278-91 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 91 v 24 (ObjNum 90 snap 0 seq_num 90) dirty exists 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:91: left oid 91 (ObjNum 90 snap 0 seq_num 90) 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:92: finishing write tid 1 to smithi093153278-92 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:92: finishing write tid 2 to smithi093153278-92 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:92: finishing write tid 3 to smithi093153278-92 2023-08-27T20:52:04.979 INFO:tasks.rados.rados.0.smithi093.stdout:92: finishing write tid 4 to smithi093153278-92 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:92: finishing write tid 5 to smithi093153278-92 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 92 v 27 (ObjNum 91 snap 0 seq_num 91) dirty exists 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:92: left oid 92 (ObjNum 91 snap 0 seq_num 91) 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:93: finishing write tid 1 to smithi093153278-93 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:93: finishing write tid 2 to smithi093153278-93 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:93: finishing write tid 3 to smithi093153278-93 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:93: finishing write tid 4 to smithi093153278-93 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:93: finishing write tid 5 to smithi093153278-93 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 93 v 23 (ObjNum 92 snap 0 seq_num 92) dirty exists 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:93: left oid 93 (ObjNum 92 snap 0 seq_num 92) 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:90: done (14 left) 2023-08-27T20:52:04.980 INFO:tasks.rados.rados.0.smithi093.stdout:91: done (13 left) 2023-08-27T20:52:04.981 INFO:tasks.rados.rados.0.smithi093.stdout:92: done (12 left) 2023-08-27T20:52:04.981 INFO:tasks.rados.rados.0.smithi093.stdout:93: done (11 left) 2023-08-27T20:52:04.981 INFO:tasks.rados.rados.0.smithi093.stdout:105: write initial oid 105 2023-08-27T20:52:04.981 INFO:tasks.rados.rados.0.smithi093.stdout:105: seq_num 104 ranges {547928=538485,1769539=409023,2706697=532054} 2023-08-27T20:52:04.984 INFO:tasks.rados.rados.0.smithi093.stdout:105: writing smithi093153278-105 from 547928 to 1086413 tid 1 2023-08-27T20:52:04.990 INFO:tasks.rados.rados.0.smithi093.stdout:105: writing smithi093153278-105 from 1769539 to 2178562 tid 2 2023-08-27T20:52:04.997 INFO:tasks.rados.rados.0.smithi093.stdout:105: writing smithi093153278-105 from 2706697 to 3238751 tid 3 2023-08-27T20:52:04.998 INFO:tasks.rados.rados.0.smithi093.stdout:106: write initial oid 106 2023-08-27T20:52:04.998 INFO:tasks.rados.rados.0.smithi093.stdout:94: finishing write tid 1 to smithi093153278-94 2023-08-27T20:52:04.998 INFO:tasks.rados.rados.0.smithi093.stdout:94: finishing write tid 2 to smithi093153278-94 2023-08-27T20:52:04.998 INFO:tasks.rados.rados.0.smithi093.stdout:106: seq_num 105 ranges {564735=613734,1828021=669034,3134120=630615} 2023-08-27T20:52:05.005 INFO:tasks.rados.rados.0.smithi093.stdout:106: writing smithi093153278-106 from 564735 to 1178469 tid 1 2023-08-27T20:52:05.013 INFO:tasks.rados.rados.0.smithi093.stdout:106: writing smithi093153278-106 from 1828021 to 2497055 tid 2 2023-08-27T20:52:05.021 INFO:tasks.rados.rados.0.smithi093.stdout:106: writing smithi093153278-106 from 3134120 to 3764735 tid 3 2023-08-27T20:52:05.022 INFO:tasks.rados.rados.0.smithi093.stdout:107: write initial oid 107 2023-08-27T20:52:05.022 INFO:tasks.rados.rados.0.smithi093.stdout:107: seq_num 106 ranges {581542=688983,1886503=412692,2761543=670951} 2023-08-27T20:52:05.029 INFO:tasks.rados.rados.0.smithi093.stdout:107: writing smithi093153278-107 from 581542 to 1270525 tid 1 2023-08-27T20:52:05.036 INFO:tasks.rados.rados.0.smithi093.stdout:107: writing smithi093153278-107 from 1886503 to 2299195 tid 2 2023-08-27T20:52:05.043 INFO:tasks.rados.rados.0.smithi093.stdout:107: writing smithi093153278-107 from 2761543 to 3432494 tid 3 2023-08-27T20:52:05.044 INFO:tasks.rados.rados.0.smithi093.stdout:108: write initial oid 108 2023-08-27T20:52:05.044 INFO:tasks.rados.rados.0.smithi093.stdout:108: seq_num 107 ranges {598349=480585,1544985=672703,2905319=482223} 2023-08-27T20:52:05.050 INFO:tasks.rados.rados.0.smithi093.stdout:108: writing smithi093153278-108 from 598349 to 1078934 tid 1 2023-08-27T20:52:05.058 INFO:tasks.rados.rados.0.smithi093.stdout:108: writing smithi093153278-108 from 1544985 to 2217688 tid 2 2023-08-27T20:52:05.065 INFO:tasks.rados.rados.0.smithi093.stdout:108: writing smithi093153278-108 from 2905319 to 3387542 tid 3 2023-08-27T20:52:05.066 INFO:tasks.rados.rados.0.smithi093.stdout:109: write initial oid 109 2023-08-27T20:52:05.066 INFO:tasks.rados.rados.0.smithi093.stdout:109: seq_num 108 ranges {615156=555834,1603467=416361,2816389=693495} 2023-08-27T20:52:05.073 INFO:tasks.rados.rados.0.smithi093.stdout:109: writing smithi093153278-109 from 615156 to 1170990 tid 1 2023-08-27T20:52:05.080 INFO:tasks.rados.rados.0.smithi093.stdout:109: writing smithi093153278-109 from 1603467 to 2019828 tid 2 2023-08-27T20:52:05.087 INFO:tasks.rados.rados.0.smithi093.stdout:109: writing smithi093153278-109 from 2816389 to 3509884 tid 3 2023-08-27T20:52:05.087 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:94: finishing write tid 3 to smithi093153278-94 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:94: finishing write tid 4 to smithi093153278-94 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:94: finishing write tid 5 to smithi093153278-94 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 94 v 11 (ObjNum 93 snap 0 seq_num 93) dirty exists 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:94: left oid 94 (ObjNum 93 snap 0 seq_num 93) 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:95: finishing write tid 1 to smithi093153278-95 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:95: finishing write tid 2 to smithi093153278-95 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:95: finishing write tid 3 to smithi093153278-95 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:95: finishing write tid 4 to smithi093153278-95 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:95: finishing write tid 5 to smithi093153278-95 2023-08-27T20:52:05.089 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 95 v 29 (ObjNum 94 snap 0 seq_num 94) dirty exists 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:95: left oid 95 (ObjNum 94 snap 0 seq_num 94) 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:96: finishing write tid 1 to smithi093153278-96 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:96: finishing write tid 2 to smithi093153278-96 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:96: finishing write tid 3 to smithi093153278-96 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:96: finishing write tid 4 to smithi093153278-96 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:96: finishing write tid 5 to smithi093153278-96 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 96 v 20 (ObjNum 95 snap 0 seq_num 95) dirty exists 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:96: left oid 96 (ObjNum 95 snap 0 seq_num 95) 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:97: finishing write tid 1 to smithi093153278-97 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:97: finishing write tid 2 to smithi093153278-97 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:97: finishing write tid 3 to smithi093153278-97 2023-08-27T20:52:05.090 INFO:tasks.rados.rados.0.smithi093.stdout:97: finishing write tid 4 to smithi093153278-97 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:97: finishing write tid 5 to smithi093153278-97 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 97 v 15 (ObjNum 96 snap 0 seq_num 96) dirty exists 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:97: left oid 97 (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:98: finishing write tid 1 to smithi093153278-98 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:94: done (15 left) 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:95: done (14 left) 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:96: done (13 left) 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:97: done (12 left) 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:110: write initial oid 110 2023-08-27T20:52:05.091 INFO:tasks.rados.rados.0.smithi093.stdout:110: seq_num 109 ranges {631963=631083,2061949=560019,3243812=504767} 2023-08-27T20:52:05.096 INFO:tasks.rados.rados.0.smithi093.stdout:110: writing smithi093153278-110 from 631963 to 1263046 tid 1 2023-08-27T20:52:05.105 INFO:tasks.rados.rados.0.smithi093.stdout:110: writing smithi093153278-110 from 2061949 to 2621968 tid 2 2023-08-27T20:52:05.112 INFO:tasks.rados.rados.0.smithi093.stdout:110: writing smithi093153278-110 from 3243812 to 3748579 tid 3 2023-08-27T20:52:05.113 INFO:tasks.rados.rados.0.smithi093.stdout:111: write initial oid 111 2023-08-27T20:52:05.113 INFO:tasks.rados.rados.0.smithi093.stdout:111: seq_num 110 ranges {648770=706332,2120431=420030,3271235=432392} 2023-08-27T20:52:05.123 INFO:tasks.rados.rados.0.smithi093.stdout:111: writing smithi093153278-111 from 648770 to 1355102 tid 1 2023-08-27T20:52:05.128 INFO:tasks.rados.rados.0.smithi093.stdout:111: writing smithi093153278-111 from 2120431 to 2540461 tid 2 2023-08-27T20:52:05.136 INFO:tasks.rados.rados.0.smithi093.stdout:111: writing smithi093153278-111 from 3271235 to 3703627 tid 3 2023-08-27T20:52:05.136 INFO:tasks.rados.rados.0.smithi093.stdout:112: write initial oid 112 2023-08-27T20:52:05.136 INFO:tasks.rados.rados.0.smithi093.stdout:112: seq_num 111 ranges {665577=781581,2062560=563688,3182305=643664} 2023-08-27T20:52:05.145 INFO:tasks.rados.rados.0.smithi093.stdout:112: writing smithi093153278-112 from 665577 to 1447158 tid 1 2023-08-27T20:52:05.153 INFO:tasks.rados.rados.0.smithi093.stdout:112: writing smithi093153278-112 from 2062560 to 2626248 tid 2 2023-08-27T20:52:05.161 INFO:tasks.rados.rados.0.smithi093.stdout:112: writing smithi093153278-112 from 3182305 to 3825969 tid 3 2023-08-27T20:52:05.161 INFO:tasks.rados.rados.0.smithi093.stdout:113: write initial oid 113 2023-08-27T20:52:05.161 INFO:tasks.rados.rados.0.smithi093.stdout:113: seq_num 112 ranges {682384=456830,1721042=423699,2809728=454936,3731357=151027} 2023-08-27T20:52:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:113: writing smithi093153278-113 from 682384 to 1139214 tid 1 2023-08-27T20:52:05.174 INFO:tasks.rados.rados.0.smithi093.stdout:113: writing smithi093153278-113 from 1721042 to 2144741 tid 2 2023-08-27T20:52:05.181 INFO:tasks.rados.rados.0.smithi093.stdout:113: writing smithi093153278-113 from 2809728 to 3264664 tid 3 2023-08-27T20:52:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:113: writing smithi093153278-113 from 3731357 to 3882384 tid 4 2023-08-27T20:52:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:98: finishing write tid 2 to smithi093153278-98 2023-08-27T20:52:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:98: finishing write tid 3 to smithi093153278-98 2023-08-27T20:52:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:98: finishing write tid 4 to smithi093153278-98 2023-08-27T20:52:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:98: finishing write tid 5 to smithi093153278-98 2023-08-27T20:52:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 98 v 33 (ObjNum 97 snap 0 seq_num 97) dirty exists 2023-08-27T20:52:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:98: left oid 98 (ObjNum 97 snap 0 seq_num 97) 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:99: finishing write tid 1 to smithi093153278-99 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:99: finishing write tid 2 to smithi093153278-99 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:99: finishing write tid 3 to smithi093153278-99 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:99: finishing write tid 4 to smithi093153278-99 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:99: finishing write tid 5 to smithi093153278-99 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 99 v 10 (ObjNum 98 snap 0 seq_num 98) dirty exists 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:99: left oid 99 (ObjNum 98 snap 0 seq_num 98) 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:100: finishing write tid 1 to smithi093153278-100 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:100: finishing write tid 2 to smithi093153278-100 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:100: finishing write tid 3 to smithi093153278-100 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:98: done (15 left) 2023-08-27T20:52:05.187 INFO:tasks.rados.rados.0.smithi093.stdout:99: done (14 left) 2023-08-27T20:52:05.188 INFO:tasks.rados.rados.0.smithi093.stdout:114: write initial oid 114 2023-08-27T20:52:05.188 INFO:tasks.rados.rados.0.smithi093.stdout:114: seq_num 113 ranges {699191=532079,1779524=567357,2837151=666208} 2023-08-27T20:52:05.193 INFO:tasks.rados.rados.0.smithi093.stdout:114: writing smithi093153278-114 from 699191 to 1231270 tid 1 2023-08-27T20:52:05.200 INFO:tasks.rados.rados.0.smithi093.stdout:114: writing smithi093153278-114 from 1779524 to 2346881 tid 2 2023-08-27T20:52:05.208 INFO:tasks.rados.rados.0.smithi093.stdout:114: writing smithi093153278-114 from 2837151 to 3503359 tid 3 2023-08-27T20:52:05.208 INFO:tasks.rados.rados.0.smithi093.stdout:115: write initial oid 115 2023-08-27T20:52:05.208 INFO:tasks.rados.rados.0.smithi093.stdout:115: seq_num 114 ranges {715998=607328,1838006=427368,2864574=477480,3863835=52163} 2023-08-27T20:52:05.216 INFO:tasks.rados.rados.0.smithi093.stdout:115: writing smithi093153278-115 from 715998 to 1323326 tid 1 2023-08-27T20:52:05.222 INFO:tasks.rados.rados.0.smithi093.stdout:115: writing smithi093153278-115 from 1838006 to 2265374 tid 2 2023-08-27T20:52:05.228 INFO:tasks.rados.rados.0.smithi093.stdout:115: writing smithi093153278-115 from 2864574 to 3342054 tid 3 2023-08-27T20:52:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:115: writing smithi093153278-115 from 3863835 to 3915998 tid 4 2023-08-27T20:52:05.232 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:100: finishing write tid 4 to smithi093153278-100 2023-08-27T20:52:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:100: finishing write tid 5 to smithi093153278-100 2023-08-27T20:52:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 100 v 24 (ObjNum 99 snap 0 seq_num 99) dirty exists 2023-08-27T20:52:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:100: left oid 100 (ObjNum 99 snap 0 seq_num 99) 2023-08-27T20:52:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:101: finishing write tid 1 to smithi093153278-101 2023-08-27T20:52:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:101: finishing write tid 2 to smithi093153278-101 2023-08-27T20:52:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:101: finishing write tid 3 to smithi093153278-101 2023-08-27T20:52:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:100: done (15 left) 2023-08-27T20:52:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:116: write initial oid 116 2023-08-27T20:52:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:116: seq_num 115 ranges {732805=798930,2296488=571026,3291997=405105} 2023-08-27T20:52:05.240 INFO:tasks.rados.rados.0.smithi093.stdout:116: writing smithi093153278-116 from 732805 to 1531735 tid 1 2023-08-27T20:52:05.249 INFO:tasks.rados.rados.0.smithi093.stdout:116: writing smithi093153278-116 from 2296488 to 2867514 tid 2 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout:116: writing smithi093153278-116 from 3291997 to 3697102 tid 3 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout:101: finishing write tid 4 to smithi093153278-101 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout:101: finishing write tid 5 to smithi093153278-101 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 101 v 31 (ObjNum 100 snap 0 seq_num 100) dirty exists 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout:101: left oid 101 (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout:102: finishing write tid 1 to smithi093153278-102 2023-08-27T20:52:05.255 INFO:tasks.rados.rados.0.smithi093.stdout:102: finishing write tid 2 to smithi093153278-102 2023-08-27T20:52:05.256 INFO:tasks.rados.rados.0.smithi093.stdout:101: done (15 left) 2023-08-27T20:52:05.256 INFO:tasks.rados.rados.0.smithi093.stdout:117: write initial oid 117 2023-08-27T20:52:05.256 INFO:tasks.rados.rados.0.smithi093.stdout:117: seq_num 116 ranges {749612=474179,1954970=431037,2919420=616377} 2023-08-27T20:52:05.262 INFO:tasks.rados.rados.0.smithi093.stdout:117: writing smithi093153278-117 from 749612 to 1223791 tid 1 2023-08-27T20:52:05.269 INFO:tasks.rados.rados.0.smithi093.stdout:117: writing smithi093153278-117 from 1954970 to 2386007 tid 2 2023-08-27T20:52:05.278 INFO:tasks.rados.rados.0.smithi093.stdout:117: writing smithi093153278-117 from 2919420 to 3535797 tid 3 2023-08-27T20:52:05.278 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:102: finishing write tid 3 to smithi093153278-102 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:102: finishing write tid 4 to smithi093153278-102 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:102: finishing write tid 5 to smithi093153278-102 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 102 v 35 (ObjNum 101 snap 0 seq_num 101) dirty exists 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:102: left oid 102 (ObjNum 101 snap 0 seq_num 101) 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:103: finishing write tid 1 to smithi093153278-103 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:102: done (15 left) 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:118: write initial oid 118 2023-08-27T20:52:05.279 INFO:tasks.rados.rados.0.smithi093.stdout:118: seq_num 117 ranges {766419=549428,2013452=574695,3346843=427649} 2023-08-27T20:52:05.284 INFO:tasks.rados.rados.0.smithi093.stdout:118: writing smithi093153278-118 from 766419 to 1315847 tid 1 2023-08-27T20:52:05.293 INFO:tasks.rados.rados.0.smithi093.stdout:118: writing smithi093153278-118 from 2013452 to 2588147 tid 2 2023-08-27T20:52:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:118: writing smithi093153278-118 from 3346843 to 3774492 tid 3 2023-08-27T20:52:05.301 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:103: finishing write tid 2 to smithi093153278-103 2023-08-27T20:52:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:103: finishing write tid 3 to smithi093153278-103 2023-08-27T20:52:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:103: finishing write tid 4 to smithi093153278-103 2023-08-27T20:52:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:103: finishing write tid 5 to smithi093153278-103 2023-08-27T20:52:05.302 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 103 v 20 (ObjNum 102 snap 0 seq_num 102) dirty exists 2023-08-27T20:52:05.302 INFO:tasks.rados.rados.0.smithi093.stdout:103: left oid 103 (ObjNum 102 snap 0 seq_num 102) 2023-08-27T20:52:05.302 INFO:tasks.rados.rados.0.smithi093.stdout:104: finishing write tid 1 to smithi093153278-104 2023-08-27T20:52:05.302 INFO:tasks.rados.rados.0.smithi093.stdout:104: finishing write tid 2 to smithi093153278-104 2023-08-27T20:52:05.302 INFO:tasks.rados.rados.0.smithi093.stdout:103: done (15 left) 2023-08-27T20:52:05.302 INFO:tasks.rados.rados.0.smithi093.stdout:119: write initial oid 119 2023-08-27T20:52:05.302 INFO:tasks.rados.rados.0.smithi093.stdout:119: seq_num 118 ranges {783226=624677,2071934=434706,2974266=638921} 2023-08-27T20:52:05.308 INFO:tasks.rados.rados.0.smithi093.stdout:119: writing smithi093153278-119 from 783226 to 1407903 tid 1 2023-08-27T20:52:05.315 INFO:tasks.rados.rados.0.smithi093.stdout:119: writing smithi093153278-119 from 2071934 to 2506640 tid 2 2023-08-27T20:52:05.322 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:05 smithi093 ceph-mon[111335]: from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:52:05.323 INFO:tasks.rados.rados.0.smithi093.stdout:119: writing smithi093153278-119 from 2974266 to 3613187 tid 3 2023-08-27T20:52:05.323 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.323 INFO:tasks.rados.rados.0.smithi093.stdout:104: finishing write tid 3 to smithi093153278-104 2023-08-27T20:52:05.324 INFO:tasks.rados.rados.0.smithi093.stdout:104: finishing write tid 4 to smithi093153278-104 2023-08-27T20:52:05.324 INFO:tasks.rados.rados.0.smithi093.stdout:104: finishing write tid 5 to smithi093153278-104 2023-08-27T20:52:05.324 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 104 v 27 (ObjNum 103 snap 0 seq_num 103) dirty exists 2023-08-27T20:52:05.324 INFO:tasks.rados.rados.0.smithi093.stdout:104: left oid 104 (ObjNum 103 snap 0 seq_num 103) 2023-08-27T20:52:05.324 INFO:tasks.rados.rados.0.smithi093.stdout:105: finishing write tid 1 to smithi093153278-105 2023-08-27T20:52:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:104: done (15 left) 2023-08-27T20:52:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:120: write initial oid 120 2023-08-27T20:52:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:120: seq_num 119 ranges {400033=699926,1614063=385970} 2023-08-27T20:52:05.325 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:05 smithi093 ceph-mon[118902]: from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:52:05.329 INFO:tasks.rados.rados.0.smithi093.stdout:120: writing smithi093153278-120 from 400033 to 1099959 tid 1 2023-08-27T20:52:05.336 INFO:tasks.rados.rados.0.smithi093.stdout:120: writing smithi093153278-120 from 1614063 to 2000033 tid 2 2023-08-27T20:52:05.336 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.336 INFO:tasks.rados.rados.0.smithi093.stdout:105: finishing write tid 2 to smithi093153278-105 2023-08-27T20:52:05.336 INFO:tasks.rados.rados.0.smithi093.stdout:105: finishing write tid 3 to smithi093153278-105 2023-08-27T20:52:05.336 INFO:tasks.rados.rados.0.smithi093.stdout:105: finishing write tid 4 to smithi093153278-105 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:105: finishing write tid 5 to smithi093153278-105 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 105 v 23 (ObjNum 104 snap 0 seq_num 104) dirty exists 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:105: left oid 105 (ObjNum 104 snap 0 seq_num 104) 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:106: finishing write tid 1 to smithi093153278-106 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:106: finishing write tid 2 to smithi093153278-106 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:106: finishing write tid 3 to smithi093153278-106 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:106: finishing write tid 4 to smithi093153278-106 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:106: finishing write tid 5 to smithi093153278-106 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 106 v 23 (ObjNum 105 snap 0 seq_num 105) dirty exists 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:106: left oid 106 (ObjNum 105 snap 0 seq_num 105) 2023-08-27T20:52:05.337 INFO:tasks.rados.rados.0.smithi093.stdout:107: finishing write tid 1 to smithi093153278-107 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:107: finishing write tid 2 to smithi093153278-107 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:107: finishing write tid 3 to smithi093153278-107 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:107: finishing write tid 4 to smithi093153278-107 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:107: finishing write tid 5 to smithi093153278-107 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 107 v 16 (ObjNum 106 snap 0 seq_num 106) dirty exists 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:107: left oid 107 (ObjNum 106 snap 0 seq_num 106) 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:108: finishing write tid 1 to smithi093153278-108 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:108: finishing write tid 2 to smithi093153278-108 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:105: done (15 left) 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:106: done (14 left) 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:107: done (13 left) 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:121: write initial oid 121 2023-08-27T20:52:05.338 INFO:tasks.rados.rados.0.smithi093.stdout:121: seq_num 120 ranges {416840=775175,1672545=344295} 2023-08-27T20:52:05.343 INFO:tasks.rados.rados.0.smithi093.stdout:121: writing smithi093153278-121 from 416840 to 1192015 tid 1 2023-08-27T20:52:05.348 INFO:tasks.rados.rados.0.smithi093.stdout:121: writing smithi093153278-121 from 1672545 to 2016840 tid 2 2023-08-27T20:52:05.348 INFO:tasks.rados.rados.0.smithi093.stdout:122: write initial oid 122 2023-08-27T20:52:05.348 INFO:tasks.rados.rados.0.smithi093.stdout:122: seq_num 121 ranges {433647=450424,1331027=582033} 2023-08-27T20:52:05.353 INFO:tasks.rados.rados.0.smithi093.stdout:122: writing smithi093153278-122 from 433647 to 884071 tid 1 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:122: writing smithi093153278-122 from 1331027 to 1913060 tid 2 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:123: write initial oid 123 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:108: finishing write tid 3 to smithi093153278-108 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:108: finishing write tid 4 to smithi093153278-108 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:108: finishing write tid 5 to smithi093153278-108 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 108 v 28 (ObjNum 107 snap 0 seq_num 107) dirty exists 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:108: left oid 108 (ObjNum 107 snap 0 seq_num 107) 2023-08-27T20:52:05.360 INFO:tasks.rados.rados.0.smithi093.stdout:109: finishing write tid 1 to smithi093153278-109 2023-08-27T20:52:05.361 INFO:tasks.rados.rados.0.smithi093.stdout:123: seq_num 122 ranges {450454=642026,1505862=544592} 2023-08-27T20:52:05.367 INFO:tasks.rados.rados.0.smithi093.stdout:123: writing smithi093153278-123 from 450454 to 1092480 tid 1 2023-08-27T20:52:05.373 INFO:tasks.rados.rados.0.smithi093.stdout:123: writing smithi093153278-123 from 1505862 to 2050454 tid 2 2023-08-27T20:52:05.373 INFO:tasks.rados.rados.0.smithi093.stdout:108: done (15 left) 2023-08-27T20:52:05.374 INFO:tasks.rados.rados.0.smithi093.stdout:124: write initial oid 124 2023-08-27T20:52:05.374 INFO:tasks.rados.rados.0.smithi093.stdout:124: seq_num 123 ranges {467261=717275,1847991=219270} 2023-08-27T20:52:05.380 INFO:tasks.rados.rados.0.smithi093.stdout:124: writing smithi093153278-124 from 467261 to 1184536 tid 1 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout:124: writing smithi093153278-124 from 1847991 to 2067261 tid 2 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout:109: finishing write tid 2 to smithi093153278-109 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout:109: finishing write tid 3 to smithi093153278-109 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout:109: finishing write tid 4 to smithi093153278-109 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout:109: finishing write tid 5 to smithi093153278-109 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 109 v 24 (ObjNum 108 snap 0 seq_num 108) dirty exists 2023-08-27T20:52:05.386 INFO:tasks.rados.rados.0.smithi093.stdout:109: left oid 109 (ObjNum 108 snap 0 seq_num 108) 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:110: finishing write tid 1 to smithi093153278-110 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:110: finishing write tid 2 to smithi093153278-110 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:110: finishing write tid 3 to smithi093153278-110 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:110: finishing write tid 4 to smithi093153278-110 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:110: finishing write tid 5 to smithi093153278-110 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 110 v 28 (ObjNum 109 snap 0 seq_num 109) dirty exists 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:110: left oid 110 (ObjNum 109 snap 0 seq_num 109) 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:111: finishing write tid 1 to smithi093153278-111 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:109: done (15 left) 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:110: done (14 left) 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:125: write initial oid 125 2023-08-27T20:52:05.387 INFO:tasks.rados.rados.0.smithi093.stdout:125: seq_num 124 ranges {484068=792524,1906473=177595} 2023-08-27T20:52:05.394 INFO:tasks.rados.rados.0.smithi093.stdout:125: writing smithi093153278-125 from 484068 to 1276592 tid 1 2023-08-27T20:52:05.399 INFO:tasks.rados.rados.0.smithi093.stdout:125: writing smithi093153278-125 from 1906473 to 2084068 tid 2 2023-08-27T20:52:05.399 INFO:tasks.rados.rados.0.smithi093.stdout:126: write initial oid 126 2023-08-27T20:52:05.399 INFO:tasks.rados.rados.0.smithi093.stdout:126: seq_num 125 ranges {500875=467773,1564955=535920} 2023-08-27T20:52:05.404 INFO:tasks.rados.rados.0.smithi093.stdout:126: writing smithi093153278-126 from 500875 to 968648 tid 1 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:126: writing smithi093153278-126 from 1564955 to 2100875 tid 2 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:111: finishing write tid 2 to smithi093153278-111 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:111: finishing write tid 3 to smithi093153278-111 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:111: finishing write tid 4 to smithi093153278-111 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:111: finishing write tid 5 to smithi093153278-111 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 111 v 32 (ObjNum 110 snap 0 seq_num 110) dirty exists 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:111: left oid 111 (ObjNum 110 snap 0 seq_num 110) 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:112: finishing write tid 1 to smithi093153278-112 2023-08-27T20:52:05.412 INFO:tasks.rados.rados.0.smithi093.stdout:112: finishing write tid 2 to smithi093153278-112 2023-08-27T20:52:05.413 INFO:tasks.rados.rados.0.smithi093.stdout:112: finishing write tid 3 to smithi093153278-112 2023-08-27T20:52:05.413 INFO:tasks.rados.rados.0.smithi093.stdout:111: done (15 left) 2023-08-27T20:52:05.413 INFO:tasks.rados.rados.0.smithi093.stdout:127: write initial oid 127 2023-08-27T20:52:05.413 INFO:tasks.rados.rados.0.smithi093.stdout:127: seq_num 126 ranges {517682=543022,1623437=494245} 2023-08-27T20:52:05.418 INFO:tasks.rados.rados.0.smithi093.stdout:127: writing smithi093153278-127 from 517682 to 1060704 tid 1 2023-08-27T20:52:05.425 INFO:tasks.rados.rados.0.smithi093.stdout:127: writing smithi093153278-127 from 1623437 to 2117682 tid 2 2023-08-27T20:52:05.425 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.425 INFO:tasks.rados.rados.0.smithi093.stdout:112: finishing write tid 4 to smithi093153278-112 2023-08-27T20:52:05.425 INFO:tasks.rados.rados.0.smithi093.stdout:112: finishing write tid 5 to smithi093153278-112 2023-08-27T20:52:05.425 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 112 v 28 (ObjNum 111 snap 0 seq_num 111) dirty exists 2023-08-27T20:52:05.425 INFO:tasks.rados.rados.0.smithi093.stdout:112: left oid 112 (ObjNum 111 snap 0 seq_num 111) 2023-08-27T20:52:05.425 INFO:tasks.rados.rados.0.smithi093.stdout:113: finishing write tid 1 to smithi093153278-113 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:113: finishing write tid 2 to smithi093153278-113 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:113: finishing write tid 3 to smithi093153278-113 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:112: done (15 left) 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:128: write initial oid 128 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:113: finishing write tid 4 to smithi093153278-113 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:113: finishing write tid 5 to smithi093153278-113 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:113: finishing write tid 6 to smithi093153278-113 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 113 v 25 (ObjNum 112 snap 0 seq_num 112) dirty exists 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:113: left oid 113 (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:114: finishing write tid 1 to smithi093153278-114 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:114: finishing write tid 2 to smithi093153278-114 2023-08-27T20:52:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:114: finishing write tid 3 to smithi093153278-114 2023-08-27T20:52:05.427 INFO:tasks.rados.rados.0.smithi093.stdout:128: seq_num 127 ranges {534489=618271,1565566=568923} 2023-08-27T20:52:05.432 INFO:tasks.rados.rados.0.smithi093.stdout:128: writing smithi093153278-128 from 534489 to 1152760 tid 1 2023-08-27T20:52:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:05 smithi136 ceph-mon[113531]: from='client.14853 -' entity='client.admin' cmd=[{"prefix": "orch ps", "target": ["mon-mgr", ""]}]: dispatch 2023-08-27T20:52:05.439 INFO:tasks.rados.rados.0.smithi093.stdout:128: writing smithi093153278-128 from 1565566 to 2134489 tid 2 2023-08-27T20:52:05.439 INFO:tasks.rados.rados.0.smithi093.stdout:113: done (15 left) 2023-08-27T20:52:05.439 INFO:tasks.rados.rados.0.smithi093.stdout:129: write initial oid 129 2023-08-27T20:52:05.439 INFO:tasks.rados.rados.0.smithi093.stdout:129: seq_num 128 ranges {551296=693520,2024048=127248} 2023-08-27T20:52:05.447 INFO:tasks.rados.rados.0.smithi093.stdout:129: writing smithi093153278-129 from 551296 to 1244816 tid 1 2023-08-27T20:52:05.452 INFO:tasks.rados.rados.0.smithi093.stdout:129: writing smithi093153278-129 from 2024048 to 2151296 tid 2 2023-08-27T20:52:05.452 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.452 INFO:tasks.rados.rados.0.smithi093.stdout:114: finishing write tid 4 to smithi093153278-114 2023-08-27T20:52:05.452 INFO:tasks.rados.rados.0.smithi093.stdout:114: finishing write tid 5 to smithi093153278-114 2023-08-27T20:52:05.453 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 114 v 29 (ObjNum 113 snap 0 seq_num 113) dirty exists 2023-08-27T20:52:05.453 INFO:tasks.rados.rados.0.smithi093.stdout:114: left oid 114 (ObjNum 113 snap 0 seq_num 113) 2023-08-27T20:52:05.453 INFO:tasks.rados.rados.0.smithi093.stdout:115: finishing write tid 1 to smithi093153278-115 2023-08-27T20:52:05.453 INFO:tasks.rados.rados.0.smithi093.stdout:114: done (15 left) 2023-08-27T20:52:05.453 INFO:tasks.rados.rados.0.smithi093.stdout:130: write initial oid 130 2023-08-27T20:52:05.453 INFO:tasks.rados.rados.0.smithi093.stdout:130: seq_num 129 ranges {568103=768769,2082530=85573} 2023-08-27T20:52:05.460 INFO:tasks.rados.rados.0.smithi093.stdout:130: writing smithi093153278-130 from 568103 to 1336872 tid 1 2023-08-27T20:52:05.467 INFO:tasks.rados.rados.0.smithi093.stdout:130: writing smithi093153278-130 from 2082530 to 2168103 tid 2 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:115: finishing write tid 2 to smithi093153278-115 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:115: finishing write tid 3 to smithi093153278-115 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:115: finishing write tid 4 to smithi093153278-115 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:115: finishing write tid 5 to smithi093153278-115 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:115: finishing write tid 6 to smithi093153278-115 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 115 v 34 (ObjNum 114 snap 0 seq_num 114) dirty exists 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:115: left oid 115 (ObjNum 114 snap 0 seq_num 114) 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:116: finishing write tid 1 to smithi093153278-116 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:116: finishing write tid 2 to smithi093153278-116 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:116: finishing write tid 3 to smithi093153278-116 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:116: finishing write tid 4 to smithi093153278-116 2023-08-27T20:52:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:116: finishing write tid 5 to smithi093153278-116 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 116 v 27 (ObjNum 115 snap 0 seq_num 115) dirty exists 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:116: left oid 116 (ObjNum 115 snap 0 seq_num 115) 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:117: finishing write tid 1 to smithi093153278-117 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:117: finishing write tid 2 to smithi093153278-117 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:117: finishing write tid 3 to smithi093153278-117 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:117: finishing write tid 4 to smithi093153278-117 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:117: finishing write tid 5 to smithi093153278-117 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 117 v 37 (ObjNum 116 snap 0 seq_num 116) dirty exists 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:117: left oid 117 (ObjNum 116 snap 0 seq_num 116) 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:118: finishing write tid 1 to smithi093153278-118 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:118: finishing write tid 2 to smithi093153278-118 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:118: finishing write tid 3 to smithi093153278-118 2023-08-27T20:52:05.469 INFO:tasks.rados.rados.0.smithi093.stdout:118: finishing write tid 4 to smithi093153278-118 2023-08-27T20:52:05.470 INFO:tasks.rados.rados.0.smithi093.stdout:118: finishing write tid 5 to smithi093153278-118 2023-08-27T20:52:05.470 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 118 v 31 (ObjNum 117 snap 0 seq_num 117) dirty exists 2023-08-27T20:52:05.470 INFO:tasks.rados.rados.0.smithi093.stdout:118: left oid 118 (ObjNum 117 snap 0 seq_num 117) 2023-08-27T20:52:05.470 INFO:tasks.rados.rados.0.smithi093.stdout:119: finishing write tid 1 to smithi093153278-119 2023-08-27T20:52:05.470 INFO:tasks.rados.rados.0.smithi093.stdout:119: finishing write tid 2 to smithi093153278-119 2023-08-27T20:52:05.470 INFO:tasks.rados.rados.0.smithi093.stdout:119: finishing write tid 3 to smithi093153278-119 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:119: finishing write tid 4 to smithi093153278-119 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:119: finishing write tid 5 to smithi093153278-119 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 119 v 39 (ObjNum 118 snap 0 seq_num 118) dirty exists 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:119: left oid 119 (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:120: finishing write tid 1 to smithi093153278-120 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:120: finishing write tid 2 to smithi093153278-120 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:120: finishing write tid 3 to smithi093153278-120 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:120: finishing write tid 4 to smithi093153278-120 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 120 v 31 (ObjNum 119 snap 0 seq_num 119) dirty exists 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:120: left oid 120 (ObjNum 119 snap 0 seq_num 119) 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:121: finishing write tid 1 to smithi093153278-121 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:121: finishing write tid 2 to smithi093153278-121 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:121: finishing write tid 3 to smithi093153278-121 2023-08-27T20:52:05.471 INFO:tasks.rados.rados.0.smithi093.stdout:121: finishing write tid 4 to smithi093153278-121 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 121 v 13 (ObjNum 120 snap 0 seq_num 120) dirty exists 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:121: left oid 121 (ObjNum 120 snap 0 seq_num 120) 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:122: finishing write tid 1 to smithi093153278-122 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:122: finishing write tid 2 to smithi093153278-122 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:122: finishing write tid 3 to smithi093153278-122 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:122: finishing write tid 4 to smithi093153278-122 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 122 v 18 (ObjNum 121 snap 0 seq_num 121) dirty exists 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:122: left oid 122 (ObjNum 121 snap 0 seq_num 121) 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:123: finishing write tid 1 to smithi093153278-123 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:123: finishing write tid 2 to smithi093153278-123 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:123: finishing write tid 3 to smithi093153278-123 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:123: finishing write tid 4 to smithi093153278-123 2023-08-27T20:52:05.472 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 123 v 34 (ObjNum 122 snap 0 seq_num 122) dirty exists 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:123: left oid 123 (ObjNum 122 snap 0 seq_num 122) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:124: finishing write tid 1 to smithi093153278-124 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:115: done (15 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:116: done (14 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:117: done (13 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:118: done (12 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:119: done (11 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:120: done (10 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:121: done (9 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:122: done (8 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:123: done (7 left) 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:131: write initial oid 131 2023-08-27T20:52:05.473 INFO:tasks.rados.rados.0.smithi093.stdout:131: seq_num 130 ranges {584910=560371,1857365=327545} 2023-08-27T20:52:05.474 INFO:tasks.rados.rados.0.smithi093.stdout:131: writing smithi093153278-131 from 584910 to 1145281 tid 1 2023-08-27T20:52:05.479 INFO:tasks.rados.rados.0.smithi093.stdout:131: writing smithi093153278-131 from 1857365 to 2184910 tid 2 2023-08-27T20:52:05.479 INFO:tasks.rados.rados.0.smithi093.stdout:132: write initial oid 132 2023-08-27T20:52:05.479 INFO:tasks.rados.rados.0.smithi093.stdout:132: seq_num 131 ranges {601717=635620,1799494=402223} 2023-08-27T20:52:05.486 INFO:tasks.rados.rados.0.smithi093.stdout:132: writing smithi093153278-132 from 601717 to 1237337 tid 1 2023-08-27T20:52:05.492 INFO:tasks.rados.rados.0.smithi093.stdout:132: writing smithi093153278-132 from 1799494 to 2201717 tid 2 2023-08-27T20:52:05.493 INFO:tasks.rados.rados.0.smithi093.stdout:133: write initial oid 133 2023-08-27T20:52:05.493 INFO:tasks.rados.rados.0.smithi093.stdout:133: seq_num 132 ranges {618524=710869,1857976=360548} 2023-08-27T20:52:05.501 INFO:tasks.rados.rados.0.smithi093.stdout:133: writing smithi093153278-133 from 618524 to 1329393 tid 1 2023-08-27T20:52:05.507 INFO:tasks.rados.rados.0.smithi093.stdout:133: writing smithi093153278-133 from 1857976 to 2218524 tid 2 2023-08-27T20:52:05.507 INFO:tasks.rados.rados.0.smithi093.stdout:134: write initial oid 134 2023-08-27T20:52:05.507 INFO:tasks.rados.rados.0.smithi093.stdout:134: seq_num 133 ranges {635331=786118,1916458=318873} 2023-08-27T20:52:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:134: writing smithi093153278-134 from 635331 to 1421449 tid 1 2023-08-27T20:52:05.521 INFO:tasks.rados.rados.0.smithi093.stdout:134: writing smithi093153278-134 from 1916458 to 2235331 tid 2 2023-08-27T20:52:05.521 INFO:tasks.rados.rados.0.smithi093.stdout:135: write initial oid 135 2023-08-27T20:52:05.521 INFO:tasks.rados.rados.0.smithi093.stdout:135: seq_num 134 ranges {652138=461367,1574940=677198} 2023-08-27T20:52:05.528 INFO:tasks.rados.rados.0.smithi093.stdout:135: writing smithi093153278-135 from 652138 to 1113505 tid 1 2023-08-27T20:52:05.535 INFO:tasks.rados.rados.0.smithi093.stdout:135: writing smithi093153278-135 from 1574940 to 2252138 tid 2 2023-08-27T20:52:05.535 INFO:tasks.rados.rados.0.smithi093.stdout:136: write initial oid 136 2023-08-27T20:52:05.536 INFO:tasks.rados.rados.0.smithi093.stdout:136: seq_num 135 ranges {668945=536616,1633422=491363} 2023-08-27T20:52:05.543 INFO:tasks.rados.rados.0.smithi093.stdout:136: writing smithi093153278-136 from 668945 to 1205561 tid 1 2023-08-27T20:52:05.549 INFO:tasks.rados.rados.0.smithi093.stdout:136: writing smithi093153278-136 from 1633422 to 2124785 tid 2 2023-08-27T20:52:05.549 INFO:tasks.rados.rados.0.smithi093.stdout:137: write initial oid 137 2023-08-27T20:52:05.549 INFO:tasks.rados.rados.0.smithi093.stdout:137: seq_num 136 ranges {685752=611865,1975551=310201} 2023-08-27T20:52:05.556 INFO:tasks.rados.rados.0.smithi093.stdout:137: writing smithi093153278-137 from 685752 to 1297617 tid 1 2023-08-27T20:52:05.562 INFO:tasks.rados.rados.0.smithi093.stdout:137: writing smithi093153278-137 from 1975551 to 2285752 tid 2 2023-08-27T20:52:05.562 INFO:tasks.rados.rados.0.smithi093.stdout:138: write initial oid 138 2023-08-27T20:52:05.562 INFO:tasks.rados.rados.0.smithi093.stdout:138: seq_num 137 ranges {702559=403467,1750386=495032} 2023-08-27T20:52:05.569 INFO:tasks.rados.rados.0.smithi093.stdout:138: writing smithi093153278-138 from 702559 to 1106026 tid 1 2023-08-27T20:52:05.576 INFO:tasks.rados.rados.0.smithi093.stdout:138: writing smithi093153278-138 from 1750386 to 2245418 tid 2 2023-08-27T20:52:05.576 INFO:tasks.rados.rados.0.smithi093.stdout:139: write initial oid 139 2023-08-27T20:52:05.576 INFO:tasks.rados.rados.0.smithi093.stdout:139: seq_num 138 ranges {719366=478716,1808868=510498} 2023-08-27T20:52:05.583 INFO:tasks.rados.rados.0.smithi093.stdout:139: writing smithi093153278-139 from 719366 to 1198082 tid 1 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:139: writing smithi093153278-139 from 1808868 to 2319366 tid 2 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:124: finishing write tid 2 to smithi093153278-124 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:124: finishing write tid 3 to smithi093153278-124 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:124: finishing write tid 4 to smithi093153278-124 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 124 v 31 (ObjNum 123 snap 0 seq_num 123) dirty exists 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:124: left oid 124 (ObjNum 123 snap 0 seq_num 123) 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:125: finishing write tid 1 to smithi093153278-125 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:125: finishing write tid 2 to smithi093153278-125 2023-08-27T20:52:05.591 INFO:tasks.rados.rados.0.smithi093.stdout:125: finishing write tid 3 to smithi093153278-125 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:125: finishing write tid 4 to smithi093153278-125 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 125 v 40 (ObjNum 124 snap 0 seq_num 124) dirty exists 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:125: left oid 125 (ObjNum 124 snap 0 seq_num 124) 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:126: finishing write tid 1 to smithi093153278-126 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:124: done (15 left) 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:125: done (14 left) 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:140: write initial oid 140 2023-08-27T20:52:05.592 INFO:tasks.rados.rados.0.smithi093.stdout:140: seq_num 139 ranges {736173=553965,1867350=468823} 2023-08-27T20:52:05.598 INFO:tasks.rados.rados.0.smithi093.stdout:140: writing smithi093153278-140 from 736173 to 1290138 tid 1 2023-08-27T20:52:05.604 INFO:tasks.rados.rados.0.smithi093.stdout:140: writing smithi093153278-140 from 1867350 to 2336173 tid 2 2023-08-27T20:52:05.604 INFO:tasks.rados.rados.0.smithi093.stdout:141: write initial oid 141 2023-08-27T20:52:05.605 INFO:tasks.rados.rados.0.smithi093.stdout:141: seq_num 140 ranges {752980=629214,1809479=543501} 2023-08-27T20:52:05.613 INFO:tasks.rados.rados.0.smithi093.stdout:141: writing smithi093153278-141 from 752980 to 1382194 tid 1 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:141: writing smithi093153278-141 from 1809479 to 2352980 tid 2 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:126: finishing write tid 2 to smithi093153278-126 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:126: finishing write tid 3 to smithi093153278-126 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:126: finishing write tid 4 to smithi093153278-126 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 26 (ObjNum 125 snap 0 seq_num 125) dirty exists 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:126: left oid 126 (ObjNum 125 snap 0 seq_num 125) 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:127: finishing write tid 1 to smithi093153278-127 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:126: done (15 left) 2023-08-27T20:52:05.620 INFO:tasks.rados.rados.0.smithi093.stdout:142: write initial oid 142 2023-08-27T20:52:05.621 INFO:tasks.rados.rados.0.smithi093.stdout:142: seq_num 141 ranges {769787=704463,2267961=101826} 2023-08-27T20:52:05.628 INFO:tasks.rados.rados.0.smithi093.stdout:142: writing smithi093153278-142 from 769787 to 1474250 tid 1 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout:142: writing smithi093153278-142 from 2267961 to 2369787 tid 2 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout:127: finishing write tid 2 to smithi093153278-127 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout:127: finishing write tid 3 to smithi093153278-127 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout:127: finishing write tid 4 to smithi093153278-127 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 127 v 35 (ObjNum 126 snap 0 seq_num 126) dirty exists 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout:127: left oid 127 (ObjNum 126 snap 0 seq_num 126) 2023-08-27T20:52:05.635 INFO:tasks.rados.rados.0.smithi093.stdout:128: finishing write tid 1 to smithi093153278-128 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:128: finishing write tid 2 to smithi093153278-128 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:128: finishing write tid 3 to smithi093153278-128 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:128: finishing write tid 4 to smithi093153278-128 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 128 v 28 (ObjNum 127 snap 0 seq_num 127) dirty exists 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:128: left oid 128 (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:129: finishing write tid 1 to smithi093153278-129 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:129: finishing write tid 2 to smithi093153278-129 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:129: finishing write tid 3 to smithi093153278-129 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:129: finishing write tid 4 to smithi093153278-129 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 129 v 42 (ObjNum 128 snap 0 seq_num 128) dirty exists 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:129: left oid 129 (ObjNum 128 snap 0 seq_num 128) 2023-08-27T20:52:05.636 INFO:tasks.rados.rados.0.smithi093.stdout:130: finishing write tid 1 to smithi093153278-130 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:130: finishing write tid 2 to smithi093153278-130 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:130: finishing write tid 3 to smithi093153278-130 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:130: finishing write tid 4 to smithi093153278-130 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 130 v 38 (ObjNum 129 snap 0 seq_num 129) dirty exists 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:130: left oid 130 (ObjNum 129 snap 0 seq_num 129) 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:131: finishing write tid 1 to smithi093153278-131 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:131: finishing write tid 2 to smithi093153278-131 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:131: finishing write tid 3 to smithi093153278-131 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:131: finishing write tid 4 to smithi093153278-131 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 131 v 30 (ObjNum 130 snap 0 seq_num 130) dirty exists 2023-08-27T20:52:05.637 INFO:tasks.rados.rados.0.smithi093.stdout:131: left oid 131 (ObjNum 130 snap 0 seq_num 130) 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:132: finishing write tid 1 to smithi093153278-132 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:132: finishing write tid 2 to smithi093153278-132 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:132: finishing write tid 3 to smithi093153278-132 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:132: finishing write tid 4 to smithi093153278-132 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 132 v 37 (ObjNum 131 snap 0 seq_num 131) dirty exists 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:132: left oid 132 (ObjNum 131 snap 0 seq_num 131) 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:133: finishing write tid 1 to smithi093153278-133 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:133: finishing write tid 2 to smithi093153278-133 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:133: finishing write tid 3 to smithi093153278-133 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:133: finishing write tid 4 to smithi093153278-133 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 133 v 29 (ObjNum 132 snap 0 seq_num 132) dirty exists 2023-08-27T20:52:05.638 INFO:tasks.rados.rados.0.smithi093.stdout:133: left oid 133 (ObjNum 132 snap 0 seq_num 132) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:134: finishing write tid 1 to smithi093153278-134 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:134: finishing write tid 2 to smithi093153278-134 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:127: done (15 left) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:128: done (14 left) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:129: done (13 left) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:130: done (12 left) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:131: done (11 left) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:132: done (10 left) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:133: done (9 left) 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:143: write initial oid 143 2023-08-27T20:52:05.639 INFO:tasks.rados.rados.0.smithi093.stdout:143: seq_num 142 ranges {786594=779712,2326443=60151} 2023-08-27T20:52:05.645 INFO:tasks.rados.rados.0.smithi093.stdout:143: writing smithi093153278-143 from 786594 to 1566306 tid 1 2023-08-27T20:52:05.649 INFO:tasks.rados.rados.0.smithi093.stdout:143: writing smithi093153278-143 from 2326443 to 2386594 tid 2 2023-08-27T20:52:05.649 INFO:tasks.rados.rados.0.smithi093.stdout:144: write initial oid 144 2023-08-27T20:52:05.649 INFO:tasks.rados.rados.0.smithi093.stdout:144: seq_num 143 ranges {403401=454961,1584925=506039} 2023-08-27T20:52:05.654 INFO:tasks.rados.rados.0.smithi093.stdout:144: writing smithi093153278-144 from 403401 to 858362 tid 1 2023-08-27T20:52:05.662 INFO:tasks.rados.rados.0.smithi093.stdout:144: writing smithi093153278-144 from 1584925 to 2090964 tid 2 2023-08-27T20:52:05.663 INFO:tasks.rados.rados.0.smithi093.stdout:145: write initial oid 145 2023-08-27T20:52:05.663 INFO:tasks.rados.rados.0.smithi093.stdout:145: seq_num 144 ranges {420208=530210,1527054=766050} 2023-08-27T20:52:05.668 INFO:tasks.rados.rados.0.smithi093.stdout:145: writing smithi093153278-145 from 420208 to 950418 tid 1 2023-08-27T20:52:05.677 INFO:tasks.rados.rados.0.smithi093.stdout:145: writing smithi093153278-145 from 1527054 to 2293104 tid 2 2023-08-27T20:52:05.677 INFO:tasks.rados.rados.0.smithi093.stdout:146: write initial oid 146 2023-08-27T20:52:05.677 INFO:tasks.rados.rados.0.smithi093.stdout:146: seq_num 145 ranges {437015=721812,1701889=509708} 2023-08-27T20:52:05.685 INFO:tasks.rados.rados.0.smithi093.stdout:146: writing smithi093153278-146 from 437015 to 1158827 tid 1 2023-08-27T20:52:05.691 INFO:tasks.rados.rados.0.smithi093.stdout:146: writing smithi093153278-146 from 1701889 to 2211597 tid 2 2023-08-27T20:52:05.692 INFO:tasks.rados.rados.0.smithi093.stdout:147: write initial oid 147 2023-08-27T20:52:05.692 INFO:tasks.rados.rados.0.smithi093.stdout:147: seq_num 146 ranges {453822=797061,1760371=653366} 2023-08-27T20:52:05.700 INFO:tasks.rados.rados.0.smithi093.stdout:147: writing smithi093153278-147 from 453822 to 1250883 tid 1 2023-08-27T20:52:05.707 INFO:tasks.rados.rados.0.smithi093.stdout:147: writing smithi093153278-147 from 1760371 to 2413737 tid 2 2023-08-27T20:52:05.707 INFO:tasks.rados.rados.0.smithi093.stdout:148: write initial oid 148 2023-08-27T20:52:05.707 INFO:tasks.rados.rados.0.smithi093.stdout:148: seq_num 147 ranges {470629=472310,1418853=513377} 2023-08-27T20:52:05.712 INFO:tasks.rados.rados.0.smithi093.stdout:148: writing smithi093153278-148 from 470629 to 942939 tid 1 2023-08-27T20:52:05.719 INFO:tasks.rados.rados.0.smithi093.stdout:148: writing smithi093153278-148 from 1418853 to 1932230 tid 2 2023-08-27T20:52:05.719 INFO:tasks.rados.rados.0.smithi093.stdout:149: write initial oid 149 2023-08-27T20:52:05.719 INFO:tasks.rados.rados.0.smithi093.stdout:149: seq_num 148 ranges {487436=547559,1760982=657035} 2023-08-27T20:52:05.725 INFO:tasks.rados.rados.0.smithi093.stdout:149: writing smithi093153278-149 from 487436 to 1034995 tid 1 2023-08-27T20:52:05.734 INFO:tasks.rados.rados.0.smithi093.stdout:149: writing smithi093153278-149 from 1760982 to 2418017 tid 2 2023-08-27T20:52:05.734 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.734 INFO:tasks.rados.rados.0.smithi093.stdout:134: finishing write tid 3 to smithi093153278-134 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:134: finishing write tid 4 to smithi093153278-134 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 134 v 34 (ObjNum 133 snap 0 seq_num 133) dirty exists 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:134: left oid 134 (ObjNum 133 snap 0 seq_num 133) 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:135: finishing write tid 1 to smithi093153278-135 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:135: finishing write tid 2 to smithi093153278-135 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:134: done (15 left) 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:150: write initial oid 150 2023-08-27T20:52:05.735 INFO:tasks.rados.rados.0.smithi093.stdout:150: seq_num 149 ranges {504243=622808,1819464=517046} 2023-08-27T20:52:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:52:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:05] "GET /metrics HTTP/1.1" 200 238755 "" "Prometheus/2.43.0" 2023-08-27T20:52:05.741 INFO:tasks.rados.rados.0.smithi093.stdout:150: writing smithi093153278-150 from 504243 to 1127051 tid 1 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:150: writing smithi093153278-150 from 1819464 to 2336510 tid 2 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:135: finishing write tid 3 to smithi093153278-135 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:135: finishing write tid 4 to smithi093153278-135 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 135 v 31 (ObjNum 134 snap 0 seq_num 134) dirty exists 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:135: left oid 135 (ObjNum 134 snap 0 seq_num 134) 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:136: finishing write tid 1 to smithi093153278-136 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:136: finishing write tid 2 to smithi093153278-136 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:136: finishing write tid 3 to smithi093153278-136 2023-08-27T20:52:05.749 INFO:tasks.rados.rados.0.smithi093.stdout:135: done (15 left) 2023-08-27T20:52:05.750 INFO:tasks.rados.rados.0.smithi093.stdout:151: write initial oid 151 2023-08-27T20:52:05.750 INFO:tasks.rados.rados.0.smithi093.stdout:151: seq_num 150 ranges {521050=698057,1877946=643104} 2023-08-27T20:52:05.758 INFO:tasks.rados.rados.0.smithi093.stdout:151: writing smithi093153278-151 from 521050 to 1219107 tid 1 2023-08-27T20:52:05.764 INFO:tasks.rados.rados.0.smithi093.stdout:151: writing smithi093153278-151 from 1877946 to 2521050 tid 2 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout:136: finishing write tid 4 to smithi093153278-136 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 136 v 41 (ObjNum 135 snap 0 seq_num 135) dirty exists 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout:136: left oid 136 (ObjNum 135 snap 0 seq_num 135) 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout:137: finishing write tid 1 to smithi093153278-137 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout:136: done (15 left) 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout:152: write initial oid 152 2023-08-27T20:52:05.765 INFO:tasks.rados.rados.0.smithi093.stdout:152: seq_num 151 ranges {537857=773306,1936428=520715} 2023-08-27T20:52:05.773 INFO:tasks.rados.rados.0.smithi093.stdout:152: writing smithi093153278-152 from 537857 to 1311163 tid 1 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:152: writing smithi093153278-152 from 1936428 to 2457143 tid 2 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:137: finishing write tid 2 to smithi093153278-137 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:137: finishing write tid 3 to smithi093153278-137 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:137: finishing write tid 4 to smithi093153278-137 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 137 v 32 (ObjNum 136 snap 0 seq_num 136) dirty exists 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:137: left oid 137 (ObjNum 136 snap 0 seq_num 136) 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:138: finishing write tid 1 to smithi093153278-138 2023-08-27T20:52:05.781 INFO:tasks.rados.rados.0.smithi093.stdout:138: finishing write tid 2 to smithi093153278-138 2023-08-27T20:52:05.782 INFO:tasks.rados.rados.0.smithi093.stdout:137: done (15 left) 2023-08-27T20:52:05.782 INFO:tasks.rados.rados.0.smithi093.stdout:153: write initial oid 153 2023-08-27T20:52:05.782 INFO:tasks.rados.rados.0.smithi093.stdout:153: seq_num 152 ranges {554664=448555,1478557=664373} 2023-08-27T20:52:05.786 INFO:tasks.rados.rados.0.smithi093.stdout:153: writing smithi093153278-153 from 554664 to 1003219 tid 1 2023-08-27T20:52:05.794 INFO:tasks.rados.rados.0.smithi093.stdout:153: writing smithi093153278-153 from 1478557 to 2142930 tid 2 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:138: finishing write tid 3 to smithi093153278-138 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:138: finishing write tid 4 to smithi093153278-138 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 138 v 35 (ObjNum 137 snap 0 seq_num 137) dirty exists 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:138: left oid 138 (ObjNum 137 snap 0 seq_num 137) 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:139: finishing write tid 1 to smithi093153278-139 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:139: finishing write tid 2 to smithi093153278-139 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:139: finishing write tid 3 to smithi093153278-139 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:139: finishing write tid 4 to smithi093153278-139 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 139 v 40 (ObjNum 138 snap 0 seq_num 138) dirty exists 2023-08-27T20:52:05.795 INFO:tasks.rados.rados.0.smithi093.stdout:139: left oid 139 (ObjNum 138 snap 0 seq_num 138) 2023-08-27T20:52:05.796 INFO:tasks.rados.rados.0.smithi093.stdout:138: done (15 left) 2023-08-27T20:52:05.796 INFO:tasks.rados.rados.0.smithi093.stdout:139: done (14 left) 2023-08-27T20:52:05.796 INFO:tasks.rados.rados.0.smithi093.stdout:154: write initial oid 154 2023-08-27T20:52:05.796 INFO:tasks.rados.rados.0.smithi093.stdout:154: seq_num 153 ranges {571471=640157,1653392=524384} 2023-08-27T20:52:05.802 INFO:tasks.rados.rados.0.smithi093.stdout:154: writing smithi093153278-154 from 571471 to 1211628 tid 1 2023-08-27T20:52:05.808 INFO:tasks.rados.rados.0.smithi093.stdout:154: writing smithi093153278-154 from 1653392 to 2177776 tid 2 2023-08-27T20:52:05.808 INFO:tasks.rados.rados.0.smithi093.stdout:155: write initial oid 155 2023-08-27T20:52:05.808 INFO:tasks.rados.rados.0.smithi093.stdout:155: seq_num 154 ranges {588278=715406,1711874=668042} 2023-08-27T20:52:05.816 INFO:tasks.rados.rados.0.smithi093.stdout:155: writing smithi093153278-155 from 588278 to 1303684 tid 1 2023-08-27T20:52:05.823 INFO:tasks.rados.rados.0.smithi093.stdout:155: writing smithi093153278-155 from 1711874 to 2379916 tid 2 2023-08-27T20:52:05.823 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.824 INFO:tasks.rados.rados.0.smithi093.stdout:140: finishing write tid 1 to smithi093153278-140 2023-08-27T20:52:05.824 INFO:tasks.rados.rados.0.smithi093.stdout:140: finishing write tid 2 to smithi093153278-140 2023-08-27T20:52:05.824 INFO:tasks.rados.rados.0.smithi093.stdout:140: finishing write tid 3 to smithi093153278-140 2023-08-27T20:52:05.824 INFO:tasks.rados.rados.0.smithi093.stdout:140: finishing write tid 4 to smithi093153278-140 2023-08-27T20:52:05.825 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 140 v 33 (ObjNum 139 snap 0 seq_num 139) dirty exists 2023-08-27T20:52:05.825 INFO:tasks.rados.rados.0.smithi093.stdout:140: left oid 140 (ObjNum 139 snap 0 seq_num 139) 2023-08-27T20:52:05.825 INFO:tasks.rados.rados.0.smithi093.stdout:141: finishing write tid 1 to smithi093153278-141 2023-08-27T20:52:05.825 INFO:tasks.rados.rados.0.smithi093.stdout:141: finishing write tid 2 to smithi093153278-141 2023-08-27T20:52:05.825 INFO:tasks.rados.rados.0.smithi093.stdout:140: done (15 left) 2023-08-27T20:52:05.825 INFO:tasks.rados.rados.0.smithi093.stdout:156: write initial oid 156 2023-08-27T20:52:05.825 INFO:tasks.rados.rados.0.smithi093.stdout:156: seq_num 155 ranges {605085=790655,2170356=434729} 2023-08-27T20:52:05.833 INFO:tasks.rados.rados.0.smithi093.stdout:156: writing smithi093153278-156 from 605085 to 1395740 tid 1 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:156: writing smithi093153278-156 from 2170356 to 2605085 tid 2 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:141: finishing write tid 3 to smithi093153278-141 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:141: finishing write tid 4 to smithi093153278-141 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 141 v 43 (ObjNum 140 snap 0 seq_num 140) dirty exists 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:141: left oid 141 (ObjNum 140 snap 0 seq_num 140) 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:142: finishing write tid 1 to smithi093153278-142 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:141: done (15 left) 2023-08-27T20:52:05.841 INFO:tasks.rados.rados.0.smithi093.stdout:157: write initial oid 157 2023-08-27T20:52:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:157: seq_num 156 ranges {621892=465904,1712485=671711} 2023-08-27T20:52:05.847 INFO:tasks.rados.rados.0.smithi093.stdout:157: writing smithi093153278-157 from 621892 to 1087796 tid 1 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:157: writing smithi093153278-157 from 1712485 to 2384196 tid 2 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:142: finishing write tid 2 to smithi093153278-142 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:142: finishing write tid 3 to smithi093153278-142 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:142: finishing write tid 4 to smithi093153278-142 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 142 v 46 (ObjNum 141 snap 0 seq_num 141) dirty exists 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:142: left oid 142 (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:143: finishing write tid 1 to smithi093153278-143 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:143: finishing write tid 2 to smithi093153278-143 2023-08-27T20:52:05.856 INFO:tasks.rados.rados.0.smithi093.stdout:142: done (15 left) 2023-08-27T20:52:05.857 INFO:tasks.rados.rados.0.smithi093.stdout:158: write initial oid 158 2023-08-27T20:52:05.857 INFO:tasks.rados.rados.0.smithi093.stdout:158: seq_num 157 ranges {638699=541153,1770967=531722} 2023-08-27T20:52:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:158: writing smithi093153278-158 from 638699 to 1179852 tid 1 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:158: writing smithi093153278-158 from 1770967 to 2302689 tid 2 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:143: finishing write tid 3 to smithi093153278-143 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:143: finishing write tid 4 to smithi093153278-143 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 143 v 16 (ObjNum 142 snap 0 seq_num 142) dirty exists 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:143: left oid 143 (ObjNum 142 snap 0 seq_num 142) 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:144: finishing write tid 1 to smithi093153278-144 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:144: finishing write tid 2 to smithi093153278-144 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:144: finishing write tid 3 to smithi093153278-144 2023-08-27T20:52:05.870 INFO:tasks.rados.rados.0.smithi093.stdout:144: finishing write tid 4 to smithi093153278-144 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 144 v 22 (ObjNum 143 snap 0 seq_num 143) dirty exists 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:144: left oid 144 (ObjNum 143 snap 0 seq_num 143) 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:145: finishing write tid 1 to smithi093153278-145 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:145: finishing write tid 2 to smithi093153278-145 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:145: finishing write tid 3 to smithi093153278-145 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:145: finishing write tid 4 to smithi093153278-145 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 145 v 37 (ObjNum 144 snap 0 seq_num 144) dirty exists 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:145: left oid 145 (ObjNum 144 snap 0 seq_num 144) 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:146: finishing write tid 1 to smithi093153278-146 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:143: done (15 left) 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:144: done (14 left) 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:145: done (13 left) 2023-08-27T20:52:05.871 INFO:tasks.rados.rados.0.smithi093.stdout:159: write initial oid 159 2023-08-27T20:52:05.872 INFO:tasks.rados.rados.0.smithi093.stdout:159: seq_num 158 ranges {655506=616402,1829449=675380} 2023-08-27T20:52:05.876 INFO:tasks.rados.rados.0.smithi093.stdout:159: writing smithi093153278-159 from 655506 to 1271908 tid 1 2023-08-27T20:52:05.881 INFO:teuthology.orchestra.run.smithi093.stdout:wait for minority of mons to upgrade 2023-08-27T20:52:05.885 INFO:tasks.rados.rados.0.smithi093.stdout:159: writing smithi093153278-159 from 1829449 to 2504829 tid 2 2023-08-27T20:52:05.885 INFO:tasks.rados.rados.0.smithi093.stdout:160: write initial oid 160 2023-08-27T20:52:05.885 INFO:tasks.rados.rados.0.smithi093.stdout:160: seq_num 159 ranges {672313=691651,1887931=419038} 2023-08-27T20:52:05.894 INFO:tasks.rados.rados.0.smithi093.stdout:160: writing smithi093153278-160 from 672313 to 1363964 tid 1 2023-08-27T20:52:05.900 INFO:tasks.rados.rados.0.smithi093.stdout:160: writing smithi093153278-160 from 1887931 to 2306969 tid 2 2023-08-27T20:52:05.900 INFO:tasks.rados.rados.0.smithi093.stdout:161: write initial oid 161 2023-08-27T20:52:05.900 INFO:tasks.rados.rados.0.smithi093.stdout:161: seq_num 160 ranges {689120=483253,1946413=679049} 2023-08-27T20:52:05.907 INFO:tasks.rados.rados.0.smithi093.stdout:161: writing smithi093153278-161 from 689120 to 1172373 tid 1 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:161: writing smithi093153278-161 from 1946413 to 2625462 tid 2 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:146: finishing write tid 2 to smithi093153278-146 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:146: finishing write tid 3 to smithi093153278-146 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:146: finishing write tid 4 to smithi093153278-146 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 146 v 29 (ObjNum 145 snap 0 seq_num 145) dirty exists 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:146: left oid 146 (ObjNum 145 snap 0 seq_num 145) 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:147: finishing write tid 1 to smithi093153278-147 2023-08-27T20:52:05.918 INFO:tasks.rados.rados.0.smithi093.stdout:147: finishing write tid 2 to smithi093153278-147 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:147: finishing write tid 3 to smithi093153278-147 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:147: finishing write tid 4 to smithi093153278-147 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 147 v 34 (ObjNum 146 snap 0 seq_num 146) dirty exists 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:147: left oid 147 (ObjNum 146 snap 0 seq_num 146) 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:148: finishing write tid 1 to smithi093153278-148 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:148: finishing write tid 2 to smithi093153278-148 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:148: finishing write tid 3 to smithi093153278-148 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:148: finishing write tid 4 to smithi093153278-148 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 148 v 37 (ObjNum 147 snap 0 seq_num 147) dirty exists 2023-08-27T20:52:05.919 INFO:tasks.rados.rados.0.smithi093.stdout:148: left oid 148 (ObjNum 147 snap 0 seq_num 147) 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:149: finishing write tid 1 to smithi093153278-149 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:149: finishing write tid 2 to smithi093153278-149 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:146: done (15 left) 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:147: done (14 left) 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:148: done (13 left) 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:162: write initial oid 162 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:149: finishing write tid 3 to smithi093153278-149 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:149: finishing write tid 4 to smithi093153278-149 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 149 v 25 (ObjNum 148 snap 0 seq_num 148) dirty exists 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:149: left oid 149 (ObjNum 148 snap 0 seq_num 148) 2023-08-27T20:52:05.920 INFO:tasks.rados.rados.0.smithi093.stdout:150: finishing write tid 1 to smithi093153278-150 2023-08-27T20:52:05.921 INFO:tasks.rados.rados.0.smithi093.stdout:150: finishing write tid 2 to smithi093153278-150 2023-08-27T20:52:05.921 INFO:tasks.rados.rados.0.smithi093.stdout:162: seq_num 161 ranges {705927=558502,2004895=422707} 2023-08-27T20:52:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:162: writing smithi093153278-162 from 705927 to 1264429 tid 1 2023-08-27T20:52:05.932 INFO:tasks.rados.rados.0.smithi093.stdout:162: writing smithi093153278-162 from 2004895 to 2427602 tid 2 2023-08-27T20:52:05.932 INFO:tasks.rados.rados.0.smithi093.stdout:149: done (13 left) 2023-08-27T20:52:05.932 INFO:tasks.rados.rados.0.smithi093.stdout:163: write initial oid 163 2023-08-27T20:52:05.932 INFO:tasks.rados.rados.0.smithi093.stdout:163: seq_num 162 ranges {722734=633751,2063377=659357} 2023-08-27T20:52:05.940 INFO:tasks.rados.rados.0.smithi093.stdout:163: writing smithi093153278-163 from 722734 to 1356485 tid 1 2023-08-27T20:52:05.949 INFO:tasks.rados.rados.0.smithi093.stdout:163: writing smithi093153278-163 from 2063377 to 2722734 tid 2 2023-08-27T20:52:05.949 INFO:tasks.rados.rados.0.smithi093.stdout:164: write initial oid 164 2023-08-27T20:52:05.949 INFO:tasks.rados.rados.0.smithi093.stdout:164: seq_num 163 ranges {739541=709000,2121859=426376} 2023-08-27T20:52:05.957 INFO:tasks.rados.rados.0.smithi093.stdout:164: writing smithi093153278-164 from 739541 to 1448541 tid 1 2023-08-27T20:52:05.964 INFO:tasks.rados.rados.0.smithi093.stdout:164: writing smithi093153278-164 from 2121859 to 2548235 tid 2 2023-08-27T20:52:05.965 INFO:tasks.rados.rados.0.smithi093.stdout:165: write initial oid 165 2023-08-27T20:52:05.965 INFO:tasks.rados.rados.0.smithi093.stdout:165: seq_num 164 ranges {756348=784249,2063988=686387} 2023-08-27T20:52:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:165: writing smithi093153278-165 from 756348 to 1540597 tid 1 2023-08-27T20:52:05.982 INFO:tasks.rados.rados.0.smithi093.stdout:165: writing smithi093153278-165 from 2063988 to 2750375 tid 2 2023-08-27T20:52:05.982 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.982 INFO:tasks.rados.rados.0.smithi093.stdout:150: finishing write tid 3 to smithi093153278-150 2023-08-27T20:52:05.983 INFO:tasks.rados.rados.0.smithi093.stdout:150: finishing write tid 4 to smithi093153278-150 2023-08-27T20:52:05.983 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 150 v 34 (ObjNum 149 snap 0 seq_num 149) dirty exists 2023-08-27T20:52:05.983 INFO:tasks.rados.rados.0.smithi093.stdout:150: left oid 150 (ObjNum 149 snap 0 seq_num 149) 2023-08-27T20:52:05.983 INFO:tasks.rados.rados.0.smithi093.stdout:151: finishing write tid 1 to smithi093153278-151 2023-08-27T20:52:05.983 INFO:tasks.rados.rados.0.smithi093.stdout:151: finishing write tid 2 to smithi093153278-151 2023-08-27T20:52:05.983 INFO:tasks.rados.rados.0.smithi093.stdout:151: finishing write tid 3 to smithi093153278-151 2023-08-27T20:52:05.983 INFO:tasks.rados.rados.0.smithi093.stdout:150: done (15 left) 2023-08-27T20:52:05.984 INFO:tasks.rados.rados.0.smithi093.stdout:166: write initial oid 166 2023-08-27T20:52:05.984 INFO:tasks.rados.rados.0.smithi093.stdout:166: seq_num 165 ranges {773155=459498,1722470=430045,2648676=124479} 2023-08-27T20:52:05.989 INFO:tasks.rados.rados.0.smithi093.stdout:166: writing smithi093153278-166 from 773155 to 1232653 tid 1 2023-08-27T20:52:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:166: writing smithi093153278-166 from 1722470 to 2152515 tid 2 2023-08-27T20:52:05.999 INFO:tasks.rados.rados.0.smithi093.stdout:166: writing smithi093153278-166 from 2648676 to 2773155 tid 3 2023-08-27T20:52:05.999 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:05.999 INFO:tasks.rados.rados.0.smithi093.stdout:151: finishing write tid 4 to smithi093153278-151 2023-08-27T20:52:05.999 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 151 v 37 (ObjNum 150 snap 0 seq_num 150) dirty exists 2023-08-27T20:52:06.000 INFO:tasks.rados.rados.0.smithi093.stdout:151: left oid 151 (ObjNum 150 snap 0 seq_num 150) 2023-08-27T20:52:06.000 INFO:tasks.rados.rados.0.smithi093.stdout:152: finishing write tid 1 to smithi093153278-152 2023-08-27T20:52:06.000 INFO:tasks.rados.rados.0.smithi093.stdout:151: done (15 left) 2023-08-27T20:52:06.000 INFO:tasks.rados.rados.0.smithi093.stdout:167: write initial oid 167 2023-08-27T20:52:06.000 INFO:tasks.rados.rados.0.smithi093.stdout:167: seq_num 166 ranges {789962=534747,1780952=690056} 2023-08-27T20:52:06.007 INFO:tasks.rados.rados.0.smithi093.stdout:167: writing smithi093153278-167 from 789962 to 1324709 tid 1 2023-08-27T20:52:06.015 INFO:tasks.rados.rados.0.smithi093.stdout:167: writing smithi093153278-167 from 1780952 to 2471008 tid 2 2023-08-27T20:52:06.015 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.015 INFO:tasks.rados.rados.0.smithi093.stdout:152: finishing write tid 2 to smithi093153278-152 2023-08-27T20:52:06.015 INFO:tasks.rados.rados.0.smithi093.stdout:152: finishing write tid 3 to smithi093153278-152 2023-08-27T20:52:06.015 INFO:tasks.rados.rados.0.smithi093.stdout:152: finishing write tid 4 to smithi093153278-152 2023-08-27T20:52:06.015 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 152 v 43 (ObjNum 151 snap 0 seq_num 151) dirty exists 2023-08-27T20:52:06.016 INFO:tasks.rados.rados.0.smithi093.stdout:152: left oid 152 (ObjNum 151 snap 0 seq_num 151) 2023-08-27T20:52:06.016 INFO:tasks.rados.rados.0.smithi093.stdout:153: finishing write tid 1 to smithi093153278-153 2023-08-27T20:52:06.016 INFO:tasks.rados.rados.0.smithi093.stdout:153: finishing write tid 2 to smithi093153278-153 2023-08-27T20:52:06.016 INFO:tasks.rados.rados.0.smithi093.stdout:152: done (15 left) 2023-08-27T20:52:06.016 INFO:tasks.rados.rados.0.smithi093.stdout:168: write initial oid 168 2023-08-27T20:52:06.016 INFO:tasks.rados.rados.0.smithi093.stdout:168: seq_num 167 ranges {406769=609996,1439434=433714,2303522=503247} 2023-08-27T20:52:06.021 INFO:tasks.rados.rados.0.smithi093.stdout:168: writing smithi093153278-168 from 406769 to 1016765 tid 1 2023-08-27T20:52:06.026 INFO:tasks.rados.rados.0.smithi093.stdout:168: writing smithi093153278-168 from 1439434 to 1873148 tid 2 2023-08-27T20:52:06.033 INFO:tasks.rados.rados.0.smithi093.stdout:168: writing smithi093153278-168 from 2303522 to 2806769 tid 3 2023-08-27T20:52:06.033 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.033 INFO:tasks.rados.rados.0.smithi093.stdout:153: finishing write tid 3 to smithi093153278-153 2023-08-27T20:52:06.033 INFO:tasks.rados.rados.0.smithi093.stdout:153: finishing write tid 4 to smithi093153278-153 2023-08-27T20:52:06.033 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 153 v 46 (ObjNum 152 snap 0 seq_num 152) dirty exists 2023-08-27T20:52:06.034 INFO:tasks.rados.rados.0.smithi093.stdout:153: left oid 153 (ObjNum 152 snap 0 seq_num 152) 2023-08-27T20:52:06.034 INFO:tasks.rados.rados.0.smithi093.stdout:154: finishing write tid 1 to smithi093153278-154 2023-08-27T20:52:06.034 INFO:tasks.rados.rados.0.smithi093.stdout:154: finishing write tid 2 to smithi093153278-154 2023-08-27T20:52:06.034 INFO:tasks.rados.rados.0.smithi093.stdout:154: finishing write tid 3 to smithi093153278-154 2023-08-27T20:52:06.034 INFO:tasks.rados.rados.0.smithi093.stdout:153: done (15 left) 2023-08-27T20:52:06.034 INFO:tasks.rados.rados.0.smithi093.stdout:169: write initial oid 169 2023-08-27T20:52:06.034 INFO:tasks.rados.rados.0.smithi093.stdout:169: seq_num 168 ranges {423576=401598,1497916=693725} 2023-08-27T20:52:06.037 INFO:tasks.rados.rados.0.smithi093.stdout:169: writing smithi093153278-169 from 423576 to 825174 tid 1 2023-08-27T20:52:06.046 INFO:tasks.rados.rados.0.smithi093.stdout:169: writing smithi093153278-169 from 1497916 to 2191641 tid 2 2023-08-27T20:52:06.046 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.046 INFO:tasks.rados.rados.0.smithi093.stdout:154: finishing write tid 4 to smithi093153278-154 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 154 v 28 (ObjNum 153 snap 0 seq_num 153) dirty exists 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:154: left oid 154 (ObjNum 153 snap 0 seq_num 153) 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:155: finishing write tid 1 to smithi093153278-155 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:155: finishing write tid 2 to smithi093153278-155 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:155: finishing write tid 3 to smithi093153278-155 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:154: done (15 left) 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:170: write initial oid 170 2023-08-27T20:52:06.047 INFO:tasks.rados.rados.0.smithi093.stdout:170: seq_num 169 ranges {440383=476847,1556398=437383,2474721=365662} 2023-08-27T20:52:06.051 INFO:tasks.rados.rados.0.smithi093.stdout:170: writing smithi093153278-170 from 440383 to 917230 tid 1 2023-08-27T20:52:06.058 INFO:tasks.rados.rados.0.smithi093.stdout:170: writing smithi093153278-170 from 1556398 to 1993781 tid 2 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout:170: writing smithi093153278-170 from 2474721 to 2840383 tid 3 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout:155: finishing write tid 4 to smithi093153278-155 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 155 v 19 (ObjNum 154 snap 0 seq_num 154) dirty exists 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout:155: left oid 155 (ObjNum 154 snap 0 seq_num 154) 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout:156: finishing write tid 1 to smithi093153278-156 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout:155: done (15 left) 2023-08-27T20:52:06.064 INFO:tasks.rados.rados.0.smithi093.stdout:171: write initial oid 171 2023-08-27T20:52:06.065 INFO:tasks.rados.rados.0.smithi093.stdout:171: seq_num 170 ranges {457190=552096,1614880=581041,2785791=71399} 2023-08-27T20:52:06.070 INFO:tasks.rados.rados.0.smithi093.stdout:171: writing smithi093153278-171 from 457190 to 1009286 tid 1 2023-08-27T20:52:06.077 INFO:tasks.rados.rados.0.smithi093.stdout:171: writing smithi093153278-171 from 1614880 to 2195921 tid 2 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout:171: writing smithi093153278-171 from 2785791 to 2857190 tid 3 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout:156: finishing write tid 2 to smithi093153278-156 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout:156: finishing write tid 3 to smithi093153278-156 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout:156: finishing write tid 4 to smithi093153278-156 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 156 v 32 (ObjNum 155 snap 0 seq_num 155) dirty exists 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout:156: left oid 156 (ObjNum 155 snap 0 seq_num 155) 2023-08-27T20:52:06.082 INFO:tasks.rados.rados.0.smithi093.stdout:157: finishing write tid 1 to smithi093153278-157 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:157: finishing write tid 2 to smithi093153278-157 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:157: finishing write tid 3 to smithi093153278-157 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:157: finishing write tid 4 to smithi093153278-157 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 157 v 44 (ObjNum 156 snap 0 seq_num 156) dirty exists 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:157: left oid 157 (ObjNum 156 snap 0 seq_num 156) 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:158: finishing write tid 1 to smithi093153278-158 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:158: finishing write tid 2 to smithi093153278-158 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:158: finishing write tid 3 to smithi093153278-158 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:156: done (15 left) 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:157: done (14 left) 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:172: write initial oid 172 2023-08-27T20:52:06.083 INFO:tasks.rados.rados.0.smithi093.stdout:172: seq_num 171 ranges {473997=627345,1673362=441052,2529567=344430} 2023-08-27T20:52:06.088 INFO:tasks.rados.rados.0.smithi093.stdout:172: writing smithi093153278-172 from 473997 to 1101342 tid 1 2023-08-27T20:52:06.094 INFO:tasks.rados.rados.0.smithi093.stdout:172: writing smithi093153278-172 from 1673362 to 2114414 tid 2 2023-08-27T20:52:06.099 INFO:tasks.rados.rados.0.smithi093.stdout:172: writing smithi093153278-172 from 2529567 to 2873997 tid 3 2023-08-27T20:52:06.099 INFO:tasks.rados.rados.0.smithi093.stdout:173: write initial oid 173 2023-08-27T20:52:06.100 INFO:tasks.rados.rados.0.smithi093.stdout:173: seq_num 172 ranges {490804=702594,1615491=584710,2724284=166520} 2023-08-27T20:52:06.106 INFO:tasks.rados.rados.0.smithi093.stdout:173: writing smithi093153278-173 from 490804 to 1193398 tid 1 2023-08-27T20:52:06.113 INFO:tasks.rados.rados.0.smithi093.stdout:173: writing smithi093153278-173 from 1615491 to 2200201 tid 2 2023-08-27T20:52:06.118 INFO:tasks.rados.rados.0.smithi093.stdout:173: writing smithi093153278-173 from 2724284 to 2890804 tid 3 2023-08-27T20:52:06.118 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.118 INFO:tasks.rados.rados.0.smithi093.stdout:158: finishing write tid 4 to smithi093153278-158 2023-08-27T20:52:06.118 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 158 v 19 (ObjNum 157 snap 0 seq_num 157) dirty exists 2023-08-27T20:52:06.118 INFO:tasks.rados.rados.0.smithi093.stdout:158: left oid 158 (ObjNum 157 snap 0 seq_num 157) 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:159: finishing write tid 1 to smithi093153278-159 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:159: finishing write tid 2 to smithi093153278-159 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:159: finishing write tid 3 to smithi093153278-159 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:159: finishing write tid 4 to smithi093153278-159 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 159 v 40 (ObjNum 158 snap 0 seq_num 158) dirty exists 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:159: left oid 159 (ObjNum 158 snap 0 seq_num 158) 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:160: finishing write tid 1 to smithi093153278-160 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:160: finishing write tid 2 to smithi093153278-160 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:160: finishing write tid 3 to smithi093153278-160 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:160: finishing write tid 4 to smithi093153278-160 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 160 v 38 (ObjNum 159 snap 0 seq_num 159) dirty exists 2023-08-27T20:52:06.119 INFO:tasks.rados.rados.0.smithi093.stdout:160: left oid 160 (ObjNum 159 snap 0 seq_num 159) 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:161: finishing write tid 1 to smithi093153278-161 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:161: finishing write tid 2 to smithi093153278-161 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:161: finishing write tid 3 to smithi093153278-161 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:161: finishing write tid 4 to smithi093153278-161 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 161 v 21 (ObjNum 160 snap 0 seq_num 160) dirty exists 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:161: left oid 161 (ObjNum 160 snap 0 seq_num 160) 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:162: finishing write tid 1 to smithi093153278-162 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:162: finishing write tid 2 to smithi093153278-162 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:162: finishing write tid 3 to smithi093153278-162 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:162: finishing write tid 4 to smithi093153278-162 2023-08-27T20:52:06.120 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 162 v 45 (ObjNum 161 snap 0 seq_num 161) dirty exists 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:162: left oid 162 (ObjNum 161 snap 0 seq_num 161) 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:163: finishing write tid 1 to smithi093153278-163 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:163: finishing write tid 2 to smithi093153278-163 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:158: done (15 left) 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:159: done (14 left) 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:160: done (13 left) 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:161: done (12 left) 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:162: done (11 left) 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:174: write initial oid 174 2023-08-27T20:52:06.121 INFO:tasks.rados.rados.0.smithi093.stdout:174: seq_num 173 ranges {507611=777843,2073973=444721} 2023-08-27T20:52:06.125 INFO:tasks.rados.rados.0.smithi093.stdout:174: writing smithi093153278-174 from 507611 to 1285454 tid 1 2023-08-27T20:52:06.133 INFO:tasks.rados.rados.0.smithi093.stdout:174: writing smithi093153278-174 from 2073973 to 2518694 tid 2 2023-08-27T20:52:06.133 INFO:tasks.rados.rados.0.smithi093.stdout:175: write initial oid 175 2023-08-27T20:52:06.133 INFO:tasks.rados.rados.0.smithi093.stdout:175: seq_num 174 ranges {524418=453092,1732455=588379,2779130=145288} 2023-08-27T20:52:06.139 INFO:tasks.rados.rados.0.smithi093.stdout:175: writing smithi093153278-175 from 524418 to 977510 tid 1 2023-08-27T20:52:06.147 INFO:tasks.rados.rados.0.smithi093.stdout:175: writing smithi093153278-175 from 1732455 to 2320834 tid 2 2023-08-27T20:52:06.152 INFO:tasks.rados.rados.0.smithi093.stdout:175: writing smithi093153278-175 from 2779130 to 2924418 tid 3 2023-08-27T20:52:06.152 INFO:tasks.rados.rados.0.smithi093.stdout:176: write initial oid 176 2023-08-27T20:52:06.152 INFO:tasks.rados.rados.0.smithi093.stdout:176: seq_num 175 ranges {541225=644694,1907290=448390} 2023-08-27T20:52:06.158 INFO:tasks.rados.rados.0.smithi093.stdout:176: writing smithi093153278-176 from 541225 to 1185919 tid 1 2023-08-27T20:52:06.166 INFO:tasks.rados.rados.0.smithi093.stdout:176: writing smithi093153278-176 from 1907290 to 2355680 tid 2 2023-08-27T20:52:06.166 INFO:tasks.rados.rados.0.smithi093.stdout:177: write initial oid 177 2023-08-27T20:52:06.166 INFO:tasks.rados.rados.0.smithi093.stdout:177: seq_num 176 ranges {558032=719943,1849419=592048} 2023-08-27T20:52:06.174 INFO:tasks.rados.rados.0.smithi093.stdout:177: writing smithi093153278-177 from 558032 to 1277975 tid 1 2023-08-27T20:52:06.182 INFO:tasks.rados.rados.0.smithi093.stdout:177: writing smithi093153278-177 from 1849419 to 2441467 tid 2 2023-08-27T20:52:06.182 INFO:tasks.rados.rados.0.smithi093.stdout:178: write initial oid 178 2023-08-27T20:52:06.182 INFO:tasks.rados.rados.0.smithi093.stdout:163: finishing write tid 3 to smithi093153278-163 2023-08-27T20:52:06.182 INFO:tasks.rados.rados.0.smithi093.stdout:163: finishing write tid 4 to smithi093153278-163 2023-08-27T20:52:06.182 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 163 v 49 (ObjNum 162 snap 0 seq_num 162) dirty exists 2023-08-27T20:52:06.182 INFO:tasks.rados.rados.0.smithi093.stdout:163: left oid 163 (ObjNum 162 snap 0 seq_num 162) 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:164: finishing write tid 1 to smithi093153278-164 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:164: finishing write tid 2 to smithi093153278-164 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:164: finishing write tid 3 to smithi093153278-164 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:164: finishing write tid 4 to smithi093153278-164 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 164 v 40 (ObjNum 163 snap 0 seq_num 163) dirty exists 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:164: left oid 164 (ObjNum 163 snap 0 seq_num 163) 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:165: finishing write tid 1 to smithi093153278-165 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:165: finishing write tid 2 to smithi093153278-165 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:165: finishing write tid 3 to smithi093153278-165 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:165: finishing write tid 4 to smithi093153278-165 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 165 v 41 (ObjNum 164 snap 0 seq_num 164) dirty exists 2023-08-27T20:52:06.183 INFO:tasks.rados.rados.0.smithi093.stdout:165: left oid 165 (ObjNum 164 snap 0 seq_num 164) 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:166: finishing write tid 1 to smithi093153278-166 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:166: finishing write tid 2 to smithi093153278-166 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:166: finishing write tid 3 to smithi093153278-166 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:166: finishing write tid 4 to smithi093153278-166 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:166: finishing write tid 5 to smithi093153278-166 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 166 v 37 (ObjNum 165 snap 0 seq_num 165) dirty exists 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:166: left oid 166 (ObjNum 165 snap 0 seq_num 165) 2023-08-27T20:52:06.184 INFO:tasks.rados.rados.0.smithi093.stdout:178: seq_num 177 ranges {574839=795192,1907901=452059} 2023-08-27T20:52:06.190 INFO:tasks.rados.rados.0.smithi093.stdout:178: writing smithi093153278-178 from 574839 to 1370031 tid 1 2023-08-27T20:52:06.197 INFO:tasks.rados.rados.0.smithi093.stdout:178: writing smithi093153278-178 from 1907901 to 2359960 tid 2 2023-08-27T20:52:06.197 INFO:tasks.rados.rados.0.smithi093.stdout:163: done (15 left) 2023-08-27T20:52:06.197 INFO:tasks.rados.rados.0.smithi093.stdout:164: done (14 left) 2023-08-27T20:52:06.197 INFO:tasks.rados.rados.0.smithi093.stdout:165: done (13 left) 2023-08-27T20:52:06.197 INFO:tasks.rados.rados.0.smithi093.stdout:166: done (12 left) 2023-08-27T20:52:06.197 INFO:tasks.rados.rados.0.smithi093.stdout:179: write initial oid 179 2023-08-27T20:52:06.197 INFO:tasks.rados.rados.0.smithi093.stdout:179: seq_num 178 ranges {591646=470441,1566383=595717,2888822=102824} 2023-08-27T20:52:06.203 INFO:tasks.rados.rados.0.smithi093.stdout:179: writing smithi093153278-179 from 591646 to 1062087 tid 1 2023-08-27T20:52:06.210 INFO:tasks.rados.rados.0.smithi093.stdout:179: writing smithi093153278-179 from 1566383 to 2162100 tid 2 2023-08-27T20:52:06.215 INFO:tasks.rados.rados.0.smithi093.stdout:179: writing smithi093153278-179 from 2888822 to 2991646 tid 3 2023-08-27T20:52:06.215 INFO:tasks.rados.rados.0.smithi093.stdout:180: write initial oid 180 2023-08-27T20:52:06.215 INFO:tasks.rados.rados.0.smithi093.stdout:180: seq_num 179 ranges {608453=545690,1624865=455728,2632598=375855} 2023-08-27T20:52:06.221 INFO:tasks.rados.rados.0.smithi093.stdout:180: writing smithi093153278-180 from 608453 to 1154143 tid 1 2023-08-27T20:52:06.227 INFO:tasks.rados.rados.0.smithi093.stdout:180: writing smithi093153278-180 from 1624865 to 2080593 tid 2 2023-08-27T20:52:06.232 INFO:tasks.rados.rados.0.smithi093.stdout:180: writing smithi093153278-180 from 2632598 to 3008453 tid 3 2023-08-27T20:52:06.232 INFO:tasks.rados.rados.0.smithi093.stdout:181: write initial oid 181 2023-08-27T20:52:06.232 INFO:tasks.rados.rados.0.smithi093.stdout:181: seq_num 180 ranges {625260=620939,1683347=599386,2943668=81592} 2023-08-27T20:52:06.239 INFO:tasks.rados.rados.0.smithi093.stdout:181: writing smithi093153278-181 from 625260 to 1246199 tid 1 2023-08-27T20:52:06.246 INFO:tasks.rados.rados.0.smithi093.stdout:181: writing smithi093153278-181 from 1683347 to 2282733 tid 2 2023-08-27T20:52:06.250 INFO:tasks.rados.rados.0.smithi093.stdout:181: writing smithi093153278-181 from 2943668 to 3025260 tid 3 2023-08-27T20:52:06.250 INFO:tasks.rados.rados.0.smithi093.stdout:182: write initial oid 182 2023-08-27T20:52:06.250 INFO:tasks.rados.rados.0.smithi093.stdout:167: finishing write tid 1 to smithi093153278-167 2023-08-27T20:52:06.250 INFO:tasks.rados.rados.0.smithi093.stdout:167: finishing write tid 2 to smithi093153278-167 2023-08-27T20:52:06.251 INFO:tasks.rados.rados.0.smithi093.stdout:167: finishing write tid 3 to smithi093153278-167 2023-08-27T20:52:06.251 INFO:tasks.rados.rados.0.smithi093.stdout:182: seq_num 181 ranges {642067=696188,2025476=459397,2971091=70976} 2023-08-27T20:52:06.258 INFO:tasks.rados.rados.0.smithi093.stdout:182: writing smithi093153278-182 from 642067 to 1338255 tid 1 2023-08-27T20:52:06.266 INFO:tasks.rados.rados.0.smithi093.stdout:182: writing smithi093153278-182 from 2025476 to 2484873 tid 2 2023-08-27T20:52:06.269 INFO:tasks.rados.rados.0.smithi093.stdout:182: writing smithi093153278-182 from 2971091 to 3042067 tid 3 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout:167: finishing write tid 4 to smithi093153278-167 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 167 v 37 (ObjNum 166 snap 0 seq_num 166) dirty exists 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout:167: left oid 167 (ObjNum 166 snap 0 seq_num 166) 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout:168: finishing write tid 1 to smithi093153278-168 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout:167: done (15 left) 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout:183: write initial oid 183 2023-08-27T20:52:06.270 INFO:tasks.rados.rados.0.smithi093.stdout:183: seq_num 182 ranges {658874=771437,2083958=603055} 2023-08-27T20:52:06.278 INFO:tasks.rados.rados.0.smithi093.stdout:183: writing smithi093153278-183 from 658874 to 1430311 tid 1 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:183: writing smithi093153278-183 from 2083958 to 2687013 tid 2 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:168: finishing write tid 2 to smithi093153278-168 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:168: finishing write tid 3 to smithi093153278-168 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:168: finishing write tid 4 to smithi093153278-168 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:168: finishing write tid 5 to smithi093153278-168 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 168 v 32 (ObjNum 167 snap 0 seq_num 167) dirty exists 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:168: left oid 168 (ObjNum 167 snap 0 seq_num 167) 2023-08-27T20:52:06.287 INFO:tasks.rados.rados.0.smithi093.stdout:169: finishing write tid 1 to smithi093153278-169 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:169: finishing write tid 2 to smithi093153278-169 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:169: finishing write tid 3 to smithi093153278-169 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:169: finishing write tid 4 to smithi093153278-169 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 169 v 35 (ObjNum 168 snap 0 seq_num 168) dirty exists 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:169: left oid 169 (ObjNum 168 snap 0 seq_num 168) 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:170: finishing write tid 1 to smithi093153278-170 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:170: finishing write tid 2 to smithi093153278-170 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:168: done (15 left) 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:169: done (14 left) 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:184: write initial oid 184 2023-08-27T20:52:06.288 INFO:tasks.rados.rados.0.smithi093.stdout:184: seq_num 183 ranges {675681=563039,1858793=746713,3025937=49744} 2023-08-27T20:52:06.294 INFO:tasks.rados.rados.0.smithi093.stdout:184: writing smithi093153278-184 from 675681 to 1238720 tid 1 2023-08-27T20:52:06.303 INFO:tasks.rados.rados.0.smithi093.stdout:184: writing smithi093153278-184 from 1858793 to 2605506 tid 2 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:184: writing smithi093153278-184 from 3025937 to 3075681 tid 3 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:185: write initial oid 185 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:170: finishing write tid 3 to smithi093153278-170 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:170: finishing write tid 4 to smithi093153278-170 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:170: finishing write tid 5 to smithi093153278-170 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 170 v 23 (ObjNum 169 snap 0 seq_num 169) dirty exists 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:170: left oid 170 (ObjNum 169 snap 0 seq_num 169) 2023-08-27T20:52:06.306 INFO:tasks.rados.rados.0.smithi093.stdout:171: finishing write tid 1 to smithi093153278-171 2023-08-27T20:52:06.307 INFO:tasks.rados.rados.0.smithi093.stdout:171: finishing write tid 2 to smithi093153278-171 2023-08-27T20:52:06.307 INFO:tasks.rados.rados.0.smithi093.stdout:185: seq_num 184 ranges {692488=638288,1917275=606724} 2023-08-27T20:52:06.314 INFO:tasks.rados.rados.0.smithi093.stdout:185: writing smithi093153278-185 from 692488 to 1330776 tid 1 2023-08-27T20:52:06.322 INFO:tasks.rados.rados.0.smithi093.stdout:185: writing smithi093153278-185 from 1917275 to 2523999 tid 2 2023-08-27T20:52:06.322 INFO:tasks.rados.rados.0.smithi093.stdout:170: done (15 left) 2023-08-27T20:52:06.322 INFO:tasks.rados.rados.0.smithi093.stdout:186: write initial oid 186 2023-08-27T20:52:06.322 INFO:tasks.rados.rados.0.smithi093.stdout:186: seq_num 185 ranges {709295=713537,1859404=750382} 2023-08-27T20:52:06.331 INFO:tasks.rados.rados.0.smithi093.stdout:186: writing smithi093153278-186 from 709295 to 1422832 tid 1 2023-08-27T20:52:06.340 INFO:tasks.rados.rados.0.smithi093.stdout:186: writing smithi093153278-186 from 1859404 to 2609786 tid 2 2023-08-27T20:52:06.340 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.340 INFO:tasks.rados.rados.0.smithi093.stdout:171: finishing write tid 3 to smithi093153278-171 2023-08-27T20:52:06.340 INFO:tasks.rados.rados.0.smithi093.stdout:171: finishing write tid 4 to smithi093153278-171 2023-08-27T20:52:06.340 INFO:tasks.rados.rados.0.smithi093.stdout:171: finishing write tid 5 to smithi093153278-171 2023-08-27T20:52:06.340 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 171 v 36 (ObjNum 170 snap 0 seq_num 170) dirty exists 2023-08-27T20:52:06.340 INFO:tasks.rados.rados.0.smithi093.stdout:171: left oid 171 (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:172: finishing write tid 1 to smithi093153278-172 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:172: finishing write tid 2 to smithi093153278-172 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:172: finishing write tid 3 to smithi093153278-172 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:172: finishing write tid 4 to smithi093153278-172 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:172: finishing write tid 5 to smithi093153278-172 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 172 v 41 (ObjNum 171 snap 0 seq_num 171) dirty exists 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:172: left oid 172 (ObjNum 171 snap 0 seq_num 171) 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:173: finishing write tid 1 to smithi093153278-173 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:173: finishing write tid 2 to smithi093153278-173 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:173: finishing write tid 3 to smithi093153278-173 2023-08-27T20:52:06.341 INFO:tasks.rados.rados.0.smithi093.stdout:173: finishing write tid 4 to smithi093153278-173 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:173: finishing write tid 5 to smithi093153278-173 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 173 v 53 (ObjNum 172 snap 0 seq_num 172) dirty exists 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:173: left oid 173 (ObjNum 172 snap 0 seq_num 172) 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:174: finishing write tid 1 to smithi093153278-174 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:174: finishing write tid 2 to smithi093153278-174 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:174: finishing write tid 3 to smithi093153278-174 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:174: finishing write tid 4 to smithi093153278-174 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 174 v 43 (ObjNum 173 snap 0 seq_num 173) dirty exists 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:174: left oid 174 (ObjNum 173 snap 0 seq_num 173) 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:171: done (15 left) 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:172: done (14 left) 2023-08-27T20:52:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:173: done (13 left) 2023-08-27T20:52:06.343 INFO:tasks.rados.rados.0.smithi093.stdout:174: done (12 left) 2023-08-27T20:52:06.343 INFO:tasks.rados.rados.0.smithi093.stdout:187: write initial oid 187 2023-08-27T20:52:06.343 INFO:tasks.rados.rados.0.smithi093.stdout:187: seq_num 186 ranges {726102=788786,1917886=610393,3108206=17896} 2023-08-27T20:52:06.344 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell -c /etc/ceph/ceph.conf -k /etc/ceph/ceph.client.admin.keyring --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -e sha1=9cf208207dfe23ffccb33308b22e51570cb45e51 -- bash -c 'while ! ceph mon versions | grep $sha1 ; do sleep 2 ; done' 2023-08-27T20:52:06.349 INFO:tasks.rados.rados.0.smithi093.stdout:187: writing smithi093153278-187 from 726102 to 1514888 tid 1 2023-08-27T20:52:06.354 INFO:tasks.rados.rados.0.smithi093.stdout:187: writing smithi093153278-187 from 1917886 to 2528279 tid 2 2023-08-27T20:52:06.358 INFO:tasks.rados.rados.0.smithi093.stdout:187: writing smithi093153278-187 from 3108206 to 3126102 tid 3 2023-08-27T20:52:06.358 INFO:tasks.rados.rados.0.smithi093.stdout:188: write initial oid 188 2023-08-27T20:52:06.358 INFO:tasks.rados.rados.0.smithi093.stdout:188: seq_num 187 ranges {742909=464035,1976368=754051} 2023-08-27T20:52:06.365 INFO:tasks.rados.rados.0.smithi093.stdout:188: writing smithi093153278-188 from 742909 to 1206944 tid 1 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:188: writing smithi093153278-188 from 1976368 to 2730419 tid 2 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:189: write initial oid 189 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:175: finishing write tid 1 to smithi093153278-175 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:175: finishing write tid 2 to smithi093153278-175 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:175: finishing write tid 3 to smithi093153278-175 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:175: finishing write tid 4 to smithi093153278-175 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:175: finishing write tid 5 to smithi093153278-175 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 175 v 44 (ObjNum 174 snap 0 seq_num 174) dirty exists 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:175: left oid 175 (ObjNum 174 snap 0 seq_num 174) 2023-08-27T20:52:06.376 INFO:tasks.rados.rados.0.smithi093.stdout:176: finishing write tid 1 to smithi093153278-176 2023-08-27T20:52:06.377 INFO:tasks.rados.rados.0.smithi093.stdout:189: seq_num 188 ranges {759716=539284,2034850=614062} 2023-08-27T20:52:06.383 INFO:tasks.rados.rados.0.smithi093.stdout:189: writing smithi093153278-189 from 759716 to 1299000 tid 1 2023-08-27T20:52:06.392 INFO:tasks.rados.rados.0.smithi093.stdout:189: writing smithi093153278-189 from 2034850 to 2648912 tid 2 2023-08-27T20:52:06.392 INFO:tasks.rados.rados.0.smithi093.stdout:175: done (14 left) 2023-08-27T20:52:06.392 INFO:tasks.rados.rados.0.smithi093.stdout:190: write initial oid 190 2023-08-27T20:52:06.392 INFO:tasks.rados.rados.0.smithi093.stdout:190: seq_num 189 ranges {776523=614533,1976979=757720} 2023-08-27T20:52:06.400 INFO:tasks.rados.rados.0.smithi093.stdout:190: writing smithi093153278-190 from 776523 to 1391056 tid 1 2023-08-27T20:52:06.409 INFO:tasks.rados.rados.0.smithi093.stdout:190: writing smithi093153278-190 from 1976979 to 2734699 tid 2 2023-08-27T20:52:06.409 INFO:tasks.rados.rados.0.smithi093.stdout:191: write initial oid 191 2023-08-27T20:52:06.409 INFO:tasks.rados.rados.0.smithi093.stdout:191: seq_num 190 ranges {793330=689782,2035461=617731,3101545=91785} 2023-08-27T20:52:06.417 INFO:tasks.rados.rados.0.smithi093.stdout:191: writing smithi093153278-191 from 793330 to 1483112 tid 1 2023-08-27T20:52:06.423 INFO:tasks.rados.rados.0.smithi093.stdout:191: writing smithi093153278-191 from 2035461 to 2653192 tid 2 2023-08-27T20:52:06.427 INFO:tasks.rados.rados.0.smithi093.stdout:191: writing smithi093153278-191 from 3101545 to 3193330 tid 3 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:176: finishing write tid 2 to smithi093153278-176 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:176: finishing write tid 3 to smithi093153278-176 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:176: finishing write tid 4 to smithi093153278-176 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 176 v 31 (ObjNum 175 snap 0 seq_num 175) dirty exists 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:176: left oid 176 (ObjNum 175 snap 0 seq_num 175) 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:177: finishing write tid 1 to smithi093153278-177 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:177: finishing write tid 2 to smithi093153278-177 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:177: finishing write tid 3 to smithi093153278-177 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:177: finishing write tid 4 to smithi093153278-177 2023-08-27T20:52:06.428 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 177 v 26 (ObjNum 176 snap 0 seq_num 176) dirty exists 2023-08-27T20:52:06.429 INFO:tasks.rados.rados.0.smithi093.stdout:177: left oid 177 (ObjNum 176 snap 0 seq_num 176) 2023-08-27T20:52:06.429 INFO:tasks.rados.rados.0.smithi093.stdout:178: finishing write tid 1 to smithi093153278-178 2023-08-27T20:52:06.429 INFO:tasks.rados.rados.0.smithi093.stdout:176: done (15 left) 2023-08-27T20:52:06.429 INFO:tasks.rados.rados.0.smithi093.stdout:177: done (14 left) 2023-08-27T20:52:06.429 INFO:tasks.rados.rados.0.smithi093.stdout:192: write initial oid 192 2023-08-27T20:52:06.429 INFO:tasks.rados.rados.0.smithi093.stdout:192: seq_num 191 ranges {410137=481384,1410296=761389,2728968=481169} 2023-08-27T20:52:06.432 INFO:tasks.rados.rados.0.smithi093.stdout:192: writing smithi093153278-192 from 410137 to 891521 tid 1 2023-08-27T20:52:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:06 smithi136 ceph-mon[113531]: pgmap v57: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:06.441 INFO:tasks.rados.rados.0.smithi093.stdout:192: writing smithi093153278-192 from 1410296 to 2171685 tid 2 2023-08-27T20:52:06.447 INFO:tasks.rados.rados.0.smithi093.stdout:192: writing smithi093153278-192 from 2728968 to 3210137 tid 3 2023-08-27T20:52:06.447 INFO:tasks.rados.rados.0.smithi093.stdout:193: write initial oid 193 2023-08-27T20:52:06.448 INFO:tasks.rados.rados.0.smithi093.stdout:193: seq_num 192 ranges {426944=556633,1468778=621400,2872744=354200} 2023-08-27T20:52:06.453 INFO:tasks.rados.rados.0.smithi093.stdout:193: writing smithi093153278-193 from 426944 to 983577 tid 1 2023-08-27T20:52:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:193: writing smithi093153278-193 from 1468778 to 2090178 tid 2 2023-08-27T20:52:06.468 INFO:tasks.rados.rados.0.smithi093.stdout:193: writing smithi093153278-193 from 2872744 to 3226944 tid 3 2023-08-27T20:52:06.468 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:178: finishing write tid 2 to smithi093153278-178 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:178: finishing write tid 3 to smithi093153278-178 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:178: finishing write tid 4 to smithi093153278-178 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 178 v 24 (ObjNum 177 snap 0 seq_num 177) dirty exists 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:178: left oid 178 (ObjNum 177 snap 0 seq_num 177) 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:179: finishing write tid 1 to smithi093153278-179 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:179: finishing write tid 2 to smithi093153278-179 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:179: finishing write tid 3 to smithi093153278-179 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:179: finishing write tid 4 to smithi093153278-179 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:178: done (15 left) 2023-08-27T20:52:06.469 INFO:tasks.rados.rados.0.smithi093.stdout:194: write initial oid 194 2023-08-27T20:52:06.470 INFO:tasks.rados.rados.0.smithi093.stdout:194: seq_num 193 ranges {443751=631882,1810907=765058,3067461=176290} 2023-08-27T20:52:06.475 INFO:tasks.rados.rados.0.smithi093.stdout:194: writing smithi093153278-194 from 443751 to 1075633 tid 1 2023-08-27T20:52:06.484 INFO:tasks.rados.rados.0.smithi093.stdout:194: writing smithi093153278-194 from 1810907 to 2575965 tid 2 2023-08-27T20:52:06.488 INFO:tasks.rados.rados.0.smithi093.stdout:194: writing smithi093153278-194 from 3067461 to 3243751 tid 3 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:179: finishing write tid 5 to smithi093153278-179 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 179 v 39 (ObjNum 178 snap 0 seq_num 178) dirty exists 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:179: left oid 179 (ObjNum 178 snap 0 seq_num 178) 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:180: finishing write tid 1 to smithi093153278-180 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:180: finishing write tid 2 to smithi093153278-180 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:180: finishing write tid 3 to smithi093153278-180 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:180: finishing write tid 4 to smithi093153278-180 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:180: finishing write tid 5 to smithi093153278-180 2023-08-27T20:52:06.489 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 180 v 30 (ObjNum 179 snap 0 seq_num 179) dirty exists 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:180: left oid 180 (ObjNum 179 snap 0 seq_num 179) 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:181: finishing write tid 1 to smithi093153278-181 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:181: finishing write tid 2 to smithi093153278-181 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:181: finishing write tid 3 to smithi093153278-181 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:181: finishing write tid 4 to smithi093153278-181 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:179: done (15 left) 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:180: done (14 left) 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:195: write initial oid 195 2023-08-27T20:52:06.490 INFO:tasks.rados.rados.0.smithi093.stdout:195: seq_num 194 ranges {460558=707131,1869389=508716,3094884=165674} 2023-08-27T20:52:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:06 smithi093 ceph-mon[111335]: pgmap v57: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:06.495 INFO:tasks.rados.rados.0.smithi093.stdout:195: writing smithi093153278-195 from 460558 to 1167689 tid 1 2023-08-27T20:52:06.498 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:06 smithi093 ceph-mon[118902]: pgmap v57: 89 pgs: 89 active+clean; 579 KiB data, 52 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:52:06.503 INFO:tasks.rados.rados.0.smithi093.stdout:195: writing smithi093153278-195 from 1869389 to 2378105 tid 2 2023-08-27T20:52:06.508 INFO:tasks.rados.rados.0.smithi093.stdout:195: writing smithi093153278-195 from 3094884 to 3260558 tid 3 2023-08-27T20:52:06.508 INFO:tasks.rados.rados.0.smithi093.stdout:196: write initial oid 196 2023-08-27T20:52:06.508 INFO:tasks.rados.rados.0.smithi093.stdout:196: seq_num 195 ranges {477365=782380,1927871=768727,3122307=155058} 2023-08-27T20:52:06.516 INFO:tasks.rados.rados.0.smithi093.stdout:196: writing smithi093153278-196 from 477365 to 1259745 tid 1 2023-08-27T20:52:06.526 INFO:tasks.rados.rados.0.smithi093.stdout:196: writing smithi093153278-196 from 1927871 to 2696598 tid 2 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:196: writing smithi093153278-196 from 3122307 to 3277365 tid 3 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:181: finishing write tid 5 to smithi093153278-181 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 181 v 40 (ObjNum 180 snap 0 seq_num 180) dirty exists 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:181: left oid 181 (ObjNum 180 snap 0 seq_num 180) 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:182: finishing write tid 1 to smithi093153278-182 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:182: finishing write tid 2 to smithi093153278-182 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:182: finishing write tid 3 to smithi093153278-182 2023-08-27T20:52:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:182: finishing write tid 4 to smithi093153278-182 2023-08-27T20:52:06.531 INFO:tasks.rados.rados.0.smithi093.stdout:181: done (15 left) 2023-08-27T20:52:06.531 INFO:tasks.rados.rados.0.smithi093.stdout:197: write initial oid 197 2023-08-27T20:52:06.531 INFO:tasks.rados.rados.0.smithi093.stdout:197: seq_num 196 ranges {494172=457629,1586353=512385,2749730=544442} 2023-08-27T20:52:06.535 INFO:tasks.rados.rados.0.smithi093.stdout:197: writing smithi093153278-197 from 494172 to 951801 tid 1 2023-08-27T20:52:06.542 INFO:tasks.rados.rados.0.smithi093.stdout:197: writing smithi093153278-197 from 1586353 to 2098738 tid 2 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:197: writing smithi093153278-197 from 2749730 to 3294172 tid 3 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:182: finishing write tid 5 to smithi093153278-182 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 182 v 23 (ObjNum 181 snap 0 seq_num 181) dirty exists 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:182: left oid 182 (ObjNum 181 snap 0 seq_num 181) 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:183: finishing write tid 1 to smithi093153278-183 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:183: finishing write tid 2 to smithi093153278-183 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:183: finishing write tid 3 to smithi093153278-183 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:183: finishing write tid 4 to smithi093153278-183 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 183 v 43 (ObjNum 182 snap 0 seq_num 182) dirty exists 2023-08-27T20:52:06.552 INFO:tasks.rados.rados.0.smithi093.stdout:183: left oid 183 (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:184: finishing write tid 1 to smithi093153278-184 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:184: finishing write tid 2 to smithi093153278-184 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:184: finishing write tid 3 to smithi093153278-184 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:184: finishing write tid 4 to smithi093153278-184 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:184: finishing write tid 5 to smithi093153278-184 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 184 v 48 (ObjNum 183 snap 0 seq_num 183) dirty exists 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:184: left oid 184 (ObjNum 183 snap 0 seq_num 183) 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:185: finishing write tid 1 to smithi093153278-185 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:185: finishing write tid 2 to smithi093153278-185 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:185: finishing write tid 3 to smithi093153278-185 2023-08-27T20:52:06.553 INFO:tasks.rados.rados.0.smithi093.stdout:185: finishing write tid 4 to smithi093153278-185 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 185 v 42 (ObjNum 184 snap 0 seq_num 184) dirty exists 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:185: left oid 185 (ObjNum 184 snap 0 seq_num 184) 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:186: finishing write tid 1 to smithi093153278-186 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:186: finishing write tid 2 to smithi093153278-186 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:186: finishing write tid 3 to smithi093153278-186 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:186: finishing write tid 4 to smithi093153278-186 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 186 v 44 (ObjNum 185 snap 0 seq_num 185) dirty exists 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:186: left oid 186 (ObjNum 185 snap 0 seq_num 185) 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:187: finishing write tid 1 to smithi093153278-187 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:187: finishing write tid 2 to smithi093153278-187 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:187: finishing write tid 3 to smithi093153278-187 2023-08-27T20:52:06.554 INFO:tasks.rados.rados.0.smithi093.stdout:187: finishing write tid 4 to smithi093153278-187 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:187: finishing write tid 5 to smithi093153278-187 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 187 v 57 (ObjNum 186 snap 0 seq_num 186) dirty exists 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:187: left oid 187 (ObjNum 186 snap 0 seq_num 186) 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:188: finishing write tid 1 to smithi093153278-188 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:188: finishing write tid 2 to smithi093153278-188 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:188: finishing write tid 3 to smithi093153278-188 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:188: finishing write tid 4 to smithi093153278-188 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 188 v 44 (ObjNum 187 snap 0 seq_num 187) dirty exists 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:188: left oid 188 (ObjNum 187 snap 0 seq_num 187) 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:189: finishing write tid 1 to smithi093153278-189 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:189: finishing write tid 2 to smithi093153278-189 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:182: done (15 left) 2023-08-27T20:52:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:183: done (14 left) 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:184: done (13 left) 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:185: done (12 left) 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:186: done (11 left) 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:187: done (10 left) 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:188: done (9 left) 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:198: write initial oid 198 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:198: seq_num 197 ranges {510979=532878,1528482=772396,3060800=250179} 2023-08-27T20:52:06.556 INFO:tasks.rados.rados.0.smithi093.stdout:198: writing smithi093153278-198 from 510979 to 1043857 tid 1 2023-08-27T20:52:06.563 INFO:tasks.rados.rados.0.smithi093.stdout:198: writing smithi093153278-198 from 1528482 to 2300878 tid 2 2023-08-27T20:52:06.574 INFO:tasks.rados.rados.0.smithi093.stdout:198: writing smithi093153278-198 from 3060800 to 3310979 tid 3 2023-08-27T20:52:06.574 INFO:tasks.rados.rados.0.smithi093.stdout:199: write initial oid 199 2023-08-27T20:52:06.574 INFO:tasks.rados.rados.0.smithi093.stdout:199: seq_num 198 ranges {527786=724480,1703317=516054,2804576=435035} 2023-08-27T20:52:06.577 INFO:tasks.rados.rados.0.smithi093.stdout:199: writing smithi093153278-199 from 527786 to 1252266 tid 1 2023-08-27T20:52:06.583 INFO:tasks.rados.rados.0.smithi093.stdout:199: writing smithi093153278-199 from 1703317 to 2219371 tid 2 2023-08-27T20:52:06.589 INFO:tasks.rados.rados.0.smithi093.stdout:199: writing smithi093153278-199 from 2804576 to 3239611 tid 3 2023-08-27T20:52:06.589 INFO:tasks.rados.rados.0.smithi093.stdout:200: write initial oid 200 2023-08-27T20:52:06.589 INFO:tasks.rados.rados.0.smithi093.stdout:200: seq_num 199 ranges {544593=799729,1761799=776065,2948352=396241} 2023-08-27T20:52:06.597 INFO:tasks.rados.rados.0.smithi093.stdout:200: writing smithi093153278-200 from 544593 to 1344322 tid 1 2023-08-27T20:52:06.604 INFO:tasks.rados.rados.0.smithi093.stdout:200: writing smithi093153278-200 from 1761799 to 2537864 tid 2 2023-08-27T20:52:06.610 INFO:tasks.rados.rados.0.smithi093.stdout:200: writing smithi093153278-200 from 2948352 to 3344593 tid 3 2023-08-27T20:52:06.610 INFO:tasks.rados.rados.0.smithi093.stdout:201: write initial oid 201 2023-08-27T20:52:06.610 INFO:tasks.rados.rados.0.smithi093.stdout:201: seq_num 200 ranges {561400=474978,1820281=519723,2859422=457579} 2023-08-27T20:52:06.616 INFO:tasks.rados.rados.0.smithi093.stdout:201: writing smithi093153278-201 from 561400 to 1036378 tid 1 2023-08-27T20:52:06.625 INFO:tasks.rados.rados.0.smithi093.stdout:201: writing smithi093153278-201 from 1820281 to 2340004 tid 2 2023-08-27T20:52:06.631 INFO:tasks.rados.rados.0.smithi093.stdout:201: writing smithi093153278-201 from 2859422 to 3317001 tid 3 2023-08-27T20:52:06.631 INFO:tasks.rados.rados.0.smithi093.stdout:202: write initial oid 202 2023-08-27T20:52:06.631 INFO:tasks.rados.rados.0.smithi093.stdout:202: seq_num 201 ranges {578207=550227,1762410=779734,3286845=91362} 2023-08-27T20:52:06.638 INFO:tasks.rados.rados.0.smithi093.stdout:202: writing smithi093153278-202 from 578207 to 1128434 tid 1 2023-08-27T20:52:06.647 INFO:tasks.rados.rados.0.smithi093.stdout:202: writing smithi093153278-202 from 1762410 to 2542144 tid 2 2023-08-27T20:52:06.652 INFO:tasks.rados.rados.0.smithi093.stdout:202: writing smithi093153278-202 from 3286845 to 3378207 tid 3 2023-08-27T20:52:06.652 INFO:tasks.rados.rados.0.smithi093.stdout:203: write initial oid 203 2023-08-27T20:52:06.652 INFO:tasks.rados.rados.0.smithi093.stdout:203: seq_num 202 ranges {595014=625476,1820892=523392,2797915=596476} 2023-08-27T20:52:06.659 INFO:tasks.rados.rados.0.smithi093.stdout:203: writing smithi093153278-203 from 595014 to 1220490 tid 1 2023-08-27T20:52:06.667 INFO:tasks.rados.rados.0.smithi093.stdout:203: writing smithi093153278-203 from 1820892 to 2344284 tid 2 2023-08-27T20:52:06.673 INFO:tasks.rados.rados.0.smithi093.stdout:203: writing smithi093153278-203 from 2797915 to 3394391 tid 3 2023-08-27T20:52:06.674 INFO:tasks.rados.rados.0.smithi093.stdout:204: write initial oid 204 2023-08-27T20:52:06.674 INFO:tasks.rados.rados.0.smithi093.stdout:204: seq_num 203 ranges {611821=700725,1879374=783403,3341691=70130} 2023-08-27T20:52:06.682 INFO:tasks.rados.rados.0.smithi093.stdout:204: writing smithi093153278-204 from 611821 to 1312546 tid 1 2023-08-27T20:52:06.690 INFO:tasks.rados.rados.0.smithi093.stdout:204: writing smithi093153278-204 from 1879374 to 2662777 tid 2 2023-08-27T20:52:06.695 INFO:tasks.rados.rados.0.smithi093.stdout:204: writing smithi093153278-204 from 3341691 to 3411821 tid 3 2023-08-27T20:52:06.695 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.695 INFO:tasks.rados.rados.0.smithi093.stdout:189: finishing write tid 3 to smithi093153278-189 2023-08-27T20:52:06.695 INFO:tasks.rados.rados.0.smithi093.stdout:189: finishing write tid 4 to smithi093153278-189 2023-08-27T20:52:06.696 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 189 v 33 (ObjNum 188 snap 0 seq_num 188) dirty exists 2023-08-27T20:52:06.696 INFO:tasks.rados.rados.0.smithi093.stdout:189: left oid 189 (ObjNum 188 snap 0 seq_num 188) 2023-08-27T20:52:06.696 INFO:tasks.rados.rados.0.smithi093.stdout:190: finishing write tid 1 to smithi093153278-190 2023-08-27T20:52:06.696 INFO:tasks.rados.rados.0.smithi093.stdout:189: done (15 left) 2023-08-27T20:52:06.696 INFO:tasks.rados.rados.0.smithi093.stdout:205: write initial oid 205 2023-08-27T20:52:06.696 INFO:tasks.rados.rados.0.smithi093.stdout:205: seq_num 204 ranges {628628=775974,1937856=527061,3252761=175867} 2023-08-27T20:52:06.702 INFO:tasks.rados.rados.0.smithi093.stdout:205: writing smithi093153278-205 from 628628 to 1404602 tid 1 2023-08-27T20:52:06.708 INFO:tasks.rados.rados.0.smithi093.stdout:205: writing smithi093153278-205 from 1937856 to 2464917 tid 2 2023-08-27T20:52:06.715 INFO:tasks.rados.rados.0.smithi093.stdout:205: writing smithi093153278-205 from 3252761 to 3428628 tid 3 2023-08-27T20:52:06.715 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.715 INFO:tasks.rados.rados.0.smithi093.stdout:190: finishing write tid 2 to smithi093153278-190 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:190: finishing write tid 3 to smithi093153278-190 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:190: finishing write tid 4 to smithi093153278-190 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 190 v 47 (ObjNum 189 snap 0 seq_num 189) dirty exists 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:190: left oid 190 (ObjNum 189 snap 0 seq_num 189) 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:191: finishing write tid 1 to smithi093153278-191 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:191: finishing write tid 2 to smithi093153278-191 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:191: finishing write tid 3 to smithi093153278-191 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:191: finishing write tid 4 to smithi093153278-191 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:191: finishing write tid 5 to smithi093153278-191 2023-08-27T20:52:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 191 v 52 (ObjNum 190 snap 0 seq_num 190) dirty exists 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:191: left oid 191 (ObjNum 190 snap 0 seq_num 190) 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:192: finishing write tid 1 to smithi093153278-192 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:192: finishing write tid 2 to smithi093153278-192 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:192: finishing write tid 3 to smithi093153278-192 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:192: finishing write tid 4 to smithi093153278-192 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:192: finishing write tid 5 to smithi093153278-192 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 192 v 41 (ObjNum 191 snap 0 seq_num 191) dirty exists 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:192: left oid 192 (ObjNum 191 snap 0 seq_num 191) 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:193: finishing write tid 1 to smithi093153278-193 2023-08-27T20:52:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:193: finishing write tid 2 to smithi093153278-193 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:193: finishing write tid 3 to smithi093153278-193 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:193: finishing write tid 4 to smithi093153278-193 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:193: finishing write tid 5 to smithi093153278-193 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 193 v 28 (ObjNum 192 snap 0 seq_num 192) dirty exists 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:193: left oid 193 (ObjNum 192 snap 0 seq_num 192) 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:194: finishing write tid 1 to smithi093153278-194 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:194: finishing write tid 2 to smithi093153278-194 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:194: finishing write tid 3 to smithi093153278-194 2023-08-27T20:52:06.718 INFO:tasks.rados.rados.0.smithi093.stdout:190: done (15 left) 2023-08-27T20:52:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:191: done (14 left) 2023-08-27T20:52:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:192: done (13 left) 2023-08-27T20:52:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:193: done (12 left) 2023-08-27T20:52:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:206: write initial oid 206 2023-08-27T20:52:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:206: seq_num 205 ranges {645435=451223,1879985=787072,3280184=165251} 2023-08-27T20:52:06.721 INFO:tasks.rados.rados.0.smithi093.stdout:206: writing smithi093153278-206 from 645435 to 1096658 tid 1 2023-08-27T20:52:06.730 INFO:tasks.rados.rados.0.smithi093.stdout:206: writing smithi093153278-206 from 1879985 to 2667057 tid 2 2023-08-27T20:52:06.735 INFO:tasks.rados.rados.0.smithi093.stdout:206: writing smithi093153278-206 from 3280184 to 3445435 tid 3 2023-08-27T20:52:06.735 INFO:tasks.rados.rados.0.smithi093.stdout:207: write initial oid 207 2023-08-27T20:52:06.735 INFO:tasks.rados.rados.0.smithi093.stdout:207: seq_num 206 ranges {662242=642825,2054820=530730,3307607=154635} 2023-08-27T20:52:06.742 INFO:tasks.rados.rados.0.smithi093.stdout:207: writing smithi093153278-207 from 662242 to 1305067 tid 1 2023-08-27T20:52:06.750 INFO:tasks.rados.rados.0.smithi093.stdout:207: writing smithi093153278-207 from 2054820 to 2585550 tid 2 2023-08-27T20:52:06.755 INFO:tasks.rados.rados.0.smithi093.stdout:207: writing smithi093153278-207 from 3307607 to 3462242 tid 3 2023-08-27T20:52:06.756 INFO:tasks.rados.rados.0.smithi093.stdout:208: write initial oid 208 2023-08-27T20:52:06.756 INFO:tasks.rados.rados.0.smithi093.stdout:208: seq_num 207 ranges {679049=718074,2113302=674388,3335030=144019} 2023-08-27T20:52:06.764 INFO:tasks.rados.rados.0.smithi093.stdout:208: writing smithi093153278-208 from 679049 to 1397123 tid 1 2023-08-27T20:52:06.772 INFO:tasks.rados.rados.0.smithi093.stdout:208: writing smithi093153278-208 from 2113302 to 2787690 tid 2 2023-08-27T20:52:06.777 INFO:tasks.rados.rados.0.smithi093.stdout:208: writing smithi093153278-208 from 3335030 to 3479049 tid 3 2023-08-27T20:52:06.777 INFO:tasks.rados.rados.0.smithi093.stdout:209: write initial oid 209 2023-08-27T20:52:06.777 INFO:tasks.rados.rados.0.smithi093.stdout:209: seq_num 208 ranges {695856=793323,2171784=534399,3362453=133403} 2023-08-27T20:52:06.785 INFO:tasks.rados.rados.0.smithi093.stdout:209: writing smithi093153278-209 from 695856 to 1489179 tid 1 2023-08-27T20:52:06.793 INFO:tasks.rados.rados.0.smithi093.stdout:209: writing smithi093153278-209 from 2171784 to 2706183 tid 2 2023-08-27T20:52:06.797 INFO:tasks.rados.rados.0.smithi093.stdout:209: writing smithi093153278-209 from 3362453 to 3495856 tid 3 2023-08-27T20:52:06.797 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.800 INFO:tasks.rados.rados.0.smithi093.stdout:194: finishing write tid 4 to smithi093153278-194 2023-08-27T20:52:06.800 INFO:tasks.rados.rados.0.smithi093.stdout:194: finishing write tid 5 to smithi093153278-194 2023-08-27T20:52:06.800 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 194 v 32 (ObjNum 193 snap 0 seq_num 193) dirty exists 2023-08-27T20:52:06.800 INFO:tasks.rados.rados.0.smithi093.stdout:194: left oid 194 (ObjNum 193 snap 0 seq_num 193) 2023-08-27T20:52:06.800 INFO:tasks.rados.rados.0.smithi093.stdout:195: finishing write tid 1 to smithi093153278-195 2023-08-27T20:52:06.800 INFO:tasks.rados.rados.0.smithi093.stdout:194: done (15 left) 2023-08-27T20:52:06.800 INFO:tasks.rados.rados.0.smithi093.stdout:210: write initial oid 210 2023-08-27T20:52:06.801 INFO:tasks.rados.rados.0.smithi093.stdout:210: seq_num 209 ranges {712663=468572,1713913=678057,2873523=591733} 2023-08-27T20:52:06.807 INFO:tasks.rados.rados.0.smithi093.stdout:210: writing smithi093153278-210 from 712663 to 1181235 tid 1 2023-08-27T20:52:06.814 INFO:tasks.rados.rados.0.smithi093.stdout:210: writing smithi093153278-210 from 1713913 to 2391970 tid 2 2023-08-27T20:52:06.821 INFO:tasks.rados.rados.0.smithi093.stdout:210: writing smithi093153278-210 from 2873523 to 3465256 tid 3 2023-08-27T20:52:06.822 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.823 INFO:tasks.rados.rados.0.smithi093.stdout:195: finishing write tid 2 to smithi093153278-195 2023-08-27T20:52:06.823 INFO:tasks.rados.rados.0.smithi093.stdout:195: finishing write tid 3 to smithi093153278-195 2023-08-27T20:52:06.823 INFO:tasks.rados.rados.0.smithi093.stdout:195: finishing write tid 4 to smithi093153278-195 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:195: finishing write tid 5 to smithi093153278-195 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 49 (ObjNum 194 snap 0 seq_num 194) dirty exists 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:195: left oid 195 (ObjNum 194 snap 0 seq_num 194) 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:196: finishing write tid 1 to smithi093153278-196 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:196: finishing write tid 2 to smithi093153278-196 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:196: finishing write tid 3 to smithi093153278-196 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:196: finishing write tid 4 to smithi093153278-196 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:196: finishing write tid 5 to smithi093153278-196 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 196 v 47 (ObjNum 195 snap 0 seq_num 195) dirty exists 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:196: left oid 196 (ObjNum 195 snap 0 seq_num 195) 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:197: finishing write tid 1 to smithi093153278-197 2023-08-27T20:52:06.824 INFO:tasks.rados.rados.0.smithi093.stdout:197: finishing write tid 2 to smithi093153278-197 2023-08-27T20:52:06.825 INFO:tasks.rados.rados.0.smithi093.stdout:195: done (15 left) 2023-08-27T20:52:06.825 INFO:tasks.rados.rados.0.smithi093.stdout:196: done (14 left) 2023-08-27T20:52:06.825 INFO:tasks.rados.rados.0.smithi093.stdout:211: write initial oid 211 2023-08-27T20:52:06.825 INFO:tasks.rados.rados.0.smithi093.stdout:211: seq_num 210 ranges {729470=543821,1772395=538068,2900946=403005} 2023-08-27T20:52:06.830 INFO:tasks.rados.rados.0.smithi093.stdout:211: writing smithi093153278-211 from 729470 to 1273291 tid 1 2023-08-27T20:52:06.836 INFO:tasks.rados.rados.0.smithi093.stdout:211: writing smithi093153278-211 from 1772395 to 2310463 tid 2 2023-08-27T20:52:06.843 INFO:tasks.rados.rados.0.smithi093.stdout:211: writing smithi093153278-211 from 2900946 to 3303951 tid 3 2023-08-27T20:52:06.843 INFO:tasks.rados.rados.0.smithi093.stdout:212: write initial oid 212 2023-08-27T20:52:06.843 INFO:tasks.rados.rados.0.smithi093.stdout:212: seq_num 211 ranges {746277=619070,1830877=681726,2928369=617908} 2023-08-27T20:52:06.851 INFO:tasks.rados.rados.0.smithi093.stdout:212: writing smithi093153278-212 from 746277 to 1365347 tid 1 2023-08-27T20:52:06.858 INFO:tasks.rados.rados.0.smithi093.stdout:212: writing smithi093153278-212 from 1830877 to 2512603 tid 2 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout:212: writing smithi093153278-212 from 2928369 to 3546277 tid 3 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout:197: finishing write tid 3 to smithi093153278-197 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout:197: finishing write tid 4 to smithi093153278-197 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout:197: finishing write tid 5 to smithi093153278-197 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 197 v 37 (ObjNum 196 snap 0 seq_num 196) dirty exists 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout:197: left oid 197 (ObjNum 196 snap 0 seq_num 196) 2023-08-27T20:52:06.866 INFO:tasks.rados.rados.0.smithi093.stdout:198: finishing write tid 1 to smithi093153278-198 2023-08-27T20:52:06.867 INFO:tasks.rados.rados.0.smithi093.stdout:198: finishing write tid 2 to smithi093153278-198 2023-08-27T20:52:06.867 INFO:tasks.rados.rados.0.smithi093.stdout:198: finishing write tid 3 to smithi093153278-198 2023-08-27T20:52:06.867 INFO:tasks.rados.rados.0.smithi093.stdout:198: finishing write tid 4 to smithi093153278-198 2023-08-27T20:52:06.867 INFO:tasks.rados.rados.0.smithi093.stdout:197: done (15 left) 2023-08-27T20:52:06.867 INFO:tasks.rados.rados.0.smithi093.stdout:213: write initial oid 213 2023-08-27T20:52:06.867 INFO:tasks.rados.rados.0.smithi093.stdout:213: seq_num 212 ranges {763084=694319,1889359=541737,2955792=541902} 2023-08-27T20:52:06.874 INFO:tasks.rados.rados.0.smithi093.stdout:213: writing smithi093153278-213 from 763084 to 1457403 tid 1 2023-08-27T20:52:06.880 INFO:tasks.rados.rados.0.smithi093.stdout:213: writing smithi093153278-213 from 1889359 to 2431096 tid 2 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:213: writing smithi093153278-213 from 2955792 to 3497694 tid 3 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:198: finishing write tid 5 to smithi093153278-198 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 198 v 35 (ObjNum 197 snap 0 seq_num 197) dirty exists 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:198: left oid 198 (ObjNum 197 snap 0 seq_num 197) 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:199: finishing write tid 1 to smithi093153278-199 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:199: finishing write tid 2 to smithi093153278-199 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:199: finishing write tid 3 to smithi093153278-199 2023-08-27T20:52:06.888 INFO:tasks.rados.rados.0.smithi093.stdout:199: finishing write tid 4 to smithi093153278-199 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:199: finishing write tid 5 to smithi093153278-199 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 199 v 39 (ObjNum 198 snap 0 seq_num 198) dirty exists 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:199: left oid 199 (ObjNum 198 snap 0 seq_num 198) 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:200: finishing write tid 1 to smithi093153278-200 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:200: finishing write tid 2 to smithi093153278-200 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:200: finishing write tid 3 to smithi093153278-200 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:200: finishing write tid 4 to smithi093153278-200 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:200: finishing write tid 5 to smithi093153278-200 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 200 v 51 (ObjNum 199 snap 0 seq_num 199) dirty exists 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:200: left oid 200 (ObjNum 199 snap 0 seq_num 199) 2023-08-27T20:52:06.889 INFO:tasks.rados.rados.0.smithi093.stdout:201: finishing write tid 1 to smithi093153278-201 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:201: finishing write tid 2 to smithi093153278-201 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:201: finishing write tid 3 to smithi093153278-201 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:201: finishing write tid 4 to smithi093153278-201 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:201: finishing write tid 5 to smithi093153278-201 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 201 v 51 (ObjNum 200 snap 0 seq_num 200) dirty exists 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:201: left oid 201 (ObjNum 200 snap 0 seq_num 200) 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:202: finishing write tid 1 to smithi093153278-202 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:202: finishing write tid 2 to smithi093153278-202 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:202: finishing write tid 3 to smithi093153278-202 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:202: finishing write tid 4 to smithi093153278-202 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:202: finishing write tid 5 to smithi093153278-202 2023-08-27T20:52:06.890 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 202 v 48 (ObjNum 201 snap 0 seq_num 201) dirty exists 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:202: left oid 202 (ObjNum 201 snap 0 seq_num 201) 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:203: finishing write tid 1 to smithi093153278-203 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:203: finishing write tid 2 to smithi093153278-203 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:203: finishing write tid 3 to smithi093153278-203 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:203: finishing write tid 4 to smithi093153278-203 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:203: finishing write tid 5 to smithi093153278-203 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 203 v 48 (ObjNum 202 snap 0 seq_num 202) dirty exists 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:203: left oid 203 (ObjNum 202 snap 0 seq_num 202) 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:204: finishing write tid 1 to smithi093153278-204 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:204: finishing write tid 2 to smithi093153278-204 2023-08-27T20:52:06.891 INFO:tasks.rados.rados.0.smithi093.stdout:204: finishing write tid 3 to smithi093153278-204 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:204: finishing write tid 4 to smithi093153278-204 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:204: finishing write tid 5 to smithi093153278-204 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 204 v 56 (ObjNum 203 snap 0 seq_num 203) dirty exists 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:204: left oid 204 (ObjNum 203 snap 0 seq_num 203) 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:205: finishing write tid 1 to smithi093153278-205 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:205: finishing write tid 2 to smithi093153278-205 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:205: finishing write tid 3 to smithi093153278-205 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:205: finishing write tid 4 to smithi093153278-205 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:198: done (15 left) 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:199: done (14 left) 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:200: done (13 left) 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:201: done (12 left) 2023-08-27T20:52:06.892 INFO:tasks.rados.rados.0.smithi093.stdout:202: done (11 left) 2023-08-27T20:52:06.893 INFO:tasks.rados.rados.0.smithi093.stdout:203: done (10 left) 2023-08-27T20:52:06.893 INFO:tasks.rados.rados.0.smithi093.stdout:204: done (9 left) 2023-08-27T20:52:06.893 INFO:tasks.rados.rados.0.smithi093.stdout:214: write initial oid 214 2023-08-27T20:52:06.893 INFO:tasks.rados.rados.0.smithi093.stdout:214: seq_num 213 ranges {779891=485921,1947841=685395,3383215=196676} 2023-08-27T20:52:06.895 INFO:tasks.rados.rados.0.smithi093.stdout:214: writing smithi093153278-214 from 779891 to 1265812 tid 1 2023-08-27T20:52:06.904 INFO:tasks.rados.rados.0.smithi093.stdout:214: writing smithi093153278-214 from 1947841 to 2633236 tid 2 2023-08-27T20:52:06.910 INFO:tasks.rados.rados.0.smithi093.stdout:214: writing smithi093153278-214 from 3383215 to 3579891 tid 3 2023-08-27T20:52:06.910 INFO:tasks.rados.rados.0.smithi093.stdout:215: write initial oid 215 2023-08-27T20:52:06.910 INFO:tasks.rados.rados.0.smithi093.stdout:215: seq_num 214 ranges {796698=561170,2006323=545406,3126991=469707} 2023-08-27T20:52:06.918 INFO:tasks.rados.rados.0.smithi093.stdout:215: writing smithi093153278-215 from 796698 to 1357868 tid 1 2023-08-27T20:52:06.925 INFO:tasks.rados.rados.0.smithi093.stdout:215: writing smithi093153278-215 from 2006323 to 2551729 tid 2 2023-08-27T20:52:06.932 INFO:tasks.rados.rados.0.smithi093.stdout:215: writing smithi093153278-215 from 3126991 to 3596698 tid 3 2023-08-27T20:52:06.932 INFO:tasks.rados.rados.0.smithi093.stdout:216: write initial oid 216 2023-08-27T20:52:06.932 INFO:tasks.rados.rados.0.smithi093.stdout:216: seq_num 215 ranges {413505=636419,1664805=689064,3038061=492071} 2023-08-27T20:52:06.938 INFO:tasks.rados.rados.0.smithi093.stdout:216: writing smithi093153278-216 from 413505 to 1049924 tid 1 2023-08-27T20:52:06.946 INFO:tasks.rados.rados.0.smithi093.stdout:216: writing smithi093153278-216 from 1664805 to 2353869 tid 2 2023-08-27T20:52:06.954 INFO:tasks.rados.rados.0.smithi093.stdout:216: writing smithi093153278-216 from 3038061 to 3530132 tid 3 2023-08-27T20:52:06.954 INFO:tasks.rados.rados.0.smithi093.stdout:217: write initial oid 217 2023-08-27T20:52:06.954 INFO:tasks.rados.rados.0.smithi093.stdout:217: seq_num 216 ranges {430312=711668,1723287=549075,2781837=703343} 2023-08-27T20:52:06.961 INFO:tasks.rados.rados.0.smithi093.stdout:217: writing smithi093153278-217 from 430312 to 1141980 tid 1 2023-08-27T20:52:06.968 INFO:tasks.rados.rados.0.smithi093.stdout:217: writing smithi093153278-217 from 1723287 to 2272362 tid 2 2023-08-27T20:52:06.976 INFO:tasks.rados.rados.0.smithi093.stdout:217: writing smithi093153278-217 from 2781837 to 3485180 tid 3 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:218: write initial oid 218 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:205: finishing write tid 5 to smithi093153278-205 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 205 v 36 (ObjNum 204 snap 0 seq_num 204) dirty exists 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:205: left oid 205 (ObjNum 204 snap 0 seq_num 204) 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:206: finishing write tid 1 to smithi093153278-206 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:206: finishing write tid 2 to smithi093153278-206 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:206: finishing write tid 3 to smithi093153278-206 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:206: finishing write tid 4 to smithi093153278-206 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:206: finishing write tid 5 to smithi093153278-206 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 206 v 55 (ObjNum 205 snap 0 seq_num 205) dirty exists 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:206: left oid 206 (ObjNum 205 snap 0 seq_num 205) 2023-08-27T20:52:06.977 INFO:tasks.rados.rados.0.smithi093.stdout:207: finishing write tid 1 to smithi093153278-207 2023-08-27T20:52:06.978 INFO:tasks.rados.rados.0.smithi093.stdout:207: finishing write tid 2 to smithi093153278-207 2023-08-27T20:52:06.978 INFO:tasks.rados.rados.0.smithi093.stdout:218: seq_num 217 ranges {447119=786917,1665416=692733,2976554=514615} 2023-08-27T20:52:06.984 INFO:tasks.rados.rados.0.smithi093.stdout:218: writing smithi093153278-218 from 447119 to 1234036 tid 1 2023-08-27T20:52:06.991 INFO:tasks.rados.rados.0.smithi093.stdout:218: writing smithi093153278-218 from 1665416 to 2358149 tid 2 2023-08-27T20:52:06.999 INFO:tasks.rados.rados.0.smithi093.stdout:218: writing smithi093153278-218 from 2976554 to 3491169 tid 3 2023-08-27T20:52:06.999 INFO:tasks.rados.rados.0.smithi093.stdout:205: done (13 left) 2023-08-27T20:52:06.999 INFO:tasks.rados.rados.0.smithi093.stdout:206: done (12 left) 2023-08-27T20:52:06.999 INFO:tasks.rados.rados.0.smithi093.stdout:219: write initial oid 219 2023-08-27T20:52:06.999 INFO:tasks.rados.rados.0.smithi093.stdout:219: seq_num 218 ranges {463926=462166,1723898=436391,2603977=725887} 2023-08-27T20:52:07.004 INFO:tasks.rados.rados.0.smithi093.stdout:219: writing smithi093153278-219 from 463926 to 926092 tid 1 2023-08-27T20:52:07.011 INFO:tasks.rados.rados.0.smithi093.stdout:219: writing smithi093153278-219 from 1723898 to 2160289 tid 2 2023-08-27T20:52:07.019 INFO:tasks.rados.rados.0.smithi093.stdout:219: writing smithi093153278-219 from 2603977 to 3329864 tid 3 2023-08-27T20:52:07.019 INFO:tasks.rados.rados.0.smithi093.stdout:220: write initial oid 220 2023-08-27T20:52:07.019 INFO:tasks.rados.rados.0.smithi093.stdout:220: seq_num 219 ranges {480733=537415,1782380=696402,3031400=537159} 2023-08-27T20:52:07.024 INFO:tasks.rados.rados.0.smithi093.stdout:220: writing smithi093153278-220 from 480733 to 1018148 tid 1 2023-08-27T20:52:07.032 INFO:tasks.rados.rados.0.smithi093.stdout:220: writing smithi093153278-220 from 1782380 to 2478782 tid 2 2023-08-27T20:52:07.039 INFO:tasks.rados.rados.0.smithi093.stdout:220: writing smithi093153278-220 from 3031400 to 3568559 tid 3 2023-08-27T20:52:07.039 INFO:tasks.rados.rados.0.smithi093.stdout:221: write initial oid 221 2023-08-27T20:52:07.039 INFO:tasks.rados.rados.0.smithi093.stdout:221: seq_num 220 ranges {497540=612664,1840862=440060,3058823=464784} 2023-08-27T20:52:07.045 INFO:tasks.rados.rados.0.smithi093.stdout:221: writing smithi093153278-221 from 497540 to 1110204 tid 1 2023-08-27T20:52:07.051 INFO:tasks.rados.rados.0.smithi093.stdout:221: writing smithi093153278-221 from 1840862 to 2280922 tid 2 2023-08-27T20:52:07.059 INFO:tasks.rados.rados.0.smithi093.stdout:221: writing smithi093153278-221 from 3058823 to 3523607 tid 3 2023-08-27T20:52:07.059 INFO:tasks.rados.rados.0.smithi093.stdout:222: write initial oid 222 2023-08-27T20:52:07.059 INFO:tasks.rados.rados.0.smithi093.stdout:222: seq_num 221 ranges {514347=404266,1499344=700071,2686246=676056} 2023-08-27T20:52:07.064 INFO:tasks.rados.rados.0.smithi093.stdout:222: writing smithi093153278-222 from 514347 to 918613 tid 1 2023-08-27T20:52:07.072 INFO:tasks.rados.rados.0.smithi093.stdout:222: writing smithi093153278-222 from 1499344 to 2199415 tid 2 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:222: writing smithi093153278-222 from 2686246 to 3362302 tid 3 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:207: finishing write tid 3 to smithi093153278-207 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:207: finishing write tid 4 to smithi093153278-207 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:207: finishing write tid 5 to smithi093153278-207 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 207 v 27 (ObjNum 206 snap 0 seq_num 206) dirty exists 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:207: left oid 207 (ObjNum 206 snap 0 seq_num 206) 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:207: done (15 left) 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:223: write initial oid 223 2023-08-27T20:52:07.081 INFO:tasks.rados.rados.0.smithi093.stdout:223: seq_num 222 ranges {531154=479515,1557826=443729,2713669=487328,3679295=51859} 2023-08-27T20:52:07.086 INFO:tasks.rados.rados.0.smithi093.stdout:223: writing smithi093153278-223 from 531154 to 1010669 tid 1 2023-08-27T20:52:07.092 INFO:tasks.rados.rados.0.smithi093.stdout:223: writing smithi093153278-223 from 1557826 to 2001555 tid 2 2023-08-27T20:52:07.100 INFO:tasks.rados.rados.0.smithi093.stdout:223: writing smithi093153278-223 from 2713669 to 3200997 tid 3 2023-08-27T20:52:07.103 INFO:tasks.rados.rados.0.smithi093.stdout:223: writing smithi093153278-223 from 3679295 to 3731154 tid 4 2023-08-27T20:52:07.103 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.103 INFO:tasks.rados.rados.0.smithi093.stdout:208: finishing write tid 1 to smithi093153278-208 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:208: finishing write tid 2 to smithi093153278-208 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:208: finishing write tid 3 to smithi093153278-208 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:208: finishing write tid 4 to smithi093153278-208 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:208: finishing write tid 5 to smithi093153278-208 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 208 v 43 (ObjNum 207 snap 0 seq_num 207) dirty exists 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:208: left oid 208 (ObjNum 207 snap 0 seq_num 207) 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:209: finishing write tid 1 to smithi093153278-209 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:209: finishing write tid 2 to smithi093153278-209 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:209: finishing write tid 3 to smithi093153278-209 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:209: finishing write tid 4 to smithi093153278-209 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:209: finishing write tid 5 to smithi093153278-209 2023-08-27T20:52:07.104 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 209 v 60 (ObjNum 208 snap 0 seq_num 208) dirty exists 2023-08-27T20:52:07.105 INFO:tasks.rados.rados.0.smithi093.stdout:209: left oid 209 (ObjNum 208 snap 0 seq_num 208) 2023-08-27T20:52:07.105 INFO:tasks.rados.rados.0.smithi093.stdout:210: finishing write tid 1 to smithi093153278-210 2023-08-27T20:52:07.105 INFO:tasks.rados.rados.0.smithi093.stdout:210: finishing write tid 2 to smithi093153278-210 2023-08-27T20:52:07.105 INFO:tasks.rados.rados.0.smithi093.stdout:208: done (15 left) 2023-08-27T20:52:07.105 INFO:tasks.rados.rados.0.smithi093.stdout:209: done (14 left) 2023-08-27T20:52:07.105 INFO:tasks.rados.rados.0.smithi093.stdout:224: write initial oid 224 2023-08-27T20:52:07.105 INFO:tasks.rados.rados.0.smithi093.stdout:224: seq_num 223 ranges {547961=554764,1616308=703740,2741092=698600} 2023-08-27T20:52:07.109 INFO:tasks.rados.rados.0.smithi093.stdout:224: writing smithi093153278-224 from 547961 to 1102725 tid 1 2023-08-27T20:52:07.116 INFO:tasks.rados.rados.0.smithi093.stdout:224: writing smithi093153278-224 from 1616308 to 2320048 tid 2 2023-08-27T20:52:07.122 INFO:tasks.rados.rados.0.smithi093.stdout:224: writing smithi093153278-224 from 2741092 to 3439692 tid 3 2023-08-27T20:52:07.123 INFO:tasks.rados.rados.0.smithi093.stdout:225: write initial oid 225 2023-08-27T20:52:07.123 INFO:tasks.rados.rados.0.smithi093.stdout:225: seq_num 224 ranges {564768=630013,1674790=447398,2768515=626225} 2023-08-27T20:52:07.129 INFO:tasks.rados.rados.0.smithi093.stdout:225: writing smithi093153278-225 from 564768 to 1194781 tid 1 2023-08-27T20:52:07.135 INFO:tasks.rados.rados.0.smithi093.stdout:225: writing smithi093153278-225 from 1674790 to 2122188 tid 2 2023-08-27T20:52:07.143 INFO:tasks.rados.rados.0.smithi093.stdout:225: writing smithi093153278-225 from 2768515 to 3394740 tid 3 2023-08-27T20:52:07.143 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.145 INFO:tasks.rados.rados.0.smithi093.stdout:210: finishing write tid 3 to smithi093153278-210 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:210: finishing write tid 4 to smithi093153278-210 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:210: finishing write tid 5 to smithi093153278-210 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 210 v 47 (ObjNum 209 snap 0 seq_num 209) dirty exists 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:210: left oid 210 (ObjNum 209 snap 0 seq_num 209) 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:211: finishing write tid 1 to smithi093153278-211 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:211: finishing write tid 2 to smithi093153278-211 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:211: finishing write tid 3 to smithi093153278-211 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:211: finishing write tid 4 to smithi093153278-211 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:210: done (15 left) 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:226: write initial oid 226 2023-08-27T20:52:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:226: seq_num 225 ranges {581575=705262,1733272=707409,3195938=437497} 2023-08-27T20:52:07.153 INFO:tasks.rados.rados.0.smithi093.stdout:226: writing smithi093153278-226 from 581575 to 1286837 tid 1 2023-08-27T20:52:07.161 INFO:tasks.rados.rados.0.smithi093.stdout:226: writing smithi093153278-226 from 1733272 to 2440681 tid 2 2023-08-27T20:52:07.169 INFO:tasks.rados.rados.0.smithi093.stdout:226: writing smithi093153278-226 from 3195938 to 3633435 tid 3 2023-08-27T20:52:07.169 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.169 INFO:tasks.rados.rados.0.smithi093.stdout:211: finishing write tid 5 to smithi093153278-211 2023-08-27T20:52:07.169 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 211 v 51 (ObjNum 210 snap 0 seq_num 210) dirty exists 2023-08-27T20:52:07.169 INFO:tasks.rados.rados.0.smithi093.stdout:211: left oid 211 (ObjNum 210 snap 0 seq_num 210) 2023-08-27T20:52:07.169 INFO:tasks.rados.rados.0.smithi093.stdout:212: finishing write tid 1 to smithi093153278-212 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:211: done (15 left) 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:227: write initial oid 227 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:212: finishing write tid 2 to smithi093153278-212 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:212: finishing write tid 3 to smithi093153278-212 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:212: finishing write tid 4 to smithi093153278-212 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:212: finishing write tid 5 to smithi093153278-212 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 212 v 41 (ObjNum 211 snap 0 seq_num 211) dirty exists 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:212: left oid 212 (ObjNum 211 snap 0 seq_num 211) 2023-08-27T20:52:07.170 INFO:tasks.rados.rados.0.smithi093.stdout:227: seq_num 226 ranges {598382=780511,2075401=451067,3107008=648769} 2023-08-27T20:52:07.176 INFO:tasks.rados.rados.0.smithi093.stdout:227: writing smithi093153278-227 from 598382 to 1378893 tid 1 2023-08-27T20:52:07.183 INFO:tasks.rados.rados.0.smithi093.stdout:227: writing smithi093153278-227 from 2075401 to 2526468 tid 2 2023-08-27T20:52:07.190 INFO:tasks.rados.rados.0.smithi093.stdout:227: writing smithi093153278-227 from 3107008 to 3755777 tid 3 2023-08-27T20:52:07.190 INFO:tasks.rados.rados.0.smithi093.stdout:212: done (15 left) 2023-08-27T20:52:07.190 INFO:tasks.rados.rados.0.smithi093.stdout:228: write initial oid 228 2023-08-27T20:52:07.190 INFO:tasks.rados.rados.0.smithi093.stdout:228: seq_num 227 ranges {615189=455760,1733883=711078,3134431=460041} 2023-08-27T20:52:07.196 INFO:tasks.rados.rados.0.smithi093.stdout:228: writing smithi093153278-228 from 615189 to 1070949 tid 1 2023-08-27T20:52:07.204 INFO:tasks.rados.rados.0.smithi093.stdout:228: writing smithi093153278-228 from 1733883 to 2444961 tid 2 2023-08-27T20:52:07.211 INFO:tasks.rados.rados.0.smithi093.stdout:228: writing smithi093153278-228 from 3134431 to 3594472 tid 3 2023-08-27T20:52:07.211 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.211 INFO:tasks.rados.rados.0.smithi093.stdout:213: finishing write tid 1 to smithi093153278-213 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:213: finishing write tid 2 to smithi093153278-213 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:213: finishing write tid 3 to smithi093153278-213 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:213: finishing write tid 4 to smithi093153278-213 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:213: finishing write tid 5 to smithi093153278-213 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 213 v 52 (ObjNum 212 snap 0 seq_num 212) dirty exists 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:213: left oid 213 (ObjNum 212 snap 0 seq_num 212) 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:214: finishing write tid 1 to smithi093153278-214 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:214: finishing write tid 2 to smithi093153278-214 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:213: done (15 left) 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:229: write initial oid 229 2023-08-27T20:52:07.212 INFO:tasks.rados.rados.0.smithi093.stdout:229: seq_num 228 ranges {631996=531009,1792365=454736,2761854=671313} 2023-08-27T20:52:07.217 INFO:tasks.rados.rados.0.smithi093.stdout:229: writing smithi093153278-229 from 631996 to 1163005 tid 1 2023-08-27T20:52:07.224 INFO:tasks.rados.rados.0.smithi093.stdout:229: writing smithi093153278-229 from 1792365 to 2247101 tid 2 2023-08-27T20:52:07.231 INFO:tasks.rados.rados.0.smithi093.stdout:229: writing smithi093153278-229 from 2761854 to 3433167 tid 3 2023-08-27T20:52:07.231 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.231 INFO:tasks.rados.rados.0.smithi093.stdout:214: finishing write tid 3 to smithi093153278-214 2023-08-27T20:52:07.231 INFO:tasks.rados.rados.0.smithi093.stdout:214: finishing write tid 4 to smithi093153278-214 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:214: finishing write tid 5 to smithi093153278-214 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 214 v 40 (ObjNum 213 snap 0 seq_num 213) dirty exists 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:214: left oid 214 (ObjNum 213 snap 0 seq_num 213) 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:215: finishing write tid 1 to smithi093153278-215 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:215: finishing write tid 2 to smithi093153278-215 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:215: finishing write tid 3 to smithi093153278-215 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:215: finishing write tid 4 to smithi093153278-215 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:215: finishing write tid 5 to smithi093153278-215 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 215 v 47 (ObjNum 214 snap 0 seq_num 214) dirty exists 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:215: left oid 215 (ObjNum 214 snap 0 seq_num 214) 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:216: finishing write tid 1 to smithi093153278-216 2023-08-27T20:52:07.232 INFO:tasks.rados.rados.0.smithi093.stdout:216: finishing write tid 2 to smithi093153278-216 2023-08-27T20:52:07.233 INFO:tasks.rados.rados.0.smithi093.stdout:214: done (15 left) 2023-08-27T20:52:07.233 INFO:tasks.rados.rados.0.smithi093.stdout:215: done (14 left) 2023-08-27T20:52:07.233 INFO:tasks.rados.rados.0.smithi093.stdout:230: write initial oid 230 2023-08-27T20:52:07.233 INFO:tasks.rados.rados.0.smithi093.stdout:230: seq_num 229 ranges {648803=722611,1967200=714747,3421983=426820} 2023-08-27T20:52:07.239 INFO:tasks.rados.rados.0.smithi093.stdout:230: writing smithi093153278-230 from 648803 to 1371414 tid 1 2023-08-27T20:52:07.246 INFO:tasks.rados.rados.0.smithi093.stdout:230: writing smithi093153278-230 from 1967200 to 2681947 tid 2 2023-08-27T20:52:07.253 INFO:tasks.rados.rados.0.smithi093.stdout:230: writing smithi093153278-230 from 3421983 to 3848803 tid 3 2023-08-27T20:52:07.253 INFO:tasks.rados.rados.0.smithi093.stdout:231: write initial oid 231 2023-08-27T20:52:07.254 INFO:tasks.rados.rados.0.smithi093.stdout:231: seq_num 230 ranges {665610=797860,1909329=458405,2816700=410210} 2023-08-27T20:52:07.262 INFO:tasks.rados.rados.0.smithi093.stdout:231: writing smithi093153278-231 from 665610 to 1463470 tid 1 2023-08-27T20:52:07.268 INFO:tasks.rados.rados.0.smithi093.stdout:231: writing smithi093153278-231 from 1909329 to 2367734 tid 2 2023-08-27T20:52:07.274 INFO:tasks.rados.rados.0.smithi093.stdout:231: writing smithi093153278-231 from 2816700 to 3226910 tid 3 2023-08-27T20:52:07.274 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.274 INFO:tasks.rados.rados.0.smithi093.stdout:216: finishing write tid 3 to smithi093153278-216 2023-08-27T20:52:07.274 INFO:tasks.rados.rados.0.smithi093.stdout:216: finishing write tid 4 to smithi093153278-216 2023-08-27T20:52:07.274 INFO:tasks.rados.rados.0.smithi093.stdout:216: finishing write tid 5 to smithi093153278-216 2023-08-27T20:52:07.274 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 216 v 31 (ObjNum 215 snap 0 seq_num 215) dirty exists 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:216: left oid 216 (ObjNum 215 snap 0 seq_num 215) 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:217: finishing write tid 1 to smithi093153278-217 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:217: finishing write tid 2 to smithi093153278-217 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:217: finishing write tid 3 to smithi093153278-217 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:217: finishing write tid 4 to smithi093153278-217 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:216: done (15 left) 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:232: write initial oid 232 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:217: finishing write tid 5 to smithi093153278-217 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 217 v 52 (ObjNum 216 snap 0 seq_num 216) dirty exists 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:217: left oid 217 (ObjNum 216 snap 0 seq_num 216) 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:218: finishing write tid 1 to smithi093153278-218 2023-08-27T20:52:07.275 INFO:tasks.rados.rados.0.smithi093.stdout:232: seq_num 231 ranges {682417=473109,1567811=602063,2844123=621482} 2023-08-27T20:52:07.280 INFO:tasks.rados.rados.0.smithi093.stdout:232: writing smithi093153278-232 from 682417 to 1155526 tid 1 2023-08-27T20:52:07.286 INFO:tasks.rados.rados.0.smithi093.stdout:232: writing smithi093153278-232 from 1567811 to 2169874 tid 2 2023-08-27T20:52:07.294 INFO:tasks.rados.rados.0.smithi093.stdout:232: writing smithi093153278-232 from 2844123 to 3465605 tid 3 2023-08-27T20:52:07.295 INFO:tasks.rados.rados.0.smithi093.stdout:217: done (15 left) 2023-08-27T20:52:07.295 INFO:tasks.rados.rados.0.smithi093.stdout:233: write initial oid 233 2023-08-27T20:52:07.295 INFO:tasks.rados.rados.0.smithi093.stdout:233: seq_num 232 ranges {699224=548358,2026293=462074,3271546=432754} 2023-08-27T20:52:07.302 INFO:tasks.rados.rados.0.smithi093.stdout:233: writing smithi093153278-233 from 699224 to 1247582 tid 1 2023-08-27T20:52:07.309 INFO:tasks.rados.rados.0.smithi093.stdout:233: writing smithi093153278-233 from 2026293 to 2488367 tid 2 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:233: writing smithi093153278-233 from 3271546 to 3704300 tid 3 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:218: finishing write tid 2 to smithi093153278-218 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:218: finishing write tid 3 to smithi093153278-218 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:218: finishing write tid 4 to smithi093153278-218 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:218: finishing write tid 5 to smithi093153278-218 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 218 v 45 (ObjNum 217 snap 0 seq_num 217) dirty exists 2023-08-27T20:52:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:218: left oid 218 (ObjNum 217 snap 0 seq_num 217) 2023-08-27T20:52:07.318 INFO:tasks.rados.rados.0.smithi093.stdout:219: finishing write tid 1 to smithi093153278-219 2023-08-27T20:52:07.318 INFO:tasks.rados.rados.0.smithi093.stdout:219: finishing write tid 2 to smithi093153278-219 2023-08-27T20:52:07.318 INFO:tasks.rados.rados.0.smithi093.stdout:218: done (15 left) 2023-08-27T20:52:07.318 INFO:tasks.rados.rados.0.smithi093.stdout:234: write initial oid 234 2023-08-27T20:52:07.318 INFO:tasks.rados.rados.0.smithi093.stdout:234: seq_num 233 ranges {716031=623607,2084775=605732,3298969=617062} 2023-08-27T20:52:07.326 INFO:tasks.rados.rados.0.smithi093.stdout:234: writing smithi093153278-234 from 716031 to 1339638 tid 1 2023-08-27T20:52:07.333 INFO:tasks.rados.rados.0.smithi093.stdout:234: writing smithi093153278-234 from 2084775 to 2690507 tid 2 2023-08-27T20:52:07.341 INFO:tasks.rados.rados.0.smithi093.stdout:234: writing smithi093153278-234 from 3298969 to 3916031 tid 3 2023-08-27T20:52:07.341 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.341 INFO:tasks.rados.rados.0.smithi093.stdout:219: finishing write tid 3 to smithi093153278-219 2023-08-27T20:52:07.341 INFO:tasks.rados.rados.0.smithi093.stdout:219: finishing write tid 4 to smithi093153278-219 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:219: finishing write tid 5 to smithi093153278-219 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 219 v 61 (ObjNum 218 snap 0 seq_num 218) dirty exists 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:219: left oid 219 (ObjNum 218 snap 0 seq_num 218) 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:220: finishing write tid 1 to smithi093153278-220 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:220: finishing write tid 2 to smithi093153278-220 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:220: finishing write tid 3 to smithi093153278-220 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:220: finishing write tid 4 to smithi093153278-220 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:220: finishing write tid 5 to smithi093153278-220 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 220 v 55 (ObjNum 219 snap 0 seq_num 219) dirty exists 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:220: left oid 220 (ObjNum 219 snap 0 seq_num 219) 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:221: finishing write tid 1 to smithi093153278-221 2023-08-27T20:52:07.342 INFO:tasks.rados.rados.0.smithi093.stdout:221: finishing write tid 2 to smithi093153278-221 2023-08-27T20:52:07.343 INFO:tasks.rados.rados.0.smithi093.stdout:221: finishing write tid 3 to smithi093153278-221 2023-08-27T20:52:07.343 INFO:tasks.rados.rados.0.smithi093.stdout:221: finishing write tid 4 to smithi093153278-221 2023-08-27T20:52:07.343 INFO:tasks.rados.rados.0.smithi093.stdout:219: done (15 left) 2023-08-27T20:52:07.343 INFO:tasks.rados.rados.0.smithi093.stdout:220: done (14 left) 2023-08-27T20:52:07.343 INFO:tasks.rados.rados.0.smithi093.stdout:235: write initial oid 235 2023-08-27T20:52:07.343 INFO:tasks.rados.rados.0.smithi093.stdout:235: seq_num 234 ranges {732838=698856,2026904=465743,3210039=571651} 2023-08-27T20:52:07.350 INFO:tasks.rados.rados.0.smithi093.stdout:235: writing smithi093153278-235 from 732838 to 1431694 tid 1 2023-08-27T20:52:07.356 INFO:tasks.rados.rados.0.smithi093.stdout:235: writing smithi093153278-235 from 2026904 to 2492647 tid 2 2023-08-27T20:52:07.363 INFO:tasks.rados.rados.0.smithi093.stdout:235: writing smithi093153278-235 from 3210039 to 3781690 tid 3 2023-08-27T20:52:07.364 INFO:tasks.rados.rados.0.smithi093.stdout:236: write initial oid 236 2023-08-27T20:52:07.364 INFO:tasks.rados.rados.0.smithi093.stdout:236: seq_num 235 ranges {749645=774105,2085386=609401,3237462=712183} 2023-08-27T20:52:07.372 INFO:tasks.rados.rados.0.smithi093.stdout:236: writing smithi093153278-236 from 749645 to 1523750 tid 1 2023-08-27T20:52:07.380 INFO:tasks.rados.rados.0.smithi093.stdout:236: writing smithi093153278-236 from 2085386 to 2694787 tid 2 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout:236: writing smithi093153278-236 from 3237462 to 3949645 tid 3 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout:221: finishing write tid 5 to smithi093153278-221 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 221 v 55 (ObjNum 220 snap 0 seq_num 220) dirty exists 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout:221: left oid 221 (ObjNum 220 snap 0 seq_num 220) 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout:221: done (15 left) 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout:237: write initial oid 237 2023-08-27T20:52:07.389 INFO:tasks.rados.rados.0.smithi093.stdout:237: seq_num 236 ranges {766452=565707,1860221=469412,2981238=594195} 2023-08-27T20:52:07.397 INFO:tasks.rados.rados.0.smithi093.stdout:237: writing smithi093153278-237 from 766452 to 1332159 tid 1 2023-08-27T20:52:07.402 INFO:tasks.rados.rados.0.smithi093.stdout:237: writing smithi093153278-237 from 1860221 to 2329633 tid 2 2023-08-27T20:52:07.410 INFO:tasks.rados.rados.0.smithi093.stdout:237: writing smithi093153278-237 from 2981238 to 3575433 tid 3 2023-08-27T20:52:07.410 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.410 INFO:tasks.rados.rados.0.smithi093.stdout:222: finishing write tid 1 to smithi093153278-222 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:222: finishing write tid 2 to smithi093153278-222 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:222: finishing write tid 3 to smithi093153278-222 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:222: finishing write tid 4 to smithi093153278-222 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:222: finishing write tid 5 to smithi093153278-222 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 222 v 64 (ObjNum 221 snap 0 seq_num 221) dirty exists 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:222: left oid 222 (ObjNum 221 snap 0 seq_num 221) 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:223: finishing write tid 1 to smithi093153278-223 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:222: done (15 left) 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:238: write initial oid 238 2023-08-27T20:52:07.411 INFO:tasks.rados.rados.0.smithi093.stdout:238: seq_num 237 ranges {783259=640956,1918703=613070,3008661=405467,3821939=161320} 2023-08-27T20:52:07.418 INFO:tasks.rados.rados.0.smithi093.stdout:238: writing smithi093153278-238 from 783259 to 1424215 tid 1 2023-08-27T20:52:07.425 INFO:tasks.rados.rados.0.smithi093.stdout:238: writing smithi093153278-238 from 1918703 to 2531773 tid 2 2023-08-27T20:52:07.431 INFO:tasks.rados.rados.0.smithi093.stdout:238: writing smithi093153278-238 from 3008661 to 3414128 tid 3 2023-08-27T20:52:07.434 INFO:tasks.rados.rados.0.smithi093.stdout:238: writing smithi093153278-238 from 3821939 to 3983259 tid 4 2023-08-27T20:52:07.434 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.437 INFO:tasks.rados.rados.0.smithi093.stdout:223: finishing write tid 2 to smithi093153278-223 2023-08-27T20:52:07.437 INFO:tasks.rados.rados.0.smithi093.stdout:223: finishing write tid 3 to smithi093153278-223 2023-08-27T20:52:07.437 INFO:tasks.rados.rados.0.smithi093.stdout:223: finishing write tid 4 to smithi093153278-223 2023-08-27T20:52:07.437 INFO:tasks.rados.rados.0.smithi093.stdout:223: finishing write tid 5 to smithi093153278-223 2023-08-27T20:52:07.437 INFO:tasks.rados.rados.0.smithi093.stdout:223: finishing write tid 6 to smithi093153278-223 2023-08-27T20:52:07.437 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 223 v 45 (ObjNum 222 snap 0 seq_num 222) dirty exists 2023-08-27T20:52:07.437 INFO:tasks.rados.rados.0.smithi093.stdout:223: left oid 223 (ObjNum 222 snap 0 seq_num 222) 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:224: finishing write tid 1 to smithi093153278-224 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:224: finishing write tid 2 to smithi093153278-224 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:224: finishing write tid 3 to smithi093153278-224 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:224: finishing write tid 4 to smithi093153278-224 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:224: finishing write tid 5 to smithi093153278-224 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 224 v 53 (ObjNum 223 snap 0 seq_num 223) dirty exists 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:224: left oid 224 (ObjNum 223 snap 0 seq_num 223) 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:225: finishing write tid 1 to smithi093153278-225 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:225: finishing write tid 2 to smithi093153278-225 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:225: finishing write tid 3 to smithi093153278-225 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:225: finishing write tid 4 to smithi093153278-225 2023-08-27T20:52:07.438 INFO:tasks.rados.rados.0.smithi093.stdout:225: finishing write tid 5 to smithi093153278-225 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 225 v 65 (ObjNum 224 snap 0 seq_num 224) dirty exists 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:225: left oid 225 (ObjNum 224 snap 0 seq_num 224) 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:226: finishing write tid 1 to smithi093153278-226 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:226: finishing write tid 2 to smithi093153278-226 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:226: finishing write tid 3 to smithi093153278-226 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:226: finishing write tid 4 to smithi093153278-226 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:226: finishing write tid 5 to smithi093153278-226 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 226 v 50 (ObjNum 225 snap 0 seq_num 225) dirty exists 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:226: left oid 226 (ObjNum 225 snap 0 seq_num 225) 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:223: done (15 left) 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:224: done (14 left) 2023-08-27T20:52:07.439 INFO:tasks.rados.rados.0.smithi093.stdout:225: done (13 left) 2023-08-27T20:52:07.440 INFO:tasks.rados.rados.0.smithi093.stdout:226: done (12 left) 2023-08-27T20:52:07.440 INFO:tasks.rados.rados.0.smithi093.stdout:239: write initial oid 239 2023-08-27T20:52:07.440 INFO:tasks.rados.rados.0.smithi093.stdout:239: seq_num 238 ranges {400066=716205,1860832=139234} 2023-08-27T20:52:07.443 INFO:tasks.rados.rados.0.smithi093.stdout:239: writing smithi093153278-239 from 400066 to 1116271 tid 1 2023-08-27T20:52:07.448 INFO:tasks.rados.rados.0.smithi093.stdout:239: writing smithi093153278-239 from 1860832 to 2000066 tid 2 2023-08-27T20:52:07.448 INFO:tasks.rados.rados.0.smithi093.stdout:240: write initial oid 240 2023-08-27T20:52:07.449 INFO:tasks.rados.rados.0.smithi093.stdout:240: seq_num 239 ranges {416873=791454,1919314=97559} 2023-08-27T20:52:07.455 INFO:tasks.rados.rados.0.smithi093.stdout:240: writing smithi093153278-240 from 416873 to 1208327 tid 1 2023-08-27T20:52:07.462 INFO:tasks.rados.rados.0.smithi093.stdout:240: writing smithi093153278-240 from 1919314 to 2016873 tid 2 2023-08-27T20:52:07.462 INFO:tasks.rados.rados.0.smithi093.stdout:241: write initial oid 241 2023-08-27T20:52:07.462 INFO:tasks.rados.rados.0.smithi093.stdout:227: finishing write tid 1 to smithi093153278-227 2023-08-27T20:52:07.462 INFO:tasks.rados.rados.0.smithi093.stdout:227: finishing write tid 2 to smithi093153278-227 2023-08-27T20:52:07.462 INFO:tasks.rados.rados.0.smithi093.stdout:227: finishing write tid 3 to smithi093153278-227 2023-08-27T20:52:07.462 INFO:tasks.rados.rados.0.smithi093.stdout:227: finishing write tid 4 to smithi093153278-227 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:227: finishing write tid 5 to smithi093153278-227 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 227 v 45 (ObjNum 226 snap 0 seq_num 226) dirty exists 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:227: left oid 227 (ObjNum 226 snap 0 seq_num 226) 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:228: finishing write tid 1 to smithi093153278-228 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:228: finishing write tid 2 to smithi093153278-228 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:228: finishing write tid 3 to smithi093153278-228 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:228: finishing write tid 4 to smithi093153278-228 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:228: finishing write tid 5 to smithi093153278-228 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 228 v 57 (ObjNum 227 snap 0 seq_num 227) dirty exists 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:228: left oid 228 (ObjNum 227 snap 0 seq_num 227) 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:229: finishing write tid 1 to smithi093153278-229 2023-08-27T20:52:07.463 INFO:tasks.rados.rados.0.smithi093.stdout:229: finishing write tid 2 to smithi093153278-229 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:229: finishing write tid 3 to smithi093153278-229 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:229: finishing write tid 4 to smithi093153278-229 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:229: finishing write tid 5 to smithi093153278-229 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 229 v 46 (ObjNum 228 snap 0 seq_num 228) dirty exists 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:229: left oid 229 (ObjNum 228 snap 0 seq_num 228) 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:230: finishing write tid 1 to smithi093153278-230 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:230: finishing write tid 2 to smithi093153278-230 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:230: finishing write tid 3 to smithi093153278-230 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:230: finishing write tid 4 to smithi093153278-230 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:230: finishing write tid 5 to smithi093153278-230 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 50 (ObjNum 229 snap 0 seq_num 229) dirty exists 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:230: left oid 230 (ObjNum 229 snap 0 seq_num 229) 2023-08-27T20:52:07.464 INFO:tasks.rados.rados.0.smithi093.stdout:231: finishing write tid 1 to smithi093153278-231 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:231: finishing write tid 2 to smithi093153278-231 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:231: finishing write tid 3 to smithi093153278-231 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:231: finishing write tid 4 to smithi093153278-231 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:231: finishing write tid 5 to smithi093153278-231 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 231 v 56 (ObjNum 230 snap 0 seq_num 230) dirty exists 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:231: left oid 231 (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:232: finishing write tid 1 to smithi093153278-232 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:232: finishing write tid 2 to smithi093153278-232 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:232: finishing write tid 3 to smithi093153278-232 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:232: finishing write tid 4 to smithi093153278-232 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:232: finishing write tid 5 to smithi093153278-232 2023-08-27T20:52:07.465 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 232 v 35 (ObjNum 231 snap 0 seq_num 231) dirty exists 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:232: left oid 232 (ObjNum 231 snap 0 seq_num 231) 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:233: finishing write tid 1 to smithi093153278-233 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:233: finishing write tid 2 to smithi093153278-233 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:233: finishing write tid 3 to smithi093153278-233 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:233: finishing write tid 4 to smithi093153278-233 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:233: finishing write tid 5 to smithi093153278-233 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 233 v 54 (ObjNum 232 snap 0 seq_num 232) dirty exists 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:233: left oid 233 (ObjNum 232 snap 0 seq_num 232) 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:234: finishing write tid 1 to smithi093153278-234 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:234: finishing write tid 2 to smithi093153278-234 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:234: finishing write tid 3 to smithi093153278-234 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:234: finishing write tid 4 to smithi093153278-234 2023-08-27T20:52:07.466 INFO:tasks.rados.rados.0.smithi093.stdout:234: finishing write tid 5 to smithi093153278-234 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 234 v 59 (ObjNum 233 snap 0 seq_num 233) dirty exists 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:234: left oid 234 (ObjNum 233 snap 0 seq_num 233) 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:235: finishing write tid 1 to smithi093153278-235 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:235: finishing write tid 2 to smithi093153278-235 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:235: finishing write tid 3 to smithi093153278-235 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:235: finishing write tid 4 to smithi093153278-235 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:235: finishing write tid 5 to smithi093153278-235 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 235 v 51 (ObjNum 234 snap 0 seq_num 234) dirty exists 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:235: left oid 235 (ObjNum 234 snap 0 seq_num 234) 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:236: finishing write tid 1 to smithi093153278-236 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:236: finishing write tid 2 to smithi093153278-236 2023-08-27T20:52:07.467 INFO:tasks.rados.rados.0.smithi093.stdout:236: finishing write tid 3 to smithi093153278-236 2023-08-27T20:52:07.468 INFO:tasks.rados.rados.0.smithi093.stdout:236: finishing write tid 4 to smithi093153278-236 2023-08-27T20:52:07.468 INFO:tasks.rados.rados.0.smithi093.stdout:241: seq_num 240 ranges {433680=466703,1577796=455884} 2023-08-27T20:52:07.468 INFO:tasks.rados.rados.0.smithi093.stdout:241: writing smithi093153278-241 from 433680 to 900383 tid 1 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:241: writing smithi093153278-241 from 1577796 to 2033680 tid 2 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:227: done (14 left) 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:228: done (13 left) 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:229: done (12 left) 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:230: done (11 left) 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:231: done (10 left) 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:232: done (9 left) 2023-08-27T20:52:07.474 INFO:tasks.rados.rados.0.smithi093.stdout:233: done (8 left) 2023-08-27T20:52:07.475 INFO:tasks.rados.rados.0.smithi093.stdout:234: done (7 left) 2023-08-27T20:52:07.475 INFO:tasks.rados.rados.0.smithi093.stdout:235: done (6 left) 2023-08-27T20:52:07.475 INFO:tasks.rados.rados.0.smithi093.stdout:242: write initial oid 242 2023-08-27T20:52:07.475 INFO:tasks.rados.rados.0.smithi093.stdout:242: seq_num 241 ranges {450487=541952,1636278=414209} 2023-08-27T20:52:07.480 INFO:tasks.rados.rados.0.smithi093.stdout:242: writing smithi093153278-242 from 450487 to 992439 tid 1 2023-08-27T20:52:07.487 INFO:tasks.rados.rados.0.smithi093.stdout:242: writing smithi093153278-242 from 1636278 to 2050487 tid 2 2023-08-27T20:52:07.487 INFO:tasks.rados.rados.0.smithi093.stdout:243: write initial oid 243 2023-08-27T20:52:07.487 INFO:tasks.rados.rados.0.smithi093.stdout:243: seq_num 242 ranges {467294=617201,1578407=488887} 2023-08-27T20:52:07.493 INFO:tasks.rados.rados.0.smithi093.stdout:243: writing smithi093153278-243 from 467294 to 1084495 tid 1 2023-08-27T20:52:07.499 INFO:tasks.rados.rados.0.smithi093.stdout:243: writing smithi093153278-243 from 1578407 to 2067294 tid 2 2023-08-27T20:52:07.499 INFO:tasks.rados.rados.0.smithi093.stdout:244: write initial oid 244 2023-08-27T20:52:07.499 INFO:tasks.rados.rados.0.smithi093.stdout:244: seq_num 243 ranges {484101=692450,1636889=447212} 2023-08-27T20:52:07.506 INFO:tasks.rados.rados.0.smithi093.stdout:244: writing smithi093153278-244 from 484101 to 1176551 tid 1 2023-08-27T20:52:07.513 INFO:tasks.rados.rados.0.smithi093.stdout:244: writing smithi093153278-244 from 1636889 to 2084101 tid 2 2023-08-27T20:52:07.513 INFO:tasks.rados.rados.0.smithi093.stdout:245: write initial oid 245 2023-08-27T20:52:07.513 INFO:tasks.rados.rados.0.smithi093.stdout:245: seq_num 244 ranges {500908=484052,1411724=689184} 2023-08-27T20:52:07.518 INFO:tasks.rados.rados.0.smithi093.stdout:245: writing smithi093153278-245 from 500908 to 984960 tid 1 2023-08-27T20:52:07.525 INFO:tasks.rados.rados.0.smithi093.stdout:245: writing smithi093153278-245 from 1411724 to 2100908 tid 2 2023-08-27T20:52:07.525 INFO:tasks.rados.rados.0.smithi093.stdout:246: write initial oid 246 2023-08-27T20:52:07.526 INFO:tasks.rados.rados.0.smithi093.stdout:246: seq_num 245 ranges {517715=559301,1870206=247509} 2023-08-27T20:52:07.532 INFO:tasks.rados.rados.0.smithi093.stdout:246: writing smithi093153278-246 from 517715 to 1077016 tid 1 2023-08-27T20:52:07.538 INFO:tasks.rados.rados.0.smithi093.stdout:246: writing smithi093153278-246 from 1870206 to 2117715 tid 2 2023-08-27T20:52:07.538 INFO:tasks.rados.rados.0.smithi093.stdout:247: write initial oid 247 2023-08-27T20:52:07.539 INFO:tasks.rados.rados.0.smithi093.stdout:236: finishing write tid 5 to smithi093153278-236 2023-08-27T20:52:07.539 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 236 v 55 (ObjNum 235 snap 0 seq_num 235) dirty exists 2023-08-27T20:52:07.539 INFO:tasks.rados.rados.0.smithi093.stdout:236: left oid 236 (ObjNum 235 snap 0 seq_num 235) 2023-08-27T20:52:07.539 INFO:tasks.rados.rados.0.smithi093.stdout:247: seq_num 246 ranges {534522=634550,1812335=322187} 2023-08-27T20:52:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:247: writing smithi093153278-247 from 534522 to 1169072 tid 1 2023-08-27T20:52:07.551 INFO:tasks.rados.rados.0.smithi093.stdout:247: writing smithi093153278-247 from 1812335 to 2134522 tid 2 2023-08-27T20:52:07.551 INFO:tasks.rados.rados.0.smithi093.stdout:236: done (11 left) 2023-08-27T20:52:07.551 INFO:tasks.rados.rados.0.smithi093.stdout:248: write initial oid 248 2023-08-27T20:52:07.551 INFO:tasks.rados.rados.0.smithi093.stdout:248: seq_num 247 ranges {551329=709799,1870817=280512} 2023-08-27T20:52:07.559 INFO:tasks.rados.rados.0.smithi093.stdout:248: writing smithi093153278-248 from 551329 to 1261128 tid 1 2023-08-27T20:52:07.565 INFO:tasks.rados.rados.0.smithi093.stdout:248: writing smithi093153278-248 from 1870817 to 2151329 tid 2 2023-08-27T20:52:07.565 INFO:tasks.rados.rados.0.smithi093.stdout:249: write initial oid 249 2023-08-27T20:52:07.565 INFO:tasks.rados.rados.0.smithi093.stdout:249: seq_num 248 ranges {568136=785048,1929299=238837} 2023-08-27T20:52:07.572 INFO:tasks.rados.rados.0.smithi093.stdout:249: writing smithi093153278-249 from 568136 to 1353184 tid 1 2023-08-27T20:52:07.577 INFO:tasks.rados.rados.0.smithi093.stdout:249: writing smithi093153278-249 from 1929299 to 2168136 tid 2 2023-08-27T20:52:07.577 INFO:tasks.rados.rados.0.smithi093.stdout:250: write initial oid 250 2023-08-27T20:52:07.578 INFO:tasks.rados.rados.0.smithi093.stdout:237: finishing write tid 1 to smithi093153278-237 2023-08-27T20:52:07.578 INFO:tasks.rados.rados.0.smithi093.stdout:237: finishing write tid 2 to smithi093153278-237 2023-08-27T20:52:07.578 INFO:tasks.rados.rados.0.smithi093.stdout:250: seq_num 249 ranges {584943=460297,1587781=597162} 2023-08-27T20:52:07.583 INFO:tasks.rados.rados.0.smithi093.stdout:250: writing smithi093153278-250 from 584943 to 1045240 tid 1 2023-08-27T20:52:07.590 INFO:tasks.rados.rados.0.smithi093.stdout:250: writing smithi093153278-250 from 1587781 to 2184943 tid 2 2023-08-27T20:52:07.590 INFO:tasks.rados.rados.0.smithi093.stdout:251: write initial oid 251 2023-08-27T20:52:07.591 INFO:tasks.rados.rados.0.smithi093.stdout:237: finishing write tid 3 to smithi093153278-237 2023-08-27T20:52:07.591 INFO:tasks.rados.rados.0.smithi093.stdout:251: seq_num 250 ranges {601750=535546,1929910=271840} 2023-08-27T20:52:07.596 INFO:tasks.rados.rados.0.smithi093.stdout:251: writing smithi093153278-251 from 601750 to 1137296 tid 1 2023-08-27T20:52:07.603 INFO:tasks.rados.rados.0.smithi093.stdout:251: writing smithi093153278-251 from 1929910 to 2201750 tid 2 2023-08-27T20:52:07.603 INFO:tasks.rados.rados.0.smithi093.stdout:252: write initial oid 252 2023-08-27T20:52:07.603 INFO:tasks.rados.rados.0.smithi093.stdout:252: seq_num 251 ranges {618557=727148,2104745=113812} 2023-08-27T20:52:07.611 INFO:tasks.rados.rados.0.smithi093.stdout:252: writing smithi093153278-252 from 618557 to 1345705 tid 1 2023-08-27T20:52:07.616 INFO:tasks.rados.rados.0.smithi093.stdout:252: writing smithi093153278-252 from 2104745 to 2218557 tid 2 2023-08-27T20:52:07.616 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:237: finishing write tid 4 to smithi093153278-237 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:237: finishing write tid 5 to smithi093153278-237 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 237 v 58 (ObjNum 236 snap 0 seq_num 236) dirty exists 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:237: left oid 237 (ObjNum 236 snap 0 seq_num 236) 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:238: finishing write tid 1 to smithi093153278-238 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:237: done (15 left) 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:253: write initial oid 253 2023-08-27T20:52:07.620 INFO:tasks.rados.rados.0.smithi093.stdout:253: seq_num 252 ranges {635364=402397,1763227=472137} 2023-08-27T20:52:07.626 INFO:tasks.rados.rados.0.smithi093.stdout:253: writing smithi093153278-253 from 635364 to 1037761 tid 1 2023-08-27T20:52:07.634 INFO:tasks.rados.rados.0.smithi093.stdout:253: writing smithi093153278-253 from 1763227 to 2235364 tid 2 2023-08-27T20:52:07.634 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.634 INFO:tasks.rados.rados.0.smithi093.stdout:238: finishing write tid 2 to smithi093153278-238 2023-08-27T20:52:07.634 INFO:tasks.rados.rados.0.smithi093.stdout:238: finishing write tid 3 to smithi093153278-238 2023-08-27T20:52:07.634 INFO:tasks.rados.rados.0.smithi093.stdout:238: finishing write tid 4 to smithi093153278-238 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:238: finishing write tid 5 to smithi093153278-238 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:238: finishing write tid 6 to smithi093153278-238 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 238 v 60 (ObjNum 237 snap 0 seq_num 237) dirty exists 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:238: left oid 238 (ObjNum 237 snap 0 seq_num 237) 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:239: finishing write tid 1 to smithi093153278-239 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:239: finishing write tid 2 to smithi093153278-239 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:239: finishing write tid 3 to smithi093153278-239 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:239: finishing write tid 4 to smithi093153278-239 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 239 v 58 (ObjNum 238 snap 0 seq_num 238) dirty exists 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:239: left oid 239 (ObjNum 238 snap 0 seq_num 238) 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:240: finishing write tid 1 to smithi093153278-240 2023-08-27T20:52:07.635 INFO:tasks.rados.rados.0.smithi093.stdout:240: finishing write tid 2 to smithi093153278-240 2023-08-27T20:52:07.636 INFO:tasks.rados.rados.0.smithi093.stdout:238: done (15 left) 2023-08-27T20:52:07.636 INFO:tasks.rados.rados.0.smithi093.stdout:239: done (14 left) 2023-08-27T20:52:07.636 INFO:tasks.rados.rados.0.smithi093.stdout:254: write initial oid 254 2023-08-27T20:52:07.636 INFO:tasks.rados.rados.0.smithi093.stdout:254: seq_num 253 ranges {652171=477646,1821709=430462} 2023-08-27T20:52:07.640 INFO:tasks.rados.rados.0.smithi093.stdout:254: writing smithi093153278-254 from 652171 to 1129817 tid 1 2023-08-27T20:52:07.648 INFO:tasks.rados.rados.0.smithi093.stdout:254: writing smithi093153278-254 from 1821709 to 2252171 tid 2 2023-08-27T20:52:07.648 INFO:tasks.rados.rados.0.smithi093.stdout:255: write initial oid 255 2023-08-27T20:52:07.648 INFO:tasks.rados.rados.0.smithi093.stdout:255: seq_num 254 ranges {668978=552895,1763838=505140} 2023-08-27T20:52:07.656 INFO:tasks.rados.rados.0.smithi093.stdout:255: writing smithi093153278-255 from 668978 to 1221873 tid 1 2023-08-27T20:52:07.663 INFO:tasks.rados.rados.0.smithi093.stdout:255: writing smithi093153278-255 from 1763838 to 2268978 tid 2 2023-08-27T20:52:07.663 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.663 INFO:tasks.rados.rados.0.smithi093.stdout:240: finishing write tid 3 to smithi093153278-240 2023-08-27T20:52:07.663 INFO:tasks.rados.rados.0.smithi093.stdout:240: finishing write tid 4 to smithi093153278-240 2023-08-27T20:52:07.663 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 240 v 40 (ObjNum 239 snap 0 seq_num 239) dirty exists 2023-08-27T20:52:07.663 INFO:tasks.rados.rados.0.smithi093.stdout:240: left oid 240 (ObjNum 239 snap 0 seq_num 239) 2023-08-27T20:52:07.663 INFO:tasks.rados.rados.0.smithi093.stdout:241: finishing write tid 1 to smithi093153278-241 2023-08-27T20:52:07.664 INFO:tasks.rados.rados.0.smithi093.stdout:241: finishing write tid 2 to smithi093153278-241 2023-08-27T20:52:07.664 INFO:tasks.rados.rados.0.smithi093.stdout:240: done (15 left) 2023-08-27T20:52:07.664 INFO:tasks.rados.rados.0.smithi093.stdout:256: write initial oid 256 2023-08-27T20:52:07.664 INFO:tasks.rados.rados.0.smithi093.stdout:256: seq_num 255 ranges {685785=628144,1822320=463465} 2023-08-27T20:52:07.671 INFO:tasks.rados.rados.0.smithi093.stdout:256: writing smithi093153278-256 from 685785 to 1313929 tid 1 2023-08-27T20:52:07.677 INFO:tasks.rados.rados.0.smithi093.stdout:256: writing smithi093153278-256 from 1822320 to 2285785 tid 2 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout:241: finishing write tid 3 to smithi093153278-241 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout:241: finishing write tid 4 to smithi093153278-241 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 241 v 48 (ObjNum 240 snap 0 seq_num 240) dirty exists 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout:241: left oid 241 (ObjNum 240 snap 0 seq_num 240) 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout:242: finishing write tid 1 to smithi093153278-242 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout:242: finishing write tid 2 to smithi093153278-242 2023-08-27T20:52:07.678 INFO:tasks.rados.rados.0.smithi093.stdout:242: finishing write tid 3 to smithi093153278-242 2023-08-27T20:52:07.679 INFO:tasks.rados.rados.0.smithi093.stdout:241: done (15 left) 2023-08-27T20:52:07.679 INFO:tasks.rados.rados.0.smithi093.stdout:257: write initial oid 257 2023-08-27T20:52:07.679 INFO:tasks.rados.rados.0.smithi093.stdout:257: seq_num 256 ranges {702592=703393,1880802=421790} 2023-08-27T20:52:07.685 INFO:tasks.rados.rados.0.smithi093.stdout:257: writing smithi093153278-257 from 702592 to 1405985 tid 1 2023-08-27T20:52:07.691 INFO:tasks.rados.rados.0.smithi093.stdout:257: writing smithi093153278-257 from 1880802 to 2302592 tid 2 2023-08-27T20:52:07.691 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.691 INFO:tasks.rados.rados.0.smithi093.stdout:242: finishing write tid 4 to smithi093153278-242 2023-08-27T20:52:07.691 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 242 v 43 (ObjNum 241 snap 0 seq_num 241) dirty exists 2023-08-27T20:52:07.692 INFO:tasks.rados.rados.0.smithi093.stdout:242: left oid 242 (ObjNum 241 snap 0 seq_num 241) 2023-08-27T20:52:07.692 INFO:tasks.rados.rados.0.smithi093.stdout:242: done (15 left) 2023-08-27T20:52:07.692 INFO:tasks.rados.rados.0.smithi093.stdout:258: write initial oid 258 2023-08-27T20:52:07.692 INFO:tasks.rados.rados.0.smithi093.stdout:258: seq_num 257 ranges {719399=778642,1939284=380115} 2023-08-27T20:52:07.699 INFO:tasks.rados.rados.0.smithi093.stdout:258: writing smithi093153278-258 from 719399 to 1498041 tid 1 2023-08-27T20:52:07.705 INFO:tasks.rados.rados.0.smithi093.stdout:258: writing smithi093153278-258 from 1939284 to 2319399 tid 2 2023-08-27T20:52:07.705 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.705 INFO:tasks.rados.rados.0.smithi093.stdout:243: finishing write tid 1 to smithi093153278-243 2023-08-27T20:52:07.705 INFO:tasks.rados.rados.0.smithi093.stdout:243: finishing write tid 2 to smithi093153278-243 2023-08-27T20:52:07.705 INFO:tasks.rados.rados.0.smithi093.stdout:243: finishing write tid 3 to smithi093153278-243 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:243: finishing write tid 4 to smithi093153278-243 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 243 v 59 (ObjNum 242 snap 0 seq_num 242) dirty exists 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:243: left oid 243 (ObjNum 242 snap 0 seq_num 242) 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:244: finishing write tid 1 to smithi093153278-244 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:244: finishing write tid 2 to smithi093153278-244 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:244: finishing write tid 3 to smithi093153278-244 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:243: done (15 left) 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:259: write initial oid 259 2023-08-27T20:52:07.706 INFO:tasks.rados.rados.0.smithi093.stdout:259: seq_num 258 ranges {736206=453891,1881413=454793} 2023-08-27T20:52:07.711 INFO:tasks.rados.rados.0.smithi093.stdout:259: writing smithi093153278-259 from 736206 to 1190097 tid 1 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:259: writing smithi093153278-259 from 1881413 to 2336206 tid 2 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:244: finishing write tid 4 to smithi093153278-244 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 244 v 48 (ObjNum 243 snap 0 seq_num 243) dirty exists 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:244: left oid 244 (ObjNum 243 snap 0 seq_num 243) 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:245: finishing write tid 1 to smithi093153278-245 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:245: finishing write tid 2 to smithi093153278-245 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:245: finishing write tid 3 to smithi093153278-245 2023-08-27T20:52:07.719 INFO:tasks.rados.rados.0.smithi093.stdout:245: finishing write tid 4 to smithi093153278-245 2023-08-27T20:52:07.720 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 245 v 63 (ObjNum 244 snap 0 seq_num 244) dirty exists 2023-08-27T20:52:07.720 INFO:tasks.rados.rados.0.smithi093.stdout:245: left oid 245 (ObjNum 244 snap 0 seq_num 244) 2023-08-27T20:52:07.720 INFO:tasks.rados.rados.0.smithi093.stdout:246: finishing write tid 1 to smithi093153278-246 2023-08-27T20:52:07.720 INFO:tasks.rados.rados.0.smithi093.stdout:244: done (15 left) 2023-08-27T20:52:07.720 INFO:tasks.rados.rados.0.smithi093.stdout:245: done (14 left) 2023-08-27T20:52:07.720 INFO:tasks.rados.rados.0.smithi093.stdout:260: write initial oid 260 2023-08-27T20:52:07.720 INFO:tasks.rados.rados.0.smithi093.stdout:260: seq_num 259 ranges {753013=645493,2056248=296765} 2023-08-27T20:52:07.726 INFO:tasks.rados.rados.0.smithi093.stdout:260: writing smithi093153278-260 from 753013 to 1398506 tid 1 2023-08-27T20:52:07.732 INFO:tasks.rados.rados.0.smithi093.stdout:260: writing smithi093153278-260 from 2056248 to 2353013 tid 2 2023-08-27T20:52:07.732 INFO:tasks.rados.rados.0.smithi093.stdout:261: write initial oid 261 2023-08-27T20:52:07.732 INFO:tasks.rados.rados.0.smithi093.stdout:261: seq_num 260 ranges {769820=720742,2114730=255090} 2023-08-27T20:52:07.741 INFO:tasks.rados.rados.0.smithi093.stdout:261: writing smithi093153278-261 from 769820 to 1490562 tid 1 2023-08-27T20:52:07.747 INFO:tasks.rados.rados.0.smithi093.stdout:261: writing smithi093153278-261 from 2114730 to 2369820 tid 2 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:246: finishing write tid 2 to smithi093153278-246 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:246: finishing write tid 3 to smithi093153278-246 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:246: finishing write tid 4 to smithi093153278-246 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 246 v 51 (ObjNum 245 snap 0 seq_num 245) dirty exists 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:246: left oid 246 (ObjNum 245 snap 0 seq_num 245) 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:247: finishing write tid 1 to smithi093153278-247 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:247: finishing write tid 2 to smithi093153278-247 2023-08-27T20:52:07.748 INFO:tasks.rados.rados.0.smithi093.stdout:247: finishing write tid 3 to smithi093153278-247 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:247: finishing write tid 4 to smithi093153278-247 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 247 v 54 (ObjNum 246 snap 0 seq_num 246) dirty exists 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:247: left oid 247 (ObjNum 246 snap 0 seq_num 246) 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:248: finishing write tid 1 to smithi093153278-248 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:246: done (15 left) 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:247: done (14 left) 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:262: write initial oid 262 2023-08-27T20:52:07.749 INFO:tasks.rados.rados.0.smithi093.stdout:262: seq_num 261 ranges {786627=795991,2173212=213415} 2023-08-27T20:52:07.755 INFO:tasks.rados.rados.0.smithi093.stdout:262: writing smithi093153278-262 from 786627 to 1582618 tid 1 2023-08-27T20:52:07.760 INFO:tasks.rados.rados.0.smithi093.stdout:262: writing smithi093153278-262 from 2173212 to 2386627 tid 2 2023-08-27T20:52:07.760 INFO:tasks.rados.rados.0.smithi093.stdout:263: write initial oid 263 2023-08-27T20:52:07.760 INFO:tasks.rados.rados.0.smithi093.stdout:263: seq_num 262 ranges {403434=471240,1315341=400756} 2023-08-27T20:52:07.764 INFO:tasks.rados.rados.0.smithi093.stdout:263: writing smithi093153278-263 from 403434 to 874674 tid 1 2023-08-27T20:52:07.771 INFO:tasks.rados.rados.0.smithi093.stdout:263: writing smithi093153278-263 from 1315341 to 1716097 tid 2 2023-08-27T20:52:07.771 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:248: finishing write tid 2 to smithi093153278-248 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:248: finishing write tid 3 to smithi093153278-248 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:248: finishing write tid 4 to smithi093153278-248 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 248 v 51 (ObjNum 247 snap 0 seq_num 247) dirty exists 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:248: left oid 248 (ObjNum 247 snap 0 seq_num 247) 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:249: finishing write tid 1 to smithi093153278-249 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:249: finishing write tid 2 to smithi093153278-249 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:249: finishing write tid 3 to smithi093153278-249 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:249: finishing write tid 4 to smithi093153278-249 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 249 v 61 (ObjNum 248 snap 0 seq_num 248) dirty exists 2023-08-27T20:52:07.772 INFO:tasks.rados.rados.0.smithi093.stdout:249: left oid 249 (ObjNum 248 snap 0 seq_num 248) 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:250: finishing write tid 1 to smithi093153278-250 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:250: finishing write tid 2 to smithi093153278-250 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:250: finishing write tid 3 to smithi093153278-250 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:250: finishing write tid 4 to smithi093153278-250 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 250 v 66 (ObjNum 249 snap 0 seq_num 249) dirty exists 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:250: left oid 250 (ObjNum 249 snap 0 seq_num 249) 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:251: finishing write tid 1 to smithi093153278-251 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:251: finishing write tid 2 to smithi093153278-251 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:251: finishing write tid 3 to smithi093153278-251 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:251: finishing write tid 4 to smithi093153278-251 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 251 v 67 (ObjNum 250 snap 0 seq_num 250) dirty exists 2023-08-27T20:52:07.773 INFO:tasks.rados.rados.0.smithi093.stdout:251: left oid 251 (ObjNum 250 snap 0 seq_num 250) 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:252: finishing write tid 1 to smithi093153278-252 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:252: finishing write tid 2 to smithi093153278-252 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:252: finishing write tid 3 to smithi093153278-252 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:252: finishing write tid 4 to smithi093153278-252 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 252 v 57 (ObjNum 251 snap 0 seq_num 251) dirty exists 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:252: left oid 252 (ObjNum 251 snap 0 seq_num 251) 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:253: finishing write tid 1 to smithi093153278-253 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:253: finishing write tid 2 to smithi093153278-253 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:253: finishing write tid 3 to smithi093153278-253 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:253: finishing write tid 4 to smithi093153278-253 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 38 (ObjNum 252 snap 0 seq_num 252) dirty exists 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:253: left oid 253 (ObjNum 252 snap 0 seq_num 252) 2023-08-27T20:52:07.774 INFO:tasks.rados.rados.0.smithi093.stdout:254: finishing write tid 1 to smithi093153278-254 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:254: finishing write tid 2 to smithi093153278-254 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:254: finishing write tid 3 to smithi093153278-254 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:254: finishing write tid 4 to smithi093153278-254 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 254 v 68 (ObjNum 253 snap 0 seq_num 253) dirty exists 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:254: left oid 254 (ObjNum 253 snap 0 seq_num 253) 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:255: finishing write tid 1 to smithi093153278-255 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:255: finishing write tid 2 to smithi093153278-255 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:255: finishing write tid 3 to smithi093153278-255 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:255: finishing write tid 4 to smithi093153278-255 2023-08-27T20:52:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 255 v 69 (ObjNum 254 snap 0 seq_num 254) dirty exists 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:255: left oid 255 (ObjNum 254 snap 0 seq_num 254) 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:256: finishing write tid 1 to smithi093153278-256 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:256: finishing write tid 2 to smithi093153278-256 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:256: finishing write tid 3 to smithi093153278-256 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:256: finishing write tid 4 to smithi093153278-256 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 256 v 71 (ObjNum 255 snap 0 seq_num 255) dirty exists 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:256: left oid 256 (ObjNum 255 snap 0 seq_num 255) 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:257: finishing write tid 1 to smithi093153278-257 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:257: finishing write tid 2 to smithi093153278-257 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:257: finishing write tid 3 to smithi093153278-257 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:257: finishing write tid 4 to smithi093153278-257 2023-08-27T20:52:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 257 v 72 (ObjNum 256 snap 0 seq_num 256) dirty exists 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:257: left oid 257 (ObjNum 256 snap 0 seq_num 256) 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:258: finishing write tid 1 to smithi093153278-258 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:258: finishing write tid 2 to smithi093153278-258 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:258: finishing write tid 3 to smithi093153278-258 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:258: finishing write tid 4 to smithi093153278-258 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 258 v 46 (ObjNum 257 snap 0 seq_num 257) dirty exists 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:258: left oid 258 (ObjNum 257 snap 0 seq_num 257) 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:259: finishing write tid 1 to smithi093153278-259 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:259: finishing write tid 2 to smithi093153278-259 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:259: finishing write tid 3 to smithi093153278-259 2023-08-27T20:52:07.777 INFO:tasks.rados.rados.0.smithi093.stdout:259: finishing write tid 4 to smithi093153278-259 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 259 v 55 (ObjNum 258 snap 0 seq_num 258) dirty exists 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:259: left oid 259 (ObjNum 258 snap 0 seq_num 258) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:260: finishing write tid 1 to smithi093153278-260 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:248: done (15 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:249: done (14 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:250: done (13 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:251: done (12 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:252: done (11 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:253: done (10 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:254: done (9 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:255: done (8 left) 2023-08-27T20:52:07.778 INFO:tasks.rados.rados.0.smithi093.stdout:256: done (7 left) 2023-08-27T20:52:07.779 INFO:tasks.rados.rados.0.smithi093.stdout:257: done (6 left) 2023-08-27T20:52:07.779 INFO:tasks.rados.rados.0.smithi093.stdout:258: done (5 left) 2023-08-27T20:52:07.779 INFO:tasks.rados.rados.0.smithi093.stdout:259: done (4 left) 2023-08-27T20:52:07.779 INFO:tasks.rados.rados.0.smithi093.stdout:264: write initial oid 264 2023-08-27T20:52:07.779 INFO:tasks.rados.rados.0.smithi093.stdout:264: seq_num 263 ranges {420241=546489,1373823=544414} 2023-08-27T20:52:07.779 INFO:tasks.rados.rados.0.smithi093.stdout:264: writing smithi093153278-264 from 420241 to 966730 tid 1 2023-08-27T20:52:07.781 INFO:tasks.rados.rados.0.smithi093.stdout:264: writing smithi093153278-264 from 1373823 to 1918237 tid 2 2023-08-27T20:52:07.781 INFO:tasks.rados.rados.0.smithi093.stdout:265: write initial oid 265 2023-08-27T20:52:07.781 INFO:tasks.rados.rados.0.smithi093.stdout:265: seq_num 264 ranges {437048=621738,1832305=404425} 2023-08-27T20:52:07.787 INFO:tasks.rados.rados.0.smithi093.stdout:265: writing smithi093153278-265 from 437048 to 1058786 tid 1 2023-08-27T20:52:07.793 INFO:tasks.rados.rados.0.smithi093.stdout:265: writing smithi093153278-265 from 1832305 to 2236730 tid 2 2023-08-27T20:52:07.794 INFO:tasks.rados.rados.0.smithi093.stdout:266: write initial oid 266 2023-08-27T20:52:07.794 INFO:tasks.rados.rados.0.smithi093.stdout:266: seq_num 265 ranges {453855=696987,1890787=548083} 2023-08-27T20:52:07.800 INFO:tasks.rados.rados.0.smithi093.stdout:266: writing smithi093153278-266 from 453855 to 1150842 tid 1 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:266: writing smithi093153278-266 from 1890787 to 2438870 tid 2 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:267: write initial oid 267 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:260: finishing write tid 2 to smithi093153278-260 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:260: finishing write tid 3 to smithi093153278-260 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:260: finishing write tid 4 to smithi093153278-260 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 260 v 70 (ObjNum 259 snap 0 seq_num 259) dirty exists 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:260: left oid 260 (ObjNum 259 snap 0 seq_num 259) 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:261: finishing write tid 1 to smithi093153278-261 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:261: finishing write tid 2 to smithi093153278-261 2023-08-27T20:52:07.808 INFO:tasks.rados.rados.0.smithi093.stdout:267: seq_num 266 ranges {470662=772236,1832916=408094} 2023-08-27T20:52:07.814 INFO:tasks.rados.rados.0.smithi093.stdout:267: writing smithi093153278-267 from 470662 to 1242898 tid 1 2023-08-27T20:52:07.820 INFO:tasks.rados.rados.0.smithi093.stdout:267: writing smithi093153278-267 from 1832916 to 2241010 tid 2 2023-08-27T20:52:07.820 INFO:tasks.rados.rados.0.smithi093.stdout:260: done (7 left) 2023-08-27T20:52:07.820 INFO:tasks.rados.rados.0.smithi093.stdout:268: write initial oid 268 2023-08-27T20:52:07.820 INFO:tasks.rados.rados.0.smithi093.stdout:268: seq_num 267 ranges {487469=563838,1607751=551752} 2023-08-27T20:52:07.826 INFO:tasks.rados.rados.0.smithi093.stdout:268: writing smithi093153278-268 from 487469 to 1051307 tid 1 2023-08-27T20:52:07.832 INFO:tasks.rados.rados.0.smithi093.stdout:268: writing smithi093153278-268 from 1607751 to 2159503 tid 2 2023-08-27T20:52:07.832 INFO:tasks.rados.rados.0.smithi093.stdout:269: write initial oid 269 2023-08-27T20:52:07.832 INFO:tasks.rados.rados.0.smithi093.stdout:269: seq_num 268 ranges {504276=639087,1666233=695410} 2023-08-27T20:52:07.839 INFO:tasks.rados.rados.0.smithi093.stdout:269: writing smithi093153278-269 from 504276 to 1143363 tid 1 2023-08-27T20:52:07.845 INFO:tasks.rados.rados.0.smithi093.stdout:269: writing smithi093153278-269 from 1666233 to 2361643 tid 2 2023-08-27T20:52:07.846 INFO:tasks.rados.rados.0.smithi093.stdout:270: write initial oid 270 2023-08-27T20:52:07.846 INFO:tasks.rados.rados.0.smithi093.stdout:270: seq_num 269 ranges {521083=714336,1724715=555421} 2023-08-27T20:52:07.853 INFO:tasks.rados.rados.0.smithi093.stdout:270: writing smithi093153278-270 from 521083 to 1235419 tid 1 2023-08-27T20:52:07.860 INFO:tasks.rados.rados.0.smithi093.stdout:270: writing smithi093153278-270 from 1724715 to 2280136 tid 2 2023-08-27T20:52:07.860 INFO:tasks.rados.rados.0.smithi093.stdout:271: write initial oid 271 2023-08-27T20:52:07.860 INFO:tasks.rados.rados.0.smithi093.stdout:271: seq_num 270 ranges {537890=789585,1783197=699079} 2023-08-27T20:52:07.868 INFO:tasks.rados.rados.0.smithi093.stdout:271: writing smithi093153278-271 from 537890 to 1327475 tid 1 2023-08-27T20:52:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:271: writing smithi093153278-271 from 1783197 to 2482276 tid 2 2023-08-27T20:52:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:272: write initial oid 272 2023-08-27T20:52:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:272: seq_num 271 ranges {554697=464834,1725326=559090} 2023-08-27T20:52:07.881 INFO:tasks.rados.rados.0.smithi093.stdout:272: writing smithi093153278-272 from 554697 to 1019531 tid 1 2023-08-27T20:52:07.888 INFO:tasks.rados.rados.0.smithi093.stdout:272: writing smithi093153278-272 from 1725326 to 2284416 tid 2 2023-08-27T20:52:07.888 INFO:tasks.rados.rados.0.smithi093.stdout:273: write initial oid 273 2023-08-27T20:52:07.888 INFO:tasks.rados.rados.0.smithi093.stdout:273: seq_num 272 ranges {571504=540083,1783808=702748} 2023-08-27T20:52:07.894 INFO:tasks.rados.rados.0.smithi093.stdout:273: writing smithi093153278-273 from 571504 to 1111587 tid 1 2023-08-27T20:52:07.901 INFO:tasks.rados.rados.0.smithi093.stdout:273: writing smithi093153278-273 from 1783808 to 2486556 tid 2 2023-08-27T20:52:07.902 INFO:tasks.rados.rados.0.smithi093.stdout:274: write initial oid 274 2023-08-27T20:52:07.902 INFO:tasks.rados.rados.0.smithi093.stdout:274: seq_num 273 ranges {588311=615332,1842290=562759} 2023-08-27T20:52:07.908 INFO:tasks.rados.rados.0.smithi093.stdout:274: writing smithi093153278-274 from 588311 to 1203643 tid 1 2023-08-27T20:52:07.915 INFO:tasks.rados.rados.0.smithi093.stdout:274: writing smithi093153278-274 from 1842290 to 2405049 tid 2 2023-08-27T20:52:07.915 INFO:tasks.rados.rados.0.smithi093.stdout:275: write initial oid 275 2023-08-27T20:52:07.915 INFO:tasks.rados.rados.0.smithi093.stdout:275: seq_num 274 ranges {605118=406934,1617125=706417} 2023-08-27T20:52:07.921 INFO:tasks.rados.rados.0.smithi093.stdout:275: writing smithi093153278-275 from 605118 to 1012052 tid 1 2023-08-27T20:52:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:275: writing smithi093153278-275 from 1617125 to 2323542 tid 2 2023-08-27T20:52:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:276: write initial oid 276 2023-08-27T20:52:07.929 INFO:tasks.rados.rados.0.smithi093.stdout:276: seq_num 275 ranges {621925=482183,1559254=566428} 2023-08-27T20:52:07.935 INFO:tasks.rados.rados.0.smithi093.stdout:276: writing smithi093153278-276 from 621925 to 1104108 tid 1 2023-08-27T20:52:07.941 INFO:tasks.rados.rados.0.smithi093.stdout:276: writing smithi093153278-276 from 1559254 to 2125682 tid 2 2023-08-27T20:52:07.941 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:261: finishing write tid 3 to smithi093153278-261 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:261: finishing write tid 4 to smithi093153278-261 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 261 v 41 (ObjNum 260 snap 0 seq_num 260) dirty exists 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:261: left oid 261 (ObjNum 260 snap 0 seq_num 260) 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:262: finishing write tid 1 to smithi093153278-262 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:262: finishing write tid 2 to smithi093153278-262 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:262: finishing write tid 3 to smithi093153278-262 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:262: finishing write tid 4 to smithi093153278-262 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 262 v 62 (ObjNum 261 snap 0 seq_num 261) dirty exists 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:262: left oid 262 (ObjNum 261 snap 0 seq_num 261) 2023-08-27T20:52:07.942 INFO:tasks.rados.rados.0.smithi093.stdout:263: finishing write tid 1 to smithi093153278-263 2023-08-27T20:52:07.943 INFO:tasks.rados.rados.0.smithi093.stdout:261: done (15 left) 2023-08-27T20:52:07.943 INFO:tasks.rados.rados.0.smithi093.stdout:262: done (14 left) 2023-08-27T20:52:07.943 INFO:tasks.rados.rados.0.smithi093.stdout:277: write initial oid 277 2023-08-27T20:52:07.943 INFO:tasks.rados.rados.0.smithi093.stdout:277: seq_num 276 ranges {638732=557432,1617736=710086} 2023-08-27T20:52:07.948 INFO:tasks.rados.rados.0.smithi093.stdout:277: writing smithi093153278-277 from 638732 to 1196164 tid 1 2023-08-27T20:52:07.955 INFO:tasks.rados.rados.0.smithi093.stdout:277: writing smithi093153278-277 from 1617736 to 2327822 tid 2 2023-08-27T20:52:07.955 INFO:tasks.rados.rados.0.smithi093.stdout:263: finishing write tid 2 to smithi093153278-263 2023-08-27T20:52:07.955 INFO:tasks.rados.rados.0.smithi093.stdout:278: write initial oid 278 2023-08-27T20:52:07.955 INFO:tasks.rados.rados.0.smithi093.stdout:278: seq_num 277 ranges {655539=632681,2076218=570097} 2023-08-27T20:52:07.962 INFO:tasks.rados.rados.0.smithi093.stdout:278: writing smithi093153278-278 from 655539 to 1288220 tid 1 2023-08-27T20:52:07.972 INFO:tasks.rados.rados.0.smithi093.stdout:278: writing smithi093153278-278 from 2076218 to 2646315 tid 2 2023-08-27T20:52:07.972 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:07.972 INFO:tasks.rados.rados.0.smithi093.stdout:263: finishing write tid 3 to smithi093153278-263 2023-08-27T20:52:07.972 INFO:tasks.rados.rados.0.smithi093.stdout:263: finishing write tid 4 to smithi093153278-263 2023-08-27T20:52:07.972 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 263 v 58 (ObjNum 262 snap 0 seq_num 262) dirty exists 2023-08-27T20:52:07.972 INFO:tasks.rados.rados.0.smithi093.stdout:263: left oid 263 (ObjNum 262 snap 0 seq_num 262) 2023-08-27T20:52:07.972 INFO:tasks.rados.rados.0.smithi093.stdout:264: finishing write tid 1 to smithi093153278-264 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:264: finishing write tid 2 to smithi093153278-264 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:264: finishing write tid 3 to smithi093153278-264 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:264: finishing write tid 4 to smithi093153278-264 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 264 v 60 (ObjNum 263 snap 0 seq_num 263) dirty exists 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:264: left oid 264 (ObjNum 263 snap 0 seq_num 263) 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:265: finishing write tid 1 to smithi093153278-265 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:265: finishing write tid 2 to smithi093153278-265 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:265: finishing write tid 3 to smithi093153278-265 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:265: finishing write tid 4 to smithi093153278-265 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 265 v 73 (ObjNum 264 snap 0 seq_num 264) dirty exists 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:265: left oid 265 (ObjNum 264 snap 0 seq_num 264) 2023-08-27T20:52:07.973 INFO:tasks.rados.rados.0.smithi093.stdout:266: finishing write tid 1 to smithi093153278-266 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:266: finishing write tid 2 to smithi093153278-266 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:266: finishing write tid 3 to smithi093153278-266 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:266: finishing write tid 4 to smithi093153278-266 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 266 v 48 (ObjNum 265 snap 0 seq_num 265) dirty exists 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:266: left oid 266 (ObjNum 265 snap 0 seq_num 265) 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:267: finishing write tid 1 to smithi093153278-267 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:267: finishing write tid 2 to smithi093153278-267 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:267: finishing write tid 3 to smithi093153278-267 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:267: finishing write tid 4 to smithi093153278-267 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 267 v 74 (ObjNum 266 snap 0 seq_num 266) dirty exists 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:267: left oid 267 (ObjNum 266 snap 0 seq_num 266) 2023-08-27T20:52:07.974 INFO:tasks.rados.rados.0.smithi093.stdout:268: finishing write tid 1 to smithi093153278-268 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:268: finishing write tid 2 to smithi093153278-268 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:268: finishing write tid 3 to smithi093153278-268 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:268: finishing write tid 4 to smithi093153278-268 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 268 v 51 (ObjNum 267 snap 0 seq_num 267) dirty exists 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:268: left oid 268 (ObjNum 267 snap 0 seq_num 267) 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:269: finishing write tid 1 to smithi093153278-269 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:263: done (15 left) 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:264: done (14 left) 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:265: done (13 left) 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:266: done (12 left) 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:267: done (11 left) 2023-08-27T20:52:07.975 INFO:tasks.rados.rados.0.smithi093.stdout:268: done (10 left) 2023-08-27T20:52:07.976 INFO:tasks.rados.rados.0.smithi093.stdout:279: write initial oid 279 2023-08-27T20:52:07.976 INFO:tasks.rados.rados.0.smithi093.stdout:279: seq_num 278 ranges {672346=707930,2134700=537646} 2023-08-27T20:52:07.979 INFO:tasks.rados.rados.0.smithi093.stdout:279: writing smithi093153278-279 from 672346 to 1380276 tid 1 2023-08-27T20:52:07.987 INFO:tasks.rados.rados.0.smithi093.stdout:279: writing smithi093153278-279 from 2134700 to 2672346 tid 2 2023-08-27T20:52:07.987 INFO:tasks.rados.rados.0.smithi093.stdout:269: finishing write tid 2 to smithi093153278-269 2023-08-27T20:52:07.987 INFO:tasks.rados.rados.0.smithi093.stdout:269: finishing write tid 3 to smithi093153278-269 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:269: finishing write tid 4 to smithi093153278-269 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 269 v 60 (ObjNum 268 snap 0 seq_num 268) dirty exists 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:269: left oid 269 (ObjNum 268 snap 0 seq_num 268) 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:270: finishing write tid 1 to smithi093153278-270 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:270: finishing write tid 2 to smithi093153278-270 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:269: done (10 left) 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:280: write initial oid 280 2023-08-27T20:52:07.988 INFO:tasks.rados.rados.0.smithi093.stdout:280: seq_num 279 ranges {689153=783179,2076829=457413} 2023-08-27T20:52:07.996 INFO:tasks.rados.rados.0.smithi093.stdout:280: writing smithi093153278-280 from 689153 to 1472332 tid 1 2023-08-27T20:52:08.003 INFO:tasks.rados.rados.0.smithi093.stdout:280: writing smithi093153278-280 from 2076829 to 2534242 tid 2 2023-08-27T20:52:08.003 INFO:tasks.rados.rados.0.smithi093.stdout:281: write initial oid 281 2023-08-27T20:52:08.003 INFO:tasks.rados.rados.0.smithi093.stdout:281: seq_num 280 ranges {705960=458428,1735311=717424} 2023-08-27T20:52:08.010 INFO:tasks.rados.rados.0.smithi093.stdout:281: writing smithi093153278-281 from 705960 to 1164388 tid 1 2023-08-27T20:52:08.019 INFO:tasks.rados.rados.0.smithi093.stdout:281: writing smithi093153278-281 from 1735311 to 2452735 tid 2 2023-08-27T20:52:08.019 INFO:tasks.rados.rados.0.smithi093.stdout:282: write initial oid 282 2023-08-27T20:52:08.019 INFO:tasks.rados.rados.0.smithi093.stdout:282: seq_num 281 ranges {722767=533677,1793793=461082} 2023-08-27T20:52:08.026 INFO:tasks.rados.rados.0.smithi093.stdout:282: writing smithi093153278-282 from 722767 to 1256444 tid 1 2023-08-27T20:52:08.032 INFO:tasks.rados.rados.0.smithi093.stdout:282: writing smithi093153278-282 from 1793793 to 2254875 tid 2 2023-08-27T20:52:08.032 INFO:tasks.rados.rados.0.smithi093.stdout:283: write initial oid 283 2023-08-27T20:52:08.032 INFO:tasks.rados.rados.0.smithi093.stdout:283: seq_num 282 ranges {739574=725279,1968628=721093} 2023-08-27T20:52:08.041 INFO:tasks.rados.rados.0.smithi093.stdout:283: writing smithi093153278-283 from 739574 to 1464853 tid 1 2023-08-27T20:52:08.049 INFO:tasks.rados.rados.0.smithi093.stdout:283: writing smithi093153278-283 from 1968628 to 2689721 tid 2 2023-08-27T20:52:08.050 INFO:tasks.rados.rados.0.smithi093.stdout:284: write initial oid 284 2023-08-27T20:52:08.050 INFO:tasks.rados.rados.0.smithi093.stdout:284: seq_num 283 ranges {756381=400528,1910757=464751} 2023-08-27T20:52:08.056 INFO:tasks.rados.rados.0.smithi093.stdout:284: writing smithi093153278-284 from 756381 to 1156909 tid 1 2023-08-27T20:52:08.065 INFO:tasks.rados.rados.0.smithi093.stdout:284: writing smithi093153278-284 from 1910757 to 2375508 tid 2 2023-08-27T20:52:08.065 INFO:tasks.rados.rados.0.smithi093.stdout:285: write initial oid 285 2023-08-27T20:52:08.065 INFO:tasks.rados.rados.0.smithi093.stdout:270: finishing write tid 3 to smithi093153278-270 2023-08-27T20:52:08.065 INFO:tasks.rados.rados.0.smithi093.stdout:270: finishing write tid 4 to smithi093153278-270 2023-08-27T20:52:08.065 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 270 v 44 (ObjNum 269 snap 0 seq_num 269) dirty exists 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:270: left oid 270 (ObjNum 269 snap 0 seq_num 269) 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:271: finishing write tid 1 to smithi093153278-271 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:271: finishing write tid 2 to smithi093153278-271 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:271: finishing write tid 3 to smithi093153278-271 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:271: finishing write tid 4 to smithi093153278-271 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 271 v 58 (ObjNum 270 snap 0 seq_num 270) dirty exists 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:271: left oid 271 (ObjNum 270 snap 0 seq_num 270) 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:272: finishing write tid 1 to smithi093153278-272 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:272: finishing write tid 2 to smithi093153278-272 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:272: finishing write tid 3 to smithi093153278-272 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:272: finishing write tid 4 to smithi093153278-272 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 272 v 76 (ObjNum 271 snap 0 seq_num 271) dirty exists 2023-08-27T20:52:08.066 INFO:tasks.rados.rados.0.smithi093.stdout:272: left oid 272 (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:273: finishing write tid 1 to smithi093153278-273 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:273: finishing write tid 2 to smithi093153278-273 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:273: finishing write tid 3 to smithi093153278-273 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:273: finishing write tid 4 to smithi093153278-273 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 273 v 75 (ObjNum 272 snap 0 seq_num 272) dirty exists 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:273: left oid 273 (ObjNum 272 snap 0 seq_num 272) 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:274: finishing write tid 1 to smithi093153278-274 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:274: finishing write tid 2 to smithi093153278-274 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:274: finishing write tid 3 to smithi093153278-274 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:274: finishing write tid 4 to smithi093153278-274 2023-08-27T20:52:08.067 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 274 v 47 (ObjNum 273 snap 0 seq_num 273) dirty exists 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:274: left oid 274 (ObjNum 273 snap 0 seq_num 273) 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:275: finishing write tid 1 to smithi093153278-275 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:275: finishing write tid 2 to smithi093153278-275 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:275: finishing write tid 3 to smithi093153278-275 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:275: finishing write tid 4 to smithi093153278-275 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 275 v 77 (ObjNum 274 snap 0 seq_num 274) dirty exists 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:275: left oid 275 (ObjNum 274 snap 0 seq_num 274) 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:276: finishing write tid 1 to smithi093153278-276 2023-08-27T20:52:08.068 INFO:tasks.rados.rados.0.smithi093.stdout:285: seq_num 284 ranges {773188=475777,1969239=724762} 2023-08-27T20:52:08.072 INFO:tasks.rados.rados.0.smithi093.stdout:285: writing smithi093153278-285 from 773188 to 1248965 tid 1 2023-08-27T20:52:08.082 INFO:tasks.rados.rados.0.smithi093.stdout:285: writing smithi093153278-285 from 1969239 to 2694001 tid 2 2023-08-27T20:52:08.082 INFO:tasks.rados.rados.0.smithi093.stdout:270: done (15 left) 2023-08-27T20:52:08.082 INFO:tasks.rados.rados.0.smithi093.stdout:271: done (14 left) 2023-08-27T20:52:08.082 INFO:tasks.rados.rados.0.smithi093.stdout:272: done (13 left) 2023-08-27T20:52:08.082 INFO:tasks.rados.rados.0.smithi093.stdout:273: done (12 left) 2023-08-27T20:52:08.082 INFO:tasks.rados.rados.0.smithi093.stdout:274: done (11 left) 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:275: done (10 left) 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:286: write initial oid 286 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:276: finishing write tid 2 to smithi093153278-276 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:276: finishing write tid 3 to smithi093153278-276 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:276: finishing write tid 4 to smithi093153278-276 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 276 v 62 (ObjNum 275 snap 0 seq_num 275) dirty exists 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:276: left oid 276 (ObjNum 275 snap 0 seq_num 275) 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:277: finishing write tid 1 to smithi093153278-277 2023-08-27T20:52:08.083 INFO:tasks.rados.rados.0.smithi093.stdout:286: seq_num 285 ranges {789995=551026,2027721=468420} 2023-08-27T20:52:08.090 INFO:tasks.rados.rados.0.smithi093.stdout:286: writing smithi093153278-286 from 789995 to 1341021 tid 1 2023-08-27T20:52:08.097 INFO:tasks.rados.rados.0.smithi093.stdout:286: writing smithi093153278-286 from 2027721 to 2496141 tid 2 2023-08-27T20:52:08.097 INFO:tasks.rados.rados.0.smithi093.stdout:276: done (10 left) 2023-08-27T20:52:08.097 INFO:tasks.rados.rados.0.smithi093.stdout:287: write initial oid 287 2023-08-27T20:52:08.097 INFO:tasks.rados.rados.0.smithi093.stdout:287: seq_num 286 ranges {406802=626275,1686203=728431} 2023-08-27T20:52:08.104 INFO:tasks.rados.rados.0.smithi093.stdout:287: writing smithi093153278-287 from 406802 to 1033077 tid 1 2023-08-27T20:52:08.113 INFO:tasks.rados.rados.0.smithi093.stdout:287: writing smithi093153278-287 from 1686203 to 2414634 tid 2 2023-08-27T20:52:08.113 INFO:tasks.rados.rados.0.smithi093.stdout:288: write initial oid 288 2023-08-27T20:52:08.113 INFO:tasks.rados.rados.0.smithi093.stdout:288: seq_num 287 ranges {423609=701524,1628332=472089,2648987=174622} 2023-08-27T20:52:08.120 INFO:tasks.rados.rados.0.smithi093.stdout:288: writing smithi093153278-288 from 423609 to 1125133 tid 1 2023-08-27T20:52:08.126 INFO:tasks.rados.rados.0.smithi093.stdout:288: writing smithi093153278-288 from 1628332 to 2100421 tid 2 2023-08-27T20:52:08.131 INFO:tasks.rados.rados.0.smithi093.stdout:288: writing smithi093153278-288 from 2648987 to 2823609 tid 3 2023-08-27T20:52:08.131 INFO:tasks.rados.rados.0.smithi093.stdout:289: write initial oid 289 2023-08-27T20:52:08.131 INFO:tasks.rados.rados.0.smithi093.stdout:289: seq_num 288 ranges {440416=776773,1686814=732100} 2023-08-27T20:52:08.139 INFO:tasks.rados.rados.0.smithi093.stdout:289: writing smithi093153278-289 from 440416 to 1217189 tid 1 2023-08-27T20:52:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:289: writing smithi093153278-289 from 1686814 to 2418914 tid 2 2023-08-27T20:52:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:290: write initial oid 290 2023-08-27T20:52:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:277: finishing write tid 2 to smithi093153278-277 2023-08-27T20:52:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:290: seq_num 289 ranges {457223=568375,1461649=475758,2420186=437037} 2023-08-27T20:52:08.154 INFO:tasks.rados.rados.0.smithi093.stdout:290: writing smithi093153278-290 from 457223 to 1025598 tid 1 2023-08-27T20:52:08.160 INFO:tasks.rados.rados.0.smithi093.stdout:290: writing smithi093153278-290 from 1461649 to 1937407 tid 2 2023-08-27T20:52:08.166 INFO:tasks.rados.rados.0.smithi093.stdout:290: writing smithi093153278-290 from 2420186 to 2857223 tid 3 2023-08-27T20:52:08.166 INFO:tasks.rados.rados.0.smithi093.stdout:291: write initial oid 291 2023-08-27T20:52:08.166 INFO:tasks.rados.rados.0.smithi093.stdout:291: seq_num 290 ranges {474030=643624,1520131=735769} 2023-08-27T20:52:08.172 INFO:tasks.rados.rados.0.smithi093.stdout:291: writing smithi093153278-291 from 474030 to 1117654 tid 1 2023-08-27T20:52:08.179 INFO:tasks.rados.rados.0.smithi093.stdout:291: writing smithi093153278-291 from 1520131 to 2255900 tid 2 2023-08-27T20:52:08.179 INFO:tasks.rados.rados.0.smithi093.stdout:292: write initial oid 292 2023-08-27T20:52:08.179 INFO:tasks.rados.rados.0.smithi093.stdout:292: seq_num 291 ranges {490837=718873,1862260=479427,2875032=15805} 2023-08-27T20:52:08.186 INFO:tasks.rados.rados.0.smithi093.stdout:292: writing smithi093153278-292 from 490837 to 1209710 tid 1 2023-08-27T20:52:08.192 INFO:tasks.rados.rados.0.smithi093.stdout:292: writing smithi093153278-292 from 1862260 to 2341687 tid 2 2023-08-27T20:52:08.196 INFO:tasks.rados.rados.0.smithi093.stdout:292: writing smithi093153278-292 from 2875032 to 2890837 tid 3 2023-08-27T20:52:08.196 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.196 INFO:tasks.rados.rados.0.smithi093.stdout:277: finishing write tid 3 to smithi093153278-277 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:277: finishing write tid 4 to smithi093153278-277 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 277 v 79 (ObjNum 276 snap 0 seq_num 276) dirty exists 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:277: left oid 277 (ObjNum 276 snap 0 seq_num 276) 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:278: finishing write tid 1 to smithi093153278-278 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:278: finishing write tid 2 to smithi093153278-278 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:278: finishing write tid 3 to smithi093153278-278 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:278: finishing write tid 4 to smithi093153278-278 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 278 v 78 (ObjNum 277 snap 0 seq_num 277) dirty exists 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:278: left oid 278 (ObjNum 277 snap 0 seq_num 277) 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:279: finishing write tid 1 to smithi093153278-279 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:279: finishing write tid 2 to smithi093153278-279 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:279: finishing write tid 3 to smithi093153278-279 2023-08-27T20:52:08.197 INFO:tasks.rados.rados.0.smithi093.stdout:279: finishing write tid 4 to smithi093153278-279 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 279 v 65 (ObjNum 278 snap 0 seq_num 278) dirty exists 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:279: left oid 279 (ObjNum 278 snap 0 seq_num 278) 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:280: finishing write tid 1 to smithi093153278-280 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:280: finishing write tid 2 to smithi093153278-280 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:280: finishing write tid 3 to smithi093153278-280 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:280: finishing write tid 4 to smithi093153278-280 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 280 v 63 (ObjNum 279 snap 0 seq_num 279) dirty exists 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:280: left oid 280 (ObjNum 279 snap 0 seq_num 279) 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:281: finishing write tid 1 to smithi093153278-281 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:277: done (15 left) 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:278: done (14 left) 2023-08-27T20:52:08.198 INFO:tasks.rados.rados.0.smithi093.stdout:279: done (13 left) 2023-08-27T20:52:08.199 INFO:tasks.rados.rados.0.smithi093.stdout:280: done (12 left) 2023-08-27T20:52:08.199 INFO:tasks.rados.rados.0.smithi093.stdout:293: write initial oid 293 2023-08-27T20:52:08.199 INFO:tasks.rados.rados.0.smithi093.stdout:293: seq_num 292 ranges {507644=794122,1920742=623085} 2023-08-27T20:52:08.203 INFO:tasks.rados.rados.0.smithi093.stdout:293: writing smithi093153278-293 from 507644 to 1301766 tid 1 2023-08-27T20:52:08.211 INFO:tasks.rados.rados.0.smithi093.stdout:293: writing smithi093153278-293 from 1920742 to 2543827 tid 2 2023-08-27T20:52:08.211 INFO:tasks.rados.rados.0.smithi093.stdout:294: write initial oid 294 2023-08-27T20:52:08.211 INFO:tasks.rados.rados.0.smithi093.stdout:281: finishing write tid 2 to smithi093153278-281 2023-08-27T20:52:08.211 INFO:tasks.rados.rados.0.smithi093.stdout:281: finishing write tid 3 to smithi093153278-281 2023-08-27T20:52:08.211 INFO:tasks.rados.rados.0.smithi093.stdout:281: finishing write tid 4 to smithi093153278-281 2023-08-27T20:52:08.211 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 281 v 61 (ObjNum 280 snap 0 seq_num 280) dirty exists 2023-08-27T20:52:08.211 INFO:tasks.rados.rados.0.smithi093.stdout:281: left oid 281 (ObjNum 280 snap 0 seq_num 280) 2023-08-27T20:52:08.212 INFO:tasks.rados.rados.0.smithi093.stdout:282: finishing write tid 1 to smithi093153278-282 2023-08-27T20:52:08.212 INFO:tasks.rados.rados.0.smithi093.stdout:294: seq_num 293 ranges {524451=469371,1579224=483096,2529878=394573} 2023-08-27T20:52:08.216 INFO:tasks.rados.rados.0.smithi093.stdout:294: writing smithi093153278-294 from 524451 to 993822 tid 1 2023-08-27T20:52:08.222 INFO:tasks.rados.rados.0.smithi093.stdout:294: writing smithi093153278-294 from 1579224 to 2062320 tid 2 2023-08-27T20:52:08.227 INFO:tasks.rados.rados.0.smithi093.stdout:294: writing smithi093153278-294 from 2529878 to 2924451 tid 3 2023-08-27T20:52:08.227 INFO:tasks.rados.rados.0.smithi093.stdout:281: done (13 left) 2023-08-27T20:52:08.227 INFO:tasks.rados.rados.0.smithi093.stdout:295: write initial oid 295 2023-08-27T20:52:08.227 INFO:tasks.rados.rados.0.smithi093.stdout:295: seq_num 294 ranges {541258=544620,1637706=626754,2840948=100310} 2023-08-27T20:52:08.234 INFO:tasks.rados.rados.0.smithi093.stdout:295: writing smithi093153278-295 from 541258 to 1085878 tid 1 2023-08-27T20:52:08.241 INFO:tasks.rados.rados.0.smithi093.stdout:295: writing smithi093153278-295 from 1637706 to 2264460 tid 2 2023-08-27T20:52:08.246 INFO:tasks.rados.rados.0.smithi093.stdout:295: writing smithi093153278-295 from 2840948 to 2941258 tid 3 2023-08-27T20:52:08.246 INFO:tasks.rados.rados.0.smithi093.stdout:296: write initial oid 296 2023-08-27T20:52:08.246 INFO:tasks.rados.rados.0.smithi093.stdout:282: finishing write tid 2 to smithi093153278-282 2023-08-27T20:52:08.246 INFO:tasks.rados.rados.0.smithi093.stdout:282: finishing write tid 3 to smithi093153278-282 2023-08-27T20:52:08.246 INFO:tasks.rados.rados.0.smithi093.stdout:282: finishing write tid 4 to smithi093153278-282 2023-08-27T20:52:08.246 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 282 v 54 (ObjNum 281 snap 0 seq_num 281) dirty exists 2023-08-27T20:52:08.246 INFO:tasks.rados.rados.0.smithi093.stdout:282: left oid 282 (ObjNum 281 snap 0 seq_num 281) 2023-08-27T20:52:08.247 INFO:tasks.rados.rados.0.smithi093.stdout:283: finishing write tid 1 to smithi093153278-283 2023-08-27T20:52:08.247 INFO:tasks.rados.rados.0.smithi093.stdout:283: finishing write tid 2 to smithi093153278-283 2023-08-27T20:52:08.247 INFO:tasks.rados.rados.0.smithi093.stdout:296: seq_num 295 ranges {558065=619869,1579835=486765,2468371=489694} 2023-08-27T20:52:08.253 INFO:tasks.rados.rados.0.smithi093.stdout:296: writing smithi093153278-296 from 558065 to 1177934 tid 1 2023-08-27T20:52:08.258 INFO:tasks.rados.rados.0.smithi093.stdout:296: writing smithi093153278-296 from 1579835 to 2066600 tid 2 2023-08-27T20:52:08.264 INFO:tasks.rados.rados.0.smithi093.stdout:296: writing smithi093153278-296 from 2468371 to 2958065 tid 3 2023-08-27T20:52:08.264 INFO:tasks.rados.rados.0.smithi093.stdout:282: done (14 left) 2023-08-27T20:52:08.264 INFO:tasks.rados.rados.0.smithi093.stdout:297: write initial oid 297 2023-08-27T20:52:08.265 INFO:tasks.rados.rados.0.smithi093.stdout:297: seq_num 296 ranges {574872=695118,2038317=630423} 2023-08-27T20:52:08.272 INFO:tasks.rados.rados.0.smithi093.stdout:297: writing smithi093153278-297 from 574872 to 1269990 tid 1 2023-08-27T20:52:08.280 INFO:tasks.rados.rados.0.smithi093.stdout:297: writing smithi093153278-297 from 2038317 to 2668740 tid 2 2023-08-27T20:52:08.280 INFO:tasks.rados.rados.0.smithi093.stdout:298: write initial oid 298 2023-08-27T20:52:08.280 INFO:tasks.rados.rados.0.smithi093.stdout:298: seq_num 297 ranges {591679=486720,1813152=490434} 2023-08-27T20:52:08.286 INFO:tasks.rados.rados.0.smithi093.stdout:298: writing smithi093153278-298 from 591679 to 1078399 tid 1 2023-08-27T20:52:08.294 INFO:tasks.rados.rados.0.smithi093.stdout:298: writing smithi093153278-298 from 1813152 to 2303586 tid 2 2023-08-27T20:52:08.294 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.294 INFO:tasks.rados.rados.0.smithi093.stdout:283: finishing write tid 3 to smithi093153278-283 2023-08-27T20:52:08.295 INFO:tasks.rados.rados.0.smithi093.stdout:283: finishing write tid 4 to smithi093153278-283 2023-08-27T20:52:08.295 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 283 v 64 (ObjNum 282 snap 0 seq_num 282) dirty exists 2023-08-27T20:52:08.295 INFO:tasks.rados.rados.0.smithi093.stdout:283: left oid 283 (ObjNum 282 snap 0 seq_num 282) 2023-08-27T20:52:08.295 INFO:tasks.rados.rados.0.smithi093.stdout:284: finishing write tid 1 to smithi093153278-284 2023-08-27T20:52:08.295 INFO:tasks.rados.rados.0.smithi093.stdout:283: done (15 left) 2023-08-27T20:52:08.295 INFO:tasks.rados.rados.0.smithi093.stdout:299: write initial oid 299 2023-08-27T20:52:08.295 INFO:tasks.rados.rados.0.smithi093.stdout:299: seq_num 298 ranges {608486=561969,1871634=634092,2950640=57846} 2023-08-27T20:52:08.300 INFO:tasks.rados.rados.0.smithi093.stdout:299: writing smithi093153278-299 from 608486 to 1170455 tid 1 2023-08-27T20:52:08.308 INFO:tasks.rados.rados.0.smithi093.stdout:299: writing smithi093153278-299 from 1871634 to 2505726 tid 2 2023-08-27T20:52:08.312 INFO:tasks.rados.rados.0.smithi093.stdout:299: writing smithi093153278-299 from 2950640 to 3008486 tid 3 2023-08-27T20:52:08.312 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.312 INFO:tasks.rados.rados.0.smithi093.stdout:285: finishing write tid 1 to smithi093153278-285 2023-08-27T20:52:08.312 INFO:tasks.rados.rados.0.smithi093.stdout:284: finishing write tid 2 to smithi093153278-284 2023-08-27T20:52:08.312 INFO:tasks.rados.rados.0.smithi093.stdout:284: finishing write tid 3 to smithi093153278-284 2023-08-27T20:52:08.312 INFO:tasks.rados.rados.0.smithi093.stdout:284: finishing write tid 4 to smithi093153278-284 2023-08-27T20:52:08.312 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 284 v 61 (ObjNum 283 snap 0 seq_num 283) dirty exists 2023-08-27T20:52:08.313 INFO:tasks.rados.rados.0.smithi093.stdout:284: left oid 284 (ObjNum 283 snap 0 seq_num 283) 2023-08-27T20:52:08.313 INFO:tasks.rados.rados.0.smithi093.stdout:285: finishing write tid 2 to smithi093153278-285 2023-08-27T20:52:08.313 INFO:tasks.rados.rados.0.smithi093.stdout:284: done (15 left) 2023-08-27T20:52:08.313 INFO:tasks.rados.rados.0.smithi093.stdout:300: write initial oid 300 2023-08-27T20:52:08.313 INFO:tasks.rados.rados.0.smithi093.stdout:300: seq_num 299 ranges {625293=637218,1813763=494103,2978063=47230} 2023-08-27T20:52:08.318 INFO:tasks.rados.rados.0.smithi093.stdout:300: writing smithi093153278-300 from 625293 to 1262511 tid 1 2023-08-27T20:52:08.325 INFO:tasks.rados.rados.0.smithi093.stdout:300: writing smithi093153278-300 from 1813763 to 2307866 tid 2 2023-08-27T20:52:08.329 INFO:tasks.rados.rados.0.smithi093.stdout:300: writing smithi093153278-300 from 2978063 to 3025293 tid 3 2023-08-27T20:52:08.329 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:285: finishing write tid 3 to smithi093153278-285 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:285: finishing write tid 4 to smithi093153278-285 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 285 v 67 (ObjNum 284 snap 0 seq_num 284) dirty exists 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:285: left oid 285 (ObjNum 284 snap 0 seq_num 284) 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:286: finishing write tid 1 to smithi093153278-286 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:286: finishing write tid 2 to smithi093153278-286 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:285: done (15 left) 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:301: write initial oid 301 2023-08-27T20:52:08.330 INFO:tasks.rados.rados.0.smithi093.stdout:301: seq_num 300 ranges {642100=712467,1872245=637761} 2023-08-27T20:52:08.337 INFO:tasks.rados.rados.0.smithi093.stdout:301: writing smithi093153278-301 from 642100 to 1354567 tid 1 2023-08-27T20:52:08.345 INFO:tasks.rados.rados.0.smithi093.stdout:301: writing smithi093153278-301 from 1872245 to 2510006 tid 2 2023-08-27T20:52:08.345 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:286: finishing write tid 3 to smithi093153278-286 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:286: finishing write tid 4 to smithi093153278-286 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 286 v 50 (ObjNum 285 snap 0 seq_num 285) dirty exists 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:286: left oid 286 (ObjNum 285 snap 0 seq_num 285) 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:287: finishing write tid 1 to smithi093153278-287 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:287: finishing write tid 2 to smithi093153278-287 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:287: finishing write tid 3 to smithi093153278-287 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:287: finishing write tid 4 to smithi093153278-287 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 287 v 80 (ObjNum 286 snap 0 seq_num 286) dirty exists 2023-08-27T20:52:08.346 INFO:tasks.rados.rados.0.smithi093.stdout:287: left oid 287 (ObjNum 286 snap 0 seq_num 286) 2023-08-27T20:52:08.347 INFO:tasks.rados.rados.0.smithi093.stdout:288: finishing write tid 1 to smithi093153278-288 2023-08-27T20:52:08.347 INFO:tasks.rados.rados.0.smithi093.stdout:286: done (15 left) 2023-08-27T20:52:08.347 INFO:tasks.rados.rados.0.smithi093.stdout:287: done (14 left) 2023-08-27T20:52:08.347 INFO:tasks.rados.rados.0.smithi093.stdout:302: write initial oid 302 2023-08-27T20:52:08.347 INFO:tasks.rados.rados.0.smithi093.stdout:302: seq_num 301 ranges {658907=787716,1930727=497772,3032909=25998} 2023-08-27T20:52:08.354 INFO:tasks.rados.rados.0.smithi093.stdout:302: writing smithi093153278-302 from 658907 to 1446623 tid 1 2023-08-27T20:52:08.360 INFO:tasks.rados.rados.0.smithi093.stdout:302: writing smithi093153278-302 from 1930727 to 2428499 tid 2 2023-08-27T20:52:08.363 INFO:tasks.rados.rados.0.smithi093.stdout:302: writing smithi093153278-302 from 3032909 to 3058907 tid 3 2023-08-27T20:52:08.363 INFO:tasks.rados.rados.0.smithi093.stdout:303: write initial oid 303 2023-08-27T20:52:08.363 INFO:tasks.rados.rados.0.smithi093.stdout:303: seq_num 302 ranges {675714=462965,1589209=641430,2943979=131735} 2023-08-27T20:52:08.369 INFO:tasks.rados.rados.0.smithi093.stdout:303: writing smithi093153278-303 from 675714 to 1138679 tid 1 2023-08-27T20:52:08.375 INFO:tasks.rados.rados.0.smithi093.stdout:303: writing smithi093153278-303 from 1589209 to 2230639 tid 2 2023-08-27T20:52:08.381 INFO:tasks.rados.rados.0.smithi093.stdout:303: writing smithi093153278-303 from 2943979 to 3075714 tid 3 2023-08-27T20:52:08.381 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.381 INFO:tasks.rados.rados.0.smithi093.stdout:288: finishing write tid 2 to smithi093153278-288 2023-08-27T20:52:08.381 INFO:tasks.rados.rados.0.smithi093.stdout:288: finishing write tid 3 to smithi093153278-288 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:288: finishing write tid 4 to smithi093153278-288 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:288: finishing write tid 5 to smithi093153278-288 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 288 v 67 (ObjNum 287 snap 0 seq_num 287) dirty exists 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:288: left oid 288 (ObjNum 287 snap 0 seq_num 287) 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:289: finishing write tid 1 to smithi093153278-289 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:289: finishing write tid 2 to smithi093153278-289 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:289: finishing write tid 3 to smithi093153278-289 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:289: finishing write tid 4 to smithi093153278-289 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 289 v 53 (ObjNum 288 snap 0 seq_num 288) dirty exists 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:289: left oid 289 (ObjNum 288 snap 0 seq_num 288) 2023-08-27T20:52:08.382 INFO:tasks.rados.rados.0.smithi093.stdout:290: finishing write tid 1 to smithi093153278-290 2023-08-27T20:52:08.383 INFO:tasks.rados.rados.0.smithi093.stdout:290: finishing write tid 2 to smithi093153278-290 2023-08-27T20:52:08.383 INFO:tasks.rados.rados.0.smithi093.stdout:290: finishing write tid 3 to smithi093153278-290 2023-08-27T20:52:08.383 INFO:tasks.rados.rados.0.smithi093.stdout:288: done (15 left) 2023-08-27T20:52:08.383 INFO:tasks.rados.rados.0.smithi093.stdout:289: done (14 left) 2023-08-27T20:52:08.383 INFO:tasks.rados.rados.0.smithi093.stdout:304: write initial oid 304 2023-08-27T20:52:08.383 INFO:tasks.rados.rados.0.smithi093.stdout:304: seq_num 303 ranges {692521=538214,1931338=785088} 2023-08-27T20:52:08.387 INFO:tasks.rados.rados.0.smithi093.stdout:304: writing smithi093153278-304 from 692521 to 1230735 tid 1 2023-08-27T20:52:08.395 INFO:tasks.rados.rados.0.smithi093.stdout:304: writing smithi093153278-304 from 1931338 to 2716426 tid 2 2023-08-27T20:52:08.395 INFO:tasks.rados.rados.0.smithi093.stdout:305: write initial oid 305 2023-08-27T20:52:08.395 INFO:tasks.rados.rados.0.smithi093.stdout:305: seq_num 304 ranges {709328=613463,1989820=645099} 2023-08-27T20:52:08.402 INFO:tasks.rados.rados.0.smithi093.stdout:305: writing smithi093153278-305 from 709328 to 1322791 tid 1 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:305: writing smithi093153278-305 from 1989820 to 2634919 tid 2 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:290: finishing write tid 4 to smithi093153278-290 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:290: finishing write tid 5 to smithi093153278-290 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 290 v 64 (ObjNum 289 snap 0 seq_num 289) dirty exists 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:290: left oid 290 (ObjNum 289 snap 0 seq_num 289) 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:291: finishing write tid 1 to smithi093153278-291 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:291: finishing write tid 2 to smithi093153278-291 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:291: finishing write tid 3 to smithi093153278-291 2023-08-27T20:52:08.411 INFO:tasks.rados.rados.0.smithi093.stdout:291: finishing write tid 4 to smithi093153278-291 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 291 v 56 (ObjNum 290 snap 0 seq_num 290) dirty exists 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:291: left oid 291 (ObjNum 290 snap 0 seq_num 290) 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:292: finishing write tid 1 to smithi093153278-292 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:292: finishing write tid 2 to smithi093153278-292 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:292: finishing write tid 3 to smithi093153278-292 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:292: finishing write tid 4 to smithi093153278-292 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:292: finishing write tid 5 to smithi093153278-292 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 292 v 65 (ObjNum 291 snap 0 seq_num 291) dirty exists 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:292: left oid 292 (ObjNum 291 snap 0 seq_num 291) 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:293: finishing write tid 1 to smithi093153278-293 2023-08-27T20:52:08.412 INFO:tasks.rados.rados.0.smithi093.stdout:293: finishing write tid 2 to smithi093153278-293 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:293: finishing write tid 3 to smithi093153278-293 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:293: finishing write tid 4 to smithi093153278-293 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 293 v 68 (ObjNum 292 snap 0 seq_num 292) dirty exists 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:293: left oid 293 (ObjNum 292 snap 0 seq_num 292) 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:294: finishing write tid 1 to smithi093153278-294 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:290: done (15 left) 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:291: done (14 left) 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:292: done (13 left) 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:293: done (12 left) 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:306: write initial oid 306 2023-08-27T20:52:08.413 INFO:tasks.rados.rados.0.smithi093.stdout:306: seq_num 305 ranges {726135=405065,1764655=788757,3026248=99887} 2023-08-27T20:52:08.416 INFO:tasks.rados.rados.0.smithi093.stdout:306: writing smithi093153278-306 from 726135 to 1131200 tid 1 2023-08-27T20:52:08.424 INFO:tasks.rados.rados.0.smithi093.stdout:306: writing smithi093153278-306 from 1764655 to 2553412 tid 2 2023-08-27T20:52:08.427 INFO:tasks.rados.rados.0.smithi093.stdout:306: writing smithi093153278-306 from 3026248 to 3126135 tid 3 2023-08-27T20:52:08.427 INFO:tasks.rados.rados.0.smithi093.stdout:307: write initial oid 307 2023-08-27T20:52:08.427 INFO:tasks.rados.rados.0.smithi093.stdout:307: seq_num 306 ranges {742942=480314,1823137=648768} 2023-08-27T20:52:08.434 INFO:tasks.rados.rados.0.smithi093.stdout:307: writing smithi093153278-307 from 742942 to 1223256 tid 1 2023-08-27T20:52:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:08 smithi136 ceph-mon[113531]: pgmap v58: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 40 MiB/s wr, 202 op/s 2023-08-27T20:52:08.442 INFO:tasks.rados.rados.0.smithi093.stdout:307: writing smithi093153278-307 from 1823137 to 2471905 tid 2 2023-08-27T20:52:08.443 INFO:tasks.rados.rados.0.smithi093.stdout:308: write initial oid 308 2023-08-27T20:52:08.443 INFO:tasks.rados.rados.0.smithi093.stdout:308: seq_num 307 ranges {759749=555563,1765266=792426,2964741=195008} 2023-08-27T20:52:08.450 INFO:tasks.rados.rados.0.smithi093.stdout:308: writing smithi093153278-308 from 759749 to 1315312 tid 1 2023-08-27T20:52:08.458 INFO:tasks.rados.rados.0.smithi093.stdout:308: writing smithi093153278-308 from 1765266 to 2557692 tid 2 2023-08-27T20:52:08.462 INFO:tasks.rados.rados.0.smithi093.stdout:308: writing smithi093153278-308 from 2964741 to 3159749 tid 3 2023-08-27T20:52:08.463 INFO:tasks.rados.rados.0.smithi093.stdout:309: write initial oid 309 2023-08-27T20:52:08.463 INFO:tasks.rados.rados.0.smithi093.stdout:309: seq_num 308 ranges {776556=630812,1823748=652437,3108517=68039} 2023-08-27T20:52:08.471 INFO:tasks.rados.rados.0.smithi093.stdout:309: writing smithi093153278-309 from 776556 to 1407368 tid 1 2023-08-27T20:52:08.477 INFO:tasks.rados.rados.0.smithi093.stdout:309: writing smithi093153278-309 from 1823748 to 2476185 tid 2 2023-08-27T20:52:08.484 INFO:tasks.rados.rados.0.smithi093.stdout:309: writing smithi093153278-309 from 3108517 to 3176556 tid 3 2023-08-27T20:52:08.484 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.484 INFO:tasks.rados.rados.0.smithi093.stdout:294: finishing write tid 2 to smithi093153278-294 2023-08-27T20:52:08.484 INFO:tasks.rados.rados.0.smithi093.stdout:294: finishing write tid 3 to smithi093153278-294 2023-08-27T20:52:08.484 INFO:tasks.rados.rados.0.smithi093.stdout:294: finishing write tid 4 to smithi093153278-294 2023-08-27T20:52:08.484 INFO:tasks.rados.rados.0.smithi093.stdout:294: finishing write tid 5 to smithi093153278-294 2023-08-27T20:52:08.484 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 294 v 54 (ObjNum 293 snap 0 seq_num 293) dirty exists 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:294: left oid 294 (ObjNum 293 snap 0 seq_num 293) 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:295: finishing write tid 1 to smithi093153278-295 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:295: finishing write tid 2 to smithi093153278-295 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:295: finishing write tid 3 to smithi093153278-295 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:295: finishing write tid 4 to smithi093153278-295 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:295: finishing write tid 5 to smithi093153278-295 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 295 v 68 (ObjNum 294 snap 0 seq_num 294) dirty exists 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:295: left oid 295 (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:296: finishing write tid 1 to smithi093153278-296 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:296: finishing write tid 2 to smithi093153278-296 2023-08-27T20:52:08.485 INFO:tasks.rados.rados.0.smithi093.stdout:296: finishing write tid 3 to smithi093153278-296 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:296: finishing write tid 4 to smithi093153278-296 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:296: finishing write tid 5 to smithi093153278-296 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 296 v 55 (ObjNum 295 snap 0 seq_num 295) dirty exists 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:296: left oid 296 (ObjNum 295 snap 0 seq_num 295) 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:297: finishing write tid 1 to smithi093153278-297 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:297: finishing write tid 2 to smithi093153278-297 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:297: finishing write tid 3 to smithi093153278-297 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:297: finishing write tid 4 to smithi093153278-297 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 297 v 58 (ObjNum 296 snap 0 seq_num 296) dirty exists 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:297: left oid 297 (ObjNum 296 snap 0 seq_num 296) 2023-08-27T20:52:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:298: finishing write tid 1 to smithi093153278-298 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:298: finishing write tid 2 to smithi093153278-298 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:298: finishing write tid 3 to smithi093153278-298 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:298: finishing write tid 4 to smithi093153278-298 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 298 v 68 (ObjNum 297 snap 0 seq_num 297) dirty exists 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:298: left oid 298 (ObjNum 297 snap 0 seq_num 297) 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:299: finishing write tid 1 to smithi093153278-299 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:299: finishing write tid 2 to smithi093153278-299 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:299: finishing write tid 3 to smithi093153278-299 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:299: finishing write tid 4 to smithi093153278-299 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:299: finishing write tid 5 to smithi093153278-299 2023-08-27T20:52:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 299 v 62 (ObjNum 298 snap 0 seq_num 298) dirty exists 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:299: left oid 299 (ObjNum 298 snap 0 seq_num 298) 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:300: finishing write tid 1 to smithi093153278-300 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:300: finishing write tid 2 to smithi093153278-300 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:300: finishing write tid 3 to smithi093153278-300 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:300: finishing write tid 4 to smithi093153278-300 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:300: finishing write tid 5 to smithi093153278-300 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 300 v 71 (ObjNum 299 snap 0 seq_num 299) dirty exists 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:300: left oid 300 (ObjNum 299 snap 0 seq_num 299) 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:301: finishing write tid 1 to smithi093153278-301 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:301: finishing write tid 2 to smithi093153278-301 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:301: finishing write tid 3 to smithi093153278-301 2023-08-27T20:52:08.488 INFO:tasks.rados.rados.0.smithi093.stdout:301: finishing write tid 4 to smithi093153278-301 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 301 v 82 (ObjNum 300 snap 0 seq_num 300) dirty exists 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:301: left oid 301 (ObjNum 300 snap 0 seq_num 300) 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:302: finishing write tid 1 to smithi093153278-302 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:302: finishing write tid 2 to smithi093153278-302 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:302: finishing write tid 3 to smithi093153278-302 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:302: finishing write tid 4 to smithi093153278-302 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:302: finishing write tid 5 to smithi093153278-302 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 302 v 60 (ObjNum 301 snap 0 seq_num 301) dirty exists 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:302: left oid 302 (ObjNum 301 snap 0 seq_num 301) 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:303: finishing write tid 1 to smithi093153278-303 2023-08-27T20:52:08.489 INFO:tasks.rados.rados.0.smithi093.stdout:303: finishing write tid 2 to smithi093153278-303 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:303: finishing write tid 3 to smithi093153278-303 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:303: finishing write tid 4 to smithi093153278-303 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:303: finishing write tid 5 to smithi093153278-303 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 303 v 86 (ObjNum 302 snap 0 seq_num 302) dirty exists 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:303: left oid 303 (ObjNum 302 snap 0 seq_num 302) 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:304: finishing write tid 1 to smithi093153278-304 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:304: finishing write tid 2 to smithi093153278-304 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:304: finishing write tid 3 to smithi093153278-304 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:304: finishing write tid 4 to smithi093153278-304 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 304 v 71 (ObjNum 303 snap 0 seq_num 303) dirty exists 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:304: left oid 304 (ObjNum 303 snap 0 seq_num 303) 2023-08-27T20:52:08.490 INFO:tasks.rados.rados.0.smithi093.stdout:305: finishing write tid 1 to smithi093153278-305 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:305: finishing write tid 2 to smithi093153278-305 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:305: finishing write tid 3 to smithi093153278-305 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:305: finishing write tid 4 to smithi093153278-305 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 305 v 65 (ObjNum 304 snap 0 seq_num 304) dirty exists 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:305: left oid 305 (ObjNum 304 snap 0 seq_num 304) 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:306: finishing write tid 1 to smithi093153278-306 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:306: finishing write tid 2 to smithi093153278-306 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:306: finishing write tid 3 to smithi093153278-306 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:306: finishing write tid 4 to smithi093153278-306 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:306: finishing write tid 5 to smithi093153278-306 2023-08-27T20:52:08.491 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 306 v 72 (ObjNum 305 snap 0 seq_num 305) dirty exists 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:306: left oid 306 (ObjNum 305 snap 0 seq_num 305) 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:307: finishing write tid 1 to smithi093153278-307 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:307: finishing write tid 2 to smithi093153278-307 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:307: finishing write tid 3 to smithi093153278-307 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:307: finishing write tid 4 to smithi093153278-307 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 307 v 83 (ObjNum 306 snap 0 seq_num 306) dirty exists 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:307: left oid 307 (ObjNum 306 snap 0 seq_num 306) 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:308: finishing write tid 1 to smithi093153278-308 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:308: finishing write tid 2 to smithi093153278-308 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:308: finishing write tid 3 to smithi093153278-308 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:308: finishing write tid 4 to smithi093153278-308 2023-08-27T20:52:08.492 INFO:tasks.rados.rados.0.smithi093.stdout:308: finishing write tid 5 to smithi093153278-308 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 308 v 66 (ObjNum 307 snap 0 seq_num 307) dirty exists 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:308: left oid 308 (ObjNum 307 snap 0 seq_num 307) 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:309: finishing write tid 1 to smithi093153278-309 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:309: finishing write tid 2 to smithi093153278-309 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:294: done (15 left) 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:295: done (14 left) 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:296: done (13 left) 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:297: done (12 left) 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:298: done (11 left) 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:299: done (10 left) 2023-08-27T20:52:08.493 INFO:tasks.rados.rados.0.smithi093.stdout:300: done (9 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:301: done (8 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:302: done (7 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:303: done (6 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:304: done (5 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:305: done (4 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:306: done (3 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:307: done (2 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:308: done (1 left) 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:310: write initial oid 310 2023-08-27T20:52:08.494 INFO:tasks.rados.rados.0.smithi093.stdout:310: seq_num 309 ranges {793363=706061,2282230=796095} 2023-08-27T20:52:08.495 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:08 smithi093 ceph-mon[118902]: pgmap v58: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 40 MiB/s wr, 202 op/s 2023-08-27T20:52:08.495 INFO:tasks.rados.rados.0.smithi093.stdout:310: writing smithi093153278-310 from 793363 to 1499424 tid 1 2023-08-27T20:52:08.495 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:08 smithi093 ceph-mon[111335]: pgmap v58: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 31 KiB/s rd, 40 MiB/s wr, 202 op/s 2023-08-27T20:52:08.500 INFO:tasks.rados.rados.0.smithi093.stdout:310: writing smithi093153278-310 from 2282230 to 3078325 tid 2 2023-08-27T20:52:08.500 INFO:tasks.rados.rados.0.smithi093.stdout:311: write initial oid 311 2023-08-27T20:52:08.500 INFO:tasks.rados.rados.0.smithi093.stdout:311: seq_num 310 ranges {410170=781310,1940712=656106,3163363=46807} 2023-08-27T20:52:08.504 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:52:08.508 INFO:tasks.rados.rados.0.smithi093.stdout:311: writing smithi093153278-311 from 410170 to 1191480 tid 1 2023-08-27T20:52:08.517 INFO:tasks.rados.rados.0.smithi093.stdout:311: writing smithi093153278-311 from 1940712 to 2596818 tid 2 2023-08-27T20:52:08.521 INFO:tasks.rados.rados.0.smithi093.stdout:311: writing smithi093153278-311 from 3163363 to 3210170 tid 3 2023-08-27T20:52:08.521 INFO:tasks.rados.rados.0.smithi093.stdout:312: write initial oid 312 2023-08-27T20:52:08.521 INFO:tasks.rados.rados.0.smithi093.stdout:312: seq_num 311 ranges {426977=456559,1482841=799764,2958080=268897} 2023-08-27T20:52:08.526 INFO:tasks.rados.rados.0.smithi093.stdout:312: writing smithi093153278-312 from 426977 to 883536 tid 1 2023-08-27T20:52:08.534 INFO:tasks.rados.rados.0.smithi093.stdout:312: writing smithi093153278-312 from 1482841 to 2282605 tid 2 2023-08-27T20:52:08.540 INFO:tasks.rados.rados.0.smithi093.stdout:312: writing smithi093153278-312 from 2958080 to 3226977 tid 3 2023-08-27T20:52:08.540 INFO:tasks.rados.rados.0.smithi093.stdout:313: write initial oid 313 2023-08-27T20:52:08.540 INFO:tasks.rados.rados.0.smithi093.stdout:313: seq_num 312 ranges {443784=648161,1657676=659775,2818209=425575} 2023-08-27T20:52:08.548 INFO:tasks.rados.rados.0.smithi093.stdout:313: writing smithi093153278-313 from 443784 to 1091945 tid 1 2023-08-27T20:52:08.554 INFO:tasks.rados.rados.0.smithi093.stdout:313: writing smithi093153278-313 from 1657676 to 2317451 tid 2 2023-08-27T20:52:08.560 INFO:tasks.rados.rados.0.smithi093.stdout:313: writing smithi093153278-313 from 2818209 to 3243784 tid 3 2023-08-27T20:52:08.560 INFO:tasks.rados.rados.0.smithi093.stdout:314: write initial oid 314 2023-08-27T20:52:08.560 INFO:tasks.rados.rados.0.smithi093.stdout:314: seq_num 313 ranges {460591=723410,1716158=403433,2729279=440140} 2023-08-27T20:52:08.568 INFO:tasks.rados.rados.0.smithi093.stdout:314: writing smithi093153278-314 from 460591 to 1184001 tid 1 2023-08-27T20:52:08.574 INFO:tasks.rados.rados.0.smithi093.stdout:314: writing smithi093153278-314 from 1716158 to 2119591 tid 2 2023-08-27T20:52:08.580 INFO:tasks.rados.rados.0.smithi093.stdout:314: writing smithi093153278-314 from 2729279 to 3169419 tid 3 2023-08-27T20:52:08.580 INFO:tasks.rados.rados.0.smithi093.stdout:315: write initial oid 315 2023-08-27T20:52:08.581 INFO:tasks.rados.rados.0.smithi093.stdout:315: seq_num 314 ranges {477398=798659,1774640=663444,2873055=404343} 2023-08-27T20:52:08.589 INFO:tasks.rados.rados.0.smithi093.stdout:315: writing smithi093153278-315 from 477398 to 1276057 tid 1 2023-08-27T20:52:08.595 INFO:tasks.rados.rados.0.smithi093.stdout:315: writing smithi093153278-315 from 1774640 to 2438084 tid 2 2023-08-27T20:52:08.600 INFO:tasks.rados.rados.0.smithi093.stdout:315: writing smithi093153278-315 from 2873055 to 3277398 tid 3 2023-08-27T20:52:08.600 INFO:tasks.rados.rados.0.smithi093.stdout:316: write initial oid 316 2023-08-27T20:52:08.601 INFO:tasks.rados.rados.0.smithi093.stdout:316: seq_num 315 ranges {494205=473908,1433122=407102,2384125=462684} 2023-08-27T20:52:08.606 INFO:tasks.rados.rados.0.smithi093.stdout:316: writing smithi093153278-316 from 494205 to 968113 tid 1 2023-08-27T20:52:08.611 INFO:tasks.rados.rados.0.smithi093.stdout:316: writing smithi093153278-316 from 1433122 to 1840224 tid 2 2023-08-27T20:52:08.617 INFO:tasks.rados.rados.0.smithi093.stdout:316: writing smithi093153278-316 from 2384125 to 2846809 tid 3 2023-08-27T20:52:08.617 INFO:tasks.rados.rados.0.smithi093.stdout:317: write initial oid 317 2023-08-27T20:52:08.617 INFO:tasks.rados.rados.0.smithi093.stdout:317: seq_num 316 ranges {511012=549157,1775251=550760,3095195=215817} 2023-08-27T20:52:08.623 INFO:tasks.rados.rados.0.smithi093.stdout:317: writing smithi093153278-317 from 511012 to 1060169 tid 1 2023-08-27T20:52:08.630 INFO:tasks.rados.rados.0.smithi093.stdout:317: writing smithi093153278-317 from 1775251 to 2326011 tid 2 2023-08-27T20:52:08.635 INFO:tasks.rados.rados.0.smithi093.stdout:317: writing smithi093153278-317 from 3095195 to 3311012 tid 3 2023-08-27T20:52:08.635 INFO:tasks.rados.rados.0.smithi093.stdout:318: write initial oid 318 2023-08-27T20:52:08.636 INFO:tasks.rados.rados.0.smithi093.stdout:309: finishing write tid 3 to smithi093153278-309 2023-08-27T20:52:08.636 INFO:tasks.rados.rados.0.smithi093.stdout:309: finishing write tid 4 to smithi093153278-309 2023-08-27T20:52:08.636 INFO:tasks.rados.rados.0.smithi093.stdout:318: seq_num 317 ranges {527819=624406,1833733=410771,2722618=601581} 2023-08-27T20:52:08.641 INFO:tasks.rados.rados.0.smithi093.stdout:318: writing smithi093153278-318 from 527819 to 1152225 tid 1 2023-08-27T20:52:08.648 INFO:tasks.rados.rados.0.smithi093.stdout:318: writing smithi093153278-318 from 1833733 to 2244504 tid 2 2023-08-27T20:52:08.655 INFO:tasks.rados.rados.0.smithi093.stdout:318: writing smithi093153278-318 from 2722618 to 3324199 tid 3 2023-08-27T20:52:08.655 INFO:tasks.rados.rados.0.smithi093.stdout:319: write initial oid 319 2023-08-27T20:52:08.655 INFO:tasks.rados.rados.0.smithi093.stdout:319: seq_num 318 ranges {544626=699655,1892215=554429,3150041=194585} 2023-08-27T20:52:08.662 INFO:tasks.rados.rados.0.smithi093.stdout:319: writing smithi093153278-319 from 544626 to 1244281 tid 1 2023-08-27T20:52:08.669 INFO:tasks.rados.rados.0.smithi093.stdout:319: writing smithi093153278-319 from 1892215 to 2446644 tid 2 2023-08-27T20:52:08.674 INFO:tasks.rados.rados.0.smithi093.stdout:319: writing smithi093153278-319 from 3150041 to 3344626 tid 3 2023-08-27T20:52:08.674 INFO:tasks.rados.rados.0.smithi093.stdout:320: write initial oid 320 2023-08-27T20:52:08.674 INFO:tasks.rados.rados.0.smithi093.stdout:320: seq_num 319 ranges {561433=774904,1950697=414440,2777464=583969} 2023-08-27T20:52:08.681 INFO:tasks.rados.rados.0.smithi093.stdout:320: writing smithi093153278-320 from 561433 to 1336337 tid 1 2023-08-27T20:52:08.688 INFO:tasks.rados.rados.0.smithi093.stdout:320: writing smithi093153278-320 from 1950697 to 2365137 tid 2 2023-08-27T20:52:08.694 INFO:tasks.rados.rados.0.smithi093.stdout:320: writing smithi093153278-320 from 2777464 to 3361433 tid 3 2023-08-27T20:52:08.694 INFO:tasks.rados.rados.0.smithi093.stdout:321: write initial oid 321 2023-08-27T20:52:08.695 INFO:tasks.rados.rados.0.smithi093.stdout:321: seq_num 320 ranges {578240=566506,1609179=558098,2804887=551750} 2023-08-27T20:52:08.700 INFO:tasks.rados.rados.0.smithi093.stdout:321: writing smithi093153278-321 from 578240 to 1144746 tid 1 2023-08-27T20:52:08.707 INFO:tasks.rados.rados.0.smithi093.stdout:321: writing smithi093153278-321 from 1609179 to 2167277 tid 2 2023-08-27T20:52:08.715 INFO:tasks.rados.rados.0.smithi093.stdout:321: writing smithi093153278-321 from 2804887 to 3356637 tid 3 2023-08-27T20:52:08.715 INFO:tasks.rados.rados.0.smithi093.stdout:322: write initial oid 322 2023-08-27T20:52:08.715 INFO:tasks.rados.rados.0.smithi093.stdout:309: finishing write tid 5 to smithi093153278-309 2023-08-27T20:52:08.715 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 309 v 75 (ObjNum 308 snap 0 seq_num 308) dirty exists 2023-08-27T20:52:08.715 INFO:tasks.rados.rados.0.smithi093.stdout:309: left oid 309 (ObjNum 308 snap 0 seq_num 308) 2023-08-27T20:52:08.715 INFO:tasks.rados.rados.0.smithi093.stdout:310: finishing write tid 1 to smithi093153278-310 2023-08-27T20:52:08.715 INFO:tasks.rados.rados.0.smithi093.stdout:322: seq_num 321 ranges {595047=641755,1667661=418109,2548663=763022} 2023-08-27T20:52:08.721 INFO:tasks.rados.rados.0.smithi093.stdout:322: writing smithi093153278-322 from 595047 to 1236802 tid 1 2023-08-27T20:52:08.726 INFO:tasks.rados.rados.0.smithi093.stdout:322: writing smithi093153278-322 from 1667661 to 2085770 tid 2 2023-08-27T20:52:08.733 INFO:tasks.rados.rados.0.smithi093.stdout:322: writing smithi093153278-322 from 2548663 to 3311685 tid 3 2023-08-27T20:52:08.733 INFO:tasks.rados.rados.0.smithi093.stdout:309: done (13 left) 2023-08-27T20:52:08.733 INFO:tasks.rados.rados.0.smithi093.stdout:323: write initial oid 323 2023-08-27T20:52:08.733 INFO:tasks.rados.rados.0.smithi093.stdout:323: seq_num 322 ranges {611854=717004,2126143=561767,3259733=152121} 2023-08-27T20:52:08.740 INFO:tasks.rados.rados.0.smithi093.stdout:323: writing smithi093153278-323 from 611854 to 1328858 tid 1 2023-08-27T20:52:08.747 INFO:tasks.rados.rados.0.smithi093.stdout:323: writing smithi093153278-323 from 2126143 to 2687910 tid 2 2023-08-27T20:52:08.752 INFO:tasks.rados.rados.0.smithi093.stdout:323: writing smithi093153278-323 from 3259733 to 3411854 tid 3 2023-08-27T20:52:08.752 INFO:tasks.rados.rados.0.smithi093.stdout:324: write initial oid 324 2023-08-27T20:52:08.752 INFO:tasks.rados.rados.0.smithi093.stdout:324: seq_num 323 ranges {628661=792253,2184625=421778,3403509=25152} 2023-08-27T20:52:08.760 INFO:tasks.rados.rados.0.smithi093.stdout:324: writing smithi093153278-324 from 628661 to 1420914 tid 1 2023-08-27T20:52:08.767 INFO:tasks.rados.rados.0.smithi093.stdout:324: writing smithi093153278-324 from 2184625 to 2606403 tid 2 2023-08-27T20:52:08.771 INFO:tasks.rados.rados.0.smithi093.stdout:324: writing smithi093153278-324 from 3403509 to 3428661 tid 3 2023-08-27T20:52:08.772 INFO:tasks.rados.rados.0.smithi093.stdout:325: write initial oid 325 2023-08-27T20:52:08.772 INFO:tasks.rados.rados.0.smithi093.stdout:325: seq_num 324 ranges {645468=467502,1726754=565436,2798226=596838} 2023-08-27T20:52:08.778 INFO:tasks.rados.rados.0.smithi093.stdout:325: writing smithi093153278-325 from 645468 to 1112970 tid 1 2023-08-27T20:52:08.785 INFO:tasks.rados.rados.0.smithi093.stdout:325: writing smithi093153278-325 from 1726754 to 2292190 tid 2 2023-08-27T20:52:08.791 INFO:tasks.rados.rados.0.smithi093.stdout:325: writing smithi093153278-325 from 2798226 to 3395064 tid 3 2023-08-27T20:52:08.791 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.791 INFO:tasks.rados.rados.0.smithi093.stdout:310: finishing write tid 2 to smithi093153278-310 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:310: finishing write tid 3 to smithi093153278-310 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:310: finishing write tid 4 to smithi093153278-310 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 310 v 86 (ObjNum 309 snap 0 seq_num 309) dirty exists 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:310: left oid 310 (ObjNum 309 snap 0 seq_num 309) 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:311: finishing write tid 1 to smithi093153278-311 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:310: done (15 left) 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:326: write initial oid 326 2023-08-27T20:52:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:326: seq_num 325 ranges {662275=542751,1785236=425447,2942002=520273} 2023-08-27T20:52:08.798 INFO:tasks.rados.rados.0.smithi093.stdout:326: writing smithi093153278-326 from 662275 to 1205026 tid 1 2023-08-27T20:52:08.804 INFO:tasks.rados.rados.0.smithi093.stdout:326: writing smithi093153278-326 from 1785236 to 2210683 tid 2 2023-08-27T20:52:08.811 INFO:tasks.rados.rados.0.smithi093.stdout:326: writing smithi093153278-326 from 2942002 to 3462275 tid 3 2023-08-27T20:52:08.811 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.811 INFO:tasks.rados.rados.0.smithi093.stdout:311: finishing write tid 2 to smithi093153278-311 2023-08-27T20:52:08.811 INFO:tasks.rados.rados.0.smithi093.stdout:311: finishing write tid 3 to smithi093153278-311 2023-08-27T20:52:08.811 INFO:tasks.rados.rados.0.smithi093.stdout:311: finishing write tid 4 to smithi093153278-311 2023-08-27T20:52:08.811 INFO:tasks.rados.rados.0.smithi093.stdout:311: finishing write tid 5 to smithi093153278-311 2023-08-27T20:52:08.812 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 311 v 62 (ObjNum 310 snap 0 seq_num 310) dirty exists 2023-08-27T20:52:08.812 INFO:tasks.rados.rados.0.smithi093.stdout:311: left oid 311 (ObjNum 310 snap 0 seq_num 310) 2023-08-27T20:52:08.812 INFO:tasks.rados.rados.0.smithi093.stdout:311: done (15 left) 2023-08-27T20:52:08.812 INFO:tasks.rados.rados.0.smithi093.stdout:327: write initial oid 327 2023-08-27T20:52:08.812 INFO:tasks.rados.rados.0.smithi093.stdout:327: seq_num 326 ranges {679082=618000,1843718=569105,2853072=626010} 2023-08-27T20:52:08.818 INFO:tasks.rados.rados.0.smithi093.stdout:327: writing smithi093153278-327 from 679082 to 1297082 tid 1 2023-08-27T20:52:08.824 INFO:tasks.rados.rados.0.smithi093.stdout:327: writing smithi093153278-327 from 1843718 to 2412823 tid 2 2023-08-27T20:52:08.831 INFO:tasks.rados.rados.0.smithi093.stdout:327: writing smithi093153278-327 from 2853072 to 3479082 tid 3 2023-08-27T20:52:08.831 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.831 INFO:tasks.rados.rados.0.smithi093.stdout:312: finishing write tid 1 to smithi093153278-312 2023-08-27T20:52:08.831 INFO:tasks.rados.rados.0.smithi093.stdout:312: finishing write tid 2 to smithi093153278-312 2023-08-27T20:52:08.831 INFO:tasks.rados.rados.0.smithi093.stdout:312: finishing write tid 3 to smithi093153278-312 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:312: finishing write tid 4 to smithi093153278-312 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:312: finishing write tid 5 to smithi093153278-312 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 312 v 70 (ObjNum 311 snap 0 seq_num 311) dirty exists 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:312: left oid 312 (ObjNum 311 snap 0 seq_num 311) 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:313: finishing write tid 1 to smithi093153278-313 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:313: finishing write tid 2 to smithi093153278-313 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:312: done (15 left) 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:328: write initial oid 328 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:313: finishing write tid 3 to smithi093153278-313 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:313: finishing write tid 4 to smithi093153278-313 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:313: finishing write tid 5 to smithi093153278-313 2023-08-27T20:52:08.832 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 313 v 58 (ObjNum 312 snap 0 seq_num 312) dirty exists 2023-08-27T20:52:08.833 INFO:tasks.rados.rados.0.smithi093.stdout:313: left oid 313 (ObjNum 312 snap 0 seq_num 312) 2023-08-27T20:52:08.833 INFO:tasks.rados.rados.0.smithi093.stdout:314: finishing write tid 1 to smithi093153278-314 2023-08-27T20:52:08.833 INFO:tasks.rados.rados.0.smithi093.stdout:314: finishing write tid 2 to smithi093153278-314 2023-08-27T20:52:08.833 INFO:tasks.rados.rados.0.smithi093.stdout:314: finishing write tid 3 to smithi093153278-314 2023-08-27T20:52:08.833 INFO:tasks.rados.rados.0.smithi093.stdout:328: seq_num 327 ranges {695889=409602,1618553=712763,2996848=499041} 2023-08-27T20:52:08.836 INFO:tasks.rados.rados.0.smithi093.stdout:328: writing smithi093153278-328 from 695889 to 1105491 tid 1 2023-08-27T20:52:08.843 INFO:tasks.rados.rados.0.smithi093.stdout:328: writing smithi093153278-328 from 1618553 to 2331316 tid 2 2023-08-27T20:52:08.850 INFO:tasks.rados.rados.0.smithi093.stdout:328: writing smithi093153278-328 from 2996848 to 3495889 tid 3 2023-08-27T20:52:08.851 INFO:tasks.rados.rados.0.smithi093.stdout:313: done (15 left) 2023-08-27T20:52:08.851 INFO:tasks.rados.rados.0.smithi093.stdout:329: write initial oid 329 2023-08-27T20:52:08.851 INFO:tasks.rados.rados.0.smithi093.stdout:329: seq_num 328 ranges {712696=484851,1960682=572774,3307918=204778} 2023-08-27T20:52:08.857 INFO:tasks.rados.rados.0.smithi093.stdout:329: writing smithi093153278-329 from 712696 to 1197547 tid 1 2023-08-27T20:52:08.866 INFO:tasks.rados.rados.0.smithi093.stdout:329: writing smithi093153278-329 from 1960682 to 2533456 tid 2 2023-08-27T20:52:08.872 INFO:tasks.rados.rados.0.smithi093.stdout:329: writing smithi093153278-329 from 3307918 to 3512696 tid 3 2023-08-27T20:52:08.872 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.872 INFO:tasks.rados.rados.0.smithi093.stdout:314: finishing write tid 4 to smithi093153278-314 2023-08-27T20:52:08.872 INFO:tasks.rados.rados.0.smithi093.stdout:314: finishing write tid 5 to smithi093153278-314 2023-08-27T20:52:08.872 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 314 v 65 (ObjNum 313 snap 0 seq_num 313) dirty exists 2023-08-27T20:52:08.872 INFO:tasks.rados.rados.0.smithi093.stdout:314: left oid 314 (ObjNum 313 snap 0 seq_num 313) 2023-08-27T20:52:08.872 INFO:tasks.rados.rados.0.smithi093.stdout:315: finishing write tid 1 to smithi093153278-315 2023-08-27T20:52:08.873 INFO:tasks.rados.rados.0.smithi093.stdout:315: finishing write tid 2 to smithi093153278-315 2023-08-27T20:52:08.873 INFO:tasks.rados.rados.0.smithi093.stdout:315: finishing write tid 3 to smithi093153278-315 2023-08-27T20:52:08.873 INFO:tasks.rados.rados.0.smithi093.stdout:315: finishing write tid 4 to smithi093153278-315 2023-08-27T20:52:08.873 INFO:tasks.rados.rados.0.smithi093.stdout:314: done (15 left) 2023-08-27T20:52:08.873 INFO:tasks.rados.rados.0.smithi093.stdout:330: write initial oid 330 2023-08-27T20:52:08.873 INFO:tasks.rados.rados.0.smithi093.stdout:330: seq_num 329 ranges {729503=560100,2019164=716432,3335341=194162} 2023-08-27T20:52:08.879 INFO:tasks.rados.rados.0.smithi093.stdout:330: writing smithi093153278-330 from 729503 to 1289603 tid 1 2023-08-27T20:52:08.888 INFO:tasks.rados.rados.0.smithi093.stdout:330: writing smithi093153278-330 from 2019164 to 2735596 tid 2 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:330: writing smithi093153278-330 from 3335341 to 3529503 tid 3 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:315: finishing write tid 5 to smithi093153278-315 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 71 (ObjNum 314 snap 0 seq_num 314) dirty exists 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:315: left oid 315 (ObjNum 314 snap 0 seq_num 314) 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:315: done (15 left) 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:331: write initial oid 331 2023-08-27T20:52:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:331: seq_num 330 ranges {746310=635349,2077646=576443,3362764=183546} 2023-08-27T20:52:08.902 INFO:tasks.rados.rados.0.smithi093.stdout:331: writing smithi093153278-331 from 746310 to 1381659 tid 1 2023-08-27T20:52:08.910 INFO:tasks.rados.rados.0.smithi093.stdout:331: writing smithi093153278-331 from 2077646 to 2654089 tid 2 2023-08-27T20:52:08.915 INFO:tasks.rados.rados.0.smithi093.stdout:331: writing smithi093153278-331 from 3362764 to 3546310 tid 3 2023-08-27T20:52:08.915 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.915 INFO:tasks.rados.rados.0.smithi093.stdout:316: finishing write tid 1 to smithi093153278-316 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:316: finishing write tid 2 to smithi093153278-316 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:316: finishing write tid 3 to smithi093153278-316 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:316: finishing write tid 4 to smithi093153278-316 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:316: finishing write tid 5 to smithi093153278-316 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 316 v 90 (ObjNum 315 snap 0 seq_num 315) dirty exists 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:316: left oid 316 (ObjNum 315 snap 0 seq_num 315) 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:317: finishing write tid 1 to smithi093153278-317 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:317: finishing write tid 2 to smithi093153278-317 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:316: done (15 left) 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:332: write initial oid 332 2023-08-27T20:52:08.916 INFO:tasks.rados.rados.0.smithi093.stdout:332: seq_num 331 ranges {763117=710598,2136128=720101,3390187=172930} 2023-08-27T20:52:08.923 INFO:tasks.rados.rados.0.smithi093.stdout:332: writing smithi093153278-332 from 763117 to 1473715 tid 1 2023-08-27T20:52:08.931 INFO:tasks.rados.rados.0.smithi093.stdout:332: writing smithi093153278-332 from 2136128 to 2856229 tid 2 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:332: writing smithi093153278-332 from 3390187 to 3563117 tid 3 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:317: finishing write tid 3 to smithi093153278-317 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:317: finishing write tid 4 to smithi093153278-317 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:317: finishing write tid 5 to smithi093153278-317 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 317 v 74 (ObjNum 316 snap 0 seq_num 316) dirty exists 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:317: left oid 317 (ObjNum 316 snap 0 seq_num 316) 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:318: finishing write tid 1 to smithi093153278-318 2023-08-27T20:52:08.936 INFO:tasks.rados.rados.0.smithi093.stdout:317: done (15 left) 2023-08-27T20:52:08.937 INFO:tasks.rados.rados.0.smithi093.stdout:333: write initial oid 333 2023-08-27T20:52:08.937 INFO:tasks.rados.rados.0.smithi093.stdout:333: seq_num 332 ranges {779924=785847,2078257=580112,3301257=278667} 2023-08-27T20:52:08.944 INFO:tasks.rados.rados.0.smithi093.stdout:333: writing smithi093153278-333 from 779924 to 1565771 tid 1 2023-08-27T20:52:08.950 INFO:tasks.rados.rados.0.smithi093.stdout:333: writing smithi093153278-333 from 2078257 to 2658369 tid 2 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:333: writing smithi093153278-333 from 3301257 to 3579924 tid 3 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:318: finishing write tid 2 to smithi093153278-318 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:318: finishing write tid 3 to smithi093153278-318 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:318: finishing write tid 4 to smithi093153278-318 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:318: finishing write tid 5 to smithi093153278-318 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 318 v 90 (ObjNum 317 snap 0 seq_num 317) dirty exists 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:318: left oid 318 (ObjNum 317 snap 0 seq_num 317) 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:319: finishing write tid 1 to smithi093153278-319 2023-08-27T20:52:08.956 INFO:tasks.rados.rados.0.smithi093.stdout:318: done (15 left) 2023-08-27T20:52:08.957 INFO:tasks.rados.rados.0.smithi093.stdout:334: write initial oid 334 2023-08-27T20:52:08.957 INFO:tasks.rados.rados.0.smithi093.stdout:334: seq_num 333 ranges {796731=461096,1736739=723770,2928680=668051} 2023-08-27T20:52:08.962 INFO:tasks.rados.rados.0.smithi093.stdout:334: writing smithi093153278-334 from 796731 to 1257827 tid 1 2023-08-27T20:52:08.969 INFO:tasks.rados.rados.0.smithi093.stdout:334: writing smithi093153278-334 from 1736739 to 2460509 tid 2 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:334: writing smithi093153278-334 from 2928680 to 3596731 tid 3 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:319: finishing write tid 2 to smithi093153278-319 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:319: finishing write tid 3 to smithi093153278-319 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:319: finishing write tid 4 to smithi093153278-319 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:319: finishing write tid 5 to smithi093153278-319 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 319 v 76 (ObjNum 318 snap 0 seq_num 318) dirty exists 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:319: left oid 319 (ObjNum 318 snap 0 seq_num 318) 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:320: finishing write tid 1 to smithi093153278-320 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:320: finishing write tid 2 to smithi093153278-320 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:319: done (15 left) 2023-08-27T20:52:08.977 INFO:tasks.rados.rados.0.smithi093.stdout:335: write initial oid 335 2023-08-27T20:52:08.978 INFO:tasks.rados.rados.0.smithi093.stdout:335: seq_num 334 ranges {413538=536345,1395221=583781,2556103=658617} 2023-08-27T20:52:08.982 INFO:tasks.rados.rados.0.smithi093.stdout:335: writing smithi093153278-335 from 413538 to 949883 tid 1 2023-08-27T20:52:08.989 INFO:tasks.rados.rados.0.smithi093.stdout:335: writing smithi093153278-335 from 1395221 to 1979002 tid 2 2023-08-27T20:52:08.997 INFO:tasks.rados.rados.0.smithi093.stdout:335: writing smithi093153278-335 from 2556103 to 3214720 tid 3 2023-08-27T20:52:08.997 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:08.997 INFO:tasks.rados.rados.0.smithi093.stdout:320: finishing write tid 3 to smithi093153278-320 2023-08-27T20:52:08.997 INFO:tasks.rados.rados.0.smithi093.stdout:320: finishing write tid 4 to smithi093153278-320 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:320: finishing write tid 5 to smithi093153278-320 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 320 v 82 (ObjNum 319 snap 0 seq_num 319) dirty exists 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:320: left oid 320 (ObjNum 319 snap 0 seq_num 319) 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:321: finishing write tid 1 to smithi093153278-321 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:321: finishing write tid 2 to smithi093153278-321 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:321: finishing write tid 3 to smithi093153278-321 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:321: finishing write tid 4 to smithi093153278-321 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:321: finishing write tid 5 to smithi093153278-321 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 321 v 75 (ObjNum 320 snap 0 seq_num 320) dirty exists 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:321: left oid 321 (ObjNum 320 snap 0 seq_num 320) 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:322: finishing write tid 1 to smithi093153278-322 2023-08-27T20:52:08.998 INFO:tasks.rados.rados.0.smithi093.stdout:322: finishing write tid 2 to smithi093153278-322 2023-08-27T20:52:08.999 INFO:tasks.rados.rados.0.smithi093.stdout:320: done (15 left) 2023-08-27T20:52:08.999 INFO:tasks.rados.rados.0.smithi093.stdout:321: done (14 left) 2023-08-27T20:52:08.999 INFO:tasks.rados.rados.0.smithi093.stdout:336: write initial oid 336 2023-08-27T20:52:08.999 INFO:tasks.rados.rados.0.smithi093.stdout:336: seq_num 335 ranges {430345=727947,1570056=727439,2699879=469889} 2023-08-27T20:52:09.003 INFO:tasks.rados.rados.0.smithi093.stdout:336: writing smithi093153278-336 from 430345 to 1158292 tid 1 2023-08-27T20:52:09.010 INFO:tasks.rados.rados.0.smithi093.stdout:336: writing smithi093153278-336 from 1570056 to 2297495 tid 2 2023-08-27T20:52:09.015 INFO:tasks.rados.rados.0.smithi093.stdout:336: writing smithi093153278-336 from 2699879 to 3169768 tid 3 2023-08-27T20:52:09.015 INFO:tasks.rados.rados.0.smithi093.stdout:337: write initial oid 337 2023-08-27T20:52:09.015 INFO:tasks.rados.rados.0.smithi093.stdout:337: seq_num 336 ranges {447152=403196,1512185=587450,2727302=681161} 2023-08-27T20:52:09.019 INFO:tasks.rados.rados.0.smithi093.stdout:337: writing smithi093153278-337 from 447152 to 850348 tid 1 2023-08-27T20:52:09.027 INFO:tasks.rados.rados.0.smithi093.stdout:337: writing smithi093153278-337 from 1512185 to 2099635 tid 2 2023-08-27T20:52:09.035 INFO:tasks.rados.rados.0.smithi093.stdout:337: writing smithi093153278-337 from 2727302 to 3408463 tid 3 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:322: finishing write tid 3 to smithi093153278-322 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:322: finishing write tid 4 to smithi093153278-322 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:322: finishing write tid 5 to smithi093153278-322 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 322 v 64 (ObjNum 321 snap 0 seq_num 321) dirty exists 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:322: left oid 322 (ObjNum 321 snap 0 seq_num 321) 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:323: finishing write tid 1 to smithi093153278-323 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:323: finishing write tid 2 to smithi093153278-323 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:323: finishing write tid 3 to smithi093153278-323 2023-08-27T20:52:09.036 INFO:tasks.rados.rados.0.smithi093.stdout:323: finishing write tid 4 to smithi093153278-323 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:323: finishing write tid 5 to smithi093153278-323 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 323 v 68 (ObjNum 322 snap 0 seq_num 322) dirty exists 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:323: left oid 323 (ObjNum 322 snap 0 seq_num 322) 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:324: finishing write tid 1 to smithi093153278-324 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:324: finishing write tid 2 to smithi093153278-324 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:322: done (15 left) 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:323: done (14 left) 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:338: write initial oid 338 2023-08-27T20:52:09.037 INFO:tasks.rados.rados.0.smithi093.stdout:338: seq_num 337 ranges {463959=478445,1570667=731108,3038372=492433} 2023-08-27T20:52:09.041 INFO:tasks.rados.rados.0.smithi093.stdout:338: writing smithi093153278-338 from 463959 to 942404 tid 1 2023-08-27T20:52:09.049 INFO:tasks.rados.rados.0.smithi093.stdout:338: writing smithi093153278-338 from 1570667 to 2301775 tid 2 2023-08-27T20:52:09.057 INFO:tasks.rados.rados.0.smithi093.stdout:338: writing smithi093153278-338 from 3038372 to 3530805 tid 3 2023-08-27T20:52:09.057 INFO:tasks.rados.rados.0.smithi093.stdout:339: write initial oid 339 2023-08-27T20:52:09.057 INFO:tasks.rados.rados.0.smithi093.stdout:339: seq_num 338 ranges {480766=553694,1629149=591119,2782148=420058} 2023-08-27T20:52:09.063 INFO:tasks.rados.rados.0.smithi093.stdout:339: writing smithi093153278-339 from 480766 to 1034460 tid 1 2023-08-27T20:52:09.071 INFO:tasks.rados.rados.0.smithi093.stdout:339: writing smithi093153278-339 from 1629149 to 2220268 tid 2 2023-08-27T20:52:09.077 INFO:tasks.rados.rados.0.smithi093.stdout:339: writing smithi093153278-339 from 2782148 to 3202206 tid 3 2023-08-27T20:52:09.077 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.077 INFO:tasks.rados.rados.0.smithi093.stdout:324: finishing write tid 3 to smithi093153278-324 2023-08-27T20:52:09.077 INFO:tasks.rados.rados.0.smithi093.stdout:324: finishing write tid 4 to smithi093153278-324 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:324: finishing write tid 5 to smithi093153278-324 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 324 v 58 (ObjNum 323 snap 0 seq_num 323) dirty exists 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:324: left oid 324 (ObjNum 323 snap 0 seq_num 323) 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:325: finishing write tid 1 to smithi093153278-325 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:325: finishing write tid 2 to smithi093153278-325 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:325: finishing write tid 3 to smithi093153278-325 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:324: done (15 left) 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:340: write initial oid 340 2023-08-27T20:52:09.078 INFO:tasks.rados.rados.0.smithi093.stdout:340: seq_num 339 ranges {497573=628943,1687631=734777,3093218=604355} 2023-08-27T20:52:09.084 INFO:tasks.rados.rados.0.smithi093.stdout:340: writing smithi093153278-340 from 497573 to 1126516 tid 1 2023-08-27T20:52:09.092 INFO:tasks.rados.rados.0.smithi093.stdout:340: writing smithi093153278-340 from 1687631 to 2422408 tid 2 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout:340: writing smithi093153278-340 from 3093218 to 3697573 tid 3 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout:325: finishing write tid 4 to smithi093153278-325 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout:325: finishing write tid 5 to smithi093153278-325 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 325 v 94 (ObjNum 324 snap 0 seq_num 324) dirty exists 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout:325: left oid 325 (ObjNum 324 snap 0 seq_num 324) 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout:326: finishing write tid 1 to smithi093153278-326 2023-08-27T20:52:09.101 INFO:tasks.rados.rados.0.smithi093.stdout:326: finishing write tid 2 to smithi093153278-326 2023-08-27T20:52:09.102 INFO:tasks.rados.rados.0.smithi093.stdout:325: done (15 left) 2023-08-27T20:52:09.102 INFO:tasks.rados.rados.0.smithi093.stdout:341: write initial oid 341 2023-08-27T20:52:09.102 INFO:tasks.rados.rados.0.smithi093.stdout:341: seq_num 340 ranges {514380=704192,1629760=478435,2604288=442602} 2023-08-27T20:52:09.108 INFO:tasks.rados.rados.0.smithi093.stdout:341: writing smithi093153278-341 from 514380 to 1218572 tid 1 2023-08-27T20:52:09.114 INFO:tasks.rados.rados.0.smithi093.stdout:341: writing smithi093153278-341 from 1629760 to 2108195 tid 2 2023-08-27T20:52:09.121 INFO:tasks.rados.rados.0.smithi093.stdout:341: writing smithi093153278-341 from 2604288 to 3046890 tid 3 2023-08-27T20:52:09.121 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.121 INFO:tasks.rados.rados.0.smithi093.stdout:326: finishing write tid 3 to smithi093153278-326 2023-08-27T20:52:09.121 INFO:tasks.rados.rados.0.smithi093.stdout:326: finishing write tid 4 to smithi093153278-326 2023-08-27T20:52:09.121 INFO:tasks.rados.rados.0.smithi093.stdout:326: finishing write tid 5 to smithi093153278-326 2023-08-27T20:52:09.121 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 69 (ObjNum 325 snap 0 seq_num 325) dirty exists 2023-08-27T20:52:09.121 INFO:tasks.rados.rados.0.smithi093.stdout:326: left oid 326 (ObjNum 325 snap 0 seq_num 325) 2023-08-27T20:52:09.122 INFO:tasks.rados.rados.0.smithi093.stdout:326: done (15 left) 2023-08-27T20:52:09.122 INFO:tasks.rados.rados.0.smithi093.stdout:342: write initial oid 342 2023-08-27T20:52:09.122 INFO:tasks.rados.rados.0.smithi093.stdout:342: seq_num 341 ranges {531187=779441,2088242=738446,3431711=299476} 2023-08-27T20:52:09.129 INFO:tasks.rados.rados.0.smithi093.stdout:342: writing smithi093153278-342 from 531187 to 1310628 tid 1 2023-08-27T20:52:09.138 INFO:tasks.rados.rados.0.smithi093.stdout:342: writing smithi093153278-342 from 2088242 to 2826688 tid 2 2023-08-27T20:52:09.144 INFO:tasks.rados.rados.0.smithi093.stdout:342: writing smithi093153278-342 from 3431711 to 3731187 tid 3 2023-08-27T20:52:09.144 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.144 INFO:tasks.rados.rados.0.smithi093.stdout:327: finishing write tid 1 to smithi093153278-327 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:327: finishing write tid 2 to smithi093153278-327 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:327: finishing write tid 3 to smithi093153278-327 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:327: finishing write tid 4 to smithi093153278-327 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:327: finishing write tid 5 to smithi093153278-327 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 327 v 79 (ObjNum 326 snap 0 seq_num 326) dirty exists 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:327: left oid 327 (ObjNum 326 snap 0 seq_num 326) 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:328: finishing write tid 1 to smithi093153278-328 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:327: done (15 left) 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:343: write initial oid 343 2023-08-27T20:52:09.145 INFO:tasks.rados.rados.0.smithi093.stdout:343: seq_num 342 ranges {547994=454690,1746724=482104,2659134=465146,3605976=142018} 2023-08-27T20:52:09.150 INFO:tasks.rados.rados.0.smithi093.stdout:343: writing smithi093153278-343 from 547994 to 1002684 tid 1 2023-08-27T20:52:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:343: writing smithi093153278-343 from 1746724 to 2228828 tid 2 2023-08-27T20:52:09.163 INFO:tasks.rados.rados.0.smithi093.stdout:343: writing smithi093153278-343 from 2659134 to 3124280 tid 3 2023-08-27T20:52:09.167 INFO:tasks.rados.rados.0.smithi093.stdout:343: writing smithi093153278-343 from 3605976 to 3747994 tid 4 2023-08-27T20:52:09.167 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:328: finishing write tid 2 to smithi093153278-328 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:328: finishing write tid 3 to smithi093153278-328 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:328: finishing write tid 4 to smithi093153278-328 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:328: finishing write tid 5 to smithi093153278-328 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 75 (ObjNum 327 snap 0 seq_num 327) dirty exists 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:328: left oid 328 (ObjNum 327 snap 0 seq_num 327) 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:329: finishing write tid 1 to smithi093153278-329 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:329: finishing write tid 2 to smithi093153278-329 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:329: finishing write tid 3 to smithi093153278-329 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:329: finishing write tid 4 to smithi093153278-329 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:328: done (15 left) 2023-08-27T20:52:09.168 INFO:tasks.rados.rados.0.smithi093.stdout:344: write initial oid 344 2023-08-27T20:52:09.169 INFO:tasks.rados.rados.0.smithi093.stdout:344: seq_num 343 ranges {564801=646292,1921559=742115,3202910=561891} 2023-08-27T20:52:09.175 INFO:tasks.rados.rados.0.smithi093.stdout:344: writing smithi093153278-344 from 564801 to 1211093 tid 1 2023-08-27T20:52:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:344: writing smithi093153278-344 from 1921559 to 2663674 tid 2 2023-08-27T20:52:09.191 INFO:tasks.rados.rados.0.smithi093.stdout:344: writing smithi093153278-344 from 3202910 to 3764801 tid 3 2023-08-27T20:52:09.191 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:329: finishing write tid 5 to smithi093153278-329 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 329 v 78 (ObjNum 328 snap 0 seq_num 328) dirty exists 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:329: left oid 329 (ObjNum 328 snap 0 seq_num 328) 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:330: finishing write tid 1 to smithi093153278-330 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:329: done (15 left) 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:345: write initial oid 345 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:330: finishing write tid 2 to smithi093153278-330 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:330: finishing write tid 3 to smithi093153278-330 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:330: finishing write tid 4 to smithi093153278-330 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:330: finishing write tid 5 to smithi093153278-330 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 330 v 66 (ObjNum 329 snap 0 seq_num 329) dirty exists 2023-08-27T20:52:09.192 INFO:tasks.rados.rados.0.smithi093.stdout:330: left oid 330 (ObjNum 329 snap 0 seq_num 329) 2023-08-27T20:52:09.193 INFO:tasks.rados.rados.0.smithi093.stdout:331: finishing write tid 1 to smithi093153278-331 2023-08-27T20:52:09.193 INFO:tasks.rados.rados.0.smithi093.stdout:345: seq_num 344 ranges {581608=721541,1863688=485773,3113980=604043} 2023-08-27T20:52:09.199 INFO:tasks.rados.rados.0.smithi093.stdout:345: writing smithi093153278-345 from 581608 to 1303149 tid 1 2023-08-27T20:52:09.205 INFO:tasks.rados.rados.0.smithi093.stdout:345: writing smithi093153278-345 from 1863688 to 2349461 tid 2 2023-08-27T20:52:09.214 INFO:tasks.rados.rados.0.smithi093.stdout:345: writing smithi093153278-345 from 3113980 to 3718023 tid 3 2023-08-27T20:52:09.214 INFO:tasks.rados.rados.0.smithi093.stdout:330: done (15 left) 2023-08-27T20:52:09.214 INFO:tasks.rados.rados.0.smithi093.stdout:346: write initial oid 346 2023-08-27T20:52:09.214 INFO:tasks.rados.rados.0.smithi093.stdout:346: seq_num 345 ranges {598415=796790,1922170=745784,3141403=415315} 2023-08-27T20:52:09.223 INFO:tasks.rados.rados.0.smithi093.stdout:346: writing smithi093153278-346 from 598415 to 1395205 tid 1 2023-08-27T20:52:09.231 INFO:tasks.rados.rados.0.smithi093.stdout:346: writing smithi093153278-346 from 1922170 to 2667954 tid 2 2023-08-27T20:52:09.237 INFO:tasks.rados.rados.0.smithi093.stdout:346: writing smithi093153278-346 from 3141403 to 3556718 tid 3 2023-08-27T20:52:09.237 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.237 INFO:tasks.rados.rados.0.smithi093.stdout:331: finishing write tid 2 to smithi093153278-331 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:331: finishing write tid 3 to smithi093153278-331 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:331: finishing write tid 4 to smithi093153278-331 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:331: finishing write tid 5 to smithi093153278-331 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 331 v 86 (ObjNum 330 snap 0 seq_num 330) dirty exists 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:331: left oid 331 (ObjNum 330 snap 0 seq_num 330) 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:332: finishing write tid 1 to smithi093153278-332 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:332: finishing write tid 2 to smithi093153278-332 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:331: done (15 left) 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:347: write initial oid 347 2023-08-27T20:52:09.238 INFO:tasks.rados.rados.0.smithi093.stdout:347: seq_num 346 ranges {615222=472039,1580652=489442,2768826=626587} 2023-08-27T20:52:09.243 INFO:tasks.rados.rados.0.smithi093.stdout:347: writing smithi093153278-347 from 615222 to 1087261 tid 1 2023-08-27T20:52:09.249 INFO:tasks.rados.rados.0.smithi093.stdout:347: writing smithi093153278-347 from 1580652 to 2070094 tid 2 2023-08-27T20:52:09.257 INFO:tasks.rados.rados.0.smithi093.stdout:347: writing smithi093153278-347 from 2768826 to 3395413 tid 3 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:332: finishing write tid 3 to smithi093153278-332 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:332: finishing write tid 4 to smithi093153278-332 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:332: finishing write tid 5 to smithi093153278-332 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 332 v 70 (ObjNum 331 snap 0 seq_num 331) dirty exists 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:332: left oid 332 (ObjNum 331 snap 0 seq_num 331) 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:333: finishing write tid 1 to smithi093153278-333 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:333: finishing write tid 2 to smithi093153278-333 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:333: finishing write tid 3 to smithi093153278-333 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:333: finishing write tid 4 to smithi093153278-333 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:333: finishing write tid 5 to smithi093153278-333 2023-08-27T20:52:09.258 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 333 v 50 (ObjNum 332 snap 0 seq_num 332) dirty exists 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:333: left oid 333 (ObjNum 332 snap 0 seq_num 332) 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:334: finishing write tid 1 to smithi093153278-334 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:334: finishing write tid 2 to smithi093153278-334 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:334: finishing write tid 3 to smithi093153278-334 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:334: finishing write tid 4 to smithi093153278-334 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:332: done (15 left) 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:333: done (14 left) 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:348: write initial oid 348 2023-08-27T20:52:09.259 INFO:tasks.rados.rados.0.smithi093.stdout:348: seq_num 347 ranges {632029=547288,1639134=749453,2796249=437859,3653524=178505} 2023-08-27T20:52:09.264 INFO:tasks.rados.rados.0.smithi093.stdout:348: writing smithi093153278-348 from 632029 to 1179317 tid 1 2023-08-27T20:52:09.270 INFO:tasks.rados.rados.0.smithi093.stdout:348: writing smithi093153278-348 from 1639134 to 2388587 tid 2 2023-08-27T20:52:09.275 INFO:tasks.rados.rados.0.smithi093.stdout:348: writing smithi093153278-348 from 2796249 to 3234108 tid 3 2023-08-27T20:52:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:348: writing smithi093153278-348 from 3653524 to 3832029 tid 4 2023-08-27T20:52:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:349: write initial oid 349 2023-08-27T20:52:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:349: seq_num 348 ranges {648836=622537,1981263=493111,3107319=741517} 2023-08-27T20:52:09.286 INFO:tasks.rados.rados.0.smithi093.stdout:349: writing smithi093153278-349 from 648836 to 1271373 tid 1 2023-08-27T20:52:09.293 INFO:tasks.rados.rados.0.smithi093.stdout:349: writing smithi093153278-349 from 1981263 to 2474374 tid 2 2023-08-27T20:52:09.303 INFO:tasks.rados.rados.0.smithi093.stdout:349: writing smithi093153278-349 from 3107319 to 3848836 tid 3 2023-08-27T20:52:09.303 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.303 INFO:tasks.rados.rados.0.smithi093.stdout:334: finishing write tid 5 to smithi093153278-334 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 334 v 74 (ObjNum 333 snap 0 seq_num 333) dirty exists 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:334: left oid 334 (ObjNum 333 snap 0 seq_num 333) 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:335: finishing write tid 1 to smithi093153278-335 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:335: finishing write tid 2 to smithi093153278-335 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:335: finishing write tid 3 to smithi093153278-335 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:335: finishing write tid 4 to smithi093153278-335 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:335: finishing write tid 5 to smithi093153278-335 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 335 v 73 (ObjNum 334 snap 0 seq_num 334) dirty exists 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:335: left oid 335 (ObjNum 334 snap 0 seq_num 334) 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:336: finishing write tid 1 to smithi093153278-336 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:336: finishing write tid 2 to smithi093153278-336 2023-08-27T20:52:09.304 INFO:tasks.rados.rados.0.smithi093.stdout:336: finishing write tid 3 to smithi093153278-336 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:336: finishing write tid 4 to smithi093153278-336 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:336: finishing write tid 5 to smithi093153278-336 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 336 v 54 (ObjNum 335 snap 0 seq_num 335) dirty exists 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:336: left oid 336 (ObjNum 335 snap 0 seq_num 335) 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:337: finishing write tid 1 to smithi093153278-337 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:337: finishing write tid 2 to smithi093153278-337 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:337: finishing write tid 3 to smithi093153278-337 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:337: finishing write tid 4 to smithi093153278-337 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:337: finishing write tid 5 to smithi093153278-337 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 337 v 83 (ObjNum 336 snap 0 seq_num 336) dirty exists 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:337: left oid 337 (ObjNum 336 snap 0 seq_num 336) 2023-08-27T20:52:09.305 INFO:tasks.rados.rados.0.smithi093.stdout:338: finishing write tid 1 to smithi093153278-338 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:338: finishing write tid 2 to smithi093153278-338 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:338: finishing write tid 3 to smithi093153278-338 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:338: finishing write tid 4 to smithi093153278-338 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:338: finishing write tid 5 to smithi093153278-338 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 338 v 62 (ObjNum 337 snap 0 seq_num 337) dirty exists 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:338: left oid 338 (ObjNum 337 snap 0 seq_num 337) 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:339: finishing write tid 1 to smithi093153278-339 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:339: finishing write tid 2 to smithi093153278-339 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:339: finishing write tid 3 to smithi093153278-339 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:339: finishing write tid 4 to smithi093153278-339 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:339: finishing write tid 5 to smithi093153278-339 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 339 v 58 (ObjNum 338 snap 0 seq_num 338) dirty exists 2023-08-27T20:52:09.306 INFO:tasks.rados.rados.0.smithi093.stdout:339: left oid 339 (ObjNum 338 snap 0 seq_num 338) 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:340: finishing write tid 1 to smithi093153278-340 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:334: done (15 left) 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:335: done (14 left) 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:336: done (13 left) 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:337: done (12 left) 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:338: done (11 left) 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:339: done (10 left) 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:350: write initial oid 350 2023-08-27T20:52:09.307 INFO:tasks.rados.rados.0.smithi093.stdout:350: seq_num 349 ranges {665643=697786,2039745=753122,3534742=330901} 2023-08-27T20:52:09.311 INFO:tasks.rados.rados.0.smithi093.stdout:350: writing smithi093153278-350 from 665643 to 1363429 tid 1 2023-08-27T20:52:09.319 INFO:tasks.rados.rados.0.smithi093.stdout:350: writing smithi093153278-350 from 2039745 to 2792867 tid 2 2023-08-27T20:52:09.325 INFO:tasks.rados.rados.0.smithi093.stdout:350: writing smithi093153278-350 from 3534742 to 3865643 tid 3 2023-08-27T20:52:09.325 INFO:tasks.rados.rados.0.smithi093.stdout:351: write initial oid 351 2023-08-27T20:52:09.325 INFO:tasks.rados.rados.0.smithi093.stdout:351: seq_num 350 ranges {682450=489388,1814580=496780,2878518=788028} 2023-08-27T20:52:09.331 INFO:tasks.rados.rados.0.smithi093.stdout:351: writing smithi093153278-351 from 682450 to 1171838 tid 1 2023-08-27T20:52:09.337 INFO:tasks.rados.rados.0.smithi093.stdout:351: writing smithi093153278-351 from 1814580 to 2311360 tid 2 2023-08-27T20:52:09.346 INFO:tasks.rados.rados.0.smithi093.stdout:351: writing smithi093153278-351 from 2878518 to 3666546 tid 3 2023-08-27T20:52:09.346 INFO:tasks.rados.rados.0.smithi093.stdout:352: write initial oid 352 2023-08-27T20:52:09.346 INFO:tasks.rados.rados.0.smithi093.stdout:352: seq_num 351 ranges {699257=564637,1873062=640438,3305941=593316} 2023-08-27T20:52:09.354 INFO:tasks.rados.rados.0.smithi093.stdout:352: writing smithi093153278-352 from 699257 to 1263894 tid 1 2023-08-27T20:52:09.362 INFO:tasks.rados.rados.0.smithi093.stdout:352: writing smithi093153278-352 from 1873062 to 2513500 tid 2 2023-08-27T20:52:09.370 INFO:tasks.rados.rados.0.smithi093.stdout:352: writing smithi093153278-352 from 3305941 to 3899257 tid 3 2023-08-27T20:52:09.370 INFO:tasks.rados.rados.0.smithi093.stdout:353: write initial oid 353 2023-08-27T20:52:09.370 INFO:tasks.rados.rados.0.smithi093.stdout:353: seq_num 352 ranges {716064=639886,1815191=500449,2817011=526925} 2023-08-27T20:52:09.378 INFO:tasks.rados.rados.0.smithi093.stdout:353: writing smithi093153278-353 from 716064 to 1355950 tid 1 2023-08-27T20:52:09.384 INFO:tasks.rados.rados.0.smithi093.stdout:353: writing smithi093153278-353 from 1815191 to 2315640 tid 2 2023-08-27T20:52:09.391 INFO:tasks.rados.rados.0.smithi093.stdout:353: writing smithi093153278-353 from 2817011 to 3343936 tid 3 2023-08-27T20:52:09.391 INFO:tasks.rados.rados.0.smithi093.stdout:354: write initial oid 354 2023-08-27T20:52:09.391 INFO:tasks.rados.rados.0.smithi093.stdout:354: seq_num 353 ranges {732871=715135,1873673=644107,3244434=688437} 2023-08-27T20:52:09.400 INFO:tasks.rados.rados.0.smithi093.stdout:354: writing smithi093153278-354 from 732871 to 1448006 tid 1 2023-08-27T20:52:09.406 INFO:tasks.rados.rados.0.smithi093.stdout:354: writing smithi093153278-354 from 1873673 to 2517780 tid 2 2023-08-27T20:52:09.416 INFO:tasks.rados.rados.0.smithi093.stdout:354: writing smithi093153278-354 from 3244434 to 3932871 tid 3 2023-08-27T20:52:09.416 INFO:tasks.rados.rados.0.smithi093.stdout:355: write initial oid 355 2023-08-27T20:52:09.416 INFO:tasks.rados.rados.0.smithi093.stdout:340: finishing write tid 2 to smithi093153278-340 2023-08-27T20:52:09.416 INFO:tasks.rados.rados.0.smithi093.stdout:340: finishing write tid 3 to smithi093153278-340 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:340: finishing write tid 4 to smithi093153278-340 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:340: finishing write tid 5 to smithi093153278-340 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 340 v 80 (ObjNum 339 snap 0 seq_num 339) dirty exists 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:340: left oid 340 (ObjNum 339 snap 0 seq_num 339) 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:341: finishing write tid 1 to smithi093153278-341 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:341: finishing write tid 2 to smithi093153278-341 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:341: finishing write tid 3 to smithi093153278-341 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:341: finishing write tid 4 to smithi093153278-341 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:341: finishing write tid 5 to smithi093153278-341 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 341 v 66 (ObjNum 340 snap 0 seq_num 340) dirty exists 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:341: left oid 341 (ObjNum 340 snap 0 seq_num 340) 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:342: finishing write tid 1 to smithi093153278-342 2023-08-27T20:52:09.417 INFO:tasks.rados.rados.0.smithi093.stdout:342: finishing write tid 2 to smithi093153278-342 2023-08-27T20:52:09.418 INFO:tasks.rados.rados.0.smithi093.stdout:342: finishing write tid 3 to smithi093153278-342 2023-08-27T20:52:09.418 INFO:tasks.rados.rados.0.smithi093.stdout:342: finishing write tid 4 to smithi093153278-342 2023-08-27T20:52:09.418 INFO:tasks.rados.rados.0.smithi093.stdout:342: finishing write tid 5 to smithi093153278-342 2023-08-27T20:52:09.418 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 342 v 98 (ObjNum 341 snap 0 seq_num 341) dirty exists 2023-08-27T20:52:09.418 INFO:tasks.rados.rados.0.smithi093.stdout:342: left oid 342 (ObjNum 341 snap 0 seq_num 341) 2023-08-27T20:52:09.418 INFO:tasks.rados.rados.0.smithi093.stdout:343: finishing write tid 1 to smithi093153278-343 2023-08-27T20:52:09.418 INFO:tasks.rados.rados.0.smithi093.stdout:355: seq_num 354 ranges {749678=790384,2332155=504118,3271857=549469} 2023-08-27T20:52:09.425 INFO:tasks.rados.rados.0.smithi093.stdout:355: writing smithi093153278-355 from 749678 to 1540062 tid 1 2023-08-27T20:52:09.433 INFO:tasks.rados.rados.0.smithi093.stdout:355: writing smithi093153278-355 from 2332155 to 2836273 tid 2 2023-08-27T20:52:09.439 INFO:tasks.rados.rados.0.smithi093.stdout:355: writing smithi093153278-355 from 3271857 to 3821326 tid 3 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:340: done (15 left) 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:341: done (14 left) 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:342: done (13 left) 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:356: write initial oid 356 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:343: finishing write tid 2 to smithi093153278-343 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:343: finishing write tid 3 to smithi093153278-343 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:343: finishing write tid 4 to smithi093153278-343 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:343: finishing write tid 5 to smithi093153278-343 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:343: finishing write tid 6 to smithi093153278-343 2023-08-27T20:52:09.440 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 343 v 85 (ObjNum 342 snap 0 seq_num 342) dirty exists 2023-08-27T20:52:09.441 INFO:tasks.rados.rados.0.smithi093.stdout:343: left oid 343 (ObjNum 342 snap 0 seq_num 342) 2023-08-27T20:52:09.441 INFO:tasks.rados.rados.0.smithi093.stdout:344: finishing write tid 1 to smithi093153278-344 2023-08-27T20:52:09.441 INFO:tasks.rados.rados.0.smithi093.stdout:356: seq_num 355 ranges {766485=465633,1990637=647776,3299280=667205} 2023-08-27T20:52:09.446 INFO:tasks.rados.rados.0.smithi093.stdout:356: writing smithi093153278-356 from 766485 to 1232118 tid 1 2023-08-27T20:52:09.455 INFO:tasks.rados.rados.0.smithi093.stdout:356: writing smithi093153278-356 from 1990637 to 2638413 tid 2 2023-08-27T20:52:09.463 INFO:tasks.rados.rados.0.smithi093.stdout:356: writing smithi093153278-356 from 3299280 to 3966485 tid 3 2023-08-27T20:52:09.463 INFO:tasks.rados.rados.0.smithi093.stdout:343: done (13 left) 2023-08-27T20:52:09.464 INFO:tasks.rados.rados.0.smithi093.stdout:357: write initial oid 357 2023-08-27T20:52:09.464 INFO:tasks.rados.rados.0.smithi093.stdout:344: finishing write tid 2 to smithi093153278-344 2023-08-27T20:52:09.464 INFO:tasks.rados.rados.0.smithi093.stdout:344: finishing write tid 3 to smithi093153278-344 2023-08-27T20:52:09.464 INFO:tasks.rados.rados.0.smithi093.stdout:357: seq_num 356 ranges {783292=540882,1932766=507787,3210350=572013} 2023-08-27T20:52:09.471 INFO:tasks.rados.rados.0.smithi093.stdout:357: writing smithi093153278-357 from 783292 to 1324174 tid 1 2023-08-27T20:52:09.477 INFO:tasks.rados.rados.0.smithi093.stdout:357: writing smithi093153278-357 from 1932766 to 2440553 tid 2 2023-08-27T20:52:09.485 INFO:tasks.rados.rados.0.smithi093.stdout:357: writing smithi093153278-357 from 3210350 to 3782363 tid 3 2023-08-27T20:52:09.485 INFO:tasks.rados.rados.0.smithi093.stdout:358: write initial oid 358 2023-08-27T20:52:09.485 INFO:tasks.rados.rados.0.smithi093.stdout:358: seq_num 357 ranges {400099=616131,1591248=408851} 2023-08-27T20:52:09.492 INFO:tasks.rados.rados.0.smithi093.stdout:358: writing smithi093153278-358 from 400099 to 1016230 tid 1 2023-08-27T20:52:09.498 INFO:tasks.rados.rados.0.smithi093.stdout:358: writing smithi093153278-358 from 1591248 to 2000099 tid 2 2023-08-27T20:52:09.498 INFO:tasks.rados.rados.0.smithi093.stdout:359: write initial oid 359 2023-08-27T20:52:09.498 INFO:tasks.rados.rados.0.smithi093.stdout:359: seq_num 358 ranges {416906=407733,1366083=511456} 2023-08-27T20:52:09.503 INFO:tasks.rados.rados.0.smithi093.stdout:359: writing smithi093153278-359 from 416906 to 824639 tid 1 2023-08-27T20:52:09.510 INFO:tasks.rados.rados.0.smithi093.stdout:359: writing smithi093153278-359 from 1366083 to 1877539 tid 2 2023-08-27T20:52:09.510 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:344: finishing write tid 4 to smithi093153278-344 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:344: finishing write tid 5 to smithi093153278-344 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 344 v 79 (ObjNum 343 snap 0 seq_num 343) dirty exists 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:344: left oid 344 (ObjNum 343 snap 0 seq_num 343) 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:345: finishing write tid 1 to smithi093153278-345 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:345: finishing write tid 2 to smithi093153278-345 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:345: finishing write tid 3 to smithi093153278-345 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:345: finishing write tid 4 to smithi093153278-345 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:344: done (15 left) 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:360: write initial oid 360 2023-08-27T20:52:09.511 INFO:tasks.rados.rados.0.smithi093.stdout:360: seq_num 359 ranges {433713=482982,1424565=609148} 2023-08-27T20:52:09.515 INFO:tasks.rados.rados.0.smithi093.stdout:360: writing smithi093153278-360 from 433713 to 916695 tid 1 2023-08-27T20:52:09.523 INFO:tasks.rados.rados.0.smithi093.stdout:360: writing smithi093153278-360 from 1424565 to 2033713 tid 2 2023-08-27T20:52:09.523 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.523 INFO:tasks.rados.rados.0.smithi093.stdout:345: finishing write tid 5 to smithi093153278-345 2023-08-27T20:52:09.523 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 345 v 62 (ObjNum 344 snap 0 seq_num 344) dirty exists 2023-08-27T20:52:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:345: left oid 345 (ObjNum 344 snap 0 seq_num 344) 2023-08-27T20:52:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:346: finishing write tid 1 to smithi093153278-346 2023-08-27T20:52:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:346: finishing write tid 2 to smithi093153278-346 2023-08-27T20:52:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:345: done (15 left) 2023-08-27T20:52:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:361: write initial oid 361 2023-08-27T20:52:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:361: seq_num 360 ranges {450520=558231,1483047=515125} 2023-08-27T20:52:09.529 INFO:tasks.rados.rados.0.smithi093.stdout:361: writing smithi093153278-361 from 450520 to 1008751 tid 1 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:361: writing smithi093153278-361 from 1483047 to 1998172 tid 2 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:346: finishing write tid 3 to smithi093153278-346 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:346: finishing write tid 4 to smithi093153278-346 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:346: finishing write tid 5 to smithi093153278-346 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 346 v 69 (ObjNum 345 snap 0 seq_num 345) dirty exists 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:346: left oid 346 (ObjNum 345 snap 0 seq_num 345) 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:347: finishing write tid 1 to smithi093153278-347 2023-08-27T20:52:09.536 INFO:tasks.rados.rados.0.smithi093.stdout:347: finishing write tid 2 to smithi093153278-347 2023-08-27T20:52:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:347: finishing write tid 3 to smithi093153278-347 2023-08-27T20:52:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:346: done (15 left) 2023-08-27T20:52:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:362: write initial oid 362 2023-08-27T20:52:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:362: seq_num 361 ranges {467327=633480,1825176=242151} 2023-08-27T20:52:09.542 INFO:tasks.rados.rados.0.smithi093.stdout:362: writing smithi093153278-362 from 467327 to 1100807 tid 1 2023-08-27T20:52:09.548 INFO:tasks.rados.rados.0.smithi093.stdout:362: writing smithi093153278-362 from 1825176 to 2067327 tid 2 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:347: finishing write tid 4 to smithi093153278-347 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:347: finishing write tid 5 to smithi093153278-347 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 347 v 77 (ObjNum 346 snap 0 seq_num 346) dirty exists 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:347: left oid 347 (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:348: finishing write tid 1 to smithi093153278-348 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:348: finishing write tid 2 to smithi093153278-348 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:348: finishing write tid 3 to smithi093153278-348 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:348: finishing write tid 4 to smithi093153278-348 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:348: finishing write tid 5 to smithi093153278-348 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:348: finishing write tid 6 to smithi093153278-348 2023-08-27T20:52:09.549 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 348 v 91 (ObjNum 347 snap 0 seq_num 347) dirty exists 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:348: left oid 348 (ObjNum 347 snap 0 seq_num 347) 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:349: finishing write tid 1 to smithi093153278-349 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:349: finishing write tid 2 to smithi093153278-349 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:349: finishing write tid 3 to smithi093153278-349 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:349: finishing write tid 4 to smithi093153278-349 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:349: finishing write tid 5 to smithi093153278-349 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 349 v 73 (ObjNum 348 snap 0 seq_num 348) dirty exists 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:349: left oid 349 (ObjNum 348 snap 0 seq_num 348) 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:350: finishing write tid 1 to smithi093153278-350 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:350: finishing write tid 2 to smithi093153278-350 2023-08-27T20:52:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:350: finishing write tid 3 to smithi093153278-350 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:350: finishing write tid 4 to smithi093153278-350 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:350: finishing write tid 5 to smithi093153278-350 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 350 v 79 (ObjNum 349 snap 0 seq_num 349) dirty exists 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:350: left oid 350 (ObjNum 349 snap 0 seq_num 349) 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:351: finishing write tid 1 to smithi093153278-351 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:351: finishing write tid 2 to smithi093153278-351 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:351: finishing write tid 3 to smithi093153278-351 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:351: finishing write tid 4 to smithi093153278-351 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:351: finishing write tid 5 to smithi093153278-351 2023-08-27T20:52:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 351 v 78 (ObjNum 350 snap 0 seq_num 350) dirty exists 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:351: left oid 351 (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:352: finishing write tid 1 to smithi093153278-352 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:352: finishing write tid 2 to smithi093153278-352 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:352: finishing write tid 3 to smithi093153278-352 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:352: finishing write tid 4 to smithi093153278-352 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:347: done (15 left) 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:348: done (14 left) 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:349: done (13 left) 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:350: done (12 left) 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:351: done (11 left) 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:363: write initial oid 363 2023-08-27T20:52:09.552 INFO:tasks.rados.rados.0.smithi093.stdout:363: seq_num 362 ranges {484134=708729,1883658=200476} 2023-08-27T20:52:09.554 INFO:tasks.rados.rados.0.smithi093.stdout:363: writing smithi093153278-363 from 484134 to 1192863 tid 1 2023-08-27T20:52:09.559 INFO:tasks.rados.rados.0.smithi093.stdout:363: writing smithi093153278-363 from 1883658 to 2084134 tid 2 2023-08-27T20:52:09.559 INFO:tasks.rados.rados.0.smithi093.stdout:364: write initial oid 364 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:352: finishing write tid 5 to smithi093153278-352 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 352 v 102 (ObjNum 351 snap 0 seq_num 351) dirty exists 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:352: left oid 352 (ObjNum 351 snap 0 seq_num 351) 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:353: finishing write tid 1 to smithi093153278-353 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:353: finishing write tid 2 to smithi093153278-353 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:353: finishing write tid 3 to smithi093153278-353 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:353: finishing write tid 4 to smithi093153278-353 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:353: finishing write tid 5 to smithi093153278-353 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 353 v 82 (ObjNum 352 snap 0 seq_num 352) dirty exists 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:353: left oid 353 (ObjNum 352 snap 0 seq_num 352) 2023-08-27T20:52:09.560 INFO:tasks.rados.rados.0.smithi093.stdout:364: seq_num 363 ranges {500941=783978,1942140=158801} 2023-08-27T20:52:09.566 INFO:tasks.rados.rados.0.smithi093.stdout:364: writing smithi093153278-364 from 500941 to 1284919 tid 1 2023-08-27T20:52:09.574 INFO:tasks.rados.rados.0.smithi093.stdout:364: writing smithi093153278-364 from 1942140 to 2100941 tid 2 2023-08-27T20:52:09.574 INFO:tasks.rados.rados.0.smithi093.stdout:352: done (12 left) 2023-08-27T20:52:09.574 INFO:tasks.rados.rados.0.smithi093.stdout:353: done (11 left) 2023-08-27T20:52:09.574 INFO:tasks.rados.rados.0.smithi093.stdout:365: write initial oid 365 2023-08-27T20:52:09.574 INFO:tasks.rados.rados.0.smithi093.stdout:354: finishing write tid 1 to smithi093153278-354 2023-08-27T20:52:09.574 INFO:tasks.rados.rados.0.smithi093.stdout:354: finishing write tid 2 to smithi093153278-354 2023-08-27T20:52:09.575 INFO:tasks.rados.rados.0.smithi093.stdout:354: finishing write tid 3 to smithi093153278-354 2023-08-27T20:52:09.575 INFO:tasks.rados.rados.0.smithi093.stdout:354: finishing write tid 4 to smithi093153278-354 2023-08-27T20:52:09.575 INFO:tasks.rados.rados.0.smithi093.stdout:365: seq_num 364 ranges {517748=459227,1600622=406110} 2023-08-27T20:52:09.577 INFO:tasks.rados.rados.0.smithi093.stdout:365: writing smithi093153278-365 from 517748 to 976975 tid 1 2023-08-27T20:52:09.584 INFO:tasks.rados.rados.0.smithi093.stdout:365: writing smithi093153278-365 from 1600622 to 2006732 tid 2 2023-08-27T20:52:09.584 INFO:tasks.rados.rados.0.smithi093.stdout:366: write initial oid 366 2023-08-27T20:52:09.584 INFO:tasks.rados.rados.0.smithi093.stdout:366: seq_num 365 ranges {534555=650829,1659104=475451} 2023-08-27T20:52:09.591 INFO:tasks.rados.rados.0.smithi093.stdout:366: writing smithi093153278-366 from 534555 to 1185384 tid 1 2023-08-27T20:52:09.597 INFO:tasks.rados.rados.0.smithi093.stdout:366: writing smithi093153278-366 from 1659104 to 2134555 tid 2 2023-08-27T20:52:09.597 INFO:tasks.rados.rados.0.smithi093.stdout:367: write initial oid 367 2023-08-27T20:52:09.597 INFO:tasks.rados.rados.0.smithi093.stdout:367: seq_num 366 ranges {551362=726078,1717586=409779} 2023-08-27T20:52:09.604 INFO:tasks.rados.rados.0.smithi093.stdout:367: writing smithi093153278-367 from 551362 to 1277440 tid 1 2023-08-27T20:52:09.609 INFO:tasks.rados.rados.0.smithi093.stdout:367: writing smithi093153278-367 from 1717586 to 2127365 tid 2 2023-08-27T20:52:09.609 INFO:tasks.rados.rados.0.smithi093.stdout:368: write initial oid 368 2023-08-27T20:52:09.609 INFO:tasks.rados.rados.0.smithi093.stdout:368: seq_num 367 ranges {568169=401327,1376068=669790} 2023-08-27T20:52:09.614 INFO:tasks.rados.rados.0.smithi093.stdout:368: writing smithi093153278-368 from 568169 to 969496 tid 1 2023-08-27T20:52:09.622 INFO:tasks.rados.rados.0.smithi093.stdout:368: writing smithi093153278-368 from 1376068 to 2045858 tid 2 2023-08-27T20:52:09.622 INFO:tasks.rados.rados.0.smithi093.stdout:369: write initial oid 369 2023-08-27T20:52:09.622 INFO:tasks.rados.rados.0.smithi093.stdout:369: seq_num 368 ranges {584976=476576,1834550=350426} 2023-08-27T20:52:09.628 INFO:tasks.rados.rados.0.smithi093.stdout:369: writing smithi093153278-369 from 584976 to 1061552 tid 1 2023-08-27T20:52:09.636 INFO:tasks.rados.rados.0.smithi093.stdout:369: writing smithi093153278-369 from 1834550 to 2184976 tid 2 2023-08-27T20:52:09.636 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:354: finishing write tid 5 to smithi093153278-354 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 354 v 87 (ObjNum 353 snap 0 seq_num 353) dirty exists 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:354: left oid 354 (ObjNum 353 snap 0 seq_num 353) 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:355: finishing write tid 1 to smithi093153278-355 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:355: finishing write tid 2 to smithi093153278-355 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:355: finishing write tid 3 to smithi093153278-355 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:355: finishing write tid 4 to smithi093153278-355 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:355: finishing write tid 5 to smithi093153278-355 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 355 v 83 (ObjNum 354 snap 0 seq_num 354) dirty exists 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:355: left oid 355 (ObjNum 354 snap 0 seq_num 354) 2023-08-27T20:52:09.637 INFO:tasks.rados.rados.0.smithi093.stdout:356: finishing write tid 1 to smithi093153278-356 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:356: finishing write tid 2 to smithi093153278-356 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:356: finishing write tid 3 to smithi093153278-356 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:356: finishing write tid 4 to smithi093153278-356 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:356: finishing write tid 5 to smithi093153278-356 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 356 v 87 (ObjNum 355 snap 0 seq_num 355) dirty exists 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:356: left oid 356 (ObjNum 355 snap 0 seq_num 355) 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:357: finishing write tid 1 to smithi093153278-357 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:357: finishing write tid 2 to smithi093153278-357 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:357: finishing write tid 3 to smithi093153278-357 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:357: finishing write tid 4 to smithi093153278-357 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:354: done (15 left) 2023-08-27T20:52:09.638 INFO:tasks.rados.rados.0.smithi093.stdout:355: done (14 left) 2023-08-27T20:52:09.639 INFO:tasks.rados.rados.0.smithi093.stdout:356: done (13 left) 2023-08-27T20:52:09.639 INFO:tasks.rados.rados.0.smithi093.stdout:370: write initial oid 370 2023-08-27T20:52:09.639 INFO:tasks.rados.rados.0.smithi093.stdout:370: seq_num 369 ranges {601783=551825,1776679=425104} 2023-08-27T20:52:09.643 INFO:tasks.rados.rados.0.smithi093.stdout:370: writing smithi093153278-370 from 601783 to 1153608 tid 1 2023-08-27T20:52:09.649 INFO:tasks.rados.rados.0.smithi093.stdout:370: writing smithi093153278-370 from 1776679 to 2201783 tid 2 2023-08-27T20:52:09.650 INFO:tasks.rados.rados.0.smithi093.stdout:371: write initial oid 371 2023-08-27T20:52:09.650 INFO:tasks.rados.rados.0.smithi093.stdout:371: seq_num 370 ranges {618590=627074,1835161=383429} 2023-08-27T20:52:09.657 INFO:tasks.rados.rados.0.smithi093.stdout:371: writing smithi093153278-371 from 618590 to 1245664 tid 1 2023-08-27T20:52:09.663 INFO:tasks.rados.rados.0.smithi093.stdout:371: writing smithi093153278-371 from 1835161 to 2218590 tid 2 2023-08-27T20:52:09.663 INFO:tasks.rados.rados.0.smithi093.stdout:372: write initial oid 372 2023-08-27T20:52:09.663 INFO:tasks.rados.rados.0.smithi093.stdout:372: seq_num 371 ranges {635397=702323,1893643=341754} 2023-08-27T20:52:09.671 INFO:tasks.rados.rados.0.smithi093.stdout:372: writing smithi093153278-372 from 635397 to 1337720 tid 1 2023-08-27T20:52:09.677 INFO:tasks.rados.rados.0.smithi093.stdout:372: writing smithi093153278-372 from 1893643 to 2235397 tid 2 2023-08-27T20:52:09.677 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.678 INFO:tasks.rados.rados.0.smithi093.stdout:357: finishing write tid 5 to smithi093153278-357 2023-08-27T20:52:09.678 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 357 v 81 (ObjNum 356 snap 0 seq_num 356) dirty exists 2023-08-27T20:52:09.678 INFO:tasks.rados.rados.0.smithi093.stdout:357: left oid 357 (ObjNum 356 snap 0 seq_num 356) 2023-08-27T20:52:09.678 INFO:tasks.rados.rados.0.smithi093.stdout:358: finishing write tid 1 to smithi093153278-358 2023-08-27T20:52:09.678 INFO:tasks.rados.rados.0.smithi093.stdout:357: done (15 left) 2023-08-27T20:52:09.678 INFO:tasks.rados.rados.0.smithi093.stdout:373: write initial oid 373 2023-08-27T20:52:09.678 INFO:tasks.rados.rados.0.smithi093.stdout:373: seq_num 372 ranges {652204=777572,1952125=300079} 2023-08-27T20:52:09.685 INFO:tasks.rados.rados.0.smithi093.stdout:373: writing smithi093153278-373 from 652204 to 1429776 tid 1 2023-08-27T20:52:09.690 INFO:tasks.rados.rados.0.smithi093.stdout:373: writing smithi093153278-373 from 1952125 to 2252204 tid 2 2023-08-27T20:52:09.690 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:358: finishing write tid 2 to smithi093153278-358 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:358: finishing write tid 3 to smithi093153278-358 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:358: finishing write tid 4 to smithi093153278-358 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 358 v 85 (ObjNum 357 snap 0 seq_num 357) dirty exists 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:358: left oid 358 (ObjNum 357 snap 0 seq_num 357) 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:359: finishing write tid 1 to smithi093153278-359 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:359: finishing write tid 2 to smithi093153278-359 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:358: done (15 left) 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:374: write initial oid 374 2023-08-27T20:52:09.691 INFO:tasks.rados.rados.0.smithi093.stdout:374: seq_num 373 ranges {669011=569174,2010607=258404} 2023-08-27T20:52:09.697 INFO:tasks.rados.rados.0.smithi093.stdout:374: writing smithi093153278-374 from 669011 to 1238185 tid 1 2023-08-27T20:52:09.703 INFO:tasks.rados.rados.0.smithi093.stdout:374: writing smithi093153278-374 from 2010607 to 2269011 tid 2 2023-08-27T20:52:09.703 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.703 INFO:tasks.rados.rados.0.smithi093.stdout:359: finishing write tid 3 to smithi093153278-359 2023-08-27T20:52:09.703 INFO:tasks.rados.rados.0.smithi093.stdout:359: finishing write tid 4 to smithi093153278-359 2023-08-27T20:52:09.703 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 359 v 94 (ObjNum 358 snap 0 seq_num 358) dirty exists 2023-08-27T20:52:09.703 INFO:tasks.rados.rados.0.smithi093.stdout:359: left oid 359 (ObjNum 358 snap 0 seq_num 358) 2023-08-27T20:52:09.704 INFO:tasks.rados.rados.0.smithi093.stdout:360: finishing write tid 1 to smithi093153278-360 2023-08-27T20:52:09.704 INFO:tasks.rados.rados.0.smithi093.stdout:359: done (15 left) 2023-08-27T20:52:09.704 INFO:tasks.rados.rados.0.smithi093.stdout:375: write initial oid 375 2023-08-27T20:52:09.704 INFO:tasks.rados.rados.0.smithi093.stdout:375: seq_num 374 ranges {685818=644423,2069089=216729} 2023-08-27T20:52:09.710 INFO:tasks.rados.rados.0.smithi093.stdout:375: writing smithi093153278-375 from 685818 to 1330241 tid 1 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:375: writing smithi093153278-375 from 2069089 to 2285818 tid 2 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:360: finishing write tid 2 to smithi093153278-360 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:360: finishing write tid 3 to smithi093153278-360 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:360: finishing write tid 4 to smithi093153278-360 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 360 v 61 (ObjNum 359 snap 0 seq_num 359) dirty exists 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:360: left oid 360 (ObjNum 359 snap 0 seq_num 359) 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:361: finishing write tid 1 to smithi093153278-361 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:360: done (15 left) 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:376: write initial oid 376 2023-08-27T20:52:09.716 INFO:tasks.rados.rados.0.smithi093.stdout:376: seq_num 375 ranges {702625=719672,2127571=175054} 2023-08-27T20:52:09.723 INFO:tasks.rados.rados.0.smithi093.stdout:376: writing smithi093153278-376 from 702625 to 1422297 tid 1 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:376: writing smithi093153278-376 from 2127571 to 2302625 tid 2 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:361: finishing write tid 2 to smithi093153278-361 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:361: finishing write tid 3 to smithi093153278-361 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:361: finishing write tid 4 to smithi093153278-361 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 361 v 88 (ObjNum 360 snap 0 seq_num 360) dirty exists 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:361: left oid 361 (ObjNum 360 snap 0 seq_num 360) 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:362: finishing write tid 1 to smithi093153278-362 2023-08-27T20:52:09.729 INFO:tasks.rados.rados.0.smithi093.stdout:362: finishing write tid 2 to smithi093153278-362 2023-08-27T20:52:09.730 INFO:tasks.rados.rados.0.smithi093.stdout:362: finishing write tid 3 to smithi093153278-362 2023-08-27T20:52:09.730 INFO:tasks.rados.rados.0.smithi093.stdout:361: done (15 left) 2023-08-27T20:52:09.730 INFO:tasks.rados.rados.0.smithi093.stdout:377: write initial oid 377 2023-08-27T20:52:09.730 INFO:tasks.rados.rados.0.smithi093.stdout:377: seq_num 376 ranges {719432=794921,2186053=133379} 2023-08-27T20:52:09.738 INFO:tasks.rados.rados.0.smithi093.stdout:377: writing smithi093153278-377 from 719432 to 1514353 tid 1 2023-08-27T20:52:09.743 INFO:tasks.rados.rados.0.smithi093.stdout:377: writing smithi093153278-377 from 2186053 to 2319432 tid 2 2023-08-27T20:52:09.743 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.743 INFO:tasks.rados.rados.0.smithi093.stdout:362: finishing write tid 4 to smithi093153278-362 2023-08-27T20:52:09.743 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 362 v 81 (ObjNum 361 snap 0 seq_num 361) dirty exists 2023-08-27T20:52:09.744 INFO:tasks.rados.rados.0.smithi093.stdout:362: left oid 362 (ObjNum 361 snap 0 seq_num 361) 2023-08-27T20:52:09.744 INFO:tasks.rados.rados.0.smithi093.stdout:363: finishing write tid 1 to smithi093153278-363 2023-08-27T20:52:09.744 INFO:tasks.rados.rados.0.smithi093.stdout:362: done (15 left) 2023-08-27T20:52:09.744 INFO:tasks.rados.rados.0.smithi093.stdout:378: write initial oid 378 2023-08-27T20:52:09.744 INFO:tasks.rados.rados.0.smithi093.stdout:378: seq_num 377 ranges {736239=470170,1728182=571782} 2023-08-27T20:52:09.749 INFO:tasks.rados.rados.0.smithi093.stdout:378: writing smithi093153278-378 from 736239 to 1206409 tid 1 2023-08-27T20:52:09.756 INFO:tasks.rados.rados.0.smithi093.stdout:378: writing smithi093153278-378 from 1728182 to 2299964 tid 2 2023-08-27T20:52:09.756 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.756 INFO:tasks.rados.rados.0.smithi093.stdout:363: finishing write tid 2 to smithi093153278-363 2023-08-27T20:52:09.756 INFO:tasks.rados.rados.0.smithi093.stdout:363: finishing write tid 3 to smithi093153278-363 2023-08-27T20:52:09.756 INFO:tasks.rados.rados.0.smithi093.stdout:363: finishing write tid 4 to smithi093153278-363 2023-08-27T20:52:09.756 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 97 (ObjNum 362 snap 0 seq_num 362) dirty exists 2023-08-27T20:52:09.756 INFO:tasks.rados.rados.0.smithi093.stdout:363: left oid 363 (ObjNum 362 snap 0 seq_num 362) 2023-08-27T20:52:09.757 INFO:tasks.rados.rados.0.smithi093.stdout:364: finishing write tid 1 to smithi093153278-364 2023-08-27T20:52:09.757 INFO:tasks.rados.rados.0.smithi093.stdout:364: finishing write tid 2 to smithi093153278-364 2023-08-27T20:52:09.757 INFO:tasks.rados.rados.0.smithi093.stdout:363: done (15 left) 2023-08-27T20:52:09.757 INFO:tasks.rados.rados.0.smithi093.stdout:379: write initial oid 379 2023-08-27T20:52:09.757 INFO:tasks.rados.rados.0.smithi093.stdout:379: seq_num 378 ranges {753046=545419,1786664=431793} 2023-08-27T20:52:09.763 INFO:tasks.rados.rados.0.smithi093.stdout:379: writing smithi093153278-379 from 753046 to 1298465 tid 1 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout:379: writing smithi093153278-379 from 1786664 to 2218457 tid 2 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout:364: finishing write tid 3 to smithi093153278-364 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout:364: finishing write tid 4 to smithi093153278-364 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 364 v 84 (ObjNum 363 snap 0 seq_num 363) dirty exists 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout:364: left oid 364 (ObjNum 363 snap 0 seq_num 363) 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout:365: finishing write tid 1 to smithi093153278-365 2023-08-27T20:52:09.770 INFO:tasks.rados.rados.0.smithi093.stdout:365: finishing write tid 2 to smithi093153278-365 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:365: finishing write tid 3 to smithi093153278-365 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:365: finishing write tid 4 to smithi093153278-365 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 365 v 90 (ObjNum 364 snap 0 seq_num 364) dirty exists 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:365: left oid 365 (ObjNum 364 snap 0 seq_num 364) 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:366: finishing write tid 1 to smithi093153278-366 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:366: finishing write tid 2 to smithi093153278-366 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:364: done (15 left) 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:365: done (14 left) 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:380: write initial oid 380 2023-08-27T20:52:09.771 INFO:tasks.rados.rados.0.smithi093.stdout:380: seq_num 379 ranges {769853=620668,1845146=524707} 2023-08-27T20:52:09.778 INFO:tasks.rados.rados.0.smithi093.stdout:380: writing smithi093153278-380 from 769853 to 1390521 tid 1 2023-08-27T20:52:09.784 INFO:tasks.rados.rados.0.smithi093.stdout:380: writing smithi093153278-380 from 1845146 to 2369853 tid 2 2023-08-27T20:52:09.784 INFO:tasks.rados.rados.0.smithi093.stdout:381: write initial oid 381 2023-08-27T20:52:09.784 INFO:tasks.rados.rados.0.smithi093.stdout:381: seq_num 380 ranges {786660=695917,1903628=435462} 2023-08-27T20:52:09.792 INFO:tasks.rados.rados.0.smithi093.stdout:381: writing smithi093153278-381 from 786660 to 1482577 tid 1 2023-08-27T20:52:09.798 INFO:tasks.rados.rados.0.smithi093.stdout:381: writing smithi093153278-381 from 1903628 to 2339090 tid 2 2023-08-27T20:52:09.798 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.798 INFO:tasks.rados.rados.0.smithi093.stdout:366: finishing write tid 3 to smithi093153278-366 2023-08-27T20:52:09.798 INFO:tasks.rados.rados.0.smithi093.stdout:366: finishing write tid 4 to smithi093153278-366 2023-08-27T20:52:09.798 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 366 v 87 (ObjNum 365 snap 0 seq_num 365) dirty exists 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:366: left oid 366 (ObjNum 365 snap 0 seq_num 365) 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:367: finishing write tid 1 to smithi093153278-367 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:367: finishing write tid 2 to smithi093153278-367 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:367: finishing write tid 3 to smithi093153278-367 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:367: finishing write tid 4 to smithi093153278-367 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 367 v 64 (ObjNum 366 snap 0 seq_num 366) dirty exists 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:367: left oid 367 (ObjNum 366 snap 0 seq_num 366) 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:368: finishing write tid 1 to smithi093153278-368 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:368: finishing write tid 2 to smithi093153278-368 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:366: done (15 left) 2023-08-27T20:52:09.799 INFO:tasks.rados.rados.0.smithi093.stdout:367: done (14 left) 2023-08-27T20:52:09.800 INFO:tasks.rados.rados.0.smithi093.stdout:382: write initial oid 382 2023-08-27T20:52:09.800 INFO:tasks.rados.rados.0.smithi093.stdout:382: seq_num 381 ranges {403467=487519,1562110=579120} 2023-08-27T20:52:09.802 INFO:tasks.rados.rados.0.smithi093.stdout:382: writing smithi093153278-382 from 403467 to 890986 tid 1 2023-08-27T20:52:09.810 INFO:tasks.rados.rados.0.smithi093.stdout:382: writing smithi093153278-382 from 1562110 to 2141230 tid 2 2023-08-27T20:52:09.810 INFO:tasks.rados.rados.0.smithi093.stdout:383: write initial oid 383 2023-08-27T20:52:09.811 INFO:tasks.rados.rados.0.smithi093.stdout:383: seq_num 382 ranges {420274=562768,1620592=439131} 2023-08-27T20:52:09.816 INFO:tasks.rados.rados.0.smithi093.stdout:383: writing smithi093153278-383 from 420274 to 983042 tid 1 2023-08-27T20:52:09.823 INFO:tasks.rados.rados.0.smithi093.stdout:383: writing smithi093153278-383 from 1620592 to 2059723 tid 2 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:368: finishing write tid 3 to smithi093153278-368 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:368: finishing write tid 4 to smithi093153278-368 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 368 v 67 (ObjNum 367 snap 0 seq_num 367) dirty exists 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:368: left oid 368 (ObjNum 367 snap 0 seq_num 367) 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:369: finishing write tid 1 to smithi093153278-369 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:369: finishing write tid 2 to smithi093153278-369 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:369: finishing write tid 3 to smithi093153278-369 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:369: finishing write tid 4 to smithi093153278-369 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 369 v 69 (ObjNum 368 snap 0 seq_num 368) dirty exists 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:369: left oid 369 (ObjNum 368 snap 0 seq_num 368) 2023-08-27T20:52:09.824 INFO:tasks.rados.rados.0.smithi093.stdout:370: finishing write tid 1 to smithi093153278-370 2023-08-27T20:52:09.825 INFO:tasks.rados.rados.0.smithi093.stdout:370: finishing write tid 2 to smithi093153278-370 2023-08-27T20:52:09.825 INFO:tasks.rados.rados.0.smithi093.stdout:370: finishing write tid 3 to smithi093153278-370 2023-08-27T20:52:09.825 INFO:tasks.rados.rados.0.smithi093.stdout:368: done (15 left) 2023-08-27T20:52:09.825 INFO:tasks.rados.rados.0.smithi093.stdout:369: done (14 left) 2023-08-27T20:52:09.825 INFO:tasks.rados.rados.0.smithi093.stdout:384: write initial oid 384 2023-08-27T20:52:09.825 INFO:tasks.rados.rados.0.smithi093.stdout:384: seq_num 383 ranges {437081=638017,1679074=582789} 2023-08-27T20:52:09.829 INFO:tasks.rados.rados.0.smithi093.stdout:384: writing smithi093153278-384 from 437081 to 1075098 tid 1 2023-08-27T20:52:09.837 INFO:tasks.rados.rados.0.smithi093.stdout:384: writing smithi093153278-384 from 1679074 to 2261863 tid 2 2023-08-27T20:52:09.837 INFO:tasks.rados.rados.0.smithi093.stdout:385: write initial oid 385 2023-08-27T20:52:09.837 INFO:tasks.rados.rados.0.smithi093.stdout:385: seq_num 384 ranges {453888=713266,1737556=442800} 2023-08-27T20:52:09.844 INFO:tasks.rados.rados.0.smithi093.stdout:385: writing smithi093153278-385 from 453888 to 1167154 tid 1 2023-08-27T20:52:09.850 INFO:tasks.rados.rados.0.smithi093.stdout:385: writing smithi093153278-385 from 1737556 to 2180356 tid 2 2023-08-27T20:52:09.851 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.851 INFO:tasks.rados.rados.0.smithi093.stdout:370: finishing write tid 4 to smithi093153278-370 2023-08-27T20:52:09.851 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 370 v 71 (ObjNum 369 snap 0 seq_num 369) dirty exists 2023-08-27T20:52:09.851 INFO:tasks.rados.rados.0.smithi093.stdout:370: left oid 370 (ObjNum 369 snap 0 seq_num 369) 2023-08-27T20:52:09.851 INFO:tasks.rados.rados.0.smithi093.stdout:370: done (15 left) 2023-08-27T20:52:09.851 INFO:tasks.rados.rados.0.smithi093.stdout:386: write initial oid 386 2023-08-27T20:52:09.851 INFO:tasks.rados.rados.0.smithi093.stdout:386: seq_num 385 ranges {470695=788515,1679685=586458} 2023-08-27T20:52:09.857 INFO:tasks.rados.rados.0.smithi093.stdout:386: writing smithi093153278-386 from 470695 to 1259210 tid 1 2023-08-27T20:52:09.864 INFO:tasks.rados.rados.0.smithi093.stdout:386: writing smithi093153278-386 from 1679685 to 2266143 tid 2 2023-08-27T20:52:09.864 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.864 INFO:tasks.rados.rados.0.smithi093.stdout:371: finishing write tid 1 to smithi093153278-371 2023-08-27T20:52:09.864 INFO:tasks.rados.rados.0.smithi093.stdout:371: finishing write tid 2 to smithi093153278-371 2023-08-27T20:52:09.864 INFO:tasks.rados.rados.0.smithi093.stdout:371: finishing write tid 3 to smithi093153278-371 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:371: finishing write tid 4 to smithi093153278-371 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 371 v 88 (ObjNum 370 snap 0 seq_num 370) dirty exists 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:371: left oid 371 (ObjNum 370 snap 0 seq_num 370) 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:372: finishing write tid 1 to smithi093153278-372 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:372: finishing write tid 2 to smithi093153278-372 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:372: finishing write tid 3 to smithi093153278-372 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:372: finishing write tid 4 to smithi093153278-372 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 372 v 90 (ObjNum 371 snap 0 seq_num 371) dirty exists 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:372: left oid 372 (ObjNum 371 snap 0 seq_num 371) 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:373: finishing write tid 1 to smithi093153278-373 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:371: done (15 left) 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:372: done (14 left) 2023-08-27T20:52:09.865 INFO:tasks.rados.rados.0.smithi093.stdout:387: write initial oid 387 2023-08-27T20:52:09.866 INFO:tasks.rados.rados.0.smithi093.stdout:373: finishing write tid 2 to smithi093153278-373 2023-08-27T20:52:09.866 INFO:tasks.rados.rados.0.smithi093.stdout:373: finishing write tid 3 to smithi093153278-373 2023-08-27T20:52:09.866 INFO:tasks.rados.rados.0.smithi093.stdout:373: finishing write tid 4 to smithi093153278-373 2023-08-27T20:52:09.866 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 373 v 72 (ObjNum 372 snap 0 seq_num 372) dirty exists 2023-08-27T20:52:09.866 INFO:tasks.rados.rados.0.smithi093.stdout:373: left oid 373 (ObjNum 372 snap 0 seq_num 372) 2023-08-27T20:52:09.866 INFO:tasks.rados.rados.0.smithi093.stdout:374: finishing write tid 1 to smithi093153278-374 2023-08-27T20:52:09.866 INFO:tasks.rados.rados.0.smithi093.stdout:387: seq_num 386 ranges {487502=463764,1738167=446469} 2023-08-27T20:52:09.869 INFO:tasks.rados.rados.0.smithi093.stdout:387: writing smithi093153278-387 from 487502 to 951266 tid 1 2023-08-27T20:52:09.877 INFO:tasks.rados.rados.0.smithi093.stdout:387: writing smithi093153278-387 from 1738167 to 2184636 tid 2 2023-08-27T20:52:09.877 INFO:tasks.rados.rados.0.smithi093.stdout:373: done (14 left) 2023-08-27T20:52:09.878 INFO:tasks.rados.rados.0.smithi093.stdout:388: write initial oid 388 2023-08-27T20:52:09.878 INFO:tasks.rados.rados.0.smithi093.stdout:388: seq_num 387 ranges {504309=539013,1796649=590127} 2023-08-27T20:52:09.883 INFO:tasks.rados.rados.0.smithi093.stdout:388: writing smithi093153278-388 from 504309 to 1043322 tid 1 2023-08-27T20:52:09.891 INFO:tasks.rados.rados.0.smithi093.stdout:388: writing smithi093153278-388 from 1796649 to 2386776 tid 2 2023-08-27T20:52:09.891 INFO:tasks.rados.rados.0.smithi093.stdout:389: write initial oid 389 2023-08-27T20:52:09.891 INFO:tasks.rados.rados.0.smithi093.stdout:389: seq_num 388 ranges {521116=730615,1971484=549632} 2023-08-27T20:52:09.898 INFO:tasks.rados.rados.0.smithi093.stdout:389: writing smithi093153278-389 from 521116 to 1251731 tid 1 2023-08-27T20:52:09.905 INFO:tasks.rados.rados.0.smithi093.stdout:389: writing smithi093153278-389 from 1971484 to 2521116 tid 2 2023-08-27T20:52:09.905 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.905 INFO:tasks.rados.rados.0.smithi093.stdout:374: finishing write tid 2 to smithi093153278-374 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:374: finishing write tid 3 to smithi093153278-374 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:374: finishing write tid 4 to smithi093153278-374 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 374 v 70 (ObjNum 373 snap 0 seq_num 373) dirty exists 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:374: left oid 374 (ObjNum 373 snap 0 seq_num 373) 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:375: finishing write tid 1 to smithi093153278-375 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:375: finishing write tid 2 to smithi093153278-375 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:375: finishing write tid 3 to smithi093153278-375 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:374: done (15 left) 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:390: write initial oid 390 2023-08-27T20:52:09.906 INFO:tasks.rados.rados.0.smithi093.stdout:390: seq_num 389 ranges {537923=405864,1513613=593796} 2023-08-27T20:52:09.910 INFO:tasks.rados.rados.0.smithi093.stdout:390: writing smithi093153278-390 from 537923 to 943787 tid 1 2023-08-27T20:52:09.917 INFO:tasks.rados.rados.0.smithi093.stdout:390: writing smithi093153278-390 from 1513613 to 2107409 tid 2 2023-08-27T20:52:09.917 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:09.917 INFO:tasks.rados.rados.0.smithi093.stdout:375: finishing write tid 4 to smithi093153278-375 2023-08-27T20:52:09.917 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 375 v 75 (ObjNum 374 snap 0 seq_num 374) dirty exists 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:375: left oid 375 (ObjNum 374 snap 0 seq_num 374) 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:376: finishing write tid 1 to smithi093153278-376 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:376: finishing write tid 2 to smithi093153278-376 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:376: finishing write tid 3 to smithi093153278-376 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:376: finishing write tid 4 to smithi093153278-376 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 376 v 100 (ObjNum 375 snap 0 seq_num 375) dirty exists 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:376: left oid 376 (ObjNum 375 snap 0 seq_num 375) 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:377: finishing write tid 1 to smithi093153278-377 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:377: finishing write tid 2 to smithi093153278-377 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:377: finishing write tid 3 to smithi093153278-377 2023-08-27T20:52:09.918 INFO:tasks.rados.rados.0.smithi093.stdout:377: finishing write tid 4 to smithi093153278-377 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 377 v 74 (ObjNum 376 snap 0 seq_num 376) dirty exists 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:377: left oid 377 (ObjNum 376 snap 0 seq_num 376) 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:378: finishing write tid 1 to smithi093153278-378 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:378: finishing write tid 2 to smithi093153278-378 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:378: finishing write tid 3 to smithi093153278-378 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:378: finishing write tid 4 to smithi093153278-378 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 378 v 77 (ObjNum 377 snap 0 seq_num 377) dirty exists 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:378: left oid 378 (ObjNum 377 snap 0 seq_num 377) 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:379: finishing write tid 1 to smithi093153278-379 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:375: done (15 left) 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:376: done (14 left) 2023-08-27T20:52:09.919 INFO:tasks.rados.rados.0.smithi093.stdout:377: done (13 left) 2023-08-27T20:52:09.920 INFO:tasks.rados.rados.0.smithi093.stdout:378: done (12 left) 2023-08-27T20:52:09.920 INFO:tasks.rados.rados.0.smithi093.stdout:391: write initial oid 391 2023-08-27T20:52:09.920 INFO:tasks.rados.rados.0.smithi093.stdout:391: seq_num 390 ranges {554730=481113,1572095=737454} 2023-08-27T20:52:09.923 INFO:tasks.rados.rados.0.smithi093.stdout:391: writing smithi093153278-391 from 554730 to 1035843 tid 1 2023-08-27T20:52:09.931 INFO:tasks.rados.rados.0.smithi093.stdout:391: writing smithi093153278-391 from 1572095 to 2309549 tid 2 2023-08-27T20:52:09.931 INFO:tasks.rados.rados.0.smithi093.stdout:392: write initial oid 392 2023-08-27T20:52:09.931 INFO:tasks.rados.rados.0.smithi093.stdout:392: seq_num 391 ranges {571537=556362,1630577=597465} 2023-08-27T20:52:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:09 smithi136 ceph-mon[113531]: pgmap v59: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 34 MiB/s wr, 174 op/s 2023-08-27T20:52:09.938 INFO:tasks.rados.rados.0.smithi093.stdout:392: writing smithi093153278-392 from 571537 to 1127899 tid 1 2023-08-27T20:52:09.945 INFO:tasks.rados.rados.0.smithi093.stdout:392: writing smithi093153278-392 from 1630577 to 2228042 tid 2 2023-08-27T20:52:09.945 INFO:tasks.rados.rados.0.smithi093.stdout:393: write initial oid 393 2023-08-27T20:52:09.945 INFO:tasks.rados.rados.0.smithi093.stdout:379: finishing write tid 2 to smithi093153278-379 2023-08-27T20:52:09.945 INFO:tasks.rados.rados.0.smithi093.stdout:379: finishing write tid 3 to smithi093153278-379 2023-08-27T20:52:09.945 INFO:tasks.rados.rados.0.smithi093.stdout:393: seq_num 392 ranges {588344=631611,1689059=741123} 2023-08-27T20:52:09.951 INFO:tasks.rados.rados.0.smithi093.stdout:393: writing smithi093153278-393 from 588344 to 1219955 tid 1 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:393: writing smithi093153278-393 from 1689059 to 2430182 tid 2 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:394: write initial oid 394 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:379: finishing write tid 4 to smithi093153278-379 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 379 v 91 (ObjNum 378 snap 0 seq_num 378) dirty exists 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:379: left oid 379 (ObjNum 378 snap 0 seq_num 378) 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:380: finishing write tid 1 to smithi093153278-380 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:380: finishing write tid 2 to smithi093153278-380 2023-08-27T20:52:09.959 INFO:tasks.rados.rados.0.smithi093.stdout:380: finishing write tid 3 to smithi093153278-380 2023-08-27T20:52:09.960 INFO:tasks.rados.rados.0.smithi093.stdout:380: finishing write tid 4 to smithi093153278-380 2023-08-27T20:52:09.960 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 380 v 82 (ObjNum 379 snap 0 seq_num 379) dirty exists 2023-08-27T20:52:09.960 INFO:tasks.rados.rados.0.smithi093.stdout:380: left oid 380 (ObjNum 379 snap 0 seq_num 379) 2023-08-27T20:52:09.960 INFO:tasks.rados.rados.0.smithi093.stdout:381: finishing write tid 1 to smithi093153278-381 2023-08-27T20:52:09.960 INFO:tasks.rados.rados.0.smithi093.stdout:381: finishing write tid 2 to smithi093153278-381 2023-08-27T20:52:09.960 INFO:tasks.rados.rados.0.smithi093.stdout:394: seq_num 393 ranges {605151=706860,2031188=573963} 2023-08-27T20:52:09.965 INFO:tasks.rados.rados.0.smithi093.stdout:394: writing smithi093153278-394 from 605151 to 1312011 tid 1 2023-08-27T20:52:09.973 INFO:tasks.rados.rados.0.smithi093.stdout:394: writing smithi093153278-394 from 2031188 to 2605151 tid 2 2023-08-27T20:52:09.973 INFO:tasks.rados.rados.0.smithi093.stdout:379: done (15 left) 2023-08-27T20:52:09.973 INFO:tasks.rados.rados.0.smithi093.stdout:380: done (14 left) 2023-08-27T20:52:09.973 INFO:tasks.rados.rados.0.smithi093.stdout:395: write initial oid 395 2023-08-27T20:52:09.973 INFO:tasks.rados.rados.0.smithi093.stdout:381: finishing write tid 3 to smithi093153278-381 2023-08-27T20:52:09.973 INFO:tasks.rados.rados.0.smithi093.stdout:381: finishing write tid 4 to smithi093153278-381 2023-08-27T20:52:09.973 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 381 v 78 (ObjNum 380 snap 0 seq_num 380) dirty exists 2023-08-27T20:52:09.974 INFO:tasks.rados.rados.0.smithi093.stdout:381: left oid 381 (ObjNum 380 snap 0 seq_num 380) 2023-08-27T20:52:09.974 INFO:tasks.rados.rados.0.smithi093.stdout:382: finishing write tid 1 to smithi093153278-382 2023-08-27T20:52:09.974 INFO:tasks.rados.rados.0.smithi093.stdout:382: finishing write tid 2 to smithi093153278-382 2023-08-27T20:52:09.974 INFO:tasks.rados.rados.0.smithi093.stdout:395: seq_num 394 ranges {621958=782109,2089670=532288} 2023-08-27T20:52:09.982 INFO:tasks.rados.rados.0.smithi093.stdout:395: writing smithi093153278-395 from 621958 to 1404067 tid 1 2023-08-27T20:52:09.987 INFO:tasks.rados.rados.0.smithi093.stdout:395: writing smithi093153278-395 from 2089670 to 2621958 tid 2 2023-08-27T20:52:09.987 INFO:tasks.rados.rados.0.smithi093.stdout:381: done (14 left) 2023-08-27T20:52:09.987 INFO:tasks.rados.rados.0.smithi093.stdout:396: write initial oid 396 2023-08-27T20:52:09.987 INFO:tasks.rados.rados.0.smithi093.stdout:396: seq_num 395 ranges {638765=457358,1748152=604803} 2023-08-27T20:52:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:09 smithi093 ceph-mon[111335]: pgmap v59: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 34 MiB/s wr, 174 op/s 2023-08-27T20:52:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:09 smithi093 ceph-mon[118902]: pgmap v59: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 27 KiB/s rd, 34 MiB/s wr, 174 op/s 2023-08-27T20:52:09.993 INFO:tasks.rados.rados.0.smithi093.stdout:396: writing smithi093153278-396 from 638765 to 1096123 tid 1 2023-08-27T20:52:10.000 INFO:tasks.rados.rados.0.smithi093.stdout:396: writing smithi093153278-396 from 1748152 to 2352955 tid 2 2023-08-27T20:52:10.000 INFO:tasks.rados.rados.0.smithi093.stdout:397: write initial oid 397 2023-08-27T20:52:10.000 INFO:tasks.rados.rados.0.smithi093.stdout:397: seq_num 396 ranges {655572=648960,1922987=732585} 2023-08-27T20:52:10.008 INFO:tasks.rados.rados.0.smithi093.stdout:397: writing smithi093153278-397 from 655572 to 1304532 tid 1 2023-08-27T20:52:10.017 INFO:tasks.rados.rados.0.smithi093.stdout:397: writing smithi093153278-397 from 1922987 to 2655572 tid 2 2023-08-27T20:52:10.017 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.017 INFO:tasks.rados.rados.0.smithi093.stdout:382: finishing write tid 3 to smithi093153278-382 2023-08-27T20:52:10.017 INFO:tasks.rados.rados.0.smithi093.stdout:382: finishing write tid 4 to smithi093153278-382 2023-08-27T20:52:10.017 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 382 v 76 (ObjNum 381 snap 0 seq_num 381) dirty exists 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:382: left oid 382 (ObjNum 381 snap 0 seq_num 381) 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:383: finishing write tid 1 to smithi093153278-383 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:383: finishing write tid 2 to smithi093153278-383 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:383: finishing write tid 3 to smithi093153278-383 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:383: finishing write tid 4 to smithi093153278-383 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 383 v 79 (ObjNum 382 snap 0 seq_num 382) dirty exists 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:383: left oid 383 (ObjNum 382 snap 0 seq_num 382) 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:384: finishing write tid 1 to smithi093153278-384 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:382: done (15 left) 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:383: done (14 left) 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:398: write initial oid 398 2023-08-27T20:52:10.018 INFO:tasks.rados.rados.0.smithi093.stdout:398: seq_num 397 ranges {672379=724209,1865116=608472} 2023-08-27T20:52:10.025 INFO:tasks.rados.rados.0.smithi093.stdout:398: writing smithi093153278-398 from 672379 to 1396588 tid 1 2023-08-27T20:52:10.032 INFO:tasks.rados.rados.0.smithi093.stdout:398: writing smithi093153278-398 from 1865116 to 2473588 tid 2 2023-08-27T20:52:10.032 INFO:tasks.rados.rados.0.smithi093.stdout:399: write initial oid 399 2023-08-27T20:52:10.033 INFO:tasks.rados.rados.0.smithi093.stdout:384: finishing write tid 2 to smithi093153278-384 2023-08-27T20:52:10.033 INFO:tasks.rados.rados.0.smithi093.stdout:384: finishing write tid 3 to smithi093153278-384 2023-08-27T20:52:10.033 INFO:tasks.rados.rados.0.smithi093.stdout:399: seq_num 398 ranges {689186=799458,1923598=752130} 2023-08-27T20:52:10.041 INFO:tasks.rados.rados.0.smithi093.stdout:399: writing smithi093153278-399 from 689186 to 1488644 tid 1 2023-08-27T20:52:10.048 INFO:tasks.rados.rados.0.smithi093.stdout:399: writing smithi093153278-399 from 1923598 to 2675728 tid 2 2023-08-27T20:52:10.048 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.048 INFO:tasks.rados.rados.0.smithi093.stdout:384: finishing write tid 4 to smithi093153278-384 2023-08-27T20:52:10.048 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 384 v 94 (ObjNum 383 snap 0 seq_num 383) dirty exists 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:384: left oid 384 (ObjNum 383 snap 0 seq_num 383) 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:385: finishing write tid 1 to smithi093153278-385 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:384: done (15 left) 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:400: write initial oid 400 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:385: finishing write tid 2 to smithi093153278-385 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:385: finishing write tid 3 to smithi093153278-385 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:385: finishing write tid 4 to smithi093153278-385 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 385 v 85 (ObjNum 384 snap 0 seq_num 384) dirty exists 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:385: left oid 385 (ObjNum 384 snap 0 seq_num 384) 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:386: finishing write tid 1 to smithi093153278-386 2023-08-27T20:52:10.049 INFO:tasks.rados.rados.0.smithi093.stdout:400: seq_num 399 ranges {705993=474707,1582080=612141} 2023-08-27T20:52:10.054 INFO:tasks.rados.rados.0.smithi093.stdout:400: writing smithi093153278-400 from 705993 to 1180700 tid 1 2023-08-27T20:52:10.061 INFO:tasks.rados.rados.0.smithi093.stdout:400: writing smithi093153278-400 from 1582080 to 2194221 tid 2 2023-08-27T20:52:10.061 INFO:tasks.rados.rados.0.smithi093.stdout:385: done (15 left) 2023-08-27T20:52:10.061 INFO:tasks.rados.rados.0.smithi093.stdout:401: write initial oid 401 2023-08-27T20:52:10.061 INFO:tasks.rados.rados.0.smithi093.stdout:401: seq_num 400 ranges {722800=549956,2040562=682238} 2023-08-27T20:52:10.068 INFO:tasks.rados.rados.0.smithi093.stdout:401: writing smithi093153278-401 from 722800 to 1272756 tid 1 2023-08-27T20:52:10.078 INFO:tasks.rados.rados.0.smithi093.stdout:401: writing smithi093153278-401 from 2040562 to 2722800 tid 2 2023-08-27T20:52:10.078 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.078 INFO:tasks.rados.rados.0.smithi093.stdout:386: finishing write tid 2 to smithi093153278-386 2023-08-27T20:52:10.078 INFO:tasks.rados.rados.0.smithi093.stdout:386: finishing write tid 3 to smithi093153278-386 2023-08-27T20:52:10.078 INFO:tasks.rados.rados.0.smithi093.stdout:386: finishing write tid 4 to smithi093153278-386 2023-08-27T20:52:10.078 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 386 v 90 (ObjNum 385 snap 0 seq_num 385) dirty exists 2023-08-27T20:52:10.078 INFO:tasks.rados.rados.0.smithi093.stdout:386: left oid 386 (ObjNum 385 snap 0 seq_num 385) 2023-08-27T20:52:10.079 INFO:tasks.rados.rados.0.smithi093.stdout:387: finishing write tid 1 to smithi093153278-387 2023-08-27T20:52:10.079 INFO:tasks.rados.rados.0.smithi093.stdout:387: finishing write tid 2 to smithi093153278-387 2023-08-27T20:52:10.079 INFO:tasks.rados.rados.0.smithi093.stdout:386: done (15 left) 2023-08-27T20:52:10.079 INFO:tasks.rados.rados.0.smithi093.stdout:402: write initial oid 402 2023-08-27T20:52:10.079 INFO:tasks.rados.rados.0.smithi093.stdout:402: seq_num 401 ranges {739607=625205,1982691=499457} 2023-08-27T20:52:10.085 INFO:tasks.rados.rados.0.smithi093.stdout:402: writing smithi093153278-402 from 739607 to 1364812 tid 1 2023-08-27T20:52:10.091 INFO:tasks.rados.rados.0.smithi093.stdout:402: writing smithi093153278-402 from 1982691 to 2482148 tid 2 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:387: finishing write tid 3 to smithi093153278-387 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:387: finishing write tid 4 to smithi093153278-387 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 387 v 71 (ObjNum 386 snap 0 seq_num 386) dirty exists 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:387: left oid 387 (ObjNum 386 snap 0 seq_num 386) 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:388: finishing write tid 1 to smithi093153278-388 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:388: finishing write tid 2 to smithi093153278-388 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:387: done (15 left) 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:403: write initial oid 403 2023-08-27T20:52:10.092 INFO:tasks.rados.rados.0.smithi093.stdout:403: seq_num 402 ranges {756414=700454,2041173=715241} 2023-08-27T20:52:10.099 INFO:tasks.rados.rados.0.smithi093.stdout:403: writing smithi093153278-403 from 756414 to 1456868 tid 1 2023-08-27T20:52:10.107 INFO:tasks.rados.rados.0.smithi093.stdout:403: writing smithi093153278-403 from 2041173 to 2756414 tid 2 2023-08-27T20:52:10.107 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.107 INFO:tasks.rados.rados.0.smithi093.stdout:388: finishing write tid 3 to smithi093153278-388 2023-08-27T20:52:10.107 INFO:tasks.rados.rados.0.smithi093.stdout:388: finishing write tid 4 to smithi093153278-388 2023-08-27T20:52:10.107 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 388 v 88 (ObjNum 387 snap 0 seq_num 387) dirty exists 2023-08-27T20:52:10.107 INFO:tasks.rados.rados.0.smithi093.stdout:388: left oid 388 (ObjNum 387 snap 0 seq_num 387) 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:389: finishing write tid 1 to smithi093153278-389 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:389: finishing write tid 2 to smithi093153278-389 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:389: finishing write tid 3 to smithi093153278-389 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:389: finishing write tid 4 to smithi093153278-389 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 389 v 93 (ObjNum 388 snap 0 seq_num 388) dirty exists 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:389: left oid 389 (ObjNum 388 snap 0 seq_num 388) 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:390: finishing write tid 1 to smithi093153278-390 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:390: finishing write tid 2 to smithi093153278-390 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:390: finishing write tid 3 to smithi093153278-390 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:390: finishing write tid 4 to smithi093153278-390 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 390 v 84 (ObjNum 389 snap 0 seq_num 389) dirty exists 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:390: left oid 390 (ObjNum 389 snap 0 seq_num 389) 2023-08-27T20:52:10.108 INFO:tasks.rados.rados.0.smithi093.stdout:391: finishing write tid 1 to smithi093153278-391 2023-08-27T20:52:10.109 INFO:tasks.rados.rados.0.smithi093.stdout:388: done (15 left) 2023-08-27T20:52:10.109 INFO:tasks.rados.rados.0.smithi093.stdout:389: done (14 left) 2023-08-27T20:52:10.109 INFO:tasks.rados.rados.0.smithi093.stdout:390: done (13 left) 2023-08-27T20:52:10.109 INFO:tasks.rados.rados.0.smithi093.stdout:404: write initial oid 404 2023-08-27T20:52:10.109 INFO:tasks.rados.rados.0.smithi093.stdout:404: seq_num 403 ranges {773221=492056,1816008=503126} 2023-08-27T20:52:10.113 INFO:tasks.rados.rados.0.smithi093.stdout:404: writing smithi093153278-404 from 773221 to 1265277 tid 1 2023-08-27T20:52:10.119 INFO:tasks.rados.rados.0.smithi093.stdout:404: writing smithi093153278-404 from 1816008 to 2319134 tid 2 2023-08-27T20:52:10.119 INFO:tasks.rados.rados.0.smithi093.stdout:405: write initial oid 405 2023-08-27T20:52:10.120 INFO:tasks.rados.rados.0.smithi093.stdout:405: seq_num 404 ranges {790028=567305,1874490=763137} 2023-08-27T20:52:10.127 INFO:tasks.rados.rados.0.smithi093.stdout:405: writing smithi093153278-405 from 790028 to 1357333 tid 1 2023-08-27T20:52:10.135 INFO:tasks.rados.rados.0.smithi093.stdout:405: writing smithi093153278-405 from 1874490 to 2637627 tid 2 2023-08-27T20:52:10.135 INFO:tasks.rados.rados.0.smithi093.stdout:406: write initial oid 406 2023-08-27T20:52:10.135 INFO:tasks.rados.rados.0.smithi093.stdout:406: seq_num 405 ranges {406835=642554,1532972=506795,2772312=34523} 2023-08-27T20:52:10.142 INFO:tasks.rados.rados.0.smithi093.stdout:406: writing smithi093153278-406 from 406835 to 1049389 tid 1 2023-08-27T20:52:10.147 INFO:tasks.rados.rados.0.smithi093.stdout:406: writing smithi093153278-406 from 1532972 to 2039767 tid 2 2023-08-27T20:52:10.152 INFO:tasks.rados.rados.0.smithi093.stdout:406: writing smithi093153278-406 from 2772312 to 2806835 tid 3 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:391: finishing write tid 2 to smithi093153278-391 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:391: finishing write tid 3 to smithi093153278-391 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:391: finishing write tid 4 to smithi093153278-391 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 391 v 105 (ObjNum 390 snap 0 seq_num 390) dirty exists 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:391: left oid 391 (ObjNum 390 snap 0 seq_num 390) 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:392: finishing write tid 1 to smithi093153278-392 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:392: finishing write tid 2 to smithi093153278-392 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:392: finishing write tid 3 to smithi093153278-392 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:391: done (15 left) 2023-08-27T20:52:10.153 INFO:tasks.rados.rados.0.smithi093.stdout:407: write initial oid 407 2023-08-27T20:52:10.154 INFO:tasks.rados.rados.0.smithi093.stdout:407: seq_num 406 ranges {423642=717803,1875101=766806} 2023-08-27T20:52:10.158 INFO:tasks.rados.rados.0.smithi093.stdout:407: writing smithi093153278-407 from 423642 to 1141445 tid 1 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:407: writing smithi093153278-407 from 1875101 to 2641907 tid 2 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:392: finishing write tid 4 to smithi093153278-392 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 392 v 80 (ObjNum 391 snap 0 seq_num 391) dirty exists 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:392: left oid 392 (ObjNum 391 snap 0 seq_num 391) 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:393: finishing write tid 1 to smithi093153278-393 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:393: finishing write tid 2 to smithi093153278-393 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:393: finishing write tid 3 to smithi093153278-393 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:392: done (15 left) 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:408: write initial oid 408 2023-08-27T20:52:10.169 INFO:tasks.rados.rados.0.smithi093.stdout:408: seq_num 407 ranges {440449=793052,1933583=510464} 2023-08-27T20:52:10.175 INFO:tasks.rados.rados.0.smithi093.stdout:408: writing smithi093153278-408 from 440449 to 1233501 tid 1 2023-08-27T20:52:10.182 INFO:tasks.rados.rados.0.smithi093.stdout:408: writing smithi093153278-408 from 1933583 to 2444047 tid 2 2023-08-27T20:52:10.182 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.182 INFO:tasks.rados.rados.0.smithi093.stdout:393: finishing write tid 4 to smithi093153278-393 2023-08-27T20:52:10.182 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 393 v 65 (ObjNum 392 snap 0 seq_num 392) dirty exists 2023-08-27T20:52:10.183 INFO:tasks.rados.rados.0.smithi093.stdout:393: left oid 393 (ObjNum 392 snap 0 seq_num 392) 2023-08-27T20:52:10.183 INFO:tasks.rados.rados.0.smithi093.stdout:393: done (15 left) 2023-08-27T20:52:10.183 INFO:tasks.rados.rados.0.smithi093.stdout:409: write initial oid 409 2023-08-27T20:52:10.183 INFO:tasks.rados.rados.0.smithi093.stdout:409: seq_num 408 ranges {457256=468301,1592065=770475,2854581=2675} 2023-08-27T20:52:10.188 INFO:tasks.rados.rados.0.smithi093.stdout:409: writing smithi093153278-409 from 457256 to 925557 tid 1 2023-08-27T20:52:10.197 INFO:tasks.rados.rados.0.smithi093.stdout:409: writing smithi093153278-409 from 1592065 to 2362540 tid 2 2023-08-27T20:52:10.200 INFO:tasks.rados.rados.0.smithi093.stdout:409: writing smithi093153278-409 from 2854581 to 2857256 tid 3 2023-08-27T20:52:10.200 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.200 INFO:tasks.rados.rados.0.smithi093.stdout:394: finishing write tid 1 to smithi093153278-394 2023-08-27T20:52:10.200 INFO:tasks.rados.rados.0.smithi093.stdout:394: finishing write tid 2 to smithi093153278-394 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:394: finishing write tid 3 to smithi093153278-394 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:394: finishing write tid 4 to smithi093153278-394 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 394 v 91 (ObjNum 393 snap 0 seq_num 393) dirty exists 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:394: left oid 394 (ObjNum 393 snap 0 seq_num 393) 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:395: finishing write tid 1 to smithi093153278-395 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:395: finishing write tid 2 to smithi093153278-395 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:395: finishing write tid 3 to smithi093153278-395 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:394: done (15 left) 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:410: write initial oid 410 2023-08-27T20:52:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:410: seq_num 409 ranges {474063=543550,1650547=514133,2765651=108412} 2023-08-27T20:52:10.206 INFO:tasks.rados.rados.0.smithi093.stdout:410: writing smithi093153278-410 from 474063 to 1017613 tid 1 2023-08-27T20:52:10.213 INFO:tasks.rados.rados.0.smithi093.stdout:410: writing smithi093153278-410 from 1650547 to 2164680 tid 2 2023-08-27T20:52:10.217 INFO:tasks.rados.rados.0.smithi093.stdout:410: writing smithi093153278-410 from 2765651 to 2874063 tid 3 2023-08-27T20:52:10.217 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.218 INFO:tasks.rados.rados.0.smithi093.stdout:395: finishing write tid 4 to smithi093153278-395 2023-08-27T20:52:10.218 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 103 (ObjNum 394 snap 0 seq_num 394) dirty exists 2023-08-27T20:52:10.218 INFO:tasks.rados.rados.0.smithi093.stdout:395: left oid 395 (ObjNum 394 snap 0 seq_num 394) 2023-08-27T20:52:10.218 INFO:tasks.rados.rados.0.smithi093.stdout:395: done (15 left) 2023-08-27T20:52:10.218 INFO:tasks.rados.rados.0.smithi093.stdout:411: write initial oid 411 2023-08-27T20:52:10.218 INFO:tasks.rados.rados.0.smithi093.stdout:411: seq_num 410 ranges {490870=618799,1592676=774144,2793074=97796} 2023-08-27T20:52:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:411: writing smithi093153278-411 from 490870 to 1109669 tid 1 2023-08-27T20:52:10.231 INFO:tasks.rados.rados.0.smithi093.stdout:411: writing smithi093153278-411 from 1592676 to 2366820 tid 2 2023-08-27T20:52:10.234 INFO:tasks.rados.rados.0.smithi093.stdout:411: writing smithi093153278-411 from 2793074 to 2890870 tid 3 2023-08-27T20:52:10.234 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:396: finishing write tid 1 to smithi093153278-396 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:396: finishing write tid 2 to smithi093153278-396 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:396: finishing write tid 3 to smithi093153278-396 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:396: finishing write tid 4 to smithi093153278-396 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 396 v 96 (ObjNum 395 snap 0 seq_num 395) dirty exists 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:396: left oid 396 (ObjNum 395 snap 0 seq_num 395) 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:397: finishing write tid 1 to smithi093153278-397 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:397: finishing write tid 2 to smithi093153278-397 2023-08-27T20:52:10.235 INFO:tasks.rados.rados.0.smithi093.stdout:396: done (15 left) 2023-08-27T20:52:10.236 INFO:tasks.rados.rados.0.smithi093.stdout:412: write initial oid 412 2023-08-27T20:52:10.236 INFO:tasks.rados.rados.0.smithi093.stdout:412: seq_num 411 ranges {507677=410401,1367511=517802,2420497=487180} 2023-08-27T20:52:10.239 INFO:tasks.rados.rados.0.smithi093.stdout:412: writing smithi093153278-412 from 507677 to 918078 tid 1 2023-08-27T20:52:10.244 INFO:tasks.rados.rados.0.smithi093.stdout:412: writing smithi093153278-412 from 1367511 to 1885313 tid 2 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:412: writing smithi093153278-412 from 2420497 to 2907677 tid 3 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:397: finishing write tid 3 to smithi093153278-397 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:397: finishing write tid 4 to smithi093153278-397 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 397 v 99 (ObjNum 396 snap 0 seq_num 396) dirty exists 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:397: left oid 397 (ObjNum 396 snap 0 seq_num 396) 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:398: finishing write tid 1 to smithi093153278-398 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:397: done (15 left) 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:413: write initial oid 413 2023-08-27T20:52:10.251 INFO:tasks.rados.rados.0.smithi093.stdout:413: seq_num 412 ranges {524484=485650,1425993=661460,2847920=76564} 2023-08-27T20:52:10.255 INFO:tasks.rados.rados.0.smithi093.stdout:413: writing smithi093153278-413 from 524484 to 1010134 tid 1 2023-08-27T20:52:10.262 INFO:tasks.rados.rados.0.smithi093.stdout:413: writing smithi093153278-413 from 1425993 to 2087453 tid 2 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:413: writing smithi093153278-413 from 2847920 to 2924484 tid 3 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:398: finishing write tid 2 to smithi093153278-398 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:398: finishing write tid 3 to smithi093153278-398 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:398: finishing write tid 4 to smithi093153278-398 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 398 v 94 (ObjNum 397 snap 0 seq_num 397) dirty exists 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:398: left oid 398 (ObjNum 397 snap 0 seq_num 397) 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:399: finishing write tid 1 to smithi093153278-399 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:398: done (15 left) 2023-08-27T20:52:10.267 INFO:tasks.rados.rados.0.smithi093.stdout:414: write initial oid 414 2023-08-27T20:52:10.268 INFO:tasks.rados.rados.0.smithi093.stdout:414: seq_num 413 ranges {541291=560899,1884475=521471} 2023-08-27T20:52:10.272 INFO:tasks.rados.rados.0.smithi093.stdout:414: writing smithi093153278-414 from 541291 to 1102190 tid 1 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout:414: writing smithi093153278-414 from 1884475 to 2405946 tid 2 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout:399: finishing write tid 2 to smithi093153278-399 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout:399: finishing write tid 3 to smithi093153278-399 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout:399: finishing write tid 4 to smithi093153278-399 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 399 v 97 (ObjNum 398 snap 0 seq_num 398) dirty exists 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout:399: left oid 399 (ObjNum 398 snap 0 seq_num 398) 2023-08-27T20:52:10.280 INFO:tasks.rados.rados.0.smithi093.stdout:400: finishing write tid 1 to smithi093153278-400 2023-08-27T20:52:10.281 INFO:tasks.rados.rados.0.smithi093.stdout:399: done (15 left) 2023-08-27T20:52:10.281 INFO:tasks.rados.rados.0.smithi093.stdout:415: write initial oid 415 2023-08-27T20:52:10.281 INFO:tasks.rados.rados.0.smithi093.stdout:415: seq_num 414 ranges {558098=636148,1826604=665129} 2023-08-27T20:52:10.286 INFO:tasks.rados.rados.0.smithi093.stdout:415: writing smithi093153278-415 from 558098 to 1194246 tid 1 2023-08-27T20:52:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:415: writing smithi093153278-415 from 1826604 to 2491733 tid 2 2023-08-27T20:52:10.294 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:400: finishing write tid 2 to smithi093153278-400 2023-08-27T20:52:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:400: finishing write tid 3 to smithi093153278-400 2023-08-27T20:52:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:400: finishing write tid 4 to smithi093153278-400 2023-08-27T20:52:10.295 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 400 v 68 (ObjNum 399 snap 0 seq_num 399) dirty exists 2023-08-27T20:52:10.295 INFO:tasks.rados.rados.0.smithi093.stdout:400: left oid 400 (ObjNum 399 snap 0 seq_num 399) 2023-08-27T20:52:10.295 INFO:tasks.rados.rados.0.smithi093.stdout:401: finishing write tid 1 to smithi093153278-401 2023-08-27T20:52:10.295 INFO:tasks.rados.rados.0.smithi093.stdout:401: finishing write tid 2 to smithi093153278-401 2023-08-27T20:52:10.295 INFO:tasks.rados.rados.0.smithi093.stdout:400: done (15 left) 2023-08-27T20:52:10.295 INFO:tasks.rados.rados.0.smithi093.stdout:416: write initial oid 416 2023-08-27T20:52:10.295 INFO:tasks.rados.rados.0.smithi093.stdout:416: seq_num 415 ranges {574905=711397,1885086=525140,2930189=44716} 2023-08-27T20:52:10.302 INFO:tasks.rados.rados.0.smithi093.stdout:416: writing smithi093153278-416 from 574905 to 1286302 tid 1 2023-08-27T20:52:10.309 INFO:tasks.rados.rados.0.smithi093.stdout:416: writing smithi093153278-416 from 1885086 to 2410226 tid 2 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout:416: writing smithi093153278-416 from 2930189 to 2974905 tid 3 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout:401: finishing write tid 3 to smithi093153278-401 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout:401: finishing write tid 4 to smithi093153278-401 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 401 v 71 (ObjNum 400 snap 0 seq_num 400) dirty exists 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout:401: left oid 401 (ObjNum 400 snap 0 seq_num 400) 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout:402: finishing write tid 1 to smithi093153278-402 2023-08-27T20:52:10.313 INFO:tasks.rados.rados.0.smithi093.stdout:402: finishing write tid 2 to smithi093153278-402 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:401: done (15 left) 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:417: write initial oid 417 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:402: finishing write tid 3 to smithi093153278-402 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:402: finishing write tid 4 to smithi093153278-402 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 402 v 87 (ObjNum 401 snap 0 seq_num 401) dirty exists 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:402: left oid 402 (ObjNum 401 snap 0 seq_num 401) 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:403: finishing write tid 1 to smithi093153278-403 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:403: finishing write tid 2 to smithi093153278-403 2023-08-27T20:52:10.314 INFO:tasks.rados.rados.0.smithi093.stdout:417: seq_num 416 ranges {591712=786646,1943568=668798} 2023-08-27T20:52:10.320 INFO:tasks.rados.rados.0.smithi093.stdout:417: writing smithi093153278-417 from 591712 to 1378358 tid 1 2023-08-27T20:52:10.329 INFO:tasks.rados.rados.0.smithi093.stdout:417: writing smithi093153278-417 from 1943568 to 2612366 tid 2 2023-08-27T20:52:10.329 INFO:tasks.rados.rados.0.smithi093.stdout:402: done (15 left) 2023-08-27T20:52:10.329 INFO:tasks.rados.rados.0.smithi093.stdout:418: write initial oid 418 2023-08-27T20:52:10.329 INFO:tasks.rados.rados.0.smithi093.stdout:418: seq_num 417 ranges {608519=461895,1602050=528809,2585035=423484} 2023-08-27T20:52:10.334 INFO:tasks.rados.rados.0.smithi093.stdout:418: writing smithi093153278-418 from 608519 to 1070414 tid 1 2023-08-27T20:52:10.340 INFO:tasks.rados.rados.0.smithi093.stdout:418: writing smithi093153278-418 from 1602050 to 2130859 tid 2 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout:418: writing smithi093153278-418 from 2585035 to 3008519 tid 3 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout:403: finishing write tid 3 to smithi093153278-403 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout:403: finishing write tid 4 to smithi093153278-403 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 403 v 73 (ObjNum 402 snap 0 seq_num 402) dirty exists 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout:403: left oid 403 (ObjNum 402 snap 0 seq_num 402) 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout:404: finishing write tid 1 to smithi093153278-404 2023-08-27T20:52:10.346 INFO:tasks.rados.rados.0.smithi093.stdout:404: finishing write tid 2 to smithi093153278-404 2023-08-27T20:52:10.347 INFO:tasks.rados.rados.0.smithi093.stdout:404: finishing write tid 3 to smithi093153278-404 2023-08-27T20:52:10.347 INFO:tasks.rados.rados.0.smithi093.stdout:403: done (15 left) 2023-08-27T20:52:10.347 INFO:tasks.rados.rados.0.smithi093.stdout:419: write initial oid 419 2023-08-27T20:52:10.347 INFO:tasks.rados.rados.0.smithi093.stdout:419: seq_num 418 ranges {625326=537144,1944179=672467} 2023-08-27T20:52:10.352 INFO:tasks.rados.rados.0.smithi093.stdout:419: writing smithi093153278-419 from 625326 to 1162470 tid 1 2023-08-27T20:52:10.360 INFO:tasks.rados.rados.0.smithi093.stdout:419: writing smithi093153278-419 from 1944179 to 2616646 tid 2 2023-08-27T20:52:10.360 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:404: finishing write tid 4 to smithi093153278-404 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 404 v 81 (ObjNum 403 snap 0 seq_num 403) dirty exists 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:404: left oid 404 (ObjNum 403 snap 0 seq_num 403) 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:405: finishing write tid 1 to smithi093153278-405 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:405: finishing write tid 2 to smithi093153278-405 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:405: finishing write tid 3 to smithi093153278-405 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:405: finishing write tid 4 to smithi093153278-405 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 405 v 83 (ObjNum 404 snap 0 seq_num 404) dirty exists 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:405: left oid 405 (ObjNum 404 snap 0 seq_num 404) 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:404: done (15 left) 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:405: done (14 left) 2023-08-27T20:52:10.361 INFO:tasks.rados.rados.0.smithi093.stdout:420: write initial oid 420 2023-08-27T20:52:10.362 INFO:tasks.rados.rados.0.smithi093.stdout:420: seq_num 419 ranges {642133=728746,2119014=532478} 2023-08-27T20:52:10.367 INFO:tasks.rados.rados.0.smithi093.stdout:420: writing smithi093153278-420 from 642133 to 1370879 tid 1 2023-08-27T20:52:10.375 INFO:tasks.rados.rados.0.smithi093.stdout:420: writing smithi093153278-420 from 2119014 to 2651492 tid 2 2023-08-27T20:52:10.375 INFO:tasks.rados.rados.0.smithi093.stdout:421: write initial oid 421 2023-08-27T20:52:10.375 INFO:tasks.rados.rados.0.smithi093.stdout:421: seq_num 420 ranges {658940=403995,1777496=676136,2950951=107989} 2023-08-27T20:52:10.380 INFO:tasks.rados.rados.0.smithi093.stdout:421: writing smithi093153278-421 from 658940 to 1062935 tid 1 2023-08-27T20:52:10.390 INFO:tasks.rados.rados.0.smithi093.stdout:421: writing smithi093153278-421 from 1777496 to 2453632 tid 2 2023-08-27T20:52:10.393 INFO:tasks.rados.rados.0.smithi093.stdout:421: writing smithi093153278-421 from 2950951 to 3058940 tid 3 2023-08-27T20:52:10.393 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.393 INFO:tasks.rados.rados.0.smithi093.stdout:406: finishing write tid 1 to smithi093153278-406 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:406: finishing write tid 2 to smithi093153278-406 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:406: finishing write tid 3 to smithi093153278-406 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:406: finishing write tid 4 to smithi093153278-406 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:406: finishing write tid 5 to smithi093153278-406 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 406 v 101 (ObjNum 405 snap 0 seq_num 405) dirty exists 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:406: left oid 406 (ObjNum 405 snap 0 seq_num 405) 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:407: finishing write tid 1 to smithi093153278-407 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:406: done (15 left) 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:422: write initial oid 422 2023-08-27T20:52:10.394 INFO:tasks.rados.rados.0.smithi093.stdout:422: seq_num 421 ranges {675747=479244,1835978=536147} 2023-08-27T20:52:10.400 INFO:tasks.rados.rados.0.smithi093.stdout:422: writing smithi093153278-422 from 675747 to 1154991 tid 1 2023-08-27T20:52:10.408 INFO:tasks.rados.rados.0.smithi093.stdout:422: writing smithi093153278-422 from 1835978 to 2372125 tid 2 2023-08-27T20:52:10.408 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.408 INFO:tasks.rados.rados.0.smithi093.stdout:407: finishing write tid 2 to smithi093153278-407 2023-08-27T20:52:10.408 INFO:tasks.rados.rados.0.smithi093.stdout:407: finishing write tid 3 to smithi093153278-407 2023-08-27T20:52:10.408 INFO:tasks.rados.rados.0.smithi093.stdout:407: finishing write tid 4 to smithi093153278-407 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 407 v 76 (ObjNum 406 snap 0 seq_num 406) dirty exists 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:407: left oid 407 (ObjNum 406 snap 0 seq_num 406) 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:408: finishing write tid 1 to smithi093153278-408 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:408: finishing write tid 2 to smithi093153278-408 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:407: done (15 left) 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:423: write initial oid 423 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:408: finishing write tid 3 to smithi093153278-408 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:408: finishing write tid 4 to smithi093153278-408 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 408 v 82 (ObjNum 407 snap 0 seq_num 407) dirty exists 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:408: left oid 408 (ObjNum 407 snap 0 seq_num 407) 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:409: finishing write tid 1 to smithi093153278-409 2023-08-27T20:52:10.409 INFO:tasks.rados.rados.0.smithi093.stdout:423: seq_num 422 ranges {692554=554493,1778107=679805,2889444=203110} 2023-08-27T20:52:10.415 INFO:tasks.rados.rados.0.smithi093.stdout:423: writing smithi093153278-423 from 692554 to 1247047 tid 1 2023-08-27T20:52:10.422 INFO:tasks.rados.rados.0.smithi093.stdout:423: writing smithi093153278-423 from 1778107 to 2457912 tid 2 2023-08-27T20:52:10.426 INFO:tasks.rados.rados.0.smithi093.stdout:423: writing smithi093153278-423 from 2889444 to 3092554 tid 3 2023-08-27T20:52:10.426 INFO:tasks.rados.rados.0.smithi093.stdout:408: done (15 left) 2023-08-27T20:52:10.427 INFO:tasks.rados.rados.0.smithi093.stdout:424: write initial oid 424 2023-08-27T20:52:10.427 INFO:tasks.rados.rados.0.smithi093.stdout:424: seq_num 423 ranges {709361=629742,1836589=539816,3033220=76141} 2023-08-27T20:52:10.434 INFO:tasks.rados.rados.0.smithi093.stdout:424: writing smithi093153278-424 from 709361 to 1339103 tid 1 2023-08-27T20:52:10.440 INFO:tasks.rados.rados.0.smithi093.stdout:424: writing smithi093153278-424 from 1836589 to 2376405 tid 2 2023-08-27T20:52:10.444 INFO:tasks.rados.rados.0.smithi093.stdout:424: writing smithi093153278-424 from 3033220 to 3109361 tid 3 2023-08-27T20:52:10.444 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.444 INFO:tasks.rados.rados.0.smithi093.stdout:409: finishing write tid 2 to smithi093153278-409 2023-08-27T20:52:10.444 INFO:tasks.rados.rados.0.smithi093.stdout:409: finishing write tid 3 to smithi093153278-409 2023-08-27T20:52:10.444 INFO:tasks.rados.rados.0.smithi093.stdout:409: finishing write tid 4 to smithi093153278-409 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:409: finishing write tid 5 to smithi093153278-409 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 409 v 92 (ObjNum 408 snap 0 seq_num 408) dirty exists 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:409: left oid 409 (ObjNum 408 snap 0 seq_num 408) 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:410: finishing write tid 1 to smithi093153278-410 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:410: finishing write tid 2 to smithi093153278-410 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:409: done (15 left) 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:425: write initial oid 425 2023-08-27T20:52:10.445 INFO:tasks.rados.rados.0.smithi093.stdout:425: seq_num 424 ranges {726168=704991,1895071=683474} 2023-08-27T20:52:10.451 INFO:tasks.rados.rados.0.smithi093.stdout:425: writing smithi093153278-425 from 726168 to 1431159 tid 1 2023-08-27T20:52:10.459 INFO:tasks.rados.rados.0.smithi093.stdout:425: writing smithi093153278-425 from 1895071 to 2578545 tid 2 2023-08-27T20:52:10.459 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.459 INFO:tasks.rados.rados.0.smithi093.stdout:410: finishing write tid 3 to smithi093153278-410 2023-08-27T20:52:10.459 INFO:tasks.rados.rados.0.smithi093.stdout:410: finishing write tid 4 to smithi093153278-410 2023-08-27T20:52:10.459 INFO:tasks.rados.rados.0.smithi093.stdout:410: finishing write tid 5 to smithi093153278-410 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 410 v 75 (ObjNum 409 snap 0 seq_num 409) dirty exists 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:410: left oid 410 (ObjNum 409 snap 0 seq_num 409) 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:411: finishing write tid 1 to smithi093153278-411 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:411: finishing write tid 2 to smithi093153278-411 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:411: finishing write tid 3 to smithi093153278-411 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:411: finishing write tid 4 to smithi093153278-411 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:411: finishing write tid 5 to smithi093153278-411 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 411 v 105 (ObjNum 410 snap 0 seq_num 410) dirty exists 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:411: left oid 411 (ObjNum 410 snap 0 seq_num 410) 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:412: finishing write tid 1 to smithi093153278-412 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:412: finishing write tid 2 to smithi093153278-412 2023-08-27T20:52:10.460 INFO:tasks.rados.rados.0.smithi093.stdout:412: finishing write tid 3 to smithi093153278-412 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:412: finishing write tid 4 to smithi093153278-412 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:412: finishing write tid 5 to smithi093153278-412 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 412 v 91 (ObjNum 411 snap 0 seq_num 411) dirty exists 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:412: left oid 412 (ObjNum 411 snap 0 seq_num 411) 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:410: done (15 left) 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:411: done (14 left) 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:412: done (13 left) 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:426: write initial oid 426 2023-08-27T20:52:10.461 INFO:tasks.rados.rados.0.smithi093.stdout:426: seq_num 425 ranges {742975=780240,1953553=427132,2971713=171262} 2023-08-27T20:52:10.467 INFO:tasks.rados.rados.0.smithi093.stdout:426: writing smithi093153278-426 from 742975 to 1523215 tid 1 2023-08-27T20:52:10.472 INFO:tasks.rados.rados.0.smithi093.stdout:426: writing smithi093153278-426 from 1953553 to 2380685 tid 2 2023-08-27T20:52:10.476 INFO:tasks.rados.rados.0.smithi093.stdout:426: writing smithi093153278-426 from 2971713 to 3142975 tid 3 2023-08-27T20:52:10.476 INFO:tasks.rados.rados.0.smithi093.stdout:427: write initial oid 427 2023-08-27T20:52:10.476 INFO:tasks.rados.rados.0.smithi093.stdout:427: seq_num 426 ranges {759782=571842,2012035=687143} 2023-08-27T20:52:10.483 INFO:tasks.rados.rados.0.smithi093.stdout:427: writing smithi093153278-427 from 759782 to 1331624 tid 1 2023-08-27T20:52:10.491 INFO:tasks.rados.rados.0.smithi093.stdout:427: writing smithi093153278-427 from 2012035 to 2699178 tid 2 2023-08-27T20:52:10.491 INFO:tasks.rados.rados.0.smithi093.stdout:428: write initial oid 428 2023-08-27T20:52:10.491 INFO:tasks.rados.rados.0.smithi093.stdout:428: seq_num 427 ranges {776589=647091,2070517=430801,3026559=150030} 2023-08-27T20:52:10.500 INFO:tasks.rados.rados.0.smithi093.stdout:428: writing smithi093153278-428 from 776589 to 1423680 tid 1 2023-08-27T20:52:10.506 INFO:tasks.rados.rados.0.smithi093.stdout:428: writing smithi093153278-428 from 2070517 to 2501318 tid 2 2023-08-27T20:52:10.511 INFO:tasks.rados.rados.0.smithi093.stdout:428: writing smithi093153278-428 from 3026559 to 3176589 tid 3 2023-08-27T20:52:10.511 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.511 INFO:tasks.rados.rados.0.smithi093.stdout:413: finishing write tid 1 to smithi093153278-413 2023-08-27T20:52:10.511 INFO:tasks.rados.rados.0.smithi093.stdout:413: finishing write tid 2 to smithi093153278-413 2023-08-27T20:52:10.511 INFO:tasks.rados.rados.0.smithi093.stdout:413: finishing write tid 3 to smithi093153278-413 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:413: finishing write tid 4 to smithi093153278-413 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:413: finishing write tid 5 to smithi093153278-413 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 413 v 79 (ObjNum 412 snap 0 seq_num 412) dirty exists 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:413: left oid 413 (ObjNum 412 snap 0 seq_num 412) 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:414: finishing write tid 1 to smithi093153278-414 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:414: finishing write tid 2 to smithi093153278-414 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:414: finishing write tid 3 to smithi093153278-414 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:413: done (15 left) 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:429: write initial oid 429 2023-08-27T20:52:10.512 INFO:tasks.rados.rados.0.smithi093.stdout:429: seq_num 428 ranges {793396=722340,2128999=690812} 2023-08-27T20:52:10.520 INFO:tasks.rados.rados.0.smithi093.stdout:429: writing smithi093153278-429 from 793396 to 1515736 tid 1 2023-08-27T20:52:10.528 INFO:tasks.rados.rados.0.smithi093.stdout:429: writing smithi093153278-429 from 2128999 to 2819811 tid 2 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:414: finishing write tid 4 to smithi093153278-414 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 414 v 102 (ObjNum 413 snap 0 seq_num 413) dirty exists 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:414: left oid 414 (ObjNum 413 snap 0 seq_num 413) 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:415: finishing write tid 1 to smithi093153278-415 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:414: done (15 left) 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:430: write initial oid 430 2023-08-27T20:52:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:430: seq_num 429 ranges {410203=797589,1787481=434470,2681405=528798} 2023-08-27T20:52:10.535 INFO:tasks.rados.rados.0.smithi093.stdout:430: writing smithi093153278-430 from 410203 to 1207792 tid 1 2023-08-27T20:52:10.541 INFO:tasks.rados.rados.0.smithi093.stdout:430: writing smithi093153278-430 from 1787481 to 2221951 tid 2 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:430: writing smithi093153278-430 from 2681405 to 3210203 tid 3 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:415: finishing write tid 2 to smithi093153278-415 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:415: finishing write tid 3 to smithi093153278-415 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:415: finishing write tid 4 to smithi093153278-415 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 415 v 108 (ObjNum 414 snap 0 seq_num 414) dirty exists 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:415: left oid 415 (ObjNum 414 snap 0 seq_num 414) 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:416: finishing write tid 1 to smithi093153278-416 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:416: finishing write tid 2 to smithi093153278-416 2023-08-27T20:52:10.547 INFO:tasks.rados.rados.0.smithi093.stdout:416: finishing write tid 3 to smithi093153278-416 2023-08-27T20:52:10.548 INFO:tasks.rados.rados.0.smithi093.stdout:415: done (15 left) 2023-08-27T20:52:10.548 INFO:tasks.rados.rados.0.smithi093.stdout:431: write initial oid 431 2023-08-27T20:52:10.548 INFO:tasks.rados.rados.0.smithi093.stdout:431: seq_num 430 ranges {427010=472838,1329610=694481,2708828=518182} 2023-08-27T20:52:10.551 INFO:tasks.rados.rados.0.smithi093.stdout:431: writing smithi093153278-431 from 427010 to 899848 tid 1 2023-08-27T20:52:10.557 INFO:tasks.rados.rados.0.smithi093.stdout:431: writing smithi093153278-431 from 1329610 to 2024091 tid 2 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout:431: writing smithi093153278-431 from 2708828 to 3227010 tid 3 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout:416: finishing write tid 4 to smithi093153278-416 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout:416: finishing write tid 5 to smithi093153278-416 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 416 v 83 (ObjNum 415 snap 0 seq_num 415) dirty exists 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout:416: left oid 416 (ObjNum 415 snap 0 seq_num 415) 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout:417: finishing write tid 1 to smithi093153278-417 2023-08-27T20:52:10.565 INFO:tasks.rados.rados.0.smithi093.stdout:416: done (15 left) 2023-08-27T20:52:10.566 INFO:tasks.rados.rados.0.smithi093.stdout:432: write initial oid 432 2023-08-27T20:52:10.566 INFO:tasks.rados.rados.0.smithi093.stdout:432: seq_num 431 ranges {443817=548087,1788092=438139,3019898=223919} 2023-08-27T20:52:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:432: writing smithi093153278-432 from 443817 to 991904 tid 1 2023-08-27T20:52:10.578 INFO:tasks.rados.rados.0.smithi093.stdout:432: writing smithi093153278-432 from 1788092 to 2226231 tid 2 2023-08-27T20:52:10.585 INFO:tasks.rados.rados.0.smithi093.stdout:432: writing smithi093153278-432 from 3019898 to 3243817 tid 3 2023-08-27T20:52:10.585 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.585 INFO:tasks.rados.rados.0.smithi093.stdout:417: finishing write tid 2 to smithi093153278-417 2023-08-27T20:52:10.585 INFO:tasks.rados.rados.0.smithi093.stdout:417: finishing write tid 3 to smithi093153278-417 2023-08-27T20:52:10.585 INFO:tasks.rados.rados.0.smithi093.stdout:417: finishing write tid 4 to smithi093153278-417 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 417 v 111 (ObjNum 416 snap 0 seq_num 416) dirty exists 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:417: left oid 417 (ObjNum 416 snap 0 seq_num 416) 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:418: finishing write tid 1 to smithi093153278-418 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:418: finishing write tid 2 to smithi093153278-418 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:418: finishing write tid 3 to smithi093153278-418 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:418: finishing write tid 4 to smithi093153278-418 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:418: finishing write tid 5 to smithi093153278-418 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 418 v 98 (ObjNum 417 snap 0 seq_num 417) dirty exists 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:418: left oid 418 (ObjNum 417 snap 0 seq_num 417) 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:419: finishing write tid 1 to smithi093153278-419 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:417: done (15 left) 2023-08-27T20:52:10.586 INFO:tasks.rados.rados.0.smithi093.stdout:418: done (14 left) 2023-08-27T20:52:10.587 INFO:tasks.rados.rados.0.smithi093.stdout:433: write initial oid 433 2023-08-27T20:52:10.587 INFO:tasks.rados.rados.0.smithi093.stdout:433: seq_num 432 ranges {460624=623336,1846574=698150,3163674=96950} 2023-08-27T20:52:10.591 INFO:tasks.rados.rados.0.smithi093.stdout:433: writing smithi093153278-433 from 460624 to 1083960 tid 1 2023-08-27T20:52:10.599 INFO:tasks.rados.rados.0.smithi093.stdout:433: writing smithi093153278-433 from 1846574 to 2544724 tid 2 2023-08-27T20:52:10.603 INFO:tasks.rados.rados.0.smithi093.stdout:433: writing smithi093153278-433 from 3163674 to 3260624 tid 3 2023-08-27T20:52:10.603 INFO:tasks.rados.rados.0.smithi093.stdout:434: write initial oid 434 2023-08-27T20:52:10.603 INFO:tasks.rados.rados.0.smithi093.stdout:434: seq_num 433 ranges {477431=698585,1905056=441808,3074744=202687} 2023-08-27T20:52:10.609 INFO:tasks.rados.rados.0.smithi093.stdout:434: writing smithi093153278-434 from 477431 to 1176016 tid 1 2023-08-27T20:52:10.616 INFO:tasks.rados.rados.0.smithi093.stdout:434: writing smithi093153278-434 from 1905056 to 2346864 tid 2 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:434: writing smithi093153278-434 from 3074744 to 3277431 tid 3 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:419: finishing write tid 2 to smithi093153278-419 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:419: finishing write tid 3 to smithi093153278-419 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:419: finishing write tid 4 to smithi093153278-419 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 419 v 85 (ObjNum 418 snap 0 seq_num 418) dirty exists 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:419: left oid 419 (ObjNum 418 snap 0 seq_num 418) 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:420: finishing write tid 1 to smithi093153278-420 2023-08-27T20:52:10.623 INFO:tasks.rados.rados.0.smithi093.stdout:420: finishing write tid 2 to smithi093153278-420 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:420: finishing write tid 3 to smithi093153278-420 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:420: finishing write tid 4 to smithi093153278-420 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 420 v 114 (ObjNum 419 snap 0 seq_num 419) dirty exists 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:420: left oid 420 (ObjNum 419 snap 0 seq_num 419) 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:421: finishing write tid 1 to smithi093153278-421 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:421: finishing write tid 2 to smithi093153278-421 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:421: finishing write tid 3 to smithi093153278-421 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:421: finishing write tid 4 to smithi093153278-421 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:421: finishing write tid 5 to smithi093153278-421 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 421 v 106 (ObjNum 420 snap 0 seq_num 420) dirty exists 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:421: left oid 421 (ObjNum 420 snap 0 seq_num 420) 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:422: finishing write tid 1 to smithi093153278-422 2023-08-27T20:52:10.624 INFO:tasks.rados.rados.0.smithi093.stdout:422: finishing write tid 2 to smithi093153278-422 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:422: finishing write tid 3 to smithi093153278-422 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:422: finishing write tid 4 to smithi093153278-422 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 422 v 108 (ObjNum 421 snap 0 seq_num 421) dirty exists 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:422: left oid 422 (ObjNum 421 snap 0 seq_num 421) 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:423: finishing write tid 1 to smithi093153278-423 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:423: finishing write tid 2 to smithi093153278-423 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:423: finishing write tid 3 to smithi093153278-423 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:423: finishing write tid 4 to smithi093153278-423 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:423: finishing write tid 5 to smithi093153278-423 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 423 v 75 (ObjNum 422 snap 0 seq_num 422) dirty exists 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:423: left oid 423 (ObjNum 422 snap 0 seq_num 422) 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:424: finishing write tid 1 to smithi093153278-424 2023-08-27T20:52:10.625 INFO:tasks.rados.rados.0.smithi093.stdout:424: finishing write tid 2 to smithi093153278-424 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:424: finishing write tid 3 to smithi093153278-424 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:424: finishing write tid 4 to smithi093153278-424 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:424: finishing write tid 5 to smithi093153278-424 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 424 v 89 (ObjNum 423 snap 0 seq_num 423) dirty exists 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:424: left oid 424 (ObjNum 423 snap 0 seq_num 423) 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:425: finishing write tid 1 to smithi093153278-425 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:419: done (15 left) 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:420: done (14 left) 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:421: done (13 left) 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:422: done (12 left) 2023-08-27T20:52:10.626 INFO:tasks.rados.rados.0.smithi093.stdout:423: done (11 left) 2023-08-27T20:52:10.627 INFO:tasks.rados.rados.0.smithi093.stdout:424: done (10 left) 2023-08-27T20:52:10.627 INFO:tasks.rados.rados.0.smithi093.stdout:435: write initial oid 435 2023-08-27T20:52:10.627 INFO:tasks.rados.rados.0.smithi093.stdout:435: seq_num 434 ranges {494238=490187,1563538=701819,2818520=475718} 2023-08-27T20:52:10.627 INFO:tasks.rados.rados.0.smithi093.stdout:435: writing smithi093153278-435 from 494238 to 984425 tid 1 2023-08-27T20:52:10.634 INFO:tasks.rados.rados.0.smithi093.stdout:435: writing smithi093153278-435 from 1563538 to 2265357 tid 2 2023-08-27T20:52:10.640 INFO:tasks.rados.rados.0.smithi093.stdout:435: writing smithi093153278-435 from 2818520 to 3294238 tid 3 2023-08-27T20:52:10.640 INFO:tasks.rados.rados.0.smithi093.stdout:436: write initial oid 436 2023-08-27T20:52:10.641 INFO:tasks.rados.rados.0.smithi093.stdout:436: seq_num 435 ranges {511045=565436,1622020=445477,2729590=556855} 2023-08-27T20:52:10.646 INFO:tasks.rados.rados.0.smithi093.stdout:436: writing smithi093153278-436 from 511045 to 1076481 tid 1 2023-08-27T20:52:10.651 INFO:tasks.rados.rados.0.smithi093.stdout:436: writing smithi093153278-436 from 1622020 to 2067497 tid 2 2023-08-27T20:52:10.658 INFO:tasks.rados.rados.0.smithi093.stdout:436: writing smithi093153278-436 from 2729590 to 3286445 tid 3 2023-08-27T20:52:10.658 INFO:tasks.rados.rados.0.smithi093.stdout:437: write initial oid 437 2023-08-27T20:52:10.658 INFO:tasks.rados.rados.0.smithi093.stdout:437: seq_num 436 ranges {527852=640685,1680502=589135,2757013=570839} 2023-08-27T20:52:10.665 INFO:tasks.rados.rados.0.smithi093.stdout:437: writing smithi093153278-437 from 527852 to 1168537 tid 1 2023-08-27T20:52:10.671 INFO:tasks.rados.rados.0.smithi093.stdout:437: writing smithi093153278-437 from 1680502 to 2269637 tid 2 2023-08-27T20:52:10.678 INFO:tasks.rados.rados.0.smithi093.stdout:437: writing smithi093153278-437 from 2757013 to 3327852 tid 3 2023-08-27T20:52:10.678 INFO:tasks.rados.rados.0.smithi093.stdout:438: write initial oid 438 2023-08-27T20:52:10.678 INFO:tasks.rados.rados.0.smithi093.stdout:438: seq_num 437 ranges {544659=715934,1738984=449146,2784436=560223} 2023-08-27T20:52:10.685 INFO:tasks.rados.rados.0.smithi093.stdout:438: writing smithi093153278-438 from 544659 to 1260593 tid 1 2023-08-27T20:52:10.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1251398218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:10] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:52:10.691 INFO:tasks.rados.rados.0.smithi093.stdout:438: writing smithi093153278-438 from 1738984 to 2188130 tid 2 2023-08-27T20:52:10.698 INFO:tasks.rados.rados.0.smithi093.stdout:438: writing smithi093153278-438 from 2784436 to 3344659 tid 3 2023-08-27T20:52:10.698 INFO:tasks.rados.rados.0.smithi093.stdout:439: write initial oid 439 2023-08-27T20:52:10.698 INFO:tasks.rados.rados.0.smithi093.stdout:439: seq_num 438 ranges {561466=791183,2081113=592804,3095506=265960} 2023-08-27T20:52:10.705 INFO:tasks.rados.rados.0.smithi093.stdout:439: writing smithi093153278-439 from 561466 to 1352649 tid 1 2023-08-27T20:52:10.712 INFO:tasks.rados.rados.0.smithi093.stdout:439: writing smithi093153278-439 from 2081113 to 2673917 tid 2 2023-08-27T20:52:10.716 INFO:tasks.rados.rados.0.smithi093.stdout:439: writing smithi093153278-439 from 3095506 to 3361466 tid 3 2023-08-27T20:52:10.716 INFO:tasks.rados.rados.0.smithi093.stdout:440: write initial oid 440 2023-08-27T20:52:10.716 INFO:tasks.rados.rados.0.smithi093.stdout:440: seq_num 439 ranges {578273=466432,1739595=452815,2722929=655344} 2023-08-27T20:52:10.722 INFO:tasks.rados.rados.0.smithi093.stdout:440: writing smithi093153278-440 from 578273 to 1044705 tid 1 2023-08-27T20:52:10.728 INFO:tasks.rados.rados.0.smithi093.stdout:440: writing smithi093153278-440 from 1739595 to 2192410 tid 2 2023-08-27T20:52:10.736 INFO:tasks.rados.rados.0.smithi093.stdout:440: writing smithi093153278-440 from 2722929 to 3378273 tid 3 2023-08-27T20:52:10.736 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.736 INFO:tasks.rados.rados.0.smithi093.stdout:425: finishing write tid 2 to smithi093153278-425 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:425: finishing write tid 3 to smithi093153278-425 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:425: finishing write tid 4 to smithi093153278-425 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 425 v 84 (ObjNum 424 snap 0 seq_num 424) dirty exists 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:425: left oid 425 (ObjNum 424 snap 0 seq_num 424) 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:426: finishing write tid 1 to smithi093153278-426 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:426: finishing write tid 2 to smithi093153278-426 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:426: finishing write tid 3 to smithi093153278-426 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:426: finishing write tid 4 to smithi093153278-426 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:426: finishing write tid 5 to smithi093153278-426 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 426 v 87 (ObjNum 425 snap 0 seq_num 425) dirty exists 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:426: left oid 426 (ObjNum 425 snap 0 seq_num 425) 2023-08-27T20:52:10.737 INFO:tasks.rados.rados.0.smithi093.stdout:427: finishing write tid 1 to smithi093153278-427 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:427: finishing write tid 2 to smithi093153278-427 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:427: finishing write tid 3 to smithi093153278-427 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:427: finishing write tid 4 to smithi093153278-427 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 427 v 95 (ObjNum 426 snap 0 seq_num 426) dirty exists 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:427: left oid 427 (ObjNum 426 snap 0 seq_num 426) 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:425: done (15 left) 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:426: done (14 left) 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:427: done (13 left) 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:441: write initial oid 441 2023-08-27T20:52:10.738 INFO:tasks.rados.rados.0.smithi093.stdout:441: seq_num 440 ranges {595080=541681,1798077=596473,3150352=244728} 2023-08-27T20:52:10.741 INFO:tasks.rados.rados.0.smithi093.stdout:441: writing smithi093153278-441 from 595080 to 1136761 tid 1 2023-08-27T20:52:10.748 INFO:tasks.rados.rados.0.smithi093.stdout:441: writing smithi093153278-441 from 1798077 to 2394550 tid 2 2023-08-27T20:52:10.754 INFO:tasks.rados.rados.0.smithi093.stdout:441: writing smithi093153278-441 from 3150352 to 3395080 tid 3 2023-08-27T20:52:10.754 INFO:tasks.rados.rados.0.smithi093.stdout:442: write initial oid 442 2023-08-27T20:52:10.754 INFO:tasks.rados.rados.0.smithi093.stdout:442: seq_num 441 ranges {611887=733283,1972912=456484,2894128=517759} 2023-08-27T20:52:10.761 INFO:tasks.rados.rados.0.smithi093.stdout:442: writing smithi093153278-442 from 611887 to 1345170 tid 1 2023-08-27T20:52:10.767 INFO:tasks.rados.rados.0.smithi093.stdout:442: writing smithi093153278-442 from 1972912 to 2429396 tid 2 2023-08-27T20:52:10.774 INFO:tasks.rados.rados.0.smithi093.stdout:442: writing smithi093153278-442 from 2894128 to 3411887 tid 3 2023-08-27T20:52:10.774 INFO:tasks.rados.rados.0.smithi093.stdout:443: write initial oid 443 2023-08-27T20:52:10.774 INFO:tasks.rados.rados.0.smithi093.stdout:443: seq_num 442 ranges {628694=408532,1515041=600142,2805198=552112} 2023-08-27T20:52:10.780 INFO:tasks.rados.rados.0.smithi093.stdout:443: writing smithi093153278-443 from 628694 to 1037226 tid 1 2023-08-27T20:52:10.787 INFO:tasks.rados.rados.0.smithi093.stdout:443: writing smithi093153278-443 from 1515041 to 2115183 tid 2 2023-08-27T20:52:10.795 INFO:tasks.rados.rados.0.smithi093.stdout:443: writing smithi093153278-443 from 2805198 to 3357310 tid 3 2023-08-27T20:52:10.795 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.795 INFO:tasks.rados.rados.0.smithi093.stdout:428: finishing write tid 1 to smithi093153278-428 2023-08-27T20:52:10.795 INFO:tasks.rados.rados.0.smithi093.stdout:428: finishing write tid 2 to smithi093153278-428 2023-08-27T20:52:10.795 INFO:tasks.rados.rados.0.smithi093.stdout:428: finishing write tid 3 to smithi093153278-428 2023-08-27T20:52:10.795 INFO:tasks.rados.rados.0.smithi093.stdout:428: finishing write tid 4 to smithi093153278-428 2023-08-27T20:52:10.796 INFO:tasks.rados.rados.0.smithi093.stdout:428: finishing write tid 5 to smithi093153278-428 2023-08-27T20:52:10.796 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 428 v 80 (ObjNum 427 snap 0 seq_num 427) dirty exists 2023-08-27T20:52:10.796 INFO:tasks.rados.rados.0.smithi093.stdout:428: left oid 428 (ObjNum 427 snap 0 seq_num 427) 2023-08-27T20:52:10.796 INFO:tasks.rados.rados.0.smithi093.stdout:429: finishing write tid 1 to smithi093153278-429 2023-08-27T20:52:10.796 INFO:tasks.rados.rados.0.smithi093.stdout:428: done (15 left) 2023-08-27T20:52:10.796 INFO:tasks.rados.rados.0.smithi093.stdout:444: write initial oid 444 2023-08-27T20:52:10.796 INFO:tasks.rados.rados.0.smithi093.stdout:444: seq_num 443 ranges {645501=483781,1573523=460153,2548974=763384} 2023-08-27T20:52:10.801 INFO:tasks.rados.rados.0.smithi093.stdout:444: writing smithi093153278-444 from 645501 to 1129282 tid 1 2023-08-27T20:52:10.807 INFO:tasks.rados.rados.0.smithi093.stdout:444: writing smithi093153278-444 from 1573523 to 2033676 tid 2 2023-08-27T20:52:10.814 INFO:tasks.rados.rados.0.smithi093.stdout:444: writing smithi093153278-444 from 2548974 to 3312358 tid 3 2023-08-27T20:52:10.814 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.814 INFO:tasks.rados.rados.0.smithi093.stdout:429: finishing write tid 2 to smithi093153278-429 2023-08-27T20:52:10.814 INFO:tasks.rados.rados.0.smithi093.stdout:429: finishing write tid 3 to smithi093153278-429 2023-08-27T20:52:10.814 INFO:tasks.rados.rados.0.smithi093.stdout:429: finishing write tid 4 to smithi093153278-429 2023-08-27T20:52:10.814 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 429 v 117 (ObjNum 428 snap 0 seq_num 428) dirty exists 2023-08-27T20:52:10.815 INFO:tasks.rados.rados.0.smithi093.stdout:429: left oid 429 (ObjNum 428 snap 0 seq_num 428) 2023-08-27T20:52:10.815 INFO:tasks.rados.rados.0.smithi093.stdout:430: finishing write tid 1 to smithi093153278-430 2023-08-27T20:52:10.815 INFO:tasks.rados.rados.0.smithi093.stdout:430: finishing write tid 2 to smithi093153278-430 2023-08-27T20:52:10.815 INFO:tasks.rados.rados.0.smithi093.stdout:429: done (15 left) 2023-08-27T20:52:10.815 INFO:tasks.rados.rados.0.smithi093.stdout:445: write initial oid 445 2023-08-27T20:52:10.815 INFO:tasks.rados.rados.0.smithi093.stdout:445: seq_num 444 ranges {662308=559030,1632005=603811,2860044=602264} 2023-08-27T20:52:10.821 INFO:tasks.rados.rados.0.smithi093.stdout:445: writing smithi093153278-445 from 662308 to 1221338 tid 1 2023-08-27T20:52:10.828 INFO:tasks.rados.rados.0.smithi093.stdout:445: writing smithi093153278-445 from 1632005 to 2235816 tid 2 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout:445: writing smithi093153278-445 from 2860044 to 3462308 tid 3 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout:430: finishing write tid 3 to smithi093153278-430 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout:430: finishing write tid 4 to smithi093153278-430 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout:430: finishing write tid 5 to smithi093153278-430 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 430 v 107 (ObjNum 429 snap 0 seq_num 429) dirty exists 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout:430: left oid 430 (ObjNum 429 snap 0 seq_num 429) 2023-08-27T20:52:10.835 INFO:tasks.rados.rados.0.smithi093.stdout:431: finishing write tid 1 to smithi093153278-431 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:431: finishing write tid 2 to smithi093153278-431 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:431: finishing write tid 3 to smithi093153278-431 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:431: finishing write tid 4 to smithi093153278-431 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:431: finishing write tid 5 to smithi093153278-431 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 431 v 94 (ObjNum 430 snap 0 seq_num 430) dirty exists 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:431: left oid 431 (ObjNum 430 snap 0 seq_num 430) 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:432: finishing write tid 1 to smithi093153278-432 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:430: done (15 left) 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:431: done (14 left) 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:446: write initial oid 446 2023-08-27T20:52:10.836 INFO:tasks.rados.rados.0.smithi093.stdout:446: seq_num 445 ranges {679115=634279,2090487=463822,3003820=475295} 2023-08-27T20:52:10.842 INFO:tasks.rados.rados.0.smithi093.stdout:446: writing smithi093153278-446 from 679115 to 1313394 tid 1 2023-08-27T20:52:10.848 INFO:tasks.rados.rados.0.smithi093.stdout:446: writing smithi093153278-446 from 2090487 to 2554309 tid 2 2023-08-27T20:52:10.854 INFO:tasks.rados.rados.0.smithi093.stdout:446: writing smithi093153278-446 from 3003820 to 3479115 tid 3 2023-08-27T20:52:10.854 INFO:tasks.rados.rados.0.smithi093.stdout:447: write initial oid 447 2023-08-27T20:52:10.854 INFO:tasks.rados.rados.0.smithi093.stdout:447: seq_num 446 ranges {695922=709528,2032616=607480,3198537=297385} 2023-08-27T20:52:10.862 INFO:tasks.rados.rados.0.smithi093.stdout:447: writing smithi093153278-447 from 695922 to 1405450 tid 1 2023-08-27T20:52:10.870 INFO:tasks.rados.rados.0.smithi093.stdout:447: writing smithi093153278-447 from 2032616 to 2640096 tid 2 2023-08-27T20:52:10.875 INFO:tasks.rados.rados.0.smithi093.stdout:447: writing smithi093153278-447 from 3198537 to 3495922 tid 3 2023-08-27T20:52:10.875 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:432: finishing write tid 2 to smithi093153278-432 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:432: finishing write tid 3 to smithi093153278-432 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:432: finishing write tid 4 to smithi093153278-432 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:432: finishing write tid 5 to smithi093153278-432 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 432 v 94 (ObjNum 431 snap 0 seq_num 431) dirty exists 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:432: left oid 432 (ObjNum 431 snap 0 seq_num 431) 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:433: finishing write tid 1 to smithi093153278-433 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:432: done (15 left) 2023-08-27T20:52:10.876 INFO:tasks.rados.rados.0.smithi093.stdout:448: write initial oid 448 2023-08-27T20:52:10.877 INFO:tasks.rados.rados.0.smithi093.stdout:448: seq_num 447 ranges {712729=784777,2091098=467491,3342313=170416} 2023-08-27T20:52:10.883 INFO:tasks.rados.rados.0.smithi093.stdout:448: writing smithi093153278-448 from 712729 to 1497506 tid 1 2023-08-27T20:52:10.890 INFO:tasks.rados.rados.0.smithi093.stdout:448: writing smithi093153278-448 from 2091098 to 2558589 tid 2 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout:448: writing smithi093153278-448 from 3342313 to 3512729 tid 3 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout:433: finishing write tid 2 to smithi093153278-433 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout:433: finishing write tid 3 to smithi093153278-433 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout:433: finishing write tid 4 to smithi093153278-433 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout:433: finishing write tid 5 to smithi093153278-433 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 433 v 121 (ObjNum 432 snap 0 seq_num 432) dirty exists 2023-08-27T20:52:10.896 INFO:tasks.rados.rados.0.smithi093.stdout:433: left oid 433 (ObjNum 432 snap 0 seq_num 432) 2023-08-27T20:52:10.897 INFO:tasks.rados.rados.0.smithi093.stdout:434: finishing write tid 1 to smithi093153278-434 2023-08-27T20:52:10.897 INFO:tasks.rados.rados.0.smithi093.stdout:433: done (15 left) 2023-08-27T20:52:10.897 INFO:tasks.rados.rados.0.smithi093.stdout:449: write initial oid 449 2023-08-27T20:52:10.897 INFO:tasks.rados.rados.0.smithi093.stdout:449: seq_num 448 ranges {729536=460026,1749580=611149,2853383=452450} 2023-08-27T20:52:10.902 INFO:tasks.rados.rados.0.smithi093.stdout:449: writing smithi093153278-449 from 729536 to 1189562 tid 1 2023-08-27T20:52:10.908 INFO:tasks.rados.rados.0.smithi093.stdout:449: writing smithi093153278-449 from 1749580 to 2360729 tid 2 2023-08-27T20:52:10.914 INFO:tasks.rados.rados.0.smithi093.stdout:449: writing smithi093153278-449 from 2853383 to 3305833 tid 3 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:434: finishing write tid 2 to smithi093153278-434 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:434: finishing write tid 3 to smithi093153278-434 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:434: finishing write tid 4 to smithi093153278-434 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:434: finishing write tid 5 to smithi093153278-434 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 99 (ObjNum 433 snap 0 seq_num 433) dirty exists 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:434: left oid 434 (ObjNum 433 snap 0 seq_num 433) 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:435: finishing write tid 1 to smithi093153278-435 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:435: finishing write tid 2 to smithi093153278-435 2023-08-27T20:52:10.915 INFO:tasks.rados.rados.0.smithi093.stdout:435: finishing write tid 3 to smithi093153278-435 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:435: finishing write tid 4 to smithi093153278-435 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:435: finishing write tid 5 to smithi093153278-435 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 435 v 102 (ObjNum 434 snap 0 seq_num 434) dirty exists 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:435: left oid 435 (ObjNum 434 snap 0 seq_num 434) 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:436: finishing write tid 1 to smithi093153278-436 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:436: finishing write tid 2 to smithi093153278-436 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:434: done (15 left) 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:435: done (14 left) 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:450: write initial oid 450 2023-08-27T20:52:10.916 INFO:tasks.rados.rados.0.smithi093.stdout:436: finishing write tid 3 to smithi093153278-436 2023-08-27T20:52:10.917 INFO:tasks.rados.rados.0.smithi093.stdout:436: finishing write tid 4 to smithi093153278-436 2023-08-27T20:52:10.917 INFO:tasks.rados.rados.0.smithi093.stdout:436: finishing write tid 5 to smithi093153278-436 2023-08-27T20:52:10.917 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 436 v 79 (ObjNum 435 snap 0 seq_num 435) dirty exists 2023-08-27T20:52:10.917 INFO:tasks.rados.rados.0.smithi093.stdout:436: left oid 436 (ObjNum 435 snap 0 seq_num 435) 2023-08-27T20:52:10.917 INFO:tasks.rados.rados.0.smithi093.stdout:437: finishing write tid 1 to smithi093153278-437 2023-08-27T20:52:10.917 INFO:tasks.rados.rados.0.smithi093.stdout:450: seq_num 449 ranges {746343=651628,1924415=754807,3397159=149184} 2023-08-27T20:52:10.921 INFO:tasks.rados.rados.0.smithi093.stdout:450: writing smithi093153278-450 from 746343 to 1397971 tid 1 2023-08-27T20:52:10.928 INFO:tasks.rados.rados.0.smithi093.stdout:450: writing smithi093153278-450 from 1924415 to 2679222 tid 2 2023-08-27T20:52:10.933 INFO:tasks.rados.rados.0.smithi093.stdout:450: writing smithi093153278-450 from 3397159 to 3546343 tid 3 2023-08-27T20:52:10.934 INFO:tasks.rados.rados.0.smithi093.stdout:436: done (14 left) 2023-08-27T20:52:10.934 INFO:tasks.rados.rados.0.smithi093.stdout:451: write initial oid 451 2023-08-27T20:52:10.934 INFO:tasks.rados.rados.0.smithi093.stdout:451: seq_num 450 ranges {763150=726877,1982897=614818,3024582=474994} 2023-08-27T20:52:10.941 INFO:tasks.rados.rados.0.smithi093.stdout:451: writing smithi093153278-451 from 763150 to 1490027 tid 1 2023-08-27T20:52:10.947 INFO:tasks.rados.rados.0.smithi093.stdout:451: writing smithi093153278-451 from 1982897 to 2597715 tid 2 2023-08-27T20:52:10.952 INFO:tasks.rados.rados.0.smithi093.stdout:451: writing smithi093153278-451 from 3024582 to 3499576 tid 3 2023-08-27T20:52:10.952 INFO:tasks.rados.rados.0.smithi093.stdout:452: write initial oid 452 2023-08-27T20:52:10.952 INFO:tasks.rados.rados.0.smithi093.stdout:452: seq_num 451 ranges {779957=402126,1925026=758476,3335652=244305} 2023-08-27T20:52:10.958 INFO:tasks.rados.rados.0.smithi093.stdout:452: writing smithi093153278-452 from 779957 to 1182083 tid 1 2023-08-27T20:52:10.967 INFO:tasks.rados.rados.0.smithi093.stdout:452: writing smithi093153278-452 from 1925026 to 2683502 tid 2 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:452: writing smithi093153278-452 from 3335652 to 3579957 tid 3 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:437: finishing write tid 2 to smithi093153278-437 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:437: finishing write tid 3 to smithi093153278-437 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:437: finishing write tid 4 to smithi093153278-437 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:437: finishing write tid 5 to smithi093153278-437 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 437 v 110 (ObjNum 436 snap 0 seq_num 436) dirty exists 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:437: left oid 437 (ObjNum 436 snap 0 seq_num 436) 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:438: finishing write tid 1 to smithi093153278-438 2023-08-27T20:52:10.973 INFO:tasks.rados.rados.0.smithi093.stdout:438: finishing write tid 2 to smithi093153278-438 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:438: finishing write tid 3 to smithi093153278-438 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:438: finishing write tid 4 to smithi093153278-438 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:438: finishing write tid 5 to smithi093153278-438 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 438 v 84 (ObjNum 437 snap 0 seq_num 437) dirty exists 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:438: left oid 438 (ObjNum 437 snap 0 seq_num 437) 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:439: finishing write tid 1 to smithi093153278-439 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:439: finishing write tid 2 to smithi093153278-439 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:439: finishing write tid 3 to smithi093153278-439 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:439: finishing write tid 4 to smithi093153278-439 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:439: finishing write tid 5 to smithi093153278-439 2023-08-27T20:52:10.974 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 439 v 87 (ObjNum 438 snap 0 seq_num 438) dirty exists 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:439: left oid 439 (ObjNum 438 snap 0 seq_num 438) 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:440: finishing write tid 1 to smithi093153278-440 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:440: finishing write tid 2 to smithi093153278-440 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:440: finishing write tid 3 to smithi093153278-440 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:440: finishing write tid 4 to smithi093153278-440 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:440: finishing write tid 5 to smithi093153278-440 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 440 v 88 (ObjNum 439 snap 0 seq_num 439) dirty exists 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:440: left oid 440 (ObjNum 439 snap 0 seq_num 439) 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:441: finishing write tid 1 to smithi093153278-441 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:441: finishing write tid 2 to smithi093153278-441 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:441: finishing write tid 3 to smithi093153278-441 2023-08-27T20:52:10.975 INFO:tasks.rados.rados.0.smithi093.stdout:441: finishing write tid 4 to smithi093153278-441 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:441: finishing write tid 5 to smithi093153278-441 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 88 (ObjNum 440 snap 0 seq_num 440) dirty exists 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:441: left oid 441 (ObjNum 440 snap 0 seq_num 440) 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:442: finishing write tid 1 to smithi093153278-442 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:442: finishing write tid 2 to smithi093153278-442 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:442: finishing write tid 3 to smithi093153278-442 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:442: finishing write tid 4 to smithi093153278-442 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:442: finishing write tid 5 to smithi093153278-442 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 442 v 125 (ObjNum 441 snap 0 seq_num 441) dirty exists 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:442: left oid 442 (ObjNum 441 snap 0 seq_num 441) 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:443: finishing write tid 1 to smithi093153278-443 2023-08-27T20:52:10.976 INFO:tasks.rados.rados.0.smithi093.stdout:443: finishing write tid 2 to smithi093153278-443 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:437: done (15 left) 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:438: done (14 left) 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:439: done (13 left) 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:440: done (12 left) 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:441: done (11 left) 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:442: done (10 left) 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:453: write initial oid 453 2023-08-27T20:52:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:453: seq_num 452 ranges {796764=477375,1983508=618487,3363075=233689} 2023-08-27T20:52:10.979 INFO:tasks.rados.rados.0.smithi093.stdout:453: writing smithi093153278-453 from 796764 to 1274139 tid 1 2023-08-27T20:52:10.988 INFO:tasks.rados.rados.0.smithi093.stdout:453: writing smithi093153278-453 from 1983508 to 2601995 tid 2 2023-08-27T20:52:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1251398218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1251398218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:10.994 INFO:tasks.rados.rados.0.smithi093.stdout:453: writing smithi093153278-453 from 3363075 to 3596764 tid 3 2023-08-27T20:52:10.994 INFO:tasks.rados.rados.0.smithi093.stdout:454: write initial oid 454 2023-08-27T20:52:10.994 INFO:tasks.rados.rados.0.smithi093.stdout:454: seq_num 453 ranges {413571=552624,1641990=762145,2990498=425163} 2023-08-27T20:52:11.002 INFO:tasks.rados.rados.0.smithi093.stdout:454: writing smithi093153278-454 from 413571 to 966195 tid 1 2023-08-27T20:52:11.009 INFO:tasks.rados.rados.0.smithi093.stdout:454: writing smithi093153278-454 from 1641990 to 2404135 tid 2 2023-08-27T20:52:11.015 INFO:tasks.rados.rados.0.smithi093.stdout:454: writing smithi093153278-454 from 2990498 to 3415661 tid 3 2023-08-27T20:52:11.015 INFO:tasks.rados.rados.0.smithi093.stdout:455: write initial oid 455 2023-08-27T20:52:11.015 INFO:tasks.rados.rados.0.smithi093.stdout:443: finishing write tid 3 to smithi093153278-443 2023-08-27T20:52:11.015 INFO:tasks.rados.rados.0.smithi093.stdout:443: finishing write tid 4 to smithi093153278-443 2023-08-27T20:52:11.016 INFO:tasks.rados.rados.0.smithi093.stdout:455: seq_num 454 ranges {430378=627873,1700472=622156,3017921=612457} 2023-08-27T20:52:11.021 INFO:tasks.rados.rados.0.smithi093.stdout:455: writing smithi093153278-455 from 430378 to 1058251 tid 1 2023-08-27T20:52:11.029 INFO:tasks.rados.rados.0.smithi093.stdout:455: writing smithi093153278-455 from 1700472 to 2322628 tid 2 2023-08-27T20:52:11.036 INFO:tasks.rados.rados.0.smithi093.stdout:455: writing smithi093153278-455 from 3017921 to 3630378 tid 3 2023-08-27T20:52:11.037 INFO:tasks.rados.rados.0.smithi093.stdout:456: write initial oid 456 2023-08-27T20:52:11.037 INFO:tasks.rados.rados.0.smithi093.stdout:456: seq_num 455 ranges {447185=703122,1642601=765814,2928991=447707} 2023-08-27T20:52:11.043 INFO:tasks.rados.rados.0.smithi093.stdout:456: writing smithi093153278-456 from 447185 to 1150307 tid 1 2023-08-27T20:52:11.050 INFO:tasks.rados.rados.0.smithi093.stdout:456: writing smithi093153278-456 from 1642601 to 2408415 tid 2 2023-08-27T20:52:11.055 INFO:tasks.rados.rados.0.smithi093.stdout:456: writing smithi093153278-456 from 2928991 to 3376698 tid 3 2023-08-27T20:52:11.055 INFO:tasks.rados.rados.0.smithi093.stdout:457: write initial oid 457 2023-08-27T20:52:11.056 INFO:tasks.rados.rados.0.smithi093.stdout:457: seq_num 456 ranges {463992=778371,1701083=625825,2956414=658979} 2023-08-27T20:52:11.063 INFO:tasks.rados.rados.0.smithi093.stdout:457: writing smithi093153278-457 from 463992 to 1242363 tid 1 2023-08-27T20:52:11.069 INFO:tasks.rados.rados.0.smithi093.stdout:457: writing smithi093153278-457 from 1701083 to 2326908 tid 2 2023-08-27T20:52:11.078 INFO:tasks.rados.rados.0.smithi093.stdout:457: writing smithi093153278-457 from 2956414 to 3615393 tid 3 2023-08-27T20:52:11.078 INFO:tasks.rados.rados.0.smithi093.stdout:458: write initial oid 458 2023-08-27T20:52:11.078 INFO:tasks.rados.rados.0.smithi093.stdout:458: seq_num 457 ranges {480799=569973,1475918=769483,2700190=586604} 2023-08-27T20:52:11.084 INFO:tasks.rados.rados.0.smithi093.stdout:458: writing smithi093153278-458 from 480799 to 1050772 tid 1 2023-08-27T20:52:11.091 INFO:tasks.rados.rados.0.smithi093.stdout:458: writing smithi093153278-458 from 1475918 to 2245401 tid 2 2023-08-27T20:52:11.097 INFO:tasks.rados.rados.0.smithi093.stdout:458: writing smithi093153278-458 from 2700190 to 3286794 tid 3 2023-08-27T20:52:11.097 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.097 INFO:tasks.rados.rados.0.smithi093.stdout:443: finishing write tid 5 to smithi093153278-443 2023-08-27T20:52:11.097 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 129 (ObjNum 442 snap 0 seq_num 442) dirty exists 2023-08-27T20:52:11.098 INFO:tasks.rados.rados.0.smithi093.stdout:443: left oid 443 (ObjNum 442 snap 0 seq_num 442) 2023-08-27T20:52:11.098 INFO:tasks.rados.rados.0.smithi093.stdout:444: finishing write tid 1 to smithi093153278-444 2023-08-27T20:52:11.098 INFO:tasks.rados.rados.0.smithi093.stdout:444: finishing write tid 2 to smithi093153278-444 2023-08-27T20:52:11.098 INFO:tasks.rados.rados.0.smithi093.stdout:443: done (15 left) 2023-08-27T20:52:11.098 INFO:tasks.rados.rados.0.smithi093.stdout:459: write initial oid 459 2023-08-27T20:52:11.098 INFO:tasks.rados.rados.0.smithi093.stdout:459: seq_num 458 ranges {497606=645222,1934400=629494,3243966=453640} 2023-08-27T20:52:11.103 INFO:tasks.rados.rados.0.smithi093.stdout:459: writing smithi093153278-459 from 497606 to 1142828 tid 1 2023-08-27T20:52:11.111 INFO:tasks.rados.rados.0.smithi093.stdout:459: writing smithi093153278-459 from 1934400 to 2563894 tid 2 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:459: writing smithi093153278-459 from 3243966 to 3697606 tid 3 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:444: finishing write tid 3 to smithi093153278-444 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:444: finishing write tid 4 to smithi093153278-444 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:444: finishing write tid 5 to smithi093153278-444 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 444 v 91 (ObjNum 443 snap 0 seq_num 443) dirty exists 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:444: left oid 444 (ObjNum 443 snap 0 seq_num 443) 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:445: finishing write tid 1 to smithi093153278-445 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:445: finishing write tid 2 to smithi093153278-445 2023-08-27T20:52:11.120 INFO:tasks.rados.rados.0.smithi093.stdout:445: finishing write tid 3 to smithi093153278-445 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:445: finishing write tid 4 to smithi093153278-445 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:445: finishing write tid 5 to smithi093153278-445 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 445 v 111 (ObjNum 444 snap 0 seq_num 444) dirty exists 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:445: left oid 445 (ObjNum 444 snap 0 seq_num 444) 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:446: finishing write tid 1 to smithi093153278-446 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:446: finishing write tid 2 to smithi093153278-446 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:446: finishing write tid 3 to smithi093153278-446 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:446: finishing write tid 4 to smithi093153278-446 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:446: finishing write tid 5 to smithi093153278-446 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 446 v 91 (ObjNum 445 snap 0 seq_num 445) dirty exists 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:446: left oid 446 (ObjNum 445 snap 0 seq_num 445) 2023-08-27T20:52:11.121 INFO:tasks.rados.rados.0.smithi093.stdout:447: finishing write tid 1 to smithi093153278-447 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:447: finishing write tid 2 to smithi093153278-447 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:447: finishing write tid 3 to smithi093153278-447 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:447: finishing write tid 4 to smithi093153278-447 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:447: finishing write tid 5 to smithi093153278-447 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 447 v 93 (ObjNum 446 snap 0 seq_num 446) dirty exists 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:447: left oid 447 (ObjNum 446 snap 0 seq_num 446) 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:448: finishing write tid 1 to smithi093153278-448 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:448: finishing write tid 2 to smithi093153278-448 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:448: finishing write tid 3 to smithi093153278-448 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:448: finishing write tid 4 to smithi093153278-448 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:448: finishing write tid 5 to smithi093153278-448 2023-08-27T20:52:11.122 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 448 v 98 (ObjNum 447 snap 0 seq_num 447) dirty exists 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:448: left oid 448 (ObjNum 447 snap 0 seq_num 447) 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:449: finishing write tid 1 to smithi093153278-449 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:449: finishing write tid 2 to smithi093153278-449 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:449: finishing write tid 3 to smithi093153278-449 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:449: finishing write tid 4 to smithi093153278-449 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:449: finishing write tid 5 to smithi093153278-449 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 97 (ObjNum 448 snap 0 seq_num 448) dirty exists 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:449: left oid 449 (ObjNum 448 snap 0 seq_num 448) 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:450: finishing write tid 1 to smithi093153278-450 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:450: finishing write tid 2 to smithi093153278-450 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:450: finishing write tid 3 to smithi093153278-450 2023-08-27T20:52:11.123 INFO:tasks.rados.rados.0.smithi093.stdout:450: finishing write tid 4 to smithi093153278-450 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:450: finishing write tid 5 to smithi093153278-450 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 450 v 98 (ObjNum 449 snap 0 seq_num 449) dirty exists 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:450: left oid 450 (ObjNum 449 snap 0 seq_num 449) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:451: finishing write tid 1 to smithi093153278-451 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:444: done (15 left) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:445: done (14 left) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:446: done (13 left) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:447: done (12 left) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:448: done (11 left) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:449: done (10 left) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:450: done (9 left) 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:460: write initial oid 460 2023-08-27T20:52:11.124 INFO:tasks.rados.rados.0.smithi093.stdout:460: seq_num 459 ranges {514413=720471,1876529=773152,3438683=275730} 2023-08-27T20:52:11.125 INFO:tasks.rados.rados.0.smithi093.stdout:460: writing smithi093153278-460 from 514413 to 1234884 tid 1 2023-08-27T20:52:11.133 INFO:tasks.rados.rados.0.smithi093.stdout:460: writing smithi093153278-460 from 1876529 to 2649681 tid 2 2023-08-27T20:52:11.139 INFO:tasks.rados.rados.0.smithi093.stdout:460: writing smithi093153278-460 from 3438683 to 3714413 tid 3 2023-08-27T20:52:11.139 INFO:tasks.rados.rados.0.smithi093.stdout:461: write initial oid 461 2023-08-27T20:52:11.139 INFO:tasks.rados.rados.0.smithi093.stdout:461: seq_num 460 ranges {531220=795720,1935011=516810,3066106=420420} 2023-08-27T20:52:11.146 INFO:tasks.rados.rados.0.smithi093.stdout:461: writing smithi093153278-461 from 531220 to 1326940 tid 1 2023-08-27T20:52:11.152 INFO:tasks.rados.rados.0.smithi093.stdout:461: writing smithi093153278-461 from 1935011 to 2451821 tid 2 2023-08-27T20:52:11.158 INFO:tasks.rados.rados.0.smithi093.stdout:461: writing smithi093153278-461 from 3066106 to 3486526 tid 3 2023-08-27T20:52:11.159 INFO:tasks.rados.rados.0.smithi093.stdout:462: write initial oid 462 2023-08-27T20:52:11.159 INFO:tasks.rados.rados.0.smithi093.stdout:462: seq_num 461 ranges {548027=470969,1593493=776821,3093529=631692} 2023-08-27T20:52:11.164 INFO:tasks.rados.rados.0.smithi093.stdout:462: writing smithi093153278-462 from 548027 to 1018996 tid 1 2023-08-27T20:52:11.172 INFO:tasks.rados.rados.0.smithi093.stdout:462: writing smithi093153278-462 from 1593493 to 2370314 tid 2 2023-08-27T20:52:11.180 INFO:tasks.rados.rados.0.smithi093.stdout:462: writing smithi093153278-462 from 3093529 to 3725221 tid 3 2023-08-27T20:52:11.180 INFO:tasks.rados.rados.0.smithi093.stdout:463: write initial oid 463 2023-08-27T20:52:11.180 INFO:tasks.rados.rados.0.smithi093.stdout:463: seq_num 462 ranges {564834=546218,1651975=520479,2720952=559317} 2023-08-27T20:52:11.186 INFO:tasks.rados.rados.0.smithi093.stdout:463: writing smithi093153278-463 from 564834 to 1111052 tid 1 2023-08-27T20:52:11.192 INFO:tasks.rados.rados.0.smithi093.stdout:463: writing smithi093153278-463 from 1651975 to 2172454 tid 2 2023-08-27T20:52:11.198 INFO:tasks.rados.rados.0.smithi093.stdout:463: writing smithi093153278-463 from 2720952 to 3280269 tid 3 2023-08-27T20:52:11.198 INFO:tasks.rados.rados.0.smithi093.stdout:464: write initial oid 464 2023-08-27T20:52:11.198 INFO:tasks.rados.rados.0.smithi093.stdout:464: seq_num 463 ranges {581641=621467,1994104=780490,3432022=349619} 2023-08-27T20:52:11.205 INFO:tasks.rados.rados.0.smithi093.stdout:464: writing smithi093153278-464 from 581641 to 1203108 tid 1 2023-08-27T20:52:11.215 INFO:tasks.rados.rados.0.smithi093.stdout:464: writing smithi093153278-464 from 1994104 to 2774594 tid 2 2023-08-27T20:52:11.222 INFO:tasks.rados.rados.0.smithi093.stdout:464: writing smithi093153278-464 from 3432022 to 3781641 tid 3 2023-08-27T20:52:11.222 INFO:tasks.rados.rados.0.smithi093.stdout:465: write initial oid 465 2023-08-27T20:52:11.222 INFO:tasks.rados.rados.0.smithi093.stdout:465: seq_num 464 ranges {598448=413069,1768939=524148,2775798=581861,3781488=16960} 2023-08-27T20:52:11.227 INFO:tasks.rados.rados.0.smithi093.stdout:465: writing smithi093153278-465 from 598448 to 1011517 tid 1 2023-08-27T20:52:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:465: writing smithi093153278-465 from 1768939 to 2293087 tid 2 2023-08-27T20:52:11.240 INFO:tasks.rados.rados.0.smithi093.stdout:465: writing smithi093153278-465 from 2775798 to 3357659 tid 3 2023-08-27T20:52:11.243 INFO:tasks.rados.rados.0.smithi093.stdout:465: writing smithi093153278-465 from 3781488 to 3798448 tid 4 2023-08-27T20:52:11.243 INFO:tasks.rados.rados.0.smithi093.stdout:466: write initial oid 466 2023-08-27T20:52:11.243 INFO:tasks.rados.rados.0.smithi093.stdout:466: seq_num 465 ranges {615255=488318,1827421=784159,3203221=612034} 2023-08-27T20:52:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:466: writing smithi093153278-466 from 615255 to 1103573 tid 1 2023-08-27T20:52:11.257 INFO:tasks.rados.rados.0.smithi093.stdout:466: writing smithi093153278-466 from 1827421 to 2611580 tid 2 2023-08-27T20:52:11.264 INFO:tasks.rados.rados.0.smithi093.stdout:466: writing smithi093153278-466 from 3203221 to 3815255 tid 3 2023-08-27T20:52:11.264 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.264 INFO:tasks.rados.rados.0.smithi093.stdout:451: finishing write tid 2 to smithi093153278-451 2023-08-27T20:52:11.264 INFO:tasks.rados.rados.0.smithi093.stdout:451: finishing write tid 3 to smithi093153278-451 2023-08-27T20:52:11.264 INFO:tasks.rados.rados.0.smithi093.stdout:451: finishing write tid 4 to smithi093153278-451 2023-08-27T20:52:11.264 INFO:tasks.rados.rados.0.smithi093.stdout:451: finishing write tid 5 to smithi093153278-451 2023-08-27T20:52:11.264 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 451 v 92 (ObjNum 450 snap 0 seq_num 450) dirty exists 2023-08-27T20:52:11.265 INFO:tasks.rados.rados.0.smithi093.stdout:451: left oid 451 (ObjNum 450 snap 0 seq_num 450) 2023-08-27T20:52:11.265 INFO:tasks.rados.rados.0.smithi093.stdout:451: done (15 left) 2023-08-27T20:52:11.265 INFO:tasks.rados.rados.0.smithi093.stdout:467: write initial oid 467 2023-08-27T20:52:11.265 INFO:tasks.rados.rados.0.smithi093.stdout:467: seq_num 466 ranges {632062=563567,1885903=527817,2830644=720758} 2023-08-27T20:52:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:467: writing smithi093153278-467 from 632062 to 1195629 tid 1 2023-08-27T20:52:11.277 INFO:tasks.rados.rados.0.smithi093.stdout:467: writing smithi093153278-467 from 1885903 to 2413720 tid 2 2023-08-27T20:52:11.283 INFO:tasks.rados.rados.0.smithi093.stdout:467: writing smithi093153278-467 from 2830644 to 3551402 tid 3 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:452: finishing write tid 1 to smithi093153278-452 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:452: finishing write tid 2 to smithi093153278-452 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:452: finishing write tid 3 to smithi093153278-452 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:452: finishing write tid 4 to smithi093153278-452 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:452: finishing write tid 5 to smithi093153278-452 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 452 v 106 (ObjNum 451 snap 0 seq_num 451) dirty exists 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:452: left oid 452 (ObjNum 451 snap 0 seq_num 451) 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:453: finishing write tid 1 to smithi093153278-453 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:452: done (15 left) 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:468: write initial oid 468 2023-08-27T20:52:11.284 INFO:tasks.rados.rados.0.smithi093.stdout:468: seq_num 467 ranges {648869=638816,1828032=787828,3141714=532030} 2023-08-27T20:52:11.290 INFO:tasks.rados.rados.0.smithi093.stdout:468: writing smithi093153278-468 from 648869 to 1287685 tid 1 2023-08-27T20:52:11.299 INFO:tasks.rados.rados.0.smithi093.stdout:468: writing smithi093153278-468 from 1828032 to 2615860 tid 2 2023-08-27T20:52:11.305 INFO:tasks.rados.rados.0.smithi093.stdout:468: writing smithi093153278-468 from 3141714 to 3673744 tid 3 2023-08-27T20:52:11.305 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.305 INFO:tasks.rados.rados.0.smithi093.stdout:453: finishing write tid 2 to smithi093153278-453 2023-08-27T20:52:11.305 INFO:tasks.rados.rados.0.smithi093.stdout:453: finishing write tid 3 to smithi093153278-453 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:453: finishing write tid 4 to smithi093153278-453 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:453: finishing write tid 5 to smithi093153278-453 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 453 v 95 (ObjNum 452 snap 0 seq_num 452) dirty exists 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:453: left oid 453 (ObjNum 452 snap 0 seq_num 452) 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:454: finishing write tid 1 to smithi093153278-454 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:453: done (15 left) 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:469: write initial oid 469 2023-08-27T20:52:11.306 INFO:tasks.rados.rados.0.smithi093.stdout:469: seq_num 468 ranges {665676=714065,1886514=531486,3169137=696539} 2023-08-27T20:52:11.312 INFO:tasks.rados.rados.0.smithi093.stdout:469: writing smithi093153278-469 from 665676 to 1379741 tid 1 2023-08-27T20:52:11.318 INFO:tasks.rados.rados.0.smithi093.stdout:469: writing smithi093153278-469 from 1886514 to 2418000 tid 2 2023-08-27T20:52:11.326 INFO:tasks.rados.rados.0.smithi093.stdout:469: writing smithi093153278-469 from 3169137 to 3865676 tid 3 2023-08-27T20:52:11.326 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.326 INFO:tasks.rados.rados.0.smithi093.stdout:454: finishing write tid 2 to smithi093153278-454 2023-08-27T20:52:11.326 INFO:tasks.rados.rados.0.smithi093.stdout:454: finishing write tid 3 to smithi093153278-454 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:454: finishing write tid 4 to smithi093153278-454 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:454: finishing write tid 5 to smithi093153278-454 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 454 v 133 (ObjNum 453 snap 0 seq_num 453) dirty exists 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:454: left oid 454 (ObjNum 453 snap 0 seq_num 453) 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:455: finishing write tid 1 to smithi093153278-455 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:454: done (15 left) 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:470: write initial oid 470 2023-08-27T20:52:11.327 INFO:tasks.rados.rados.0.smithi093.stdout:470: seq_num 469 ranges {682483=789314,1944996=791497,3196560=554574} 2023-08-27T20:52:11.334 INFO:tasks.rados.rados.0.smithi093.stdout:470: writing smithi093153278-470 from 682483 to 1471797 tid 1 2023-08-27T20:52:11.341 INFO:tasks.rados.rados.0.smithi093.stdout:470: writing smithi093153278-470 from 1944996 to 2736493 tid 2 2023-08-27T20:52:11.347 INFO:tasks.rados.rados.0.smithi093.stdout:470: writing smithi093153278-470 from 3196560 to 3751134 tid 3 2023-08-27T20:52:11.347 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.347 INFO:tasks.rados.rados.0.smithi093.stdout:455: finishing write tid 2 to smithi093153278-455 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:455: finishing write tid 3 to smithi093153278-455 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:455: finishing write tid 4 to smithi093153278-455 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:455: finishing write tid 5 to smithi093153278-455 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 455 v 101 (ObjNum 454 snap 0 seq_num 454) dirty exists 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:455: left oid 455 (ObjNum 454 snap 0 seq_num 454) 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:456: finishing write tid 1 to smithi093153278-456 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:456: finishing write tid 2 to smithi093153278-456 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:456: finishing write tid 3 to smithi093153278-456 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:456: finishing write tid 4 to smithi093153278-456 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:456: finishing write tid 5 to smithi093153278-456 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 456 v 105 (ObjNum 455 snap 0 seq_num 455) dirty exists 2023-08-27T20:52:11.348 INFO:tasks.rados.rados.0.smithi093.stdout:456: left oid 456 (ObjNum 455 snap 0 seq_num 455) 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:457: finishing write tid 1 to smithi093153278-457 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:457: finishing write tid 2 to smithi093153278-457 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:457: finishing write tid 3 to smithi093153278-457 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:457: finishing write tid 4 to smithi093153278-457 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:455: done (15 left) 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:456: done (14 left) 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:471: write initial oid 471 2023-08-27T20:52:11.349 INFO:tasks.rados.rados.0.smithi093.stdout:471: seq_num 470 ranges {699290=464563,1603478=535155,2823983=765846} 2023-08-27T20:52:11.353 INFO:tasks.rados.rados.0.smithi093.stdout:471: writing smithi093153278-471 from 699290 to 1163853 tid 1 2023-08-27T20:52:11.358 INFO:tasks.rados.rados.0.smithi093.stdout:471: writing smithi093153278-471 from 1603478 to 2138633 tid 2 2023-08-27T20:52:11.367 INFO:tasks.rados.rados.0.smithi093.stdout:471: writing smithi093153278-471 from 2823983 to 3589829 tid 3 2023-08-27T20:52:11.367 INFO:tasks.rados.rados.0.smithi093.stdout:472: write initial oid 472 2023-08-27T20:52:11.367 INFO:tasks.rados.rados.0.smithi093.stdout:472: seq_num 471 ranges {716097=539812,1945607=795166,3535053=381044} 2023-08-27T20:52:11.374 INFO:tasks.rados.rados.0.smithi093.stdout:472: writing smithi093153278-472 from 716097 to 1255909 tid 1 2023-08-27T20:52:11.384 INFO:tasks.rados.rados.0.smithi093.stdout:472: writing smithi093153278-472 from 1945607 to 2740773 tid 2 2023-08-27T20:52:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:472: writing smithi093153278-472 from 3535053 to 3916097 tid 3 2023-08-27T20:52:11.391 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:457: finishing write tid 5 to smithi093153278-457 2023-08-27T20:52:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 457 v 95 (ObjNum 456 snap 0 seq_num 456) dirty exists 2023-08-27T20:52:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:457: left oid 457 (ObjNum 456 snap 0 seq_num 456) 2023-08-27T20:52:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:457: done (15 left) 2023-08-27T20:52:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:473: write initial oid 473 2023-08-27T20:52:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:473: seq_num 472 ranges {732904=731414,2120442=538824,3278829=504743} 2023-08-27T20:52:11.399 INFO:tasks.rados.rados.0.smithi093.stdout:473: writing smithi093153278-473 from 732904 to 1464318 tid 1 2023-08-27T20:52:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:473: writing smithi093153278-473 from 2120442 to 2659266 tid 2 2023-08-27T20:52:11.413 INFO:tasks.rados.rados.0.smithi093.stdout:473: writing smithi093153278-473 from 3278829 to 3783572 tid 3 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:458: finishing write tid 1 to smithi093153278-458 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:458: finishing write tid 2 to smithi093153278-458 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:458: finishing write tid 3 to smithi093153278-458 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:458: finishing write tid 4 to smithi093153278-458 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:458: finishing write tid 5 to smithi093153278-458 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 458 v 110 (ObjNum 457 snap 0 seq_num 457) dirty exists 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:458: left oid 458 (ObjNum 457 snap 0 seq_num 457) 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:459: finishing write tid 1 to smithi093153278-459 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:459: finishing write tid 2 to smithi093153278-459 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:458: done (15 left) 2023-08-27T20:52:11.414 INFO:tasks.rados.rados.0.smithi093.stdout:474: write initial oid 474 2023-08-27T20:52:11.415 INFO:tasks.rados.rados.0.smithi093.stdout:474: seq_num 473 ranges {749711=406663,1778924=682482,3189899=716015} 2023-08-27T20:52:11.419 INFO:tasks.rados.rados.0.smithi093.stdout:474: writing smithi093153278-474 from 749711 to 1156374 tid 1 2023-08-27T20:52:11.426 INFO:tasks.rados.rados.0.smithi093.stdout:474: writing smithi093153278-474 from 1778924 to 2461406 tid 2 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:474: writing smithi093153278-474 from 3189899 to 3905914 tid 3 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:459: finishing write tid 3 to smithi093153278-459 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:459: finishing write tid 4 to smithi093153278-459 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:459: finishing write tid 5 to smithi093153278-459 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 459 v 96 (ObjNum 458 snap 0 seq_num 458) dirty exists 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:459: left oid 459 (ObjNum 458 snap 0 seq_num 458) 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:460: finishing write tid 1 to smithi093153278-460 2023-08-27T20:52:11.435 INFO:tasks.rados.rados.0.smithi093.stdout:459: done (15 left) 2023-08-27T20:52:11.436 INFO:tasks.rados.rados.0.smithi093.stdout:475: write initial oid 475 2023-08-27T20:52:11.436 INFO:tasks.rados.rados.0.smithi093.stdout:475: seq_num 474 ranges {766518=481912,1837406=542493,2933675=527287,3876584=89934} 2023-08-27T20:52:11.441 INFO:tasks.rados.rados.0.smithi093.stdout:475: writing smithi093153278-475 from 766518 to 1248430 tid 1 2023-08-27T20:52:11.447 INFO:tasks.rados.rados.0.smithi093.stdout:475: writing smithi093153278-475 from 1837406 to 2379899 tid 2 2023-08-27T20:52:11.453 INFO:tasks.rados.rados.0.smithi093.stdout:475: writing smithi093153278-475 from 2933675 to 3460962 tid 3 2023-08-27T20:52:11.457 INFO:tasks.rados.rados.0.smithi093.stdout:475: writing smithi093153278-475 from 3876584 to 3966518 tid 4 2023-08-27T20:52:11.457 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.457 INFO:tasks.rados.rados.0.smithi093.stdout:460: finishing write tid 2 to smithi093153278-460 2023-08-27T20:52:11.457 INFO:tasks.rados.rados.0.smithi093.stdout:460: finishing write tid 3 to smithi093153278-460 2023-08-27T20:52:11.457 INFO:tasks.rados.rados.0.smithi093.stdout:460: finishing write tid 4 to smithi093153278-460 2023-08-27T20:52:11.457 INFO:tasks.rados.rados.0.smithi093.stdout:460: finishing write tid 5 to smithi093153278-460 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 460 v 102 (ObjNum 459 snap 0 seq_num 459) dirty exists 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:460: left oid 460 (ObjNum 459 snap 0 seq_num 459) 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:461: finishing write tid 1 to smithi093153278-461 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:461: finishing write tid 2 to smithi093153278-461 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:461: finishing write tid 3 to smithi093153278-461 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:461: finishing write tid 4 to smithi093153278-461 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:461: finishing write tid 5 to smithi093153278-461 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 461 v 99 (ObjNum 460 snap 0 seq_num 460) dirty exists 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:461: left oid 461 (ObjNum 460 snap 0 seq_num 460) 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:462: finishing write tid 1 to smithi093153278-462 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:462: finishing write tid 2 to smithi093153278-462 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:462: finishing write tid 3 to smithi093153278-462 2023-08-27T20:52:11.458 INFO:tasks.rados.rados.0.smithi093.stdout:462: finishing write tid 4 to smithi093153278-462 2023-08-27T20:52:11.459 INFO:tasks.rados.rados.0.smithi093.stdout:460: done (15 left) 2023-08-27T20:52:11.459 INFO:tasks.rados.rados.0.smithi093.stdout:461: done (14 left) 2023-08-27T20:52:11.459 INFO:tasks.rados.rados.0.smithi093.stdout:476: write initial oid 476 2023-08-27T20:52:11.459 INFO:tasks.rados.rados.0.smithi093.stdout:476: seq_num 475 ranges {783325=557161,1779535=686151,3244745=454912} 2023-08-27T20:52:11.463 INFO:tasks.rados.rados.0.smithi093.stdout:476: writing smithi093153278-476 from 783325 to 1340486 tid 1 2023-08-27T20:52:11.470 INFO:tasks.rados.rados.0.smithi093.stdout:476: writing smithi093153278-476 from 1779535 to 2465686 tid 2 2023-08-27T20:52:11.477 INFO:tasks.rados.rados.0.smithi093.stdout:476: writing smithi093153278-476 from 3244745 to 3699657 tid 3 2023-08-27T20:52:11.477 INFO:tasks.rados.rados.0.smithi093.stdout:477: write initial oid 477 2023-08-27T20:52:11.477 INFO:tasks.rados.rados.0.smithi093.stdout:477: seq_num 476 ranges {400132=632410,1438017=546162} 2023-08-27T20:52:11.483 INFO:tasks.rados.rados.0.smithi093.stdout:477: writing smithi093153278-477 from 400132 to 1032542 tid 1 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:477: writing smithi093153278-477 from 1438017 to 1984179 tid 2 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:462: finishing write tid 5 to smithi093153278-462 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 462 v 92 (ObjNum 461 snap 0 seq_num 461) dirty exists 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:462: left oid 462 (ObjNum 461 snap 0 seq_num 461) 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:463: finishing write tid 1 to smithi093153278-463 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:463: finishing write tid 2 to smithi093153278-463 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:462: done (15 left) 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:478: write initial oid 478 2023-08-27T20:52:11.489 INFO:tasks.rados.rados.0.smithi093.stdout:478: seq_num 477 ranges {416939=707659,1896499=120440} 2023-08-27T20:52:11.495 INFO:tasks.rados.rados.0.smithi093.stdout:478: writing smithi093153278-478 from 416939 to 1124598 tid 1 2023-08-27T20:52:11.500 INFO:tasks.rados.rados.0.smithi093.stdout:478: writing smithi093153278-478 from 1896499 to 2016939 tid 2 2023-08-27T20:52:11.500 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.500 INFO:tasks.rados.rados.0.smithi093.stdout:463: finishing write tid 3 to smithi093153278-463 2023-08-27T20:52:11.500 INFO:tasks.rados.rados.0.smithi093.stdout:463: finishing write tid 4 to smithi093153278-463 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:463: finishing write tid 5 to smithi093153278-463 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 463 v 114 (ObjNum 462 snap 0 seq_num 462) dirty exists 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:463: left oid 463 (ObjNum 462 snap 0 seq_num 462) 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:464: finishing write tid 1 to smithi093153278-464 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:464: finishing write tid 2 to smithi093153278-464 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:463: done (15 left) 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:479: write initial oid 479 2023-08-27T20:52:11.501 INFO:tasks.rados.rados.0.smithi093.stdout:479: seq_num 478 ranges {433746=782908,1954981=78765} 2023-08-27T20:52:11.507 INFO:tasks.rados.rados.0.smithi093.stdout:479: writing smithi093153278-479 from 433746 to 1216654 tid 1 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout:479: writing smithi093153278-479 from 1954981 to 2033746 tid 2 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout:464: finishing write tid 3 to smithi093153278-464 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout:464: finishing write tid 4 to smithi093153278-464 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout:464: finishing write tid 5 to smithi093153278-464 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 464 v 103 (ObjNum 463 snap 0 seq_num 463) dirty exists 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout:464: left oid 464 (ObjNum 463 snap 0 seq_num 463) 2023-08-27T20:52:11.512 INFO:tasks.rados.rados.0.smithi093.stdout:465: finishing write tid 1 to smithi093153278-465 2023-08-27T20:52:11.513 INFO:tasks.rados.rados.0.smithi093.stdout:465: finishing write tid 2 to smithi093153278-465 2023-08-27T20:52:11.513 INFO:tasks.rados.rados.0.smithi093.stdout:464: done (15 left) 2023-08-27T20:52:11.513 INFO:tasks.rados.rados.0.smithi093.stdout:480: write initial oid 480 2023-08-27T20:52:11.513 INFO:tasks.rados.rados.0.smithi093.stdout:480: seq_num 479 ranges {450553=574510,1613463=437090} 2023-08-27T20:52:11.517 INFO:tasks.rados.rados.0.smithi093.stdout:480: writing smithi093153278-480 from 450553 to 1025063 tid 1 2023-08-27T20:52:11.523 INFO:tasks.rados.rados.0.smithi093.stdout:480: writing smithi093153278-480 from 1613463 to 2050553 tid 2 2023-08-27T20:52:11.523 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.523 INFO:tasks.rados.rados.0.smithi093.stdout:465: finishing write tid 3 to smithi093153278-465 2023-08-27T20:52:11.523 INFO:tasks.rados.rados.0.smithi093.stdout:465: finishing write tid 4 to smithi093153278-465 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:465: finishing write tid 5 to smithi093153278-465 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:465: finishing write tid 6 to smithi093153278-465 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 465 v 116 (ObjNum 464 snap 0 seq_num 464) dirty exists 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:465: left oid 465 (ObjNum 464 snap 0 seq_num 464) 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:466: finishing write tid 1 to smithi093153278-466 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:465: done (15 left) 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:481: write initial oid 481 2023-08-27T20:52:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:481: seq_num 480 ranges {467360=649759,1671945=395415} 2023-08-27T20:52:11.529 INFO:tasks.rados.rados.0.smithi093.stdout:481: writing smithi093153278-481 from 467360 to 1117119 tid 1 2023-08-27T20:52:11.535 INFO:tasks.rados.rados.0.smithi093.stdout:481: writing smithi093153278-481 from 1671945 to 2067360 tid 2 2023-08-27T20:52:11.535 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.535 INFO:tasks.rados.rados.0.smithi093.stdout:466: finishing write tid 2 to smithi093153278-466 2023-08-27T20:52:11.535 INFO:tasks.rados.rados.0.smithi093.stdout:466: finishing write tid 3 to smithi093153278-466 2023-08-27T20:52:11.535 INFO:tasks.rados.rados.0.smithi093.stdout:466: finishing write tid 4 to smithi093153278-466 2023-08-27T20:52:11.535 INFO:tasks.rados.rados.0.smithi093.stdout:466: finishing write tid 5 to smithi093153278-466 2023-08-27T20:52:11.536 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 466 v 109 (ObjNum 465 snap 0 seq_num 465) dirty exists 2023-08-27T20:52:11.536 INFO:tasks.rados.rados.0.smithi093.stdout:466: left oid 466 (ObjNum 465 snap 0 seq_num 465) 2023-08-27T20:52:11.536 INFO:tasks.rados.rados.0.smithi093.stdout:467: finishing write tid 1 to smithi093153278-467 2023-08-27T20:52:11.536 INFO:tasks.rados.rados.0.smithi093.stdout:467: finishing write tid 2 to smithi093153278-467 2023-08-27T20:52:11.536 INFO:tasks.rados.rados.0.smithi093.stdout:466: done (15 left) 2023-08-27T20:52:11.536 INFO:tasks.rados.rados.0.smithi093.stdout:482: write initial oid 482 2023-08-27T20:52:11.536 INFO:tasks.rados.rados.0.smithi093.stdout:482: seq_num 481 ranges {484167=725008,1730427=353740} 2023-08-27T20:52:11.541 INFO:tasks.rados.rados.0.smithi093.stdout:482: writing smithi093153278-482 from 484167 to 1209175 tid 1 2023-08-27T20:52:11.546 INFO:tasks.rados.rados.0.smithi093.stdout:482: writing smithi093153278-482 from 1730427 to 2084167 tid 2 2023-08-27T20:52:11.546 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.546 INFO:tasks.rados.rados.0.smithi093.stdout:467: finishing write tid 3 to smithi093153278-467 2023-08-27T20:52:11.546 INFO:tasks.rados.rados.0.smithi093.stdout:467: finishing write tid 4 to smithi093153278-467 2023-08-27T20:52:11.546 INFO:tasks.rados.rados.0.smithi093.stdout:467: finishing write tid 5 to smithi093153278-467 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 467 v 83 (ObjNum 466 snap 0 seq_num 466) dirty exists 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:467: left oid 467 (ObjNum 466 snap 0 seq_num 466) 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:468: finishing write tid 1 to smithi093153278-468 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:468: finishing write tid 2 to smithi093153278-468 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:468: finishing write tid 3 to smithi093153278-468 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:467: done (15 left) 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:483: write initial oid 483 2023-08-27T20:52:11.547 INFO:tasks.rados.rados.0.smithi093.stdout:483: seq_num 482 ranges {500974=400257,1388909=557169} 2023-08-27T20:52:11.551 INFO:tasks.rados.rados.0.smithi093.stdout:483: writing smithi093153278-483 from 500974 to 901231 tid 1 2023-08-27T20:52:11.557 INFO:tasks.rados.rados.0.smithi093.stdout:483: writing smithi093153278-483 from 1388909 to 1946078 tid 2 2023-08-27T20:52:11.557 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.557 INFO:tasks.rados.rados.0.smithi093.stdout:468: finishing write tid 4 to smithi093153278-468 2023-08-27T20:52:11.557 INFO:tasks.rados.rados.0.smithi093.stdout:468: finishing write tid 5 to smithi093153278-468 2023-08-27T20:52:11.557 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 468 v 102 (ObjNum 467 snap 0 seq_num 467) dirty exists 2023-08-27T20:52:11.558 INFO:tasks.rados.rados.0.smithi093.stdout:468: left oid 468 (ObjNum 467 snap 0 seq_num 467) 2023-08-27T20:52:11.558 INFO:tasks.rados.rados.0.smithi093.stdout:469: finishing write tid 1 to smithi093153278-469 2023-08-27T20:52:11.558 INFO:tasks.rados.rados.0.smithi093.stdout:468: done (15 left) 2023-08-27T20:52:11.558 INFO:tasks.rados.rados.0.smithi093.stdout:484: write initial oid 484 2023-08-27T20:52:11.558 INFO:tasks.rados.rados.0.smithi093.stdout:484: seq_num 483 ranges {517781=475506,1731038=386743} 2023-08-27T20:52:11.562 INFO:tasks.rados.rados.0.smithi093.stdout:484: writing smithi093153278-484 from 517781 to 993287 tid 1 2023-08-27T20:52:11.569 INFO:tasks.rados.rados.0.smithi093.stdout:484: writing smithi093153278-484 from 1731038 to 2117781 tid 2 2023-08-27T20:52:11.569 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.569 INFO:tasks.rados.rados.0.smithi093.stdout:469: finishing write tid 2 to smithi093153278-469 2023-08-27T20:52:11.569 INFO:tasks.rados.rados.0.smithi093.stdout:469: finishing write tid 3 to smithi093153278-469 2023-08-27T20:52:11.569 INFO:tasks.rados.rados.0.smithi093.stdout:469: finishing write tid 4 to smithi093153278-469 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:469: finishing write tid 5 to smithi093153278-469 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 469 v 113 (ObjNum 468 snap 0 seq_num 468) dirty exists 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:469: left oid 469 (ObjNum 468 snap 0 seq_num 468) 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:470: finishing write tid 1 to smithi093153278-470 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:470: finishing write tid 2 to smithi093153278-470 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:470: finishing write tid 3 to smithi093153278-470 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:469: done (15 left) 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:485: write initial oid 485 2023-08-27T20:52:11.570 INFO:tasks.rados.rados.0.smithi093.stdout:485: seq_num 484 ranges {534588=550755,1789520=345068} 2023-08-27T20:52:11.575 INFO:tasks.rados.rados.0.smithi093.stdout:485: writing smithi093153278-485 from 534588 to 1085343 tid 1 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:485: writing smithi093153278-485 from 1789520 to 2134588 tid 2 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:470: finishing write tid 4 to smithi093153278-470 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:470: finishing write tid 5 to smithi093153278-470 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 470 v 120 (ObjNum 469 snap 0 seq_num 469) dirty exists 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:470: left oid 470 (ObjNum 469 snap 0 seq_num 469) 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:471: finishing write tid 1 to smithi093153278-471 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:470: done (15 left) 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:486: write initial oid 486 2023-08-27T20:52:11.582 INFO:tasks.rados.rados.0.smithi093.stdout:486: seq_num 485 ranges {551395=626004,1848002=303393} 2023-08-27T20:52:11.587 INFO:tasks.rados.rados.0.smithi093.stdout:486: writing smithi093153278-486 from 551395 to 1177399 tid 1 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:486: writing smithi093153278-486 from 1848002 to 2151395 tid 2 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:471: finishing write tid 2 to smithi093153278-471 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:471: finishing write tid 3 to smithi093153278-471 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:471: finishing write tid 4 to smithi093153278-471 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:471: finishing write tid 5 to smithi093153278-471 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 471 v 114 (ObjNum 470 snap 0 seq_num 470) dirty exists 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:471: left oid 471 (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:52:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:472: finishing write tid 1 to smithi093153278-472 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:472: finishing write tid 2 to smithi093153278-472 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:472: finishing write tid 3 to smithi093153278-472 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:472: finishing write tid 4 to smithi093153278-472 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:472: finishing write tid 5 to smithi093153278-472 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 472 v 99 (ObjNum 471 snap 0 seq_num 471) dirty exists 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:472: left oid 472 (ObjNum 471 snap 0 seq_num 471) 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:473: finishing write tid 1 to smithi093153278-473 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:473: finishing write tid 2 to smithi093153278-473 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:473: finishing write tid 3 to smithi093153278-473 2023-08-27T20:52:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:473: finishing write tid 4 to smithi093153278-473 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:473: finishing write tid 5 to smithi093153278-473 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 473 v 96 (ObjNum 472 snap 0 seq_num 472) dirty exists 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:473: left oid 473 (ObjNum 472 snap 0 seq_num 472) 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:474: finishing write tid 1 to smithi093153278-474 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:471: done (15 left) 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:472: done (14 left) 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:473: done (13 left) 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:487: write initial oid 487 2023-08-27T20:52:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:487: seq_num 486 ranges {568202=701253,1906484=261718} 2023-08-27T20:52:11.600 INFO:tasks.rados.rados.0.smithi093.stdout:487: writing smithi093153278-487 from 568202 to 1269455 tid 1 2023-08-27T20:52:11.605 INFO:tasks.rados.rados.0.smithi093.stdout:487: writing smithi093153278-487 from 1906484 to 2168202 tid 2 2023-08-27T20:52:11.605 INFO:tasks.rados.rados.0.smithi093.stdout:488: write initial oid 488 2023-08-27T20:52:11.605 INFO:tasks.rados.rados.0.smithi093.stdout:488: seq_num 487 ranges {585009=492855,1564966=620043} 2023-08-27T20:52:11.611 INFO:tasks.rados.rados.0.smithi093.stdout:488: writing smithi093153278-488 from 585009 to 1077864 tid 1 2023-08-27T20:52:11.617 INFO:tasks.rados.rados.0.smithi093.stdout:488: writing smithi093153278-488 from 1564966 to 2185009 tid 2 2023-08-27T20:52:11.617 INFO:tasks.rados.rados.0.smithi093.stdout:489: write initial oid 489 2023-08-27T20:52:11.617 INFO:tasks.rados.rados.0.smithi093.stdout:489: seq_num 488 ranges {601816=568104,1623448=451823} 2023-08-27T20:52:11.623 INFO:tasks.rados.rados.0.smithi093.stdout:489: writing smithi093153278-489 from 601816 to 1169920 tid 1 2023-08-27T20:52:11.629 INFO:tasks.rados.rados.0.smithi093.stdout:489: writing smithi093153278-489 from 1623448 to 2075271 tid 2 2023-08-27T20:52:11.629 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.629 INFO:tasks.rados.rados.0.smithi093.stdout:474: finishing write tid 2 to smithi093153278-474 2023-08-27T20:52:11.629 INFO:tasks.rados.rados.0.smithi093.stdout:474: finishing write tid 3 to smithi093153278-474 2023-08-27T20:52:11.629 INFO:tasks.rados.rados.0.smithi093.stdout:474: finishing write tid 4 to smithi093153278-474 2023-08-27T20:52:11.629 INFO:tasks.rados.rados.0.smithi093.stdout:474: finishing write tid 5 to smithi093153278-474 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 474 v 95 (ObjNum 473 snap 0 seq_num 473) dirty exists 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:474: left oid 474 (ObjNum 473 snap 0 seq_num 473) 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:475: finishing write tid 1 to smithi093153278-475 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:475: finishing write tid 2 to smithi093153278-475 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:475: finishing write tid 3 to smithi093153278-475 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:474: done (15 left) 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:490: write initial oid 490 2023-08-27T20:52:11.630 INFO:tasks.rados.rados.0.smithi093.stdout:490: seq_num 489 ranges {618623=643353,1681930=536693} 2023-08-27T20:52:11.636 INFO:tasks.rados.rados.0.smithi093.stdout:490: writing smithi093153278-490 from 618623 to 1261976 tid 1 2023-08-27T20:52:11.642 INFO:tasks.rados.rados.0.smithi093.stdout:490: writing smithi093153278-490 from 1681930 to 2218623 tid 2 2023-08-27T20:52:11.642 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.642 INFO:tasks.rados.rados.0.smithi093.stdout:475: finishing write tid 4 to smithi093153278-475 2023-08-27T20:52:11.642 INFO:tasks.rados.rados.0.smithi093.stdout:475: finishing write tid 5 to smithi093153278-475 2023-08-27T20:52:11.642 INFO:tasks.rados.rados.0.smithi093.stdout:475: finishing write tid 6 to smithi093153278-475 2023-08-27T20:52:11.642 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 475 v 107 (ObjNum 474 snap 0 seq_num 474) dirty exists 2023-08-27T20:52:11.642 INFO:tasks.rados.rados.0.smithi093.stdout:475: left oid 475 (ObjNum 474 snap 0 seq_num 474) 2023-08-27T20:52:11.643 INFO:tasks.rados.rados.0.smithi093.stdout:476: finishing write tid 1 to smithi093153278-476 2023-08-27T20:52:11.643 INFO:tasks.rados.rados.0.smithi093.stdout:476: finishing write tid 2 to smithi093153278-476 2023-08-27T20:52:11.643 INFO:tasks.rados.rados.0.smithi093.stdout:476: finishing write tid 3 to smithi093153278-476 2023-08-27T20:52:11.643 INFO:tasks.rados.rados.0.smithi093.stdout:476: finishing write tid 4 to smithi093153278-476 2023-08-27T20:52:11.643 INFO:tasks.rados.rados.0.smithi093.stdout:475: done (15 left) 2023-08-27T20:52:11.643 INFO:tasks.rados.rados.0.smithi093.stdout:491: write initial oid 491 2023-08-27T20:52:11.643 INFO:tasks.rados.rados.0.smithi093.stdout:491: seq_num 490 ranges {635430=718602,2140412=95018} 2023-08-27T20:52:11.649 INFO:tasks.rados.rados.0.smithi093.stdout:491: writing smithi093153278-491 from 635430 to 1354032 tid 1 2023-08-27T20:52:11.654 INFO:tasks.rados.rados.0.smithi093.stdout:491: writing smithi093153278-491 from 2140412 to 2235430 tid 2 2023-08-27T20:52:11.654 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.654 INFO:tasks.rados.rados.0.smithi093.stdout:476: finishing write tid 5 to smithi093153278-476 2023-08-27T20:52:11.654 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 476 v 118 (ObjNum 475 snap 0 seq_num 475) dirty exists 2023-08-27T20:52:11.654 INFO:tasks.rados.rados.0.smithi093.stdout:476: left oid 476 (ObjNum 475 snap 0 seq_num 475) 2023-08-27T20:52:11.654 INFO:tasks.rados.rados.0.smithi093.stdout:476: done (15 left) 2023-08-27T20:52:11.654 INFO:tasks.rados.rados.0.smithi093.stdout:492: write initial oid 492 2023-08-27T20:52:11.655 INFO:tasks.rados.rados.0.smithi093.stdout:492: seq_num 491 ranges {652237=793851,2198894=53343} 2023-08-27T20:52:11.662 INFO:tasks.rados.rados.0.smithi093.stdout:492: writing smithi093153278-492 from 652237 to 1446088 tid 1 2023-08-27T20:52:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:492: writing smithi093153278-492 from 2198894 to 2252237 tid 2 2023-08-27T20:52:11.667 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:477: finishing write tid 1 to smithi093153278-477 2023-08-27T20:52:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:477: finishing write tid 2 to smithi093153278-477 2023-08-27T20:52:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:477: finishing write tid 3 to smithi093153278-477 2023-08-27T20:52:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:477: finishing write tid 4 to smithi093153278-477 2023-08-27T20:52:11.668 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 477 v 116 (ObjNum 476 snap 0 seq_num 476) dirty exists 2023-08-27T20:52:11.668 INFO:tasks.rados.rados.0.smithi093.stdout:477: left oid 477 (ObjNum 476 snap 0 seq_num 476) 2023-08-27T20:52:11.668 INFO:tasks.rados.rados.0.smithi093.stdout:478: finishing write tid 1 to smithi093153278-478 2023-08-27T20:52:11.668 INFO:tasks.rados.rados.0.smithi093.stdout:478: finishing write tid 2 to smithi093153278-478 2023-08-27T20:52:11.668 INFO:tasks.rados.rados.0.smithi093.stdout:477: done (15 left) 2023-08-27T20:52:11.668 INFO:tasks.rados.rados.0.smithi093.stdout:493: write initial oid 493 2023-08-27T20:52:11.668 INFO:tasks.rados.rados.0.smithi093.stdout:493: seq_num 492 ranges {669044=469100,1741023=459161} 2023-08-27T20:52:11.673 INFO:tasks.rados.rados.0.smithi093.stdout:493: writing smithi093153278-493 from 669044 to 1138144 tid 1 2023-08-27T20:52:11.679 INFO:tasks.rados.rados.0.smithi093.stdout:493: writing smithi093153278-493 from 1741023 to 2200184 tid 2 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:478: finishing write tid 3 to smithi093153278-478 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:478: finishing write tid 4 to smithi093153278-478 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 117 (ObjNum 477 snap 0 seq_num 477) dirty exists 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:478: left oid 478 (ObjNum 477 snap 0 seq_num 477) 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:479: finishing write tid 1 to smithi093153278-479 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:479: finishing write tid 2 to smithi093153278-479 2023-08-27T20:52:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:479: finishing write tid 3 to smithi093153278-479 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:479: finishing write tid 4 to smithi093153278-479 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 479 v 111 (ObjNum 478 snap 0 seq_num 478) dirty exists 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:479: left oid 479 (ObjNum 478 snap 0 seq_num 478) 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:480: finishing write tid 1 to smithi093153278-480 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:480: finishing write tid 2 to smithi093153278-480 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:478: done (15 left) 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:479: done (14 left) 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:494: write initial oid 494 2023-08-27T20:52:11.681 INFO:tasks.rados.rados.0.smithi093.stdout:494: seq_num 493 ranges {685851=544349,1799505=486346} 2023-08-27T20:52:11.685 INFO:tasks.rados.rados.0.smithi093.stdout:494: writing smithi093153278-494 from 685851 to 1230200 tid 1 2023-08-27T20:52:11.692 INFO:tasks.rados.rados.0.smithi093.stdout:494: writing smithi093153278-494 from 1799505 to 2285851 tid 2 2023-08-27T20:52:11.692 INFO:tasks.rados.rados.0.smithi093.stdout:495: write initial oid 495 2023-08-27T20:52:11.692 INFO:tasks.rados.rados.0.smithi093.stdout:495: seq_num 494 ranges {702658=619598,1857987=444671} 2023-08-27T20:52:11.699 INFO:tasks.rados.rados.0.smithi093.stdout:495: writing smithi093153278-495 from 702658 to 1322256 tid 1 2023-08-27T20:52:11.705 INFO:tasks.rados.rados.0.smithi093.stdout:495: writing smithi093153278-495 from 1857987 to 2302658 tid 2 2023-08-27T20:52:11.705 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.705 INFO:tasks.rados.rados.0.smithi093.stdout:480: finishing write tid 3 to smithi093153278-480 2023-08-27T20:52:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:480: finishing write tid 4 to smithi093153278-480 2023-08-27T20:52:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 480 v 119 (ObjNum 479 snap 0 seq_num 479) dirty exists 2023-08-27T20:52:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:480: left oid 480 (ObjNum 479 snap 0 seq_num 479) 2023-08-27T20:52:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:481: finishing write tid 1 to smithi093153278-481 2023-08-27T20:52:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:480: done (15 left) 2023-08-27T20:52:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:496: write initial oid 496 2023-08-27T20:52:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:496: seq_num 495 ranges {719465=411200,1632822=686643} 2023-08-27T20:52:11.711 INFO:tasks.rados.rados.0.smithi093.stdout:496: writing smithi093153278-496 from 719465 to 1130665 tid 1 2023-08-27T20:52:11.718 INFO:tasks.rados.rados.0.smithi093.stdout:496: writing smithi093153278-496 from 1632822 to 2319465 tid 2 2023-08-27T20:52:11.718 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:481: finishing write tid 2 to smithi093153278-481 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:481: finishing write tid 3 to smithi093153278-481 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:481: finishing write tid 4 to smithi093153278-481 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 481 v 106 (ObjNum 480 snap 0 seq_num 480) dirty exists 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:481: left oid 481 (ObjNum 480 snap 0 seq_num 480) 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:482: finishing write tid 1 to smithi093153278-482 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:482: finishing write tid 2 to smithi093153278-482 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:482: finishing write tid 3 to smithi093153278-482 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:482: finishing write tid 4 to smithi093153278-482 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 482 v 93 (ObjNum 481 snap 0 seq_num 481) dirty exists 2023-08-27T20:52:11.719 INFO:tasks.rados.rados.0.smithi093.stdout:482: left oid 482 (ObjNum 481 snap 0 seq_num 481) 2023-08-27T20:52:11.720 INFO:tasks.rados.rados.0.smithi093.stdout:483: finishing write tid 1 to smithi093153278-483 2023-08-27T20:52:11.720 INFO:tasks.rados.rados.0.smithi093.stdout:483: finishing write tid 2 to smithi093153278-483 2023-08-27T20:52:11.720 INFO:tasks.rados.rados.0.smithi093.stdout:483: finishing write tid 3 to smithi093153278-483 2023-08-27T20:52:11.720 INFO:tasks.rados.rados.0.smithi093.stdout:481: done (15 left) 2023-08-27T20:52:11.720 INFO:tasks.rados.rados.0.smithi093.stdout:482: done (14 left) 2023-08-27T20:52:11.720 INFO:tasks.rados.rados.0.smithi093.stdout:497: write initial oid 497 2023-08-27T20:52:11.720 INFO:tasks.rados.rados.0.smithi093.stdout:497: seq_num 496 ranges {736272=486449,1974951=361321} 2023-08-27T20:52:11.724 INFO:tasks.rados.rados.0.smithi093.stdout:497: writing smithi093153278-497 from 736272 to 1222721 tid 1 2023-08-27T20:52:11.731 INFO:tasks.rados.rados.0.smithi093.stdout:497: writing smithi093153278-497 from 1974951 to 2336272 tid 2 2023-08-27T20:52:11.731 INFO:tasks.rados.rados.0.smithi093.stdout:498: write initial oid 498 2023-08-27T20:52:11.731 INFO:tasks.rados.rados.0.smithi093.stdout:498: seq_num 497 ranges {753079=561698,2033433=319646} 2023-08-27T20:52:11.738 INFO:tasks.rados.rados.0.smithi093.stdout:498: writing smithi093153278-498 from 753079 to 1314777 tid 1 2023-08-27T20:52:11.745 INFO:tasks.rados.rados.0.smithi093.stdout:498: writing smithi093153278-498 from 2033433 to 2353079 tid 2 2023-08-27T20:52:11.745 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:11.745 INFO:tasks.rados.rados.0.smithi093.stdout:483: finishing write tid 4 to smithi093153278-483 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 483 v 96 (ObjNum 482 snap 0 seq_num 482) dirty exists 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:483: left oid 483 (ObjNum 482 snap 0 seq_num 482) 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:484: finishing write tid 1 to smithi093153278-484 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:484: finishing write tid 2 to smithi093153278-484 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:484: finishing write tid 3 to smithi093153278-484 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:484: finishing write tid 4 to smithi093153278-484 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 484 v 123 (ObjNum 483 snap 0 seq_num 483) dirty exists 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:484: left oid 484 (ObjNum 483 snap 0 seq_num 483) 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:485: finishing write tid 1 to smithi093153278-485 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:485: finishing write tid 2 to smithi093153278-485 2023-08-27T20:52:11.746 INFO:tasks.rados.rados.0.smithi093.stdout:485: finishing write tid 3 to smithi093153278-485 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:485: finishing write tid 4 to smithi093153278-485 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 485 v 109 (ObjNum 484 snap 0 seq_num 484) dirty exists 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:485: left oid 485 (ObjNum 484 snap 0 seq_num 484) 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:486: finishing write tid 1 to smithi093153278-486 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:486: finishing write tid 2 to smithi093153278-486 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:486: finishing write tid 3 to smithi093153278-486 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:486: finishing write tid 4 to smithi093153278-486 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 120 (ObjNum 485 snap 0 seq_num 485) dirty exists 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:486: left oid 486 (ObjNum 485 snap 0 seq_num 485) 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:487: finishing write tid 1 to smithi093153278-487 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:487: finishing write tid 2 to smithi093153278-487 2023-08-27T20:52:11.747 INFO:tasks.rados.rados.0.smithi093.stdout:487: finishing write tid 3 to smithi093153278-487 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:487: finishing write tid 4 to smithi093153278-487 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 487 v 102 (ObjNum 486 snap 0 seq_num 486) dirty exists 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:487: left oid 487 (ObjNum 486 snap 0 seq_num 486) 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:488: finishing write tid 1 to smithi093153278-488 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:488: finishing write tid 2 to smithi093153278-488 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:488: finishing write tid 3 to smithi093153278-488 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:488: finishing write tid 4 to smithi093153278-488 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 488 v 121 (ObjNum 487 snap 0 seq_num 487) dirty exists 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:488: left oid 488 (ObjNum 487 snap 0 seq_num 487) 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:489: finishing write tid 1 to smithi093153278-489 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:489: finishing write tid 2 to smithi093153278-489 2023-08-27T20:52:11.748 INFO:tasks.rados.rados.0.smithi093.stdout:489: finishing write tid 3 to smithi093153278-489 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:489: finishing write tid 4 to smithi093153278-489 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 489 v 110 (ObjNum 488 snap 0 seq_num 488) dirty exists 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:489: left oid 489 (ObjNum 488 snap 0 seq_num 488) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:490: finishing write tid 1 to smithi093153278-490 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:490: finishing write tid 2 to smithi093153278-490 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:483: done (15 left) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:484: done (14 left) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:485: done (13 left) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:486: done (12 left) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:487: done (11 left) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:488: done (10 left) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:489: done (9 left) 2023-08-27T20:52:11.749 INFO:tasks.rados.rados.0.smithi093.stdout:499: write initial oid 499 2023-08-27T20:52:11.750 INFO:tasks.rados.rados.0.smithi093.stdout:499: seq_num 498 ranges {769886=636947,2091915=277971} 2023-08-27T20:52:11.751 INFO:tasks.rados.rados.0.smithi093.stdout:499: writing smithi093153278-499 from 769886 to 1406833 tid 1 2023-08-27T20:52:11.757 INFO:tasks.rados.rados.0.smithi093.stdout:499: writing smithi093153278-499 from 2091915 to 2369886 tid 2 2023-08-27T20:52:11.757 INFO:tasks.rados.rados.0.smithi093.stdout:500: write initial oid 500 2023-08-27T20:52:11.757 INFO:tasks.rados.rados.0.smithi093.stdout:500: seq_num 499 ranges {786693=712196,2150397=236296} 2023-08-27T20:52:11.765 INFO:tasks.rados.rados.0.smithi093.stdout:500: writing smithi093153278-500 from 786693 to 1498889 tid 1 2023-08-27T20:52:11.770 INFO:tasks.rados.rados.0.smithi093.stdout:500: writing smithi093153278-500 from 2150397 to 2386693 tid 2 2023-08-27T20:52:11.770 INFO:tasks.rados.rados.0.smithi093.stdout:501: write oid 10 current snap is 0 2023-08-27T20:52:11.770 INFO:tasks.rados.rados.0.smithi093.stdout:490: finishing write tid 3 to smithi093153278-490 2023-08-27T20:52:11.770 INFO:tasks.rados.rados.0.smithi093.stdout:490: finishing write tid 4 to smithi093153278-490 2023-08-27T20:52:11.770 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 490 v 113 (ObjNum 489 snap 0 seq_num 489) dirty exists 2023-08-27T20:52:11.770 INFO:tasks.rados.rados.0.smithi093.stdout:490: left oid 490 (ObjNum 489 snap 0 seq_num 489) 2023-08-27T20:52:11.771 INFO:tasks.rados.rados.0.smithi093.stdout:491: finishing write tid 1 to smithi093153278-491 2023-08-27T20:52:11.771 INFO:tasks.rados.rados.0.smithi093.stdout:491: finishing write tid 2 to smithi093153278-491 2023-08-27T20:52:11.771 INFO:tasks.rados.rados.0.smithi093.stdout:501: seq_num 500 ranges {403500=787445,1692526=473837} 2023-08-27T20:52:11.776 INFO:tasks.rados.rados.0.smithi093.stdout:501: writing smithi093153278-10 from 403500 to 1190945 tid 1 2023-08-27T20:52:11.782 INFO:tasks.rados.rados.0.smithi093.stdout:501: writing smithi093153278-10 from 1692526 to 2166363 tid 2 2023-08-27T20:52:11.782 INFO:tasks.rados.rados.0.smithi093.stdout:490: done (11 left) 2023-08-27T20:52:11.782 INFO:tasks.rados.rados.0.smithi093.stdout:502: write oid 81 current snap is 0 2023-08-27T20:52:11.782 INFO:tasks.rados.rados.0.smithi093.stdout:491: finishing write tid 3 to smithi093153278-491 2023-08-27T20:52:11.782 INFO:tasks.rados.rados.0.smithi093.stdout:502: seq_num 501 ranges {420307=462694,1351008=617495} 2023-08-27T20:52:11.787 INFO:tasks.rados.rados.0.smithi093.stdout:502: writing smithi093153278-81 from 420307 to 883001 tid 1 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:502: writing smithi093153278-81 from 1351008 to 1968503 tid 2 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:503: delete oid 474 current snap is 0 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:491: finishing write tid 4 to smithi093153278-491 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 491 v 105 (ObjNum 490 snap 0 seq_num 490) dirty exists 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:491: left oid 491 (ObjNum 490 snap 0 seq_num 490) 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:492: finishing write tid 1 to smithi093153278-492 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:492: finishing write tid 2 to smithi093153278-492 2023-08-27T20:52:11.795 INFO:tasks.rados.rados.0.smithi093.stdout:492: finishing write tid 3 to smithi093153278-492 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:492: finishing write tid 4 to smithi093153278-492 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 492 v 99 (ObjNum 491 snap 0 seq_num 491) dirty exists 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:492: left oid 492 (ObjNum 491 snap 0 seq_num 491) 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:493: finishing write tid 1 to smithi093153278-493 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:493: finishing write tid 2 to smithi093153278-493 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:493: finishing write tid 3 to smithi093153278-493 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:493: finishing write tid 4 to smithi093153278-493 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 493 v 102 (ObjNum 492 snap 0 seq_num 492) dirty exists 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:493: left oid 493 (ObjNum 492 snap 0 seq_num 492) 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:494: finishing write tid 1 to smithi093153278-494 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:494: finishing write tid 2 to smithi093153278-494 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:494: finishing write tid 3 to smithi093153278-494 2023-08-27T20:52:11.796 INFO:tasks.rados.rados.0.smithi093.stdout:494: finishing write tid 4 to smithi093153278-494 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 494 v 105 (ObjNum 493 snap 0 seq_num 493) dirty exists 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:494: left oid 494 (ObjNum 493 snap 0 seq_num 493) 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:495: finishing write tid 1 to smithi093153278-495 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:495: finishing write tid 2 to smithi093153278-495 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:495: finishing write tid 3 to smithi093153278-495 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:495: finishing write tid 4 to smithi093153278-495 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 495 v 124 (ObjNum 494 snap 0 seq_num 494) dirty exists 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:495: left oid 495 (ObjNum 494 snap 0 seq_num 494) 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:496: finishing write tid 1 to smithi093153278-496 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:496: finishing write tid 2 to smithi093153278-496 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:496: finishing write tid 3 to smithi093153278-496 2023-08-27T20:52:11.797 INFO:tasks.rados.rados.0.smithi093.stdout:496: finishing write tid 4 to smithi093153278-496 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 496 v 99 (ObjNum 495 snap 0 seq_num 495) dirty exists 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:496: left oid 496 (ObjNum 495 snap 0 seq_num 495) 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:497: finishing write tid 1 to smithi093153278-497 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:497: finishing write tid 2 to smithi093153278-497 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:497: finishing write tid 3 to smithi093153278-497 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:497: finishing write tid 4 to smithi093153278-497 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 497 v 98 (ObjNum 496 snap 0 seq_num 496) dirty exists 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:497: left oid 497 (ObjNum 496 snap 0 seq_num 496) 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:498: finishing write tid 1 to smithi093153278-498 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:498: finishing write tid 2 to smithi093153278-498 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:498: finishing write tid 3 to smithi093153278-498 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:498: finishing write tid 4 to smithi093153278-498 2023-08-27T20:52:11.798 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 498 v 86 (ObjNum 497 snap 0 seq_num 497) dirty exists 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:498: left oid 498 (ObjNum 497 snap 0 seq_num 497) 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:499: finishing write tid 1 to smithi093153278-499 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:499: finishing write tid 2 to smithi093153278-499 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:499: finishing write tid 3 to smithi093153278-499 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:499: finishing write tid 4 to smithi093153278-499 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 499 v 99 (ObjNum 498 snap 0 seq_num 498) dirty exists 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:499: left oid 499 (ObjNum 498 snap 0 seq_num 498) 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:500: finishing write tid 1 to smithi093153278-500 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:500: finishing write tid 2 to smithi093153278-500 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:500: finishing write tid 3 to smithi093153278-500 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:500: finishing write tid 4 to smithi093153278-500 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 500 v 122 (ObjNum 499 snap 0 seq_num 499) dirty exists 2023-08-27T20:52:11.799 INFO:tasks.rados.rados.0.smithi093.stdout:500: left oid 500 (ObjNum 499 snap 0 seq_num 499) 2023-08-27T20:52:11.800 INFO:tasks.rados.rados.0.smithi093.stdout:501: finishing write tid 1 to smithi093153278-10 2023-08-27T20:52:11.800 INFO:tasks.rados.rados.0.smithi093.stdout:501: finishing write tid 2 to smithi093153278-10 2023-08-27T20:52:11.800 INFO:tasks.rados.rados.0.smithi093.stdout:501: finishing write tid 3 to smithi093153278-10 2023-08-27T20:52:11.800 INFO:tasks.rados.rados.0.smithi093.stdout:501: finishing write tid 4 to smithi093153278-10 2023-08-27T20:52:11.800 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 10 v 108 (ObjNum 500 snap 0 seq_num 500) dirty exists 2023-08-27T20:52:11.800 INFO:tasks.rados.rados.0.smithi093.stdout:501: left oid 10 (ObjNum 500 snap 0 seq_num 500) 2023-08-27T20:52:11.800 INFO:tasks.rados.rados.0.smithi093.stdout:502: finishing write tid 1 to smithi093153278-81 2023-08-27T20:52:11.801 INFO:tasks.rados.rados.0.smithi093.stdout:491: done (12 left) 2023-08-27T20:52:11.801 INFO:tasks.rados.rados.0.smithi093.stdout:492: done (11 left) 2023-08-27T20:52:11.801 INFO:tasks.rados.rados.0.smithi093.stdout:493: done (10 left) 2023-08-27T20:52:11.801 INFO:tasks.rados.rados.0.smithi093.stdout:494: done (9 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:495: done (8 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:496: done (7 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:497: done (6 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:498: done (5 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:499: done (4 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:500: done (3 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:501: done (2 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:503: done (1 left) 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:504: delete oid 165 current snap is 0 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:502: finishing write tid 2 to smithi093153278-81 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:502: finishing write tid 3 to smithi093153278-81 2023-08-27T20:52:11.802 INFO:tasks.rados.rados.0.smithi093.stdout:504: done (1 left) 2023-08-27T20:52:11.803 INFO:tasks.rados.rados.0.smithi093.stdout:505: write oid 477 current snap is 0 2023-08-27T20:52:11.803 INFO:tasks.rados.rados.0.smithi093.stdout:505: seq_num 504 ranges {470728=404794,1526454=481175} 2023-08-27T20:52:11.803 INFO:tasks.rados.rados.0.smithi093.stdout:505: writing smithi093153278-477 from 470728 to 875522 tid 1 2023-08-27T20:52:11.808 INFO:tasks.rados.rados.0.smithi093.stdout:505: writing smithi093153278-477 from 1526454 to 2007629 tid 2 2023-08-27T20:52:11.809 INFO:tasks.rados.rados.0.smithi093.stdout:506: delete oid 413 current snap is 0 2023-08-27T20:52:11.809 INFO:tasks.rados.rados.0.smithi093.stdout:502: finishing write tid 4 to smithi093153278-81 2023-08-27T20:52:11.809 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 126 (ObjNum 501 snap 0 seq_num 501) dirty exists 2023-08-27T20:52:11.809 INFO:tasks.rados.rados.0.smithi093.stdout:502: left oid 81 (ObjNum 501 snap 0 seq_num 501) 2023-08-27T20:52:11.809 INFO:tasks.rados.rados.0.smithi093.stdout:505: finishing write tid 1 to smithi093153278-477 2023-08-27T20:52:11.810 INFO:tasks.rados.rados.0.smithi093.stdout:502: done (2 left) 2023-08-27T20:52:11.810 INFO:tasks.rados.rados.0.smithi093.stdout:506: done (1 left) 2023-08-27T20:52:11.810 INFO:tasks.rados.rados.0.smithi093.stdout:507: read oid 434 snap -1 2023-08-27T20:52:11.810 INFO:tasks.rados.rados.0.smithi093.stdout:507: expect (ObjNum 433 snap 0 seq_num 433) 2023-08-27T20:52:11.811 INFO:tasks.rados.rados.0.smithi093.stdout:508: write oid 2 current snap is 0 2023-08-27T20:52:11.811 INFO:tasks.rados.rados.0.smithi093.stdout:508: seq_num 506 ranges {504342=555292,1643418=484844} 2023-08-27T20:52:11.816 INFO:tasks.rados.rados.0.smithi093.stdout:508: writing smithi093153278-2 from 504342 to 1059634 tid 1 2023-08-27T20:52:11.823 INFO:tasks.rados.rados.0.smithi093.stdout:508: writing smithi093153278-2 from 1643418 to 2128262 tid 2 2023-08-27T20:52:11.823 INFO:tasks.rados.rados.0.smithi093.stdout:509: read oid 335 snap -1 2023-08-27T20:52:11.823 INFO:tasks.rados.rados.0.smithi093.stdout:509: expect (ObjNum 334 snap 0 seq_num 334) 2023-08-27T20:52:11.823 INFO:tasks.rados.rados.0.smithi093.stdout:510: read oid 194 snap -1 2023-08-27T20:52:11.824 INFO:tasks.rados.rados.0.smithi093.stdout:510: expect (ObjNum 193 snap 0 seq_num 193) 2023-08-27T20:52:11.824 INFO:tasks.rados.rados.0.smithi093.stdout:511: write oid 165 current snap is 0 2023-08-27T20:52:11.824 INFO:tasks.rados.rados.0.smithi093.stdout:505: finishing write tid 2 to smithi093153278-477 2023-08-27T20:52:11.824 INFO:tasks.rados.rados.0.smithi093.stdout:511: seq_num 507 ranges {521149=630541,1701900=628502} 2023-08-27T20:52:11.830 INFO:tasks.rados.rados.0.smithi093.stdout:511: writing smithi093153278-165 from 521149 to 1151690 tid 1 2023-08-27T20:52:11.837 INFO:tasks.rados.rados.0.smithi093.stdout:511: writing smithi093153278-165 from 1701900 to 2330402 tid 2 2023-08-27T20:52:11.837 INFO:tasks.rados.rados.0.smithi093.stdout:512: write oid 250 current snap is 0 2023-08-27T20:52:11.837 INFO:tasks.rados.rados.0.smithi093.stdout:505: finishing write tid 3 to smithi093153278-477 2023-08-27T20:52:11.837 INFO:tasks.rados.rados.0.smithi093.stdout:505: finishing write tid 4 to smithi093153278-477 2023-08-27T20:52:11.837 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 477 v 125 (ObjNum 504 snap 0 seq_num 504) dirty exists 2023-08-27T20:52:11.837 INFO:tasks.rados.rados.0.smithi093.stdout:505: left oid 477 (ObjNum 504 snap 0 seq_num 504) 2023-08-27T20:52:11.837 INFO:tasks.rados.rados.0.smithi093.stdout:508: finishing write tid 1 to smithi093153278-2 2023-08-27T20:52:11.838 INFO:tasks.rados.rados.0.smithi093.stdout:512: seq_num 508 ranges {537956=705790,1644029=772160} 2023-08-27T20:52:11.843 INFO:tasks.rados.rados.0.smithi093.stdout:512: writing smithi093153278-250 from 537956 to 1243746 tid 1 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:512: writing smithi093153278-250 from 1644029 to 2416189 tid 2 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:505: done (6 left) 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:513: read oid 291 snap -1 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:513: expect (ObjNum 290 snap 0 seq_num 290) 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:514: read oid 148 snap -1 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:514: expect (ObjNum 147 snap 0 seq_num 147) 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:515: read oid 459 snap -1 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:515: expect (ObjNum 458 snap 0 seq_num 458) 2023-08-27T20:52:11.852 INFO:tasks.rados.rados.0.smithi093.stdout:516: write oid 21 current snap is 0 2023-08-27T20:52:11.853 INFO:tasks.rados.rados.0.smithi093.stdout:508: finishing write tid 2 to smithi093153278-2 2023-08-27T20:52:11.853 INFO:tasks.rados.rados.0.smithi093.stdout:508: finishing write tid 3 to smithi093153278-2 2023-08-27T20:52:11.853 INFO:tasks.rados.rados.0.smithi093.stdout:508: finishing write tid 4 to smithi093153278-2 2023-08-27T20:52:11.853 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 2 v 136 (ObjNum 506 snap 0 seq_num 506) dirty exists 2023-08-27T20:52:11.853 INFO:tasks.rados.rados.0.smithi093.stdout:508: left oid 2 (ObjNum 506 snap 0 seq_num 506) 2023-08-27T20:52:11.853 INFO:tasks.rados.rados.0.smithi093.stdout:509: expect (ObjNum 334 snap 0 seq_num 334) 2023-08-27T20:52:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:11 smithi136 ceph-mon[113531]: pgmap v60: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 28 MiB/s wr, 141 op/s 2023-08-27T20:52:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:11 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:511: finishing write tid 1 to smithi093153278-165 2023-08-27T20:52:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:507: expect (ObjNum 433 snap 0 seq_num 433) 2023-08-27T20:52:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:11 smithi093 ceph-mon[111335]: pgmap v60: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 28 MiB/s wr, 141 op/s 2023-08-27T20:52:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:11 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:11 smithi093 ceph-mon[118902]: pgmap v60: 89 pgs: 89 active+clean; 602 MiB data, 754 MiB used, 714 GiB / 715 GiB avail; 22 KiB/s rd, 28 MiB/s wr, 141 op/s 2023-08-27T20:52:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:11 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:12.088 INFO:tasks.rados.rados.0.smithi093.stdout:511: finishing write tid 2 to smithi093153278-165 2023-08-27T20:52:12.088 INFO:tasks.rados.rados.0.smithi093.stdout:511: finishing write tid 3 to smithi093153278-165 2023-08-27T20:52:12.088 INFO:tasks.rados.rados.0.smithi093.stdout:511: finishing write tid 4 to smithi093153278-165 2023-08-27T20:52:12.088 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 165 v 128 (ObjNum 507 snap 0 seq_num 507) dirty exists 2023-08-27T20:52:12.088 INFO:tasks.rados.rados.0.smithi093.stdout:511: left oid 165 (ObjNum 507 snap 0 seq_num 507) 2023-08-27T20:52:12.088 INFO:tasks.rados.rados.0.smithi093.stdout:510: expect (ObjNum 193 snap 0 seq_num 193) 2023-08-27T20:52:12.196 INFO:tasks.rados.rados.0.smithi093.stdout:512: finishing write tid 1 to smithi093153278-250 2023-08-27T20:52:12.196 INFO:tasks.rados.rados.0.smithi093.stdout:516: seq_num 509 ranges {554763=781039,2102511=452252} 2023-08-27T20:52:12.204 INFO:tasks.rados.rados.0.smithi093.stdout:516: writing smithi093153278-21 from 554763 to 1335802 tid 1 2023-08-27T20:52:12.211 INFO:tasks.rados.rados.0.smithi093.stdout:516: writing smithi093153278-21 from 2102511 to 2554763 tid 2 2023-08-27T20:52:12.212 INFO:tasks.rados.rados.0.smithi093.stdout:507: done (9 left) 2023-08-27T20:52:12.212 INFO:tasks.rados.rados.0.smithi093.stdout:508: done (8 left) 2023-08-27T20:52:12.212 INFO:tasks.rados.rados.0.smithi093.stdout:509: done (7 left) 2023-08-27T20:52:12.213 INFO:tasks.rados.rados.0.smithi093.stdout:510: done (6 left) 2023-08-27T20:52:12.213 INFO:tasks.rados.rados.0.smithi093.stdout:511: done (5 left) 2023-08-27T20:52:12.213 INFO:tasks.rados.rados.0.smithi093.stdout:517: write oid 112 current snap is 0 2023-08-27T20:52:12.213 INFO:tasks.rados.rados.0.smithi093.stdout:513: expect (ObjNum 290 snap 0 seq_num 290) 2023-08-27T20:52:12.309 INFO:tasks.rados.rados.0.smithi093.stdout:512: finishing write tid 2 to smithi093153278-250 2023-08-27T20:52:12.309 INFO:tasks.rados.rados.0.smithi093.stdout:512: finishing write tid 3 to smithi093153278-250 2023-08-27T20:52:12.309 INFO:tasks.rados.rados.0.smithi093.stdout:512: finishing write tid 4 to smithi093153278-250 2023-08-27T20:52:12.310 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 250 v 129 (ObjNum 508 snap 0 seq_num 508) dirty exists 2023-08-27T20:52:12.310 INFO:tasks.rados.rados.0.smithi093.stdout:512: left oid 250 (ObjNum 508 snap 0 seq_num 508) 2023-08-27T20:52:12.310 INFO:tasks.rados.rados.0.smithi093.stdout:514: expect (ObjNum 147 snap 0 seq_num 147) 2023-08-27T20:52:12.396 INFO:tasks.rados.rados.0.smithi093.stdout:515: expect (ObjNum 458 snap 0 seq_num 458) 2023-08-27T20:52:12.517 INFO:tasks.rados.rados.0.smithi093.stdout:516: finishing write tid 1 to smithi093153278-21 2023-08-27T20:52:12.517 INFO:tasks.rados.rados.0.smithi093.stdout:516: finishing write tid 2 to smithi093153278-21 2023-08-27T20:52:12.517 INFO:tasks.rados.rados.0.smithi093.stdout:516: finishing write tid 3 to smithi093153278-21 2023-08-27T20:52:12.517 INFO:tasks.rados.rados.0.smithi093.stdout:516: finishing write tid 4 to smithi093153278-21 2023-08-27T20:52:12.518 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 21 v 131 (ObjNum 509 snap 0 seq_num 509) dirty exists 2023-08-27T20:52:12.518 INFO:tasks.rados.rados.0.smithi093.stdout:516: left oid 21 (ObjNum 509 snap 0 seq_num 509) 2023-08-27T20:52:12.518 INFO:tasks.rados.rados.0.smithi093.stdout:517: seq_num 510 ranges {571570=572641,1877346=694224} 2023-08-27T20:52:12.523 INFO:tasks.rados.rados.0.smithi093.stdout:517: writing smithi093153278-112 from 571570 to 1144211 tid 1 2023-08-27T20:52:12.531 INFO:tasks.rados.rados.0.smithi093.stdout:517: writing smithi093153278-112 from 1877346 to 2571570 tid 2 2023-08-27T20:52:12.531 INFO:tasks.rados.rados.0.smithi093.stdout:512: done (5 left) 2023-08-27T20:52:12.531 INFO:tasks.rados.rados.0.smithi093.stdout:513: done (4 left) 2023-08-27T20:52:12.532 INFO:tasks.rados.rados.0.smithi093.stdout:514: done (3 left) 2023-08-27T20:52:12.532 INFO:tasks.rados.rados.0.smithi093.stdout:515: done (2 left) 2023-08-27T20:52:12.532 INFO:tasks.rados.rados.0.smithi093.stdout:516: done (1 left) 2023-08-27T20:52:12.533 INFO:tasks.rados.rados.0.smithi093.stdout:518: read oid 373 snap -1 2023-08-27T20:52:12.533 INFO:tasks.rados.rados.0.smithi093.stdout:518: expect (ObjNum 372 snap 0 seq_num 372) 2023-08-27T20:52:12.533 INFO:tasks.rados.rados.0.smithi093.stdout:519: read oid 225 snap -1 2023-08-27T20:52:12.533 INFO:tasks.rados.rados.0.smithi093.stdout:519: expect (ObjNum 224 snap 0 seq_num 224) 2023-08-27T20:52:12.533 INFO:tasks.rados.rados.0.smithi093.stdout:520: write oid 191 current snap is 0 2023-08-27T20:52:12.533 INFO:tasks.rados.rados.0.smithi093.stdout:517: finishing write tid 1 to smithi093153278-112 2023-08-27T20:52:12.533 INFO:tasks.rados.rados.0.smithi093.stdout:520: seq_num 511 ranges {588377=647890,1935828=635840} 2023-08-27T20:52:12.539 INFO:tasks.rados.rados.0.smithi093.stdout:520: writing smithi093153278-191 from 588377 to 1236267 tid 1 2023-08-27T20:52:12.546 INFO:tasks.rados.rados.0.smithi093.stdout:520: writing smithi093153278-191 from 1935828 to 2571668 tid 2 2023-08-27T20:52:12.547 INFO:tasks.rados.rados.0.smithi093.stdout:521: write oid 500 current snap is 0 2023-08-27T20:52:12.547 INFO:tasks.rados.rados.0.smithi093.stdout:521: seq_num 512 ranges {605184=723139,1877957=727227} 2023-08-27T20:52:12.554 INFO:tasks.rados.rados.0.smithi093.stdout:521: writing smithi093153278-500 from 605184 to 1328323 tid 1 2023-08-27T20:52:12.561 INFO:tasks.rados.rados.0.smithi093.stdout:521: writing smithi093153278-500 from 1877957 to 2605184 tid 2 2023-08-27T20:52:12.561 INFO:tasks.rados.rados.0.smithi093.stdout:522: write oid 376 current snap is 0 2023-08-27T20:52:12.561 INFO:tasks.rados.rados.0.smithi093.stdout:517: finishing write tid 2 to smithi093153278-112 2023-08-27T20:52:12.561 INFO:tasks.rados.rados.0.smithi093.stdout:517: finishing write tid 3 to smithi093153278-112 2023-08-27T20:52:12.561 INFO:tasks.rados.rados.0.smithi093.stdout:522: seq_num 513 ranges {621991=798388,1936439=639509} 2023-08-27T20:52:12.569 INFO:tasks.rados.rados.0.smithi093.stdout:522: writing smithi093153278-376 from 621991 to 1420379 tid 1 2023-08-27T20:52:12.576 INFO:tasks.rados.rados.0.smithi093.stdout:522: writing smithi093153278-376 from 1936439 to 2575948 tid 2 2023-08-27T20:52:12.576 INFO:tasks.rados.rados.0.smithi093.stdout:523: write oid 492 current snap is 0 2023-08-27T20:52:12.576 INFO:tasks.rados.rados.0.smithi093.stdout:517: finishing write tid 4 to smithi093153278-112 2023-08-27T20:52:12.576 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 112 v 113 (ObjNum 510 snap 0 seq_num 510) dirty exists 2023-08-27T20:52:12.576 INFO:tasks.rados.rados.0.smithi093.stdout:517: left oid 112 (ObjNum 510 snap 0 seq_num 510) 2023-08-27T20:52:12.576 INFO:tasks.rados.rados.0.smithi093.stdout:518: expect (ObjNum 372 snap 0 seq_num 372) 2023-08-27T20:52:12.649 INFO:tasks.rados.rados.0.smithi093.stdout:519: expect (ObjNum 224 snap 0 seq_num 224) 2023-08-27T20:52:12.771 INFO:tasks.rados.rados.0.smithi093.stdout:520: finishing write tid 1 to smithi093153278-191 2023-08-27T20:52:12.771 INFO:tasks.rados.rados.0.smithi093.stdout:523: seq_num 514 ranges {638798=473637,1594921=783167} 2023-08-27T20:52:12.776 INFO:tasks.rados.rados.0.smithi093.stdout:523: writing smithi093153278-492 from 638798 to 1112435 tid 1 2023-08-27T20:52:12.784 INFO:tasks.rados.rados.0.smithi093.stdout:523: writing smithi093153278-492 from 1594921 to 2378088 tid 2 2023-08-27T20:52:12.784 INFO:tasks.rados.rados.0.smithi093.stdout:517: done (6 left) 2023-08-27T20:52:12.784 INFO:tasks.rados.rados.0.smithi093.stdout:518: done (5 left) 2023-08-27T20:52:12.784 INFO:tasks.rados.rados.0.smithi093.stdout:519: done (4 left) 2023-08-27T20:52:12.784 INFO:tasks.rados.rados.0.smithi093.stdout:524: read oid 27 snap -1 2023-08-27T20:52:12.784 INFO:tasks.rados.rados.0.smithi093.stdout:524: expect (ObjNum 26 snap 0 seq_num 26) 2023-08-27T20:52:12.784 INFO:tasks.rados.rados.0.smithi093.stdout:525: write oid 261 current snap is 0 2023-08-27T20:52:12.785 INFO:tasks.rados.rados.0.smithi093.stdout:520: finishing write tid 2 to smithi093153278-191 2023-08-27T20:52:12.785 INFO:tasks.rados.rados.0.smithi093.stdout:525: seq_num 515 ranges {655605=548886,1653403=643178} 2023-08-27T20:52:12.791 INFO:tasks.rados.rados.0.smithi093.stdout:525: writing smithi093153278-261 from 655605 to 1204491 tid 1 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:525: writing smithi093153278-261 from 1653403 to 2296581 tid 2 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:526: delete oid 369 current snap is 0 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:520: finishing write tid 3 to smithi093153278-191 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:520: finishing write tid 4 to smithi093153278-191 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 191 v 139 (ObjNum 511 snap 0 seq_num 511) dirty exists 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:520: left oid 191 (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:521: finishing write tid 1 to smithi093153278-500 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:521: finishing write tid 2 to smithi093153278-500 2023-08-27T20:52:12.798 INFO:tasks.rados.rados.0.smithi093.stdout:521: finishing write tid 3 to smithi093153278-500 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:521: finishing write tid 4 to smithi093153278-500 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 500 v 128 (ObjNum 512 snap 0 seq_num 512) dirty exists 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:521: left oid 500 (ObjNum 512 snap 0 seq_num 512) 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:522: finishing write tid 1 to smithi093153278-376 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:522: finishing write tid 2 to smithi093153278-376 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:522: finishing write tid 3 to smithi093153278-376 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:522: finishing write tid 4 to smithi093153278-376 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 376 v 132 (ObjNum 513 snap 0 seq_num 513) dirty exists 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:522: left oid 376 (ObjNum 513 snap 0 seq_num 513) 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:523: finishing write tid 1 to smithi093153278-492 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:523: finishing write tid 2 to smithi093153278-492 2023-08-27T20:52:12.799 INFO:tasks.rados.rados.0.smithi093.stdout:523: finishing write tid 3 to smithi093153278-492 2023-08-27T20:52:12.800 INFO:tasks.rados.rados.0.smithi093.stdout:523: finishing write tid 4 to smithi093153278-492 2023-08-27T20:52:12.800 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 492 v 105 (ObjNum 514 snap 0 seq_num 514) dirty exists 2023-08-27T20:52:12.800 INFO:tasks.rados.rados.0.smithi093.stdout:523: left oid 492 (ObjNum 514 snap 0 seq_num 514) 2023-08-27T20:52:12.800 INFO:tasks.rados.rados.0.smithi093.stdout:524: expect (ObjNum 26 snap 0 seq_num 26) 2023-08-27T20:52:12.874 INFO:tasks.rados.rados.0.smithi093.stdout:525: finishing write tid 1 to smithi093153278-261 2023-08-27T20:52:12.874 INFO:tasks.rados.rados.0.smithi093.stdout:520: done (6 left) 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:521: done (5 left) 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:522: done (4 left) 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:523: done (3 left) 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:524: done (2 left) 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:526: done (1 left) 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:527: write oid 320 current snap is 0 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:525: finishing write tid 2 to smithi093153278-261 2023-08-27T20:52:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:527: seq_num 517 ranges {689219=415737,1770367=646847} 2023-08-27T20:52:12.881 INFO:tasks.rados.rados.0.smithi093.stdout:527: writing smithi093153278-320 from 689219 to 1104956 tid 1 2023-08-27T20:52:12.889 INFO:tasks.rados.rados.0.smithi093.stdout:527: writing smithi093153278-320 from 1770367 to 2417214 tid 2 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:528: read oid 303 snap -1 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:528: expect (ObjNum 302 snap 0 seq_num 302) 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:529: delete oid 315 current snap is 0 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:525: finishing write tid 3 to smithi093153278-261 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:525: finishing write tid 4 to smithi093153278-261 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 261 v 108 (ObjNum 515 snap 0 seq_num 515) dirty exists 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:525: left oid 261 (ObjNum 515 snap 0 seq_num 515) 2023-08-27T20:52:12.890 INFO:tasks.rados.rados.0.smithi093.stdout:527: finishing write tid 1 to smithi093153278-320 2023-08-27T20:52:12.891 INFO:tasks.rados.rados.0.smithi093.stdout:525: done (3 left) 2023-08-27T20:52:12.891 INFO:tasks.rados.rados.0.smithi093.stdout:529: done (2 left) 2023-08-27T20:52:12.891 INFO:tasks.rados.rados.0.smithi093.stdout:530: write oid 55 current snap is 0 2023-08-27T20:52:12.891 INFO:tasks.rados.rados.0.smithi093.stdout:530: seq_num 519 ranges {722833=566235,1887331=650516} 2023-08-27T20:52:12.897 INFO:tasks.rados.rados.0.smithi093.stdout:530: writing smithi093153278-55 from 722833 to 1289068 tid 1 2023-08-27T20:52:12.905 INFO:tasks.rados.rados.0.smithi093.stdout:530: writing smithi093153278-55 from 1887331 to 2537847 tid 2 2023-08-27T20:52:12.905 INFO:tasks.rados.rados.0.smithi093.stdout:531: read oid 295 snap -1 2023-08-27T20:52:12.905 INFO:tasks.rados.rados.0.smithi093.stdout:531: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:52:12.905 INFO:tasks.rados.rados.0.smithi093.stdout:532: write oid 291 current snap is 0 2023-08-27T20:52:12.905 INFO:tasks.rados.rados.0.smithi093.stdout:532: seq_num 520 ranges {739640=641484,1829460=794174} 2023-08-27T20:52:12.913 INFO:tasks.rados.rados.0.smithi093.stdout:532: writing smithi093153278-291 from 739640 to 1381124 tid 1 2023-08-27T20:52:12.920 INFO:tasks.rados.rados.0.smithi093.stdout:532: writing smithi093153278-291 from 1829460 to 2623634 tid 2 2023-08-27T20:52:12.920 INFO:tasks.rados.rados.0.smithi093.stdout:533: write oid 285 current snap is 0 2023-08-27T20:52:12.920 INFO:tasks.rados.rados.0.smithi093.stdout:533: seq_num 521 ranges {756447=716733,1887942=537832} 2023-08-27T20:52:12.928 INFO:tasks.rados.rados.0.smithi093.stdout:533: writing smithi093153278-285 from 756447 to 1473180 tid 1 2023-08-27T20:52:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:533: writing smithi093153278-285 from 1887942 to 2425774 tid 2 2023-08-27T20:52:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:534: read oid 77 snap -1 2023-08-27T20:52:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:534: expect (ObjNum 76 snap 0 seq_num 76) 2023-08-27T20:52:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:535: read oid 495 snap -1 2023-08-27T20:52:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:535: expect (ObjNum 494 snap 0 seq_num 494) 2023-08-27T20:52:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:536: write oid 129 current snap is 0 2023-08-27T20:52:12.935 INFO:tasks.rados.rados.0.smithi093.stdout:527: finishing write tid 2 to smithi093153278-320 2023-08-27T20:52:12.935 INFO:tasks.rados.rados.0.smithi093.stdout:527: finishing write tid 3 to smithi093153278-320 2023-08-27T20:52:12.935 INFO:tasks.rados.rados.0.smithi093.stdout:527: finishing write tid 4 to smithi093153278-320 2023-08-27T20:52:12.935 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 320 v 135 (ObjNum 517 snap 0 seq_num 517) dirty exists 2023-08-27T20:52:12.935 INFO:tasks.rados.rados.0.smithi093.stdout:527: left oid 320 (ObjNum 517 snap 0 seq_num 517) 2023-08-27T20:52:12.935 INFO:tasks.rados.rados.0.smithi093.stdout:528: expect (ObjNum 302 snap 0 seq_num 302) 2023-08-27T20:52:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3465107800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3465107800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3465107800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:13.039 INFO:tasks.rados.rados.0.smithi093.stdout:530: finishing write tid 1 to smithi093153278-55 2023-08-27T20:52:13.039 INFO:tasks.rados.rados.0.smithi093.stdout:530: finishing write tid 2 to smithi093153278-55 2023-08-27T20:52:13.039 INFO:tasks.rados.rados.0.smithi093.stdout:530: finishing write tid 3 to smithi093153278-55 2023-08-27T20:52:13.039 INFO:tasks.rados.rados.0.smithi093.stdout:536: seq_num 522 ranges {773254=791982,2346424=426830} 2023-08-27T20:52:13.047 INFO:tasks.rados.rados.0.smithi093.stdout:536: writing smithi093153278-129 from 773254 to 1565236 tid 1 2023-08-27T20:52:13.053 INFO:tasks.rados.rados.0.smithi093.stdout:536: writing smithi093153278-129 from 2346424 to 2773254 tid 2 2023-08-27T20:52:13.053 INFO:tasks.rados.rados.0.smithi093.stdout:527: done (8 left) 2023-08-27T20:52:13.054 INFO:tasks.rados.rados.0.smithi093.stdout:528: done (7 left) 2023-08-27T20:52:13.054 INFO:tasks.rados.rados.0.smithi093.stdout:530: finishing write tid 4 to smithi093153278-55 2023-08-27T20:52:13.054 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 55 v 116 (ObjNum 519 snap 0 seq_num 519) dirty exists 2023-08-27T20:52:13.054 INFO:tasks.rados.rados.0.smithi093.stdout:530: left oid 55 (ObjNum 519 snap 0 seq_num 519) 2023-08-27T20:52:13.054 INFO:tasks.rados.rados.0.smithi093.stdout:531: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:532: finishing write tid 1 to smithi093153278-291 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:532: finishing write tid 2 to smithi093153278-291 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:532: finishing write tid 3 to smithi093153278-291 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:532: finishing write tid 4 to smithi093153278-291 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 291 v 111 (ObjNum 520 snap 0 seq_num 520) dirty exists 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:532: left oid 291 (ObjNum 520 snap 0 seq_num 520) 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:533: finishing write tid 1 to smithi093153278-285 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:533: finishing write tid 2 to smithi093153278-285 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:533: finishing write tid 3 to smithi093153278-285 2023-08-27T20:52:13.153 INFO:tasks.rados.rados.0.smithi093.stdout:533: finishing write tid 4 to smithi093153278-285 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 285 v 124 (ObjNum 521 snap 0 seq_num 521) dirty exists 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:533: left oid 285 (ObjNum 521 snap 0 seq_num 521) 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:537: read oid 101 snap -1 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:537: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:530: done (7 left) 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:531: done (6 left) 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:532: done (5 left) 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:533: done (4 left) 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:538: write oid 399 current snap is 0 2023-08-27T20:52:13.154 INFO:tasks.rados.rados.0.smithi093.stdout:535: expect (ObjNum 494 snap 0 seq_num 494) 2023-08-27T20:52:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:534: expect (ObjNum 76 snap 0 seq_num 76) 2023-08-27T20:52:13.339 INFO:tasks.rados.rados.0.smithi093.stdout:536: finishing write tid 1 to smithi093153278-129 2023-08-27T20:52:13.339 INFO:tasks.rados.rados.0.smithi093.stdout:536: finishing write tid 2 to smithi093153278-129 2023-08-27T20:52:13.339 INFO:tasks.rados.rados.0.smithi093.stdout:536: finishing write tid 3 to smithi093153278-129 2023-08-27T20:52:13.339 INFO:tasks.rados.rados.0.smithi093.stdout:536: finishing write tid 4 to smithi093153278-129 2023-08-27T20:52:13.339 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 129 v 102 (ObjNum 522 snap 0 seq_num 522) dirty exists 2023-08-27T20:52:13.339 INFO:tasks.rados.rados.0.smithi093.stdout:536: left oid 129 (ObjNum 522 snap 0 seq_num 522) 2023-08-27T20:52:13.340 INFO:tasks.rados.rados.0.smithi093.stdout:538: seq_num 523 ranges {790061=467231,2004906=541501} 2023-08-27T20:52:13.346 INFO:tasks.rados.rados.0.smithi093.stdout:538: writing smithi093153278-399 from 790061 to 1257292 tid 1 2023-08-27T20:52:13.353 INFO:tasks.rados.rados.0.smithi093.stdout:538: writing smithi093153278-399 from 2004906 to 2546407 tid 2 2023-08-27T20:52:13.353 INFO:tasks.rados.rados.0.smithi093.stdout:534: done (4 left) 2023-08-27T20:52:13.353 INFO:tasks.rados.rados.0.smithi093.stdout:535: done (3 left) 2023-08-27T20:52:13.353 INFO:tasks.rados.rados.0.smithi093.stdout:536: done (2 left) 2023-08-27T20:52:13.353 INFO:tasks.rados.rados.0.smithi093.stdout:539: write oid 104 current snap is 0 2023-08-27T20:52:13.353 INFO:tasks.rados.rados.0.smithi093.stdout:537: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:52:13.475 INFO:tasks.rados.rados.0.smithi093.stdout:539: seq_num 524 ranges {406868=542480,1547035=401512,2690354=116514} 2023-08-27T20:52:13.481 INFO:tasks.rados.rados.0.smithi093.stdout:539: writing smithi093153278-104 from 406868 to 949348 tid 1 2023-08-27T20:52:13.486 INFO:tasks.rados.rados.0.smithi093.stdout:539: writing smithi093153278-104 from 1547035 to 1948547 tid 2 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:539: writing smithi093153278-104 from 2690354 to 2806868 tid 3 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:537: done (2 left) 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:540: read oid 374 snap -1 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:540: expect (ObjNum 373 snap 0 seq_num 373) 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:541: read oid 432 snap -1 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:541: expect (ObjNum 431 snap 0 seq_num 431) 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:542: write oid 305 current snap is 0 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:538: finishing write tid 1 to smithi093153278-399 2023-08-27T20:52:13.492 INFO:tasks.rados.rados.0.smithi093.stdout:538: finishing write tid 2 to smithi093153278-399 2023-08-27T20:52:13.493 INFO:tasks.rados.rados.0.smithi093.stdout:542: seq_num 525 ranges {423675=734082,1721870=545170,2717777=105898} 2023-08-27T20:52:13.498 INFO:tasks.rados.rados.0.smithi093.stdout:542: writing smithi093153278-305 from 423675 to 1157757 tid 1 2023-08-27T20:52:13.505 INFO:tasks.rados.rados.0.smithi093.stdout:542: writing smithi093153278-305 from 1721870 to 2267040 tid 2 2023-08-27T20:52:13.508 INFO:tasks.rados.rados.0.smithi093.stdout:542: writing smithi093153278-305 from 2717777 to 2823675 tid 3 2023-08-27T20:52:13.509 INFO:tasks.rados.rados.0.smithi093.stdout:543: read oid 449 snap -1 2023-08-27T20:52:13.509 INFO:tasks.rados.rados.0.smithi093.stdout:543: expect (ObjNum 448 snap 0 seq_num 448) 2023-08-27T20:52:13.510 INFO:tasks.rados.rados.0.smithi093.stdout:544: write oid 218 current snap is 0 2023-08-27T20:52:13.510 INFO:tasks.rados.rados.0.smithi093.stdout:538: finishing write tid 3 to smithi093153278-399 2023-08-27T20:52:13.510 INFO:tasks.rados.rados.0.smithi093.stdout:538: finishing write tid 4 to smithi093153278-399 2023-08-27T20:52:13.510 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 399 v 114 (ObjNum 523 snap 0 seq_num 523) dirty exists 2023-08-27T20:52:13.510 INFO:tasks.rados.rados.0.smithi093.stdout:538: left oid 399 (ObjNum 523 snap 0 seq_num 523) 2023-08-27T20:52:13.510 INFO:tasks.rados.rados.0.smithi093.stdout:539: finishing write tid 1 to smithi093153278-104 2023-08-27T20:52:13.510 INFO:tasks.rados.rados.0.smithi093.stdout:539: finishing write tid 2 to smithi093153278-104 2023-08-27T20:52:13.511 INFO:tasks.rados.rados.0.smithi093.stdout:539: finishing write tid 3 to smithi093153278-104 2023-08-27T20:52:13.511 INFO:tasks.rados.rados.0.smithi093.stdout:539: finishing write tid 4 to smithi093153278-104 2023-08-27T20:52:13.511 INFO:tasks.rados.rados.0.smithi093.stdout:539: finishing write tid 5 to smithi093153278-104 2023-08-27T20:52:13.511 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 104 v 118 (ObjNum 524 snap 0 seq_num 524) dirty exists 2023-08-27T20:52:13.511 INFO:tasks.rados.rados.0.smithi093.stdout:539: left oid 104 (ObjNum 524 snap 0 seq_num 524) 2023-08-27T20:52:13.511 INFO:tasks.rados.rados.0.smithi093.stdout:541: expect (ObjNum 431 snap 0 seq_num 431) 2023-08-27T20:52:13.618 INFO:tasks.rados.rados.0.smithi093.stdout:540: expect (ObjNum 373 snap 0 seq_num 373) 2023-08-27T20:52:13.693 INFO:tasks.rados.rados.0.smithi093.stdout:542: finishing write tid 1 to smithi093153278-305 2023-08-27T20:52:13.693 INFO:tasks.rados.rados.0.smithi093.stdout:542: finishing write tid 2 to smithi093153278-305 2023-08-27T20:52:13.693 INFO:tasks.rados.rados.0.smithi093.stdout:544: seq_num 526 ranges {440482=409331,1380352=405181,2461553=378929} 2023-08-27T20:52:13.697 INFO:tasks.rados.rados.0.smithi093.stdout:544: writing smithi093153278-218 from 440482 to 849813 tid 1 2023-08-27T20:52:13.703 INFO:tasks.rados.rados.0.smithi093.stdout:544: writing smithi093153278-218 from 1380352 to 1785533 tid 2 2023-08-27T20:52:13.710 INFO:tasks.rados.rados.0.smithi093.stdout:544: writing smithi093153278-218 from 2461553 to 2840482 tid 3 2023-08-27T20:52:13.710 INFO:tasks.rados.rados.0.smithi093.stdout:538: done (6 left) 2023-08-27T20:52:13.710 INFO:tasks.rados.rados.0.smithi093.stdout:539: done (5 left) 2023-08-27T20:52:13.710 INFO:tasks.rados.rados.0.smithi093.stdout:540: done (4 left) 2023-08-27T20:52:13.711 INFO:tasks.rados.rados.0.smithi093.stdout:541: done (3 left) 2023-08-27T20:52:13.711 INFO:tasks.rados.rados.0.smithi093.stdout:545: write oid 126 current snap is 0 2023-08-27T20:52:13.711 INFO:tasks.rados.rados.0.smithi093.stdout:545: seq_num 527 ranges {457289=484580,1438834=548839,2772623=84666} 2023-08-27T20:52:13.714 INFO:tasks.rados.rados.0.smithi093.stdout:545: writing smithi093153278-126 from 457289 to 941869 tid 1 2023-08-27T20:52:13.720 INFO:tasks.rados.rados.0.smithi093.stdout:545: writing smithi093153278-126 from 1438834 to 1987673 tid 2 2023-08-27T20:52:13.725 INFO:tasks.rados.rados.0.smithi093.stdout:545: writing smithi093153278-126 from 2772623 to 2857289 tid 3 2023-08-27T20:52:13.725 INFO:tasks.rados.rados.0.smithi093.stdout:546: read oid 231 snap -1 2023-08-27T20:52:13.725 INFO:tasks.rados.rados.0.smithi093.stdout:546: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:52:13.725 INFO:tasks.rados.rados.0.smithi093.stdout:547: write oid 245 current snap is 0 2023-08-27T20:52:13.725 INFO:tasks.rados.rados.0.smithi093.stdout:543: expect (ObjNum 448 snap 0 seq_num 448) 2023-08-27T20:52:13.840 INFO:tasks.rados.rados.0.smithi093.stdout:542: finishing write tid 3 to smithi093153278-305 2023-08-27T20:52:13.840 INFO:tasks.rados.rados.0.smithi093.stdout:542: finishing write tid 4 to smithi093153278-305 2023-08-27T20:52:13.840 INFO:tasks.rados.rados.0.smithi093.stdout:542: finishing write tid 5 to smithi093153278-305 2023-08-27T20:52:13.840 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 305 v 112 (ObjNum 525 snap 0 seq_num 525) dirty exists 2023-08-27T20:52:13.840 INFO:tasks.rados.rados.0.smithi093.stdout:542: left oid 305 (ObjNum 525 snap 0 seq_num 525) 2023-08-27T20:52:13.840 INFO:tasks.rados.rados.0.smithi093.stdout:547: seq_num 528 ranges {474096=559829,1780963=408850,2800046=74050} 2023-08-27T20:52:13.845 INFO:tasks.rados.rados.0.smithi093.stdout:547: writing smithi093153278-245 from 474096 to 1033925 tid 1 2023-08-27T20:52:13.852 INFO:tasks.rados.rados.0.smithi093.stdout:547: writing smithi093153278-245 from 1780963 to 2189813 tid 2 2023-08-27T20:52:13.856 INFO:tasks.rados.rados.0.smithi093.stdout:547: writing smithi093153278-245 from 2800046 to 2874096 tid 3 2023-08-27T20:52:13.856 INFO:tasks.rados.rados.0.smithi093.stdout:542: done (5 left) 2023-08-27T20:52:13.856 INFO:tasks.rados.rados.0.smithi093.stdout:543: done (4 left) 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:548: read oid 153 snap -1 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:548: expect (ObjNum 152 snap 0 seq_num 152) 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:549: write oid 107 current snap is 0 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:544: finishing write tid 1 to smithi093153278-218 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:544: finishing write tid 2 to smithi093153278-218 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:544: finishing write tid 3 to smithi093153278-218 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:544: finishing write tid 4 to smithi093153278-218 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:544: finishing write tid 5 to smithi093153278-218 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 218 v 122 (ObjNum 526 snap 0 seq_num 526) dirty exists 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:544: left oid 218 (ObjNum 526 snap 0 seq_num 526) 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:545: finishing write tid 1 to smithi093153278-126 2023-08-27T20:52:13.857 INFO:tasks.rados.rados.0.smithi093.stdout:545: finishing write tid 2 to smithi093153278-126 2023-08-27T20:52:13.858 INFO:tasks.rados.rados.0.smithi093.stdout:545: finishing write tid 3 to smithi093153278-126 2023-08-27T20:52:13.858 INFO:tasks.rados.rados.0.smithi093.stdout:545: finishing write tid 4 to smithi093153278-126 2023-08-27T20:52:13.858 INFO:tasks.rados.rados.0.smithi093.stdout:545: finishing write tid 5 to smithi093153278-126 2023-08-27T20:52:13.858 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 103 (ObjNum 527 snap 0 seq_num 527) dirty exists 2023-08-27T20:52:13.858 INFO:tasks.rados.rados.0.smithi093.stdout:545: left oid 126 (ObjNum 527 snap 0 seq_num 527) 2023-08-27T20:52:13.858 INFO:tasks.rados.rados.0.smithi093.stdout:546: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:52:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:13 smithi136 ceph-mon[113531]: pgmap v61: 89 pgs: 89 active+clean; 1.4 GiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 65 MiB/s wr, 333 op/s 2023-08-27T20:52:13.987 INFO:tasks.rados.rados.0.smithi093.stdout:547: finishing write tid 1 to smithi093153278-245 2023-08-27T20:52:13.987 INFO:tasks.rados.rados.0.smithi093.stdout:547: finishing write tid 2 to smithi093153278-245 2023-08-27T20:52:13.987 INFO:tasks.rados.rados.0.smithi093.stdout:547: finishing write tid 3 to smithi093153278-245 2023-08-27T20:52:13.987 INFO:tasks.rados.rados.0.smithi093.stdout:547: finishing write tid 4 to smithi093153278-245 2023-08-27T20:52:13.987 INFO:tasks.rados.rados.0.smithi093.stdout:547: finishing write tid 5 to smithi093153278-245 2023-08-27T20:52:13.988 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 245 v 139 (ObjNum 528 snap 0 seq_num 528) dirty exists 2023-08-27T20:52:13.988 INFO:tasks.rados.rados.0.smithi093.stdout:547: left oid 245 (ObjNum 528 snap 0 seq_num 528) 2023-08-27T20:52:13.988 INFO:tasks.rados.rados.0.smithi093.stdout:548: expect (ObjNum 152 snap 0 seq_num 152) 2023-08-27T20:52:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:13 smithi093 ceph-mon[111335]: pgmap v61: 89 pgs: 89 active+clean; 1.4 GiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 65 MiB/s wr, 333 op/s 2023-08-27T20:52:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:13 smithi093 ceph-mon[118902]: pgmap v61: 89 pgs: 89 active+clean; 1.4 GiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 6.7 MiB/s rd, 65 MiB/s wr, 333 op/s 2023-08-27T20:52:14.070 INFO:tasks.rados.rados.0.smithi093.stdout:549: seq_num 529 ranges {490903=635078,1839445=552508} 2023-08-27T20:52:14.075 INFO:tasks.rados.rados.0.smithi093.stdout:549: writing smithi093153278-107 from 490903 to 1125981 tid 1 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:549: writing smithi093153278-107 from 1839445 to 2391953 tid 2 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:544: done (5 left) 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:545: done (4 left) 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:546: done (3 left) 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:547: done (2 left) 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:548: done (1 left) 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:550: read oid 393 snap -1 2023-08-27T20:52:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:550: expect (ObjNum 392 snap 0 seq_num 392) 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:551: read oid 452 snap -1 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:551: expect (ObjNum 451 snap 0 seq_num 451) 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:552: read oid 410 snap -1 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:552: expect (ObjNum 409 snap 0 seq_num 409) 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:549: finishing write tid 1 to smithi093153278-107 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:553: read oid 194 snap -1 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:553: expect (ObjNum 193 snap 0 seq_num 193) 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:554: write oid 476 current snap is 0 2023-08-27T20:52:14.085 INFO:tasks.rados.rados.0.smithi093.stdout:554: seq_num 530 ranges {507710=710327,1897927=412519,2854892=52818} 2023-08-27T20:52:14.092 INFO:tasks.rados.rados.0.smithi093.stdout:554: writing smithi093153278-476 from 507710 to 1218037 tid 1 2023-08-27T20:52:14.099 INFO:tasks.rados.rados.0.smithi093.stdout:554: writing smithi093153278-476 from 1897927 to 2310446 tid 2 2023-08-27T20:52:14.103 INFO:tasks.rados.rados.0.smithi093.stdout:554: writing smithi093153278-476 from 2854892 to 2907710 tid 3 2023-08-27T20:52:14.103 INFO:tasks.rados.rados.0.smithi093.stdout:555: read oid 414 snap -1 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:555: expect (ObjNum 413 snap 0 seq_num 413) 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:556: write oid 32 current snap is 0 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:549: finishing write tid 2 to smithi093153278-107 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:549: finishing write tid 3 to smithi093153278-107 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:549: finishing write tid 4 to smithi093153278-107 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 107 v 108 (ObjNum 529 snap 0 seq_num 529) dirty exists 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:549: left oid 107 (ObjNum 529 snap 0 seq_num 529) 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:554: finishing write tid 1 to smithi093153278-476 2023-08-27T20:52:14.104 INFO:tasks.rados.rados.0.smithi093.stdout:552: expect (ObjNum 409 snap 0 seq_num 409) 2023-08-27T20:52:14.202 INFO:tasks.rados.rados.0.smithi093.stdout:554: finishing write tid 2 to smithi093153278-476 2023-08-27T20:52:14.203 INFO:tasks.rados.rados.0.smithi093.stdout:556: seq_num 531 ranges {524517=785576,1956409=556177} 2023-08-27T20:52:14.209 INFO:tasks.rados.rados.0.smithi093.stdout:556: writing smithi093153278-32 from 524517 to 1310093 tid 1 2023-08-27T20:52:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:556: writing smithi093153278-32 from 1956409 to 2512586 tid 2 2023-08-27T20:52:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:549: done (7 left) 2023-08-27T20:52:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:552: done (6 left) 2023-08-27T20:52:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:557: read oid 14 snap -1 2023-08-27T20:52:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:557: expect (ObjNum 13 snap 0 seq_num 13) 2023-08-27T20:52:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:558: write oid 229 current snap is 0 2023-08-27T20:52:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:551: expect (ObjNum 451 snap 0 seq_num 451) 2023-08-27T20:52:14.341 INFO:tasks.rados.rados.0.smithi093.stdout:554: finishing write tid 3 to smithi093153278-476 2023-08-27T20:52:14.341 INFO:tasks.rados.rados.0.smithi093.stdout:554: finishing write tid 4 to smithi093153278-476 2023-08-27T20:52:14.341 INFO:tasks.rados.rados.0.smithi093.stdout:554: finishing write tid 5 to smithi093153278-476 2023-08-27T20:52:14.341 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 476 v 135 (ObjNum 530 snap 0 seq_num 530) dirty exists 2023-08-27T20:52:14.341 INFO:tasks.rados.rados.0.smithi093.stdout:554: left oid 476 (ObjNum 530 snap 0 seq_num 530) 2023-08-27T20:52:14.341 INFO:tasks.rados.rados.0.smithi093.stdout:555: expect (ObjNum 413 snap 0 seq_num 413) 2023-08-27T20:52:14.441 INFO:tasks.rados.rados.0.smithi093.stdout:550: expect (ObjNum 392 snap 0 seq_num 392) 2023-08-27T20:52:14.528 INFO:tasks.rados.rados.0.smithi093.stdout:553: expect (ObjNum 193 snap 0 seq_num 193) 2023-08-27T20:52:14.636 INFO:tasks.rados.rados.0.smithi093.stdout:556: finishing write tid 1 to smithi093153278-32 2023-08-27T20:52:14.636 INFO:tasks.rados.rados.0.smithi093.stdout:558: seq_num 532 ranges {541324=460825,1498538=416188,2393385=547939} 2023-08-27T20:52:14.642 INFO:tasks.rados.rados.0.smithi093.stdout:558: writing smithi093153278-229 from 541324 to 1002149 tid 1 2023-08-27T20:52:14.647 INFO:tasks.rados.rados.0.smithi093.stdout:558: writing smithi093153278-229 from 1498538 to 1914726 tid 2 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:558: writing smithi093153278-229 from 2393385 to 2941324 tid 3 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:550: done (7 left) 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:551: done (6 left) 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:553: done (5 left) 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:554: done (4 left) 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:555: done (3 left) 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:559: delete oid 216 current snap is 0 2023-08-27T20:52:14.653 INFO:tasks.rados.rados.0.smithi093.stdout:557: expect (ObjNum 13 snap 0 seq_num 13) 2023-08-27T20:52:14.730 INFO:tasks.rados.rados.0.smithi093.stdout:556: finishing write tid 2 to smithi093153278-32 2023-08-27T20:52:14.730 INFO:tasks.rados.rados.0.smithi093.stdout:556: finishing write tid 3 to smithi093153278-32 2023-08-27T20:52:14.730 INFO:tasks.rados.rados.0.smithi093.stdout:556: finishing write tid 4 to smithi093153278-32 2023-08-27T20:52:14.730 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 32 v 138 (ObjNum 531 snap 0 seq_num 531) dirty exists 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:556: left oid 32 (ObjNum 531 snap 0 seq_num 531) 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:556: done (3 left) 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:557: done (2 left) 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:559: done (1 left) 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:560: read oid 371 snap -1 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:560: expect (ObjNum 370 snap 0 seq_num 370) 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:561: write oid 90 current snap is 0 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:558: finishing write tid 1 to smithi093153278-229 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:558: finishing write tid 2 to smithi093153278-229 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:558: finishing write tid 3 to smithi093153278-229 2023-08-27T20:52:14.731 INFO:tasks.rados.rados.0.smithi093.stdout:561: seq_num 534 ranges {574938=727676,1731855=703504,2848231=126707} 2023-08-27T20:52:14.737 INFO:tasks.rados.rados.0.smithi093.stdout:561: writing smithi093153278-90 from 574938 to 1302614 tid 1 2023-08-27T20:52:14.743 INFO:tasks.rados.rados.0.smithi093.stdout:561: writing smithi093153278-90 from 1731855 to 2435359 tid 2 2023-08-27T20:52:14.746 INFO:tasks.rados.rados.0.smithi093.stdout:561: writing smithi093153278-90 from 2848231 to 2974938 tid 3 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:562: read oid 300 snap -1 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:562: expect (ObjNum 299 snap 0 seq_num 299) 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:563: write oid 32 current snap is 0 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:558: finishing write tid 4 to smithi093153278-229 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:558: finishing write tid 5 to smithi093153278-229 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 229 v 104 (ObjNum 532 snap 0 seq_num 532) dirty exists 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:558: left oid 229 (ObjNum 532 snap 0 seq_num 532) 2023-08-27T20:52:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:563: seq_num 535 ranges {591745=402925,1790337=563515} 2023-08-27T20:52:14.752 INFO:tasks.rados.rados.0.smithi093.stdout:563: writing smithi093153278-32 from 591745 to 994670 tid 1 2023-08-27T20:52:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:563: writing smithi093153278-32 from 1790337 to 2353852 tid 2 2023-08-27T20:52:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:558: done (4 left) 2023-08-27T20:52:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:564: read oid 335 snap -1 2023-08-27T20:52:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:564: expect (ObjNum 334 snap 0 seq_num 334) 2023-08-27T20:52:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:565: read oid 406 snap -1 2023-08-27T20:52:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:565: expect (ObjNum 405 snap 0 seq_num 405) 2023-08-27T20:52:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:566: write oid 278 current snap is 0 2023-08-27T20:52:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:566: seq_num 536 ranges {608552=478174,1848819=707173} 2023-08-27T20:52:14.766 INFO:tasks.rados.rados.0.smithi093.stdout:566: writing smithi093153278-278 from 608552 to 1086726 tid 1 2023-08-27T20:52:14.775 INFO:tasks.rados.rados.0.smithi093.stdout:566: writing smithi093153278-278 from 1848819 to 2555992 tid 2 2023-08-27T20:52:14.775 INFO:tasks.rados.rados.0.smithi093.stdout:567: write oid 448 current snap is 0 2023-08-27T20:52:14.775 INFO:tasks.rados.rados.0.smithi093.stdout:560: expect (ObjNum 370 snap 0 seq_num 370) 2023-08-27T20:52:14.848 INFO:tasks.rados.rados.0.smithi093.stdout:561: finishing write tid 1 to smithi093153278-90 2023-08-27T20:52:14.848 INFO:tasks.rados.rados.0.smithi093.stdout:561: finishing write tid 2 to smithi093153278-90 2023-08-27T20:52:14.848 INFO:tasks.rados.rados.0.smithi093.stdout:561: finishing write tid 3 to smithi093153278-90 2023-08-27T20:52:14.848 INFO:tasks.rados.rados.0.smithi093.stdout:561: finishing write tid 4 to smithi093153278-90 2023-08-27T20:52:14.849 INFO:tasks.rados.rados.0.smithi093.stdout:561: finishing write tid 5 to smithi093153278-90 2023-08-27T20:52:14.849 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 90 v 132 (ObjNum 534 snap 0 seq_num 534) dirty exists 2023-08-27T20:52:14.849 INFO:tasks.rados.rados.0.smithi093.stdout:561: left oid 90 (ObjNum 534 snap 0 seq_num 534) 2023-08-27T20:52:14.849 INFO:tasks.rados.rados.0.smithi093.stdout:563: finishing write tid 1 to smithi093153278-32 2023-08-27T20:52:14.849 INFO:tasks.rados.rados.0.smithi093.stdout:567: seq_num 537 ranges {625359=553423,1790948=567184,2930500=94859} 2023-08-27T20:52:14.854 INFO:tasks.rados.rados.0.smithi093.stdout:567: writing smithi093153278-448 from 625359 to 1178782 tid 1 2023-08-27T20:52:14.861 INFO:tasks.rados.rados.0.smithi093.stdout:567: writing smithi093153278-448 from 1790948 to 2358132 tid 2 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:567: writing smithi093153278-448 from 2930500 to 3025359 tid 3 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:560: done (7 left) 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:561: done (6 left) 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:568: read oid 241 snap -1 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:568: expect (ObjNum 240 snap 0 seq_num 240) 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:569: read oid 43 snap -1 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:569: expect (ObjNum 42 snap 0 seq_num 42) 2023-08-27T20:52:14.865 INFO:tasks.rados.rados.0.smithi093.stdout:570: write oid 449 current snap is 0 2023-08-27T20:52:14.866 INFO:tasks.rados.rados.0.smithi093.stdout:570: seq_num 538 ranges {642166=628672,1849430=710842} 2023-08-27T20:52:14.872 INFO:tasks.rados.rados.0.smithi093.stdout:570: writing smithi093153278-449 from 642166 to 1270838 tid 1 2023-08-27T20:52:14.879 INFO:tasks.rados.rados.0.smithi093.stdout:570: writing smithi093153278-449 from 1849430 to 2560272 tid 2 2023-08-27T20:52:14.879 INFO:tasks.rados.rados.0.smithi093.stdout:571: read oid 388 snap -1 2023-08-27T20:52:14.880 INFO:tasks.rados.rados.0.smithi093.stdout:571: expect (ObjNum 387 snap 0 seq_num 387) 2023-08-27T20:52:14.880 INFO:tasks.rados.rados.0.smithi093.stdout:572: write oid 226 current snap is 0 2023-08-27T20:52:14.880 INFO:tasks.rados.rados.0.smithi093.stdout:572: seq_num 539 ranges {658973=703921,1907912=570853,2985346=73627} 2023-08-27T20:52:14.887 INFO:tasks.rados.rados.0.smithi093.stdout:572: writing smithi093153278-226 from 658973 to 1362894 tid 1 2023-08-27T20:52:14.893 INFO:tasks.rados.rados.0.smithi093.stdout:572: writing smithi093153278-226 from 1907912 to 2478765 tid 2 2023-08-27T20:52:14.896 INFO:tasks.rados.rados.0.smithi093.stdout:572: writing smithi093153278-226 from 2985346 to 3058973 tid 3 2023-08-27T20:52:14.896 INFO:tasks.rados.rados.0.smithi093.stdout:573: write oid 106 current snap is 0 2023-08-27T20:52:14.897 INFO:tasks.rados.rados.0.smithi093.stdout:565: expect (ObjNum 405 snap 0 seq_num 405) 2023-08-27T20:52:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4168593921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4168593921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:14.989 INFO:tasks.rados.rados.0.smithi093.stdout:563: finishing write tid 2 to smithi093153278-32 2023-08-27T20:52:14.990 INFO:tasks.rados.rados.0.smithi093.stdout:563: finishing write tid 3 to smithi093153278-32 2023-08-27T20:52:14.990 INFO:tasks.rados.rados.0.smithi093.stdout:563: finishing write tid 4 to smithi093153278-32 2023-08-27T20:52:14.990 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 32 v 141 (ObjNum 535 snap 0 seq_num 535) dirty exists 2023-08-27T20:52:14.990 INFO:tasks.rados.rados.0.smithi093.stdout:563: left oid 32 (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:52:14.990 INFO:tasks.rados.rados.0.smithi093.stdout:573: seq_num 540 ranges {675780=495523,1682747=714511,3012769=63011} 2023-08-27T20:52:14.995 INFO:tasks.rados.rados.0.smithi093.stdout:573: writing smithi093153278-106 from 675780 to 1171303 tid 1 2023-08-27T20:52:15.002 INFO:tasks.rados.rados.0.smithi093.stdout:573: writing smithi093153278-106 from 1682747 to 2397258 tid 2 2023-08-27T20:52:15.006 INFO:tasks.rados.rados.0.smithi093.stdout:573: writing smithi093153278-106 from 3012769 to 3075780 tid 3 2023-08-27T20:52:15.006 INFO:tasks.rados.rados.0.smithi093.stdout:563: done (11 left) 2023-08-27T20:52:15.006 INFO:tasks.rados.rados.0.smithi093.stdout:565: done (10 left) 2023-08-27T20:52:15.006 INFO:tasks.rados.rados.0.smithi093.stdout:574: write oid 339 current snap is 0 2023-08-27T20:52:15.007 INFO:tasks.rados.rados.0.smithi093.stdout:564: expect (ObjNum 334 snap 0 seq_num 334) 2023-08-27T20:52:15.121 INFO:tasks.rados.rados.0.smithi093.stdout:566: finishing write tid 1 to smithi093153278-278 2023-08-27T20:52:15.122 INFO:tasks.rados.rados.0.smithi093.stdout:562: expect (ObjNum 299 snap 0 seq_num 299) 2023-08-27T20:52:15.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4168593921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:15.225 INFO:tasks.rados.rados.0.smithi093.stdout:566: finishing write tid 2 to smithi093153278-278 2023-08-27T20:52:15.225 INFO:tasks.rados.rados.0.smithi093.stdout:566: finishing write tid 3 to smithi093153278-278 2023-08-27T20:52:15.225 INFO:tasks.rados.rados.0.smithi093.stdout:566: finishing write tid 4 to smithi093153278-278 2023-08-27T20:52:15.225 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 278 v 142 (ObjNum 536 snap 0 seq_num 536) dirty exists 2023-08-27T20:52:15.225 INFO:tasks.rados.rados.0.smithi093.stdout:566: left oid 278 (ObjNum 536 snap 0 seq_num 536) 2023-08-27T20:52:15.225 INFO:tasks.rados.rados.0.smithi093.stdout:574: seq_num 541 ranges {692587=570772,2024876=574522,3040192=52395} 2023-08-27T20:52:15.231 INFO:tasks.rados.rados.0.smithi093.stdout:574: writing smithi093153278-339 from 692587 to 1263359 tid 1 2023-08-27T20:52:15.239 INFO:tasks.rados.rados.0.smithi093.stdout:574: writing smithi093153278-339 from 2024876 to 2599398 tid 2 2023-08-27T20:52:15.242 INFO:tasks.rados.rados.0.smithi093.stdout:574: writing smithi093153278-339 from 3040192 to 3092587 tid 3 2023-08-27T20:52:15.242 INFO:tasks.rados.rados.0.smithi093.stdout:562: done (10 left) 2023-08-27T20:52:15.242 INFO:tasks.rados.rados.0.smithi093.stdout:564: done (9 left) 2023-08-27T20:52:15.242 INFO:tasks.rados.rados.0.smithi093.stdout:566: done (8 left) 2023-08-27T20:52:15.242 INFO:tasks.rados.rados.0.smithi093.stdout:575: read oid 21 snap -1 2023-08-27T20:52:15.242 INFO:tasks.rados.rados.0.smithi093.stdout:575: expect (ObjNum 509 snap 0 seq_num 509) 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:576: delete oid 434 current snap is 0 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:567: finishing write tid 1 to smithi093153278-448 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:567: finishing write tid 2 to smithi093153278-448 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:567: finishing write tid 3 to smithi093153278-448 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:567: finishing write tid 4 to smithi093153278-448 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:567: finishing write tid 5 to smithi093153278-448 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 448 v 117 (ObjNum 537 snap 0 seq_num 537) dirty exists 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:567: left oid 448 (ObjNum 537 snap 0 seq_num 537) 2023-08-27T20:52:15.243 INFO:tasks.rados.rados.0.smithi093.stdout:569: expect (ObjNum 42 snap 0 seq_num 42) 2023-08-27T20:52:15.336 INFO:tasks.rados.rados.0.smithi093.stdout:568: expect (ObjNum 240 snap 0 seq_num 240) 2023-08-27T20:52:15.401 INFO:tasks.rados.rados.0.smithi093.stdout:570: finishing write tid 1 to smithi093153278-449 2023-08-27T20:52:15.401 INFO:tasks.rados.rados.0.smithi093.stdout:570: finishing write tid 2 to smithi093153278-449 2023-08-27T20:52:15.401 INFO:tasks.rados.rados.0.smithi093.stdout:570: finishing write tid 3 to smithi093153278-449 2023-08-27T20:52:15.401 INFO:tasks.rados.rados.0.smithi093.stdout:570: finishing write tid 4 to smithi093153278-449 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 135 (ObjNum 538 snap 0 seq_num 538) dirty exists 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:570: left oid 449 (ObjNum 538 snap 0 seq_num 538) 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:567: done (9 left) 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:568: done (8 left) 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:569: done (7 left) 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:570: done (6 left) 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:576: done (5 left) 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:577: write oid 371 current snap is 0 2023-08-27T20:52:15.402 INFO:tasks.rados.rados.0.smithi093.stdout:571: expect (ObjNum 387 snap 0 seq_num 387) 2023-08-27T20:52:15.486 INFO:tasks.rados.rados.0.smithi093.stdout:572: finishing write tid 1 to smithi093153278-226 2023-08-27T20:52:15.486 INFO:tasks.rados.rados.0.smithi093.stdout:572: finishing write tid 2 to smithi093153278-226 2023-08-27T20:52:15.486 INFO:tasks.rados.rados.0.smithi093.stdout:572: finishing write tid 3 to smithi093153278-226 2023-08-27T20:52:15.486 INFO:tasks.rados.rados.0.smithi093.stdout:577: seq_num 543 ranges {726201=721270,2141840=578191} 2023-08-27T20:52:15.494 INFO:tasks.rados.rados.0.smithi093.stdout:577: writing smithi093153278-371 from 726201 to 1447471 tid 1 2023-08-27T20:52:15.501 INFO:tasks.rados.rados.0.smithi093.stdout:577: writing smithi093153278-371 from 2141840 to 2720031 tid 2 2023-08-27T20:52:15.501 INFO:tasks.rados.rados.0.smithi093.stdout:571: done (5 left) 2023-08-27T20:52:15.501 INFO:tasks.rados.rados.0.smithi093.stdout:578: write oid 213 current snap is 0 2023-08-27T20:52:15.501 INFO:tasks.rados.rados.0.smithi093.stdout:572: finishing write tid 4 to smithi093153278-226 2023-08-27T20:52:15.501 INFO:tasks.rados.rados.0.smithi093.stdout:578: seq_num 544 ranges {743008=796519,2200322=721849} 2023-08-27T20:52:15.509 INFO:tasks.rados.rados.0.smithi093.stdout:578: writing smithi093153278-213 from 743008 to 1539527 tid 1 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:578: writing smithi093153278-213 from 2200322 to 2922171 tid 2 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:579: delete oid 222 current snap is 0 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:572: finishing write tid 5 to smithi093153278-226 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 226 v 128 (ObjNum 539 snap 0 seq_num 539) dirty exists 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:572: left oid 226 (ObjNum 539 snap 0 seq_num 539) 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:573: finishing write tid 1 to smithi093153278-106 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:573: finishing write tid 2 to smithi093153278-106 2023-08-27T20:52:15.518 INFO:tasks.rados.rados.0.smithi093.stdout:573: finishing write tid 3 to smithi093153278-106 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:573: finishing write tid 4 to smithi093153278-106 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:573: finishing write tid 5 to smithi093153278-106 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 106 v 107 (ObjNum 540 snap 0 seq_num 540) dirty exists 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:573: left oid 106 (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:574: finishing write tid 1 to smithi093153278-339 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:574: finishing write tid 2 to smithi093153278-339 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:574: finishing write tid 3 to smithi093153278-339 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:574: finishing write tid 4 to smithi093153278-339 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:574: finishing write tid 5 to smithi093153278-339 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 339 v 112 (ObjNum 541 snap 0 seq_num 541) dirty exists 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:574: left oid 339 (ObjNum 541 snap 0 seq_num 541) 2023-08-27T20:52:15.519 INFO:tasks.rados.rados.0.smithi093.stdout:575: expect (ObjNum 509 snap 0 seq_num 509) 2023-08-27T20:52:15.633 INFO:tasks.rados.rados.0.smithi093.stdout:577: finishing write tid 1 to smithi093153278-371 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:577: finishing write tid 2 to smithi093153278-371 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:577: finishing write tid 3 to smithi093153278-371 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:577: finishing write tid 4 to smithi093153278-371 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 371 v 103 (ObjNum 543 snap 0 seq_num 543) dirty exists 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:577: left oid 371 (ObjNum 543 snap 0 seq_num 543) 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:578: finishing write tid 1 to smithi093153278-213 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:572: done (6 left) 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:573: done (5 left) 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:574: done (4 left) 2023-08-27T20:52:15.634 INFO:tasks.rados.rados.0.smithi093.stdout:575: done (3 left) 2023-08-27T20:52:15.635 INFO:tasks.rados.rados.0.smithi093.stdout:577: done (2 left) 2023-08-27T20:52:15.635 INFO:tasks.rados.rados.0.smithi093.stdout:579: done (1 left) 2023-08-27T20:52:15.635 INFO:tasks.rados.rados.0.smithi093.stdout:580: write oid 441 current snap is 0 2023-08-27T20:52:15.635 INFO:tasks.rados.rados.0.smithi093.stdout:578: finishing write tid 2 to smithi093153278-213 2023-08-27T20:52:15.635 INFO:tasks.rados.rados.0.smithi093.stdout:578: finishing write tid 3 to smithi093153278-213 2023-08-27T20:52:15.635 INFO:tasks.rados.rados.0.smithi093.stdout:580: seq_num 546 ranges {776622=547017,1800933=725518,2944601=232021} 2023-08-27T20:52:15.640 INFO:tasks.rados.rados.0.smithi093.stdout:580: writing smithi093153278-441 from 776622 to 1323639 tid 1 2023-08-27T20:52:15.647 INFO:tasks.rados.rados.0.smithi093.stdout:580: writing smithi093153278-441 from 1800933 to 2526451 tid 2 2023-08-27T20:52:15.651 INFO:tasks.rados.rados.0.smithi093.stdout:580: writing smithi093153278-441 from 2944601 to 3176622 tid 3 2023-08-27T20:52:15.651 INFO:tasks.rados.rados.0.smithi093.stdout:581: delete oid 291 current snap is 0 2023-08-27T20:52:15.651 INFO:tasks.rados.rados.0.smithi093.stdout:578: finishing write tid 4 to smithi093153278-213 2023-08-27T20:52:15.651 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 213 v 115 (ObjNum 544 snap 0 seq_num 544) dirty exists 2023-08-27T20:52:15.651 INFO:tasks.rados.rados.0.smithi093.stdout:578: left oid 213 (ObjNum 544 snap 0 seq_num 544) 2023-08-27T20:52:15.651 INFO:tasks.rados.rados.0.smithi093.stdout:580: finishing write tid 1 to smithi093153278-441 2023-08-27T20:52:15.652 INFO:tasks.rados.rados.0.smithi093.stdout:578: done (2 left) 2023-08-27T20:52:15.652 INFO:tasks.rados.rados.0.smithi093.stdout:581: done (1 left) 2023-08-27T20:52:15.652 INFO:tasks.rados.rados.0.smithi093.stdout:582: read oid 310 snap -1 2023-08-27T20:52:15.652 INFO:tasks.rados.rados.0.smithi093.stdout:582: expect (ObjNum 309 snap 0 seq_num 309) 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:583: read oid 288 snap -1 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:583: expect (ObjNum 287 snap 0 seq_num 287) 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:584: read oid 401 snap -1 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:584: expect (ObjNum 400 snap 0 seq_num 400) 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:585: read oid 86 snap -1 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:585: expect (ObjNum 85 snap 0 seq_num 85) 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:586: delete oid 418 current snap is 0 2023-08-27T20:52:15.653 INFO:tasks.rados.rados.0.smithi093.stdout:580: finishing write tid 2 to smithi093153278-441 2023-08-27T20:52:15.654 INFO:tasks.rados.rados.0.smithi093.stdout:580: finishing write tid 3 to smithi093153278-441 2023-08-27T20:52:15.655 INFO:tasks.rados.rados.0.smithi093.stdout:586: done (5 left) 2023-08-27T20:52:15.655 INFO:tasks.rados.rados.0.smithi093.stdout:587: read oid 235 snap -1 2023-08-27T20:52:15.655 INFO:tasks.rados.rados.0.smithi093.stdout:587: expect (ObjNum 234 snap 0 seq_num 234) 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:588: read oid 378 snap -1 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:588: expect (ObjNum 377 snap 0 seq_num 377) 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:589: read oid 271 snap -1 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:589: expect (ObjNum 270 snap 0 seq_num 270) 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:590: delete oid 153 current snap is 0 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:580: finishing write tid 4 to smithi093153278-441 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:580: finishing write tid 5 to smithi093153278-441 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 116 (ObjNum 546 snap 0 seq_num 546) dirty exists 2023-08-27T20:52:15.656 INFO:tasks.rados.rados.0.smithi093.stdout:580: left oid 441 (ObjNum 546 snap 0 seq_num 546) 2023-08-27T20:52:15.657 INFO:tasks.rados.rados.0.smithi093.stdout:580: done (8 left) 2023-08-27T20:52:15.657 INFO:tasks.rados.rados.0.smithi093.stdout:590: done (7 left) 2023-08-27T20:52:15.657 INFO:tasks.rados.rados.0.smithi093.stdout:591: write oid 46 current snap is 0 2023-08-27T20:52:15.658 INFO:tasks.rados.rados.0.smithi093.stdout:591: seq_num 550 ranges {443850=564366,1634861=732856,2770646=473204} 2023-08-27T20:52:15.663 INFO:tasks.rados.rados.0.smithi093.stdout:591: writing smithi093153278-46 from 443850 to 1008216 tid 1 2023-08-27T20:52:15.671 INFO:tasks.rados.rados.0.smithi093.stdout:591: writing smithi093153278-46 from 1634861 to 2367717 tid 2 2023-08-27T20:52:15.677 INFO:tasks.rados.rados.0.smithi093.stdout:591: writing smithi093153278-46 from 2770646 to 3243850 tid 3 2023-08-27T20:52:15.677 INFO:tasks.rados.rados.0.smithi093.stdout:592: write oid 464 current snap is 0 2023-08-27T20:52:15.677 INFO:tasks.rados.rados.0.smithi093.stdout:582: expect (ObjNum 309 snap 0 seq_num 309) 2023-08-27T20:52:15.719 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:52:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:15] "GET /metrics HTTP/1.1" 200 241913 "" "Prometheus/2.43.0" 2023-08-27T20:52:15.782 INFO:tasks.rados.rados.0.smithi093.stdout:592: seq_num 551 ranges {460657=639615,1693343=476514,2681716=578941} 2023-08-27T20:52:15.788 INFO:tasks.rados.rados.0.smithi093.stdout:592: writing smithi093153278-464 from 460657 to 1100272 tid 1 2023-08-27T20:52:15.794 INFO:tasks.rados.rados.0.smithi093.stdout:592: writing smithi093153278-464 from 1693343 to 2169857 tid 2 2023-08-27T20:52:15.801 INFO:tasks.rados.rados.0.smithi093.stdout:592: writing smithi093153278-464 from 2681716 to 3260657 tid 3 2023-08-27T20:52:15.801 INFO:tasks.rados.rados.0.smithi093.stdout:582: done (8 left) 2023-08-27T20:52:15.801 INFO:tasks.rados.rados.0.smithi093.stdout:593: delete oid 453 current snap is 0 2023-08-27T20:52:15.801 INFO:tasks.rados.rados.0.smithi093.stdout:589: expect (ObjNum 270 snap 0 seq_num 270) 2023-08-27T20:52:15.881 INFO:tasks.rados.rados.0.smithi093.stdout:588: expect (ObjNum 377 snap 0 seq_num 377) 2023-08-27T20:52:15.957 INFO:tasks.rados.rados.0.smithi093.stdout:584: expect (ObjNum 400 snap 0 seq_num 400) 2023-08-27T20:52:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:15 smithi093 ceph-mon[111335]: pgmap v62: 89 pgs: 89 active+clean; 1.4 GiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 55 MiB/s wr, 281 op/s 2023-08-27T20:52:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:15 smithi093 ceph-mon[118902]: pgmap v62: 89 pgs: 89 active+clean; 1.4 GiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 55 MiB/s wr, 281 op/s 2023-08-27T20:52:16.049 INFO:tasks.rados.rados.0.smithi093.stdout:583: expect (ObjNum 287 snap 0 seq_num 287) 2023-08-27T20:52:16.142 INFO:tasks.rados.rados.0.smithi093.stdout:591: finishing write tid 1 to smithi093153278-46 2023-08-27T20:52:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:591: finishing write tid 2 to smithi093153278-46 2023-08-27T20:52:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:583: done (8 left) 2023-08-27T20:52:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:584: done (7 left) 2023-08-27T20:52:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:588: done (6 left) 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:589: done (5 left) 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:593: done (4 left) 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:594: delete oid 31 current snap is 0 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:591: finishing write tid 3 to smithi093153278-46 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:591: finishing write tid 4 to smithi093153278-46 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:591: finishing write tid 5 to smithi093153278-46 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 46 v 146 (ObjNum 550 snap 0 seq_num 550) dirty exists 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:591: left oid 46 (ObjNum 550 snap 0 seq_num 550) 2023-08-27T20:52:16.144 INFO:tasks.rados.rados.0.smithi093.stdout:587: expect (ObjNum 234 snap 0 seq_num 234) 2023-08-27T20:52:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:15 smithi136 ceph-mon[113531]: pgmap v62: 89 pgs: 89 active+clean; 1.4 GiB data, 1.9 GiB used, 713 GiB / 715 GiB avail; 5.7 MiB/s rd, 55 MiB/s wr, 281 op/s 2023-08-27T20:52:16.278 INFO:tasks.rados.rados.0.smithi093.stdout:585: expect (ObjNum 85 snap 0 seq_num 85) 2023-08-27T20:52:16.399 INFO:tasks.rados.rados.0.smithi093.stdout:592: finishing write tid 1 to smithi093153278-464 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:592: finishing write tid 2 to smithi093153278-464 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:592: finishing write tid 3 to smithi093153278-464 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:585: done (4 left) 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:587: done (3 left) 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:591: done (2 left) 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:594: done (1 left) 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:595: delete oid 304 current snap is 0 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:592: finishing write tid 4 to smithi093153278-464 2023-08-27T20:52:16.400 INFO:tasks.rados.rados.0.smithi093.stdout:592: finishing write tid 5 to smithi093153278-464 2023-08-27T20:52:16.401 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 464 v 120 (ObjNum 551 snap 0 seq_num 551) dirty exists 2023-08-27T20:52:16.401 INFO:tasks.rados.rados.0.smithi093.stdout:592: left oid 464 (ObjNum 551 snap 0 seq_num 551) 2023-08-27T20:52:16.401 INFO:tasks.rados.rados.0.smithi093.stdout:592: done (1 left) 2023-08-27T20:52:16.401 INFO:tasks.rados.rados.0.smithi093.stdout:595: done (0 left) 2023-08-27T20:52:16.401 INFO:tasks.rados.rados.0.smithi093.stdout:596: read oid 172 snap -1 2023-08-27T20:52:16.402 INFO:tasks.rados.rados.0.smithi093.stdout:596: expect (ObjNum 171 snap 0 seq_num 171) 2023-08-27T20:52:16.402 INFO:tasks.rados.rados.0.smithi093.stdout:597: delete oid 20 current snap is 0 2023-08-27T20:52:16.402 INFO:tasks.rados.rados.0.smithi093.stdout:597: done (1 left) 2023-08-27T20:52:16.402 INFO:tasks.rados.rados.0.smithi093.stdout:598: write oid 267 current snap is 0 2023-08-27T20:52:16.402 INFO:tasks.rados.rados.0.smithi093.stdout:598: seq_num 556 ranges {544692=732213,1985753=743863,3335184=9508} 2023-08-27T20:52:16.410 INFO:tasks.rados.rados.0.smithi093.stdout:598: writing smithi093153278-267 from 544692 to 1276905 tid 1 2023-08-27T20:52:16.418 INFO:tasks.rados.rados.0.smithi093.stdout:598: writing smithi093153278-267 from 1985753 to 2729616 tid 2 2023-08-27T20:52:16.422 INFO:tasks.rados.rados.0.smithi093.stdout:598: writing smithi093153278-267 from 3335184 to 3344692 tid 3 2023-08-27T20:52:16.422 INFO:tasks.rados.rados.0.smithi093.stdout:599: write oid 240 current snap is 0 2023-08-27T20:52:16.422 INFO:tasks.rados.rados.0.smithi093.stdout:596: expect (ObjNum 171 snap 0 seq_num 171) 2023-08-27T20:52:16.521 INFO:tasks.rados.rados.0.smithi093.stdout:598: finishing write tid 1 to smithi093153278-267 2023-08-27T20:52:16.522 INFO:tasks.rados.rados.0.smithi093.stdout:598: finishing write tid 2 to smithi093153278-267 2023-08-27T20:52:16.522 INFO:tasks.rados.rados.0.smithi093.stdout:598: finishing write tid 3 to smithi093153278-267 2023-08-27T20:52:16.522 INFO:tasks.rados.rados.0.smithi093.stdout:599: seq_num 557 ranges {561499=407462,1527882=487521,2729901=557217} 2023-08-27T20:52:16.527 INFO:tasks.rados.rados.0.smithi093.stdout:599: writing smithi093153278-240 from 561499 to 968961 tid 1 2023-08-27T20:52:16.532 INFO:tasks.rados.rados.0.smithi093.stdout:599: writing smithi093153278-240 from 1527882 to 2015403 tid 2 2023-08-27T20:52:16.540 INFO:tasks.rados.rados.0.smithi093.stdout:599: writing smithi093153278-240 from 2729901 to 3287118 tid 3 2023-08-27T20:52:16.540 INFO:tasks.rados.rados.0.smithi093.stdout:596: done (2 left) 2023-08-27T20:52:16.540 INFO:tasks.rados.rados.0.smithi093.stdout:600: write oid 195 current snap is 0 2023-08-27T20:52:16.540 INFO:tasks.rados.rados.0.smithi093.stdout:598: finishing write tid 4 to smithi093153278-267 2023-08-27T20:52:16.540 INFO:tasks.rados.rados.0.smithi093.stdout:598: finishing write tid 5 to smithi093153278-267 2023-08-27T20:52:16.541 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 267 v 121 (ObjNum 556 snap 0 seq_num 556) dirty exists 2023-08-27T20:52:16.541 INFO:tasks.rados.rados.0.smithi093.stdout:598: left oid 267 (ObjNum 556 snap 0 seq_num 556) 2023-08-27T20:52:16.541 INFO:tasks.rados.rados.0.smithi093.stdout:599: finishing write tid 1 to smithi093153278-240 2023-08-27T20:52:16.541 INFO:tasks.rados.rados.0.smithi093.stdout:599: finishing write tid 2 to smithi093153278-240 2023-08-27T20:52:16.541 INFO:tasks.rados.rados.0.smithi093.stdout:600: seq_num 558 ranges {578306=482711,1586364=631179,2757324=484842} 2023-08-27T20:52:16.545 INFO:tasks.rados.rados.0.smithi093.stdout:600: writing smithi093153278-195 from 578306 to 1061017 tid 1 2023-08-27T20:52:16.552 INFO:tasks.rados.rados.0.smithi093.stdout:600: writing smithi093153278-195 from 1586364 to 2217543 tid 2 2023-08-27T20:52:16.558 INFO:tasks.rados.rados.0.smithi093.stdout:600: writing smithi093153278-195 from 2757324 to 3242166 tid 3 2023-08-27T20:52:16.558 INFO:tasks.rados.rados.0.smithi093.stdout:598: done (2 left) 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:601: read oid 169 snap -1 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:601: expect (ObjNum 168 snap 0 seq_num 168) 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:602: write oid 161 current snap is 0 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:599: finishing write tid 3 to smithi093153278-240 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:599: finishing write tid 4 to smithi093153278-240 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:599: finishing write tid 5 to smithi093153278-240 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 240 v 120 (ObjNum 557 snap 0 seq_num 557) dirty exists 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:599: left oid 240 (ObjNum 557 snap 0 seq_num 557) 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:600: finishing write tid 1 to smithi093153278-195 2023-08-27T20:52:16.559 INFO:tasks.rados.rados.0.smithi093.stdout:600: finishing write tid 2 to smithi093153278-195 2023-08-27T20:52:16.560 INFO:tasks.rados.rados.0.smithi093.stdout:602: seq_num 559 ranges {595113=557960,1644846=491190,2784747=610366} 2023-08-27T20:52:16.564 INFO:tasks.rados.rados.0.smithi093.stdout:602: writing smithi093153278-161 from 595113 to 1153073 tid 1 2023-08-27T20:52:16.570 INFO:tasks.rados.rados.0.smithi093.stdout:602: writing smithi093153278-161 from 1644846 to 2136036 tid 2 2023-08-27T20:52:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:602: writing smithi093153278-161 from 2784747 to 3395113 tid 3 2023-08-27T20:52:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:599: done (3 left) 2023-08-27T20:52:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:603: read oid 113 snap -1 2023-08-27T20:52:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:603: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:52:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:604: read oid 320 snap -1 2023-08-27T20:52:16.579 INFO:tasks.rados.rados.0.smithi093.stdout:604: expect (ObjNum 517 snap 0 seq_num 517) 2023-08-27T20:52:16.579 INFO:tasks.rados.rados.0.smithi093.stdout:605: write oid 336 current snap is 0 2023-08-27T20:52:16.579 INFO:tasks.rados.rados.0.smithi093.stdout:600: finishing write tid 3 to smithi093153278-195 2023-08-27T20:52:16.579 INFO:tasks.rados.rados.0.smithi093.stdout:605: seq_num 560 ranges {611920=633209,1703328=634848,2812170=507386} 2023-08-27T20:52:16.585 INFO:tasks.rados.rados.0.smithi093.stdout:605: writing smithi093153278-336 from 611920 to 1245129 tid 1 2023-08-27T20:52:16.591 INFO:tasks.rados.rados.0.smithi093.stdout:605: writing smithi093153278-336 from 1703328 to 2338176 tid 2 2023-08-27T20:52:16.597 INFO:tasks.rados.rados.0.smithi093.stdout:605: writing smithi093153278-336 from 2812170 to 3319556 tid 3 2023-08-27T20:52:16.597 INFO:tasks.rados.rados.0.smithi093.stdout:606: write oid 360 current snap is 0 2023-08-27T20:52:16.597 INFO:tasks.rados.rados.0.smithi093.stdout:600: finishing write tid 4 to smithi093153278-195 2023-08-27T20:52:16.597 INFO:tasks.rados.rados.0.smithi093.stdout:606: seq_num 561 ranges {628727=708458,2045457=494859,3123240=305487} 2023-08-27T20:52:16.604 INFO:tasks.rados.rados.0.smithi093.stdout:606: writing smithi093153278-360 from 628727 to 1337185 tid 1 2023-08-27T20:52:16.611 INFO:tasks.rados.rados.0.smithi093.stdout:606: writing smithi093153278-360 from 2045457 to 2540316 tid 2 2023-08-27T20:52:16.616 INFO:tasks.rados.rados.0.smithi093.stdout:606: writing smithi093153278-360 from 3123240 to 3428727 tid 3 2023-08-27T20:52:16.616 INFO:tasks.rados.rados.0.smithi093.stdout:607: read oid 436 snap -1 2023-08-27T20:52:16.616 INFO:tasks.rados.rados.0.smithi093.stdout:607: expect (ObjNum 435 snap 0 seq_num 435) 2023-08-27T20:52:16.617 INFO:tasks.rados.rados.0.smithi093.stdout:608: write oid 180 current snap is 0 2023-08-27T20:52:16.617 INFO:tasks.rados.rados.0.smithi093.stdout:600: finishing write tid 5 to smithi093153278-195 2023-08-27T20:52:16.617 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 106 (ObjNum 558 snap 0 seq_num 558) dirty exists 2023-08-27T20:52:16.617 INFO:tasks.rados.rados.0.smithi093.stdout:600: left oid 195 (ObjNum 558 snap 0 seq_num 558) 2023-08-27T20:52:16.617 INFO:tasks.rados.rados.0.smithi093.stdout:601: expect (ObjNum 168 snap 0 seq_num 168) 2023-08-27T20:52:16.723 INFO:tasks.rados.rados.0.smithi093.stdout:602: finishing write tid 1 to smithi093153278-161 2023-08-27T20:52:16.723 INFO:tasks.rados.rados.0.smithi093.stdout:602: finishing write tid 2 to smithi093153278-161 2023-08-27T20:52:16.723 INFO:tasks.rados.rados.0.smithi093.stdout:608: seq_num 562 ranges {645534=783707,2103939=638517,3150663=294871} 2023-08-27T20:52:16.731 INFO:tasks.rados.rados.0.smithi093.stdout:608: writing smithi093153278-180 from 645534 to 1429241 tid 1 2023-08-27T20:52:16.739 INFO:tasks.rados.rados.0.smithi093.stdout:608: writing smithi093153278-180 from 2103939 to 2742456 tid 2 2023-08-27T20:52:16.744 INFO:tasks.rados.rados.0.smithi093.stdout:608: writing smithi093153278-180 from 3150663 to 3445534 tid 3 2023-08-27T20:52:16.744 INFO:tasks.rados.rados.0.smithi093.stdout:600: done (8 left) 2023-08-27T20:52:16.744 INFO:tasks.rados.rados.0.smithi093.stdout:601: done (7 left) 2023-08-27T20:52:16.744 INFO:tasks.rados.rados.0.smithi093.stdout:609: read oid 272 snap -1 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:609: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:610: write oid 96 current snap is 0 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:602: finishing write tid 3 to smithi093153278-161 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:602: finishing write tid 4 to smithi093153278-161 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:602: finishing write tid 5 to smithi093153278-161 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 161 v 90 (ObjNum 559 snap 0 seq_num 559) dirty exists 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:602: left oid 161 (ObjNum 559 snap 0 seq_num 559) 2023-08-27T20:52:16.745 INFO:tasks.rados.rados.0.smithi093.stdout:604: expect (ObjNum 517 snap 0 seq_num 517) 2023-08-27T20:52:16.860 INFO:tasks.rados.rados.0.smithi093.stdout:603: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:52:17.007 INFO:tasks.rados.rados.0.smithi093.stdout:605: finishing write tid 1 to smithi093153278-336 2023-08-27T20:52:17.007 INFO:tasks.rados.rados.0.smithi093.stdout:605: finishing write tid 2 to smithi093153278-336 2023-08-27T20:52:17.007 INFO:tasks.rados.rados.0.smithi093.stdout:610: seq_num 563 ranges {662341=575309,1878774=498528,2894439=457555} 2023-08-27T20:52:17.013 INFO:tasks.rados.rados.0.smithi093.stdout:610: writing smithi093153278-96 from 662341 to 1237650 tid 1 2023-08-27T20:52:17.020 INFO:tasks.rados.rados.0.smithi093.stdout:610: writing smithi093153278-96 from 1878774 to 2377302 tid 2 2023-08-27T20:52:17.025 INFO:tasks.rados.rados.0.smithi093.stdout:610: writing smithi093153278-96 from 2894439 to 3351994 tid 3 2023-08-27T20:52:17.025 INFO:tasks.rados.rados.0.smithi093.stdout:602: done (8 left) 2023-08-27T20:52:17.026 INFO:tasks.rados.rados.0.smithi093.stdout:603: done (7 left) 2023-08-27T20:52:17.026 INFO:tasks.rados.rados.0.smithi093.stdout:604: done (6 left) 2023-08-27T20:52:17.027 INFO:tasks.rados.rados.0.smithi093.stdout:611: read oid 57 snap -1 2023-08-27T20:52:17.027 INFO:tasks.rados.rados.0.smithi093.stdout:611: expect (ObjNum 56 snap 0 seq_num 56) 2023-08-27T20:52:17.027 INFO:tasks.rados.rados.0.smithi093.stdout:612: delete oid 174 current snap is 0 2023-08-27T20:52:17.029 INFO:tasks.rados.rados.0.smithi093.stdout:605: finishing write tid 3 to smithi093153278-336 2023-08-27T20:52:17.029 INFO:tasks.rados.rados.0.smithi093.stdout:605: finishing write tid 4 to smithi093153278-336 2023-08-27T20:52:17.029 INFO:tasks.rados.rados.0.smithi093.stdout:605: finishing write tid 5 to smithi093153278-336 2023-08-27T20:52:17.029 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 336 v 124 (ObjNum 560 snap 0 seq_num 560) dirty exists 2023-08-27T20:52:17.029 INFO:tasks.rados.rados.0.smithi093.stdout:605: left oid 336 (ObjNum 560 snap 0 seq_num 560) 2023-08-27T20:52:17.029 INFO:tasks.rados.rados.0.smithi093.stdout:606: finishing write tid 1 to smithi093153278-360 2023-08-27T20:52:17.030 INFO:tasks.rados.rados.0.smithi093.stdout:606: finishing write tid 2 to smithi093153278-360 2023-08-27T20:52:17.030 INFO:tasks.rados.rados.0.smithi093.stdout:606: finishing write tid 3 to smithi093153278-360 2023-08-27T20:52:17.030 INFO:tasks.rados.rados.0.smithi093.stdout:606: finishing write tid 4 to smithi093153278-360 2023-08-27T20:52:17.030 INFO:tasks.rados.rados.0.smithi093.stdout:606: finishing write tid 5 to smithi093153278-360 2023-08-27T20:52:17.030 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 360 v 128 (ObjNum 561 snap 0 seq_num 561) dirty exists 2023-08-27T20:52:17.030 INFO:tasks.rados.rados.0.smithi093.stdout:606: left oid 360 (ObjNum 561 snap 0 seq_num 561) 2023-08-27T20:52:17.030 INFO:tasks.rados.rados.0.smithi093.stdout:607: expect (ObjNum 435 snap 0 seq_num 435) 2023-08-27T20:52:17.154 INFO:tasks.rados.rados.0.smithi093.stdout:608: finishing write tid 1 to smithi093153278-180 2023-08-27T20:52:17.154 INFO:tasks.rados.rados.0.smithi093.stdout:608: finishing write tid 2 to smithi093153278-180 2023-08-27T20:52:17.154 INFO:tasks.rados.rados.0.smithi093.stdout:609: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:17.242 INFO:tasks.rados.rados.0.smithi093.stdout:608: finishing write tid 3 to smithi093153278-180 2023-08-27T20:52:17.242 INFO:tasks.rados.rados.0.smithi093.stdout:608: finishing write tid 4 to smithi093153278-180 2023-08-27T20:52:17.242 INFO:tasks.rados.rados.0.smithi093.stdout:605: done (7 left) 2023-08-27T20:52:17.242 INFO:tasks.rados.rados.0.smithi093.stdout:606: done (6 left) 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:607: done (5 left) 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:609: done (4 left) 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:612: done (3 left) 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:613: write oid 456 current snap is 0 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:608: finishing write tid 5 to smithi093153278-180 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 180 v 132 (ObjNum 562 snap 0 seq_num 562) dirty exists 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:608: left oid 180 (ObjNum 562 snap 0 seq_num 562) 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:610: finishing write tid 1 to smithi093153278-96 2023-08-27T20:52:17.243 INFO:tasks.rados.rados.0.smithi093.stdout:613: seq_num 565 ranges {695955=725807,1879385=502197,2949285=480099} 2023-08-27T20:52:17.250 INFO:tasks.rados.rados.0.smithi093.stdout:613: writing smithi093153278-456 from 695955 to 1421762 tid 1 2023-08-27T20:52:17.255 INFO:tasks.rados.rados.0.smithi093.stdout:613: writing smithi093153278-456 from 1879385 to 2381582 tid 2 2023-08-27T20:52:17.261 INFO:tasks.rados.rados.0.smithi093.stdout:613: writing smithi093153278-456 from 2949285 to 3429384 tid 3 2023-08-27T20:52:17.261 INFO:tasks.rados.rados.0.smithi093.stdout:608: done (3 left) 2023-08-27T20:52:17.261 INFO:tasks.rados.rados.0.smithi093.stdout:614: write oid 270 current snap is 0 2023-08-27T20:52:17.261 INFO:tasks.rados.rados.0.smithi093.stdout:610: finishing write tid 2 to smithi093153278-96 2023-08-27T20:52:17.262 INFO:tasks.rados.rados.0.smithi093.stdout:614: seq_num 566 ranges {712762=401056,1537867=645855,2860355=652407} 2023-08-27T20:52:17.268 INFO:tasks.rados.rados.0.smithi093.stdout:614: writing smithi093153278-270 from 712762 to 1113818 tid 1 2023-08-27T20:52:17.274 INFO:tasks.rados.rados.0.smithi093.stdout:614: writing smithi093153278-270 from 1537867 to 2183722 tid 2 2023-08-27T20:52:17.282 INFO:tasks.rados.rados.0.smithi093.stdout:614: writing smithi093153278-270 from 2860355 to 3512762 tid 3 2023-08-27T20:52:17.282 INFO:tasks.rados.rados.0.smithi093.stdout:615: write oid 204 current snap is 0 2023-08-27T20:52:17.282 INFO:tasks.rados.rados.0.smithi093.stdout:610: finishing write tid 3 to smithi093153278-96 2023-08-27T20:52:17.282 INFO:tasks.rados.rados.0.smithi093.stdout:615: seq_num 567 ranges {729569=476305,1996349=505866,3004131=525438} 2023-08-27T20:52:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:615: writing smithi093153278-204 from 729569 to 1205874 tid 1 2023-08-27T20:52:17.297 INFO:tasks.rados.rados.0.smithi093.stdout:615: writing smithi093153278-204 from 1996349 to 2502215 tid 2 2023-08-27T20:52:17.302 INFO:tasks.rados.rados.0.smithi093.stdout:615: writing smithi093153278-204 from 3004131 to 3529569 tid 3 2023-08-27T20:52:17.303 INFO:tasks.rados.rados.0.smithi093.stdout:616: write oid 168 current snap is 0 2023-08-27T20:52:17.303 INFO:tasks.rados.rados.0.smithi093.stdout:610: finishing write tid 4 to smithi093153278-96 2023-08-27T20:52:17.303 INFO:tasks.rados.rados.0.smithi093.stdout:616: seq_num 568 ranges {746376=551554,2054831=649524,3315201=231175} 2023-08-27T20:52:17.309 INFO:tasks.rados.rados.0.smithi093.stdout:616: writing smithi093153278-168 from 746376 to 1297930 tid 1 2023-08-27T20:52:17.317 INFO:tasks.rados.rados.0.smithi093.stdout:616: writing smithi093153278-168 from 2054831 to 2704355 tid 2 2023-08-27T20:52:17.323 INFO:tasks.rados.rados.0.smithi093.stdout:616: writing smithi093153278-168 from 3315201 to 3546376 tid 3 2023-08-27T20:52:17.323 INFO:tasks.rados.rados.0.smithi093.stdout:617: write oid 448 current snap is 0 2023-08-27T20:52:17.323 INFO:tasks.rados.rados.0.smithi093.stdout:610: finishing write tid 5 to smithi093153278-96 2023-08-27T20:52:17.323 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 96 v 144 (ObjNum 563 snap 0 seq_num 563) dirty exists 2023-08-27T20:52:17.323 INFO:tasks.rados.rados.0.smithi093.stdout:610: left oid 96 (ObjNum 563 snap 0 seq_num 563) 2023-08-27T20:52:17.323 INFO:tasks.rados.rados.0.smithi093.stdout:617: seq_num 569 ranges {763183=626803,1996960=793182,3226271=336912} 2023-08-27T20:52:17.330 INFO:tasks.rados.rados.0.smithi093.stdout:617: writing smithi093153278-448 from 763183 to 1389986 tid 1 2023-08-27T20:52:17.338 INFO:tasks.rados.rados.0.smithi093.stdout:617: writing smithi093153278-448 from 1996960 to 2790142 tid 2 2023-08-27T20:52:17.342 INFO:tasks.rados.rados.0.smithi093.stdout:617: writing smithi093153278-448 from 3226271 to 3563183 tid 3 2023-08-27T20:52:17.343 INFO:tasks.rados.rados.0.smithi093.stdout:610: done (6 left) 2023-08-27T20:52:17.343 INFO:tasks.rados.rados.0.smithi093.stdout:618: write oid 388 current snap is 0 2023-08-27T20:52:17.343 INFO:tasks.rados.rados.0.smithi093.stdout:611: expect (ObjNum 56 snap 0 seq_num 56) 2023-08-27T20:52:17.446 INFO:tasks.rados.rados.0.smithi093.stdout:618: seq_num 570 ranges {779990=702052,2055442=653193,3253694=326296} 2023-08-27T20:52:17.454 INFO:tasks.rados.rados.0.smithi093.stdout:618: writing smithi093153278-388 from 779990 to 1482042 tid 1 2023-08-27T20:52:17.461 INFO:tasks.rados.rados.0.smithi093.stdout:618: writing smithi093153278-388 from 2055442 to 2708635 tid 2 2023-08-27T20:52:17.467 INFO:tasks.rados.rados.0.smithi093.stdout:618: writing smithi093153278-388 from 3253694 to 3579990 tid 3 2023-08-27T20:52:17.467 INFO:tasks.rados.rados.0.smithi093.stdout:611: done (6 left) 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:619: read oid 131 snap -1 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:619: expect (ObjNum 130 snap 0 seq_num 130) 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:620: read oid 216 snap -1 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:620: expect deleted 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:621: delete oid 437 current snap is 0 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:613: finishing write tid 1 to smithi093153278-456 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:613: finishing write tid 2 to smithi093153278-456 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:613: finishing write tid 3 to smithi093153278-456 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:613: finishing write tid 4 to smithi093153278-456 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:613: finishing write tid 5 to smithi093153278-456 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 456 v 139 (ObjNum 565 snap 0 seq_num 565) dirty exists 2023-08-27T20:52:17.468 INFO:tasks.rados.rados.0.smithi093.stdout:613: left oid 456 (ObjNum 565 snap 0 seq_num 565) 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:614: finishing write tid 1 to smithi093153278-270 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:614: finishing write tid 2 to smithi093153278-270 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:614: finishing write tid 3 to smithi093153278-270 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:614: finishing write tid 4 to smithi093153278-270 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:614: finishing write tid 5 to smithi093153278-270 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 270 v 117 (ObjNum 566 snap 0 seq_num 566) dirty exists 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:614: left oid 270 (ObjNum 566 snap 0 seq_num 566) 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:615: finishing write tid 1 to smithi093153278-204 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:615: finishing write tid 2 to smithi093153278-204 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:615: finishing write tid 3 to smithi093153278-204 2023-08-27T20:52:17.469 INFO:tasks.rados.rados.0.smithi093.stdout:615: finishing write tid 4 to smithi093153278-204 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:615: finishing write tid 5 to smithi093153278-204 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 204 v 148 (ObjNum 567 snap 0 seq_num 567) dirty exists 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:615: left oid 204 (ObjNum 567 snap 0 seq_num 567) 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:616: finishing write tid 1 to smithi093153278-168 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:616: finishing write tid 2 to smithi093153278-168 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:616: finishing write tid 3 to smithi093153278-168 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:616: finishing write tid 4 to smithi093153278-168 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:616: finishing write tid 5 to smithi093153278-168 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 168 v 109 (ObjNum 568 snap 0 seq_num 568) dirty exists 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:616: left oid 168 (ObjNum 568 snap 0 seq_num 568) 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:617: finishing write tid 1 to smithi093153278-448 2023-08-27T20:52:17.470 INFO:tasks.rados.rados.0.smithi093.stdout:617: finishing write tid 2 to smithi093153278-448 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:617: finishing write tid 3 to smithi093153278-448 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:617: finishing write tid 4 to smithi093153278-448 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:617: finishing write tid 5 to smithi093153278-448 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 448 v 125 (ObjNum 569 snap 0 seq_num 569) dirty exists 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:617: left oid 448 (ObjNum 569 snap 0 seq_num 569) 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:618: finishing write tid 1 to smithi093153278-388 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:618: finishing write tid 2 to smithi093153278-388 2023-08-27T20:52:17.471 INFO:tasks.rados.rados.0.smithi093.stdout:613: done (8 left) 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:614: done (7 left) 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:615: done (6 left) 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:616: done (5 left) 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:617: done (4 left) 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:620: done (3 left) 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:621: done (2 left) 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:622: read oid 304 snap -1 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:622: expect deleted 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:623: write oid 215 current snap is 0 2023-08-27T20:52:17.472 INFO:tasks.rados.rados.0.smithi093.stdout:623: seq_num 572 ranges {413604=568903,1488759=656862,2624893=591709} 2023-08-27T20:52:17.473 INFO:tasks.rados.rados.0.smithi093.stdout:623: writing smithi093153278-215 from 413604 to 982507 tid 1 2023-08-27T20:52:17.480 INFO:tasks.rados.rados.0.smithi093.stdout:623: writing smithi093153278-215 from 1488759 to 2145621 tid 2 2023-08-27T20:52:17.486 INFO:tasks.rados.rados.0.smithi093.stdout:623: writing smithi093153278-215 from 2624893 to 3216602 tid 3 2023-08-27T20:52:17.487 INFO:tasks.rados.rados.0.smithi093.stdout:624: read oid 491 snap -1 2023-08-27T20:52:17.487 INFO:tasks.rados.rados.0.smithi093.stdout:624: expect (ObjNum 490 snap 0 seq_num 490) 2023-08-27T20:52:17.487 INFO:tasks.rados.rados.0.smithi093.stdout:625: delete oid 290 current snap is 0 2023-08-27T20:52:17.487 INFO:tasks.rados.rados.0.smithi093.stdout:619: expect (ObjNum 130 snap 0 seq_num 130) 2023-08-27T20:52:17.564 INFO:tasks.rados.rados.0.smithi093.stdout:618: finishing write tid 3 to smithi093153278-388 2023-08-27T20:52:17.564 INFO:tasks.rados.rados.0.smithi093.stdout:618: finishing write tid 4 to smithi093153278-388 2023-08-27T20:52:17.564 INFO:tasks.rados.rados.0.smithi093.stdout:619: done (5 left) 2023-08-27T20:52:17.564 INFO:tasks.rados.rados.0.smithi093.stdout:622: done (4 left) 2023-08-27T20:52:17.564 INFO:tasks.rados.rados.0.smithi093.stdout:625: done (3 left) 2023-08-27T20:52:17.564 INFO:tasks.rados.rados.0.smithi093.stdout:626: read oid 165 snap -1 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:626: expect (ObjNum 507 snap 0 seq_num 507) 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:627: read oid 55 snap -1 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:627: expect (ObjNum 519 snap 0 seq_num 519) 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:628: write oid 217 current snap is 0 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:618: finishing write tid 5 to smithi093153278-388 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 388 v 134 (ObjNum 570 snap 0 seq_num 570) dirty exists 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:618: left oid 388 (ObjNum 570 snap 0 seq_num 570) 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:623: finishing write tid 1 to smithi093153278-215 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:623: finishing write tid 2 to smithi093153278-215 2023-08-27T20:52:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:628: seq_num 574 ranges {447218=719401,1889370=660531,2963386=614253} 2023-08-27T20:52:17.570 INFO:tasks.rados.rados.0.smithi093.stdout:628: writing smithi093153278-217 from 447218 to 1166619 tid 1 2023-08-27T20:52:17.578 INFO:tasks.rados.rados.0.smithi093.stdout:628: writing smithi093153278-217 from 1889370 to 2549901 tid 2 2023-08-27T20:52:17.584 INFO:tasks.rados.rados.0.smithi093.stdout:628: writing smithi093153278-217 from 2963386 to 3577639 tid 3 2023-08-27T20:52:17.584 INFO:tasks.rados.rados.0.smithi093.stdout:618: done (5 left) 2023-08-27T20:52:17.584 INFO:tasks.rados.rados.0.smithi093.stdout:629: write oid 465 current snap is 0 2023-08-27T20:52:17.585 INFO:tasks.rados.rados.0.smithi093.stdout:624: expect (ObjNum 490 snap 0 seq_num 490) 2023-08-27T20:52:17.662 INFO:tasks.rados.rados.0.smithi093.stdout:623: finishing write tid 3 to smithi093153278-215 2023-08-27T20:52:17.662 INFO:tasks.rados.rados.0.smithi093.stdout:623: finishing write tid 4 to smithi093153278-215 2023-08-27T20:52:17.662 INFO:tasks.rados.rados.0.smithi093.stdout:623: finishing write tid 5 to smithi093153278-215 2023-08-27T20:52:17.662 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 215 v 126 (ObjNum 572 snap 0 seq_num 572) dirty exists 2023-08-27T20:52:17.663 INFO:tasks.rados.rados.0.smithi093.stdout:623: left oid 215 (ObjNum 572 snap 0 seq_num 572) 2023-08-27T20:52:17.663 INFO:tasks.rados.rados.0.smithi093.stdout:629: seq_num 575 ranges {464025=794650,1947852=404189,2990809=425525} 2023-08-27T20:52:17.669 INFO:tasks.rados.rados.0.smithi093.stdout:629: writing smithi093153278-465 from 464025 to 1258675 tid 1 2023-08-27T20:52:17.675 INFO:tasks.rados.rados.0.smithi093.stdout:629: writing smithi093153278-465 from 1947852 to 2352041 tid 2 2023-08-27T20:52:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:629: writing smithi093153278-465 from 2990809 to 3416334 tid 3 2023-08-27T20:52:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:623: done (5 left) 2023-08-27T20:52:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:624: done (4 left) 2023-08-27T20:52:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:630: delete oid 363 current snap is 0 2023-08-27T20:52:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:626: expect (ObjNum 507 snap 0 seq_num 507) 2023-08-27T20:52:17.761 INFO:tasks.rados.rados.0.smithi093.stdout:627: expect (ObjNum 519 snap 0 seq_num 519) 2023-08-27T20:52:17.876 INFO:tasks.rados.rados.0.smithi093.stdout:628: finishing write tid 1 to smithi093153278-217 2023-08-27T20:52:17.876 INFO:tasks.rados.rados.0.smithi093.stdout:628: finishing write tid 2 to smithi093153278-217 2023-08-27T20:52:17.876 INFO:tasks.rados.rados.0.smithi093.stdout:628: finishing write tid 3 to smithi093153278-217 2023-08-27T20:52:17.876 INFO:tasks.rados.rados.0.smithi093.stdout:628: finishing write tid 4 to smithi093153278-217 2023-08-27T20:52:17.876 INFO:tasks.rados.rados.0.smithi093.stdout:626: done (4 left) 2023-08-27T20:52:17.876 INFO:tasks.rados.rados.0.smithi093.stdout:627: done (3 left) 2023-08-27T20:52:17.876 INFO:tasks.rados.rados.0.smithi093.stdout:630: done (2 left) 2023-08-27T20:52:17.879 INFO:tasks.rados.rados.0.smithi093.stdout:631: read oid 335 snap -1 2023-08-27T20:52:17.879 INFO:tasks.rados.rados.0.smithi093.stdout:631: expect (ObjNum 334 snap 0 seq_num 334) 2023-08-27T20:52:17.879 INFO:tasks.rados.rados.0.smithi093.stdout:632: write oid 343 current snap is 0 2023-08-27T20:52:17.879 INFO:tasks.rados.rados.0.smithi093.stdout:628: finishing write tid 5 to smithi093153278-217 2023-08-27T20:52:17.880 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 217 v 147 (ObjNum 574 snap 0 seq_num 574) dirty exists 2023-08-27T20:52:17.880 INFO:tasks.rados.rados.0.smithi093.stdout:628: left oid 217 (ObjNum 574 snap 0 seq_num 574) 2023-08-27T20:52:17.880 INFO:tasks.rados.rados.0.smithi093.stdout:632: seq_num 577 ranges {497639=545148,1548463=407858,2529302=564422,3528143=169496} 2023-08-27T20:52:17.882 INFO:tasks.rados.rados.0.smithi093.stdout:632: writing smithi093153278-343 from 497639 to 1042787 tid 1 2023-08-27T20:52:17.887 INFO:tasks.rados.rados.0.smithi093.stdout:632: writing smithi093153278-343 from 1548463 to 1956321 tid 2 2023-08-27T20:52:17.893 INFO:tasks.rados.rados.0.smithi093.stdout:632: writing smithi093153278-343 from 2529302 to 3093724 tid 3 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:632: writing smithi093153278-343 from 3528143 to 3697639 tid 4 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:628: done (3 left) 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:633: read oid 34 snap -1 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:633: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:629: finishing write tid 1 to smithi093153278-465 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:629: finishing write tid 2 to smithi093153278-465 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:634: write oid 71 current snap is 0 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:629: finishing write tid 3 to smithi093153278-465 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:629: finishing write tid 4 to smithi093153278-465 2023-08-27T20:52:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:634: seq_num 578 ranges {514446=736750,1723298=667869,3073078=641368} 2023-08-27T20:52:17.904 INFO:tasks.rados.rados.0.smithi093.stdout:634: writing smithi093153278-71 from 514446 to 1251196 tid 1 2023-08-27T20:52:17.910 INFO:tasks.rados.rados.0.smithi093.stdout:634: writing smithi093153278-71 from 1723298 to 2391167 tid 2 2023-08-27T20:52:17.918 INFO:tasks.rados.rados.0.smithi093.stdout:634: writing smithi093153278-71 from 3073078 to 3714446 tid 3 2023-08-27T20:52:17.918 INFO:tasks.rados.rados.0.smithi093.stdout:635: read oid 112 snap -1 2023-08-27T20:52:17.918 INFO:tasks.rados.rados.0.smithi093.stdout:635: expect (ObjNum 510 snap 0 seq_num 510) 2023-08-27T20:52:17.918 INFO:tasks.rados.rados.0.smithi093.stdout:636: read oid 282 snap -1 2023-08-27T20:52:17.918 INFO:tasks.rados.rados.0.smithi093.stdout:636: expect (ObjNum 281 snap 0 seq_num 281) 2023-08-27T20:52:17.918 INFO:tasks.rados.rados.0.smithi093.stdout:637: write oid 412 current snap is 0 2023-08-27T20:52:17.918 INFO:tasks.rados.rados.0.smithi093.stdout:629: finishing write tid 5 to smithi093153278-465 2023-08-27T20:52:17.919 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 465 v 150 (ObjNum 575 snap 0 seq_num 575) dirty exists 2023-08-27T20:52:17.919 INFO:tasks.rados.rados.0.smithi093.stdout:629: left oid 465 (ObjNum 575 snap 0 seq_num 575) 2023-08-27T20:52:17.919 INFO:tasks.rados.rados.0.smithi093.stdout:637: seq_num 579 ranges {531253=411999,1381780=411527,2300501=586966,3376974=354279} 2023-08-27T20:52:17.923 INFO:tasks.rados.rados.0.smithi093.stdout:637: writing smithi093153278-412 from 531253 to 943252 tid 1 2023-08-27T20:52:17.927 INFO:tasks.rados.rados.0.smithi093.stdout:637: writing smithi093153278-412 from 1381780 to 1793307 tid 2 2023-08-27T20:52:17.933 INFO:tasks.rados.rados.0.smithi093.stdout:637: writing smithi093153278-412 from 2300501 to 2887467 tid 3 2023-08-27T20:52:17.938 INFO:tasks.rados.rados.0.smithi093.stdout:637: writing smithi093153278-412 from 3376974 to 3731253 tid 4 2023-08-27T20:52:17.939 INFO:tasks.rados.rados.0.smithi093.stdout:629: done (7 left) 2023-08-27T20:52:17.939 INFO:tasks.rados.rados.0.smithi093.stdout:638: write oid 81 current snap is 0 2023-08-27T20:52:17.939 INFO:tasks.rados.rados.0.smithi093.stdout:631: expect (ObjNum 334 snap 0 seq_num 334) 2023-08-27T20:52:18.056 INFO:tasks.rados.rados.0.smithi093.stdout:632: finishing write tid 1 to smithi093153278-343 2023-08-27T20:52:18.056 INFO:tasks.rados.rados.0.smithi093.stdout:632: finishing write tid 2 to smithi093153278-343 2023-08-27T20:52:18.056 INFO:tasks.rados.rados.0.smithi093.stdout:632: finishing write tid 3 to smithi093153278-343 2023-08-27T20:52:18.056 INFO:tasks.rados.rados.0.smithi093.stdout:632: finishing write tid 4 to smithi093153278-343 2023-08-27T20:52:18.056 INFO:tasks.rados.rados.0.smithi093.stdout:632: finishing write tid 5 to smithi093153278-343 2023-08-27T20:52:18.056 INFO:tasks.rados.rados.0.smithi093.stdout:632: finishing write tid 6 to smithi093153278-343 2023-08-27T20:52:18.056 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 343 v 109 (ObjNum 577 snap 0 seq_num 577) dirty exists 2023-08-27T20:52:18.057 INFO:tasks.rados.rados.0.smithi093.stdout:632: left oid 343 (ObjNum 577 snap 0 seq_num 577) 2023-08-27T20:52:18.057 INFO:tasks.rados.rados.0.smithi093.stdout:633: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:52:18.151 INFO:tasks.rados.rados.0.smithi093.stdout:634: finishing write tid 1 to smithi093153278-71 2023-08-27T20:52:18.151 INFO:tasks.rados.rados.0.smithi093.stdout:634: finishing write tid 2 to smithi093153278-71 2023-08-27T20:52:18.151 INFO:tasks.rados.rados.0.smithi093.stdout:638: seq_num 580 ranges {548060=487248,1440262=671538,2844277=798238} 2023-08-27T20:52:18.156 INFO:tasks.rados.rados.0.smithi093.stdout:638: writing smithi093153278-81 from 548060 to 1035308 tid 1 2023-08-27T20:52:18.163 INFO:tasks.rados.rados.0.smithi093.stdout:638: writing smithi093153278-81 from 1440262 to 2111800 tid 2 2023-08-27T20:52:18.172 INFO:tasks.rados.rados.0.smithi093.stdout:638: writing smithi093153278-81 from 2844277 to 3642515 tid 3 2023-08-27T20:52:18.172 INFO:tasks.rados.rados.0.smithi093.stdout:631: done (7 left) 2023-08-27T20:52:18.172 INFO:tasks.rados.rados.0.smithi093.stdout:632: done (6 left) 2023-08-27T20:52:18.172 INFO:tasks.rados.rados.0.smithi093.stdout:633: done (5 left) 2023-08-27T20:52:18.172 INFO:tasks.rados.rados.0.smithi093.stdout:639: write oid 462 current snap is 0 2023-08-27T20:52:18.172 INFO:tasks.rados.rados.0.smithi093.stdout:639: seq_num 581 ranges {564867=562497,1898744=415196,2755347=725863} 2023-08-27T20:52:18.178 INFO:tasks.rados.rados.0.smithi093.stdout:639: writing smithi093153278-462 from 564867 to 1127364 tid 1 2023-08-27T20:52:18.185 INFO:tasks.rados.rados.0.smithi093.stdout:639: writing smithi093153278-462 from 1898744 to 2313940 tid 2 2023-08-27T20:52:18.192 INFO:tasks.rados.rados.0.smithi093.stdout:639: writing smithi093153278-462 from 2755347 to 3481210 tid 3 2023-08-27T20:52:18.192 INFO:tasks.rados.rados.0.smithi093.stdout:640: write oid 169 current snap is 0 2023-08-27T20:52:18.192 INFO:tasks.rados.rados.0.smithi093.stdout:634: finishing write tid 3 to smithi093153278-71 2023-08-27T20:52:18.192 INFO:tasks.rados.rados.0.smithi093.stdout:636: expect (ObjNum 281 snap 0 seq_num 281) 2023-08-27T20:52:18.295 INFO:tasks.rados.rados.0.smithi093.stdout:634: finishing write tid 4 to smithi093153278-71 2023-08-27T20:52:18.296 INFO:tasks.rados.rados.0.smithi093.stdout:634: finishing write tid 5 to smithi093153278-71 2023-08-27T20:52:18.296 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 71 v 111 (ObjNum 578 snap 0 seq_num 578) dirty exists 2023-08-27T20:52:18.296 INFO:tasks.rados.rados.0.smithi093.stdout:634: left oid 71 (ObjNum 578 snap 0 seq_num 578) 2023-08-27T20:52:18.296 INFO:tasks.rados.rados.0.smithi093.stdout:637: finishing write tid 1 to smithi093153278-412 2023-08-27T20:52:18.296 INFO:tasks.rados.rados.0.smithi093.stdout:637: finishing write tid 2 to smithi093153278-412 2023-08-27T20:52:18.296 INFO:tasks.rados.rados.0.smithi093.stdout:635: expect (ObjNum 510 snap 0 seq_num 510) 2023-08-27T20:52:18.398 INFO:tasks.rados.rados.0.smithi093.stdout:637: finishing write tid 3 to smithi093153278-412 2023-08-27T20:52:18.398 INFO:tasks.rados.rados.0.smithi093.stdout:637: finishing write tid 4 to smithi093153278-412 2023-08-27T20:52:18.398 INFO:tasks.rados.rados.0.smithi093.stdout:637: finishing write tid 5 to smithi093153278-412 2023-08-27T20:52:18.398 INFO:tasks.rados.rados.0.smithi093.stdout:640: seq_num 582 ranges {581674=637746,1840873=558854,3066417=537135} 2023-08-27T20:52:18.404 INFO:tasks.rados.rados.0.smithi093.stdout:640: writing smithi093153278-169 from 581674 to 1219420 tid 1 2023-08-27T20:52:18.410 INFO:tasks.rados.rados.0.smithi093.stdout:640: writing smithi093153278-169 from 1840873 to 2399727 tid 2 2023-08-27T20:52:18.418 INFO:tasks.rados.rados.0.smithi093.stdout:640: writing smithi093153278-169 from 3066417 to 3603552 tid 3 2023-08-27T20:52:18.418 INFO:tasks.rados.rados.0.smithi093.stdout:634: done (6 left) 2023-08-27T20:52:18.418 INFO:tasks.rados.rados.0.smithi093.stdout:635: done (5 left) 2023-08-27T20:52:18.419 INFO:tasks.rados.rados.0.smithi093.stdout:636: done (4 left) 2023-08-27T20:52:18.419 INFO:tasks.rados.rados.0.smithi093.stdout:641: delete oid 89 current snap is 0 2023-08-27T20:52:18.419 INFO:tasks.rados.rados.0.smithi093.stdout:637: finishing write tid 6 to smithi093153278-412 2023-08-27T20:52:18.419 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 412 v 116 (ObjNum 579 snap 0 seq_num 579) dirty exists 2023-08-27T20:52:18.419 INFO:tasks.rados.rados.0.smithi093.stdout:637: left oid 412 (ObjNum 579 snap 0 seq_num 579) 2023-08-27T20:52:18.419 INFO:tasks.rados.rados.0.smithi093.stdout:638: finishing write tid 1 to smithi093153278-81 2023-08-27T20:52:18.419 INFO:tasks.rados.rados.0.smithi093.stdout:638: finishing write tid 2 to smithi093153278-81 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:638: finishing write tid 3 to smithi093153278-81 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:638: finishing write tid 4 to smithi093153278-81 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:638: finishing write tid 5 to smithi093153278-81 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 155 (ObjNum 580 snap 0 seq_num 580) dirty exists 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:638: left oid 81 (ObjNum 580 snap 0 seq_num 580) 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:639: finishing write tid 1 to smithi093153278-462 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:639: finishing write tid 2 to smithi093153278-462 2023-08-27T20:52:18.420 INFO:tasks.rados.rados.0.smithi093.stdout:639: finishing write tid 3 to smithi093153278-462 2023-08-27T20:52:18.421 INFO:tasks.rados.rados.0.smithi093.stdout:639: finishing write tid 4 to smithi093153278-462 2023-08-27T20:52:18.421 INFO:tasks.rados.rados.0.smithi093.stdout:639: finishing write tid 5 to smithi093153278-462 2023-08-27T20:52:18.421 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 462 v 113 (ObjNum 581 snap 0 seq_num 581) dirty exists 2023-08-27T20:52:18.421 INFO:tasks.rados.rados.0.smithi093.stdout:639: left oid 462 (ObjNum 581 snap 0 seq_num 581) 2023-08-27T20:52:18.421 INFO:tasks.rados.rados.0.smithi093.stdout:640: finishing write tid 1 to smithi093153278-169 2023-08-27T20:52:18.421 INFO:tasks.rados.rados.0.smithi093.stdout:640: finishing write tid 2 to smithi093153278-169 2023-08-27T20:52:18.422 INFO:tasks.rados.rados.0.smithi093.stdout:637: done (4 left) 2023-08-27T20:52:18.422 INFO:tasks.rados.rados.0.smithi093.stdout:638: done (3 left) 2023-08-27T20:52:18.422 INFO:tasks.rados.rados.0.smithi093.stdout:639: done (2 left) 2023-08-27T20:52:18.422 INFO:tasks.rados.rados.0.smithi093.stdout:641: done (1 left) 2023-08-27T20:52:18.422 INFO:tasks.rados.rados.0.smithi093.stdout:642: write oid 115 current snap is 0 2023-08-27T20:52:18.422 INFO:tasks.rados.rados.0.smithi093.stdout:642: seq_num 584 ranges {615288=788244,1957837=562523,3121263=559679} 2023-08-27T20:52:18.427 INFO:tasks.rados.rados.0.smithi093.stdout:642: writing smithi093153278-115 from 615288 to 1403532 tid 1 2023-08-27T20:52:18.433 INFO:tasks.rados.rados.0.smithi093.stdout:642: writing smithi093153278-115 from 1957837 to 2520360 tid 2 2023-08-27T20:52:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:18 smithi136 ceph-mon[113531]: pgmap v63: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 59 MiB/s wr, 326 op/s 2023-08-27T20:52:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1618287180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:18.440 INFO:tasks.rados.rados.0.smithi093.stdout:642: writing smithi093153278-115 from 3121263 to 3680942 tid 3 2023-08-27T20:52:18.440 INFO:tasks.rados.rados.0.smithi093.stdout:643: write oid 261 current snap is 0 2023-08-27T20:52:18.440 INFO:tasks.rados.rados.0.smithi093.stdout:640: finishing write tid 3 to smithi093153278-169 2023-08-27T20:52:18.440 INFO:tasks.rados.rados.0.smithi093.stdout:643: seq_num 585 ranges {632095=463493,1616319=422534,2748686=487304} 2023-08-27T20:52:18.446 INFO:tasks.rados.rados.0.smithi093.stdout:643: writing smithi093153278-261 from 632095 to 1095588 tid 1 2023-08-27T20:52:18.451 INFO:tasks.rados.rados.0.smithi093.stdout:643: writing smithi093153278-261 from 1616319 to 2038853 tid 2 2023-08-27T20:52:18.458 INFO:tasks.rados.rados.0.smithi093.stdout:643: writing smithi093153278-261 from 2748686 to 3235990 tid 3 2023-08-27T20:52:18.458 INFO:tasks.rados.rados.0.smithi093.stdout:644: write oid 366 current snap is 0 2023-08-27T20:52:18.459 INFO:tasks.rados.rados.0.smithi093.stdout:640: finishing write tid 4 to smithi093153278-169 2023-08-27T20:52:18.459 INFO:tasks.rados.rados.0.smithi093.stdout:640: finishing write tid 5 to smithi093153278-169 2023-08-27T20:52:18.459 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 169 v 117 (ObjNum 582 snap 0 seq_num 582) dirty exists 2023-08-27T20:52:18.459 INFO:tasks.rados.rados.0.smithi093.stdout:640: left oid 169 (ObjNum 582 snap 0 seq_num 582) 2023-08-27T20:52:18.459 INFO:tasks.rados.rados.0.smithi093.stdout:642: finishing write tid 1 to smithi093153278-115 2023-08-27T20:52:18.459 INFO:tasks.rados.rados.0.smithi093.stdout:644: seq_num 586 ranges {648902=655095,2074801=566192,3176109=672793} 2023-08-27T20:52:18.465 INFO:tasks.rados.rados.0.smithi093.stdout:644: writing smithi093153278-366 from 648902 to 1303997 tid 1 2023-08-27T20:52:18.473 INFO:tasks.rados.rados.0.smithi093.stdout:644: writing smithi093153278-366 from 2074801 to 2640993 tid 2 2023-08-27T20:52:18.480 INFO:tasks.rados.rados.0.smithi093.stdout:644: writing smithi093153278-366 from 3176109 to 3848902 tid 3 2023-08-27T20:52:18.480 INFO:tasks.rados.rados.0.smithi093.stdout:640: done (3 left) 2023-08-27T20:52:18.480 INFO:tasks.rados.rados.0.smithi093.stdout:645: write oid 252 current snap is 0 2023-08-27T20:52:18.480 INFO:tasks.rados.rados.0.smithi093.stdout:642: finishing write tid 2 to smithi093153278-115 2023-08-27T20:52:18.480 INFO:tasks.rados.rados.0.smithi093.stdout:645: seq_num 587 ranges {665709=730344,2133283=426203,3203532=509848} 2023-08-27T20:52:18.487 INFO:tasks.rados.rados.0.smithi093.stdout:645: writing smithi093153278-252 from 665709 to 1396053 tid 1 2023-08-27T20:52:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:18 smithi093 ceph-mon[111335]: pgmap v63: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 59 MiB/s wr, 326 op/s 2023-08-27T20:52:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1618287180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:18 smithi093 ceph-mon[118902]: pgmap v63: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 59 MiB/s wr, 326 op/s 2023-08-27T20:52:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1618287180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:18.494 INFO:tasks.rados.rados.0.smithi093.stdout:645: writing smithi093153278-252 from 2133283 to 2559486 tid 2 2023-08-27T20:52:18.501 INFO:tasks.rados.rados.0.smithi093.stdout:645: writing smithi093153278-252 from 3203532 to 3713380 tid 3 2023-08-27T20:52:18.501 INFO:tasks.rados.rados.0.smithi093.stdout:646: read oid 216 snap -1 2023-08-27T20:52:18.501 INFO:tasks.rados.rados.0.smithi093.stdout:646: expect deleted 2023-08-27T20:52:18.501 INFO:tasks.rados.rados.0.smithi093.stdout:647: read oid 450 snap -1 2023-08-27T20:52:18.501 INFO:tasks.rados.rados.0.smithi093.stdout:647: expect (ObjNum 449 snap 0 seq_num 449) 2023-08-27T20:52:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:648: read oid 175 snap -1 2023-08-27T20:52:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:648: expect (ObjNum 174 snap 0 seq_num 174) 2023-08-27T20:52:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:649: read oid 454 snap -1 2023-08-27T20:52:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:649: expect (ObjNum 453 snap 0 seq_num 453) 2023-08-27T20:52:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:650: write oid 224 current snap is 0 2023-08-27T20:52:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:650: seq_num 588 ranges {682516=405593,1791765=569861,2830955=721120} 2023-08-27T20:52:18.507 INFO:tasks.rados.rados.0.smithi093.stdout:650: writing smithi093153278-224 from 682516 to 1088109 tid 1 2023-08-27T20:52:18.514 INFO:tasks.rados.rados.0.smithi093.stdout:650: writing smithi093153278-224 from 1791765 to 2361626 tid 2 2023-08-27T20:52:18.521 INFO:tasks.rados.rados.0.smithi093.stdout:650: writing smithi093153278-224 from 2830955 to 3552075 tid 3 2023-08-27T20:52:18.522 INFO:tasks.rados.rados.0.smithi093.stdout:651: write oid 234 current snap is 0 2023-08-27T20:52:18.522 INFO:tasks.rados.rados.0.smithi093.stdout:642: finishing write tid 3 to smithi093153278-115 2023-08-27T20:52:18.522 INFO:tasks.rados.rados.0.smithi093.stdout:642: finishing write tid 4 to smithi093153278-115 2023-08-27T20:52:18.522 INFO:tasks.rados.rados.0.smithi093.stdout:642: finishing write tid 5 to smithi093153278-115 2023-08-27T20:52:18.522 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 115 v 120 (ObjNum 584 snap 0 seq_num 584) dirty exists 2023-08-27T20:52:18.522 INFO:tasks.rados.rados.0.smithi093.stdout:642: left oid 115 (ObjNum 584 snap 0 seq_num 584) 2023-08-27T20:52:18.522 INFO:tasks.rados.rados.0.smithi093.stdout:651: seq_num 589 ranges {699323=480842,1850247=429872,2858378=532392} 2023-08-27T20:52:18.527 INFO:tasks.rados.rados.0.smithi093.stdout:651: writing smithi093153278-234 from 699323 to 1180165 tid 1 2023-08-27T20:52:18.534 INFO:tasks.rados.rados.0.smithi093.stdout:651: writing smithi093153278-234 from 1850247 to 2280119 tid 2 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:651: writing smithi093153278-234 from 2858378 to 3390770 tid 3 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:642: done (9 left) 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:652: read oid 95 snap -1 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:652: expect (ObjNum 94 snap 0 seq_num 94) 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:653: read oid 115 snap -1 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:653: expect (ObjNum 584 snap 0 seq_num 584) 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:654: write oid 395 current snap is 0 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:643: finishing write tid 1 to smithi093153278-261 2023-08-27T20:52:18.541 INFO:tasks.rados.rados.0.smithi093.stdout:643: finishing write tid 2 to smithi093153278-261 2023-08-27T20:52:18.542 INFO:tasks.rados.rados.0.smithi093.stdout:643: finishing write tid 3 to smithi093153278-261 2023-08-27T20:52:18.542 INFO:tasks.rados.rados.0.smithi093.stdout:654: seq_num 590 ranges {716130=556091,1792376=573530,2769448=460017,3821956=94174} 2023-08-27T20:52:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:654: writing smithi093153278-395 from 716130 to 1272221 tid 1 2023-08-27T20:52:18.554 INFO:tasks.rados.rados.0.smithi093.stdout:654: writing smithi093153278-395 from 1792376 to 2365906 tid 2 2023-08-27T20:52:18.559 INFO:tasks.rados.rados.0.smithi093.stdout:654: writing smithi093153278-395 from 2769448 to 3229465 tid 3 2023-08-27T20:52:18.564 INFO:tasks.rados.rados.0.smithi093.stdout:654: writing smithi093153278-395 from 3821956 to 3916130 tid 4 2023-08-27T20:52:18.564 INFO:tasks.rados.rados.0.smithi093.stdout:655: write oid 441 current snap is 0 2023-08-27T20:52:18.564 INFO:tasks.rados.rados.0.smithi093.stdout:643: finishing write tid 4 to smithi093153278-261 2023-08-27T20:52:18.565 INFO:tasks.rados.rados.0.smithi093.stdout:643: finishing write tid 5 to smithi093153278-261 2023-08-27T20:52:18.565 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 261 v 121 (ObjNum 585 snap 0 seq_num 585) dirty exists 2023-08-27T20:52:18.565 INFO:tasks.rados.rados.0.smithi093.stdout:643: left oid 261 (ObjNum 585 snap 0 seq_num 585) 2023-08-27T20:52:18.565 INFO:tasks.rados.rados.0.smithi093.stdout:644: finishing write tid 1 to smithi093153278-366 2023-08-27T20:52:18.565 INFO:tasks.rados.rados.0.smithi093.stdout:644: finishing write tid 2 to smithi093153278-366 2023-08-27T20:52:18.565 INFO:tasks.rados.rados.0.smithi093.stdout:655: seq_num 591 ranges {732937=631340,1850858=433541,2796871=671289,3888195=44742} 2023-08-27T20:52:18.572 INFO:tasks.rados.rados.0.smithi093.stdout:655: writing smithi093153278-441 from 732937 to 1364277 tid 1 2023-08-27T20:52:18.577 INFO:tasks.rados.rados.0.smithi093.stdout:655: writing smithi093153278-441 from 1850858 to 2284399 tid 2 2023-08-27T20:52:18.584 INFO:tasks.rados.rados.0.smithi093.stdout:655: writing smithi093153278-441 from 2796871 to 3468160 tid 3 2023-08-27T20:52:18.587 INFO:tasks.rados.rados.0.smithi093.stdout:655: writing smithi093153278-441 from 3888195 to 3932937 tid 4 2023-08-27T20:52:18.587 INFO:tasks.rados.rados.0.smithi093.stdout:643: done (12 left) 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:656: read oid 179 snap -1 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:656: expect (ObjNum 178 snap 0 seq_num 178) 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:657: read oid 186 snap -1 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:657: expect (ObjNum 185 snap 0 seq_num 185) 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:658: write oid 36 current snap is 0 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:644: finishing write tid 3 to smithi093153278-366 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:644: finishing write tid 4 to smithi093153278-366 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:644: finishing write tid 5 to smithi093153278-366 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 366 v 124 (ObjNum 586 snap 0 seq_num 586) dirty exists 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:644: left oid 366 (ObjNum 586 snap 0 seq_num 586) 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:645: finishing write tid 1 to smithi093153278-252 2023-08-27T20:52:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:645: finishing write tid 2 to smithi093153278-252 2023-08-27T20:52:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:658: seq_num 592 ranges {749744=706589,1909340=577199,3224294=482561} 2023-08-27T20:52:18.596 INFO:tasks.rados.rados.0.smithi093.stdout:658: writing smithi093153278-36 from 749744 to 1456333 tid 1 2023-08-27T20:52:18.601 INFO:tasks.rados.rados.0.smithi093.stdout:658: writing smithi093153278-36 from 1909340 to 2486539 tid 2 2023-08-27T20:52:18.609 INFO:tasks.rados.rados.0.smithi093.stdout:658: writing smithi093153278-36 from 3224294 to 3706855 tid 3 2023-08-27T20:52:18.609 INFO:tasks.rados.rados.0.smithi093.stdout:644: done (14 left) 2023-08-27T20:52:18.609 INFO:tasks.rados.rados.0.smithi093.stdout:646: done (13 left) 2023-08-27T20:52:18.609 INFO:tasks.rados.rados.0.smithi093.stdout:659: write oid 378 current snap is 0 2023-08-27T20:52:18.609 INFO:tasks.rados.rados.0.smithi093.stdout:645: finishing write tid 3 to smithi093153278-252 2023-08-27T20:52:18.609 INFO:tasks.rados.rados.0.smithi093.stdout:649: expect (ObjNum 453 snap 0 seq_num 453) 2023-08-27T20:52:18.725 INFO:tasks.rados.rados.0.smithi093.stdout:647: expect (ObjNum 449 snap 0 seq_num 449) 2023-08-27T20:52:18.848 INFO:tasks.rados.rados.0.smithi093.stdout:645: finishing write tid 4 to smithi093153278-252 2023-08-27T20:52:18.849 INFO:tasks.rados.rados.0.smithi093.stdout:645: finishing write tid 5 to smithi093153278-252 2023-08-27T20:52:18.849 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 252 v 113 (ObjNum 587 snap 0 seq_num 587) dirty exists 2023-08-27T20:52:18.849 INFO:tasks.rados.rados.0.smithi093.stdout:645: left oid 252 (ObjNum 587 snap 0 seq_num 587) 2023-08-27T20:52:18.849 INFO:tasks.rados.rados.0.smithi093.stdout:650: finishing write tid 1 to smithi093153278-224 2023-08-27T20:52:18.849 INFO:tasks.rados.rados.0.smithi093.stdout:648: expect (ObjNum 174 snap 0 seq_num 174) 2023-08-27T20:52:18.949 INFO:tasks.rados.rados.0.smithi093.stdout:650: finishing write tid 2 to smithi093153278-224 2023-08-27T20:52:18.949 INFO:tasks.rados.rados.0.smithi093.stdout:650: finishing write tid 3 to smithi093153278-224 2023-08-27T20:52:18.949 INFO:tasks.rados.rados.0.smithi093.stdout:650: finishing write tid 4 to smithi093153278-224 2023-08-27T20:52:18.949 INFO:tasks.rados.rados.0.smithi093.stdout:659: seq_num 593 ranges {766551=498191,1684175=720857,2851717=693833} 2023-08-27T20:52:18.956 INFO:tasks.rados.rados.0.smithi093.stdout:659: writing smithi093153278-378 from 766551 to 1264742 tid 1 2023-08-27T20:52:18.962 INFO:tasks.rados.rados.0.smithi093.stdout:659: writing smithi093153278-378 from 1684175 to 2405032 tid 2 2023-08-27T20:52:18.969 INFO:tasks.rados.rados.0.smithi093.stdout:659: writing smithi093153278-378 from 2851717 to 3545550 tid 3 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:645: done (13 left) 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:647: done (12 left) 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:648: done (11 left) 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:649: done (10 left) 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:660: delete oid 410 current snap is 0 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:650: finishing write tid 5 to smithi093153278-224 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 224 v 110 (ObjNum 588 snap 0 seq_num 588) dirty exists 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:650: left oid 224 (ObjNum 588 snap 0 seq_num 588) 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:651: finishing write tid 1 to smithi093153278-234 2023-08-27T20:52:18.970 INFO:tasks.rados.rados.0.smithi093.stdout:651: finishing write tid 2 to smithi093153278-234 2023-08-27T20:52:18.971 INFO:tasks.rados.rados.0.smithi093.stdout:651: finishing write tid 3 to smithi093153278-234 2023-08-27T20:52:18.971 INFO:tasks.rados.rados.0.smithi093.stdout:651: finishing write tid 4 to smithi093153278-234 2023-08-27T20:52:18.971 INFO:tasks.rados.rados.0.smithi093.stdout:651: finishing write tid 5 to smithi093153278-234 2023-08-27T20:52:18.971 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 234 v 120 (ObjNum 589 snap 0 seq_num 589) dirty exists 2023-08-27T20:52:18.971 INFO:tasks.rados.rados.0.smithi093.stdout:651: left oid 234 (ObjNum 589 snap 0 seq_num 589) 2023-08-27T20:52:18.971 INFO:tasks.rados.rados.0.smithi093.stdout:653: expect (ObjNum 584 snap 0 seq_num 584) 2023-08-27T20:52:19.146 INFO:tasks.rados.rados.0.smithi093.stdout:654: finishing write tid 1 to smithi093153278-395 2023-08-27T20:52:19.146 INFO:tasks.rados.rados.0.smithi093.stdout:654: finishing write tid 2 to smithi093153278-395 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:654: finishing write tid 3 to smithi093153278-395 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:650: done (10 left) 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:651: done (9 left) 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:653: done (8 left) 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:660: done (7 left) 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:661: delete oid 5 current snap is 0 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:654: finishing write tid 4 to smithi093153278-395 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:654: finishing write tid 5 to smithi093153278-395 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:654: finishing write tid 6 to smithi093153278-395 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 160 (ObjNum 590 snap 0 seq_num 590) dirty exists 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:654: left oid 395 (ObjNum 590 snap 0 seq_num 590) 2023-08-27T20:52:19.147 INFO:tasks.rados.rados.0.smithi093.stdout:652: expect (ObjNum 94 snap 0 seq_num 94) 2023-08-27T20:52:19.289 INFO:tasks.rados.rados.0.smithi093.stdout:655: finishing write tid 1 to smithi093153278-441 2023-08-27T20:52:19.289 INFO:tasks.rados.rados.0.smithi093.stdout:655: finishing write tid 2 to smithi093153278-441 2023-08-27T20:52:19.289 INFO:tasks.rados.rados.0.smithi093.stdout:655: finishing write tid 3 to smithi093153278-441 2023-08-27T20:52:19.289 INFO:tasks.rados.rados.0.smithi093.stdout:655: finishing write tid 4 to smithi093153278-441 2023-08-27T20:52:19.289 INFO:tasks.rados.rados.0.smithi093.stdout:655: finishing write tid 5 to smithi093153278-441 2023-08-27T20:52:19.289 INFO:tasks.rados.rados.0.smithi093.stdout:655: finishing write tid 6 to smithi093153278-441 2023-08-27T20:52:19.290 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 138 (ObjNum 591 snap 0 seq_num 591) dirty exists 2023-08-27T20:52:19.290 INFO:tasks.rados.rados.0.smithi093.stdout:655: left oid 441 (ObjNum 591 snap 0 seq_num 591) 2023-08-27T20:52:19.290 INFO:tasks.rados.rados.0.smithi093.stdout:657: expect (ObjNum 185 snap 0 seq_num 185) 2023-08-27T20:52:19.412 INFO:tasks.rados.rados.0.smithi093.stdout:656: expect (ObjNum 178 snap 0 seq_num 178) 2023-08-27T20:52:19.525 INFO:tasks.rados.rados.0.smithi093.stdout:658: finishing write tid 1 to smithi093153278-36 2023-08-27T20:52:19.525 INFO:tasks.rados.rados.0.smithi093.stdout:658: finishing write tid 2 to smithi093153278-36 2023-08-27T20:52:19.525 INFO:tasks.rados.rados.0.smithi093.stdout:658: finishing write tid 3 to smithi093153278-36 2023-08-27T20:52:19.525 INFO:tasks.rados.rados.0.smithi093.stdout:658: finishing write tid 4 to smithi093153278-36 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:658: finishing write tid 5 to smithi093153278-36 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 121 (ObjNum 592 snap 0 seq_num 592) dirty exists 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:658: left oid 36 (ObjNum 592 snap 0 seq_num 592) 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:659: finishing write tid 1 to smithi093153278-378 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:659: finishing write tid 2 to smithi093153278-378 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:659: finishing write tid 3 to smithi093153278-378 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:659: finishing write tid 4 to smithi093153278-378 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:659: finishing write tid 5 to smithi093153278-378 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 378 v 125 (ObjNum 593 snap 0 seq_num 593) dirty exists 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:659: left oid 378 (ObjNum 593 snap 0 seq_num 593) 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:652: done (7 left) 2023-08-27T20:52:19.526 INFO:tasks.rados.rados.0.smithi093.stdout:654: done (6 left) 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:655: done (5 left) 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:656: done (4 left) 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:657: done (3 left) 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:658: done (2 left) 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:659: done (1 left) 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:661: done (0 left) 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:662: write oid 426 current snap is 0 2023-08-27T20:52:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:662: seq_num 596 ranges {416972=723938,1743268=273704} 2023-08-27T20:52:19.530 INFO:tasks.rados.rados.0.smithi093.stdout:662: writing smithi093153278-426 from 416972 to 1140910 tid 1 2023-08-27T20:52:19.536 INFO:tasks.rados.rados.0.smithi093.stdout:662: writing smithi093153278-426 from 1743268 to 2016972 tid 2 2023-08-27T20:52:19.536 INFO:tasks.rados.rados.0.smithi093.stdout:663: read oid 456 snap -1 2023-08-27T20:52:19.536 INFO:tasks.rados.rados.0.smithi093.stdout:663: expect (ObjNum 565 snap 0 seq_num 565) 2023-08-27T20:52:19.536 INFO:tasks.rados.rados.0.smithi093.stdout:664: write oid 74 current snap is 0 2023-08-27T20:52:19.536 INFO:tasks.rados.rados.0.smithi093.stdout:662: finishing write tid 1 to smithi093153278-426 2023-08-27T20:52:19.536 INFO:tasks.rados.rados.0.smithi093.stdout:664: seq_num 597 ranges {433779=799187,1801750=232029} 2023-08-27T20:52:19.543 INFO:tasks.rados.rados.0.smithi093.stdout:664: writing smithi093153278-74 from 433779 to 1232966 tid 1 2023-08-27T20:52:19.548 INFO:tasks.rados.rados.0.smithi093.stdout:664: writing smithi093153278-74 from 1801750 to 2033779 tid 2 2023-08-27T20:52:19.548 INFO:tasks.rados.rados.0.smithi093.stdout:665: delete oid 371 current snap is 0 2023-08-27T20:52:19.548 INFO:tasks.rados.rados.0.smithi093.stdout:662: finishing write tid 2 to smithi093153278-426 2023-08-27T20:52:19.549 INFO:tasks.rados.rados.0.smithi093.stdout:662: finishing write tid 3 to smithi093153278-426 2023-08-27T20:52:19.549 INFO:tasks.rados.rados.0.smithi093.stdout:662: finishing write tid 4 to smithi093153278-426 2023-08-27T20:52:19.549 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 426 v 128 (ObjNum 596 snap 0 seq_num 596) dirty exists 2023-08-27T20:52:19.549 INFO:tasks.rados.rados.0.smithi093.stdout:662: left oid 426 (ObjNum 596 snap 0 seq_num 596) 2023-08-27T20:52:19.549 INFO:tasks.rados.rados.0.smithi093.stdout:663: expect (ObjNum 565 snap 0 seq_num 565) 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:664: finishing write tid 1 to smithi093153278-74 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:662: done (3 left) 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:663: done (2 left) 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:665: done (1 left) 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:666: read oid 430 snap -1 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:666: expect (ObjNum 429 snap 0 seq_num 429) 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:667: write oid 2 current snap is 0 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:664: finishing write tid 2 to smithi093153278-74 2023-08-27T20:52:19.705 INFO:tasks.rados.rados.0.smithi093.stdout:664: finishing write tid 3 to smithi093153278-74 2023-08-27T20:52:19.706 INFO:tasks.rados.rados.0.smithi093.stdout:667: seq_num 599 ranges {467393=549685,1802361=265032} 2023-08-27T20:52:19.710 INFO:tasks.rados.rados.0.smithi093.stdout:667: writing smithi093153278-2 from 467393 to 1017078 tid 1 2023-08-27T20:52:19.716 INFO:tasks.rados.rados.0.smithi093.stdout:667: writing smithi093153278-2 from 1802361 to 2067393 tid 2 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:668: read oid 253 snap -1 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:668: expect (ObjNum 252 snap 0 seq_num 252) 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:669: read oid 485 snap -1 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:669: expect (ObjNum 484 snap 0 seq_num 484) 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:670: read oid 122 snap -1 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:670: expect (ObjNum 121 snap 0 seq_num 121) 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:671: read oid 238 snap -1 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:671: expect (ObjNum 237 snap 0 seq_num 237) 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:672: write oid 118 current snap is 0 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:664: finishing write tid 4 to smithi093153278-74 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 74 v 163 (ObjNum 597 snap 0 seq_num 597) dirty exists 2023-08-27T20:52:19.717 INFO:tasks.rados.rados.0.smithi093.stdout:664: left oid 74 (ObjNum 597 snap 0 seq_num 597) 2023-08-27T20:52:19.718 INFO:tasks.rados.rados.0.smithi093.stdout:666: expect (ObjNum 429 snap 0 seq_num 429) 2023-08-27T20:52:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:19 smithi093 ceph-mon[111335]: pgmap v64: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 209 op/s 2023-08-27T20:52:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:19 smithi093 ceph-mon[118902]: pgmap v64: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 209 op/s 2023-08-27T20:52:19.821 INFO:tasks.rados.rados.0.smithi093.stdout:667: finishing write tid 1 to smithi093153278-2 2023-08-27T20:52:19.821 INFO:tasks.rados.rados.0.smithi093.stdout:668: expect (ObjNum 252 snap 0 seq_num 252) 2023-08-27T20:52:19.896 INFO:tasks.rados.rados.0.smithi093.stdout:667: finishing write tid 2 to smithi093153278-2 2023-08-27T20:52:19.896 INFO:tasks.rados.rados.0.smithi093.stdout:672: seq_num 600 ranges {484200=624934,1860843=223357} 2023-08-27T20:52:19.902 INFO:tasks.rados.rados.0.smithi093.stdout:672: writing smithi093153278-118 from 484200 to 1109134 tid 1 2023-08-27T20:52:19.908 INFO:tasks.rados.rados.0.smithi093.stdout:672: writing smithi093153278-118 from 1860843 to 2084200 tid 2 2023-08-27T20:52:19.908 INFO:tasks.rados.rados.0.smithi093.stdout:664: done (7 left) 2023-08-27T20:52:19.908 INFO:tasks.rados.rados.0.smithi093.stdout:666: done (6 left) 2023-08-27T20:52:19.908 INFO:tasks.rados.rados.0.smithi093.stdout:668: done (5 left) 2023-08-27T20:52:19.908 INFO:tasks.rados.rados.0.smithi093.stdout:673: read oid 379 snap -1 2023-08-27T20:52:19.909 INFO:tasks.rados.rados.0.smithi093.stdout:673: expect (ObjNum 378 snap 0 seq_num 378) 2023-08-27T20:52:19.909 INFO:tasks.rados.rados.0.smithi093.stdout:674: read oid 185 snap -1 2023-08-27T20:52:19.909 INFO:tasks.rados.rados.0.smithi093.stdout:674: expect (ObjNum 184 snap 0 seq_num 184) 2023-08-27T20:52:19.909 INFO:tasks.rados.rados.0.smithi093.stdout:675: write oid 21 current snap is 0 2023-08-27T20:52:19.909 INFO:tasks.rados.rados.0.smithi093.stdout:670: expect (ObjNum 121 snap 0 seq_num 121) 2023-08-27T20:52:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:19 smithi136 ceph-mon[113531]: pgmap v64: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 209 op/s 2023-08-27T20:52:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:671: expect (ObjNum 237 snap 0 seq_num 237) 2023-08-27T20:52:20.108 INFO:tasks.rados.rados.0.smithi093.stdout:667: finishing write tid 3 to smithi093153278-2 2023-08-27T20:52:20.109 INFO:tasks.rados.rados.0.smithi093.stdout:667: finishing write tid 4 to smithi093153278-2 2023-08-27T20:52:20.109 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 2 v 151 (ObjNum 599 snap 0 seq_num 599) dirty exists 2023-08-27T20:52:20.109 INFO:tasks.rados.rados.0.smithi093.stdout:667: left oid 2 (ObjNum 599 snap 0 seq_num 599) 2023-08-27T20:52:20.109 INFO:tasks.rados.rados.0.smithi093.stdout:669: expect (ObjNum 484 snap 0 seq_num 484) 2023-08-27T20:52:20.187 INFO:tasks.rados.rados.0.smithi093.stdout:672: finishing write tid 1 to smithi093153278-118 2023-08-27T20:52:20.187 INFO:tasks.rados.rados.0.smithi093.stdout:672: finishing write tid 2 to smithi093153278-118 2023-08-27T20:52:20.187 INFO:tasks.rados.rados.0.smithi093.stdout:675: seq_num 601 ranges {501007=416536,1635678=465329} 2023-08-27T20:52:20.191 INFO:tasks.rados.rados.0.smithi093.stdout:675: writing smithi093153278-21 from 501007 to 917543 tid 1 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:675: writing smithi093153278-21 from 1635678 to 2101007 tid 2 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:667: done (7 left) 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:669: done (6 left) 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:670: done (5 left) 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:671: done (4 left) 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:676: read oid 131 snap -1 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:676: expect (ObjNum 130 snap 0 seq_num 130) 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:677: read oid 401 snap -1 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:677: expect (ObjNum 400 snap 0 seq_num 400) 2023-08-27T20:52:20.199 INFO:tasks.rados.rados.0.smithi093.stdout:678: write oid 233 current snap is 0 2023-08-27T20:52:20.200 INFO:tasks.rados.rados.0.smithi093.stdout:672: finishing write tid 3 to smithi093153278-118 2023-08-27T20:52:20.200 INFO:tasks.rados.rados.0.smithi093.stdout:678: seq_num 602 ranges {517814=491785,1577807=540007} 2023-08-27T20:52:20.205 INFO:tasks.rados.rados.0.smithi093.stdout:678: writing smithi093153278-233 from 517814 to 1009599 tid 1 2023-08-27T20:52:20.212 INFO:tasks.rados.rados.0.smithi093.stdout:678: writing smithi093153278-233 from 1577807 to 2117814 tid 2 2023-08-27T20:52:20.212 INFO:tasks.rados.rados.0.smithi093.stdout:679: read oid 429 snap -1 2023-08-27T20:52:20.212 INFO:tasks.rados.rados.0.smithi093.stdout:679: expect (ObjNum 428 snap 0 seq_num 428) 2023-08-27T20:52:20.213 INFO:tasks.rados.rados.0.smithi093.stdout:680: read oid 107 snap -1 2023-08-27T20:52:20.213 INFO:tasks.rados.rados.0.smithi093.stdout:680: expect (ObjNum 529 snap 0 seq_num 529) 2023-08-27T20:52:20.213 INFO:tasks.rados.rados.0.smithi093.stdout:681: write oid 88 current snap is 0 2023-08-27T20:52:20.213 INFO:tasks.rados.rados.0.smithi093.stdout:681: seq_num 603 ranges {534621=567034,1636289=498332} 2023-08-27T20:52:20.217 INFO:tasks.rados.rados.0.smithi093.stdout:681: writing smithi093153278-88 from 534621 to 1101655 tid 1 2023-08-27T20:52:20.223 INFO:tasks.rados.rados.0.smithi093.stdout:681: writing smithi093153278-88 from 1636289 to 2134621 tid 2 2023-08-27T20:52:20.223 INFO:tasks.rados.rados.0.smithi093.stdout:682: write oid 270 current snap is 0 2023-08-27T20:52:20.224 INFO:tasks.rados.rados.0.smithi093.stdout:682: seq_num 604 ranges {551428=642283,1694771=456657} 2023-08-27T20:52:20.231 INFO:tasks.rados.rados.0.smithi093.stdout:682: writing smithi093153278-270 from 551428 to 1193711 tid 1 2023-08-27T20:52:20.237 INFO:tasks.rados.rados.0.smithi093.stdout:682: writing smithi093153278-270 from 1694771 to 2151428 tid 2 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:683: read oid 119 snap -1 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:683: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:684: read oid 400 snap -1 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:684: expect (ObjNum 399 snap 0 seq_num 399) 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:685: read oid 99 snap -1 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:685: expect (ObjNum 98 snap 0 seq_num 98) 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:686: read oid 254 snap -1 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:686: expect (ObjNum 253 snap 0 seq_num 253) 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:687: read oid 97 snap -1 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout:687: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:52:20.238 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout:672: finishing write tid 4 to smithi093153278-118 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 118 v 128 (ObjNum 600 snap 0 seq_num 600) dirty exists 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout:672: left oid 118 (ObjNum 600 snap 0 seq_num 600) 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout:672: done (15 left) 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout:688: read oid 217 snap -1 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout:688: expect (ObjNum 574 snap 0 seq_num 574) 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:20.239 INFO:tasks.rados.rados.0.smithi093.stdout:674: expect (ObjNum 184 snap 0 seq_num 184) 2023-08-27T20:52:20.355 INFO:tasks.rados.rados.0.smithi093.stdout:673: expect (ObjNum 378 snap 0 seq_num 378) 2023-08-27T20:52:20.448 INFO:tasks.rados.rados.0.smithi093.stdout:673: done (15 left) 2023-08-27T20:52:20.448 INFO:tasks.rados.rados.0.smithi093.stdout:674: done (14 left) 2023-08-27T20:52:20.448 INFO:tasks.rados.rados.0.smithi093.stdout:689: write oid 122 current snap is 0 2023-08-27T20:52:20.448 INFO:tasks.rados.rados.0.smithi093.stdout:675: finishing write tid 1 to smithi093153278-21 2023-08-27T20:52:20.448 INFO:tasks.rados.rados.0.smithi093.stdout:689: seq_num 605 ranges {568235=717532,1753253=414982} 2023-08-27T20:52:20.455 INFO:tasks.rados.rados.0.smithi093.stdout:689: writing smithi093153278-122 from 568235 to 1285767 tid 1 2023-08-27T20:52:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:689: writing smithi093153278-122 from 1753253 to 2168235 tid 2 2023-08-27T20:52:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:690: write oid 60 current snap is 0 2023-08-27T20:52:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:675: finishing write tid 2 to smithi093153278-21 2023-08-27T20:52:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:676: expect (ObjNum 130 snap 0 seq_num 130) 2023-08-27T20:52:20.543 INFO:tasks.rados.rados.0.smithi093.stdout:675: finishing write tid 3 to smithi093153278-21 2023-08-27T20:52:20.543 INFO:tasks.rados.rados.0.smithi093.stdout:675: finishing write tid 4 to smithi093153278-21 2023-08-27T20:52:20.543 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 21 v 150 (ObjNum 601 snap 0 seq_num 601) dirty exists 2023-08-27T20:52:20.543 INFO:tasks.rados.rados.0.smithi093.stdout:675: left oid 21 (ObjNum 601 snap 0 seq_num 601) 2023-08-27T20:52:20.543 INFO:tasks.rados.rados.0.smithi093.stdout:690: seq_num 606 ranges {585042=792781,2095382=89660} 2023-08-27T20:52:20.550 INFO:tasks.rados.rados.0.smithi093.stdout:690: writing smithi093153278-60 from 585042 to 1377823 tid 1 2023-08-27T20:52:20.555 INFO:tasks.rados.rados.0.smithi093.stdout:690: writing smithi093153278-60 from 2095382 to 2185042 tid 2 2023-08-27T20:52:20.555 INFO:tasks.rados.rados.0.smithi093.stdout:675: done (15 left) 2023-08-27T20:52:20.555 INFO:tasks.rados.rados.0.smithi093.stdout:676: done (14 left) 2023-08-27T20:52:20.555 INFO:tasks.rados.rados.0.smithi093.stdout:691: write oid 273 current snap is 0 2023-08-27T20:52:20.555 INFO:tasks.rados.rados.0.smithi093.stdout:677: expect (ObjNum 400 snap 0 seq_num 400) 2023-08-27T20:52:20.648 INFO:tasks.rados.rados.0.smithi093.stdout:678: finishing write tid 1 to smithi093153278-233 2023-08-27T20:52:20.648 INFO:tasks.rados.rados.0.smithi093.stdout:678: finishing write tid 2 to smithi093153278-233 2023-08-27T20:52:20.648 INFO:tasks.rados.rados.0.smithi093.stdout:678: finishing write tid 3 to smithi093153278-233 2023-08-27T20:52:20.648 INFO:tasks.rados.rados.0.smithi093.stdout:691: seq_num 607 ranges {601849=468030,1753864=447985} 2023-08-27T20:52:20.654 INFO:tasks.rados.rados.0.smithi093.stdout:691: writing smithi093153278-273 from 601849 to 1069879 tid 1 2023-08-27T20:52:20.661 INFO:tasks.rados.rados.0.smithi093.stdout:691: writing smithi093153278-273 from 1753864 to 2201849 tid 2 2023-08-27T20:52:20.661 INFO:tasks.rados.rados.0.smithi093.stdout:677: done (14 left) 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:692: read oid 197 snap -1 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:692: expect (ObjNum 196 snap 0 seq_num 196) 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:693: read oid 100 snap -1 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:693: expect (ObjNum 99 snap 0 seq_num 99) 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:678: finishing write tid 4 to smithi093153278-233 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 233 v 137 (ObjNum 602 snap 0 seq_num 602) dirty exists 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:678: left oid 233 (ObjNum 602 snap 0 seq_num 602) 2023-08-27T20:52:20.662 INFO:tasks.rados.rados.0.smithi093.stdout:679: expect (ObjNum 428 snap 0 seq_num 428) 2023-08-27T20:52:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3049142211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:20] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:52:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3049142211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3049142211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:20.781 INFO:tasks.rados.rados.0.smithi093.stdout:680: expect (ObjNum 529 snap 0 seq_num 529) 2023-08-27T20:52:20.920 INFO:tasks.rados.rados.0.smithi093.stdout:681: finishing write tid 1 to smithi093153278-88 2023-08-27T20:52:20.921 INFO:tasks.rados.rados.0.smithi093.stdout:681: finishing write tid 2 to smithi093153278-88 2023-08-27T20:52:20.921 INFO:tasks.rados.rados.0.smithi093.stdout:678: done (15 left) 2023-08-27T20:52:20.921 INFO:tasks.rados.rados.0.smithi093.stdout:679: done (14 left) 2023-08-27T20:52:20.921 INFO:tasks.rados.rados.0.smithi093.stdout:680: done (13 left) 2023-08-27T20:52:20.921 INFO:tasks.rados.rados.0.smithi093.stdout:694: write oid 316 current snap is 0 2023-08-27T20:52:20.921 INFO:tasks.rados.rados.0.smithi093.stdout:694: seq_num 608 ranges {618656=543279,1812346=406310} 2023-08-27T20:52:20.927 INFO:tasks.rados.rados.0.smithi093.stdout:694: writing smithi093153278-316 from 618656 to 1161935 tid 1 2023-08-27T20:52:20.933 INFO:tasks.rados.rados.0.smithi093.stdout:694: writing smithi093153278-316 from 1812346 to 2218656 tid 2 2023-08-27T20:52:20.933 INFO:tasks.rados.rados.0.smithi093.stdout:695: write oid 201 current snap is 0 2023-08-27T20:52:20.933 INFO:tasks.rados.rados.0.smithi093.stdout:681: finishing write tid 3 to smithi093153278-88 2023-08-27T20:52:20.933 INFO:tasks.rados.rados.0.smithi093.stdout:681: finishing write tid 4 to smithi093153278-88 2023-08-27T20:52:20.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 153 (ObjNum 603 snap 0 seq_num 603) dirty exists 2023-08-27T20:52:20.933 INFO:tasks.rados.rados.0.smithi093.stdout:681: left oid 88 (ObjNum 603 snap 0 seq_num 603) 2023-08-27T20:52:20.934 INFO:tasks.rados.rados.0.smithi093.stdout:682: finishing write tid 1 to smithi093153278-270 2023-08-27T20:52:20.934 INFO:tasks.rados.rados.0.smithi093.stdout:695: seq_num 609 ranges {635463=734881,1987181=248282} 2023-08-27T20:52:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:695: writing smithi093153278-201 from 635463 to 1370344 tid 1 2023-08-27T20:52:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:695: writing smithi093153278-201 from 1987181 to 2235463 tid 2 2023-08-27T20:52:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:681: done (14 left) 2023-08-27T20:52:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:696: write oid 466 current snap is 0 2023-08-27T20:52:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:686: expect (ObjNum 253 snap 0 seq_num 253) 2023-08-27T20:52:21.021 INFO:tasks.rados.rados.0.smithi093.stdout:688: expect (ObjNum 574 snap 0 seq_num 574) 2023-08-27T20:52:21.196 INFO:tasks.rados.rados.0.smithi093.stdout:684: expect (ObjNum 399 snap 0 seq_num 399) 2023-08-27T20:52:21.287 INFO:tasks.rados.rados.0.smithi093.stdout:682: finishing write tid 2 to smithi093153278-270 2023-08-27T20:52:21.287 INFO:tasks.rados.rados.0.smithi093.stdout:682: finishing write tid 3 to smithi093153278-270 2023-08-27T20:52:21.287 INFO:tasks.rados.rados.0.smithi093.stdout:682: finishing write tid 4 to smithi093153278-270 2023-08-27T20:52:21.288 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 270 v 131 (ObjNum 604 snap 0 seq_num 604) dirty exists 2023-08-27T20:52:21.288 INFO:tasks.rados.rados.0.smithi093.stdout:682: left oid 270 (ObjNum 604 snap 0 seq_num 604) 2023-08-27T20:52:21.288 INFO:tasks.rados.rados.0.smithi093.stdout:685: expect (ObjNum 98 snap 0 seq_num 98) 2023-08-27T20:52:21.408 INFO:tasks.rados.rados.0.smithi093.stdout:687: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:52:21.527 INFO:tasks.rados.rados.0.smithi093.stdout:683: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:52:21.660 INFO:tasks.rados.rados.0.smithi093.stdout:696: seq_num 610 ranges {652270=410130,1529310=493867} 2023-08-27T20:52:21.665 INFO:tasks.rados.rados.0.smithi093.stdout:696: writing smithi093153278-466 from 652270 to 1062400 tid 1 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:696: writing smithi093153278-466 from 1529310 to 2023177 tid 2 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:682: done (14 left) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:683: done (13 left) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:684: done (12 left) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:685: done (11 left) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:686: done (10 left) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:687: done (9 left) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:688: done (8 left) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:697: read oid 367 snap -1 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:697: expect (ObjNum 366 snap 0 seq_num 366) 2023-08-27T20:52:21.672 INFO:tasks.rados.rados.0.smithi093.stdout:698: read oid 267 snap -1 2023-08-27T20:52:21.673 INFO:tasks.rados.rados.0.smithi093.stdout:698: expect (ObjNum 556 snap 0 seq_num 556) 2023-08-27T20:52:21.673 INFO:tasks.rados.rados.0.smithi093.stdout:699: read oid 297 snap -1 2023-08-27T20:52:21.673 INFO:tasks.rados.rados.0.smithi093.stdout:699: expect (ObjNum 296 snap 0 seq_num 296) 2023-08-27T20:52:21.673 INFO:tasks.rados.rados.0.smithi093.stdout:700: write oid 91 current snap is 0 2023-08-27T20:52:21.673 INFO:tasks.rados.rados.0.smithi093.stdout:700: seq_num 611 ranges {669077=485379,1587792=681285} 2023-08-27T20:52:21.678 INFO:tasks.rados.rados.0.smithi093.stdout:700: writing smithi093153278-91 from 669077 to 1154456 tid 1 2023-08-27T20:52:21.686 INFO:tasks.rados.rados.0.smithi093.stdout:700: writing smithi093153278-91 from 1587792 to 2269077 tid 2 2023-08-27T20:52:21.686 INFO:tasks.rados.rados.0.smithi093.stdout:701: read oid 111 snap -1 2023-08-27T20:52:21.686 INFO:tasks.rados.rados.0.smithi093.stdout:701: expect (ObjNum 110 snap 0 seq_num 110) 2023-08-27T20:52:21.686 INFO:tasks.rados.rados.0.smithi093.stdout:702: read oid 467 snap -1 2023-08-27T20:52:21.686 INFO:tasks.rados.rados.0.smithi093.stdout:702: expect (ObjNum 466 snap 0 seq_num 466) 2023-08-27T20:52:21.686 INFO:tasks.rados.rados.0.smithi093.stdout:703: read oid 347 snap -1 2023-08-27T20:52:21.686 INFO:tasks.rados.rados.0.smithi093.stdout:703: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:52:21.687 INFO:tasks.rados.rados.0.smithi093.stdout:704: write oid 395 current snap is 0 2023-08-27T20:52:21.687 INFO:tasks.rados.rados.0.smithi093.stdout:689: finishing write tid 1 to smithi093153278-122 2023-08-27T20:52:21.687 INFO:tasks.rados.rados.0.smithi093.stdout:689: finishing write tid 2 to smithi093153278-122 2023-08-27T20:52:21.687 INFO:tasks.rados.rados.0.smithi093.stdout:704: seq_num 612 ranges {685884=560628,2046274=239610} 2023-08-27T20:52:21.694 INFO:tasks.rados.rados.0.smithi093.stdout:704: writing smithi093153278-395 from 685884 to 1246512 tid 1 2023-08-27T20:52:21.701 INFO:tasks.rados.rados.0.smithi093.stdout:704: writing smithi093153278-395 from 2046274 to 2285884 tid 2 2023-08-27T20:52:21.701 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:689: finishing write tid 3 to smithi093153278-122 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:689: finishing write tid 4 to smithi093153278-122 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 122 v 93 (ObjNum 605 snap 0 seq_num 605) dirty exists 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:689: left oid 122 (ObjNum 605 snap 0 seq_num 605) 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:690: finishing write tid 1 to smithi093153278-60 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:689: done (15 left) 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:705: write oid 337 current snap is 0 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:690: finishing write tid 2 to smithi093153278-60 2023-08-27T20:52:21.702 INFO:tasks.rados.rados.0.smithi093.stdout:705: seq_num 613 ranges {702691=635877,2104756=197935} 2023-08-27T20:52:21.711 INFO:tasks.rados.rados.0.smithi093.stdout:705: writing smithi093153278-337 from 702691 to 1338568 tid 1 2023-08-27T20:52:21.717 INFO:tasks.rados.rados.0.smithi093.stdout:705: writing smithi093153278-337 from 2104756 to 2302691 tid 2 2023-08-27T20:52:21.717 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:690: finishing write tid 3 to smithi093153278-60 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:690: finishing write tid 4 to smithi093153278-60 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 60 v 117 (ObjNum 606 snap 0 seq_num 606) dirty exists 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:690: left oid 60 (ObjNum 606 snap 0 seq_num 606) 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:691: finishing write tid 1 to smithi093153278-273 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:691: finishing write tid 2 to smithi093153278-273 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:691: finishing write tid 3 to smithi093153278-273 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:691: finishing write tid 4 to smithi093153278-273 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 273 v 166 (ObjNum 607 snap 0 seq_num 607) dirty exists 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:691: left oid 273 (ObjNum 607 snap 0 seq_num 607) 2023-08-27T20:52:21.718 INFO:tasks.rados.rados.0.smithi093.stdout:693: expect (ObjNum 99 snap 0 seq_num 99) 2023-08-27T20:52:21.840 INFO:tasks.rados.rados.0.smithi093.stdout:692: expect (ObjNum 196 snap 0 seq_num 196) 2023-08-27T20:52:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:21 smithi136 ceph-mon[113531]: pgmap v65: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 209 op/s 2023-08-27T20:52:21.950 INFO:tasks.rados.rados.0.smithi093.stdout:694: finishing write tid 1 to smithi093153278-316 2023-08-27T20:52:21.950 INFO:tasks.rados.rados.0.smithi093.stdout:694: finishing write tid 2 to smithi093153278-316 2023-08-27T20:52:21.950 INFO:tasks.rados.rados.0.smithi093.stdout:690: done (15 left) 2023-08-27T20:52:21.950 INFO:tasks.rados.rados.0.smithi093.stdout:691: done (14 left) 2023-08-27T20:52:21.950 INFO:tasks.rados.rados.0.smithi093.stdout:692: done (13 left) 2023-08-27T20:52:21.951 INFO:tasks.rados.rados.0.smithi093.stdout:693: done (12 left) 2023-08-27T20:52:21.951 INFO:tasks.rados.rados.0.smithi093.stdout:706: write oid 278 current snap is 0 2023-08-27T20:52:21.951 INFO:tasks.rados.rados.0.smithi093.stdout:694: finishing write tid 3 to smithi093153278-316 2023-08-27T20:52:21.951 INFO:tasks.rados.rados.0.smithi093.stdout:694: finishing write tid 4 to smithi093153278-316 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 316 v 154 (ObjNum 608 snap 0 seq_num 608) dirty exists 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:694: left oid 316 (ObjNum 608 snap 0 seq_num 608) 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:695: finishing write tid 1 to smithi093153278-201 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:695: finishing write tid 2 to smithi093153278-201 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:695: finishing write tid 3 to smithi093153278-201 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:695: finishing write tid 4 to smithi093153278-201 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 201 v 124 (ObjNum 609 snap 0 seq_num 609) dirty exists 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:695: left oid 201 (ObjNum 609 snap 0 seq_num 609) 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:696: finishing write tid 1 to smithi093153278-466 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:696: finishing write tid 2 to smithi093153278-466 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:696: finishing write tid 3 to smithi093153278-466 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:696: finishing write tid 4 to smithi093153278-466 2023-08-27T20:52:21.952 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 466 v 142 (ObjNum 610 snap 0 seq_num 610) dirty exists 2023-08-27T20:52:21.953 INFO:tasks.rados.rados.0.smithi093.stdout:696: left oid 466 (ObjNum 610 snap 0 seq_num 610) 2023-08-27T20:52:21.953 INFO:tasks.rados.rados.0.smithi093.stdout:697: expect (ObjNum 366 snap 0 seq_num 366) 2023-08-27T20:52:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:21 smithi093 ceph-mon[111335]: pgmap v65: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 209 op/s 2023-08-27T20:52:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:21 smithi093 ceph-mon[118902]: pgmap v65: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 39 MiB/s rd, 36 MiB/s wr, 209 op/s 2023-08-27T20:52:22.020 INFO:tasks.rados.rados.0.smithi093.stdout:706: seq_num 614 ranges {719498=711126,2046885=272613} 2023-08-27T20:52:22.028 INFO:tasks.rados.rados.0.smithi093.stdout:706: writing smithi093153278-278 from 719498 to 1430624 tid 1 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:706: writing smithi093153278-278 from 2046885 to 2319498 tid 2 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:694: done (12 left) 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:695: done (11 left) 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:696: done (10 left) 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:697: done (9 left) 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:707: read oid 145 snap -1 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:707: expect (ObjNum 144 snap 0 seq_num 144) 2023-08-27T20:52:22.034 INFO:tasks.rados.rados.0.smithi093.stdout:708: write oid 380 current snap is 0 2023-08-27T20:52:22.035 INFO:tasks.rados.rados.0.smithi093.stdout:698: expect (ObjNum 556 snap 0 seq_num 556) 2023-08-27T20:52:22.189 INFO:tasks.rados.rados.0.smithi093.stdout:699: expect (ObjNum 296 snap 0 seq_num 296) 2023-08-27T20:52:22.290 INFO:tasks.rados.rados.0.smithi093.stdout:700: finishing write tid 1 to smithi093153278-91 2023-08-27T20:52:22.290 INFO:tasks.rados.rados.0.smithi093.stdout:708: seq_num 615 ranges {736305=786375,2105367=230938} 2023-08-27T20:52:22.298 INFO:tasks.rados.rados.0.smithi093.stdout:708: writing smithi093153278-380 from 736305 to 1522680 tid 1 2023-08-27T20:52:22.303 INFO:tasks.rados.rados.0.smithi093.stdout:708: writing smithi093153278-380 from 2105367 to 2336305 tid 2 2023-08-27T20:52:22.303 INFO:tasks.rados.rados.0.smithi093.stdout:698: done (10 left) 2023-08-27T20:52:22.304 INFO:tasks.rados.rados.0.smithi093.stdout:699: done (9 left) 2023-08-27T20:52:22.304 INFO:tasks.rados.rados.0.smithi093.stdout:709: write oid 459 current snap is 0 2023-08-27T20:52:22.304 INFO:tasks.rados.rados.0.smithi093.stdout:700: finishing write tid 2 to smithi093153278-91 2023-08-27T20:52:22.304 INFO:tasks.rados.rados.0.smithi093.stdout:709: seq_num 616 ranges {753112=577977,1880202=472910} 2023-08-27T20:52:22.311 INFO:tasks.rados.rados.0.smithi093.stdout:709: writing smithi093153278-459 from 753112 to 1331089 tid 1 2023-08-27T20:52:22.317 INFO:tasks.rados.rados.0.smithi093.stdout:709: writing smithi093153278-459 from 1880202 to 2353112 tid 2 2023-08-27T20:52:22.317 INFO:tasks.rados.rados.0.smithi093.stdout:710: write oid 438 current snap is 0 2023-08-27T20:52:22.317 INFO:tasks.rados.rados.0.smithi093.stdout:700: finishing write tid 3 to smithi093153278-91 2023-08-27T20:52:22.318 INFO:tasks.rados.rados.0.smithi093.stdout:700: finishing write tid 4 to smithi093153278-91 2023-08-27T20:52:22.318 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 91 v 145 (ObjNum 611 snap 0 seq_num 611) dirty exists 2023-08-27T20:52:22.318 INFO:tasks.rados.rados.0.smithi093.stdout:700: left oid 91 (ObjNum 611 snap 0 seq_num 611) 2023-08-27T20:52:22.318 INFO:tasks.rados.rados.0.smithi093.stdout:703: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:52:22.445 INFO:tasks.rados.rados.0.smithi093.stdout:701: expect (ObjNum 110 snap 0 seq_num 110) 2023-08-27T20:52:22.577 INFO:tasks.rados.rados.0.smithi093.stdout:704: finishing write tid 1 to smithi093153278-395 2023-08-27T20:52:22.577 INFO:tasks.rados.rados.0.smithi093.stdout:710: seq_num 617 ranges {769919=653226,1938684=431235} 2023-08-27T20:52:22.584 INFO:tasks.rados.rados.0.smithi093.stdout:710: writing smithi093153278-438 from 769919 to 1423145 tid 1 2023-08-27T20:52:22.590 INFO:tasks.rados.rados.0.smithi093.stdout:710: writing smithi093153278-438 from 1938684 to 2369919 tid 2 2023-08-27T20:52:22.590 INFO:tasks.rados.rados.0.smithi093.stdout:700: done (10 left) 2023-08-27T20:52:22.590 INFO:tasks.rados.rados.0.smithi093.stdout:701: done (9 left) 2023-08-27T20:52:22.590 INFO:tasks.rados.rados.0.smithi093.stdout:703: done (8 left) 2023-08-27T20:52:22.590 INFO:tasks.rados.rados.0.smithi093.stdout:711: read oid 377 snap -1 2023-08-27T20:52:22.590 INFO:tasks.rados.rados.0.smithi093.stdout:711: expect (ObjNum 376 snap 0 seq_num 376) 2023-08-27T20:52:22.590 INFO:tasks.rados.rados.0.smithi093.stdout:712: write oid 195 current snap is 0 2023-08-27T20:52:22.591 INFO:tasks.rados.rados.0.smithi093.stdout:704: finishing write tid 2 to smithi093153278-395 2023-08-27T20:52:22.591 INFO:tasks.rados.rados.0.smithi093.stdout:712: seq_num 618 ranges {786726=728475,2280813=105913} 2023-08-27T20:52:22.598 INFO:tasks.rados.rados.0.smithi093.stdout:712: writing smithi093153278-195 from 786726 to 1515201 tid 1 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:712: writing smithi093153278-195 from 2280813 to 2386726 tid 2 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:713: read oid 446 snap -1 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:713: expect (ObjNum 445 snap 0 seq_num 445) 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:714: delete oid 86 current snap is 0 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:704: finishing write tid 3 to smithi093153278-395 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:704: finishing write tid 4 to smithi093153278-395 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 169 (ObjNum 612 snap 0 seq_num 612) dirty exists 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:704: left oid 395 (ObjNum 612 snap 0 seq_num 612) 2023-08-27T20:52:22.604 INFO:tasks.rados.rados.0.smithi093.stdout:702: expect (ObjNum 466 snap 0 seq_num 466) 2023-08-27T20:52:22.728 INFO:tasks.rados.rados.0.smithi093.stdout:705: finishing write tid 1 to smithi093153278-337 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:705: finishing write tid 2 to smithi093153278-337 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:705: finishing write tid 3 to smithi093153278-337 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:705: finishing write tid 4 to smithi093153278-337 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 337 v 156 (ObjNum 613 snap 0 seq_num 613) dirty exists 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:705: left oid 337 (ObjNum 613 snap 0 seq_num 613) 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:702: done (11 left) 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:704: done (10 left) 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:705: done (9 left) 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:714: done (8 left) 2023-08-27T20:52:22.729 INFO:tasks.rados.rados.0.smithi093.stdout:715: delete oid 360 current snap is 0 2023-08-27T20:52:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:706: finishing write tid 1 to smithi093153278-278 2023-08-27T20:52:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:706: finishing write tid 2 to smithi093153278-278 2023-08-27T20:52:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:706: finishing write tid 3 to smithi093153278-278 2023-08-27T20:52:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:706: finishing write tid 4 to smithi093153278-278 2023-08-27T20:52:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 278 v 172 (ObjNum 614 snap 0 seq_num 614) dirty exists 2023-08-27T20:52:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:706: left oid 278 (ObjNum 614 snap 0 seq_num 614) 2023-08-27T20:52:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:707: expect (ObjNum 144 snap 0 seq_num 144) 2023-08-27T20:52:22.807 INFO:tasks.rados.rados.0.smithi093.stdout:708: finishing write tid 1 to smithi093153278-380 2023-08-27T20:52:22.807 INFO:tasks.rados.rados.0.smithi093.stdout:708: finishing write tid 2 to smithi093153278-380 2023-08-27T20:52:22.807 INFO:tasks.rados.rados.0.smithi093.stdout:708: finishing write tid 3 to smithi093153278-380 2023-08-27T20:52:22.807 INFO:tasks.rados.rados.0.smithi093.stdout:708: finishing write tid 4 to smithi093153278-380 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 380 v 140 (ObjNum 615 snap 0 seq_num 615) dirty exists 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:708: left oid 380 (ObjNum 615 snap 0 seq_num 615) 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:709: finishing write tid 1 to smithi093153278-459 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:709: finishing write tid 2 to smithi093153278-459 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:706: done (8 left) 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:707: done (7 left) 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:708: done (6 left) 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:715: done (5 left) 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:716: write oid 406 current snap is 0 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:709: finishing write tid 3 to smithi093153278-459 2023-08-27T20:52:22.808 INFO:tasks.rados.rados.0.smithi093.stdout:709: finishing write tid 4 to smithi093153278-459 2023-08-27T20:52:22.809 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 459 v 141 (ObjNum 616 snap 0 seq_num 616) dirty exists 2023-08-27T20:52:22.809 INFO:tasks.rados.rados.0.smithi093.stdout:709: left oid 459 (ObjNum 616 snap 0 seq_num 616) 2023-08-27T20:52:22.809 INFO:tasks.rados.rados.0.smithi093.stdout:710: finishing write tid 1 to smithi093153278-438 2023-08-27T20:52:22.809 INFO:tasks.rados.rados.0.smithi093.stdout:716: seq_num 621 ranges {437147=554222,1656259=655870} 2023-08-27T20:52:22.812 INFO:tasks.rados.rados.0.smithi093.stdout:716: writing smithi093153278-406 from 437147 to 991369 tid 1 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:716: writing smithi093153278-406 from 1656259 to 2312129 tid 2 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:709: done (5 left) 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:717: write oid 353 current snap is 0 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:710: finishing write tid 2 to smithi093153278-438 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:710: finishing write tid 3 to smithi093153278-438 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:710: finishing write tid 4 to smithi093153278-438 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 438 v 144 (ObjNum 617 snap 0 seq_num 617) dirty exists 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:710: left oid 438 (ObjNum 617 snap 0 seq_num 617) 2023-08-27T20:52:22.820 INFO:tasks.rados.rados.0.smithi093.stdout:711: expect (ObjNum 376 snap 0 seq_num 376) 2023-08-27T20:52:22.899 INFO:tasks.rados.rados.0.smithi093.stdout:712: finishing write tid 1 to smithi093153278-195 2023-08-27T20:52:22.900 INFO:tasks.rados.rados.0.smithi093.stdout:717: seq_num 622 ranges {453954=629471,1598388=515881} 2023-08-27T20:52:22.905 INFO:tasks.rados.rados.0.smithi093.stdout:717: writing smithi093153278-353 from 453954 to 1083425 tid 1 2023-08-27T20:52:22.911 INFO:tasks.rados.rados.0.smithi093.stdout:717: writing smithi093153278-353 from 1598388 to 2114269 tid 2 2023-08-27T20:52:22.911 INFO:tasks.rados.rados.0.smithi093.stdout:710: done (5 left) 2023-08-27T20:52:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:711: done (4 left) 2023-08-27T20:52:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:718: read oid 208 snap -1 2023-08-27T20:52:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:718: expect (ObjNum 207 snap 0 seq_num 207) 2023-08-27T20:52:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:719: write oid 44 current snap is 0 2023-08-27T20:52:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:712: finishing write tid 2 to smithi093153278-195 2023-08-27T20:52:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:712: finishing write tid 3 to smithi093153278-195 2023-08-27T20:52:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:713: expect (ObjNum 445 snap 0 seq_num 445) 2023-08-27T20:52:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3991015449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3991015449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3991015449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:23.039 INFO:tasks.rados.rados.0.smithi093.stdout:712: finishing write tid 4 to smithi093153278-195 2023-08-27T20:52:23.039 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 113 (ObjNum 618 snap 0 seq_num 618) dirty exists 2023-08-27T20:52:23.039 INFO:tasks.rados.rados.0.smithi093.stdout:712: left oid 195 (ObjNum 618 snap 0 seq_num 618) 2023-08-27T20:52:23.039 INFO:tasks.rados.rados.0.smithi093.stdout:716: finishing write tid 1 to smithi093153278-406 2023-08-27T20:52:23.039 INFO:tasks.rados.rados.0.smithi093.stdout:716: finishing write tid 2 to smithi093153278-406 2023-08-27T20:52:23.039 INFO:tasks.rados.rados.0.smithi093.stdout:719: seq_num 623 ranges {470761=704720,1656870=659539} 2023-08-27T20:52:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:719: writing smithi093153278-44 from 470761 to 1175481 tid 1 2023-08-27T20:52:23.052 INFO:tasks.rados.rados.0.smithi093.stdout:719: writing smithi093153278-44 from 1656870 to 2316409 tid 2 2023-08-27T20:52:23.052 INFO:tasks.rados.rados.0.smithi093.stdout:712: done (5 left) 2023-08-27T20:52:23.052 INFO:tasks.rados.rados.0.smithi093.stdout:713: done (4 left) 2023-08-27T20:52:23.053 INFO:tasks.rados.rados.0.smithi093.stdout:720: write oid 486 current snap is 0 2023-08-27T20:52:23.053 INFO:tasks.rados.rados.0.smithi093.stdout:716: finishing write tid 3 to smithi093153278-406 2023-08-27T20:52:23.053 INFO:tasks.rados.rados.0.smithi093.stdout:716: finishing write tid 4 to smithi093153278-406 2023-08-27T20:52:23.053 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 406 v 125 (ObjNum 621 snap 0 seq_num 621) dirty exists 2023-08-27T20:52:23.053 INFO:tasks.rados.rados.0.smithi093.stdout:716: left oid 406 (ObjNum 621 snap 0 seq_num 621) 2023-08-27T20:52:23.053 INFO:tasks.rados.rados.0.smithi093.stdout:717: finishing write tid 1 to smithi093153278-353 2023-08-27T20:52:23.053 INFO:tasks.rados.rados.0.smithi093.stdout:720: seq_num 624 ranges {487568=496322,1431705=519550} 2023-08-27T20:52:23.057 INFO:tasks.rados.rados.0.smithi093.stdout:720: writing smithi093153278-486 from 487568 to 983890 tid 1 2023-08-27T20:52:23.063 INFO:tasks.rados.rados.0.smithi093.stdout:720: writing smithi093153278-486 from 1431705 to 1951255 tid 2 2023-08-27T20:52:23.063 INFO:tasks.rados.rados.0.smithi093.stdout:716: done (4 left) 2023-08-27T20:52:23.063 INFO:tasks.rados.rados.0.smithi093.stdout:721: read oid 65 snap -1 2023-08-27T20:52:23.063 INFO:tasks.rados.rados.0.smithi093.stdout:721: expect (ObjNum 64 snap 0 seq_num 64) 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:722: read oid 132 snap -1 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:722: expect (ObjNum 131 snap 0 seq_num 131) 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:723: write oid 495 current snap is 0 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:717: finishing write tid 2 to smithi093153278-353 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:717: finishing write tid 3 to smithi093153278-353 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:717: finishing write tid 4 to smithi093153278-353 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 353 v 128 (ObjNum 622 snap 0 seq_num 622) dirty exists 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:717: left oid 353 (ObjNum 622 snap 0 seq_num 622) 2023-08-27T20:52:23.064 INFO:tasks.rados.rados.0.smithi093.stdout:718: expect (ObjNum 207 snap 0 seq_num 207) 2023-08-27T20:52:23.186 INFO:tasks.rados.rados.0.smithi093.stdout:723: seq_num 625 ranges {504375=571571,1490187=663208} 2023-08-27T20:52:23.191 INFO:tasks.rados.rados.0.smithi093.stdout:723: writing smithi093153278-495 from 504375 to 1075946 tid 1 2023-08-27T20:52:23.198 INFO:tasks.rados.rados.0.smithi093.stdout:723: writing smithi093153278-495 from 1490187 to 2153395 tid 2 2023-08-27T20:52:23.198 INFO:tasks.rados.rados.0.smithi093.stdout:717: done (6 left) 2023-08-27T20:52:23.198 INFO:tasks.rados.rados.0.smithi093.stdout:718: done (5 left) 2023-08-27T20:52:23.198 INFO:tasks.rados.rados.0.smithi093.stdout:724: write oid 212 current snap is 0 2023-08-27T20:52:23.199 INFO:tasks.rados.rados.0.smithi093.stdout:719: finishing write tid 1 to smithi093153278-44 2023-08-27T20:52:23.199 INFO:tasks.rados.rados.0.smithi093.stdout:719: finishing write tid 2 to smithi093153278-44 2023-08-27T20:52:23.199 INFO:tasks.rados.rados.0.smithi093.stdout:719: finishing write tid 3 to smithi093153278-44 2023-08-27T20:52:23.199 INFO:tasks.rados.rados.0.smithi093.stdout:724: seq_num 626 ranges {521182=646820,1948669=523219} 2023-08-27T20:52:23.204 INFO:tasks.rados.rados.0.smithi093.stdout:724: writing smithi093153278-212 from 521182 to 1168002 tid 1 2023-08-27T20:52:23.212 INFO:tasks.rados.rados.0.smithi093.stdout:724: writing smithi093153278-212 from 1948669 to 2471888 tid 2 2023-08-27T20:52:23.212 INFO:tasks.rados.rados.0.smithi093.stdout:725: write oid 434 current snap is 0 2023-08-27T20:52:23.212 INFO:tasks.rados.rados.0.smithi093.stdout:725: seq_num 627 ranges {537989=722069,1890798=647191} 2023-08-27T20:52:23.219 INFO:tasks.rados.rados.0.smithi093.stdout:725: writing smithi093153278-434 from 537989 to 1260058 tid 1 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:725: writing smithi093153278-434 from 1890798 to 2537989 tid 2 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:726: read oid 75 snap -1 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:726: expect (ObjNum 74 snap 0 seq_num 74) 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:727: read oid 40 snap -1 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:727: expect (ObjNum 39 snap 0 seq_num 39) 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:728: read oid 272 snap -1 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:728: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:729: write oid 205 current snap is 0 2023-08-27T20:52:23.227 INFO:tasks.rados.rados.0.smithi093.stdout:729: seq_num 628 ranges {554796=797318,1949280=526888} 2023-08-27T20:52:23.235 INFO:tasks.rados.rados.0.smithi093.stdout:729: writing smithi093153278-205 from 554796 to 1352114 tid 1 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:729: writing smithi093153278-205 from 1949280 to 2476168 tid 2 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:730: read oid 51 snap -1 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:730: expect (ObjNum 50 snap 0 seq_num 50) 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:731: read oid 358 snap -1 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:731: expect (ObjNum 357 snap 0 seq_num 357) 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:732: write oid 186 current snap is 0 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:719: finishing write tid 4 to smithi093153278-44 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 44 v 131 (ObjNum 623 snap 0 seq_num 623) dirty exists 2023-08-27T20:52:23.242 INFO:tasks.rados.rados.0.smithi093.stdout:719: left oid 44 (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:52:23.243 INFO:tasks.rados.rados.0.smithi093.stdout:720: finishing write tid 1 to smithi093153278-486 2023-08-27T20:52:23.243 INFO:tasks.rados.rados.0.smithi093.stdout:732: seq_num 629 ranges {571603=472567,1607762=670546} 2023-08-27T20:52:23.247 INFO:tasks.rados.rados.0.smithi093.stdout:732: writing smithi093153278-186 from 571603 to 1044170 tid 1 2023-08-27T20:52:23.254 INFO:tasks.rados.rados.0.smithi093.stdout:732: writing smithi093153278-186 from 1607762 to 2278308 tid 2 2023-08-27T20:52:23.255 INFO:tasks.rados.rados.0.smithi093.stdout:719: done (13 left) 2023-08-27T20:52:23.255 INFO:tasks.rados.rados.0.smithi093.stdout:733: write oid 156 current snap is 0 2023-08-27T20:52:23.255 INFO:tasks.rados.rados.0.smithi093.stdout:733: seq_num 630 ranges {588410=547816,1666244=414204,2484603=103807} 2023-08-27T20:52:23.261 INFO:tasks.rados.rados.0.smithi093.stdout:733: writing smithi093153278-156 from 588410 to 1136226 tid 1 2023-08-27T20:52:23.266 INFO:tasks.rados.rados.0.smithi093.stdout:733: writing smithi093153278-156 from 1666244 to 2080448 tid 2 2023-08-27T20:52:23.269 INFO:tasks.rados.rados.0.smithi093.stdout:733: writing smithi093153278-156 from 2484603 to 2588410 tid 3 2023-08-27T20:52:23.269 INFO:tasks.rados.rados.0.smithi093.stdout:734: write oid 89 current snap is 0 2023-08-27T20:52:23.269 INFO:tasks.rados.rados.0.smithi093.stdout:720: finishing write tid 2 to smithi093153278-486 2023-08-27T20:52:23.269 INFO:tasks.rados.rados.0.smithi093.stdout:722: expect (ObjNum 131 snap 0 seq_num 131) 2023-08-27T20:52:23.372 INFO:tasks.rados.rados.0.smithi093.stdout:720: finishing write tid 3 to smithi093153278-486 2023-08-27T20:52:23.373 INFO:tasks.rados.rados.0.smithi093.stdout:734: seq_num 631 ranges {605217=739418,2124726=480491} 2023-08-27T20:52:23.380 INFO:tasks.rados.rados.0.smithi093.stdout:734: writing smithi093153278-89 from 605217 to 1344635 tid 1 2023-08-27T20:52:23.389 INFO:tasks.rados.rados.0.smithi093.stdout:734: writing smithi093153278-89 from 2124726 to 2605217 tid 2 2023-08-27T20:52:23.389 INFO:tasks.rados.rados.0.smithi093.stdout:722: done (14 left) 2023-08-27T20:52:23.389 INFO:tasks.rados.rados.0.smithi093.stdout:735: write oid 61 current snap is 0 2023-08-27T20:52:23.389 INFO:tasks.rados.rados.0.smithi093.stdout:720: finishing write tid 4 to smithi093153278-486 2023-08-27T20:52:23.389 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 143 (ObjNum 624 snap 0 seq_num 624) dirty exists 2023-08-27T20:52:23.389 INFO:tasks.rados.rados.0.smithi093.stdout:720: left oid 486 (ObjNum 624 snap 0 seq_num 624) 2023-08-27T20:52:23.389 INFO:tasks.rados.rados.0.smithi093.stdout:721: expect (ObjNum 64 snap 0 seq_num 64) 2023-08-27T20:52:23.507 INFO:tasks.rados.rados.0.smithi093.stdout:735: seq_num 632 ranges {622024=414667,1783208=417873} 2023-08-27T20:52:23.513 INFO:tasks.rados.rados.0.smithi093.stdout:735: writing smithi093153278-61 from 622024 to 1036691 tid 1 2023-08-27T20:52:23.520 INFO:tasks.rados.rados.0.smithi093.stdout:735: writing smithi093153278-61 from 1783208 to 2201081 tid 2 2023-08-27T20:52:23.520 INFO:tasks.rados.rados.0.smithi093.stdout:720: done (14 left) 2023-08-27T20:52:23.520 INFO:tasks.rados.rados.0.smithi093.stdout:721: done (13 left) 2023-08-27T20:52:23.520 INFO:tasks.rados.rados.0.smithi093.stdout:736: write oid 368 current snap is 0 2023-08-27T20:52:23.520 INFO:tasks.rados.rados.0.smithi093.stdout:723: finishing write tid 1 to smithi093153278-495 2023-08-27T20:52:23.520 INFO:tasks.rados.rados.0.smithi093.stdout:736: seq_num 633 ranges {638831=489916,1841690=677884} 2023-08-27T20:52:23.526 INFO:tasks.rados.rados.0.smithi093.stdout:736: writing smithi093153278-368 from 638831 to 1128747 tid 1 2023-08-27T20:52:23.534 INFO:tasks.rados.rados.0.smithi093.stdout:736: writing smithi093153278-368 from 1841690 to 2519574 tid 2 2023-08-27T20:52:23.534 INFO:tasks.rados.rados.0.smithi093.stdout:737: write oid 486 current snap is 0 2023-08-27T20:52:23.534 INFO:tasks.rados.rados.0.smithi093.stdout:723: finishing write tid 2 to smithi093153278-495 2023-08-27T20:52:23.535 INFO:tasks.rados.rados.0.smithi093.stdout:737: seq_num 634 ranges {655638=565165,1900172=421542} 2023-08-27T20:52:23.541 INFO:tasks.rados.rados.0.smithi093.stdout:737: writing smithi093153278-486 from 655638 to 1220803 tid 1 2023-08-27T20:52:23.548 INFO:tasks.rados.rados.0.smithi093.stdout:737: writing smithi093153278-486 from 1900172 to 2321714 tid 2 2023-08-27T20:52:23.548 INFO:tasks.rados.rados.0.smithi093.stdout:738: read oid 436 snap -1 2023-08-27T20:52:23.548 INFO:tasks.rados.rados.0.smithi093.stdout:738: expect (ObjNum 435 snap 0 seq_num 435) 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:723: finishing write tid 3 to smithi093153278-495 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:723: finishing write tid 4 to smithi093153278-495 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 495 v 159 (ObjNum 625 snap 0 seq_num 625) dirty exists 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:723: left oid 495 (ObjNum 625 snap 0 seq_num 625) 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:724: finishing write tid 1 to smithi093153278-212 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:724: finishing write tid 2 to smithi093153278-212 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:724: finishing write tid 3 to smithi093153278-212 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:724: finishing write tid 4 to smithi093153278-212 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 212 v 134 (ObjNum 626 snap 0 seq_num 626) dirty exists 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:724: left oid 212 (ObjNum 626 snap 0 seq_num 626) 2023-08-27T20:52:23.549 INFO:tasks.rados.rados.0.smithi093.stdout:723: done (15 left) 2023-08-27T20:52:23.550 INFO:tasks.rados.rados.0.smithi093.stdout:724: done (14 left) 2023-08-27T20:52:23.550 INFO:tasks.rados.rados.0.smithi093.stdout:739: write oid 104 current snap is 0 2023-08-27T20:52:23.550 INFO:tasks.rados.rados.0.smithi093.stdout:739: seq_num 635 ranges {672445=640414,1842301=681553} 2023-08-27T20:52:23.555 INFO:tasks.rados.rados.0.smithi093.stdout:739: writing smithi093153278-104 from 672445 to 1312859 tid 1 2023-08-27T20:52:23.562 INFO:tasks.rados.rados.0.smithi093.stdout:739: writing smithi093153278-104 from 1842301 to 2523854 tid 2 2023-08-27T20:52:23.562 INFO:tasks.rados.rados.0.smithi093.stdout:740: write oid 146 current snap is 0 2023-08-27T20:52:23.563 INFO:tasks.rados.rados.0.smithi093.stdout:725: finishing write tid 1 to smithi093153278-434 2023-08-27T20:52:23.563 INFO:tasks.rados.rados.0.smithi093.stdout:728: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:23.655 INFO:tasks.rados.rados.0.smithi093.stdout:727: expect (ObjNum 39 snap 0 seq_num 39) 2023-08-27T20:52:23.748 INFO:tasks.rados.rados.0.smithi093.stdout:725: finishing write tid 2 to smithi093153278-434 2023-08-27T20:52:23.748 INFO:tasks.rados.rados.0.smithi093.stdout:725: finishing write tid 3 to smithi093153278-434 2023-08-27T20:52:23.748 INFO:tasks.rados.rados.0.smithi093.stdout:725: finishing write tid 4 to smithi093153278-434 2023-08-27T20:52:23.748 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 120 (ObjNum 627 snap 0 seq_num 627) dirty exists 2023-08-27T20:52:23.748 INFO:tasks.rados.rados.0.smithi093.stdout:725: left oid 434 (ObjNum 627 snap 0 seq_num 627) 2023-08-27T20:52:23.748 INFO:tasks.rados.rados.0.smithi093.stdout:740: seq_num 636 ranges {689252=715663,1900783=425211} 2023-08-27T20:52:23.755 INFO:tasks.rados.rados.0.smithi093.stdout:740: writing smithi093153278-146 from 689252 to 1404915 tid 1 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:740: writing smithi093153278-146 from 1900783 to 2325994 tid 2 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:725: done (15 left) 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:727: done (14 left) 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:728: done (13 left) 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:741: read oid 422 snap -1 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:741: expect (ObjNum 421 snap 0 seq_num 421) 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:742: delete oid 389 current snap is 0 2023-08-27T20:52:23.761 INFO:tasks.rados.rados.0.smithi093.stdout:726: expect (ObjNum 74 snap 0 seq_num 74) 2023-08-27T20:52:23.867 INFO:tasks.rados.rados.0.smithi093.stdout:729: finishing write tid 1 to smithi093153278-205 2023-08-27T20:52:23.867 INFO:tasks.rados.rados.0.smithi093.stdout:726: done (14 left) 2023-08-27T20:52:23.867 INFO:tasks.rados.rados.0.smithi093.stdout:742: done (13 left) 2023-08-27T20:52:23.867 INFO:tasks.rados.rados.0.smithi093.stdout:743: read oid 151 snap -1 2023-08-27T20:52:23.867 INFO:tasks.rados.rados.0.smithi093.stdout:743: expect (ObjNum 150 snap 0 seq_num 150) 2023-08-27T20:52:23.867 INFO:tasks.rados.rados.0.smithi093.stdout:744: read oid 4 snap -1 2023-08-27T20:52:23.867 INFO:tasks.rados.rados.0.smithi093.stdout:744: expect (ObjNum 3 snap 0 seq_num 3) 2023-08-27T20:52:23.868 INFO:tasks.rados.rados.0.smithi093.stdout:745: read oid 188 snap -1 2023-08-27T20:52:23.868 INFO:tasks.rados.rados.0.smithi093.stdout:745: expect (ObjNum 187 snap 0 seq_num 187) 2023-08-27T20:52:23.868 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:23.868 INFO:tasks.rados.rados.0.smithi093.stdout:729: finishing write tid 2 to smithi093153278-205 2023-08-27T20:52:23.868 INFO:tasks.rados.rados.0.smithi093.stdout:731: expect (ObjNum 357 snap 0 seq_num 357) 2023-08-27T20:52:23.933 INFO:tasks.rados.rados.0.smithi093.stdout:729: finishing write tid 3 to smithi093153278-205 2023-08-27T20:52:23.933 INFO:tasks.rados.rados.0.smithi093.stdout:729: finishing write tid 4 to smithi093153278-205 2023-08-27T20:52:23.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 205 v 96 (ObjNum 628 snap 0 seq_num 628) dirty exists 2023-08-27T20:52:23.933 INFO:tasks.rados.rados.0.smithi093.stdout:729: left oid 205 (ObjNum 628 snap 0 seq_num 628) 2023-08-27T20:52:23.933 INFO:tasks.rados.rados.0.smithi093.stdout:730: expect (ObjNum 50 snap 0 seq_num 50) 2023-08-27T20:52:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:23 smithi136 ceph-mon[113531]: pgmap v66: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s rd, 41 MiB/s wr, 261 op/s 2023-08-27T20:52:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:23 smithi093 ceph-mon[111335]: pgmap v66: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s rd, 41 MiB/s wr, 261 op/s 2023-08-27T20:52:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:23 smithi093 ceph-mon[118902]: pgmap v66: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 77 MiB/s rd, 41 MiB/s wr, 261 op/s 2023-08-27T20:52:24.025 INFO:tasks.rados.rados.0.smithi093.stdout:732: finishing write tid 1 to smithi093153278-186 2023-08-27T20:52:24.025 INFO:tasks.rados.rados.0.smithi093.stdout:732: finishing write tid 2 to smithi093153278-186 2023-08-27T20:52:24.025 INFO:tasks.rados.rados.0.smithi093.stdout:729: done (15 left) 2023-08-27T20:52:24.025 INFO:tasks.rados.rados.0.smithi093.stdout:730: done (14 left) 2023-08-27T20:52:24.025 INFO:tasks.rados.rados.0.smithi093.stdout:731: done (13 left) 2023-08-27T20:52:24.026 INFO:tasks.rados.rados.0.smithi093.stdout:746: write oid 326 current snap is 0 2023-08-27T20:52:24.026 INFO:tasks.rados.rados.0.smithi093.stdout:732: finishing write tid 3 to smithi093153278-186 2023-08-27T20:52:24.026 INFO:tasks.rados.rados.0.smithi093.stdout:732: finishing write tid 4 to smithi093153278-186 2023-08-27T20:52:24.026 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 186 v 148 (ObjNum 629 snap 0 seq_num 629) dirty exists 2023-08-27T20:52:24.027 INFO:tasks.rados.rados.0.smithi093.stdout:732: left oid 186 (ObjNum 629 snap 0 seq_num 629) 2023-08-27T20:52:24.027 INFO:tasks.rados.rados.0.smithi093.stdout:733: finishing write tid 1 to smithi093153278-156 2023-08-27T20:52:24.027 INFO:tasks.rados.rados.0.smithi093.stdout:733: finishing write tid 2 to smithi093153278-156 2023-08-27T20:52:24.027 INFO:tasks.rados.rados.0.smithi093.stdout:733: finishing write tid 3 to smithi093153278-156 2023-08-27T20:52:24.027 INFO:tasks.rados.rados.0.smithi093.stdout:746: seq_num 638 ranges {722866=466161,1617747=428880,2587634=135232} 2023-08-27T20:52:24.032 INFO:tasks.rados.rados.0.smithi093.stdout:746: writing smithi093153278-326 from 722866 to 1189027 tid 1 2023-08-27T20:52:24.038 INFO:tasks.rados.rados.0.smithi093.stdout:746: writing smithi093153278-326 from 1617747 to 2046627 tid 2 2023-08-27T20:52:24.043 INFO:tasks.rados.rados.0.smithi093.stdout:746: writing smithi093153278-326 from 2587634 to 2722866 tid 3 2023-08-27T20:52:24.043 INFO:tasks.rados.rados.0.smithi093.stdout:732: done (13 left) 2023-08-27T20:52:24.043 INFO:tasks.rados.rados.0.smithi093.stdout:747: read oid 31 snap -1 2023-08-27T20:52:24.043 INFO:tasks.rados.rados.0.smithi093.stdout:747: expect deleted 2023-08-27T20:52:24.043 INFO:tasks.rados.rados.0.smithi093.stdout:733: finishing write tid 4 to smithi093153278-156 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:733: finishing write tid 5 to smithi093153278-156 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 156 v 129 (ObjNum 630 snap 0 seq_num 630) dirty exists 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:733: left oid 156 (ObjNum 630 snap 0 seq_num 630) 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:734: finishing write tid 1 to smithi093153278-89 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:734: finishing write tid 2 to smithi093153278-89 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:734: finishing write tid 3 to smithi093153278-89 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:748: read oid 179 snap -1 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:748: expect (ObjNum 178 snap 0 seq_num 178) 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:733: done (14 left) 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:749: delete oid 362 current snap is 0 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:734: finishing write tid 4 to smithi093153278-89 2023-08-27T20:52:24.044 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 89 v 148 (ObjNum 631 snap 0 seq_num 631) dirty exists 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:734: left oid 89 (ObjNum 631 snap 0 seq_num 631) 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:735: finishing write tid 1 to smithi093153278-61 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:735: finishing write tid 2 to smithi093153278-61 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:735: finishing write tid 3 to smithi093153278-61 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:735: finishing write tid 4 to smithi093153278-61 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 61 v 146 (ObjNum 632 snap 0 seq_num 632) dirty exists 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:735: left oid 61 (ObjNum 632 snap 0 seq_num 632) 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:736: finishing write tid 1 to smithi093153278-368 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:736: finishing write tid 2 to smithi093153278-368 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:736: finishing write tid 3 to smithi093153278-368 2023-08-27T20:52:24.045 INFO:tasks.rados.rados.0.smithi093.stdout:736: finishing write tid 4 to smithi093153278-368 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 368 v 151 (ObjNum 633 snap 0 seq_num 633) dirty exists 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:736: left oid 368 (ObjNum 633 snap 0 seq_num 633) 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:737: finishing write tid 1 to smithi093153278-486 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:737: finishing write tid 2 to smithi093153278-486 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:737: finishing write tid 3 to smithi093153278-486 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:737: finishing write tid 4 to smithi093153278-486 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 149 (ObjNum 634 snap 0 seq_num 634) dirty exists 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:737: left oid 486 (ObjNum 634 snap 0 seq_num 634) 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:739: finishing write tid 1 to smithi093153278-104 2023-08-27T20:52:24.046 INFO:tasks.rados.rados.0.smithi093.stdout:738: expect (ObjNum 435 snap 0 seq_num 435) 2023-08-27T20:52:24.155 INFO:tasks.rados.rados.0.smithi093.stdout:739: finishing write tid 2 to smithi093153278-104 2023-08-27T20:52:24.155 INFO:tasks.rados.rados.0.smithi093.stdout:739: finishing write tid 3 to smithi093153278-104 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:734: done (14 left) 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:735: done (13 left) 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:736: done (12 left) 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:737: done (11 left) 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:738: done (10 left) 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:749: done (9 left) 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:750: write oid 265 current snap is 0 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:739: finishing write tid 4 to smithi093153278-104 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 104 v 137 (ObjNum 635 snap 0 seq_num 635) dirty exists 2023-08-27T20:52:24.156 INFO:tasks.rados.rados.0.smithi093.stdout:739: left oid 104 (ObjNum 635 snap 0 seq_num 635) 2023-08-27T20:52:24.157 INFO:tasks.rados.rados.0.smithi093.stdout:740: finishing write tid 1 to smithi093153278-146 2023-08-27T20:52:24.157 INFO:tasks.rados.rados.0.smithi093.stdout:750: seq_num 640 ranges {756480=733012,2134711=432549} 2023-08-27T20:52:24.165 INFO:tasks.rados.rados.0.smithi093.stdout:750: writing smithi093153278-265 from 756480 to 1489492 tid 1 2023-08-27T20:52:24.172 INFO:tasks.rados.rados.0.smithi093.stdout:750: writing smithi093153278-265 from 2134711 to 2567260 tid 2 2023-08-27T20:52:24.172 INFO:tasks.rados.rados.0.smithi093.stdout:739: done (9 left) 2023-08-27T20:52:24.172 INFO:tasks.rados.rados.0.smithi093.stdout:751: write oid 500 current snap is 0 2023-08-27T20:52:24.172 INFO:tasks.rados.rados.0.smithi093.stdout:751: seq_num 641 ranges {773287=408261,1793193=576207} 2023-08-27T20:52:24.179 INFO:tasks.rados.rados.0.smithi093.stdout:751: writing smithi093153278-500 from 773287 to 1181548 tid 1 2023-08-27T20:52:24.187 INFO:tasks.rados.rados.0.smithi093.stdout:751: writing smithi093153278-500 from 1793193 to 2369400 tid 2 2023-08-27T20:52:24.187 INFO:tasks.rados.rados.0.smithi093.stdout:752: write oid 483 current snap is 0 2023-08-27T20:52:24.187 INFO:tasks.rados.rados.0.smithi093.stdout:752: seq_num 642 ranges {790094=483510,1851675=436218} 2023-08-27T20:52:24.195 INFO:tasks.rados.rados.0.smithi093.stdout:752: writing smithi093153278-483 from 790094 to 1273604 tid 1 2023-08-27T20:52:24.201 INFO:tasks.rados.rados.0.smithi093.stdout:752: writing smithi093153278-483 from 1851675 to 2287893 tid 2 2023-08-27T20:52:24.201 INFO:tasks.rados.rados.0.smithi093.stdout:753: delete oid 318 current snap is 0 2023-08-27T20:52:24.202 INFO:tasks.rados.rados.0.smithi093.stdout:740: finishing write tid 2 to smithi093153278-146 2023-08-27T20:52:24.202 INFO:tasks.rados.rados.0.smithi093.stdout:740: finishing write tid 3 to smithi093153278-146 2023-08-27T20:52:24.202 INFO:tasks.rados.rados.0.smithi093.stdout:740: finishing write tid 4 to smithi093153278-146 2023-08-27T20:52:24.202 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 146 v 132 (ObjNum 636 snap 0 seq_num 636) dirty exists 2023-08-27T20:52:24.202 INFO:tasks.rados.rados.0.smithi093.stdout:740: left oid 146 (ObjNum 636 snap 0 seq_num 636) 2023-08-27T20:52:24.202 INFO:tasks.rados.rados.0.smithi093.stdout:741: expect (ObjNum 421 snap 0 seq_num 421) 2023-08-27T20:52:24.310 INFO:tasks.rados.rados.0.smithi093.stdout:744: expect (ObjNum 3 snap 0 seq_num 3) 2023-08-27T20:52:24.378 INFO:tasks.rados.rados.0.smithi093.stdout:745: expect (ObjNum 187 snap 0 seq_num 187) 2023-08-27T20:52:24.485 INFO:tasks.rados.rados.0.smithi093.stdout:743: expect (ObjNum 150 snap 0 seq_num 150) 2023-08-27T20:52:24.567 INFO:tasks.rados.rados.0.smithi093.stdout:746: finishing write tid 1 to smithi093153278-326 2023-08-27T20:52:24.617 INFO:tasks.rados.rados.0.smithi093.stdout:746: finishing write tid 2 to smithi093153278-326 2023-08-27T20:52:24.617 INFO:tasks.rados.rados.0.smithi093.stdout:748: expect (ObjNum 178 snap 0 seq_num 178) 2023-08-27T20:52:24.672 INFO:tasks.rados.rados.0.smithi093.stdout:746: finishing write tid 3 to smithi093153278-326 2023-08-27T20:52:24.672 INFO:tasks.rados.rados.0.smithi093.stdout:746: finishing write tid 4 to smithi093153278-326 2023-08-27T20:52:24.672 INFO:tasks.rados.rados.0.smithi093.stdout:746: finishing write tid 5 to smithi093153278-326 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 128 (ObjNum 638 snap 0 seq_num 638) dirty exists 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:746: left oid 326 (ObjNum 638 snap 0 seq_num 638) 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:750: finishing write tid 1 to smithi093153278-265 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:750: finishing write tid 2 to smithi093153278-265 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:750: finishing write tid 3 to smithi093153278-265 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:750: finishing write tid 4 to smithi093153278-265 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 265 v 157 (ObjNum 640 snap 0 seq_num 640) dirty exists 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:750: left oid 265 (ObjNum 640 snap 0 seq_num 640) 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:751: finishing write tid 1 to smithi093153278-500 2023-08-27T20:52:24.673 INFO:tasks.rados.rados.0.smithi093.stdout:751: finishing write tid 2 to smithi093153278-500 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:751: finishing write tid 3 to smithi093153278-500 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:751: finishing write tid 4 to smithi093153278-500 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 500 v 151 (ObjNum 641 snap 0 seq_num 641) dirty exists 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:751: left oid 500 (ObjNum 641 snap 0 seq_num 641) 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:752: finishing write tid 1 to smithi093153278-483 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:752: finishing write tid 2 to smithi093153278-483 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:752: finishing write tid 3 to smithi093153278-483 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:752: finishing write tid 4 to smithi093153278-483 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 483 v 116 (ObjNum 642 snap 0 seq_num 642) dirty exists 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:752: left oid 483 (ObjNum 642 snap 0 seq_num 642) 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:740: done (11 left) 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:741: done (10 left) 2023-08-27T20:52:24.674 INFO:tasks.rados.rados.0.smithi093.stdout:743: done (9 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:744: done (8 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:745: done (7 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:746: done (6 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:747: done (5 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:748: done (4 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:750: done (3 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:751: done (2 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:752: done (1 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:753: done (0 left) 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:754: write oid 114 current snap is 0 2023-08-27T20:52:24.675 INFO:tasks.rados.rados.0.smithi093.stdout:754: seq_num 644 ranges {423708=634008,1852286=439887,2752172=71536} 2023-08-27T20:52:24.677 INFO:tasks.rados.rados.0.smithi093.stdout:754: writing smithi093153278-114 from 423708 to 1057716 tid 1 2023-08-27T20:52:24.684 INFO:tasks.rados.rados.0.smithi093.stdout:754: writing smithi093153278-114 from 1852286 to 2292173 tid 2 2023-08-27T20:52:24.688 INFO:tasks.rados.rados.0.smithi093.stdout:754: writing smithi093153278-114 from 2752172 to 2823708 tid 3 2023-08-27T20:52:24.688 INFO:tasks.rados.rados.0.smithi093.stdout:755: read oid 346 snap -1 2023-08-27T20:52:24.688 INFO:tasks.rados.rados.0.smithi093.stdout:755: expect (ObjNum 345 snap 0 seq_num 345) 2023-08-27T20:52:24.688 INFO:tasks.rados.rados.0.smithi093.stdout:756: write oid 164 current snap is 0 2023-08-27T20:52:24.688 INFO:tasks.rados.rados.0.smithi093.stdout:754: finishing write tid 1 to smithi093153278-114 2023-08-27T20:52:24.688 INFO:tasks.rados.rados.0.smithi093.stdout:756: seq_num 645 ranges {440515=709257,1910768=583545} 2023-08-27T20:52:24.694 INFO:tasks.rados.rados.0.smithi093.stdout:756: writing smithi093153278-164 from 440515 to 1149772 tid 1 2023-08-27T20:52:24.702 INFO:tasks.rados.rados.0.smithi093.stdout:756: writing smithi093153278-164 from 1910768 to 2494313 tid 2 2023-08-27T20:52:24.702 INFO:tasks.rados.rados.0.smithi093.stdout:757: write oid 324 current snap is 0 2023-08-27T20:52:24.702 INFO:tasks.rados.rados.0.smithi093.stdout:754: finishing write tid 2 to smithi093153278-114 2023-08-27T20:52:24.702 INFO:tasks.rados.rados.0.smithi093.stdout:757: seq_num 646 ranges {457322=784506,1969250=443556} 2023-08-27T20:52:24.709 INFO:tasks.rados.rados.0.smithi093.stdout:757: writing smithi093153278-324 from 457322 to 1241828 tid 1 2023-08-27T20:52:24.715 INFO:tasks.rados.rados.0.smithi093.stdout:757: writing smithi093153278-324 from 1969250 to 2412806 tid 2 2023-08-27T20:52:24.716 INFO:tasks.rados.rados.0.smithi093.stdout:758: write oid 348 current snap is 0 2023-08-27T20:52:24.716 INFO:tasks.rados.rados.0.smithi093.stdout:754: finishing write tid 3 to smithi093153278-114 2023-08-27T20:52:24.716 INFO:tasks.rados.rados.0.smithi093.stdout:758: seq_num 647 ranges {474129=576108,1627732=587214,2718088=156041} 2023-08-27T20:52:24.721 INFO:tasks.rados.rados.0.smithi093.stdout:758: writing smithi093153278-348 from 474129 to 1050237 tid 1 2023-08-27T20:52:24.728 INFO:tasks.rados.rados.0.smithi093.stdout:758: writing smithi093153278-348 from 1627732 to 2214946 tid 2 2023-08-27T20:52:24.732 INFO:tasks.rados.rados.0.smithi093.stdout:758: writing smithi093153278-348 from 2718088 to 2874129 tid 3 2023-08-27T20:52:24.732 INFO:tasks.rados.rados.0.smithi093.stdout:759: read oid 52 snap -1 2023-08-27T20:52:24.732 INFO:tasks.rados.rados.0.smithi093.stdout:759: expect (ObjNum 51 snap 0 seq_num 51) 2023-08-27T20:52:24.732 INFO:tasks.rados.rados.0.smithi093.stdout:760: write oid 5 current snap is 0 2023-08-27T20:52:24.732 INFO:tasks.rados.rados.0.smithi093.stdout:754: finishing write tid 4 to smithi093153278-114 2023-08-27T20:52:24.733 INFO:tasks.rados.rados.0.smithi093.stdout:760: seq_num 648 ranges {490936=651357,1686214=447225,2861864=29072} 2023-08-27T20:52:24.738 INFO:tasks.rados.rados.0.smithi093.stdout:760: writing smithi093153278-5 from 490936 to 1142293 tid 1 2023-08-27T20:52:24.744 INFO:tasks.rados.rados.0.smithi093.stdout:760: writing smithi093153278-5 from 1686214 to 2133439 tid 2 2023-08-27T20:52:24.748 INFO:tasks.rados.rados.0.smithi093.stdout:760: writing smithi093153278-5 from 2861864 to 2890936 tid 3 2023-08-27T20:52:24.749 INFO:tasks.rados.rados.0.smithi093.stdout:761: read oid 310 snap -1 2023-08-27T20:52:24.749 INFO:tasks.rados.rados.0.smithi093.stdout:761: expect (ObjNum 309 snap 0 seq_num 309) 2023-08-27T20:52:24.749 INFO:tasks.rados.rados.0.smithi093.stdout:762: write oid 82 current snap is 0 2023-08-27T20:52:24.749 INFO:tasks.rados.rados.0.smithi093.stdout:754: finishing write tid 5 to smithi093153278-114 2023-08-27T20:52:24.749 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 114 v 132 (ObjNum 644 snap 0 seq_num 644) dirty exists 2023-08-27T20:52:24.749 INFO:tasks.rados.rados.0.smithi093.stdout:754: left oid 114 (ObjNum 644 snap 0 seq_num 644) 2023-08-27T20:52:24.749 INFO:tasks.rados.rados.0.smithi093.stdout:762: seq_num 649 ranges {507743=726606,1744696=590883,2772934=134809} 2023-08-27T20:52:24.755 INFO:tasks.rados.rados.0.smithi093.stdout:762: writing smithi093153278-82 from 507743 to 1234349 tid 1 2023-08-27T20:52:24.762 INFO:tasks.rados.rados.0.smithi093.stdout:762: writing smithi093153278-82 from 1744696 to 2335579 tid 2 2023-08-27T20:52:24.765 INFO:tasks.rados.rados.0.smithi093.stdout:762: writing smithi093153278-82 from 2772934 to 2907743 tid 3 2023-08-27T20:52:24.765 INFO:tasks.rados.rados.0.smithi093.stdout:754: done (8 left) 2023-08-27T20:52:24.765 INFO:tasks.rados.rados.0.smithi093.stdout:763: delete oid 75 current snap is 0 2023-08-27T20:52:24.766 INFO:tasks.rados.rados.0.smithi093.stdout:755: expect (ObjNum 345 snap 0 seq_num 345) 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:756: finishing write tid 1 to smithi093153278-164 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:755: done (8 left) 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:763: done (7 left) 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:764: read oid 244 snap -1 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:764: expect (ObjNum 243 snap 0 seq_num 243) 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:765: write oid 167 current snap is 0 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:756: finishing write tid 2 to smithi093153278-164 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:756: finishing write tid 3 to smithi093153278-164 2023-08-27T20:52:24.893 INFO:tasks.rados.rados.0.smithi093.stdout:765: seq_num 651 ranges {541357=477104,1745307=594552} 2023-08-27T20:52:24.898 INFO:tasks.rados.rados.0.smithi093.stdout:765: writing smithi093153278-167 from 541357 to 1018461 tid 1 2023-08-27T20:52:24.906 INFO:tasks.rados.rados.0.smithi093.stdout:765: writing smithi093153278-167 from 1745307 to 2339859 tid 2 2023-08-27T20:52:24.906 INFO:tasks.rados.rados.0.smithi093.stdout:766: write oid 269 current snap is 0 2023-08-27T20:52:24.906 INFO:tasks.rados.rados.0.smithi093.stdout:756: finishing write tid 4 to smithi093153278-164 2023-08-27T20:52:24.906 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 164 v 130 (ObjNum 645 snap 0 seq_num 645) dirty exists 2023-08-27T20:52:24.906 INFO:tasks.rados.rados.0.smithi093.stdout:756: left oid 164 (ObjNum 645 snap 0 seq_num 645) 2023-08-27T20:52:24.906 INFO:tasks.rados.rados.0.smithi093.stdout:757: finishing write tid 1 to smithi093153278-324 2023-08-27T20:52:24.907 INFO:tasks.rados.rados.0.smithi093.stdout:757: finishing write tid 2 to smithi093153278-324 2023-08-27T20:52:24.907 INFO:tasks.rados.rados.0.smithi093.stdout:757: finishing write tid 3 to smithi093153278-324 2023-08-27T20:52:24.907 INFO:tasks.rados.rados.0.smithi093.stdout:766: seq_num 652 ranges {558164=552353,1803789=454563,2855203=102961} 2023-08-27T20:52:24.912 INFO:tasks.rados.rados.0.smithi093.stdout:766: writing smithi093153278-269 from 558164 to 1110517 tid 1 2023-08-27T20:52:24.918 INFO:tasks.rados.rados.0.smithi093.stdout:766: writing smithi093153278-269 from 1803789 to 2258352 tid 2 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:766: writing smithi093153278-269 from 2855203 to 2958164 tid 3 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:756: done (9 left) 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:767: read oid 461 snap -1 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:767: expect (ObjNum 460 snap 0 seq_num 460) 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:768: read oid 488 snap -1 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:768: expect (ObjNum 487 snap 0 seq_num 487) 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:769: write oid 35 current snap is 0 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:757: finishing write tid 4 to smithi093153278-324 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 324 v 124 (ObjNum 646 snap 0 seq_num 646) dirty exists 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:757: left oid 324 (ObjNum 646 snap 0 seq_num 646) 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:758: finishing write tid 1 to smithi093153278-348 2023-08-27T20:52:24.923 INFO:tasks.rados.rados.0.smithi093.stdout:758: finishing write tid 2 to smithi093153278-348 2023-08-27T20:52:24.924 INFO:tasks.rados.rados.0.smithi093.stdout:758: finishing write tid 3 to smithi093153278-348 2023-08-27T20:52:24.924 INFO:tasks.rados.rados.0.smithi093.stdout:758: finishing write tid 4 to smithi093153278-348 2023-08-27T20:52:24.924 INFO:tasks.rados.rados.0.smithi093.stdout:769: seq_num 653 ranges {574971=627602,1862271=598221} 2023-08-27T20:52:24.930 INFO:tasks.rados.rados.0.smithi093.stdout:769: writing smithi093153278-35 from 574971 to 1202573 tid 1 2023-08-27T20:52:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2886450237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:24.939 INFO:tasks.rados.rados.0.smithi093.stdout:769: writing smithi093153278-35 from 1862271 to 2460492 tid 2 2023-08-27T20:52:24.939 INFO:tasks.rados.rados.0.smithi093.stdout:757: done (11 left) 2023-08-27T20:52:24.939 INFO:tasks.rados.rados.0.smithi093.stdout:770: write oid 276 current snap is 0 2023-08-27T20:52:24.939 INFO:tasks.rados.rados.0.smithi093.stdout:758: finishing write tid 5 to smithi093153278-348 2023-08-27T20:52:24.939 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 348 v 176 (ObjNum 647 snap 0 seq_num 647) dirty exists 2023-08-27T20:52:24.939 INFO:tasks.rados.rados.0.smithi093.stdout:758: left oid 348 (ObjNum 647 snap 0 seq_num 647) 2023-08-27T20:52:24.940 INFO:tasks.rados.rados.0.smithi093.stdout:759: expect (ObjNum 51 snap 0 seq_num 51) 2023-08-27T20:52:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2886450237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2886450237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:25.043 INFO:tasks.rados.rados.0.smithi093.stdout:760: finishing write tid 1 to smithi093153278-5 2023-08-27T20:52:25.043 INFO:tasks.rados.rados.0.smithi093.stdout:760: finishing write tid 2 to smithi093153278-5 2023-08-27T20:52:25.043 INFO:tasks.rados.rados.0.smithi093.stdout:770: seq_num 654 ranges {591778=419204,1637106=741879,2910049=81729} 2023-08-27T20:52:25.048 INFO:tasks.rados.rados.0.smithi093.stdout:770: writing smithi093153278-276 from 591778 to 1010982 tid 1 2023-08-27T20:52:25.058 INFO:tasks.rados.rados.0.smithi093.stdout:770: writing smithi093153278-276 from 1637106 to 2378985 tid 2 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:770: writing smithi093153278-276 from 2910049 to 2991778 tid 3 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:758: done (11 left) 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:759: done (10 left) 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:771: write oid 132 current snap is 0 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:760: finishing write tid 3 to smithi093153278-5 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:760: finishing write tid 4 to smithi093153278-5 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:760: finishing write tid 5 to smithi093153278-5 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 5 v 137 (ObjNum 648 snap 0 seq_num 648) dirty exists 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:760: left oid 5 (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:52:25.062 INFO:tasks.rados.rados.0.smithi093.stdout:761: expect (ObjNum 309 snap 0 seq_num 309) 2023-08-27T20:52:25.167 INFO:tasks.rados.rados.0.smithi093.stdout:762: finishing write tid 1 to smithi093153278-82 2023-08-27T20:52:25.167 INFO:tasks.rados.rados.0.smithi093.stdout:762: finishing write tid 2 to smithi093153278-82 2023-08-27T20:52:25.167 INFO:tasks.rados.rados.0.smithi093.stdout:771: seq_num 655 ranges {608585=494453,1579235=601890,2821119=187466} 2023-08-27T20:52:25.174 INFO:tasks.rados.rados.0.smithi093.stdout:771: writing smithi093153278-132 from 608585 to 1103038 tid 1 2023-08-27T20:52:25.181 INFO:tasks.rados.rados.0.smithi093.stdout:771: writing smithi093153278-132 from 1579235 to 2181125 tid 2 2023-08-27T20:52:25.186 INFO:tasks.rados.rados.0.smithi093.stdout:771: writing smithi093153278-132 from 2821119 to 3008585 tid 3 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:760: done (10 left) 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:761: done (9 left) 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:772: read oid 485 snap -1 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:772: expect (ObjNum 484 snap 0 seq_num 484) 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:773: write oid 255 current snap is 0 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:762: finishing write tid 3 to smithi093153278-82 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:762: finishing write tid 4 to smithi093153278-82 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:762: finishing write tid 5 to smithi093153278-82 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 82 v 128 (ObjNum 649 snap 0 seq_num 649) dirty exists 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:762: left oid 82 (ObjNum 649 snap 0 seq_num 649) 2023-08-27T20:52:25.187 INFO:tasks.rados.rados.0.smithi093.stdout:773: seq_num 656 ranges {625392=569702,1637717=745548,2848542=176850} 2023-08-27T20:52:25.194 INFO:tasks.rados.rados.0.smithi093.stdout:773: writing smithi093153278-255 from 625392 to 1195094 tid 1 2023-08-27T20:52:25.202 INFO:tasks.rados.rados.0.smithi093.stdout:773: writing smithi093153278-255 from 1637717 to 2383265 tid 2 2023-08-27T20:52:25.206 INFO:tasks.rados.rados.0.smithi093.stdout:773: writing smithi093153278-255 from 2848542 to 3025392 tid 3 2023-08-27T20:52:25.206 INFO:tasks.rados.rados.0.smithi093.stdout:762: done (10 left) 2023-08-27T20:52:25.206 INFO:tasks.rados.rados.0.smithi093.stdout:774: read oid 339 snap -1 2023-08-27T20:52:25.206 INFO:tasks.rados.rados.0.smithi093.stdout:774: expect (ObjNum 541 snap 0 seq_num 541) 2023-08-27T20:52:25.206 INFO:tasks.rados.rados.0.smithi093.stdout:775: read oid 325 snap -1 2023-08-27T20:52:25.206 INFO:tasks.rados.rados.0.smithi093.stdout:775: expect (ObjNum 324 snap 0 seq_num 324) 2023-08-27T20:52:25.207 INFO:tasks.rados.rados.0.smithi093.stdout:776: read oid 76 snap -1 2023-08-27T20:52:25.207 INFO:tasks.rados.rados.0.smithi093.stdout:776: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:52:25.207 INFO:tasks.rados.rados.0.smithi093.stdout:777: write oid 455 current snap is 0 2023-08-27T20:52:25.207 INFO:tasks.rados.rados.0.smithi093.stdout:777: seq_num 657 ranges {642199=644951,1696199=605559,2992318=49881} 2023-08-27T20:52:25.213 INFO:tasks.rados.rados.0.smithi093.stdout:777: writing smithi093153278-455 from 642199 to 1287150 tid 1 2023-08-27T20:52:25.219 INFO:tasks.rados.rados.0.smithi093.stdout:777: writing smithi093153278-455 from 1696199 to 2301758 tid 2 2023-08-27T20:52:25.223 INFO:tasks.rados.rados.0.smithi093.stdout:777: writing smithi093153278-455 from 2992318 to 3042199 tid 3 2023-08-27T20:52:25.223 INFO:tasks.rados.rados.0.smithi093.stdout:778: write oid 318 current snap is 0 2023-08-27T20:52:25.224 INFO:tasks.rados.rados.0.smithi093.stdout:778: seq_num 658 ranges {659006=720200,2154681=749217} 2023-08-27T20:52:25.231 INFO:tasks.rados.rados.0.smithi093.stdout:778: writing smithi093153278-318 from 659006 to 1379206 tid 1 2023-08-27T20:52:25.240 INFO:tasks.rados.rados.0.smithi093.stdout:778: writing smithi093153278-318 from 2154681 to 2903898 tid 2 2023-08-27T20:52:25.240 INFO:tasks.rados.rados.0.smithi093.stdout:779: read oid 64 snap -1 2023-08-27T20:52:25.240 INFO:tasks.rados.rados.0.smithi093.stdout:779: expect (ObjNum 63 snap 0 seq_num 63) 2023-08-27T20:52:25.240 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:25.240 INFO:tasks.rados.rados.0.smithi093.stdout:764: expect (ObjNum 243 snap 0 seq_num 243) 2023-08-27T20:52:25.322 INFO:tasks.rados.rados.0.smithi093.stdout:765: finishing write tid 1 to smithi093153278-167 2023-08-27T20:52:25.322 INFO:tasks.rados.rados.0.smithi093.stdout:764: done (15 left) 2023-08-27T20:52:25.323 INFO:tasks.rados.rados.0.smithi093.stdout:780: write oid 403 current snap is 0 2023-08-27T20:52:25.323 INFO:tasks.rados.rados.0.smithi093.stdout:765: finishing write tid 2 to smithi093153278-167 2023-08-27T20:52:25.323 INFO:tasks.rados.rados.0.smithi093.stdout:765: finishing write tid 3 to smithi093153278-167 2023-08-27T20:52:25.323 INFO:tasks.rados.rados.0.smithi093.stdout:780: seq_num 659 ranges {675813=795449,2096810=609228} 2023-08-27T20:52:25.329 INFO:tasks.rados.rados.0.smithi093.stdout:780: writing smithi093153278-403 from 675813 to 1471262 tid 1 2023-08-27T20:52:25.337 INFO:tasks.rados.rados.0.smithi093.stdout:780: writing smithi093153278-403 from 2096810 to 2706038 tid 2 2023-08-27T20:52:25.337 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:25.338 INFO:tasks.rados.rados.0.smithi093.stdout:765: finishing write tid 4 to smithi093153278-167 2023-08-27T20:52:25.338 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 167 v 154 (ObjNum 651 snap 0 seq_num 651) dirty exists 2023-08-27T20:52:25.338 INFO:tasks.rados.rados.0.smithi093.stdout:765: left oid 167 (ObjNum 651 snap 0 seq_num 651) 2023-08-27T20:52:25.338 INFO:tasks.rados.rados.0.smithi093.stdout:766: finishing write tid 1 to smithi093153278-269 2023-08-27T20:52:25.338 INFO:tasks.rados.rados.0.smithi093.stdout:766: finishing write tid 2 to smithi093153278-269 2023-08-27T20:52:25.339 INFO:tasks.rados.rados.0.smithi093.stdout:765: done (15 left) 2023-08-27T20:52:25.339 INFO:tasks.rados.rados.0.smithi093.stdout:781: write oid 202 current snap is 0 2023-08-27T20:52:25.339 INFO:tasks.rados.rados.0.smithi093.stdout:768: expect (ObjNum 487 snap 0 seq_num 487) 2023-08-27T20:52:25.407 INFO:tasks.rados.rados.0.smithi093.stdout:766: finishing write tid 3 to smithi093153278-269 2023-08-27T20:52:25.407 INFO:tasks.rados.rados.0.smithi093.stdout:766: finishing write tid 4 to smithi093153278-269 2023-08-27T20:52:25.407 INFO:tasks.rados.rados.0.smithi093.stdout:766: finishing write tid 5 to smithi093153278-269 2023-08-27T20:52:25.407 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 269 v 120 (ObjNum 652 snap 0 seq_num 652) dirty exists 2023-08-27T20:52:25.407 INFO:tasks.rados.rados.0.smithi093.stdout:766: left oid 269 (ObjNum 652 snap 0 seq_num 652) 2023-08-27T20:52:25.407 INFO:tasks.rados.rados.0.smithi093.stdout:767: expect (ObjNum 460 snap 0 seq_num 460) 2023-08-27T20:52:25.541 INFO:tasks.rados.rados.0.smithi093.stdout:769: finishing write tid 1 to smithi093153278-35 2023-08-27T20:52:25.541 INFO:tasks.rados.rados.0.smithi093.stdout:769: finishing write tid 2 to smithi093153278-35 2023-08-27T20:52:25.541 INFO:tasks.rados.rados.0.smithi093.stdout:769: finishing write tid 3 to smithi093153278-35 2023-08-27T20:52:25.541 INFO:tasks.rados.rados.0.smithi093.stdout:769: finishing write tid 4 to smithi093153278-35 2023-08-27T20:52:25.541 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 35 v 140 (ObjNum 653 snap 0 seq_num 653) dirty exists 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:769: left oid 35 (ObjNum 653 snap 0 seq_num 653) 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:770: finishing write tid 1 to smithi093153278-276 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:770: finishing write tid 2 to smithi093153278-276 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:770: finishing write tid 3 to smithi093153278-276 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:770: finishing write tid 4 to smithi093153278-276 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:770: finishing write tid 5 to smithi093153278-276 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 276 v 141 (ObjNum 654 snap 0 seq_num 654) dirty exists 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:770: left oid 276 (ObjNum 654 snap 0 seq_num 654) 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:771: finishing write tid 1 to smithi093153278-132 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:771: finishing write tid 2 to smithi093153278-132 2023-08-27T20:52:25.542 INFO:tasks.rados.rados.0.smithi093.stdout:781: seq_num 660 ranges {692620=470698,1755292=752886} 2023-08-27T20:52:25.548 INFO:tasks.rados.rados.0.smithi093.stdout:781: writing smithi093153278-202 from 692620 to 1163318 tid 1 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:781: writing smithi093153278-202 from 1755292 to 2508178 tid 2 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:766: done (15 left) 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:767: done (14 left) 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:768: done (13 left) 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:769: done (12 left) 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:770: done (11 left) 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:782: read oid 442 snap -1 2023-08-27T20:52:25.558 INFO:tasks.rados.rados.0.smithi093.stdout:782: expect (ObjNum 441 snap 0 seq_num 441) 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:783: read oid 423 snap -1 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:783: expect (ObjNum 422 snap 0 seq_num 422) 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:784: delete oid 276 current snap is 0 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:771: finishing write tid 3 to smithi093153278-132 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:771: finishing write tid 4 to smithi093153278-132 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:771: finishing write tid 5 to smithi093153278-132 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 132 v 133 (ObjNum 655 snap 0 seq_num 655) dirty exists 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:771: left oid 132 (ObjNum 655 snap 0 seq_num 655) 2023-08-27T20:52:25.559 INFO:tasks.rados.rados.0.smithi093.stdout:772: expect (ObjNum 484 snap 0 seq_num 484) 2023-08-27T20:52:25.635 INFO:tasks.rados.rados.0.smithi093.stdout:773: finishing write tid 1 to smithi093153278-255 2023-08-27T20:52:25.635 INFO:tasks.rados.rados.0.smithi093.stdout:773: finishing write tid 2 to smithi093153278-255 2023-08-27T20:52:25.635 INFO:tasks.rados.rados.0.smithi093.stdout:773: finishing write tid 3 to smithi093153278-255 2023-08-27T20:52:25.635 INFO:tasks.rados.rados.0.smithi093.stdout:773: finishing write tid 4 to smithi093153278-255 2023-08-27T20:52:25.635 INFO:tasks.rados.rados.0.smithi093.stdout:773: finishing write tid 5 to smithi093153278-255 2023-08-27T20:52:25.635 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 255 v 181 (ObjNum 656 snap 0 seq_num 656) dirty exists 2023-08-27T20:52:25.635 INFO:tasks.rados.rados.0.smithi093.stdout:773: left oid 255 (ObjNum 656 snap 0 seq_num 656) 2023-08-27T20:52:25.636 INFO:tasks.rados.rados.0.smithi093.stdout:775: expect (ObjNum 324 snap 0 seq_num 324) 2023-08-27T20:52:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:25 smithi093 ceph-mon[111335]: pgmap v67: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s rd, 9.3 MiB/s wr, 99 op/s 2023-08-27T20:52:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:25 smithi093 ceph-mon[118902]: pgmap v67: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s rd, 9.3 MiB/s wr, 99 op/s 2023-08-27T20:52:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:52:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:25] "GET /metrics HTTP/1.1" 200 241913 "" "Prometheus/2.43.0" 2023-08-27T20:52:25.752 INFO:tasks.rados.rados.0.smithi093.stdout:771: done (13 left) 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:772: done (12 left) 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:773: done (11 left) 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:775: done (10 left) 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:784: done (9 left) 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:785: read oid 441 snap -1 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:785: expect (ObjNum 591 snap 0 seq_num 591) 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:786: write oid 210 current snap is 0 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:777: finishing write tid 1 to smithi093153278-455 2023-08-27T20:52:25.753 INFO:tasks.rados.rados.0.smithi093.stdout:774: expect (ObjNum 541 snap 0 seq_num 541) 2023-08-27T20:52:25.927 INFO:tasks.rados.rados.0.smithi093.stdout:777: finishing write tid 2 to smithi093153278-455 2023-08-27T20:52:25.927 INFO:tasks.rados.rados.0.smithi093.stdout:777: finishing write tid 3 to smithi093153278-455 2023-08-27T20:52:25.927 INFO:tasks.rados.rados.0.smithi093.stdout:786: seq_num 662 ranges {726234=737549,1988609=756555} 2023-08-27T20:52:25.935 INFO:tasks.rados.rados.0.smithi093.stdout:786: writing smithi093153278-210 from 726234 to 1463783 tid 1 2023-08-27T20:52:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:25 smithi136 ceph-mon[113531]: pgmap v67: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 71 MiB/s rd, 9.3 MiB/s wr, 99 op/s 2023-08-27T20:52:25.942 INFO:tasks.rados.rados.0.smithi093.stdout:786: writing smithi093153278-210 from 1988609 to 2745164 tid 2 2023-08-27T20:52:25.942 INFO:tasks.rados.rados.0.smithi093.stdout:774: done (10 left) 2023-08-27T20:52:25.942 INFO:tasks.rados.rados.0.smithi093.stdout:787: read oid 28 snap -1 2023-08-27T20:52:25.942 INFO:tasks.rados.rados.0.smithi093.stdout:787: expect (ObjNum 27 snap 0 seq_num 27) 2023-08-27T20:52:25.945 INFO:tasks.rados.rados.0.smithi093.stdout:788: write oid 221 current snap is 0 2023-08-27T20:52:25.945 INFO:tasks.rados.rados.0.smithi093.stdout:788: seq_num 663 ranges {743041=412798,1930738=616566,3040503=102538} 2023-08-27T20:52:25.949 INFO:tasks.rados.rados.0.smithi093.stdout:788: writing smithi093153278-221 from 743041 to 1155839 tid 1 2023-08-27T20:52:25.957 INFO:tasks.rados.rados.0.smithi093.stdout:788: writing smithi093153278-221 from 1930738 to 2547304 tid 2 2023-08-27T20:52:25.962 INFO:tasks.rados.rados.0.smithi093.stdout:788: writing smithi093153278-221 from 3040503 to 3143041 tid 3 2023-08-27T20:52:25.962 INFO:tasks.rados.rados.0.smithi093.stdout:789: write oid 434 current snap is 0 2023-08-27T20:52:25.962 INFO:tasks.rados.rados.0.smithi093.stdout:777: finishing write tid 4 to smithi093153278-455 2023-08-27T20:52:25.962 INFO:tasks.rados.rados.0.smithi093.stdout:777: finishing write tid 5 to smithi093153278-455 2023-08-27T20:52:25.963 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 455 v 158 (ObjNum 657 snap 0 seq_num 657) dirty exists 2023-08-27T20:52:25.963 INFO:tasks.rados.rados.0.smithi093.stdout:777: left oid 455 (ObjNum 657 snap 0 seq_num 657) 2023-08-27T20:52:25.963 INFO:tasks.rados.rados.0.smithi093.stdout:789: seq_num 664 ranges {759848=488047,1989220=760224} 2023-08-27T20:52:25.967 INFO:tasks.rados.rados.0.smithi093.stdout:789: writing smithi093153278-434 from 759848 to 1247895 tid 1 2023-08-27T20:52:25.976 INFO:tasks.rados.rados.0.smithi093.stdout:789: writing smithi093153278-434 from 1989220 to 2749444 tid 2 2023-08-27T20:52:25.976 INFO:tasks.rados.rados.0.smithi093.stdout:777: done (12 left) 2023-08-27T20:52:25.977 INFO:tasks.rados.rados.0.smithi093.stdout:790: read oid 452 snap -1 2023-08-27T20:52:25.977 INFO:tasks.rados.rados.0.smithi093.stdout:790: expect (ObjNum 451 snap 0 seq_num 451) 2023-08-27T20:52:25.977 INFO:tasks.rados.rados.0.smithi093.stdout:791: write oid 404 current snap is 0 2023-08-27T20:52:25.977 INFO:tasks.rados.rados.0.smithi093.stdout:778: finishing write tid 1 to smithi093153278-318 2023-08-27T20:52:25.977 INFO:tasks.rados.rados.0.smithi093.stdout:776: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:52:26.087 INFO:tasks.rados.rados.0.smithi093.stdout:778: finishing write tid 2 to smithi093153278-318 2023-08-27T20:52:26.090 INFO:tasks.rados.rados.0.smithi093.stdout:778: finishing write tid 3 to smithi093153278-318 2023-08-27T20:52:26.090 INFO:tasks.rados.rados.0.smithi093.stdout:778: finishing write tid 4 to smithi093153278-318 2023-08-27T20:52:26.090 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 318 v 136 (ObjNum 658 snap 0 seq_num 658) dirty exists 2023-08-27T20:52:26.090 INFO:tasks.rados.rados.0.smithi093.stdout:778: left oid 318 (ObjNum 658 snap 0 seq_num 658) 2023-08-27T20:52:26.090 INFO:tasks.rados.rados.0.smithi093.stdout:791: seq_num 665 ranges {776655=563296,2047702=620235,3095349=81306} 2023-08-27T20:52:26.095 INFO:tasks.rados.rados.0.smithi093.stdout:791: writing smithi093153278-404 from 776655 to 1339951 tid 1 2023-08-27T20:52:26.102 INFO:tasks.rados.rados.0.smithi093.stdout:791: writing smithi093153278-404 from 2047702 to 2667937 tid 2 2023-08-27T20:52:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:791: writing smithi093153278-404 from 3095349 to 3176655 tid 3 2023-08-27T20:52:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:776: done (13 left) 2023-08-27T20:52:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:778: done (12 left) 2023-08-27T20:52:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:792: write oid 296 current snap is 0 2023-08-27T20:52:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:779: expect (ObjNum 63 snap 0 seq_num 63) 2023-08-27T20:52:26.218 INFO:tasks.rados.rados.0.smithi093.stdout:780: finishing write tid 1 to smithi093153278-403 2023-08-27T20:52:26.218 INFO:tasks.rados.rados.0.smithi093.stdout:780: finishing write tid 2 to smithi093153278-403 2023-08-27T20:52:26.218 INFO:tasks.rados.rados.0.smithi093.stdout:780: finishing write tid 3 to smithi093153278-403 2023-08-27T20:52:26.218 INFO:tasks.rados.rados.0.smithi093.stdout:780: finishing write tid 4 to smithi093153278-403 2023-08-27T20:52:26.218 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 403 v 154 (ObjNum 659 snap 0 seq_num 659) dirty exists 2023-08-27T20:52:26.218 INFO:tasks.rados.rados.0.smithi093.stdout:780: left oid 403 (ObjNum 659 snap 0 seq_num 659) 2023-08-27T20:52:26.219 INFO:tasks.rados.rados.0.smithi093.stdout:792: seq_num 666 ranges {793462=638545,2106184=763893} 2023-08-27T20:52:26.226 INFO:tasks.rados.rados.0.smithi093.stdout:792: writing smithi093153278-296 from 793462 to 1432007 tid 1 2023-08-27T20:52:26.234 INFO:tasks.rados.rados.0.smithi093.stdout:792: writing smithi093153278-296 from 2106184 to 2870077 tid 2 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:779: done (12 left) 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:780: done (11 left) 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:793: read oid 108 snap -1 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:793: expect (ObjNum 107 snap 0 seq_num 107) 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:781: finishing write tid 1 to smithi093153278-202 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:794: read oid 165 snap -1 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:794: expect (ObjNum 507 snap 0 seq_num 507) 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:795: write oid 90 current snap is 0 2023-08-27T20:52:26.235 INFO:tasks.rados.rados.0.smithi093.stdout:782: expect (ObjNum 441 snap 0 seq_num 441) 2023-08-27T20:52:26.348 INFO:tasks.rados.rados.0.smithi093.stdout:781: finishing write tid 2 to smithi093153278-202 2023-08-27T20:52:26.348 INFO:tasks.rados.rados.0.smithi093.stdout:781: finishing write tid 3 to smithi093153278-202 2023-08-27T20:52:26.348 INFO:tasks.rados.rados.0.smithi093.stdout:781: finishing write tid 4 to smithi093153278-202 2023-08-27T20:52:26.349 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 202 v 163 (ObjNum 660 snap 0 seq_num 660) dirty exists 2023-08-27T20:52:26.349 INFO:tasks.rados.rados.0.smithi093.stdout:781: left oid 202 (ObjNum 660 snap 0 seq_num 660) 2023-08-27T20:52:26.349 INFO:tasks.rados.rados.0.smithi093.stdout:795: seq_num 667 ranges {410269=713794,1648313=507551,2917489=292780} 2023-08-27T20:52:26.354 INFO:tasks.rados.rados.0.smithi093.stdout:795: writing smithi093153278-90 from 410269 to 1124063 tid 1 2023-08-27T20:52:26.360 INFO:tasks.rados.rados.0.smithi093.stdout:795: writing smithi093153278-90 from 1648313 to 2155864 tid 2 2023-08-27T20:52:26.366 INFO:tasks.rados.rados.0.smithi093.stdout:795: writing smithi093153278-90 from 2917489 to 3210269 tid 3 2023-08-27T20:52:26.367 INFO:tasks.rados.rados.0.smithi093.stdout:781: done (13 left) 2023-08-27T20:52:26.367 INFO:tasks.rados.rados.0.smithi093.stdout:782: done (12 left) 2023-08-27T20:52:26.367 INFO:tasks.rados.rados.0.smithi093.stdout:796: write oid 77 current snap is 0 2023-08-27T20:52:26.367 INFO:tasks.rados.rados.0.smithi093.stdout:783: expect (ObjNum 422 snap 0 seq_num 422) 2023-08-27T20:52:26.493 INFO:tasks.rados.rados.0.smithi093.stdout:785: expect (ObjNum 591 snap 0 seq_num 591) 2023-08-27T20:52:26.722 INFO:tasks.rados.rados.0.smithi093.stdout:786: finishing write tid 1 to smithi093153278-210 2023-08-27T20:52:26.723 INFO:tasks.rados.rados.0.smithi093.stdout:786: finishing write tid 2 to smithi093153278-210 2023-08-27T20:52:26.723 INFO:tasks.rados.rados.0.smithi093.stdout:786: finishing write tid 3 to smithi093153278-210 2023-08-27T20:52:26.723 INFO:tasks.rados.rados.0.smithi093.stdout:786: finishing write tid 4 to smithi093153278-210 2023-08-27T20:52:26.723 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 210 v 184 (ObjNum 662 snap 0 seq_num 662) dirty exists 2023-08-27T20:52:26.723 INFO:tasks.rados.rados.0.smithi093.stdout:786: left oid 210 (ObjNum 662 snap 0 seq_num 662) 2023-08-27T20:52:26.723 INFO:tasks.rados.rados.0.smithi093.stdout:787: expect (ObjNum 27 snap 0 seq_num 27) 2023-08-27T20:52:26.803 INFO:tasks.rados.rados.0.smithi093.stdout:788: finishing write tid 1 to smithi093153278-221 2023-08-27T20:52:26.803 INFO:tasks.rados.rados.0.smithi093.stdout:788: finishing write tid 2 to smithi093153278-221 2023-08-27T20:52:26.803 INFO:tasks.rados.rados.0.smithi093.stdout:796: seq_num 668 ranges {427076=789043,1706795=767562,2944912=282164} 2023-08-27T20:52:26.809 INFO:tasks.rados.rados.0.smithi093.stdout:796: writing smithi093153278-77 from 427076 to 1216119 tid 1 2023-08-27T20:52:26.817 INFO:tasks.rados.rados.0.smithi093.stdout:796: writing smithi093153278-77 from 1706795 to 2474357 tid 2 2023-08-27T20:52:26.821 INFO:tasks.rados.rados.0.smithi093.stdout:796: writing smithi093153278-77 from 2944912 to 3227076 tid 3 2023-08-27T20:52:26.821 INFO:tasks.rados.rados.0.smithi093.stdout:783: done (12 left) 2023-08-27T20:52:26.821 INFO:tasks.rados.rados.0.smithi093.stdout:785: done (11 left) 2023-08-27T20:52:26.822 INFO:tasks.rados.rados.0.smithi093.stdout:786: done (10 left) 2023-08-27T20:52:26.822 INFO:tasks.rados.rados.0.smithi093.stdout:787: done (9 left) 2023-08-27T20:52:26.822 INFO:tasks.rados.rados.0.smithi093.stdout:797: write oid 349 current snap is 0 2023-08-27T20:52:26.822 INFO:tasks.rados.rados.0.smithi093.stdout:797: seq_num 669 ranges {443883=464292,1365277=511220,2572335=671548} 2023-08-27T20:52:26.826 INFO:tasks.rados.rados.0.smithi093.stdout:797: writing smithi093153278-349 from 443883 to 908175 tid 1 2023-08-27T20:52:26.832 INFO:tasks.rados.rados.0.smithi093.stdout:797: writing smithi093153278-349 from 1365277 to 1876497 tid 2 2023-08-27T20:52:26.840 INFO:tasks.rados.rados.0.smithi093.stdout:797: writing smithi093153278-349 from 2572335 to 3243883 tid 3 2023-08-27T20:52:26.840 INFO:tasks.rados.rados.0.smithi093.stdout:798: read oid 249 snap -1 2023-08-27T20:52:26.840 INFO:tasks.rados.rados.0.smithi093.stdout:798: expect (ObjNum 248 snap 0 seq_num 248) 2023-08-27T20:52:26.840 INFO:tasks.rados.rados.0.smithi093.stdout:799: write oid 81 current snap is 0 2023-08-27T20:52:26.840 INFO:tasks.rados.rados.0.smithi093.stdout:788: finishing write tid 3 to smithi093153278-221 2023-08-27T20:52:26.840 INFO:tasks.rados.rados.0.smithi093.stdout:788: finishing write tid 4 to smithi093153278-221 2023-08-27T20:52:26.840 INFO:tasks.rados.rados.0.smithi093.stdout:788: finishing write tid 5 to smithi093153278-221 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 221 v 188 (ObjNum 663 snap 0 seq_num 663) dirty exists 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:788: left oid 221 (ObjNum 663 snap 0 seq_num 663) 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:789: finishing write tid 1 to smithi093153278-434 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:789: finishing write tid 2 to smithi093153278-434 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:789: finishing write tid 3 to smithi093153278-434 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:789: finishing write tid 4 to smithi093153278-434 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 123 (ObjNum 664 snap 0 seq_num 664) dirty exists 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:789: left oid 434 (ObjNum 664 snap 0 seq_num 664) 2023-08-27T20:52:26.841 INFO:tasks.rados.rados.0.smithi093.stdout:790: expect (ObjNum 451 snap 0 seq_num 451) 2023-08-27T20:52:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:26 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:26.979 INFO:tasks.rados.rados.0.smithi093.stdout:791: finishing write tid 1 to smithi093153278-404 2023-08-27T20:52:26.979 INFO:tasks.rados.rados.0.smithi093.stdout:791: finishing write tid 2 to smithi093153278-404 2023-08-27T20:52:26.979 INFO:tasks.rados.rados.0.smithi093.stdout:799: seq_num 670 ranges {460690=655894,1540112=771231,2716111=539778} 2023-08-27T20:52:26.985 INFO:tasks.rados.rados.0.smithi093.stdout:799: writing smithi093153278-81 from 460690 to 1116584 tid 1 2023-08-27T20:52:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:26 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:26 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:26.993 INFO:tasks.rados.rados.0.smithi093.stdout:799: writing smithi093153278-81 from 1540112 to 2311343 tid 2 2023-08-27T20:52:26.999 INFO:tasks.rados.rados.0.smithi093.stdout:799: writing smithi093153278-81 from 2716111 to 3255889 tid 3 2023-08-27T20:52:26.999 INFO:tasks.rados.rados.0.smithi093.stdout:788: done (11 left) 2023-08-27T20:52:26.999 INFO:tasks.rados.rados.0.smithi093.stdout:789: done (10 left) 2023-08-27T20:52:26.999 INFO:tasks.rados.rados.0.smithi093.stdout:790: done (9 left) 2023-08-27T20:52:26.999 INFO:tasks.rados.rados.0.smithi093.stdout:800: write oid 230 current snap is 0 2023-08-27T20:52:26.999 INFO:tasks.rados.rados.0.smithi093.stdout:791: finishing write tid 3 to smithi093153278-404 2023-08-27T20:52:26.999 INFO:tasks.rados.rados.0.smithi093.stdout:800: seq_num 671 ranges {477497=731143,1998594=514889,3143534=133963} 2023-08-27T20:52:27.005 INFO:tasks.rados.rados.0.smithi093.stdout:800: writing smithi093153278-230 from 477497 to 1208640 tid 1 2023-08-27T20:52:27.012 INFO:tasks.rados.rados.0.smithi093.stdout:800: writing smithi093153278-230 from 1998594 to 2513483 tid 2 2023-08-27T20:52:27.017 INFO:tasks.rados.rados.0.smithi093.stdout:800: writing smithi093153278-230 from 3143534 to 3277497 tid 3 2023-08-27T20:52:27.017 INFO:tasks.rados.rados.0.smithi093.stdout:801: delete oid 383 current snap is 0 2023-08-27T20:52:27.017 INFO:tasks.rados.rados.0.smithi093.stdout:791: finishing write tid 4 to smithi093153278-404 2023-08-27T20:52:27.017 INFO:tasks.rados.rados.0.smithi093.stdout:791: finishing write tid 5 to smithi093153278-404 2023-08-27T20:52:27.017 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 404 v 132 (ObjNum 665 snap 0 seq_num 665) dirty exists 2023-08-27T20:52:27.018 INFO:tasks.rados.rados.0.smithi093.stdout:791: left oid 404 (ObjNum 665 snap 0 seq_num 665) 2023-08-27T20:52:27.018 INFO:tasks.rados.rados.0.smithi093.stdout:792: finishing write tid 1 to smithi093153278-296 2023-08-27T20:52:27.018 INFO:tasks.rados.rados.0.smithi093.stdout:794: expect (ObjNum 507 snap 0 seq_num 507) 2023-08-27T20:52:27.101 INFO:tasks.rados.rados.0.smithi093.stdout:792: finishing write tid 2 to smithi093153278-296 2023-08-27T20:52:27.101 INFO:tasks.rados.rados.0.smithi093.stdout:792: finishing write tid 3 to smithi093153278-296 2023-08-27T20:52:27.101 INFO:tasks.rados.rados.0.smithi093.stdout:792: finishing write tid 4 to smithi093153278-296 2023-08-27T20:52:27.101 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 296 v 143 (ObjNum 666 snap 0 seq_num 666) dirty exists 2023-08-27T20:52:27.101 INFO:tasks.rados.rados.0.smithi093.stdout:792: left oid 296 (ObjNum 666 snap 0 seq_num 666) 2023-08-27T20:52:27.101 INFO:tasks.rados.rados.0.smithi093.stdout:793: expect (ObjNum 107 snap 0 seq_num 107) 2023-08-27T20:52:27.232 INFO:tasks.rados.rados.0.smithi093.stdout:795: finishing write tid 1 to smithi093153278-90 2023-08-27T20:52:27.232 INFO:tasks.rados.rados.0.smithi093.stdout:795: finishing write tid 2 to smithi093153278-90 2023-08-27T20:52:27.232 INFO:tasks.rados.rados.0.smithi093.stdout:795: finishing write tid 3 to smithi093153278-90 2023-08-27T20:52:27.232 INFO:tasks.rados.rados.0.smithi093.stdout:795: finishing write tid 4 to smithi093153278-90 2023-08-27T20:52:27.232 INFO:tasks.rados.rados.0.smithi093.stdout:791: done (10 left) 2023-08-27T20:52:27.232 INFO:tasks.rados.rados.0.smithi093.stdout:792: done (9 left) 2023-08-27T20:52:27.232 INFO:tasks.rados.rados.0.smithi093.stdout:793: done (8 left) 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:794: done (7 left) 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:801: done (6 left) 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:802: write oid 213 current snap is 0 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:795: finishing write tid 5 to smithi093153278-90 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 90 v 162 (ObjNum 667 snap 0 seq_num 667) dirty exists 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:795: left oid 90 (ObjNum 667 snap 0 seq_num 667) 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:796: finishing write tid 1 to smithi093153278-77 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:796: finishing write tid 2 to smithi093153278-77 2023-08-27T20:52:27.233 INFO:tasks.rados.rados.0.smithi093.stdout:796: finishing write tid 3 to smithi093153278-77 2023-08-27T20:52:27.234 INFO:tasks.rados.rados.0.smithi093.stdout:802: seq_num 673 ranges {511111=481641,1599205=518558,2682027=489947} 2023-08-27T20:52:27.237 INFO:tasks.rados.rados.0.smithi093.stdout:802: writing smithi093153278-213 from 511111 to 992752 tid 1 2023-08-27T20:52:27.243 INFO:tasks.rados.rados.0.smithi093.stdout:802: writing smithi093153278-213 from 1599205 to 2117763 tid 2 2023-08-27T20:52:27.250 INFO:tasks.rados.rados.0.smithi093.stdout:802: writing smithi093153278-213 from 2682027 to 3171974 tid 3 2023-08-27T20:52:27.250 INFO:tasks.rados.rados.0.smithi093.stdout:795: done (6 left) 2023-08-27T20:52:27.250 INFO:tasks.rados.rados.0.smithi093.stdout:803: read oid 498 snap -1 2023-08-27T20:52:27.250 INFO:tasks.rados.rados.0.smithi093.stdout:803: expect (ObjNum 497 snap 0 seq_num 497) 2023-08-27T20:52:27.250 INFO:tasks.rados.rados.0.smithi093.stdout:804: write oid 327 current snap is 0 2023-08-27T20:52:27.250 INFO:tasks.rados.rados.0.smithi093.stdout:804: seq_num 674 ranges {527918=556890,1657687=778569,3225803=102115} 2023-08-27T20:52:27.256 INFO:tasks.rados.rados.0.smithi093.stdout:804: writing smithi093153278-327 from 527918 to 1084808 tid 1 2023-08-27T20:52:27.264 INFO:tasks.rados.rados.0.smithi093.stdout:804: writing smithi093153278-327 from 1657687 to 2436256 tid 2 2023-08-27T20:52:27.269 INFO:tasks.rados.rados.0.smithi093.stdout:804: writing smithi093153278-327 from 3225803 to 3327918 tid 3 2023-08-27T20:52:27.269 INFO:tasks.rados.rados.0.smithi093.stdout:805: write oid 461 current snap is 0 2023-08-27T20:52:27.269 INFO:tasks.rados.rados.0.smithi093.stdout:805: seq_num 675 ranges {544725=632139,1716169=522227,2736873=512491} 2023-08-27T20:52:27.275 INFO:tasks.rados.rados.0.smithi093.stdout:805: writing smithi093153278-461 from 544725 to 1176864 tid 1 2023-08-27T20:52:27.282 INFO:tasks.rados.rados.0.smithi093.stdout:805: writing smithi093153278-461 from 1716169 to 2238396 tid 2 2023-08-27T20:52:27.289 INFO:tasks.rados.rados.0.smithi093.stdout:805: writing smithi093153278-461 from 2736873 to 3249364 tid 3 2023-08-27T20:52:27.289 INFO:tasks.rados.rados.0.smithi093.stdout:806: write oid 168 current snap is 0 2023-08-27T20:52:27.289 INFO:tasks.rados.rados.0.smithi093.stdout:796: finishing write tid 4 to smithi093153278-77 2023-08-27T20:52:27.289 INFO:tasks.rados.rados.0.smithi093.stdout:806: seq_num 676 ranges {561532=707388,2058298=782238} 2023-08-27T20:52:27.296 INFO:tasks.rados.rados.0.smithi093.stdout:806: writing smithi093153278-168 from 561532 to 1268920 tid 1 2023-08-27T20:52:27.305 INFO:tasks.rados.rados.0.smithi093.stdout:806: writing smithi093153278-168 from 2058298 to 2840536 tid 2 2023-08-27T20:52:27.305 INFO:tasks.rados.rados.0.smithi093.stdout:807: read oid 201 snap -1 2023-08-27T20:52:27.305 INFO:tasks.rados.rados.0.smithi093.stdout:807: expect (ObjNum 609 snap 0 seq_num 609) 2023-08-27T20:52:27.305 INFO:tasks.rados.rados.0.smithi093.stdout:808: write oid 473 current snap is 0 2023-08-27T20:52:27.305 INFO:tasks.rados.rados.0.smithi093.stdout:796: finishing write tid 5 to smithi093153278-77 2023-08-27T20:52:27.305 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 77 v 158 (ObjNum 668 snap 0 seq_num 668) dirty exists 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:796: left oid 77 (ObjNum 668 snap 0 seq_num 668) 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:797: finishing write tid 1 to smithi093153278-349 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:797: finishing write tid 2 to smithi093153278-349 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:797: finishing write tid 3 to smithi093153278-349 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:797: finishing write tid 4 to smithi093153278-349 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:797: finishing write tid 5 to smithi093153278-349 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 349 v 166 (ObjNum 669 snap 0 seq_num 669) dirty exists 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:797: left oid 349 (ObjNum 669 snap 0 seq_num 669) 2023-08-27T20:52:27.306 INFO:tasks.rados.rados.0.smithi093.stdout:798: expect (ObjNum 248 snap 0 seq_num 248) 2023-08-27T20:52:27.381 INFO:tasks.rados.rados.0.smithi093.stdout:799: finishing write tid 1 to smithi093153278-81 2023-08-27T20:52:27.381 INFO:tasks.rados.rados.0.smithi093.stdout:799: finishing write tid 2 to smithi093153278-81 2023-08-27T20:52:27.381 INFO:tasks.rados.rados.0.smithi093.stdout:799: finishing write tid 3 to smithi093153278-81 2023-08-27T20:52:27.381 INFO:tasks.rados.rados.0.smithi093.stdout:799: finishing write tid 4 to smithi093153278-81 2023-08-27T20:52:27.382 INFO:tasks.rados.rados.0.smithi093.stdout:808: seq_num 677 ranges {578339=498990,1833133=525896,2791719=586620} 2023-08-27T20:52:27.387 INFO:tasks.rados.rados.0.smithi093.stdout:808: writing smithi093153278-473 from 578339 to 1077329 tid 1 2023-08-27T20:52:27.394 INFO:tasks.rados.rados.0.smithi093.stdout:808: writing smithi093153278-473 from 1833133 to 2359029 tid 2 2023-08-27T20:52:27.400 INFO:tasks.rados.rados.0.smithi093.stdout:808: writing smithi093153278-473 from 2791719 to 3378339 tid 3 2023-08-27T20:52:27.400 INFO:tasks.rados.rados.0.smithi093.stdout:796: done (11 left) 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:797: done (10 left) 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:798: done (9 left) 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:809: write oid 175 current snap is 0 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:799: finishing write tid 5 to smithi093153278-81 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 192 (ObjNum 670 snap 0 seq_num 670) dirty exists 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:799: left oid 81 (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:800: finishing write tid 1 to smithi093153278-230 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:800: finishing write tid 2 to smithi093153278-230 2023-08-27T20:52:27.401 INFO:tasks.rados.rados.0.smithi093.stdout:809: seq_num 678 ranges {595146=574239,1891615=669554,3219142=176004} 2023-08-27T20:52:27.406 INFO:tasks.rados.rados.0.smithi093.stdout:809: writing smithi093153278-175 from 595146 to 1169385 tid 1 2023-08-27T20:52:27.414 INFO:tasks.rados.rados.0.smithi093.stdout:809: writing smithi093153278-175 from 1891615 to 2561169 tid 2 2023-08-27T20:52:27.419 INFO:tasks.rados.rados.0.smithi093.stdout:809: writing smithi093153278-175 from 3219142 to 3395146 tid 3 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:799: done (9 left) 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:810: delete oid 16 current snap is 0 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:800: finishing write tid 3 to smithi093153278-230 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:800: finishing write tid 4 to smithi093153278-230 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:800: finishing write tid 5 to smithi093153278-230 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 136 (ObjNum 671 snap 0 seq_num 671) dirty exists 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:800: left oid 230 (ObjNum 671 snap 0 seq_num 671) 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:802: finishing write tid 1 to smithi093153278-213 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:802: finishing write tid 2 to smithi093153278-213 2023-08-27T20:52:27.420 INFO:tasks.rados.rados.0.smithi093.stdout:802: finishing write tid 3 to smithi093153278-213 2023-08-27T20:52:27.421 INFO:tasks.rados.rados.0.smithi093.stdout:802: finishing write tid 4 to smithi093153278-213 2023-08-27T20:52:27.421 INFO:tasks.rados.rados.0.smithi093.stdout:802: finishing write tid 5 to smithi093153278-213 2023-08-27T20:52:27.421 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 213 v 136 (ObjNum 673 snap 0 seq_num 673) dirty exists 2023-08-27T20:52:27.421 INFO:tasks.rados.rados.0.smithi093.stdout:802: left oid 213 (ObjNum 673 snap 0 seq_num 673) 2023-08-27T20:52:27.421 INFO:tasks.rados.rados.0.smithi093.stdout:803: expect (ObjNum 497 snap 0 seq_num 497) 2023-08-27T20:52:27.501 INFO:tasks.rados.rados.0.smithi093.stdout:804: finishing write tid 1 to smithi093153278-327 2023-08-27T20:52:27.501 INFO:tasks.rados.rados.0.smithi093.stdout:804: finishing write tid 2 to smithi093153278-327 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:804: finishing write tid 3 to smithi093153278-327 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:804: finishing write tid 4 to smithi093153278-327 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:804: finishing write tid 5 to smithi093153278-327 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 327 v 167 (ObjNum 674 snap 0 seq_num 674) dirty exists 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:804: left oid 327 (ObjNum 674 snap 0 seq_num 674) 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:805: finishing write tid 1 to smithi093153278-461 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:800: done (9 left) 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:802: done (8 left) 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:803: done (7 left) 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:804: done (6 left) 2023-08-27T20:52:27.502 INFO:tasks.rados.rados.0.smithi093.stdout:810: done (5 left) 2023-08-27T20:52:27.503 INFO:tasks.rados.rados.0.smithi093.stdout:811: read oid 241 snap -1 2023-08-27T20:52:27.503 INFO:tasks.rados.rados.0.smithi093.stdout:811: expect (ObjNum 240 snap 0 seq_num 240) 2023-08-27T20:52:27.503 INFO:tasks.rados.rados.0.smithi093.stdout:812: write oid 197 current snap is 0 2023-08-27T20:52:27.503 INFO:tasks.rados.rados.0.smithi093.stdout:805: finishing write tid 2 to smithi093153278-461 2023-08-27T20:52:27.503 INFO:tasks.rados.rados.0.smithi093.stdout:805: finishing write tid 3 to smithi093153278-461 2023-08-27T20:52:27.503 INFO:tasks.rados.rados.0.smithi093.stdout:812: seq_num 680 ranges {628760=724737,1892226=673223,3157635=271125} 2023-08-27T20:52:27.508 INFO:tasks.rados.rados.0.smithi093.stdout:812: writing smithi093153278-197 from 628760 to 1353497 tid 1 2023-08-27T20:52:27.515 INFO:tasks.rados.rados.0.smithi093.stdout:812: writing smithi093153278-197 from 1892226 to 2565449 tid 2 2023-08-27T20:52:27.521 INFO:tasks.rados.rados.0.smithi093.stdout:812: writing smithi093153278-197 from 3157635 to 3428760 tid 3 2023-08-27T20:52:27.521 INFO:tasks.rados.rados.0.smithi093.stdout:813: read oid 48 snap -1 2023-08-27T20:52:27.521 INFO:tasks.rados.rados.0.smithi093.stdout:813: expect (ObjNum 47 snap 0 seq_num 47) 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:814: read oid 495 snap -1 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:814: expect (ObjNum 625 snap 0 seq_num 625) 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:815: read oid 291 snap -1 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:815: expect deleted 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:816: read oid 42 snap -1 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:816: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:817: read oid 78 snap -1 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:817: expect (ObjNum 77 snap 0 seq_num 77) 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:818: delete oid 374 current snap is 0 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:805: finishing write tid 4 to smithi093153278-461 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:805: finishing write tid 5 to smithi093153278-461 2023-08-27T20:52:27.522 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 461 v 134 (ObjNum 675 snap 0 seq_num 675) dirty exists 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:805: left oid 461 (ObjNum 675 snap 0 seq_num 675) 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:806: finishing write tid 1 to smithi093153278-168 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:806: finishing write tid 2 to smithi093153278-168 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:806: finishing write tid 3 to smithi093153278-168 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:806: finishing write tid 4 to smithi093153278-168 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 168 v 139 (ObjNum 676 snap 0 seq_num 676) dirty exists 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:806: left oid 168 (ObjNum 676 snap 0 seq_num 676) 2023-08-27T20:52:27.523 INFO:tasks.rados.rados.0.smithi093.stdout:807: expect (ObjNum 609 snap 0 seq_num 609) 2023-08-27T20:52:27.635 INFO:tasks.rados.rados.0.smithi093.stdout:808: finishing write tid 1 to smithi093153278-473 2023-08-27T20:52:27.635 INFO:tasks.rados.rados.0.smithi093.stdout:808: finishing write tid 2 to smithi093153278-473 2023-08-27T20:52:27.635 INFO:tasks.rados.rados.0.smithi093.stdout:805: done (12 left) 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:806: done (11 left) 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:807: done (10 left) 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:818: done (9 left) 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:819: write oid 215 current snap is 0 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:808: finishing write tid 3 to smithi093153278-473 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:808: finishing write tid 4 to smithi093153278-473 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:808: finishing write tid 5 to smithi093153278-473 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 473 v 143 (ObjNum 677 snap 0 seq_num 677) dirty exists 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:808: left oid 473 (ObjNum 677 snap 0 seq_num 677) 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:809: finishing write tid 1 to smithi093153278-175 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:809: finishing write tid 2 to smithi093153278-175 2023-08-27T20:52:27.636 INFO:tasks.rados.rados.0.smithi093.stdout:809: finishing write tid 3 to smithi093153278-175 2023-08-27T20:52:27.637 INFO:tasks.rados.rados.0.smithi093.stdout:809: finishing write tid 4 to smithi093153278-175 2023-08-27T20:52:27.637 INFO:tasks.rados.rados.0.smithi093.stdout:809: finishing write tid 5 to smithi093153278-175 2023-08-27T20:52:27.637 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 175 v 140 (ObjNum 678 snap 0 seq_num 678) dirty exists 2023-08-27T20:52:27.637 INFO:tasks.rados.rados.0.smithi093.stdout:809: left oid 175 (ObjNum 678 snap 0 seq_num 678) 2023-08-27T20:52:27.637 INFO:tasks.rados.rados.0.smithi093.stdout:819: seq_num 682 ranges {662374=475235,1609190=676892,2812481=624101} 2023-08-27T20:52:27.641 INFO:tasks.rados.rados.0.smithi093.stdout:819: writing smithi093153278-215 from 662374 to 1137609 tid 1 2023-08-27T20:52:27.648 INFO:tasks.rados.rados.0.smithi093.stdout:819: writing smithi093153278-215 from 1609190 to 2286082 tid 2 2023-08-27T20:52:27.655 INFO:tasks.rados.rados.0.smithi093.stdout:819: writing smithi093153278-215 from 2812481 to 3436582 tid 3 2023-08-27T20:52:27.655 INFO:tasks.rados.rados.0.smithi093.stdout:808: done (9 left) 2023-08-27T20:52:27.655 INFO:tasks.rados.rados.0.smithi093.stdout:809: done (8 left) 2023-08-27T20:52:27.655 INFO:tasks.rados.rados.0.smithi093.stdout:820: write oid 93 current snap is 0 2023-08-27T20:52:27.655 INFO:tasks.rados.rados.0.smithi093.stdout:811: expect (ObjNum 240 snap 0 seq_num 240) 2023-08-27T20:52:27.726 INFO:tasks.rados.rados.0.smithi093.stdout:812: finishing write tid 1 to smithi093153278-197 2023-08-27T20:52:27.726 INFO:tasks.rados.rados.0.smithi093.stdout:812: finishing write tid 2 to smithi093153278-197 2023-08-27T20:52:27.726 INFO:tasks.rados.rados.0.smithi093.stdout:820: seq_num 683 ranges {679181=550484,1667672=536903,2839904=435373} 2023-08-27T20:52:27.732 INFO:tasks.rados.rados.0.smithi093.stdout:820: writing smithi093153278-93 from 679181 to 1229665 tid 1 2023-08-27T20:52:27.737 INFO:tasks.rados.rados.0.smithi093.stdout:820: writing smithi093153278-93 from 1667672 to 2204575 tid 2 2023-08-27T20:52:27.744 INFO:tasks.rados.rados.0.smithi093.stdout:820: writing smithi093153278-93 from 2839904 to 3275277 tid 3 2023-08-27T20:52:27.744 INFO:tasks.rados.rados.0.smithi093.stdout:811: done (8 left) 2023-08-27T20:52:27.744 INFO:tasks.rados.rados.0.smithi093.stdout:821: read oid 300 snap -1 2023-08-27T20:52:27.744 INFO:tasks.rados.rados.0.smithi093.stdout:821: expect (ObjNum 299 snap 0 seq_num 299) 2023-08-27T20:52:27.744 INFO:tasks.rados.rados.0.smithi093.stdout:822: write oid 7 current snap is 0 2023-08-27T20:52:27.744 INFO:tasks.rados.rados.0.smithi093.stdout:813: expect (ObjNum 47 snap 0 seq_num 47) 2023-08-27T20:52:27.833 INFO:tasks.rados.rados.0.smithi093.stdout:814: expect (ObjNum 625 snap 0 seq_num 625) 2023-08-27T20:52:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/635684947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:27 smithi136 ceph-mon[113531]: pgmap v68: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 102 MiB/s rd, 14 MiB/s wr, 148 op/s 2023-08-27T20:52:27.943 INFO:tasks.rados.rados.0.smithi093.stdout:816: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:52:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/635684947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:27 smithi093 ceph-mon[118902]: pgmap v68: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 102 MiB/s rd, 14 MiB/s wr, 148 op/s 2023-08-27T20:52:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/635684947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:27 smithi093 ceph-mon[111335]: pgmap v68: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 102 MiB/s rd, 14 MiB/s wr, 148 op/s 2023-08-27T20:52:28.027 INFO:tasks.rados.rados.0.smithi093.stdout:817: expect (ObjNum 77 snap 0 seq_num 77) 2023-08-27T20:52:28.127 INFO:tasks.rados.rados.0.smithi093.stdout:812: finishing write tid 3 to smithi093153278-197 2023-08-27T20:52:28.127 INFO:tasks.rados.rados.0.smithi093.stdout:812: finishing write tid 4 to smithi093153278-197 2023-08-27T20:52:28.128 INFO:tasks.rados.rados.0.smithi093.stdout:812: finishing write tid 5 to smithi093153278-197 2023-08-27T20:52:28.128 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 197 v 162 (ObjNum 680 snap 0 seq_num 680) dirty exists 2023-08-27T20:52:28.128 INFO:tasks.rados.rados.0.smithi093.stdout:812: left oid 197 (ObjNum 680 snap 0 seq_num 680) 2023-08-27T20:52:28.128 INFO:tasks.rados.rados.0.smithi093.stdout:822: seq_num 684 ranges {695988=625733,2009801=680561,3150974=345014} 2023-08-27T20:52:28.134 INFO:tasks.rados.rados.0.smithi093.stdout:822: writing smithi093153278-7 from 695988 to 1321721 tid 1 2023-08-27T20:52:28.142 INFO:tasks.rados.rados.0.smithi093.stdout:822: writing smithi093153278-7 from 2009801 to 2690362 tid 2 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:822: writing smithi093153278-7 from 3150974 to 3495988 tid 3 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:812: done (9 left) 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:813: done (8 left) 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:814: done (7 left) 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:815: done (6 left) 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:816: done (5 left) 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:817: done (4 left) 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:823: write oid 344 current snap is 0 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:819: finishing write tid 1 to smithi093153278-215 2023-08-27T20:52:28.147 INFO:tasks.rados.rados.0.smithi093.stdout:819: finishing write tid 2 to smithi093153278-215 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:819: finishing write tid 3 to smithi093153278-215 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:819: finishing write tid 4 to smithi093153278-215 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:819: finishing write tid 5 to smithi093153278-215 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 215 v 138 (ObjNum 682 snap 0 seq_num 682) dirty exists 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:819: left oid 215 (ObjNum 682 snap 0 seq_num 682) 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:820: finishing write tid 1 to smithi093153278-93 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:820: finishing write tid 2 to smithi093153278-93 2023-08-27T20:52:28.148 INFO:tasks.rados.rados.0.smithi093.stdout:823: seq_num 685 ranges {712795=417335,1784636=540572,2894750=457917} 2023-08-27T20:52:28.152 INFO:tasks.rados.rados.0.smithi093.stdout:823: writing smithi093153278-344 from 712795 to 1130130 tid 1 2023-08-27T20:52:28.159 INFO:tasks.rados.rados.0.smithi093.stdout:823: writing smithi093153278-344 from 1784636 to 2325208 tid 2 2023-08-27T20:52:28.164 INFO:tasks.rados.rados.0.smithi093.stdout:823: writing smithi093153278-344 from 2894750 to 3352667 tid 3 2023-08-27T20:52:28.165 INFO:tasks.rados.rados.0.smithi093.stdout:819: done (4 left) 2023-08-27T20:52:28.165 INFO:tasks.rados.rados.0.smithi093.stdout:824: write oid 152 current snap is 0 2023-08-27T20:52:28.165 INFO:tasks.rados.rados.0.smithi093.stdout:824: seq_num 686 ranges {729602=492584,1843118=684230,3322173=207429} 2023-08-27T20:52:28.171 INFO:tasks.rados.rados.0.smithi093.stdout:824: writing smithi093153278-152 from 729602 to 1222186 tid 1 2023-08-27T20:52:28.178 INFO:tasks.rados.rados.0.smithi093.stdout:824: writing smithi093153278-152 from 1843118 to 2527348 tid 2 2023-08-27T20:52:28.184 INFO:tasks.rados.rados.0.smithi093.stdout:824: writing smithi093153278-152 from 3322173 to 3529602 tid 3 2023-08-27T20:52:28.184 INFO:tasks.rados.rados.0.smithi093.stdout:825: write oid 180 current snap is 0 2023-08-27T20:52:28.184 INFO:tasks.rados.rados.0.smithi093.stdout:825: seq_num 687 ranges {746409=567833,1901600=544241,3065949=480460} 2023-08-27T20:52:28.191 INFO:tasks.rados.rados.0.smithi093.stdout:825: writing smithi093153278-180 from 746409 to 1314242 tid 1 2023-08-27T20:52:28.197 INFO:tasks.rados.rados.0.smithi093.stdout:825: writing smithi093153278-180 from 1901600 to 2445841 tid 2 2023-08-27T20:52:28.204 INFO:tasks.rados.rados.0.smithi093.stdout:825: writing smithi093153278-180 from 3065949 to 3546409 tid 3 2023-08-27T20:52:28.204 INFO:tasks.rados.rados.0.smithi093.stdout:826: write oid 75 current snap is 0 2023-08-27T20:52:28.204 INFO:tasks.rados.rados.0.smithi093.stdout:820: finishing write tid 3 to smithi093153278-93 2023-08-27T20:52:28.204 INFO:tasks.rados.rados.0.smithi093.stdout:826: seq_num 688 ranges {763216=643082,1843729=687899,3260666=302550} 2023-08-27T20:52:28.212 INFO:tasks.rados.rados.0.smithi093.stdout:826: writing smithi093153278-75 from 763216 to 1406298 tid 1 2023-08-27T20:52:28.218 INFO:tasks.rados.rados.0.smithi093.stdout:826: writing smithi093153278-75 from 1843729 to 2531628 tid 2 2023-08-27T20:52:28.224 INFO:tasks.rados.rados.0.smithi093.stdout:826: writing smithi093153278-75 from 3260666 to 3563216 tid 3 2023-08-27T20:52:28.224 INFO:tasks.rados.rados.0.smithi093.stdout:827: read oid 177 snap -1 2023-08-27T20:52:28.224 INFO:tasks.rados.rados.0.smithi093.stdout:827: expect (ObjNum 176 snap 0 seq_num 176) 2023-08-27T20:52:28.225 INFO:tasks.rados.rados.0.smithi093.stdout:828: write oid 67 current snap is 0 2023-08-27T20:52:28.225 INFO:tasks.rados.rados.0.smithi093.stdout:820: finishing write tid 4 to smithi093153278-93 2023-08-27T20:52:28.225 INFO:tasks.rados.rados.0.smithi093.stdout:820: finishing write tid 5 to smithi093153278-93 2023-08-27T20:52:28.225 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 93 v 147 (ObjNum 683 snap 0 seq_num 683) dirty exists 2023-08-27T20:52:28.225 INFO:tasks.rados.rados.0.smithi093.stdout:820: left oid 93 (ObjNum 683 snap 0 seq_num 683) 2023-08-27T20:52:28.225 INFO:tasks.rados.rados.0.smithi093.stdout:828: seq_num 689 ranges {780023=718331,1902211=547910,3004442=575581} 2023-08-27T20:52:28.233 INFO:tasks.rados.rados.0.smithi093.stdout:828: writing smithi093153278-67 from 780023 to 1498354 tid 1 2023-08-27T20:52:28.239 INFO:tasks.rados.rados.0.smithi093.stdout:828: writing smithi093153278-67 from 1902211 to 2450121 tid 2 2023-08-27T20:52:28.245 INFO:tasks.rados.rados.0.smithi093.stdout:828: writing smithi093153278-67 from 3004442 to 3580023 tid 3 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:820: done (8 left) 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:829: read oid 35 snap -1 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:829: expect (ObjNum 653 snap 0 seq_num 653) 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:830: read oid 247 snap -1 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:830: expect (ObjNum 246 snap 0 seq_num 246) 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:831: read oid 273 snap -1 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:831: expect (ObjNum 607 snap 0 seq_num 607) 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:832: write oid 328 current snap is 0 2023-08-27T20:52:28.246 INFO:tasks.rados.rados.0.smithi093.stdout:821: expect (ObjNum 299 snap 0 seq_num 299) 2023-08-27T20:52:28.344 INFO:tasks.rados.rados.0.smithi093.stdout:822: finishing write tid 1 to smithi093153278-7 2023-08-27T20:52:28.344 INFO:tasks.rados.rados.0.smithi093.stdout:822: finishing write tid 2 to smithi093153278-7 2023-08-27T20:52:28.344 INFO:tasks.rados.rados.0.smithi093.stdout:822: finishing write tid 3 to smithi093153278-7 2023-08-27T20:52:28.344 INFO:tasks.rados.rados.0.smithi093.stdout:822: finishing write tid 4 to smithi093153278-7 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:822: finishing write tid 5 to smithi093153278-7 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 7 v 124 (ObjNum 684 snap 0 seq_num 684) dirty exists 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:822: left oid 7 (ObjNum 684 snap 0 seq_num 684) 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:823: finishing write tid 1 to smithi093153278-344 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:823: finishing write tid 2 to smithi093153278-344 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:823: finishing write tid 3 to smithi093153278-344 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:823: finishing write tid 4 to smithi093153278-344 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:823: finishing write tid 5 to smithi093153278-344 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 344 v 153 (ObjNum 685 snap 0 seq_num 685) dirty exists 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:823: left oid 344 (ObjNum 685 snap 0 seq_num 685) 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:824: finishing write tid 1 to smithi093153278-152 2023-08-27T20:52:28.345 INFO:tasks.rados.rados.0.smithi093.stdout:832: seq_num 690 ranges {796830=793580,2360693=691568} 2023-08-27T20:52:28.352 INFO:tasks.rados.rados.0.smithi093.stdout:832: writing smithi093153278-328 from 796830 to 1590410 tid 1 2023-08-27T20:52:28.361 INFO:tasks.rados.rados.0.smithi093.stdout:832: writing smithi093153278-328 from 2360693 to 3052261 tid 2 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:821: done (11 left) 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:822: done (10 left) 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:823: done (9 left) 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:833: read oid 56 snap -1 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:833: expect (ObjNum 55 snap 0 seq_num 55) 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:834: read oid 445 snap -1 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:834: expect (ObjNum 444 snap 0 seq_num 444) 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:835: read oid 489 snap -1 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:835: expect (ObjNum 488 snap 0 seq_num 488) 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:836: write oid 335 current snap is 0 2023-08-27T20:52:28.362 INFO:tasks.rados.rados.0.smithi093.stdout:824: finishing write tid 2 to smithi093153278-152 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:824: finishing write tid 3 to smithi093153278-152 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:824: finishing write tid 4 to smithi093153278-152 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:824: finishing write tid 5 to smithi093153278-152 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 152 v 157 (ObjNum 686 snap 0 seq_num 686) dirty exists 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:824: left oid 152 (ObjNum 686 snap 0 seq_num 686) 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:825: finishing write tid 1 to smithi093153278-180 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:825: finishing write tid 2 to smithi093153278-180 2023-08-27T20:52:28.363 INFO:tasks.rados.rados.0.smithi093.stdout:836: seq_num 691 ranges {413637=468829,1619175=435226,2542935=758255} 2023-08-27T20:52:28.366 INFO:tasks.rados.rados.0.smithi093.stdout:836: writing smithi093153278-335 from 413637 to 882466 tid 1 2023-08-27T20:52:28.372 INFO:tasks.rados.rados.0.smithi093.stdout:836: writing smithi093153278-335 from 1619175 to 2054401 tid 2 2023-08-27T20:52:28.380 INFO:tasks.rados.rados.0.smithi093.stdout:836: writing smithi093153278-335 from 2542935 to 3301190 tid 3 2023-08-27T20:52:28.380 INFO:tasks.rados.rados.0.smithi093.stdout:824: done (12 left) 2023-08-27T20:52:28.380 INFO:tasks.rados.rados.0.smithi093.stdout:837: read oid 301 snap -1 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:837: expect (ObjNum 300 snap 0 seq_num 300) 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:838: read oid 270 snap -1 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:838: expect (ObjNum 604 snap 0 seq_num 604) 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:839: read oid 390 snap -1 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:839: expect (ObjNum 389 snap 0 seq_num 389) 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:840: write oid 190 current snap is 0 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:825: finishing write tid 3 to smithi093153278-180 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:825: finishing write tid 4 to smithi093153278-180 2023-08-27T20:52:28.381 INFO:tasks.rados.rados.0.smithi093.stdout:840: seq_num 692 ranges {430444=660431,1677657=695237,2970358=569527} 2023-08-27T20:52:28.386 INFO:tasks.rados.rados.0.smithi093.stdout:840: writing smithi093153278-190 from 430444 to 1090875 tid 1 2023-08-27T20:52:28.394 INFO:tasks.rados.rados.0.smithi093.stdout:840: writing smithi093153278-190 from 1677657 to 2372894 tid 2 2023-08-27T20:52:28.400 INFO:tasks.rados.rados.0.smithi093.stdout:840: writing smithi093153278-190 from 2970358 to 3539885 tid 3 2023-08-27T20:52:28.400 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:825: finishing write tid 5 to smithi093153278-180 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 180 v 167 (ObjNum 687 snap 0 seq_num 687) dirty exists 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:825: left oid 180 (ObjNum 687 snap 0 seq_num 687) 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:826: finishing write tid 1 to smithi093153278-75 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:826: finishing write tid 2 to smithi093153278-75 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:825: done (15 left) 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:841: write oid 297 current snap is 0 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:826: finishing write tid 3 to smithi093153278-75 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:826: finishing write tid 4 to smithi093153278-75 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:826: finishing write tid 5 to smithi093153278-75 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 75 v 196 (ObjNum 688 snap 0 seq_num 688) dirty exists 2023-08-27T20:52:28.401 INFO:tasks.rados.rados.0.smithi093.stdout:826: left oid 75 (ObjNum 688 snap 0 seq_num 688) 2023-08-27T20:52:28.402 INFO:tasks.rados.rados.0.smithi093.stdout:841: seq_num 693 ranges {447251=735680,1736139=438895,2597781=780799} 2023-08-27T20:52:28.407 INFO:tasks.rados.rados.0.smithi093.stdout:841: writing smithi093153278-297 from 447251 to 1182931 tid 1 2023-08-27T20:52:28.412 INFO:tasks.rados.rados.0.smithi093.stdout:841: writing smithi093153278-297 from 1736139 to 2175034 tid 2 2023-08-27T20:52:28.419 INFO:tasks.rados.rados.0.smithi093.stdout:841: writing smithi093153278-297 from 2597781 to 3378580 tid 3 2023-08-27T20:52:28.419 INFO:tasks.rados.rados.0.smithi093.stdout:826: done (15 left) 2023-08-27T20:52:28.419 INFO:tasks.rados.rados.0.smithi093.stdout:842: write oid 473 current snap is 0 2023-08-27T20:52:28.419 INFO:tasks.rados.rados.0.smithi093.stdout:827: expect (ObjNum 176 snap 0 seq_num 176) 2023-08-27T20:52:28.517 INFO:tasks.rados.rados.0.smithi093.stdout:828: finishing write tid 1 to smithi093153278-67 2023-08-27T20:52:28.517 INFO:tasks.rados.rados.0.smithi093.stdout:828: finishing write tid 2 to smithi093153278-67 2023-08-27T20:52:28.517 INFO:tasks.rados.rados.0.smithi093.stdout:842: seq_num 694 ranges {464058=410929,1394621=698906,2625204=592071,3656107=7951} 2023-08-27T20:52:28.522 INFO:tasks.rados.rados.0.smithi093.stdout:842: writing smithi093153278-473 from 464058 to 874987 tid 1 2023-08-27T20:52:28.528 INFO:tasks.rados.rados.0.smithi093.stdout:842: writing smithi093153278-473 from 1394621 to 2093527 tid 2 2023-08-27T20:52:28.535 INFO:tasks.rados.rados.0.smithi093.stdout:842: writing smithi093153278-473 from 2625204 to 3217275 tid 3 2023-08-27T20:52:28.538 INFO:tasks.rados.rados.0.smithi093.stdout:842: writing smithi093153278-473 from 3656107 to 3664058 tid 4 2023-08-27T20:52:28.538 INFO:tasks.rados.rados.0.smithi093.stdout:827: done (15 left) 2023-08-27T20:52:28.538 INFO:tasks.rados.rados.0.smithi093.stdout:843: read oid 142 snap -1 2023-08-27T20:52:28.538 INFO:tasks.rados.rados.0.smithi093.stdout:843: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:52:28.538 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:28.538 INFO:tasks.rados.rados.0.smithi093.stdout:831: expect (ObjNum 607 snap 0 seq_num 607) 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:828: finishing write tid 3 to smithi093153278-67 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:828: finishing write tid 4 to smithi093153278-67 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:831: done (15 left) 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:844: delete oid 318 current snap is 0 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:828: finishing write tid 5 to smithi093153278-67 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 67 v 128 (ObjNum 689 snap 0 seq_num 689) dirty exists 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:828: left oid 67 (ObjNum 689 snap 0 seq_num 689) 2023-08-27T20:52:28.627 INFO:tasks.rados.rados.0.smithi093.stdout:830: expect (ObjNum 246 snap 0 seq_num 246) 2023-08-27T20:52:28.696 INFO:tasks.rados.rados.0.smithi093.stdout:829: expect (ObjNum 653 snap 0 seq_num 653) 2023-08-27T20:52:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:832: finishing write tid 1 to smithi093153278-328 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:828: done (15 left) 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:829: done (14 left) 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:830: done (13 left) 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:844: done (12 left) 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:845: read oid 227 snap -1 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:845: expect (ObjNum 226 snap 0 seq_num 226) 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:846: read oid 13 snap -1 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:846: expect (ObjNum 12 snap 0 seq_num 12) 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:847: read oid 296 snap -1 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:847: expect (ObjNum 666 snap 0 seq_num 666) 2023-08-27T20:52:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:848: write oid 3 current snap is 0 2023-08-27T20:52:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:832: finishing write tid 2 to smithi093153278-328 2023-08-27T20:52:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:848: seq_num 696 ranges {497672=561427,1795232=702575,2963697=730968} 2023-08-27T20:52:28.840 INFO:tasks.rados.rados.0.smithi093.stdout:848: writing smithi093153278-3 from 497672 to 1059099 tid 1 2023-08-27T20:52:28.849 INFO:tasks.rados.rados.0.smithi093.stdout:848: writing smithi093153278-3 from 1795232 to 2497807 tid 2 2023-08-27T20:52:28.856 INFO:tasks.rados.rados.0.smithi093.stdout:848: writing smithi093153278-3 from 2963697 to 3694665 tid 3 2023-08-27T20:52:28.856 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:28.857 INFO:tasks.rados.rados.0.smithi093.stdout:832: finishing write tid 3 to smithi093153278-328 2023-08-27T20:52:28.857 INFO:tasks.rados.rados.0.smithi093.stdout:832: finishing write tid 4 to smithi093153278-328 2023-08-27T20:52:28.857 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 160 (ObjNum 690 snap 0 seq_num 690) dirty exists 2023-08-27T20:52:28.857 INFO:tasks.rados.rados.0.smithi093.stdout:832: left oid 328 (ObjNum 690 snap 0 seq_num 690) 2023-08-27T20:52:28.857 INFO:tasks.rados.rados.0.smithi093.stdout:835: expect (ObjNum 488 snap 0 seq_num 488) 2023-08-27T20:52:28.928 INFO:tasks.rados.rados.0.smithi093.stdout:836: finishing write tid 1 to smithi093153278-335 2023-08-27T20:52:28.928 INFO:tasks.rados.rados.0.smithi093.stdout:834: expect (ObjNum 444 snap 0 seq_num 444) 2023-08-27T20:52:29.039 INFO:tasks.rados.rados.0.smithi093.stdout:833: expect (ObjNum 55 snap 0 seq_num 55) 2023-08-27T20:52:29.133 INFO:tasks.rados.rados.0.smithi093.stdout:836: finishing write tid 2 to smithi093153278-335 2023-08-27T20:52:29.133 INFO:tasks.rados.rados.0.smithi093.stdout:832: done (15 left) 2023-08-27T20:52:29.133 INFO:tasks.rados.rados.0.smithi093.stdout:833: done (14 left) 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:834: done (13 left) 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:835: done (12 left) 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:849: delete oid 373 current snap is 0 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:836: finishing write tid 3 to smithi093153278-335 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:836: finishing write tid 4 to smithi093153278-335 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:836: finishing write tid 5 to smithi093153278-335 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 335 v 144 (ObjNum 691 snap 0 seq_num 691) dirty exists 2023-08-27T20:52:29.134 INFO:tasks.rados.rados.0.smithi093.stdout:836: left oid 335 (ObjNum 691 snap 0 seq_num 691) 2023-08-27T20:52:29.135 INFO:tasks.rados.rados.0.smithi093.stdout:837: expect (ObjNum 300 snap 0 seq_num 300) 2023-08-27T20:52:29.233 INFO:tasks.rados.rados.0.smithi093.stdout:839: expect (ObjNum 389 snap 0 seq_num 389) 2023-08-27T20:52:29.317 INFO:tasks.rados.rados.0.smithi093.stdout:838: expect (ObjNum 604 snap 0 seq_num 604) 2023-08-27T20:52:29.426 INFO:tasks.rados.rados.0.smithi093.stdout:840: finishing write tid 1 to smithi093153278-190 2023-08-27T20:52:29.427 INFO:tasks.rados.rados.0.smithi093.stdout:840: finishing write tid 2 to smithi093153278-190 2023-08-27T20:52:29.427 INFO:tasks.rados.rados.0.smithi093.stdout:840: finishing write tid 3 to smithi093153278-190 2023-08-27T20:52:29.427 INFO:tasks.rados.rados.0.smithi093.stdout:840: finishing write tid 4 to smithi093153278-190 2023-08-27T20:52:29.427 INFO:tasks.rados.rados.0.smithi093.stdout:840: finishing write tid 5 to smithi093153278-190 2023-08-27T20:52:29.427 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 190 v 171 (ObjNum 692 snap 0 seq_num 692) dirty exists 2023-08-27T20:52:29.427 INFO:tasks.rados.rados.0.smithi093.stdout:840: left oid 190 (ObjNum 692 snap 0 seq_num 692) 2023-08-27T20:52:29.428 INFO:tasks.rados.rados.0.smithi093.stdout:841: finishing write tid 1 to smithi093153278-297 2023-08-27T20:52:29.428 INFO:tasks.rados.rados.0.smithi093.stdout:841: finishing write tid 2 to smithi093153278-297 2023-08-27T20:52:29.428 INFO:tasks.rados.rados.0.smithi093.stdout:841: finishing write tid 3 to smithi093153278-297 2023-08-27T20:52:29.428 INFO:tasks.rados.rados.0.smithi093.stdout:841: finishing write tid 4 to smithi093153278-297 2023-08-27T20:52:29.428 INFO:tasks.rados.rados.0.smithi093.stdout:841: finishing write tid 5 to smithi093153278-297 2023-08-27T20:52:29.428 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 297 v 151 (ObjNum 693 snap 0 seq_num 693) dirty exists 2023-08-27T20:52:29.429 INFO:tasks.rados.rados.0.smithi093.stdout:841: left oid 297 (ObjNum 693 snap 0 seq_num 693) 2023-08-27T20:52:29.429 INFO:tasks.rados.rados.0.smithi093.stdout:842: finishing write tid 1 to smithi093153278-473 2023-08-27T20:52:29.429 INFO:tasks.rados.rados.0.smithi093.stdout:842: finishing write tid 2 to smithi093153278-473 2023-08-27T20:52:29.429 INFO:tasks.rados.rados.0.smithi093.stdout:842: finishing write tid 3 to smithi093153278-473 2023-08-27T20:52:29.429 INFO:tasks.rados.rados.0.smithi093.stdout:842: finishing write tid 4 to smithi093153278-473 2023-08-27T20:52:29.429 INFO:tasks.rados.rados.0.smithi093.stdout:836: done (12 left) 2023-08-27T20:52:29.430 INFO:tasks.rados.rados.0.smithi093.stdout:837: done (11 left) 2023-08-27T20:52:29.430 INFO:tasks.rados.rados.0.smithi093.stdout:838: done (10 left) 2023-08-27T20:52:29.430 INFO:tasks.rados.rados.0.smithi093.stdout:839: done (9 left) 2023-08-27T20:52:29.430 INFO:tasks.rados.rados.0.smithi093.stdout:840: done (8 left) 2023-08-27T20:52:29.430 INFO:tasks.rados.rados.0.smithi093.stdout:841: done (7 left) 2023-08-27T20:52:29.431 INFO:tasks.rados.rados.0.smithi093.stdout:849: done (6 left) 2023-08-27T20:52:29.431 INFO:tasks.rados.rados.0.smithi093.stdout:850: delete oid 172 current snap is 0 2023-08-27T20:52:29.431 INFO:tasks.rados.rados.0.smithi093.stdout:842: finishing write tid 5 to smithi093153278-473 2023-08-27T20:52:29.431 INFO:tasks.rados.rados.0.smithi093.stdout:842: finishing write tid 6 to smithi093153278-473 2023-08-27T20:52:29.431 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 473 v 148 (ObjNum 694 snap 0 seq_num 694) dirty exists 2023-08-27T20:52:29.431 INFO:tasks.rados.rados.0.smithi093.stdout:842: left oid 473 (ObjNum 694 snap 0 seq_num 694) 2023-08-27T20:52:29.431 INFO:tasks.rados.rados.0.smithi093.stdout:843: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:52:29.504 INFO:tasks.rados.rados.0.smithi093.stdout:847: expect (ObjNum 666 snap 0 seq_num 666) 2023-08-27T20:52:29.641 INFO:tasks.rados.rados.0.smithi093.stdout:848: finishing write tid 1 to smithi093153278-3 2023-08-27T20:52:29.642 INFO:tasks.rados.rados.0.smithi093.stdout:846: expect (ObjNum 12 snap 0 seq_num 12) 2023-08-27T20:52:29.706 INFO:tasks.rados.rados.0.smithi093.stdout:845: expect (ObjNum 226 snap 0 seq_num 226) 2023-08-27T20:52:29.821 INFO:tasks.rados.rados.0.smithi093.stdout:848: finishing write tid 2 to smithi093153278-3 2023-08-27T20:52:29.821 INFO:tasks.rados.rados.0.smithi093.stdout:848: finishing write tid 3 to smithi093153278-3 2023-08-27T20:52:29.821 INFO:tasks.rados.rados.0.smithi093.stdout:848: finishing write tid 4 to smithi093153278-3 2023-08-27T20:52:29.821 INFO:tasks.rados.rados.0.smithi093.stdout:848: finishing write tid 5 to smithi093153278-3 2023-08-27T20:52:29.821 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 3 v 175 (ObjNum 696 snap 0 seq_num 696) dirty exists 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:848: left oid 3 (ObjNum 696 snap 0 seq_num 696) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:842: done (6 left) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:843: done (5 left) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:845: done (4 left) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:846: done (3 left) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:847: done (2 left) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:848: done (1 left) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:850: done (0 left) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:851: read oid 258 snap -1 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:851: expect (ObjNum 257 snap 0 seq_num 257) 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:852: read oid 475 snap -1 2023-08-27T20:52:29.822 INFO:tasks.rados.rados.0.smithi093.stdout:852: expect (ObjNum 474 snap 0 seq_num 474) 2023-08-27T20:52:29.823 INFO:tasks.rados.rados.0.smithi093.stdout:853: write oid 413 current snap is 0 2023-08-27T20:52:29.823 INFO:tasks.rados.rados.0.smithi093.stdout:853: seq_num 699 ranges {548093=787174,1970678=449902,3045966=681137} 2023-08-27T20:52:29.829 INFO:tasks.rados.rados.0.smithi093.stdout:853: writing smithi093153278-413 from 548093 to 1335267 tid 1 2023-08-27T20:52:29.835 INFO:tasks.rados.rados.0.smithi093.stdout:853: writing smithi093153278-413 from 1970678 to 2420580 tid 2 2023-08-27T20:52:29.842 INFO:tasks.rados.rados.0.smithi093.stdout:853: writing smithi093153278-413 from 3045966 to 3727103 tid 3 2023-08-27T20:52:29.842 INFO:tasks.rados.rados.0.smithi093.stdout:854: write oid 210 current snap is 0 2023-08-27T20:52:29.842 INFO:tasks.rados.rados.0.smithi093.stdout:854: seq_num 700 ranges {564900=578776,1629160=709913,3073389=492409} 2023-08-27T20:52:29.848 INFO:tasks.rados.rados.0.smithi093.stdout:854: writing smithi093153278-210 from 564900 to 1143676 tid 1 2023-08-27T20:52:29.855 INFO:tasks.rados.rados.0.smithi093.stdout:854: writing smithi093153278-210 from 1629160 to 2339073 tid 2 2023-08-27T20:52:29.862 INFO:tasks.rados.rados.0.smithi093.stdout:854: writing smithi093153278-210 from 3073389 to 3565798 tid 3 2023-08-27T20:52:29.862 INFO:tasks.rados.rados.0.smithi093.stdout:855: read oid 118 snap -1 2023-08-27T20:52:29.862 INFO:tasks.rados.rados.0.smithi093.stdout:855: expect (ObjNum 600 snap 0 seq_num 600) 2023-08-27T20:52:29.862 INFO:tasks.rados.rados.0.smithi093.stdout:856: delete oid 98 current snap is 0 2023-08-27T20:52:29.863 INFO:tasks.rados.rados.0.smithi093.stdout:852: expect (ObjNum 474 snap 0 seq_num 474) 2023-08-27T20:52:29.987 INFO:tasks.rados.rados.0.smithi093.stdout:851: expect (ObjNum 257 snap 0 seq_num 257) 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:853: finishing write tid 1 to smithi093153278-413 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:853: finishing write tid 2 to smithi093153278-413 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:853: finishing write tid 3 to smithi093153278-413 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:853: finishing write tid 4 to smithi093153278-413 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:853: finishing write tid 5 to smithi093153278-413 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 413 v 142 (ObjNum 699 snap 0 seq_num 699) dirty exists 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:853: left oid 413 (ObjNum 699 snap 0 seq_num 699) 2023-08-27T20:52:30.058 INFO:tasks.rados.rados.0.smithi093.stdout:854: finishing write tid 1 to smithi093153278-210 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:851: done (5 left) 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:852: done (4 left) 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:853: done (3 left) 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:856: done (2 left) 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:857: read oid 76 snap -1 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:857: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:858: delete oid 129 current snap is 0 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:854: finishing write tid 2 to smithi093153278-210 2023-08-27T20:52:30.059 INFO:tasks.rados.rados.0.smithi093.stdout:855: expect (ObjNum 600 snap 0 seq_num 600) 2023-08-27T20:52:30.145 INFO:tasks.rados.rados.0.smithi093.stdout:854: finishing write tid 3 to smithi093153278-210 2023-08-27T20:52:30.145 INFO:tasks.rados.rados.0.smithi093.stdout:854: finishing write tid 4 to smithi093153278-210 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:854: finishing write tid 5 to smithi093153278-210 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 210 v 200 (ObjNum 700 snap 0 seq_num 700) dirty exists 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:854: left oid 210 (ObjNum 700 snap 0 seq_num 700) 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:854: done (3 left) 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:855: done (2 left) 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:858: done (1 left) 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:859: read oid 426 snap -1 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:859: expect (ObjNum 596 snap 0 seq_num 596) 2023-08-27T20:52:30.146 INFO:tasks.rados.rados.0.smithi093.stdout:857: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:52:30.246 INFO:tasks.rados.rados.0.smithi093.stdout:857: done (1 left) 2023-08-27T20:52:30.246 INFO:tasks.rados.rados.0.smithi093.stdout:860: write oid 59 current snap is 0 2023-08-27T20:52:30.246 INFO:tasks.rados.rados.0.smithi093.stdout:860: seq_num 703 ranges {615321=404523,1804606=457240,2755658=726225} 2023-08-27T20:52:30.251 INFO:tasks.rados.rados.0.smithi093.stdout:860: writing smithi093153278-59 from 615321 to 1019844 tid 1 2023-08-27T20:52:30.258 INFO:tasks.rados.rados.0.smithi093.stdout:860: writing smithi093153278-59 from 1804606 to 2261846 tid 2 2023-08-27T20:52:30.265 INFO:tasks.rados.rados.0.smithi093.stdout:860: writing smithi093153278-59 from 2755658 to 3481883 tid 3 2023-08-27T20:52:30.265 INFO:tasks.rados.rados.0.smithi093.stdout:861: read oid 301 snap -1 2023-08-27T20:52:30.265 INFO:tasks.rados.rados.0.smithi093.stdout:861: expect (ObjNum 300 snap 0 seq_num 300) 2023-08-27T20:52:30.265 INFO:tasks.rados.rados.0.smithi093.stdout:862: read oid 484 snap -1 2023-08-27T20:52:30.265 INFO:tasks.rados.rados.0.smithi093.stdout:862: expect (ObjNum 483 snap 0 seq_num 483) 2023-08-27T20:52:30.265 INFO:tasks.rados.rados.0.smithi093.stdout:863: write oid 461 current snap is 0 2023-08-27T20:52:30.265 INFO:tasks.rados.rados.0.smithi093.stdout:863: seq_num 704 ranges {632128=479772,1746735=600898,3066728=653850} 2023-08-27T20:52:30.271 INFO:tasks.rados.rados.0.smithi093.stdout:863: writing smithi093153278-461 from 632128 to 1111900 tid 1 2023-08-27T20:52:30.278 INFO:tasks.rados.rados.0.smithi093.stdout:863: writing smithi093153278-461 from 1746735 to 2347633 tid 2 2023-08-27T20:52:30.286 INFO:tasks.rados.rados.0.smithi093.stdout:863: writing smithi093153278-461 from 3066728 to 3720578 tid 3 2023-08-27T20:52:30.286 INFO:tasks.rados.rados.0.smithi093.stdout:864: write oid 111 current snap is 0 2023-08-27T20:52:30.286 INFO:tasks.rados.rados.0.smithi093.stdout:859: expect (ObjNum 596 snap 0 seq_num 596) 2023-08-27T20:52:30.367 INFO:tasks.rados.rados.0.smithi093.stdout:860: finishing write tid 1 to smithi093153278-59 2023-08-27T20:52:30.367 INFO:tasks.rados.rados.0.smithi093.stdout:860: finishing write tid 2 to smithi093153278-59 2023-08-27T20:52:30.367 INFO:tasks.rados.rados.0.smithi093.stdout:861: expect (ObjNum 300 snap 0 seq_num 300) 2023-08-27T20:52:30.461 INFO:tasks.rados.rados.0.smithi093.stdout:860: finishing write tid 3 to smithi093153278-59 2023-08-27T20:52:30.461 INFO:tasks.rados.rados.0.smithi093.stdout:860: finishing write tid 4 to smithi093153278-59 2023-08-27T20:52:30.461 INFO:tasks.rados.rados.0.smithi093.stdout:860: finishing write tid 5 to smithi093153278-59 2023-08-27T20:52:30.461 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 59 v 127 (ObjNum 703 snap 0 seq_num 703) dirty exists 2023-08-27T20:52:30.462 INFO:tasks.rados.rados.0.smithi093.stdout:860: left oid 59 (ObjNum 703 snap 0 seq_num 703) 2023-08-27T20:52:30.462 INFO:tasks.rados.rados.0.smithi093.stdout:862: expect (ObjNum 483 snap 0 seq_num 483) 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: pgmap v69: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 69 MiB/s rd, 9.6 MiB/s wr, 99 op/s 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:52:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1429055772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: pgmap v69: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 69 MiB/s rd, 9.6 MiB/s wr, 99 op/s 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:52:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1429055772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:30.529 INFO:tasks.rados.rados.0.smithi093.stdout:863: finishing write tid 1 to smithi093153278-461 2023-08-27T20:52:30.530 INFO:tasks.rados.rados.0.smithi093.stdout:863: finishing write tid 2 to smithi093153278-461 2023-08-27T20:52:30.530 INFO:tasks.rados.rados.0.smithi093.stdout:863: finishing write tid 3 to smithi093153278-461 2023-08-27T20:52:30.530 INFO:tasks.rados.rados.0.smithi093.stdout:863: finishing write tid 4 to smithi093153278-461 2023-08-27T20:52:30.530 INFO:tasks.rados.rados.0.smithi093.stdout:863: finishing write tid 5 to smithi093153278-461 2023-08-27T20:52:30.530 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 461 v 146 (ObjNum 704 snap 0 seq_num 704) dirty exists 2023-08-27T20:52:30.530 INFO:tasks.rados.rados.0.smithi093.stdout:863: left oid 461 (ObjNum 704 snap 0 seq_num 704) 2023-08-27T20:52:30.530 INFO:tasks.rados.rados.0.smithi093.stdout:864: seq_num 705 ranges {648935=555021,1805217=460909,2694151=465122,3817442=31493} 2023-08-27T20:52:30.535 INFO:tasks.rados.rados.0.smithi093.stdout:864: writing smithi093153278-111 from 648935 to 1203956 tid 1 2023-08-27T20:52:30.541 INFO:tasks.rados.rados.0.smithi093.stdout:864: writing smithi093153278-111 from 1805217 to 2266126 tid 2 2023-08-27T20:52:30.546 INFO:tasks.rados.rados.0.smithi093.stdout:864: writing smithi093153278-111 from 2694151 to 3159273 tid 3 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:864: writing smithi093153278-111 from 3817442 to 3848935 tid 4 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:859: done (5 left) 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:860: done (4 left) 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:861: done (3 left) 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:862: done (2 left) 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:863: done (1 left) 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:865: read oid 239 snap -1 2023-08-27T20:52:30.551 INFO:tasks.rados.rados.0.smithi093.stdout:865: expect (ObjNum 238 snap 0 seq_num 238) 2023-08-27T20:52:30.552 INFO:tasks.rados.rados.0.smithi093.stdout:866: read oid 137 snap -1 2023-08-27T20:52:30.552 INFO:tasks.rados.rados.0.smithi093.stdout:866: expect (ObjNum 136 snap 0 seq_num 136) 2023-08-27T20:52:30.552 INFO:tasks.rados.rados.0.smithi093.stdout:867: write oid 437 current snap is 0 2023-08-27T20:52:30.552 INFO:tasks.rados.rados.0.smithi093.stdout:864: finishing write tid 1 to smithi093153278-111 2023-08-27T20:52:30.552 INFO:tasks.rados.rados.0.smithi093.stdout:867: seq_num 706 ranges {665742=630270,1863699=604567,3121574=676394} 2023-08-27T20:52:30.558 INFO:tasks.rados.rados.0.smithi093.stdout:867: writing smithi093153278-437 from 665742 to 1296012 tid 1 2023-08-27T20:52:30.565 INFO:tasks.rados.rados.0.smithi093.stdout:867: writing smithi093153278-437 from 1863699 to 2468266 tid 2 2023-08-27T20:52:30.573 INFO:tasks.rados.rados.0.smithi093.stdout:867: writing smithi093153278-437 from 3121574 to 3797968 tid 3 2023-08-27T20:52:30.573 INFO:tasks.rados.rados.0.smithi093.stdout:868: read oid 366 snap -1 2023-08-27T20:52:30.573 INFO:tasks.rados.rados.0.smithi093.stdout:868: expect (ObjNum 586 snap 0 seq_num 586) 2023-08-27T20:52:30.573 INFO:tasks.rados.rados.0.smithi093.stdout:869: read oid 465 snap -1 2023-08-27T20:52:30.573 INFO:tasks.rados.rados.0.smithi093.stdout:869: expect (ObjNum 575 snap 0 seq_num 575) 2023-08-27T20:52:30.573 INFO:tasks.rados.rados.0.smithi093.stdout:870: delete oid 78 current snap is 0 2023-08-27T20:52:30.573 INFO:tasks.rados.rados.0.smithi093.stdout:864: finishing write tid 2 to smithi093153278-111 2023-08-27T20:52:30.574 INFO:tasks.rados.rados.0.smithi093.stdout:864: finishing write tid 3 to smithi093153278-111 2023-08-27T20:52:30.574 INFO:tasks.rados.rados.0.smithi093.stdout:864: finishing write tid 4 to smithi093153278-111 2023-08-27T20:52:30.574 INFO:tasks.rados.rados.0.smithi093.stdout:864: finishing write tid 5 to smithi093153278-111 2023-08-27T20:52:30.574 INFO:tasks.rados.rados.0.smithi093.stdout:866: expect (ObjNum 136 snap 0 seq_num 136) 2023-08-27T20:52:30.645 INFO:tasks.rados.rados.0.smithi093.stdout:864: finishing write tid 6 to smithi093153278-111 2023-08-27T20:52:30.645 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 111 v 162 (ObjNum 705 snap 0 seq_num 705) dirty exists 2023-08-27T20:52:30.645 INFO:tasks.rados.rados.0.smithi093.stdout:864: left oid 111 (ObjNum 705 snap 0 seq_num 705) 2023-08-27T20:52:30.645 INFO:tasks.rados.rados.0.smithi093.stdout:865: expect (ObjNum 238 snap 0 seq_num 238) 2023-08-27T20:52:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:30] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: pgmap v69: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 69 MiB/s rd, 9.6 MiB/s wr, 99 op/s 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: Upgrade: Target is version 18.2.0-187-g9cf20820 (unknown) 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: Upgrade: Target container is quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8, digests ['quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8'] 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1429055772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:30.707 INFO:tasks.rados.rados.0.smithi093.stdout:867: finishing write tid 1 to smithi093153278-437 2023-08-27T20:52:30.707 INFO:tasks.rados.rados.0.smithi093.stdout:867: finishing write tid 2 to smithi093153278-437 2023-08-27T20:52:30.707 INFO:tasks.rados.rados.0.smithi093.stdout:867: finishing write tid 3 to smithi093153278-437 2023-08-27T20:52:30.707 INFO:tasks.rados.rados.0.smithi093.stdout:867: finishing write tid 4 to smithi093153278-437 2023-08-27T20:52:30.707 INFO:tasks.rados.rados.0.smithi093.stdout:867: finishing write tid 5 to smithi093153278-437 2023-08-27T20:52:30.707 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 437 v 171 (ObjNum 706 snap 0 seq_num 706) dirty exists 2023-08-27T20:52:30.708 INFO:tasks.rados.rados.0.smithi093.stdout:867: left oid 437 (ObjNum 706 snap 0 seq_num 706) 2023-08-27T20:52:30.708 INFO:tasks.rados.rados.0.smithi093.stdout:868: expect (ObjNum 586 snap 0 seq_num 586) 2023-08-27T20:52:30.857 INFO:tasks.rados.rados.0.smithi093.stdout:869: expect (ObjNum 575 snap 0 seq_num 575) 2023-08-27T20:52:31.013 INFO:tasks.rados.rados.0.smithi093.stdout:864: done (6 left) 2023-08-27T20:52:31.013 INFO:tasks.rados.rados.0.smithi093.stdout:865: done (5 left) 2023-08-27T20:52:31.013 INFO:tasks.rados.rados.0.smithi093.stdout:866: done (4 left) 2023-08-27T20:52:31.013 INFO:tasks.rados.rados.0.smithi093.stdout:867: done (3 left) 2023-08-27T20:52:31.013 INFO:tasks.rados.rados.0.smithi093.stdout:868: done (2 left) 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:869: done (1 left) 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:870: done (0 left) 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:871: read oid 493 snap -1 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:871: expect (ObjNum 492 snap 0 seq_num 492) 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:872: read oid 469 snap -1 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:872: expect (ObjNum 468 snap 0 seq_num 468) 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:873: read oid 433 snap -1 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:873: expect (ObjNum 432 snap 0 seq_num 432) 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:874: read oid 223 snap -1 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:874: expect (ObjNum 222 snap 0 seq_num 222) 2023-08-27T20:52:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:875: read oid 276 snap -1 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:875: expect deleted 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:876: read oid 107 snap -1 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:876: expect (ObjNum 529 snap 0 seq_num 529) 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:877: read oid 160 snap -1 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:877: expect (ObjNum 159 snap 0 seq_num 159) 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:878: read oid 97 snap -1 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:878: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:879: write oid 310 current snap is 0 2023-08-27T20:52:31.015 INFO:tasks.rados.rados.0.smithi093.stdout:879: seq_num 708 ranges {699356=497121,1980663=608236,3176420=415291} 2023-08-27T20:52:31.020 INFO:tasks.rados.rados.0.smithi093.stdout:879: writing smithi093153278-310 from 699356 to 1196477 tid 1 2023-08-27T20:52:31.029 INFO:tasks.rados.rados.0.smithi093.stdout:879: writing smithi093153278-310 from 1980663 to 2588899 tid 2 2023-08-27T20:52:31.036 INFO:tasks.rados.rados.0.smithi093.stdout:879: writing smithi093153278-310 from 3176420 to 3591711 tid 3 2023-08-27T20:52:31.036 INFO:tasks.rados.rados.0.smithi093.stdout:880: read oid 471 snap -1 2023-08-27T20:52:31.036 INFO:tasks.rados.rados.0.smithi093.stdout:880: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:52:31.036 INFO:tasks.rados.rados.0.smithi093.stdout:881: write oid 134 current snap is 0 2023-08-27T20:52:31.036 INFO:tasks.rados.rados.0.smithi093.stdout:881: seq_num 709 ranges {716163=572370,2039145=468247,3203843=626563} 2023-08-27T20:52:31.044 INFO:tasks.rados.rados.0.smithi093.stdout:881: writing smithi093153278-134 from 716163 to 1288533 tid 1 2023-08-27T20:52:31.052 INFO:tasks.rados.rados.0.smithi093.stdout:881: writing smithi093153278-134 from 2039145 to 2507392 tid 2 2023-08-27T20:52:31.059 INFO:tasks.rados.rados.0.smithi093.stdout:881: writing smithi093153278-134 from 3203843 to 3830406 tid 3 2023-08-27T20:52:31.060 INFO:tasks.rados.rados.0.smithi093.stdout:882: write oid 363 current snap is 0 2023-08-27T20:52:31.060 INFO:tasks.rados.rados.0.smithi093.stdout:882: seq_num 710 ranges {732970=647619,2097627=611905,3231266=437835} 2023-08-27T20:52:31.067 INFO:tasks.rados.rados.0.smithi093.stdout:882: writing smithi093153278-363 from 732970 to 1380589 tid 1 2023-08-27T20:52:31.074 INFO:tasks.rados.rados.0.smithi093.stdout:882: writing smithi093153278-363 from 2097627 to 2709532 tid 2 2023-08-27T20:52:31.080 INFO:tasks.rados.rados.0.smithi093.stdout:882: writing smithi093153278-363 from 3231266 to 3669101 tid 3 2023-08-27T20:52:31.080 INFO:tasks.rados.rados.0.smithi093.stdout:883: write oid 290 current snap is 0 2023-08-27T20:52:31.080 INFO:tasks.rados.rados.0.smithi093.stdout:871: expect (ObjNum 492 snap 0 seq_num 492) 2023-08-27T20:52:31.151 INFO:tasks.rados.rados.0.smithi093.stdout:873: expect (ObjNum 432 snap 0 seq_num 432) 2023-08-27T20:52:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:872: expect (ObjNum 468 snap 0 seq_num 468) 2023-08-27T20:52:31.376 INFO:tasks.rados.rados.0.smithi093.stdout:877: expect (ObjNum 159 snap 0 seq_num 159) 2023-08-27T20:52:31.466 INFO:tasks.rados.rados.0.smithi093.stdout:879: finishing write tid 1 to smithi093153278-310 2023-08-27T20:52:31.466 INFO:tasks.rados.rados.0.smithi093.stdout:879: finishing write tid 2 to smithi093153278-310 2023-08-27T20:52:31.466 INFO:tasks.rados.rados.0.smithi093.stdout:883: seq_num 711 ranges {749777=722868,2156109=471916,3258689=649107} 2023-08-27T20:52:31.475 INFO:tasks.rados.rados.0.smithi093.stdout:883: writing smithi093153278-290 from 749777 to 1472645 tid 1 2023-08-27T20:52:31.483 INFO:tasks.rados.rados.0.smithi093.stdout:883: writing smithi093153278-290 from 2156109 to 2628025 tid 2 2023-08-27T20:52:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:31 smithi093 ceph-mon[111335]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi136 2023-08-27T20:52:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:31 smithi093 ceph-mon[118902]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi136 2023-08-27T20:52:31.492 INFO:tasks.rados.rados.0.smithi093.stdout:883: writing smithi093153278-290 from 3258689 to 3907796 tid 3 2023-08-27T20:52:31.492 INFO:tasks.rados.rados.0.smithi093.stdout:871: done (12 left) 2023-08-27T20:52:31.492 INFO:tasks.rados.rados.0.smithi093.stdout:872: done (11 left) 2023-08-27T20:52:31.492 INFO:tasks.rados.rados.0.smithi093.stdout:873: done (10 left) 2023-08-27T20:52:31.493 INFO:tasks.rados.rados.0.smithi093.stdout:875: done (9 left) 2023-08-27T20:52:31.493 INFO:tasks.rados.rados.0.smithi093.stdout:877: done (8 left) 2023-08-27T20:52:31.493 INFO:tasks.rados.rados.0.smithi093.stdout:884: delete oid 11 current snap is 0 2023-08-27T20:52:31.493 INFO:tasks.rados.rados.0.smithi093.stdout:874: expect (ObjNum 222 snap 0 seq_num 222) 2023-08-27T20:52:31.616 INFO:tasks.rados.rados.0.smithi093.stdout:876: expect (ObjNum 529 snap 0 seq_num 529) 2023-08-27T20:52:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:31 smithi136 ceph-mon[113531]: Upgrade: Pulling quay.ceph.io/ceph-ci/ceph@sha256:90e74e9dbaafb6b50d53d065fb7fed5018d4dfbdd16f414a8aaabb53a6e7a3a8 on smithi136 2023-08-27T20:52:31.752 INFO:tasks.rados.rados.0.smithi093.stdout:879: finishing write tid 3 to smithi093153278-310 2023-08-27T20:52:31.752 INFO:tasks.rados.rados.0.smithi093.stdout:879: finishing write tid 4 to smithi093153278-310 2023-08-27T20:52:31.752 INFO:tasks.rados.rados.0.smithi093.stdout:879: finishing write tid 5 to smithi093153278-310 2023-08-27T20:52:31.752 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 310 v 141 (ObjNum 708 snap 0 seq_num 708) dirty exists 2023-08-27T20:52:31.752 INFO:tasks.rados.rados.0.smithi093.stdout:879: left oid 310 (ObjNum 708 snap 0 seq_num 708) 2023-08-27T20:52:31.752 INFO:tasks.rados.rados.0.smithi093.stdout:878: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:52:31.863 INFO:tasks.rados.rados.0.smithi093.stdout:880: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:52:31.983 INFO:tasks.rados.rados.0.smithi093.stdout:881: finishing write tid 1 to smithi093153278-134 2023-08-27T20:52:31.983 INFO:tasks.rados.rados.0.smithi093.stdout:881: finishing write tid 2 to smithi093153278-134 2023-08-27T20:52:31.983 INFO:tasks.rados.rados.0.smithi093.stdout:881: finishing write tid 3 to smithi093153278-134 2023-08-27T20:52:31.983 INFO:tasks.rados.rados.0.smithi093.stdout:881: finishing write tid 4 to smithi093153278-134 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:881: finishing write tid 5 to smithi093153278-134 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 134 v 150 (ObjNum 709 snap 0 seq_num 709) dirty exists 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:881: left oid 134 (ObjNum 709 snap 0 seq_num 709) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:882: finishing write tid 1 to smithi093153278-363 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:874: done (8 left) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:876: done (7 left) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:878: done (6 left) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:879: done (5 left) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:880: done (4 left) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:881: done (3 left) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:884: done (2 left) 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:885: read oid 16 snap -1 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:885: expect deleted 2023-08-27T20:52:31.984 INFO:tasks.rados.rados.0.smithi093.stdout:886: write oid 198 current snap is 0 2023-08-27T20:52:31.985 INFO:tasks.rados.rados.0.smithi093.stdout:882: finishing write tid 2 to smithi093153278-363 2023-08-27T20:52:31.985 INFO:tasks.rados.rados.0.smithi093.stdout:886: seq_num 713 ranges {783391=473366,1756720=475585,2797182=788004} 2023-08-27T20:52:31.989 INFO:tasks.rados.rados.0.smithi093.stdout:886: writing smithi093153278-198 from 783391 to 1256757 tid 1 2023-08-27T20:52:31.995 INFO:tasks.rados.rados.0.smithi093.stdout:886: writing smithi093153278-198 from 1756720 to 2232305 tid 2 2023-08-27T20:52:32.002 INFO:tasks.rados.rados.0.smithi093.stdout:886: writing smithi093153278-198 from 2797182 to 3585186 tid 3 2023-08-27T20:52:32.002 INFO:tasks.rados.rados.0.smithi093.stdout:887: write oid 104 current snap is 0 2023-08-27T20:52:32.003 INFO:tasks.rados.rados.0.smithi093.stdout:887: seq_num 714 ranges {400198=548615,1415202=584996} 2023-08-27T20:52:32.008 INFO:tasks.rados.rados.0.smithi093.stdout:887: writing smithi093153278-104 from 400198 to 948813 tid 1 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:887: writing smithi093153278-104 from 1415202 to 2000198 tid 2 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:888: delete oid 386 current snap is 0 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:882: finishing write tid 3 to smithi093153278-363 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:882: finishing write tid 4 to smithi093153278-363 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:882: finishing write tid 5 to smithi093153278-363 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 204 (ObjNum 710 snap 0 seq_num 710) dirty exists 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:882: left oid 363 (ObjNum 710 snap 0 seq_num 710) 2023-08-27T20:52:32.014 INFO:tasks.rados.rados.0.smithi093.stdout:883: finishing write tid 1 to smithi093153278-290 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:883: finishing write tid 2 to smithi093153278-290 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:883: finishing write tid 3 to smithi093153278-290 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:883: finishing write tid 4 to smithi093153278-290 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:883: finishing write tid 5 to smithi093153278-290 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 290 v 132 (ObjNum 711 snap 0 seq_num 711) dirty exists 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:883: left oid 290 (ObjNum 711 snap 0 seq_num 711) 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:886: finishing write tid 1 to smithi093153278-198 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:886: finishing write tid 2 to smithi093153278-198 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:886: finishing write tid 3 to smithi093153278-198 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:886: finishing write tid 4 to smithi093153278-198 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:886: finishing write tid 5 to smithi093153278-198 2023-08-27T20:52:32.015 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 198 v 208 (ObjNum 713 snap 0 seq_num 713) dirty exists 2023-08-27T20:52:32.016 INFO:tasks.rados.rados.0.smithi093.stdout:886: left oid 198 (ObjNum 713 snap 0 seq_num 713) 2023-08-27T20:52:32.016 INFO:tasks.rados.rados.0.smithi093.stdout:887: finishing write tid 1 to smithi093153278-104 2023-08-27T20:52:32.016 INFO:tasks.rados.rados.0.smithi093.stdout:882: done (5 left) 2023-08-27T20:52:32.016 INFO:tasks.rados.rados.0.smithi093.stdout:883: done (4 left) 2023-08-27T20:52:32.016 INFO:tasks.rados.rados.0.smithi093.stdout:885: done (3 left) 2023-08-27T20:52:32.016 INFO:tasks.rados.rados.0.smithi093.stdout:886: done (2 left) 2023-08-27T20:52:32.017 INFO:tasks.rados.rados.0.smithi093.stdout:888: done (1 left) 2023-08-27T20:52:32.017 INFO:tasks.rados.rados.0.smithi093.stdout:889: write oid 86 current snap is 0 2023-08-27T20:52:32.017 INFO:tasks.rados.rados.0.smithi093.stdout:889: seq_num 716 ranges {433812=415466,1648519=385293} 2023-08-27T20:52:32.020 INFO:tasks.rados.rados.0.smithi093.stdout:889: writing smithi093153278-86 from 433812 to 849278 tid 1 2023-08-27T20:52:32.026 INFO:tasks.rados.rados.0.smithi093.stdout:889: writing smithi093153278-86 from 1648519 to 2033812 tid 2 2023-08-27T20:52:32.026 INFO:tasks.rados.rados.0.smithi093.stdout:890: write oid 484 current snap is 0 2023-08-27T20:52:32.026 INFO:tasks.rados.rados.0.smithi093.stdout:887: finishing write tid 2 to smithi093153278-104 2023-08-27T20:52:32.026 INFO:tasks.rados.rados.0.smithi093.stdout:887: finishing write tid 3 to smithi093153278-104 2023-08-27T20:52:32.027 INFO:tasks.rados.rados.0.smithi093.stdout:887: finishing write tid 4 to smithi093153278-104 2023-08-27T20:52:32.027 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 104 v 154 (ObjNum 714 snap 0 seq_num 714) dirty exists 2023-08-27T20:52:32.027 INFO:tasks.rados.rados.0.smithi093.stdout:887: left oid 104 (ObjNum 714 snap 0 seq_num 714) 2023-08-27T20:52:32.027 INFO:tasks.rados.rados.0.smithi093.stdout:889: finishing write tid 1 to smithi093153278-86 2023-08-27T20:52:32.027 INFO:tasks.rados.rados.0.smithi093.stdout:890: seq_num 717 ranges {450619=490715,1590648=459971} 2023-08-27T20:52:32.031 INFO:tasks.rados.rados.0.smithi093.stdout:890: writing smithi093153278-484 from 450619 to 941334 tid 1 2023-08-27T20:52:32.038 INFO:tasks.rados.rados.0.smithi093.stdout:890: writing smithi093153278-484 from 1590648 to 2050619 tid 2 2023-08-27T20:52:32.038 INFO:tasks.rados.rados.0.smithi093.stdout:887: done (2 left) 2023-08-27T20:52:32.038 INFO:tasks.rados.rados.0.smithi093.stdout:891: write oid 303 current snap is 0 2023-08-27T20:52:32.038 INFO:tasks.rados.rados.0.smithi093.stdout:889: finishing write tid 2 to smithi093153278-86 2023-08-27T20:52:32.038 INFO:tasks.rados.rados.0.smithi093.stdout:889: finishing write tid 3 to smithi093153278-86 2023-08-27T20:52:32.038 INFO:tasks.rados.rados.0.smithi093.stdout:889: finishing write tid 4 to smithi093153278-86 2023-08-27T20:52:32.038 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 86 v 146 (ObjNum 716 snap 0 seq_num 716) dirty exists 2023-08-27T20:52:32.039 INFO:tasks.rados.rados.0.smithi093.stdout:889: left oid 86 (ObjNum 716 snap 0 seq_num 716) 2023-08-27T20:52:32.039 INFO:tasks.rados.rados.0.smithi093.stdout:890: finishing write tid 1 to smithi093153278-484 2023-08-27T20:52:32.039 INFO:tasks.rados.rados.0.smithi093.stdout:891: seq_num 718 ranges {467426=565964,1649130=418296} 2023-08-27T20:52:32.043 INFO:tasks.rados.rados.0.smithi093.stdout:891: writing smithi093153278-303 from 467426 to 1033390 tid 1 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:891: writing smithi093153278-303 from 1649130 to 2067426 tid 2 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:889: done (2 left) 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:892: read oid 301 snap -1 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:892: expect (ObjNum 300 snap 0 seq_num 300) 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:893: write oid 367 current snap is 0 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:890: finishing write tid 2 to smithi093153278-484 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:890: finishing write tid 3 to smithi093153278-484 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:890: finishing write tid 4 to smithi093153278-484 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 484 v 211 (ObjNum 717 snap 0 seq_num 717) dirty exists 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:890: left oid 484 (ObjNum 717 snap 0 seq_num 717) 2023-08-27T20:52:32.050 INFO:tasks.rados.rados.0.smithi093.stdout:891: finishing write tid 1 to smithi093153278-303 2023-08-27T20:52:32.051 INFO:tasks.rados.rados.0.smithi093.stdout:893: seq_num 719 ranges {484233=641213,1707612=376621} 2023-08-27T20:52:32.056 INFO:tasks.rados.rados.0.smithi093.stdout:893: writing smithi093153278-367 from 484233 to 1125446 tid 1 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:893: writing smithi093153278-367 from 1707612 to 2084233 tid 2 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:890: done (3 left) 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:894: delete oid 383 current snap is 0 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:891: finishing write tid 2 to smithi093153278-303 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:891: finishing write tid 3 to smithi093153278-303 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:891: finishing write tid 4 to smithi093153278-303 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 303 v 165 (ObjNum 718 snap 0 seq_num 718) dirty exists 2023-08-27T20:52:32.062 INFO:tasks.rados.rados.0.smithi093.stdout:891: left oid 303 (ObjNum 718 snap 0 seq_num 718) 2023-08-27T20:52:32.063 INFO:tasks.rados.rados.0.smithi093.stdout:892: expect (ObjNum 300 snap 0 seq_num 300) 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:893: finishing write tid 1 to smithi093153278-367 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:891: done (3 left) 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:892: done (2 left) 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:894: done (1 left) 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:895: write oid 149 current snap is 0 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:893: finishing write tid 2 to smithi093153278-367 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:893: finishing write tid 3 to smithi093153278-367 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:893: finishing write tid 4 to smithi093153278-367 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 367 v 170 (ObjNum 719 snap 0 seq_num 719) dirty exists 2023-08-27T20:52:32.157 INFO:tasks.rados.rados.0.smithi093.stdout:893: left oid 367 (ObjNum 719 snap 0 seq_num 719) 2023-08-27T20:52:32.158 INFO:tasks.rados.rados.0.smithi093.stdout:895: seq_num 721 ranges {517847=791711,2108223=9624} 2023-08-27T20:52:32.163 INFO:tasks.rados.rados.0.smithi093.stdout:895: writing smithi093153278-149 from 517847 to 1309558 tid 1 2023-08-27T20:52:32.168 INFO:tasks.rados.rados.0.smithi093.stdout:895: writing smithi093153278-149 from 2108223 to 2117847 tid 2 2023-08-27T20:52:32.168 INFO:tasks.rados.rados.0.smithi093.stdout:893: done (1 left) 2023-08-27T20:52:32.168 INFO:tasks.rados.rados.0.smithi093.stdout:896: read oid 226 snap -1 2023-08-27T20:52:32.168 INFO:tasks.rados.rados.0.smithi093.stdout:896: expect (ObjNum 539 snap 0 seq_num 539) 2023-08-27T20:52:32.169 INFO:tasks.rados.rados.0.smithi093.stdout:897: write oid 197 current snap is 0 2023-08-27T20:52:32.169 INFO:tasks.rados.rados.0.smithi093.stdout:895: finishing write tid 1 to smithi093153278-149 2023-08-27T20:52:32.169 INFO:tasks.rados.rados.0.smithi093.stdout:897: seq_num 722 ranges {534654=466960,1766705=367949} 2023-08-27T20:52:32.173 INFO:tasks.rados.rados.0.smithi093.stdout:897: writing smithi093153278-197 from 534654 to 1001614 tid 1 2023-08-27T20:52:32.180 INFO:tasks.rados.rados.0.smithi093.stdout:897: writing smithi093153278-197 from 1766705 to 2134654 tid 2 2023-08-27T20:52:32.180 INFO:tasks.rados.rados.0.smithi093.stdout:898: read oid 402 snap -1 2023-08-27T20:52:32.180 INFO:tasks.rados.rados.0.smithi093.stdout:898: expect (ObjNum 401 snap 0 seq_num 401) 2023-08-27T20:52:32.181 INFO:tasks.rados.rados.0.smithi093.stdout:899: read oid 355 snap -1 2023-08-27T20:52:32.181 INFO:tasks.rados.rados.0.smithi093.stdout:899: expect (ObjNum 354 snap 0 seq_num 354) 2023-08-27T20:52:32.181 INFO:tasks.rados.rados.0.smithi093.stdout:900: write oid 326 current snap is 0 2023-08-27T20:52:32.181 INFO:tasks.rados.rados.0.smithi093.stdout:895: finishing write tid 2 to smithi093153278-149 2023-08-27T20:52:32.181 INFO:tasks.rados.rados.0.smithi093.stdout:900: seq_num 723 ranges {551461=658562,1941540=209921} 2023-08-27T20:52:32.186 INFO:tasks.rados.rados.0.smithi093.stdout:900: writing smithi093153278-326 from 551461 to 1210023 tid 1 2023-08-27T20:52:32.192 INFO:tasks.rados.rados.0.smithi093.stdout:900: writing smithi093153278-326 from 1941540 to 2151461 tid 2 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:901: read oid 295 snap -1 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:901: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:902: read oid 24 snap -1 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:902: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:903: write oid 49 current snap is 0 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:895: finishing write tid 3 to smithi093153278-149 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:895: finishing write tid 4 to smithi093153278-149 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 149 v 152 (ObjNum 721 snap 0 seq_num 721) dirty exists 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:895: left oid 149 (ObjNum 721 snap 0 seq_num 721) 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:897: finishing write tid 1 to smithi093153278-197 2023-08-27T20:52:32.193 INFO:tasks.rados.rados.0.smithi093.stdout:896: expect (ObjNum 539 snap 0 seq_num 539) 2023-08-27T20:52:32.317 INFO:tasks.rados.rados.0.smithi093.stdout:897: finishing write tid 2 to smithi093153278-197 2023-08-27T20:52:32.317 INFO:tasks.rados.rados.0.smithi093.stdout:897: finishing write tid 3 to smithi093153278-197 2023-08-27T20:52:32.317 INFO:tasks.rados.rados.0.smithi093.stdout:897: finishing write tid 4 to smithi093153278-197 2023-08-27T20:52:32.317 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 197 v 173 (ObjNum 722 snap 0 seq_num 722) dirty exists 2023-08-27T20:52:32.317 INFO:tasks.rados.rados.0.smithi093.stdout:897: left oid 197 (ObjNum 722 snap 0 seq_num 722) 2023-08-27T20:52:32.317 INFO:tasks.rados.rados.0.smithi093.stdout:900: finishing write tid 1 to smithi093153278-326 2023-08-27T20:52:32.317 INFO:tasks.rados.rados.0.smithi093.stdout:898: expect (ObjNum 401 snap 0 seq_num 401) 2023-08-27T20:52:32.403 INFO:tasks.rados.rados.0.smithi093.stdout:900: finishing write tid 2 to smithi093153278-326 2023-08-27T20:52:32.403 INFO:tasks.rados.rados.0.smithi093.stdout:900: finishing write tid 3 to smithi093153278-326 2023-08-27T20:52:32.403 INFO:tasks.rados.rados.0.smithi093.stdout:900: finishing write tid 4 to smithi093153278-326 2023-08-27T20:52:32.403 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 148 (ObjNum 723 snap 0 seq_num 723) dirty exists 2023-08-27T20:52:32.403 INFO:tasks.rados.rados.0.smithi093.stdout:900: left oid 326 (ObjNum 723 snap 0 seq_num 723) 2023-08-27T20:52:32.403 INFO:tasks.rados.rados.0.smithi093.stdout:899: expect (ObjNum 354 snap 0 seq_num 354) 2023-08-27T20:52:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:32 smithi093 ceph-mon[111335]: pgmap v70: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 69 MiB/s rd, 9.6 MiB/s wr, 99 op/s 2023-08-27T20:52:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1643876059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:32 smithi093 ceph-mon[118902]: pgmap v70: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 69 MiB/s rd, 9.6 MiB/s wr, 99 op/s 2023-08-27T20:52:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1643876059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:32.524 INFO:tasks.rados.rados.0.smithi093.stdout:901: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:52:32.615 INFO:tasks.rados.rados.0.smithi093.stdout:902: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:52:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:32 smithi136 ceph-mon[113531]: pgmap v70: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 69 MiB/s rd, 9.6 MiB/s wr, 99 op/s 2023-08-27T20:52:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1643876059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:32.690 INFO:tasks.rados.rados.0.smithi093.stdout:903: seq_num 724 ranges {568268=733811,2000022=168246} 2023-08-27T20:52:32.697 INFO:tasks.rados.rados.0.smithi093.stdout:903: writing smithi093153278-49 from 568268 to 1302079 tid 1 2023-08-27T20:52:32.702 INFO:tasks.rados.rados.0.smithi093.stdout:903: writing smithi093153278-49 from 2000022 to 2168268 tid 2 2023-08-27T20:52:32.702 INFO:tasks.rados.rados.0.smithi093.stdout:895: done (8 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:896: done (7 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:897: done (6 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:898: done (5 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:899: done (4 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:900: done (3 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:901: done (2 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:902: done (1 left) 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:904: write oid 48 current snap is 0 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:903: finishing write tid 1 to smithi093153278-49 2023-08-27T20:52:32.703 INFO:tasks.rados.rados.0.smithi093.stdout:904: seq_num 725 ranges {585075=409060,1542151=642924} 2023-08-27T20:52:32.707 INFO:tasks.rados.rados.0.smithi093.stdout:904: writing smithi093153278-48 from 585075 to 994135 tid 1 2023-08-27T20:52:32.714 INFO:tasks.rados.rados.0.smithi093.stdout:904: writing smithi093153278-48 from 1542151 to 2185075 tid 2 2023-08-27T20:52:32.714 INFO:tasks.rados.rados.0.smithi093.stdout:905: delete oid 215 current snap is 0 2023-08-27T20:52:32.714 INFO:tasks.rados.rados.0.smithi093.stdout:903: finishing write tid 2 to smithi093153278-49 2023-08-27T20:52:32.714 INFO:tasks.rados.rados.0.smithi093.stdout:903: finishing write tid 3 to smithi093153278-49 2023-08-27T20:52:32.715 INFO:tasks.rados.rados.0.smithi093.stdout:903: finishing write tid 4 to smithi093153278-49 2023-08-27T20:52:32.715 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 49 v 99 (ObjNum 724 snap 0 seq_num 724) dirty exists 2023-08-27T20:52:32.715 INFO:tasks.rados.rados.0.smithi093.stdout:903: left oid 49 (ObjNum 724 snap 0 seq_num 724) 2023-08-27T20:52:32.715 INFO:tasks.rados.rados.0.smithi093.stdout:904: finishing write tid 1 to smithi093153278-48 2023-08-27T20:52:32.716 INFO:tasks.rados.rados.0.smithi093.stdout:903: done (2 left) 2023-08-27T20:52:32.716 INFO:tasks.rados.rados.0.smithi093.stdout:905: done (1 left) 2023-08-27T20:52:32.716 INFO:tasks.rados.rados.0.smithi093.stdout:906: write oid 51 current snap is 0 2023-08-27T20:52:32.716 INFO:tasks.rados.rados.0.smithi093.stdout:906: seq_num 727 ranges {618689=559558,1659115=559574} 2023-08-27T20:52:32.722 INFO:tasks.rados.rados.0.smithi093.stdout:906: writing smithi093153278-51 from 618689 to 1178247 tid 1 2023-08-27T20:52:32.728 INFO:tasks.rados.rados.0.smithi093.stdout:906: writing smithi093153278-51 from 1659115 to 2218689 tid 2 2023-08-27T20:52:32.728 INFO:tasks.rados.rados.0.smithi093.stdout:907: read oid 252 snap -1 2023-08-27T20:52:32.728 INFO:tasks.rados.rados.0.smithi093.stdout:907: expect (ObjNum 587 snap 0 seq_num 587) 2023-08-27T20:52:32.729 INFO:tasks.rados.rados.0.smithi093.stdout:908: write oid 152 current snap is 0 2023-08-27T20:52:32.729 INFO:tasks.rados.rados.0.smithi093.stdout:904: finishing write tid 2 to smithi093153278-48 2023-08-27T20:52:32.729 INFO:tasks.rados.rados.0.smithi093.stdout:908: seq_num 728 ranges {635496=634807,1717597=517899} 2023-08-27T20:52:32.735 INFO:tasks.rados.rados.0.smithi093.stdout:908: writing smithi093153278-152 from 635496 to 1270303 tid 1 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:908: writing smithi093153278-152 from 1717597 to 2235496 tid 2 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:909: read oid 75 snap -1 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:909: expect (ObjNum 688 snap 0 seq_num 688) 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:910: read oid 39 snap -1 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:910: expect (ObjNum 38 snap 0 seq_num 38) 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:911: delete oid 233 current snap is 0 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:904: finishing write tid 3 to smithi093153278-48 2023-08-27T20:52:32.742 INFO:tasks.rados.rados.0.smithi093.stdout:904: finishing write tid 4 to smithi093153278-48 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 48 v 144 (ObjNum 725 snap 0 seq_num 725) dirty exists 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:904: left oid 48 (ObjNum 725 snap 0 seq_num 725) 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:906: finishing write tid 1 to smithi093153278-51 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:906: finishing write tid 2 to smithi093153278-51 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:906: finishing write tid 3 to smithi093153278-51 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:906: finishing write tid 4 to smithi093153278-51 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 51 v 135 (ObjNum 727 snap 0 seq_num 727) dirty exists 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:906: left oid 51 (ObjNum 727 snap 0 seq_num 727) 2023-08-27T20:52:32.743 INFO:tasks.rados.rados.0.smithi093.stdout:907: expect (ObjNum 587 snap 0 seq_num 587) 2023-08-27T20:52:32.920 INFO:tasks.rados.rados.0.smithi093.stdout:908: finishing write tid 1 to smithi093153278-152 2023-08-27T20:52:32.920 INFO:tasks.rados.rados.0.smithi093.stdout:908: finishing write tid 2 to smithi093153278-152 2023-08-27T20:52:32.921 INFO:tasks.rados.rados.0.smithi093.stdout:908: finishing write tid 3 to smithi093153278-152 2023-08-27T20:52:32.921 INFO:tasks.rados.rados.0.smithi093.stdout:908: finishing write tid 4 to smithi093153278-152 2023-08-27T20:52:32.921 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 152 v 164 (ObjNum 728 snap 0 seq_num 728) dirty exists 2023-08-27T20:52:32.921 INFO:tasks.rados.rados.0.smithi093.stdout:908: left oid 152 (ObjNum 728 snap 0 seq_num 728) 2023-08-27T20:52:32.921 INFO:tasks.rados.rados.0.smithi093.stdout:910: expect (ObjNum 38 snap 0 seq_num 38) 2023-08-27T20:52:33.001 INFO:tasks.rados.rados.0.smithi093.stdout:909: expect (ObjNum 688 snap 0 seq_num 688) 2023-08-27T20:52:33.111 INFO:tasks.rados.rados.0.smithi093.stdout:904: done (6 left) 2023-08-27T20:52:33.111 INFO:tasks.rados.rados.0.smithi093.stdout:906: done (5 left) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:907: done (4 left) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:908: done (3 left) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:909: done (2 left) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:910: done (1 left) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:911: done (0 left) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:912: read oid 417 snap -1 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:912: expect (ObjNum 416 snap 0 seq_num 416) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:913: read oid 126 snap -1 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:913: expect (ObjNum 527 snap 0 seq_num 527) 2023-08-27T20:52:33.112 INFO:tasks.rados.rados.0.smithi093.stdout:914: delete oid 412 current snap is 0 2023-08-27T20:52:33.116 INFO:tasks.rados.rados.0.smithi093.stdout:912: expect (ObjNum 416 snap 0 seq_num 416) 2023-08-27T20:52:33.207 INFO:tasks.rados.rados.0.smithi093.stdout:913: expect (ObjNum 527 snap 0 seq_num 527) 2023-08-27T20:52:33.328 INFO:tasks.rados.rados.0.smithi093.stdout:912: done (2 left) 2023-08-27T20:52:33.329 INFO:tasks.rados.rados.0.smithi093.stdout:913: done (1 left) 2023-08-27T20:52:33.329 INFO:tasks.rados.rados.0.smithi093.stdout:914: done (0 left) 2023-08-27T20:52:33.329 INFO:tasks.rados.rados.0.smithi093.stdout:915: delete oid 391 current snap is 0 2023-08-27T20:52:33.330 INFO:tasks.rados.rados.0.smithi093.stdout:915: done (0 left) 2023-08-27T20:52:33.330 INFO:tasks.rados.rados.0.smithi093.stdout:916: write oid 55 current snap is 0 2023-08-27T20:52:33.330 INFO:tasks.rados.rados.0.smithi093.stdout:916: seq_num 732 ranges {702724=652156,1951525=351199} 2023-08-27T20:52:33.337 INFO:tasks.rados.rados.0.smithi093.stdout:916: writing smithi093153278-55 from 702724 to 1354880 tid 1 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:916: writing smithi093153278-55 from 1951525 to 2302724 tid 2 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:917: read oid 281 snap -1 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:917: expect (ObjNum 280 snap 0 seq_num 280) 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:918: read oid 103 snap -1 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:918: expect (ObjNum 102 snap 0 seq_num 102) 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:916: finishing write tid 1 to smithi093153278-55 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:919: read oid 189 snap -1 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:919: expect (ObjNum 188 snap 0 seq_num 188) 2023-08-27T20:52:33.343 INFO:tasks.rados.rados.0.smithi093.stdout:920: write oid 330 current snap is 0 2023-08-27T20:52:33.344 INFO:tasks.rados.rados.0.smithi093.stdout:920: seq_num 733 ranges {719531=727405,1893654=425877} 2023-08-27T20:52:33.351 INFO:tasks.rados.rados.0.smithi093.stdout:920: writing smithi093153278-330 from 719531 to 1446936 tid 1 2023-08-27T20:52:33.356 INFO:tasks.rados.rados.0.smithi093.stdout:920: writing smithi093153278-330 from 1893654 to 2319531 tid 2 2023-08-27T20:52:33.356 INFO:tasks.rados.rados.0.smithi093.stdout:921: write oid 278 current snap is 0 2023-08-27T20:52:33.357 INFO:tasks.rados.rados.0.smithi093.stdout:916: finishing write tid 2 to smithi093153278-55 2023-08-27T20:52:33.357 INFO:tasks.rados.rados.0.smithi093.stdout:916: finishing write tid 3 to smithi093153278-55 2023-08-27T20:52:33.357 INFO:tasks.rados.rados.0.smithi093.stdout:916: finishing write tid 4 to smithi093153278-55 2023-08-27T20:52:33.357 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 55 v 154 (ObjNum 732 snap 0 seq_num 732) dirty exists 2023-08-27T20:52:33.357 INFO:tasks.rados.rados.0.smithi093.stdout:916: left oid 55 (ObjNum 732 snap 0 seq_num 732) 2023-08-27T20:52:33.357 INFO:tasks.rados.rados.0.smithi093.stdout:918: expect (ObjNum 102 snap 0 seq_num 102) 2023-08-27T20:52:33.471 INFO:tasks.rados.rados.0.smithi093.stdout:917: expect (ObjNum 280 snap 0 seq_num 280) 2023-08-27T20:52:33.557 INFO:tasks.rados.rados.0.smithi093.stdout:919: expect (ObjNum 188 snap 0 seq_num 188) 2023-08-27T20:52:33.657 INFO:tasks.rados.rados.0.smithi093.stdout:920: finishing write tid 1 to smithi093153278-330 2023-08-27T20:52:33.658 INFO:tasks.rados.rados.0.smithi093.stdout:920: finishing write tid 2 to smithi093153278-330 2023-08-27T20:52:33.658 INFO:tasks.rados.rados.0.smithi093.stdout:920: finishing write tid 3 to smithi093153278-330 2023-08-27T20:52:33.658 INFO:tasks.rados.rados.0.smithi093.stdout:920: finishing write tid 4 to smithi093153278-330 2023-08-27T20:52:33.658 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 330 v 157 (ObjNum 733 snap 0 seq_num 733) dirty exists 2023-08-27T20:52:33.658 INFO:tasks.rados.rados.0.smithi093.stdout:920: left oid 330 (ObjNum 733 snap 0 seq_num 733) 2023-08-27T20:52:33.658 INFO:tasks.rados.rados.0.smithi093.stdout:921: seq_num 734 ranges {736338=402654,1552136=539580} 2023-08-27T20:52:33.663 INFO:tasks.rados.rados.0.smithi093.stdout:921: writing smithi093153278-278 from 736338 to 1138992 tid 1 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:921: writing smithi093153278-278 from 1552136 to 2091716 tid 2 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:916: done (5 left) 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:917: done (4 left) 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:918: done (3 left) 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:919: done (2 left) 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:920: done (1 left) 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:922: write oid 477 current snap is 0 2023-08-27T20:52:33.669 INFO:tasks.rados.rados.0.smithi093.stdout:922: seq_num 735 ranges {753145=477903,2010618=342527} 2023-08-27T20:52:33.675 INFO:tasks.rados.rados.0.smithi093.stdout:922: writing smithi093153278-477 from 753145 to 1231048 tid 1 2023-08-27T20:52:33.682 INFO:tasks.rados.rados.0.smithi093.stdout:922: writing smithi093153278-477 from 2010618 to 2353145 tid 2 2023-08-27T20:52:33.682 INFO:tasks.rados.rados.0.smithi093.stdout:923: read oid 199 snap -1 2023-08-27T20:52:33.682 INFO:tasks.rados.rados.0.smithi093.stdout:923: expect (ObjNum 198 snap 0 seq_num 198) 2023-08-27T20:52:33.682 INFO:tasks.rados.rados.0.smithi093.stdout:924: delete oid 261 current snap is 0 2023-08-27T20:52:33.682 INFO:tasks.rados.rados.0.smithi093.stdout:921: finishing write tid 1 to smithi093153278-278 2023-08-27T20:52:33.682 INFO:tasks.rados.rados.0.smithi093.stdout:921: finishing write tid 2 to smithi093153278-278 2023-08-27T20:52:33.682 INFO:tasks.rados.rados.0.smithi093.stdout:921: finishing write tid 3 to smithi093153278-278 2023-08-27T20:52:33.683 INFO:tasks.rados.rados.0.smithi093.stdout:921: finishing write tid 4 to smithi093153278-278 2023-08-27T20:52:33.683 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 278 v 214 (ObjNum 734 snap 0 seq_num 734) dirty exists 2023-08-27T20:52:33.683 INFO:tasks.rados.rados.0.smithi093.stdout:921: left oid 278 (ObjNum 734 snap 0 seq_num 734) 2023-08-27T20:52:33.683 INFO:tasks.rados.rados.0.smithi093.stdout:922: finishing write tid 1 to smithi093153278-477 2023-08-27T20:52:33.683 INFO:tasks.rados.rados.0.smithi093.stdout:921: done (3 left) 2023-08-27T20:52:33.684 INFO:tasks.rados.rados.0.smithi093.stdout:924: done (2 left) 2023-08-27T20:52:33.684 INFO:tasks.rados.rados.0.smithi093.stdout:925: read oid 10 snap -1 2023-08-27T20:52:33.684 INFO:tasks.rados.rados.0.smithi093.stdout:925: expect (ObjNum 500 snap 0 seq_num 500) 2023-08-27T20:52:33.684 INFO:tasks.rados.rados.0.smithi093.stdout:926: write oid 359 current snap is 0 2023-08-27T20:52:33.684 INFO:tasks.rados.rados.0.smithi093.stdout:926: seq_num 737 ranges {786759=628401,2011229=375530} 2023-08-27T20:52:33.691 INFO:tasks.rados.rados.0.smithi093.stdout:926: writing smithi093153278-359 from 786759 to 1415160 tid 1 2023-08-27T20:52:33.697 INFO:tasks.rados.rados.0.smithi093.stdout:926: writing smithi093153278-359 from 2011229 to 2386759 tid 2 2023-08-27T20:52:33.697 INFO:tasks.rados.rados.0.smithi093.stdout:927: read oid 495 snap -1 2023-08-27T20:52:33.697 INFO:tasks.rados.rados.0.smithi093.stdout:927: expect (ObjNum 625 snap 0 seq_num 625) 2023-08-27T20:52:33.697 INFO:tasks.rados.rados.0.smithi093.stdout:928: write oid 82 current snap is 0 2023-08-27T20:52:33.697 INFO:tasks.rados.rados.0.smithi093.stdout:922: finishing write tid 2 to smithi093153278-477 2023-08-27T20:52:33.698 INFO:tasks.rados.rados.0.smithi093.stdout:928: seq_num 738 ranges {403566=420003,1386064=546918} 2023-08-27T20:52:33.702 INFO:tasks.rados.rados.0.smithi093.stdout:928: writing smithi093153278-82 from 403566 to 823569 tid 1 2023-08-27T20:52:33.708 INFO:tasks.rados.rados.0.smithi093.stdout:928: writing smithi093153278-82 from 1386064 to 1932982 tid 2 2023-08-27T20:52:33.708 INFO:tasks.rados.rados.0.smithi093.stdout:929: write oid 475 current snap is 0 2023-08-27T20:52:33.708 INFO:tasks.rados.rados.0.smithi093.stdout:922: finishing write tid 3 to smithi093153278-477 2023-08-27T20:52:33.708 INFO:tasks.rados.rados.0.smithi093.stdout:929: seq_num 739 ranges {420373=495252,1444546=690576} 2023-08-27T20:52:33.713 INFO:tasks.rados.rados.0.smithi093.stdout:929: writing smithi093153278-475 from 420373 to 915625 tid 1 2023-08-27T20:52:33.720 INFO:tasks.rados.rados.0.smithi093.stdout:929: writing smithi093153278-475 from 1444546 to 2135122 tid 2 2023-08-27T20:52:33.720 INFO:tasks.rados.rados.0.smithi093.stdout:930: write oid 167 current snap is 0 2023-08-27T20:52:33.720 INFO:tasks.rados.rados.0.smithi093.stdout:922: finishing write tid 4 to smithi093153278-477 2023-08-27T20:52:33.720 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 477 v 180 (ObjNum 735 snap 0 seq_num 735) dirty exists 2023-08-27T20:52:33.720 INFO:tasks.rados.rados.0.smithi093.stdout:922: left oid 477 (ObjNum 735 snap 0 seq_num 735) 2023-08-27T20:52:33.721 INFO:tasks.rados.rados.0.smithi093.stdout:930: seq_num 740 ranges {437180=570501,1503028=550587} 2023-08-27T20:52:33.725 INFO:tasks.rados.rados.0.smithi093.stdout:930: writing smithi093153278-167 from 437180 to 1007681 tid 1 2023-08-27T20:52:33.731 INFO:tasks.rados.rados.0.smithi093.stdout:930: writing smithi093153278-167 from 1503028 to 2053615 tid 2 2023-08-27T20:52:33.731 INFO:tasks.rados.rados.0.smithi093.stdout:922: done (7 left) 2023-08-27T20:52:33.731 INFO:tasks.rados.rados.0.smithi093.stdout:931: write oid 478 current snap is 0 2023-08-27T20:52:33.732 INFO:tasks.rados.rados.0.smithi093.stdout:931: seq_num 741 ranges {453987=645750,1845157=608830} 2023-08-27T20:52:33.737 INFO:tasks.rados.rados.0.smithi093.stdout:931: writing smithi093153278-478 from 453987 to 1099737 tid 1 2023-08-27T20:52:33.745 INFO:tasks.rados.rados.0.smithi093.stdout:931: writing smithi093153278-478 from 1845157 to 2453987 tid 2 2023-08-27T20:52:33.745 INFO:tasks.rados.rados.0.smithi093.stdout:932: write oid 390 current snap is 0 2023-08-27T20:52:33.745 INFO:tasks.rados.rados.0.smithi093.stdout:923: expect (ObjNum 198 snap 0 seq_num 198) 2023-08-27T20:52:33.851 INFO:tasks.rados.rados.0.smithi093.stdout:925: expect (ObjNum 500 snap 0 seq_num 500) 2023-08-27T20:52:33.959 INFO:tasks.rados.rados.0.smithi093.stdout:926: finishing write tid 1 to smithi093153278-359 2023-08-27T20:52:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:932: seq_num 742 ranges {470794=720999,1903639=554256} 2023-08-27T20:52:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:932: writing smithi093153278-390 from 470794 to 1191793 tid 1 2023-08-27T20:52:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:932: writing smithi093153278-390 from 1903639 to 2457895 tid 2 2023-08-27T20:52:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:923: done (8 left) 2023-08-27T20:52:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:925: done (7 left) 2023-08-27T20:52:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:933: read oid 125 snap -1 2023-08-27T20:52:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:933: expect (ObjNum 124 snap 0 seq_num 124) 2023-08-27T20:52:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:934: delete oid 487 current snap is 0 2023-08-27T20:52:33.994 INFO:tasks.rados.rados.0.smithi093.stdout:926: finishing write tid 2 to smithi093153278-359 2023-08-27T20:52:33.994 INFO:tasks.rados.rados.0.smithi093.stdout:926: finishing write tid 3 to smithi093153278-359 2023-08-27T20:52:33.994 INFO:tasks.rados.rados.0.smithi093.stdout:927: expect (ObjNum 625 snap 0 seq_num 625) 2023-08-27T20:52:34.086 INFO:tasks.rados.rados.0.smithi093.stdout:926: finishing write tid 4 to smithi093153278-359 2023-08-27T20:52:34.086 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 359 v 217 (ObjNum 737 snap 0 seq_num 737) dirty exists 2023-08-27T20:52:34.086 INFO:tasks.rados.rados.0.smithi093.stdout:926: left oid 359 (ObjNum 737 snap 0 seq_num 737) 2023-08-27T20:52:34.086 INFO:tasks.rados.rados.0.smithi093.stdout:928: finishing write tid 1 to smithi093153278-82 2023-08-27T20:52:34.086 INFO:tasks.rados.rados.0.smithi093.stdout:926: done (8 left) 2023-08-27T20:52:34.087 INFO:tasks.rados.rados.0.smithi093.stdout:927: done (7 left) 2023-08-27T20:52:34.087 INFO:tasks.rados.rados.0.smithi093.stdout:934: done (6 left) 2023-08-27T20:52:34.087 INFO:tasks.rados.rados.0.smithi093.stdout:935: read oid 452 snap -1 2023-08-27T20:52:34.087 INFO:tasks.rados.rados.0.smithi093.stdout:935: expect (ObjNum 451 snap 0 seq_num 451) 2023-08-27T20:52:34.087 INFO:tasks.rados.rados.0.smithi093.stdout:936: write oid 80 current snap is 0 2023-08-27T20:52:34.087 INFO:tasks.rados.rados.0.smithi093.stdout:928: finishing write tid 2 to smithi093153278-82 2023-08-27T20:52:34.087 INFO:tasks.rados.rados.0.smithi093.stdout:936: seq_num 744 ranges {504408=471497,1620603=557925} 2023-08-27T20:52:34.091 INFO:tasks.rados.rados.0.smithi093.stdout:936: writing smithi093153278-80 from 504408 to 975905 tid 1 2023-08-27T20:52:34.099 INFO:tasks.rados.rados.0.smithi093.stdout:936: writing smithi093153278-80 from 1620603 to 2178528 tid 2 2023-08-27T20:52:34.099 INFO:tasks.rados.rados.0.smithi093.stdout:937: read oid 280 snap -1 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:937: expect (ObjNum 279 snap 0 seq_num 279) 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:938: write oid 256 current snap is 0 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:928: finishing write tid 3 to smithi093153278-82 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:928: finishing write tid 4 to smithi093153278-82 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 82 v 155 (ObjNum 738 snap 0 seq_num 738) dirty exists 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:928: left oid 82 (ObjNum 738 snap 0 seq_num 738) 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:929: finishing write tid 1 to smithi093153278-475 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:929: finishing write tid 2 to smithi093153278-475 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:929: finishing write tid 3 to smithi093153278-475 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:929: finishing write tid 4 to smithi093153278-475 2023-08-27T20:52:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 475 v 147 (ObjNum 739 snap 0 seq_num 739) dirty exists 2023-08-27T20:52:34.101 INFO:tasks.rados.rados.0.smithi093.stdout:929: left oid 475 (ObjNum 739 snap 0 seq_num 739) 2023-08-27T20:52:34.101 INFO:tasks.rados.rados.0.smithi093.stdout:930: finishing write tid 1 to smithi093153278-167 2023-08-27T20:52:34.101 INFO:tasks.rados.rados.0.smithi093.stdout:930: finishing write tid 2 to smithi093153278-167 2023-08-27T20:52:34.101 INFO:tasks.rados.rados.0.smithi093.stdout:930: finishing write tid 3 to smithi093153278-167 2023-08-27T20:52:34.101 INFO:tasks.rados.rados.0.smithi093.stdout:938: seq_num 745 ranges {521215=546746,1562732=701583} 2023-08-27T20:52:34.105 INFO:tasks.rados.rados.0.smithi093.stdout:938: writing smithi093153278-256 from 521215 to 1067961 tid 1 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:938: writing smithi093153278-256 from 1562732 to 2264315 tid 2 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:928: done (9 left) 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:929: done (8 left) 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:939: write oid 217 current snap is 0 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:930: finishing write tid 4 to smithi093153278-167 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 167 v 183 (ObjNum 740 snap 0 seq_num 740) dirty exists 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:930: left oid 167 (ObjNum 740 snap 0 seq_num 740) 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:931: finishing write tid 1 to smithi093153278-478 2023-08-27T20:52:34.112 INFO:tasks.rados.rados.0.smithi093.stdout:931: finishing write tid 2 to smithi093153278-478 2023-08-27T20:52:34.113 INFO:tasks.rados.rados.0.smithi093.stdout:931: finishing write tid 3 to smithi093153278-478 2023-08-27T20:52:34.113 INFO:tasks.rados.rados.0.smithi093.stdout:939: seq_num 746 ranges {538022=738348,1737567=561594} 2023-08-27T20:52:34.119 INFO:tasks.rados.rados.0.smithi093.stdout:939: writing smithi093153278-217 from 538022 to 1276370 tid 1 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:939: writing smithi093153278-217 from 1737567 to 2299161 tid 2 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:930: done (8 left) 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:940: read oid 182 snap -1 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:940: expect (ObjNum 181 snap 0 seq_num 181) 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:941: write oid 49 current snap is 0 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:931: finishing write tid 4 to smithi093153278-478 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 168 (ObjNum 741 snap 0 seq_num 741) dirty exists 2023-08-27T20:52:34.180 INFO:tasks.rados.rados.0.smithi093.stdout:931: left oid 478 (ObjNum 741 snap 0 seq_num 741) 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:932: finishing write tid 1 to smithi093153278-390 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:932: finishing write tid 2 to smithi093153278-390 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:941: seq_num 747 ranges {554829=413597,1396049=705252} 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:941: writing smithi093153278-49 from 554829 to 968426 tid 1 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:941: writing smithi093153278-49 from 1396049 to 2101301 tid 2 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:931: done (9 left) 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:942: write oid 118 current snap is 0 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:932: finishing write tid 3 to smithi093153278-390 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:932: finishing write tid 4 to smithi093153278-390 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 390 v 150 (ObjNum 742 snap 0 seq_num 742) dirty exists 2023-08-27T20:52:34.181 INFO:tasks.rados.rados.0.smithi093.stdout:932: left oid 390 (ObjNum 742 snap 0 seq_num 742) 2023-08-27T20:52:34.182 INFO:tasks.rados.rados.0.smithi093.stdout:933: expect (ObjNum 124 snap 0 seq_num 124) 2023-08-27T20:52:34.201 INFO:tasks.rados.rados.0.smithi093.stdout:936: finishing write tid 1 to smithi093153278-80 2023-08-27T20:52:34.202 INFO:tasks.rados.rados.0.smithi093.stdout:935: expect (ObjNum 451 snap 0 seq_num 451) 2023-08-27T20:52:34.315 INFO:tasks.rados.rados.0.smithi093.stdout:936: finishing write tid 2 to smithi093153278-80 2023-08-27T20:52:34.315 INFO:tasks.rados.rados.0.smithi093.stdout:936: finishing write tid 3 to smithi093153278-80 2023-08-27T20:52:34.316 INFO:tasks.rados.rados.0.smithi093.stdout:936: finishing write tid 4 to smithi093153278-80 2023-08-27T20:52:34.316 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 80 v 150 (ObjNum 744 snap 0 seq_num 744) dirty exists 2023-08-27T20:52:34.316 INFO:tasks.rados.rados.0.smithi093.stdout:936: left oid 80 (ObjNum 744 snap 0 seq_num 744) 2023-08-27T20:52:34.316 INFO:tasks.rados.rados.0.smithi093.stdout:937: expect (ObjNum 279 snap 0 seq_num 279) 2023-08-27T20:52:34.397 INFO:tasks.rados.rados.0.smithi093.stdout:938: finishing write tid 1 to smithi093153278-256 2023-08-27T20:52:34.397 INFO:tasks.rados.rados.0.smithi093.stdout:938: finishing write tid 2 to smithi093153278-256 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:938: finishing write tid 3 to smithi093153278-256 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:938: finishing write tid 4 to smithi093153278-256 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 256 v 153 (ObjNum 745 snap 0 seq_num 745) dirty exists 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:938: left oid 256 (ObjNum 745 snap 0 seq_num 745) 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:939: finishing write tid 1 to smithi093153278-217 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:939: finishing write tid 2 to smithi093153278-217 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:939: finishing write tid 3 to smithi093153278-217 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:939: finishing write tid 4 to smithi093153278-217 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 217 v 174 (ObjNum 746 snap 0 seq_num 746) dirty exists 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:939: left oid 217 (ObjNum 746 snap 0 seq_num 746) 2023-08-27T20:52:34.398 INFO:tasks.rados.rados.0.smithi093.stdout:940: expect (ObjNum 181 snap 0 seq_num 181) 2023-08-27T20:52:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:34 smithi093 ceph-mon[111335]: pgmap v71: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 14 MiB/s wr, 145 op/s 2023-08-27T20:52:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4091642098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:34 smithi093 ceph-mon[118902]: pgmap v71: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 14 MiB/s wr, 145 op/s 2023-08-27T20:52:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4091642098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:34.493 INFO:tasks.rados.rados.0.smithi093.stdout:941: finishing write tid 1 to smithi093153278-49 2023-08-27T20:52:34.493 INFO:tasks.rados.rados.0.smithi093.stdout:941: finishing write tid 2 to smithi093153278-49 2023-08-27T20:52:34.493 INFO:tasks.rados.rados.0.smithi093.stdout:941: finishing write tid 3 to smithi093153278-49 2023-08-27T20:52:34.493 INFO:tasks.rados.rados.0.smithi093.stdout:941: finishing write tid 4 to smithi093153278-49 2023-08-27T20:52:34.493 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 49 v 102 (ObjNum 747 snap 0 seq_num 747) dirty exists 2023-08-27T20:52:34.493 INFO:tasks.rados.rados.0.smithi093.stdout:941: left oid 49 (ObjNum 747 snap 0 seq_num 747) 2023-08-27T20:52:34.493 INFO:tasks.rados.rados.0.smithi093.stdout:942: seq_num 748 ranges {571636=488846,1854531=565263} 2023-08-27T20:52:34.498 INFO:tasks.rados.rados.0.smithi093.stdout:942: writing smithi093153278-118 from 571636 to 1060482 tid 1 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:942: writing smithi093153278-118 from 1854531 to 2419794 tid 2 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:932: done (9 left) 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:933: done (8 left) 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:935: done (7 left) 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:936: done (6 left) 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:937: done (5 left) 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:938: done (4 left) 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:939: done (3 left) 2023-08-27T20:52:34.507 INFO:tasks.rados.rados.0.smithi093.stdout:940: done (2 left) 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:941: done (1 left) 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:943: read oid 495 snap -1 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:943: expect (ObjNum 625 snap 0 seq_num 625) 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:944: read oid 332 snap -1 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:944: expect (ObjNum 331 snap 0 seq_num 331) 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:945: read oid 469 snap -1 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:945: expect (ObjNum 468 snap 0 seq_num 468) 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:946: write oid 465 current snap is 0 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:942: finishing write tid 1 to smithi093153278-118 2023-08-27T20:52:34.508 INFO:tasks.rados.rados.0.smithi093.stdout:946: seq_num 749 ranges {588443=564095,1796660=708921} 2023-08-27T20:52:34.513 INFO:tasks.rados.rados.0.smithi093.stdout:946: writing smithi093153278-465 from 588443 to 1152538 tid 1 2023-08-27T20:52:34.521 INFO:tasks.rados.rados.0.smithi093.stdout:946: writing smithi093153278-465 from 1796660 to 2505581 tid 2 2023-08-27T20:52:34.521 INFO:tasks.rados.rados.0.smithi093.stdout:947: write oid 62 current snap is 0 2023-08-27T20:52:34.521 INFO:tasks.rados.rados.0.smithi093.stdout:947: seq_num 750 ranges {605250=639344,1855142=452579} 2023-08-27T20:52:34.527 INFO:tasks.rados.rados.0.smithi093.stdout:947: writing smithi093153278-62 from 605250 to 1244594 tid 1 2023-08-27T20:52:34.534 INFO:tasks.rados.rados.0.smithi093.stdout:947: writing smithi093153278-62 from 1855142 to 2307721 tid 2 2023-08-27T20:52:34.534 INFO:tasks.rados.rados.0.smithi093.stdout:948: write oid 229 current snap is 0 2023-08-27T20:52:34.534 INFO:tasks.rados.rados.0.smithi093.stdout:943: expect (ObjNum 625 snap 0 seq_num 625) 2023-08-27T20:52:34.640 INFO:tasks.rados.rados.0.smithi093.stdout:945: expect (ObjNum 468 snap 0 seq_num 468) 2023-08-27T20:52:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:34 smithi136 ceph-mon[113531]: pgmap v71: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 14 MiB/s wr, 145 op/s 2023-08-27T20:52:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4091642098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:34.758 INFO:tasks.rados.rados.0.smithi093.stdout:942: finishing write tid 2 to smithi093153278-118 2023-08-27T20:52:34.758 INFO:tasks.rados.rados.0.smithi093.stdout:942: finishing write tid 3 to smithi093153278-118 2023-08-27T20:52:34.758 INFO:tasks.rados.rados.0.smithi093.stdout:942: finishing write tid 4 to smithi093153278-118 2023-08-27T20:52:34.758 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 118 v 156 (ObjNum 748 snap 0 seq_num 748) dirty exists 2023-08-27T20:52:34.758 INFO:tasks.rados.rados.0.smithi093.stdout:942: left oid 118 (ObjNum 748 snap 0 seq_num 748) 2023-08-27T20:52:34.758 INFO:tasks.rados.rados.0.smithi093.stdout:944: expect (ObjNum 331 snap 0 seq_num 331) 2023-08-27T20:52:34.866 INFO:tasks.rados.rados.0.smithi093.stdout:946: finishing write tid 1 to smithi093153278-465 2023-08-27T20:52:34.866 INFO:tasks.rados.rados.0.smithi093.stdout:946: finishing write tid 2 to smithi093153278-465 2023-08-27T20:52:34.867 INFO:tasks.rados.rados.0.smithi093.stdout:946: finishing write tid 3 to smithi093153278-465 2023-08-27T20:52:34.867 INFO:tasks.rados.rados.0.smithi093.stdout:946: finishing write tid 4 to smithi093153278-465 2023-08-27T20:52:34.867 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 465 v 220 (ObjNum 749 snap 0 seq_num 749) dirty exists 2023-08-27T20:52:34.867 INFO:tasks.rados.rados.0.smithi093.stdout:946: left oid 465 (ObjNum 749 snap 0 seq_num 749) 2023-08-27T20:52:34.867 INFO:tasks.rados.rados.0.smithi093.stdout:947: finishing write tid 1 to smithi093153278-62 2023-08-27T20:52:34.867 INFO:tasks.rados.rados.0.smithi093.stdout:948: seq_num 751 ranges {622057=714593,1913624=708433} 2023-08-27T20:52:34.873 INFO:tasks.rados.rados.0.smithi093.stdout:948: writing smithi093153278-229 from 622057 to 1336650 tid 1 2023-08-27T20:52:34.881 INFO:tasks.rados.rados.0.smithi093.stdout:948: writing smithi093153278-229 from 1913624 to 2622057 tid 2 2023-08-27T20:52:34.881 INFO:tasks.rados.rados.0.smithi093.stdout:942: done (6 left) 2023-08-27T20:52:34.881 INFO:tasks.rados.rados.0.smithi093.stdout:943: done (5 left) 2023-08-27T20:52:34.881 INFO:tasks.rados.rados.0.smithi093.stdout:944: done (4 left) 2023-08-27T20:52:34.881 INFO:tasks.rados.rados.0.smithi093.stdout:945: done (3 left) 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:946: done (2 left) 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:949: read oid 443 snap -1 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:949: expect (ObjNum 442 snap 0 seq_num 442) 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:950: write oid 242 current snap is 0 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:947: finishing write tid 2 to smithi093153278-62 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:947: finishing write tid 3 to smithi093153278-62 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:947: finishing write tid 4 to smithi093153278-62 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 62 v 158 (ObjNum 750 snap 0 seq_num 750) dirty exists 2023-08-27T20:52:34.882 INFO:tasks.rados.rados.0.smithi093.stdout:947: left oid 62 (ObjNum 750 snap 0 seq_num 750) 2023-08-27T20:52:34.883 INFO:tasks.rados.rados.0.smithi093.stdout:948: finishing write tid 1 to smithi093153278-229 2023-08-27T20:52:34.883 INFO:tasks.rados.rados.0.smithi093.stdout:950: seq_num 752 ranges {638864=789842,1972106=456248} 2023-08-27T20:52:34.889 INFO:tasks.rados.rados.0.smithi093.stdout:950: writing smithi093153278-242 from 638864 to 1428706 tid 1 2023-08-27T20:52:34.895 INFO:tasks.rados.rados.0.smithi093.stdout:950: writing smithi093153278-242 from 1972106 to 2428354 tid 2 2023-08-27T20:52:34.895 INFO:tasks.rados.rados.0.smithi093.stdout:947: done (3 left) 2023-08-27T20:52:34.895 INFO:tasks.rados.rados.0.smithi093.stdout:951: write oid 402 current snap is 0 2023-08-27T20:52:34.895 INFO:tasks.rados.rados.0.smithi093.stdout:948: finishing write tid 2 to smithi093153278-229 2023-08-27T20:52:34.895 INFO:tasks.rados.rados.0.smithi093.stdout:951: seq_num 753 ranges {655671=581444,2030588=625083} 2023-08-27T20:52:34.901 INFO:tasks.rados.rados.0.smithi093.stdout:951: writing smithi093153278-402 from 655671 to 1237115 tid 1 2023-08-27T20:52:34.909 INFO:tasks.rados.rados.0.smithi093.stdout:951: writing smithi093153278-402 from 2030588 to 2655671 tid 2 2023-08-27T20:52:34.909 INFO:tasks.rados.rados.0.smithi093.stdout:952: read oid 272 snap -1 2023-08-27T20:52:34.909 INFO:tasks.rados.rados.0.smithi093.stdout:952: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:34.909 INFO:tasks.rados.rados.0.smithi093.stdout:953: write oid 96 current snap is 0 2023-08-27T20:52:34.909 INFO:tasks.rados.rados.0.smithi093.stdout:948: finishing write tid 3 to smithi093153278-229 2023-08-27T20:52:34.910 INFO:tasks.rados.rados.0.smithi093.stdout:953: seq_num 754 ranges {672478=656693,2089070=459917} 2023-08-27T20:52:34.916 INFO:tasks.rados.rados.0.smithi093.stdout:953: writing smithi093153278-96 from 672478 to 1329171 tid 1 2023-08-27T20:52:34.923 INFO:tasks.rados.rados.0.smithi093.stdout:953: writing smithi093153278-96 from 2089070 to 2548987 tid 2 2023-08-27T20:52:34.923 INFO:tasks.rados.rados.0.smithi093.stdout:954: write oid 498 current snap is 0 2023-08-27T20:52:34.924 INFO:tasks.rados.rados.0.smithi093.stdout:948: finishing write tid 4 to smithi093153278-229 2023-08-27T20:52:34.924 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 229 v 161 (ObjNum 751 snap 0 seq_num 751) dirty exists 2023-08-27T20:52:34.924 INFO:tasks.rados.rados.0.smithi093.stdout:948: left oid 229 (ObjNum 751 snap 0 seq_num 751) 2023-08-27T20:52:34.924 INFO:tasks.rados.rados.0.smithi093.stdout:949: expect (ObjNum 442 snap 0 seq_num 442) 2023-08-27T20:52:35.030 INFO:tasks.rados.rados.0.smithi093.stdout:950: finishing write tid 1 to smithi093153278-242 2023-08-27T20:52:35.030 INFO:tasks.rados.rados.0.smithi093.stdout:950: finishing write tid 2 to smithi093153278-242 2023-08-27T20:52:35.030 INFO:tasks.rados.rados.0.smithi093.stdout:950: finishing write tid 3 to smithi093153278-242 2023-08-27T20:52:35.030 INFO:tasks.rados.rados.0.smithi093.stdout:950: finishing write tid 4 to smithi093153278-242 2023-08-27T20:52:35.030 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 242 v 176 (ObjNum 752 snap 0 seq_num 752) dirty exists 2023-08-27T20:52:35.030 INFO:tasks.rados.rados.0.smithi093.stdout:950: left oid 242 (ObjNum 752 snap 0 seq_num 752) 2023-08-27T20:52:35.031 INFO:tasks.rados.rados.0.smithi093.stdout:951: finishing write tid 1 to smithi093153278-402 2023-08-27T20:52:35.031 INFO:tasks.rados.rados.0.smithi093.stdout:954: seq_num 755 ranges {689285=731942,2147552=541733} 2023-08-27T20:52:35.037 INFO:tasks.rados.rados.0.smithi093.stdout:954: writing smithi093153278-498 from 689285 to 1421227 tid 1 2023-08-27T20:52:35.045 INFO:tasks.rados.rados.0.smithi093.stdout:954: writing smithi093153278-498 from 2147552 to 2689285 tid 2 2023-08-27T20:52:35.045 INFO:tasks.rados.rados.0.smithi093.stdout:948: done (6 left) 2023-08-27T20:52:35.045 INFO:tasks.rados.rados.0.smithi093.stdout:949: done (5 left) 2023-08-27T20:52:35.045 INFO:tasks.rados.rados.0.smithi093.stdout:950: done (4 left) 2023-08-27T20:52:35.045 INFO:tasks.rados.rados.0.smithi093.stdout:955: write oid 334 current snap is 0 2023-08-27T20:52:35.045 INFO:tasks.rados.rados.0.smithi093.stdout:955: seq_num 756 ranges {706092=407191,1806034=463586} 2023-08-27T20:52:35.051 INFO:tasks.rados.rados.0.smithi093.stdout:955: writing smithi093153278-334 from 706092 to 1113283 tid 1 2023-08-27T20:52:35.057 INFO:tasks.rados.rados.0.smithi093.stdout:955: writing smithi093153278-334 from 1806034 to 2269620 tid 2 2023-08-27T20:52:35.057 INFO:tasks.rados.rados.0.smithi093.stdout:956: read oid 385 snap -1 2023-08-27T20:52:35.058 INFO:tasks.rados.rados.0.smithi093.stdout:956: expect (ObjNum 384 snap 0 seq_num 384) 2023-08-27T20:52:35.058 INFO:tasks.rados.rados.0.smithi093.stdout:957: write oid 331 current snap is 0 2023-08-27T20:52:35.058 INFO:tasks.rados.rados.0.smithi093.stdout:957: seq_num 757 ranges {722899=482440,1748163=723597} 2023-08-27T20:52:35.064 INFO:tasks.rados.rados.0.smithi093.stdout:957: writing smithi093153278-331 from 722899 to 1205339 tid 1 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:957: writing smithi093153278-331 from 1748163 to 2471760 tid 2 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:958: delete oid 335 current snap is 0 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:951: finishing write tid 2 to smithi093153278-402 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:951: finishing write tid 3 to smithi093153278-402 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:951: finishing write tid 4 to smithi093153278-402 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 402 v 153 (ObjNum 753 snap 0 seq_num 753) dirty exists 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:951: left oid 402 (ObjNum 753 snap 0 seq_num 753) 2023-08-27T20:52:35.072 INFO:tasks.rados.rados.0.smithi093.stdout:952: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:35.153 INFO:tasks.rados.rados.0.smithi093.stdout:953: finishing write tid 1 to smithi093153278-96 2023-08-27T20:52:35.153 INFO:tasks.rados.rados.0.smithi093.stdout:953: finishing write tid 2 to smithi093153278-96 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:953: finishing write tid 3 to smithi093153278-96 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:953: finishing write tid 4 to smithi093153278-96 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 96 v 169 (ObjNum 754 snap 0 seq_num 754) dirty exists 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:953: left oid 96 (ObjNum 754 snap 0 seq_num 754) 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:951: done (7 left) 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:952: done (6 left) 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:953: done (5 left) 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:958: done (4 left) 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:959: write oid 259 current snap is 0 2023-08-27T20:52:35.154 INFO:tasks.rados.rados.0.smithi093.stdout:959: seq_num 759 ranges {756513=632938,1865127=727266} 2023-08-27T20:52:35.160 INFO:tasks.rados.rados.0.smithi093.stdout:959: writing smithi093153278-259 from 756513 to 1389451 tid 1 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:959: writing smithi093153278-259 from 1865127 to 2592393 tid 2 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:960: write oid 31 current snap is 0 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:954: finishing write tid 1 to smithi093153278-498 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:954: finishing write tid 2 to smithi093153278-498 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:954: finishing write tid 3 to smithi093153278-498 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:954: finishing write tid 4 to smithi093153278-498 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 498 v 105 (ObjNum 755 snap 0 seq_num 755) dirty exists 2023-08-27T20:52:35.168 INFO:tasks.rados.rados.0.smithi093.stdout:954: left oid 498 (ObjNum 755 snap 0 seq_num 755) 2023-08-27T20:52:35.169 INFO:tasks.rados.rados.0.smithi093.stdout:955: finishing write tid 1 to smithi093153278-334 2023-08-27T20:52:35.169 INFO:tasks.rados.rados.0.smithi093.stdout:955: finishing write tid 2 to smithi093153278-334 2023-08-27T20:52:35.169 INFO:tasks.rados.rados.0.smithi093.stdout:955: finishing write tid 3 to smithi093153278-334 2023-08-27T20:52:35.169 INFO:tasks.rados.rados.0.smithi093.stdout:955: finishing write tid 4 to smithi093153278-334 2023-08-27T20:52:35.169 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 334 v 160 (ObjNum 756 snap 0 seq_num 756) dirty exists 2023-08-27T20:52:35.169 INFO:tasks.rados.rados.0.smithi093.stdout:955: left oid 334 (ObjNum 756 snap 0 seq_num 756) 2023-08-27T20:52:35.169 INFO:tasks.rados.rados.0.smithi093.stdout:956: expect (ObjNum 384 snap 0 seq_num 384) 2023-08-27T20:52:35.244 INFO:tasks.rados.rados.0.smithi093.stdout:957: finishing write tid 1 to smithi093153278-331 2023-08-27T20:52:35.244 INFO:tasks.rados.rados.0.smithi093.stdout:957: finishing write tid 2 to smithi093153278-331 2023-08-27T20:52:35.244 INFO:tasks.rados.rados.0.smithi093.stdout:957: finishing write tid 3 to smithi093153278-331 2023-08-27T20:52:35.245 INFO:tasks.rados.rados.0.smithi093.stdout:957: finishing write tid 4 to smithi093153278-331 2023-08-27T20:52:35.245 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 331 v 223 (ObjNum 757 snap 0 seq_num 757) dirty exists 2023-08-27T20:52:35.245 INFO:tasks.rados.rados.0.smithi093.stdout:957: left oid 331 (ObjNum 757 snap 0 seq_num 757) 2023-08-27T20:52:35.245 INFO:tasks.rados.rados.0.smithi093.stdout:960: seq_num 760 ranges {773320=708187,1923609=470924} 2023-08-27T20:52:35.252 INFO:tasks.rados.rados.0.smithi093.stdout:960: writing smithi093153278-31 from 773320 to 1481507 tid 1 2023-08-27T20:52:35.257 INFO:tasks.rados.rados.0.smithi093.stdout:960: writing smithi093153278-31 from 1923609 to 2394533 tid 2 2023-08-27T20:52:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:954: done (5 left) 2023-08-27T20:52:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:955: done (4 left) 2023-08-27T20:52:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:956: done (3 left) 2023-08-27T20:52:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:957: done (2 left) 2023-08-27T20:52:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:961: write oid 205 current snap is 0 2023-08-27T20:52:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:959: finishing write tid 1 to smithi093153278-259 2023-08-27T20:52:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:961: seq_num 761 ranges {790127=499789,1698444=730935} 2023-08-27T20:52:35.264 INFO:tasks.rados.rados.0.smithi093.stdout:961: writing smithi093153278-205 from 790127 to 1289916 tid 1 2023-08-27T20:52:35.271 INFO:tasks.rados.rados.0.smithi093.stdout:961: writing smithi093153278-205 from 1698444 to 2429379 tid 2 2023-08-27T20:52:35.271 INFO:tasks.rados.rados.0.smithi093.stdout:962: write oid 444 current snap is 0 2023-08-27T20:52:35.271 INFO:tasks.rados.rados.0.smithi093.stdout:959: finishing write tid 2 to smithi093153278-259 2023-08-27T20:52:35.271 INFO:tasks.rados.rados.0.smithi093.stdout:959: finishing write tid 3 to smithi093153278-259 2023-08-27T20:52:35.272 INFO:tasks.rados.rados.0.smithi093.stdout:959: finishing write tid 4 to smithi093153278-259 2023-08-27T20:52:35.272 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 259 v 152 (ObjNum 759 snap 0 seq_num 759) dirty exists 2023-08-27T20:52:35.272 INFO:tasks.rados.rados.0.smithi093.stdout:959: left oid 259 (ObjNum 759 snap 0 seq_num 759) 2023-08-27T20:52:35.272 INFO:tasks.rados.rados.0.smithi093.stdout:960: finishing write tid 1 to smithi093153278-31 2023-08-27T20:52:35.272 INFO:tasks.rados.rados.0.smithi093.stdout:960: finishing write tid 2 to smithi093153278-31 2023-08-27T20:52:35.272 INFO:tasks.rados.rados.0.smithi093.stdout:960: finishing write tid 3 to smithi093153278-31 2023-08-27T20:52:35.272 INFO:tasks.rados.rados.0.smithi093.stdout:962: seq_num 762 ranges {406934=575038,1640573=474593,2642791=164143} 2023-08-27T20:52:35.276 INFO:tasks.rados.rados.0.smithi093.stdout:962: writing smithi093153278-444 from 406934 to 981972 tid 1 2023-08-27T20:52:35.282 INFO:tasks.rados.rados.0.smithi093.stdout:962: writing smithi093153278-444 from 1640573 to 2115166 tid 2 2023-08-27T20:52:35.287 INFO:tasks.rados.rados.0.smithi093.stdout:962: writing smithi093153278-444 from 2642791 to 2806934 tid 3 2023-08-27T20:52:35.287 INFO:tasks.rados.rados.0.smithi093.stdout:959: done (3 left) 2023-08-27T20:52:35.287 INFO:tasks.rados.rados.0.smithi093.stdout:963: write oid 447 current snap is 0 2023-08-27T20:52:35.287 INFO:tasks.rados.rados.0.smithi093.stdout:960: finishing write tid 4 to smithi093153278-31 2023-08-27T20:52:35.287 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 31 v 155 (ObjNum 760 snap 0 seq_num 760) dirty exists 2023-08-27T20:52:35.288 INFO:tasks.rados.rados.0.smithi093.stdout:960: left oid 31 (ObjNum 760 snap 0 seq_num 760) 2023-08-27T20:52:35.288 INFO:tasks.rados.rados.0.smithi093.stdout:961: finishing write tid 1 to smithi093153278-205 2023-08-27T20:52:35.288 INFO:tasks.rados.rados.0.smithi093.stdout:963: seq_num 763 ranges {423741=650287,1699055=618251} 2023-08-27T20:52:35.292 INFO:tasks.rados.rados.0.smithi093.stdout:963: writing smithi093153278-447 from 423741 to 1074028 tid 1 2023-08-27T20:52:35.300 INFO:tasks.rados.rados.0.smithi093.stdout:963: writing smithi093153278-447 from 1699055 to 2317306 tid 2 2023-08-27T20:52:35.300 INFO:tasks.rados.rados.0.smithi093.stdout:960: done (3 left) 2023-08-27T20:52:35.300 INFO:tasks.rados.rados.0.smithi093.stdout:964: read oid 111 snap -1 2023-08-27T20:52:35.300 INFO:tasks.rados.rados.0.smithi093.stdout:964: expect (ObjNum 705 snap 0 seq_num 705) 2023-08-27T20:52:35.300 INFO:tasks.rados.rados.0.smithi093.stdout:965: read oid 135 snap -1 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:965: expect (ObjNum 134 snap 0 seq_num 134) 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:966: write oid 328 current snap is 0 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:961: finishing write tid 2 to smithi093153278-205 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:961: finishing write tid 3 to smithi093153278-205 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:961: finishing write tid 4 to smithi093153278-205 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 205 v 108 (ObjNum 761 snap 0 seq_num 761) dirty exists 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:961: left oid 205 (ObjNum 761 snap 0 seq_num 761) 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:962: finishing write tid 1 to smithi093153278-444 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:962: finishing write tid 2 to smithi093153278-444 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:962: finishing write tid 3 to smithi093153278-444 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:962: finishing write tid 4 to smithi093153278-444 2023-08-27T20:52:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:962: finishing write tid 5 to smithi093153278-444 2023-08-27T20:52:35.302 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 444 v 137 (ObjNum 762 snap 0 seq_num 762) dirty exists 2023-08-27T20:52:35.302 INFO:tasks.rados.rados.0.smithi093.stdout:962: left oid 444 (ObjNum 762 snap 0 seq_num 762) 2023-08-27T20:52:35.302 INFO:tasks.rados.rados.0.smithi093.stdout:963: finishing write tid 1 to smithi093153278-447 2023-08-27T20:52:35.302 INFO:tasks.rados.rados.0.smithi093.stdout:966: seq_num 764 ranges {440548=725536,1757537=478262,2813990=26558} 2023-08-27T20:52:35.306 INFO:tasks.rados.rados.0.smithi093.stdout:966: writing smithi093153278-328 from 440548 to 1166084 tid 1 2023-08-27T20:52:35.312 INFO:tasks.rados.rados.0.smithi093.stdout:966: writing smithi093153278-328 from 1757537 to 2235799 tid 2 2023-08-27T20:52:35.316 INFO:tasks.rados.rados.0.smithi093.stdout:966: writing smithi093153278-328 from 2813990 to 2840548 tid 3 2023-08-27T20:52:35.316 INFO:tasks.rados.rados.0.smithi093.stdout:961: done (5 left) 2023-08-27T20:52:35.316 INFO:tasks.rados.rados.0.smithi093.stdout:962: done (4 left) 2023-08-27T20:52:35.316 INFO:tasks.rados.rados.0.smithi093.stdout:967: read oid 192 snap -1 2023-08-27T20:52:35.316 INFO:tasks.rados.rados.0.smithi093.stdout:967: expect (ObjNum 191 snap 0 seq_num 191) 2023-08-27T20:52:35.316 INFO:tasks.rados.rados.0.smithi093.stdout:968: write oid 62 current snap is 0 2023-08-27T20:52:35.316 INFO:tasks.rados.rados.0.smithi093.stdout:968: seq_num 765 ranges {457355=400785,1416019=621920,2725060=132295} 2023-08-27T20:52:35.321 INFO:tasks.rados.rados.0.smithi093.stdout:968: writing smithi093153278-62 from 457355 to 858140 tid 1 2023-08-27T20:52:35.328 INFO:tasks.rados.rados.0.smithi093.stdout:968: writing smithi093153278-62 from 1416019 to 2037939 tid 2 2023-08-27T20:52:35.333 INFO:tasks.rados.rados.0.smithi093.stdout:968: writing smithi093153278-62 from 2725060 to 2857355 tid 3 2023-08-27T20:52:35.333 INFO:tasks.rados.rados.0.smithi093.stdout:969: read oid 188 snap -1 2023-08-27T20:52:35.333 INFO:tasks.rados.rados.0.smithi093.stdout:969: expect (ObjNum 187 snap 0 seq_num 187) 2023-08-27T20:52:35.333 INFO:tasks.rados.rados.0.smithi093.stdout:970: write oid 430 current snap is 0 2023-08-27T20:52:35.334 INFO:tasks.rados.rados.0.smithi093.stdout:963: finishing write tid 2 to smithi093153278-447 2023-08-27T20:52:35.334 INFO:tasks.rados.rados.0.smithi093.stdout:963: finishing write tid 3 to smithi093153278-447 2023-08-27T20:52:35.334 INFO:tasks.rados.rados.0.smithi093.stdout:963: finishing write tid 4 to smithi093153278-447 2023-08-27T20:52:35.334 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 447 v 186 (ObjNum 763 snap 0 seq_num 763) dirty exists 2023-08-27T20:52:35.334 INFO:tasks.rados.rados.0.smithi093.stdout:963: left oid 447 (ObjNum 763 snap 0 seq_num 763) 2023-08-27T20:52:35.334 INFO:tasks.rados.rados.0.smithi093.stdout:964: expect (ObjNum 705 snap 0 seq_num 705) 2023-08-27T20:52:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:965: expect (ObjNum 134 snap 0 seq_num 134) 2023-08-27T20:52:35.569 INFO:tasks.rados.rados.0.smithi093.stdout:966: finishing write tid 1 to smithi093153278-328 2023-08-27T20:52:35.569 INFO:tasks.rados.rados.0.smithi093.stdout:966: finishing write tid 2 to smithi093153278-328 2023-08-27T20:52:35.570 INFO:tasks.rados.rados.0.smithi093.stdout:966: finishing write tid 3 to smithi093153278-328 2023-08-27T20:52:35.570 INFO:tasks.rados.rados.0.smithi093.stdout:966: finishing write tid 4 to smithi093153278-328 2023-08-27T20:52:35.570 INFO:tasks.rados.rados.0.smithi093.stdout:970: seq_num 766 ranges {474162=476034,1358148=481931,2352483=465303} 2023-08-27T20:52:35.575 INFO:tasks.rados.rados.0.smithi093.stdout:970: writing smithi093153278-430 from 474162 to 950196 tid 1 2023-08-27T20:52:35.579 INFO:tasks.rados.rados.0.smithi093.stdout:970: writing smithi093153278-430 from 1358148 to 1840079 tid 2 2023-08-27T20:52:35.585 INFO:tasks.rados.rados.0.smithi093.stdout:970: writing smithi093153278-430 from 2352483 to 2817786 tid 3 2023-08-27T20:52:35.585 INFO:tasks.rados.rados.0.smithi093.stdout:963: done (7 left) 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:964: done (6 left) 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:965: done (5 left) 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:971: read oid 5 snap -1 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:971: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:972: read oid 216 snap -1 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:972: expect deleted 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:973: delete oid 208 current snap is 0 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:966: finishing write tid 5 to smithi093153278-328 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 172 (ObjNum 764 snap 0 seq_num 764) dirty exists 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:966: left oid 328 (ObjNum 764 snap 0 seq_num 764) 2023-08-27T20:52:35.586 INFO:tasks.rados.rados.0.smithi093.stdout:968: finishing write tid 1 to smithi093153278-62 2023-08-27T20:52:35.587 INFO:tasks.rados.rados.0.smithi093.stdout:967: expect (ObjNum 191 snap 0 seq_num 191) 2023-08-27T20:52:35.681 INFO:tasks.rados.rados.0.smithi093.stdout:968: finishing write tid 2 to smithi093153278-62 2023-08-27T20:52:35.681 INFO:tasks.rados.rados.0.smithi093.stdout:968: finishing write tid 3 to smithi093153278-62 2023-08-27T20:52:35.681 INFO:tasks.rados.rados.0.smithi093.stdout:968: finishing write tid 4 to smithi093153278-62 2023-08-27T20:52:35.681 INFO:tasks.rados.rados.0.smithi093.stdout:968: finishing write tid 5 to smithi093153278-62 2023-08-27T20:52:35.682 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 62 v 165 (ObjNum 765 snap 0 seq_num 765) dirty exists 2023-08-27T20:52:35.682 INFO:tasks.rados.rados.0.smithi093.stdout:968: left oid 62 (ObjNum 765 snap 0 seq_num 765) 2023-08-27T20:52:35.682 INFO:tasks.rados.rados.0.smithi093.stdout:969: expect (ObjNum 187 snap 0 seq_num 187) 2023-08-27T20:52:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:52:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:35] "GET /metrics HTTP/1.1" 200 242379 "" "Prometheus/2.43.0" 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:970: finishing write tid 1 to smithi093153278-430 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:970: finishing write tid 2 to smithi093153278-430 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:970: finishing write tid 3 to smithi093153278-430 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:970: finishing write tid 4 to smithi093153278-430 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:970: finishing write tid 5 to smithi093153278-430 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 430 v 227 (ObjNum 766 snap 0 seq_num 766) dirty exists 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:970: left oid 430 (ObjNum 766 snap 0 seq_num 766) 2023-08-27T20:52:35.781 INFO:tasks.rados.rados.0.smithi093.stdout:971: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:966: done (7 left) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:967: done (6 left) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:968: done (5 left) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:969: done (4 left) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:970: done (3 left) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:971: done (2 left) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:972: done (1 left) 2023-08-27T20:52:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:973: done (0 left) 2023-08-27T20:52:35.873 INFO:tasks.rados.rados.0.smithi093.stdout:974: read oid 180 snap -1 2023-08-27T20:52:35.873 INFO:tasks.rados.rados.0.smithi093.stdout:974: expect (ObjNum 687 snap 0 seq_num 687) 2023-08-27T20:52:35.873 INFO:tasks.rados.rados.0.smithi093.stdout:975: read oid 480 snap -1 2023-08-27T20:52:35.873 INFO:tasks.rados.rados.0.smithi093.stdout:975: expect (ObjNum 479 snap 0 seq_num 479) 2023-08-27T20:52:35.873 INFO:tasks.rados.rados.0.smithi093.stdout:976: write oid 78 current snap is 0 2023-08-27T20:52:35.873 INFO:tasks.rados.rados.0.smithi093.stdout:976: seq_num 768 ranges {507776=742885,1991465=485600} 2023-08-27T20:52:35.878 INFO:tasks.rados.rados.0.smithi093.stdout:976: writing smithi093153278-78 from 507776 to 1250661 tid 1 2023-08-27T20:52:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:976: writing smithi093153278-78 from 1991465 to 2477065 tid 2 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:977: read oid 414 snap -1 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:977: expect (ObjNum 413 snap 0 seq_num 413) 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:978: read oid 94 snap -1 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:978: expect (ObjNum 93 snap 0 seq_num 93) 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:979: read oid 81 snap -1 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:979: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:980: read oid 229 snap -1 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:980: expect (ObjNum 751 snap 0 seq_num 751) 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:981: read oid 248 snap -1 2023-08-27T20:52:35.887 INFO:tasks.rados.rados.0.smithi093.stdout:981: expect (ObjNum 247 snap 0 seq_num 247) 2023-08-27T20:52:35.888 INFO:tasks.rados.rados.0.smithi093.stdout:982: write oid 474 current snap is 0 2023-08-27T20:52:35.888 INFO:tasks.rados.rados.0.smithi093.stdout:982: seq_num 769 ranges {524583=418134,1649947=629258,2834752=89831} 2023-08-27T20:52:35.891 INFO:tasks.rados.rados.0.smithi093.stdout:982: writing smithi093153278-474 from 524583 to 942717 tid 1 2023-08-27T20:52:35.899 INFO:tasks.rados.rados.0.smithi093.stdout:982: writing smithi093153278-474 from 1649947 to 2279205 tid 2 2023-08-27T20:52:35.902 INFO:tasks.rados.rados.0.smithi093.stdout:982: writing smithi093153278-474 from 2834752 to 2924583 tid 3 2023-08-27T20:52:35.902 INFO:tasks.rados.rados.0.smithi093.stdout:983: write oid 189 current snap is 0 2023-08-27T20:52:35.902 INFO:tasks.rados.rados.0.smithi093.stdout:983: seq_num 770 ranges {541390=493383,1592076=489269,2862175=79215} 2023-08-27T20:52:35.908 INFO:tasks.rados.rados.0.smithi093.stdout:983: writing smithi093153278-189 from 541390 to 1034773 tid 1 2023-08-27T20:52:35.914 INFO:tasks.rados.rados.0.smithi093.stdout:983: writing smithi093153278-189 from 1592076 to 2081345 tid 2 2023-08-27T20:52:35.919 INFO:tasks.rados.rados.0.smithi093.stdout:983: writing smithi093153278-189 from 2862175 to 2941390 tid 3 2023-08-27T20:52:35.919 INFO:tasks.rados.rados.0.smithi093.stdout:984: read oid 443 snap -1 2023-08-27T20:52:35.919 INFO:tasks.rados.rados.0.smithi093.stdout:984: expect (ObjNum 442 snap 0 seq_num 442) 2023-08-27T20:52:35.919 INFO:tasks.rados.rados.0.smithi093.stdout:985: read oid 54 snap -1 2023-08-27T20:52:35.919 INFO:tasks.rados.rados.0.smithi093.stdout:985: expect (ObjNum 53 snap 0 seq_num 53) 2023-08-27T20:52:35.920 INFO:tasks.rados.rados.0.smithi093.stdout:986: write oid 244 current snap is 0 2023-08-27T20:52:35.920 INFO:tasks.rados.rados.0.smithi093.stdout:975: expect (ObjNum 479 snap 0 seq_num 479) 2023-08-27T20:52:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:974: expect (ObjNum 687 snap 0 seq_num 687) 2023-08-27T20:52:36.171 INFO:tasks.rados.rados.0.smithi093.stdout:976: finishing write tid 1 to smithi093153278-78 2023-08-27T20:52:36.171 INFO:tasks.rados.rados.0.smithi093.stdout:986: seq_num 771 ranges {558197=568632,1650558=632927,2773245=184952} 2023-08-27T20:52:36.177 INFO:tasks.rados.rados.0.smithi093.stdout:986: writing smithi093153278-244 from 558197 to 1126829 tid 1 2023-08-27T20:52:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:986: writing smithi093153278-244 from 1650558 to 2283485 tid 2 2023-08-27T20:52:36.188 INFO:tasks.rados.rados.0.smithi093.stdout:986: writing smithi093153278-244 from 2773245 to 2958197 tid 3 2023-08-27T20:52:36.188 INFO:tasks.rados.rados.0.smithi093.stdout:974: done (12 left) 2023-08-27T20:52:36.188 INFO:tasks.rados.rados.0.smithi093.stdout:975: done (11 left) 2023-08-27T20:52:36.188 INFO:tasks.rados.rados.0.smithi093.stdout:987: write oid 16 current snap is 0 2023-08-27T20:52:36.188 INFO:tasks.rados.rados.0.smithi093.stdout:977: expect (ObjNum 413 snap 0 seq_num 413) 2023-08-27T20:52:36.285 INFO:tasks.rados.rados.0.smithi093.stdout:976: finishing write tid 2 to smithi093153278-78 2023-08-27T20:52:36.286 INFO:tasks.rados.rados.0.smithi093.stdout:987: seq_num 772 ranges {575004=643881,1709040=492938,2917021=57983} 2023-08-27T20:52:36.292 INFO:tasks.rados.rados.0.smithi093.stdout:987: writing smithi093153278-16 from 575004 to 1218885 tid 1 2023-08-27T20:52:36.297 INFO:tasks.rados.rados.0.smithi093.stdout:987: writing smithi093153278-16 from 1709040 to 2201978 tid 2 2023-08-27T20:52:36.302 INFO:tasks.rados.rados.0.smithi093.stdout:987: writing smithi093153278-16 from 2917021 to 2975004 tid 3 2023-08-27T20:52:36.302 INFO:tasks.rados.rados.0.smithi093.stdout:977: done (11 left) 2023-08-27T20:52:36.302 INFO:tasks.rados.rados.0.smithi093.stdout:988: read oid 327 snap -1 2023-08-27T20:52:36.302 INFO:tasks.rados.rados.0.smithi093.stdout:988: expect (ObjNum 674 snap 0 seq_num 674) 2023-08-27T20:52:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:989: write oid 299 current snap is 0 2023-08-27T20:52:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:976: finishing write tid 3 to smithi093153278-78 2023-08-27T20:52:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:976: finishing write tid 4 to smithi093153278-78 2023-08-27T20:52:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 138 (ObjNum 768 snap 0 seq_num 768) dirty exists 2023-08-27T20:52:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:976: left oid 78 (ObjNum 768 snap 0 seq_num 768) 2023-08-27T20:52:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:989: seq_num 773 ranges {591811=719130,1767522=636596,2828091=163720} 2023-08-27T20:52:36.309 INFO:tasks.rados.rados.0.smithi093.stdout:989: writing smithi093153278-299 from 591811 to 1310941 tid 1 2023-08-27T20:52:36.315 INFO:tasks.rados.rados.0.smithi093.stdout:989: writing smithi093153278-299 from 1767522 to 2404118 tid 2 2023-08-27T20:52:36.319 INFO:tasks.rados.rados.0.smithi093.stdout:989: writing smithi093153278-299 from 2828091 to 2991811 tid 3 2023-08-27T20:52:36.319 INFO:tasks.rados.rados.0.smithi093.stdout:976: done (12 left) 2023-08-27T20:52:36.319 INFO:tasks.rados.rados.0.smithi093.stdout:990: delete oid 235 current snap is 0 2023-08-27T20:52:36.319 INFO:tasks.rados.rados.0.smithi093.stdout:979: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:52:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:36 smithi093 ceph-mon[111335]: pgmap v72: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 9.1 MiB/s wr, 94 op/s 2023-08-27T20:52:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:36 smithi093 ceph-mon[118902]: pgmap v72: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 9.1 MiB/s wr, 94 op/s 2023-08-27T20:52:36.540 INFO:tasks.rados.rados.0.smithi093.stdout:980: expect (ObjNum 751 snap 0 seq_num 751) 2023-08-27T20:52:36.641 INFO:tasks.rados.rados.0.smithi093.stdout:978: expect (ObjNum 93 snap 0 seq_num 93) 2023-08-27T20:52:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:36 smithi136 ceph-mon[113531]: pgmap v72: 89 pgs: 89 active+clean; 1.3 GiB data, 2.0 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 9.1 MiB/s wr, 94 op/s 2023-08-27T20:52:36.750 INFO:tasks.rados.rados.0.smithi093.stdout:981: expect (ObjNum 247 snap 0 seq_num 247) 2023-08-27T20:52:36.816 INFO:tasks.rados.rados.0.smithi093.stdout:982: finishing write tid 1 to smithi093153278-474 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:982: finishing write tid 2 to smithi093153278-474 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:978: done (12 left) 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:979: done (11 left) 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:980: done (10 left) 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:981: done (9 left) 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:990: done (8 left) 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:991: write oid 192 current snap is 0 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:982: finishing write tid 3 to smithi093153278-474 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:982: finishing write tid 4 to smithi093153278-474 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:982: finishing write tid 5 to smithi093153278-474 2023-08-27T20:52:36.817 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 474 v 157 (ObjNum 769 snap 0 seq_num 769) dirty exists 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:982: left oid 474 (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:983: finishing write tid 1 to smithi093153278-189 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:983: finishing write tid 2 to smithi093153278-189 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:983: finishing write tid 3 to smithi093153278-189 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:983: finishing write tid 4 to smithi093153278-189 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:983: finishing write tid 5 to smithi093153278-189 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 189 v 180 (ObjNum 770 snap 0 seq_num 770) dirty exists 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:983: left oid 189 (ObjNum 770 snap 0 seq_num 770) 2023-08-27T20:52:36.818 INFO:tasks.rados.rados.0.smithi093.stdout:991: seq_num 775 ranges {625425=469628,1768133=640265} 2023-08-27T20:52:36.822 INFO:tasks.rados.rados.0.smithi093.stdout:991: writing smithi093153278-192 from 625425 to 1095053 tid 1 2023-08-27T20:52:36.829 INFO:tasks.rados.rados.0.smithi093.stdout:991: writing smithi093153278-192 from 1768133 to 2408398 tid 2 2023-08-27T20:52:36.829 INFO:tasks.rados.rados.0.smithi093.stdout:982: done (8 left) 2023-08-27T20:52:36.830 INFO:tasks.rados.rados.0.smithi093.stdout:983: done (7 left) 2023-08-27T20:52:36.830 INFO:tasks.rados.rados.0.smithi093.stdout:992: write oid 136 current snap is 0 2023-08-27T20:52:36.830 INFO:tasks.rados.rados.0.smithi093.stdout:984: expect (ObjNum 442 snap 0 seq_num 442) 2023-08-27T20:52:36.935 INFO:tasks.rados.rados.0.smithi093.stdout:985: expect (ObjNum 53 snap 0 seq_num 53) 2023-08-27T20:52:37.028 INFO:tasks.rados.rados.0.smithi093.stdout:986: finishing write tid 1 to smithi093153278-244 2023-08-27T20:52:37.028 INFO:tasks.rados.rados.0.smithi093.stdout:986: finishing write tid 2 to smithi093153278-244 2023-08-27T20:52:37.028 INFO:tasks.rados.rados.0.smithi093.stdout:986: finishing write tid 3 to smithi093153278-244 2023-08-27T20:52:37.028 INFO:tasks.rados.rados.0.smithi093.stdout:986: finishing write tid 4 to smithi093153278-244 2023-08-27T20:52:37.028 INFO:tasks.rados.rados.0.smithi093.stdout:986: finishing write tid 5 to smithi093153278-244 2023-08-27T20:52:37.029 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 244 v 112 (ObjNum 771 snap 0 seq_num 771) dirty exists 2023-08-27T20:52:37.029 INFO:tasks.rados.rados.0.smithi093.stdout:986: left oid 244 (ObjNum 771 snap 0 seq_num 771) 2023-08-27T20:52:37.029 INFO:tasks.rados.rados.0.smithi093.stdout:987: finishing write tid 1 to smithi093153278-16 2023-08-27T20:52:37.029 INFO:tasks.rados.rados.0.smithi093.stdout:987: finishing write tid 2 to smithi093153278-16 2023-08-27T20:52:37.029 INFO:tasks.rados.rados.0.smithi093.stdout:992: seq_num 776 ranges {642232=661230,1942968=783923} 2023-08-27T20:52:37.035 INFO:tasks.rados.rados.0.smithi093.stdout:992: writing smithi093153278-136 from 642232 to 1303462 tid 1 2023-08-27T20:52:37.043 INFO:tasks.rados.rados.0.smithi093.stdout:992: writing smithi093153278-136 from 1942968 to 2726891 tid 2 2023-08-27T20:52:37.043 INFO:tasks.rados.rados.0.smithi093.stdout:984: done (7 left) 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:985: done (6 left) 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:986: done (5 left) 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:993: delete oid 270 current snap is 0 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:987: finishing write tid 3 to smithi093153278-16 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:987: finishing write tid 4 to smithi093153278-16 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:987: finishing write tid 5 to smithi093153278-16 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 16 v 161 (ObjNum 772 snap 0 seq_num 772) dirty exists 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:987: left oid 16 (ObjNum 772 snap 0 seq_num 772) 2023-08-27T20:52:37.044 INFO:tasks.rados.rados.0.smithi093.stdout:988: expect (ObjNum 674 snap 0 seq_num 674) 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:989: finishing write tid 1 to smithi093153278-299 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:989: finishing write tid 2 to smithi093153278-299 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:989: finishing write tid 3 to smithi093153278-299 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:989: finishing write tid 4 to smithi093153278-299 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:987: done (5 left) 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:988: done (4 left) 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:993: done (3 left) 2023-08-27T20:52:37.190 INFO:tasks.rados.rados.0.smithi093.stdout:994: delete oid 236 current snap is 0 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:989: finishing write tid 5 to smithi093153278-299 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 299 v 159 (ObjNum 773 snap 0 seq_num 773) dirty exists 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:989: left oid 299 (ObjNum 773 snap 0 seq_num 773) 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:991: finishing write tid 1 to smithi093153278-192 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:991: finishing write tid 2 to smithi093153278-192 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:991: finishing write tid 3 to smithi093153278-192 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:991: finishing write tid 4 to smithi093153278-192 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 192 v 141 (ObjNum 775 snap 0 seq_num 775) dirty exists 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:991: left oid 192 (ObjNum 775 snap 0 seq_num 775) 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:992: finishing write tid 1 to smithi093153278-136 2023-08-27T20:52:37.191 INFO:tasks.rados.rados.0.smithi093.stdout:992: finishing write tid 2 to smithi093153278-136 2023-08-27T20:52:37.192 INFO:tasks.rados.rados.0.smithi093.stdout:992: finishing write tid 3 to smithi093153278-136 2023-08-27T20:52:37.192 INFO:tasks.rados.rados.0.smithi093.stdout:992: finishing write tid 4 to smithi093153278-136 2023-08-27T20:52:37.192 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 136 v 172 (ObjNum 776 snap 0 seq_num 776) dirty exists 2023-08-27T20:52:37.192 INFO:tasks.rados.rados.0.smithi093.stdout:992: left oid 136 (ObjNum 776 snap 0 seq_num 776) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:989: done (3 left) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:991: done (2 left) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:992: done (1 left) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:994: done (0 left) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:995: read oid 148 snap -1 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:995: expect (ObjNum 147 snap 0 seq_num 147) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:996: read oid 69 snap -1 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:996: expect (ObjNum 68 snap 0 seq_num 68) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:997: read oid 288 snap -1 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:997: expect (ObjNum 287 snap 0 seq_num 287) 2023-08-27T20:52:37.193 INFO:tasks.rados.rados.0.smithi093.stdout:998: write oid 346 current snap is 0 2023-08-27T20:52:37.194 INFO:tasks.rados.rados.0.smithi093.stdout:998: seq_num 779 ranges {692653=486977,1602061=647603,2992629=100024} 2023-08-27T20:52:37.198 INFO:tasks.rados.rados.0.smithi093.stdout:998: writing smithi093153278-346 from 692653 to 1179630 tid 1 2023-08-27T20:52:37.206 INFO:tasks.rados.rados.0.smithi093.stdout:998: writing smithi093153278-346 from 1602061 to 2249664 tid 2 2023-08-27T20:52:37.209 INFO:tasks.rados.rados.0.smithi093.stdout:998: writing smithi093153278-346 from 2992629 to 3092653 tid 3 2023-08-27T20:52:37.209 INFO:tasks.rados.rados.0.smithi093.stdout:999: write oid 454 current snap is 0 2023-08-27T20:52:37.209 INFO:tasks.rados.rados.0.smithi093.stdout:996: expect (ObjNum 68 snap 0 seq_num 68) 2023-08-27T20:52:37.314 INFO:tasks.rados.rados.0.smithi093.stdout:999: seq_num 780 ranges {709460=562226,2060543=791261} 2023-08-27T20:52:37.321 INFO:tasks.rados.rados.0.smithi093.stdout:999: writing smithi093153278-454 from 709460 to 1271686 tid 1 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:999: writing smithi093153278-454 from 2060543 to 2851804 tid 2 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:996: done (4 left) 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:1000: read oid 342 snap -1 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:1000: expect (ObjNum 341 snap 0 seq_num 341) 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:1001: read oid 23 snap -1 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:1001: expect (ObjNum 22 snap 0 seq_num 22) 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:1002: write oid 67 current snap is 0 2023-08-27T20:52:37.330 INFO:tasks.rados.rados.0.smithi093.stdout:995: expect (ObjNum 147 snap 0 seq_num 147) 2023-08-27T20:52:37.411 INFO:tasks.rados.rados.0.smithi093.stdout:997: expect (ObjNum 287 snap 0 seq_num 287) 2023-08-27T20:52:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2798755031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2798755031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:998: finishing write tid 1 to smithi093153278-346 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:998: finishing write tid 2 to smithi093153278-346 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:998: finishing write tid 3 to smithi093153278-346 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:998: finishing write tid 4 to smithi093153278-346 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:998: finishing write tid 5 to smithi093153278-346 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 346 v 190 (ObjNum 779 snap 0 seq_num 779) dirty exists 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:998: left oid 346 (ObjNum 779 snap 0 seq_num 779) 2023-08-27T20:52:37.495 INFO:tasks.rados.rados.0.smithi093.stdout:999: finishing write tid 1 to smithi093153278-454 2023-08-27T20:52:37.496 INFO:tasks.rados.rados.0.smithi093.stdout:1002: seq_num 781 ranges {726267=637475,2119025=651272} 2023-08-27T20:52:37.502 INFO:tasks.rados.rados.0.smithi093.stdout:1002: writing smithi093153278-67 from 726267 to 1363742 tid 1 2023-08-27T20:52:37.510 INFO:tasks.rados.rados.0.smithi093.stdout:1002: writing smithi093153278-67 from 2119025 to 2770297 tid 2 2023-08-27T20:52:37.510 INFO:tasks.rados.rados.0.smithi093.stdout:995: done (6 left) 2023-08-27T20:52:37.510 INFO:tasks.rados.rados.0.smithi093.stdout:997: done (5 left) 2023-08-27T20:52:37.510 INFO:tasks.rados.rados.0.smithi093.stdout:998: done (4 left) 2023-08-27T20:52:37.510 INFO:tasks.rados.rados.0.smithi093.stdout:1003: write oid 341 current snap is 0 2023-08-27T20:52:37.510 INFO:tasks.rados.rados.0.smithi093.stdout:1001: expect (ObjNum 22 snap 0 seq_num 22) 2023-08-27T20:52:37.584 INFO:tasks.rados.rados.0.smithi093.stdout:999: finishing write tid 2 to smithi093153278-454 2023-08-27T20:52:37.584 INFO:tasks.rados.rados.0.smithi093.stdout:999: finishing write tid 3 to smithi093153278-454 2023-08-27T20:52:37.585 INFO:tasks.rados.rados.0.smithi093.stdout:999: finishing write tid 4 to smithi093153278-454 2023-08-27T20:52:37.585 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 454 v 175 (ObjNum 780 snap 0 seq_num 780) dirty exists 2023-08-27T20:52:37.585 INFO:tasks.rados.rados.0.smithi093.stdout:999: left oid 454 (ObjNum 780 snap 0 seq_num 780) 2023-08-27T20:52:37.585 INFO:tasks.rados.rados.0.smithi093.stdout:1000: expect (ObjNum 341 snap 0 seq_num 341) 2023-08-27T20:52:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2798755031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:37.697 INFO:tasks.rados.rados.0.smithi093.stdout:1002: finishing write tid 1 to smithi093153278-67 2023-08-27T20:52:37.697 INFO:tasks.rados.rados.0.smithi093.stdout:1002: finishing write tid 2 to smithi093153278-67 2023-08-27T20:52:37.697 INFO:tasks.rados.rados.0.smithi093.stdout:1002: finishing write tid 3 to smithi093153278-67 2023-08-27T20:52:37.697 INFO:tasks.rados.rados.0.smithi093.stdout:1002: finishing write tid 4 to smithi093153278-67 2023-08-27T20:52:37.697 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 67 v 133 (ObjNum 781 snap 0 seq_num 781) dirty exists 2023-08-27T20:52:37.697 INFO:tasks.rados.rados.0.smithi093.stdout:1002: left oid 67 (ObjNum 781 snap 0 seq_num 781) 2023-08-27T20:52:37.697 INFO:tasks.rados.rados.0.smithi093.stdout:1003: seq_num 782 ranges {743074=712724,2061154=794930} 2023-08-27T20:52:37.704 INFO:tasks.rados.rados.0.smithi093.stdout:1003: writing smithi093153278-341 from 743074 to 1455798 tid 1 2023-08-27T20:52:37.712 INFO:tasks.rados.rados.0.smithi093.stdout:1003: writing smithi093153278-341 from 2061154 to 2856084 tid 2 2023-08-27T20:52:37.712 INFO:tasks.rados.rados.0.smithi093.stdout:999: done (4 left) 2023-08-27T20:52:37.713 INFO:tasks.rados.rados.0.smithi093.stdout:1000: done (3 left) 2023-08-27T20:52:37.713 INFO:tasks.rados.rados.0.smithi093.stdout:1001: done (2 left) 2023-08-27T20:52:37.713 INFO:tasks.rados.rados.0.smithi093.stdout:1002: done (1 left) 2023-08-27T20:52:37.713 INFO:tasks.rados.rados.0.smithi093.stdout:1004: write oid 466 current snap is 0 2023-08-27T20:52:37.713 INFO:tasks.rados.rados.0.smithi093.stdout:1003: finishing write tid 1 to smithi093153278-341 2023-08-27T20:52:37.713 INFO:tasks.rados.rados.0.smithi093.stdout:1004: seq_num 783 ranges {759881=787973,2119636=654941} 2023-08-27T20:52:37.720 INFO:tasks.rados.rados.0.smithi093.stdout:1004: writing smithi093153278-466 from 759881 to 1547854 tid 1 2023-08-27T20:52:37.727 INFO:tasks.rados.rados.0.smithi093.stdout:1004: writing smithi093153278-466 from 2119636 to 2774577 tid 2 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:1005: write oid 245 current snap is 0 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:1003: finishing write tid 2 to smithi093153278-341 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:1003: finishing write tid 3 to smithi093153278-341 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:1003: finishing write tid 4 to smithi093153278-341 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 341 v 168 (ObjNum 782 snap 0 seq_num 782) dirty exists 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:1003: left oid 341 (ObjNum 782 snap 0 seq_num 782) 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:1004: finishing write tid 1 to smithi093153278-466 2023-08-27T20:52:37.728 INFO:tasks.rados.rados.0.smithi093.stdout:1005: seq_num 784 ranges {776688=579575,1894471=798599} 2023-08-27T20:52:37.735 INFO:tasks.rados.rados.0.smithi093.stdout:1005: writing smithi093153278-245 from 776688 to 1356263 tid 1 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1005: writing smithi093153278-245 from 1894471 to 2693070 tid 2 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1003: done (2 left) 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1006: read oid 371 snap -1 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1006: expect deleted 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1007: read oid 42 snap -1 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1007: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1008: read oid 441 snap -1 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1008: expect (ObjNum 591 snap 0 seq_num 591) 2023-08-27T20:52:37.743 INFO:tasks.rados.rados.0.smithi093.stdout:1009: write oid 394 current snap is 0 2023-08-27T20:52:37.744 INFO:tasks.rados.rados.0.smithi093.stdout:1004: finishing write tid 2 to smithi093153278-466 2023-08-27T20:52:37.744 INFO:tasks.rados.rados.0.smithi093.stdout:1009: seq_num 785 ranges {793495=654824,1952953=658610,3157167=36328} 2023-08-27T20:52:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:1009: writing smithi093153278-394 from 793495 to 1448319 tid 1 2023-08-27T20:52:37.757 INFO:tasks.rados.rados.0.smithi093.stdout:1009: writing smithi093153278-394 from 1952953 to 2611563 tid 2 2023-08-27T20:52:37.761 INFO:tasks.rados.rados.0.smithi093.stdout:1009: writing smithi093153278-394 from 3157167 to 3193495 tid 3 2023-08-27T20:52:37.761 INFO:tasks.rados.rados.0.smithi093.stdout:1010: read oid 406 snap -1 2023-08-27T20:52:37.761 INFO:tasks.rados.rados.0.smithi093.stdout:1010: expect (ObjNum 621 snap 0 seq_num 621) 2023-08-27T20:52:37.761 INFO:tasks.rados.rados.0.smithi093.stdout:1011: read oid 5 snap -1 2023-08-27T20:52:37.761 INFO:tasks.rados.rados.0.smithi093.stdout:1011: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:52:37.761 INFO:tasks.rados.rados.0.smithi093.stdout:1012: write oid 137 current snap is 0 2023-08-27T20:52:37.761 INFO:tasks.rados.rados.0.smithi093.stdout:1012: seq_num 786 ranges {410302=730073,1895082=402268,2951884=258418} 2023-08-27T20:52:37.767 INFO:tasks.rados.rados.0.smithi093.stdout:1012: writing smithi093153278-137 from 410302 to 1140375 tid 1 2023-08-27T20:52:37.774 INFO:tasks.rados.rados.0.smithi093.stdout:1012: writing smithi093153278-137 from 1895082 to 2297350 tid 2 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1012: writing smithi093153278-137 from 2951884 to 3210302 tid 3 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1013: read oid 94 snap -1 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1013: expect (ObjNum 93 snap 0 seq_num 93) 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1014: read oid 51 snap -1 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1014: expect (ObjNum 727 snap 0 seq_num 727) 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1015: read oid 230 snap -1 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1015: expect (ObjNum 671 snap 0 seq_num 671) 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1016: write oid 337 current snap is 0 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1004: finishing write tid 3 to smithi093153278-466 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:1004: finishing write tid 4 to smithi093153278-466 2023-08-27T20:52:37.780 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 466 v 193 (ObjNum 783 snap 0 seq_num 783) dirty exists 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:1004: left oid 466 (ObjNum 783 snap 0 seq_num 783) 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:1005: finishing write tid 1 to smithi093153278-245 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:1005: finishing write tid 2 to smithi093153278-245 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:1005: finishing write tid 3 to smithi093153278-245 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:1005: finishing write tid 4 to smithi093153278-245 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 245 v 231 (ObjNum 784 snap 0 seq_num 784) dirty exists 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:1005: left oid 245 (ObjNum 784 snap 0 seq_num 784) 2023-08-27T20:52:37.781 INFO:tasks.rados.rados.0.smithi093.stdout:1016: seq_num 787 ranges {427109=405322,1553564=545926,2579307=647802} 2023-08-27T20:52:37.784 INFO:tasks.rados.rados.0.smithi093.stdout:1016: writing smithi093153278-337 from 427109 to 832431 tid 1 2023-08-27T20:52:37.791 INFO:tasks.rados.rados.0.smithi093.stdout:1016: writing smithi093153278-337 from 1553564 to 2099490 tid 2 2023-08-27T20:52:37.798 INFO:tasks.rados.rados.0.smithi093.stdout:1016: writing smithi093153278-337 from 2579307 to 3227109 tid 3 2023-08-27T20:52:37.798 INFO:tasks.rados.rados.0.smithi093.stdout:1004: done (12 left) 2023-08-27T20:52:37.798 INFO:tasks.rados.rados.0.smithi093.stdout:1005: done (11 left) 2023-08-27T20:52:37.798 INFO:tasks.rados.rados.0.smithi093.stdout:1006: done (10 left) 2023-08-27T20:52:37.798 INFO:tasks.rados.rados.0.smithi093.stdout:1017: write oid 276 current snap is 0 2023-08-27T20:52:37.798 INFO:tasks.rados.rados.0.smithi093.stdout:1009: finishing write tid 1 to smithi093153278-394 2023-08-27T20:52:37.798 INFO:tasks.rados.rados.0.smithi093.stdout:1007: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:52:37.881 INFO:tasks.rados.rados.0.smithi093.stdout:1009: finishing write tid 2 to smithi093153278-394 2023-08-27T20:52:37.882 INFO:tasks.rados.rados.0.smithi093.stdout:1008: expect (ObjNum 591 snap 0 seq_num 591) 2023-08-27T20:52:38.080 INFO:tasks.rados.rados.0.smithi093.stdout:1009: finishing write tid 3 to smithi093153278-394 2023-08-27T20:52:38.080 INFO:tasks.rados.rados.0.smithi093.stdout:1009: finishing write tid 4 to smithi093153278-394 2023-08-27T20:52:38.081 INFO:tasks.rados.rados.0.smithi093.stdout:1009: finishing write tid 5 to smithi093153278-394 2023-08-27T20:52:38.081 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 394 v 176 (ObjNum 785 snap 0 seq_num 785) dirty exists 2023-08-27T20:52:38.081 INFO:tasks.rados.rados.0.smithi093.stdout:1009: left oid 394 (ObjNum 785 snap 0 seq_num 785) 2023-08-27T20:52:38.081 INFO:tasks.rados.rados.0.smithi093.stdout:1010: expect (ObjNum 621 snap 0 seq_num 621) 2023-08-27T20:52:38.181 INFO:tasks.rados.rados.0.smithi093.stdout:1011: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:52:38.274 INFO:tasks.rados.rados.0.smithi093.stdout:1012: finishing write tid 1 to smithi093153278-137 2023-08-27T20:52:38.274 INFO:tasks.rados.rados.0.smithi093.stdout:1012: finishing write tid 2 to smithi093153278-137 2023-08-27T20:52:38.274 INFO:tasks.rados.rados.0.smithi093.stdout:1017: seq_num 788 ranges {443916=480571,1612046=405937,2606730=495052} 2023-08-27T20:52:38.279 INFO:tasks.rados.rados.0.smithi093.stdout:1017: writing smithi093153278-276 from 443916 to 924487 tid 1 2023-08-27T20:52:38.285 INFO:tasks.rados.rados.0.smithi093.stdout:1017: writing smithi093153278-276 from 1612046 to 2017983 tid 2 2023-08-27T20:52:38.291 INFO:tasks.rados.rados.0.smithi093.stdout:1017: writing smithi093153278-276 from 2606730 to 3101782 tid 3 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1007: done (10 left) 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1008: done (9 left) 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1009: done (8 left) 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1010: done (7 left) 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1011: done (6 left) 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1018: read oid 102 snap -1 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1018: expect (ObjNum 101 snap 0 seq_num 101) 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1019: read oid 340 snap -1 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1019: expect (ObjNum 339 snap 0 seq_num 339) 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1020: write oid 465 current snap is 0 2023-08-27T20:52:38.292 INFO:tasks.rados.rados.0.smithi093.stdout:1015: expect (ObjNum 671 snap 0 seq_num 671) 2023-08-27T20:52:38.433 INFO:tasks.rados.rados.0.smithi093.stdout:1012: finishing write tid 3 to smithi093153278-137 2023-08-27T20:52:38.433 INFO:tasks.rados.rados.0.smithi093.stdout:1012: finishing write tid 4 to smithi093153278-137 2023-08-27T20:52:38.433 INFO:tasks.rados.rados.0.smithi093.stdout:1012: finishing write tid 5 to smithi093153278-137 2023-08-27T20:52:38.433 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 137 v 178 (ObjNum 786 snap 0 seq_num 786) dirty exists 2023-08-27T20:52:38.433 INFO:tasks.rados.rados.0.smithi093.stdout:1012: left oid 137 (ObjNum 786 snap 0 seq_num 786) 2023-08-27T20:52:38.433 INFO:tasks.rados.rados.0.smithi093.stdout:1020: seq_num 789 ranges {460723=555820,1670528=549595,2634153=626570} 2023-08-27T20:52:38.438 INFO:tasks.rados.rados.0.smithi093.stdout:1020: writing smithi093153278-465 from 460723 to 1016543 tid 1 2023-08-27T20:52:38.445 INFO:tasks.rados.rados.0.smithi093.stdout:1020: writing smithi093153278-465 from 1670528 to 2220123 tid 2 2023-08-27T20:52:38.451 INFO:tasks.rados.rados.0.smithi093.stdout:1020: writing smithi093153278-465 from 2634153 to 3260723 tid 3 2023-08-27T20:52:38.451 INFO:tasks.rados.rados.0.smithi093.stdout:1012: done (8 left) 2023-08-27T20:52:38.451 INFO:tasks.rados.rados.0.smithi093.stdout:1015: done (7 left) 2023-08-27T20:52:38.451 INFO:tasks.rados.rados.0.smithi093.stdout:1021: write oid 149 current snap is 0 2023-08-27T20:52:38.451 INFO:tasks.rados.rados.0.smithi093.stdout:1016: finishing write tid 1 to smithi093153278-337 2023-08-27T20:52:38.451 INFO:tasks.rados.rados.0.smithi093.stdout:1021: seq_num 790 ranges {477530=631069,1612657=409606,2545223=517596} 2023-08-27T20:52:38.457 INFO:tasks.rados.rados.0.smithi093.stdout:1021: writing smithi093153278-149 from 477530 to 1108599 tid 1 2023-08-27T20:52:38.462 INFO:tasks.rados.rados.0.smithi093.stdout:1021: writing smithi093153278-149 from 1612657 to 2022263 tid 2 2023-08-27T20:52:38.468 INFO:tasks.rados.rados.0.smithi093.stdout:1021: writing smithi093153278-149 from 2545223 to 3062819 tid 3 2023-08-27T20:52:38.468 INFO:tasks.rados.rados.0.smithi093.stdout:1022: write oid 406 current snap is 0 2023-08-27T20:52:38.468 INFO:tasks.rados.rados.0.smithi093.stdout:1013: expect (ObjNum 93 snap 0 seq_num 93) 2023-08-27T20:52:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:38 smithi093 ceph-mon[111335]: pgmap v73: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 14 MiB/s wr, 147 op/s 2023-08-27T20:52:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:38 smithi093 ceph-mon[118902]: pgmap v73: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 14 MiB/s wr, 147 op/s 2023-08-27T20:52:38.586 INFO:tasks.rados.rados.0.smithi093.stdout:1014: expect (ObjNum 727 snap 0 seq_num 727) 2023-08-27T20:52:38.677 INFO:tasks.rados.rados.0.smithi093.stdout:1016: finishing write tid 2 to smithi093153278-337 2023-08-27T20:52:38.677 INFO:tasks.rados.rados.0.smithi093.stdout:1016: finishing write tid 3 to smithi093153278-337 2023-08-27T20:52:38.677 INFO:tasks.rados.rados.0.smithi093.stdout:1016: finishing write tid 4 to smithi093153278-337 2023-08-27T20:52:38.677 INFO:tasks.rados.rados.0.smithi093.stdout:1016: finishing write tid 5 to smithi093153278-337 2023-08-27T20:52:38.677 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 337 v 182 (ObjNum 787 snap 0 seq_num 787) dirty exists 2023-08-27T20:52:38.677 INFO:tasks.rados.rados.0.smithi093.stdout:1016: left oid 337 (ObjNum 787 snap 0 seq_num 787) 2023-08-27T20:52:38.677 INFO:tasks.rados.rados.0.smithi093.stdout:1022: seq_num 791 ranges {494337=422671,1387492=553264,2688999=445221} 2023-08-27T20:52:38.681 INFO:tasks.rados.rados.0.smithi093.stdout:1022: writing smithi093153278-406 from 494337 to 917008 tid 1 2023-08-27T20:52:38.687 INFO:tasks.rados.rados.0.smithi093.stdout:1022: writing smithi093153278-406 from 1387492 to 1940756 tid 2 2023-08-27T20:52:38.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:38 smithi136 ceph-mon[113531]: pgmap v73: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 14 MiB/s wr, 147 op/s 2023-08-27T20:52:38.694 INFO:tasks.rados.rados.0.smithi093.stdout:1022: writing smithi093153278-406 from 2688999 to 3134220 tid 3 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1013: done (8 left) 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1014: done (7 left) 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1016: done (6 left) 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1023: read oid 447 snap -1 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1023: expect (ObjNum 763 snap 0 seq_num 763) 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1024: read oid 451 snap -1 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1024: expect (ObjNum 450 snap 0 seq_num 450) 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1025: write oid 323 current snap is 0 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1017: finishing write tid 1 to smithi093153278-276 2023-08-27T20:52:38.695 INFO:tasks.rados.rados.0.smithi093.stdout:1017: finishing write tid 2 to smithi093153278-276 2023-08-27T20:52:38.696 INFO:tasks.rados.rados.0.smithi093.stdout:1017: finishing write tid 3 to smithi093153278-276 2023-08-27T20:52:38.696 INFO:tasks.rados.rados.0.smithi093.stdout:1017: finishing write tid 4 to smithi093153278-276 2023-08-27T20:52:38.696 INFO:tasks.rados.rados.0.smithi093.stdout:1017: finishing write tid 5 to smithi093153278-276 2023-08-27T20:52:38.696 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 276 v 165 (ObjNum 788 snap 0 seq_num 788) dirty exists 2023-08-27T20:52:38.696 INFO:tasks.rados.rados.0.smithi093.stdout:1017: left oid 276 (ObjNum 788 snap 0 seq_num 788) 2023-08-27T20:52:38.696 INFO:tasks.rados.rados.0.smithi093.stdout:1019: expect (ObjNum 339 snap 0 seq_num 339) 2023-08-27T20:52:38.811 INFO:tasks.rados.rados.0.smithi093.stdout:1018: expect (ObjNum 101 snap 0 seq_num 101) 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:1020: finishing write tid 1 to smithi093153278-465 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:1020: finishing write tid 2 to smithi093153278-465 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:1020: finishing write tid 3 to smithi093153278-465 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:1020: finishing write tid 4 to smithi093153278-465 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:1020: finishing write tid 5 to smithi093153278-465 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 465 v 235 (ObjNum 789 snap 0 seq_num 789) dirty exists 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:1020: left oid 465 (ObjNum 789 snap 0 seq_num 789) 2023-08-27T20:52:38.927 INFO:tasks.rados.rados.0.smithi093.stdout:1021: finishing write tid 1 to smithi093153278-149 2023-08-27T20:52:38.928 INFO:tasks.rados.rados.0.smithi093.stdout:1025: seq_num 792 ranges {511144=497920,1445974=413275,2316422=656493} 2023-08-27T20:52:38.932 INFO:tasks.rados.rados.0.smithi093.stdout:1025: writing smithi093153278-323 from 511144 to 1009064 tid 1 2023-08-27T20:52:38.937 INFO:tasks.rados.rados.0.smithi093.stdout:1025: writing smithi093153278-323 from 1445974 to 1859249 tid 2 2023-08-27T20:52:38.943 INFO:tasks.rados.rados.0.smithi093.stdout:1025: writing smithi093153278-323 from 2316422 to 2972915 tid 3 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1017: done (8 left) 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1018: done (7 left) 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1019: done (6 left) 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1020: done (5 left) 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1026: read oid 41 snap -1 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1026: expect (ObjNum 40 snap 0 seq_num 40) 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1027: read oid 163 snap -1 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1027: expect (ObjNum 162 snap 0 seq_num 162) 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1028: write oid 313 current snap is 0 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1021: finishing write tid 2 to smithi093153278-149 2023-08-27T20:52:38.944 INFO:tasks.rados.rados.0.smithi093.stdout:1028: seq_num 793 ranges {527951=573169,1504456=556933,2743845=467765} 2023-08-27T20:52:38.950 INFO:tasks.rados.rados.0.smithi093.stdout:1028: writing smithi093153278-313 from 527951 to 1101120 tid 1 2023-08-27T20:52:38.955 INFO:tasks.rados.rados.0.smithi093.stdout:1028: writing smithi093153278-313 from 1504456 to 2061389 tid 2 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:1028: writing smithi093153278-313 from 2743845 to 3211610 tid 3 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:1029: write oid 409 current snap is 0 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:1021: finishing write tid 3 to smithi093153278-149 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:1021: finishing write tid 4 to smithi093153278-149 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:1021: finishing write tid 5 to smithi093153278-149 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 149 v 172 (ObjNum 790 snap 0 seq_num 790) dirty exists 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:1021: left oid 149 (ObjNum 790 snap 0 seq_num 790) 2023-08-27T20:52:38.962 INFO:tasks.rados.rados.0.smithi093.stdout:1029: seq_num 794 ranges {544758=648418,1846585=416944,2771268=573490} 2023-08-27T20:52:38.968 INFO:tasks.rados.rados.0.smithi093.stdout:1029: writing smithi093153278-409 from 544758 to 1193176 tid 1 2023-08-27T20:52:38.974 INFO:tasks.rados.rados.0.smithi093.stdout:1029: writing smithi093153278-409 from 1846585 to 2263529 tid 2 2023-08-27T20:52:38.980 INFO:tasks.rados.rados.0.smithi093.stdout:1029: writing smithi093153278-409 from 2771268 to 3344758 tid 3 2023-08-27T20:52:38.981 INFO:tasks.rados.rados.0.smithi093.stdout:1021: done (8 left) 2023-08-27T20:52:38.981 INFO:tasks.rados.rados.0.smithi093.stdout:1030: read oid 109 snap -1 2023-08-27T20:52:38.981 INFO:tasks.rados.rados.0.smithi093.stdout:1030: expect (ObjNum 108 snap 0 seq_num 108) 2023-08-27T20:52:38.981 INFO:tasks.rados.rados.0.smithi093.stdout:1031: write oid 86 current snap is 0 2023-08-27T20:52:38.981 INFO:tasks.rados.rados.0.smithi093.stdout:1022: finishing write tid 1 to smithi093153278-406 2023-08-27T20:52:38.981 INFO:tasks.rados.rados.0.smithi093.stdout:1022: finishing write tid 2 to smithi093153278-406 2023-08-27T20:52:38.981 INFO:tasks.rados.rados.0.smithi093.stdout:1031: seq_num 795 ranges {561565=723667,1905067=560602,3082338=279227} 2023-08-27T20:52:38.987 INFO:tasks.rados.rados.0.smithi093.stdout:1031: writing smithi093153278-86 from 561565 to 1285232 tid 1 2023-08-27T20:52:38.994 INFO:tasks.rados.rados.0.smithi093.stdout:1031: writing smithi093153278-86 from 1905067 to 2465669 tid 2 2023-08-27T20:52:38.999 INFO:tasks.rados.rados.0.smithi093.stdout:1031: writing smithi093153278-86 from 3082338 to 3361565 tid 3 2023-08-27T20:52:38.999 INFO:tasks.rados.rados.0.smithi093.stdout:1032: delete oid 102 current snap is 0 2023-08-27T20:52:38.999 INFO:tasks.rados.rados.0.smithi093.stdout:1022: finishing write tid 3 to smithi093153278-406 2023-08-27T20:52:39.000 INFO:tasks.rados.rados.0.smithi093.stdout:1022: finishing write tid 4 to smithi093153278-406 2023-08-27T20:52:39.000 INFO:tasks.rados.rados.0.smithi093.stdout:1022: finishing write tid 5 to smithi093153278-406 2023-08-27T20:52:39.000 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 406 v 164 (ObjNum 791 snap 0 seq_num 791) dirty exists 2023-08-27T20:52:39.000 INFO:tasks.rados.rados.0.smithi093.stdout:1022: left oid 406 (ObjNum 791 snap 0 seq_num 791) 2023-08-27T20:52:39.000 INFO:tasks.rados.rados.0.smithi093.stdout:1023: expect (ObjNum 763 snap 0 seq_num 763) 2023-08-27T20:52:39.120 INFO:tasks.rados.rados.0.smithi093.stdout:1024: expect (ObjNum 450 snap 0 seq_num 450) 2023-08-27T20:52:39.228 INFO:tasks.rados.rados.0.smithi093.stdout:1025: finishing write tid 1 to smithi093153278-323 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1025: finishing write tid 2 to smithi093153278-323 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1022: done (10 left) 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1023: done (9 left) 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1024: done (8 left) 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1032: done (7 left) 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1033: read oid 105 snap -1 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1033: expect (ObjNum 104 snap 0 seq_num 104) 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1034: read oid 420 snap -1 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1034: expect (ObjNum 419 snap 0 seq_num 419) 2023-08-27T20:52:39.229 INFO:tasks.rados.rados.0.smithi093.stdout:1035: write oid 342 current snap is 0 2023-08-27T20:52:39.230 INFO:tasks.rados.rados.0.smithi093.stdout:1025: finishing write tid 3 to smithi093153278-323 2023-08-27T20:52:39.230 INFO:tasks.rados.rados.0.smithi093.stdout:1027: expect (ObjNum 162 snap 0 seq_num 162) 2023-08-27T20:52:39.313 INFO:tasks.rados.rados.0.smithi093.stdout:1025: finishing write tid 4 to smithi093153278-323 2023-08-27T20:52:39.313 INFO:tasks.rados.rados.0.smithi093.stdout:1025: finishing write tid 5 to smithi093153278-323 2023-08-27T20:52:39.314 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 323 v 142 (ObjNum 792 snap 0 seq_num 792) dirty exists 2023-08-27T20:52:39.314 INFO:tasks.rados.rados.0.smithi093.stdout:1025: left oid 323 (ObjNum 792 snap 0 seq_num 792) 2023-08-27T20:52:39.314 INFO:tasks.rados.rados.0.smithi093.stdout:1026: expect (ObjNum 40 snap 0 seq_num 40) 2023-08-27T20:52:39.394 INFO:tasks.rados.rados.0.smithi093.stdout:1028: finishing write tid 1 to smithi093153278-313 2023-08-27T20:52:39.394 INFO:tasks.rados.rados.0.smithi093.stdout:1028: finishing write tid 2 to smithi093153278-313 2023-08-27T20:52:39.394 INFO:tasks.rados.rados.0.smithi093.stdout:1028: finishing write tid 3 to smithi093153278-313 2023-08-27T20:52:39.394 INFO:tasks.rados.rados.0.smithi093.stdout:1028: finishing write tid 4 to smithi093153278-313 2023-08-27T20:52:39.394 INFO:tasks.rados.rados.0.smithi093.stdout:1028: finishing write tid 5 to smithi093153278-313 2023-08-27T20:52:39.394 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 313 v 116 (ObjNum 793 snap 0 seq_num 793) dirty exists 2023-08-27T20:52:39.395 INFO:tasks.rados.rados.0.smithi093.stdout:1028: left oid 313 (ObjNum 793 snap 0 seq_num 793) 2023-08-27T20:52:39.395 INFO:tasks.rados.rados.0.smithi093.stdout:1029: finishing write tid 1 to smithi093153278-409 2023-08-27T20:52:39.395 INFO:tasks.rados.rados.0.smithi093.stdout:1035: seq_num 797 ranges {595179=474165,1622031=564271,2737184=629206} 2023-08-27T20:52:39.399 INFO:tasks.rados.rados.0.smithi093.stdout:1035: writing smithi093153278-342 from 595179 to 1069344 tid 1 2023-08-27T20:52:39.406 INFO:tasks.rados.rados.0.smithi093.stdout:1035: writing smithi093153278-342 from 1622031 to 2186302 tid 2 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1035: writing smithi093153278-342 from 2737184 to 3366390 tid 3 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1025: done (9 left) 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1026: done (8 left) 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1027: done (7 left) 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1028: done (6 left) 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1036: write oid 204 current snap is 0 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1029: finishing write tid 2 to smithi093153278-409 2023-08-27T20:52:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:1029: finishing write tid 3 to smithi093153278-409 2023-08-27T20:52:39.414 INFO:tasks.rados.rados.0.smithi093.stdout:1029: finishing write tid 4 to smithi093153278-409 2023-08-27T20:52:39.414 INFO:tasks.rados.rados.0.smithi093.stdout:1029: finishing write tid 5 to smithi093153278-409 2023-08-27T20:52:39.414 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 409 v 145 (ObjNum 794 snap 0 seq_num 794) dirty exists 2023-08-27T20:52:39.414 INFO:tasks.rados.rados.0.smithi093.stdout:1029: left oid 409 (ObjNum 794 snap 0 seq_num 794) 2023-08-27T20:52:39.414 INFO:tasks.rados.rados.0.smithi093.stdout:1030: expect (ObjNum 108 snap 0 seq_num 108) 2023-08-27T20:52:39.533 INFO:tasks.rados.rados.0.smithi093.stdout:1031: finishing write tid 1 to smithi093153278-86 2023-08-27T20:52:39.534 INFO:tasks.rados.rados.0.smithi093.stdout:1031: finishing write tid 2 to smithi093153278-86 2023-08-27T20:52:39.534 INFO:tasks.rados.rados.0.smithi093.stdout:1031: finishing write tid 3 to smithi093153278-86 2023-08-27T20:52:39.534 INFO:tasks.rados.rados.0.smithi093.stdout:1031: finishing write tid 4 to smithi093153278-86 2023-08-27T20:52:39.534 INFO:tasks.rados.rados.0.smithi093.stdout:1031: finishing write tid 5 to smithi093153278-86 2023-08-27T20:52:39.534 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 86 v 169 (ObjNum 795 snap 0 seq_num 795) dirty exists 2023-08-27T20:52:39.534 INFO:tasks.rados.rados.0.smithi093.stdout:1031: left oid 86 (ObjNum 795 snap 0 seq_num 795) 2023-08-27T20:52:39.534 INFO:tasks.rados.rados.0.smithi093.stdout:1036: seq_num 798 ranges {611986=549414,1564160=424282,2764607=440478} 2023-08-27T20:52:39.539 INFO:tasks.rados.rados.0.smithi093.stdout:1036: writing smithi093153278-204 from 611986 to 1161400 tid 1 2023-08-27T20:52:39.544 INFO:tasks.rados.rados.0.smithi093.stdout:1036: writing smithi093153278-204 from 1564160 to 1988442 tid 2 2023-08-27T20:52:39.551 INFO:tasks.rados.rados.0.smithi093.stdout:1036: writing smithi093153278-204 from 2764607 to 3205085 tid 3 2023-08-27T20:52:39.551 INFO:tasks.rados.rados.0.smithi093.stdout:1029: done (6 left) 2023-08-27T20:52:39.551 INFO:tasks.rados.rados.0.smithi093.stdout:1030: done (5 left) 2023-08-27T20:52:39.551 INFO:tasks.rados.rados.0.smithi093.stdout:1031: done (4 left) 2023-08-27T20:52:39.551 INFO:tasks.rados.rados.0.smithi093.stdout:1037: write oid 319 current snap is 0 2023-08-27T20:52:39.552 INFO:tasks.rados.rados.0.smithi093.stdout:1037: seq_num 799 ranges {628793=741016,2138995=567940,3192030=236763} 2023-08-27T20:52:39.558 INFO:tasks.rados.rados.0.smithi093.stdout:1037: writing smithi093153278-319 from 628793 to 1369809 tid 1 2023-08-27T20:52:39.566 INFO:tasks.rados.rados.0.smithi093.stdout:1037: writing smithi093153278-319 from 2138995 to 2706935 tid 2 2023-08-27T20:52:39.570 INFO:tasks.rados.rados.0.smithi093.stdout:1037: writing smithi093153278-319 from 3192030 to 3428793 tid 3 2023-08-27T20:52:39.570 INFO:tasks.rados.rados.0.smithi093.stdout:1038: delete oid 401 current snap is 0 2023-08-27T20:52:39.571 INFO:tasks.rados.rados.0.smithi093.stdout:1033: expect (ObjNum 104 snap 0 seq_num 104) 2023-08-27T20:52:39.689 INFO:tasks.rados.rados.0.smithi093.stdout:1034: expect (ObjNum 419 snap 0 seq_num 419) 2023-08-27T20:52:39.784 INFO:tasks.rados.rados.0.smithi093.stdout:1035: finishing write tid 1 to smithi093153278-342 2023-08-27T20:52:39.784 INFO:tasks.rados.rados.0.smithi093.stdout:1035: finishing write tid 2 to smithi093153278-342 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1035: finishing write tid 3 to smithi093153278-342 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1035: finishing write tid 4 to smithi093153278-342 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1035: finishing write tid 5 to smithi093153278-342 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 342 v 179 (ObjNum 797 snap 0 seq_num 797) dirty exists 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1035: left oid 342 (ObjNum 797 snap 0 seq_num 797) 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1036: finishing write tid 1 to smithi093153278-204 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1033: done (5 left) 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1034: done (4 left) 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1035: done (3 left) 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1038: done (2 left) 2023-08-27T20:52:39.785 INFO:tasks.rados.rados.0.smithi093.stdout:1039: read oid 268 snap -1 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1039: expect (ObjNum 267 snap 0 seq_num 267) 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1040: read oid 127 snap -1 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1040: expect (ObjNum 126 snap 0 seq_num 126) 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1041: delete oid 383 current snap is 0 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1036: finishing write tid 2 to smithi093153278-204 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1036: finishing write tid 3 to smithi093153278-204 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1036: finishing write tid 4 to smithi093153278-204 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1036: finishing write tid 5 to smithi093153278-204 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 204 v 183 (ObjNum 798 snap 0 seq_num 798) dirty exists 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1036: left oid 204 (ObjNum 798 snap 0 seq_num 798) 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1037: finishing write tid 1 to smithi093153278-319 2023-08-27T20:52:39.786 INFO:tasks.rados.rados.0.smithi093.stdout:1037: finishing write tid 2 to smithi093153278-319 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:1037: finishing write tid 3 to smithi093153278-319 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:1037: finishing write tid 4 to smithi093153278-319 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:1037: finishing write tid 5 to smithi093153278-319 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 319 v 149 (ObjNum 799 snap 0 seq_num 799) dirty exists 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:1037: left oid 319 (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:1036: done (4 left) 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:1037: done (3 left) 2023-08-27T20:52:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:1041: done (2 left) 2023-08-27T20:52:39.788 INFO:tasks.rados.rados.0.smithi093.stdout:1042: write oid 253 current snap is 0 2023-08-27T20:52:39.788 INFO:tasks.rados.rados.0.smithi093.stdout:1042: seq_num 802 ranges {679214=566763,1798088=715267,3157946=321268} 2023-08-27T20:52:39.792 INFO:tasks.rados.rados.0.smithi093.stdout:1042: writing smithi093153278-253 from 679214 to 1245977 tid 1 2023-08-27T20:52:39.799 INFO:tasks.rados.rados.0.smithi093.stdout:1042: writing smithi093153278-253 from 1798088 to 2513355 tid 2 2023-08-27T20:52:39.804 INFO:tasks.rados.rados.0.smithi093.stdout:1042: writing smithi093153278-253 from 3157946 to 3479214 tid 3 2023-08-27T20:52:39.804 INFO:tasks.rados.rados.0.smithi093.stdout:1043: write oid 87 current snap is 0 2023-08-27T20:52:39.804 INFO:tasks.rados.rados.0.smithi093.stdout:1043: seq_num 803 ranges {696021=642012,1856570=575278,3185369=310652} 2023-08-27T20:52:39.811 INFO:tasks.rados.rados.0.smithi093.stdout:1043: writing smithi093153278-87 from 696021 to 1338033 tid 1 2023-08-27T20:52:39.817 INFO:tasks.rados.rados.0.smithi093.stdout:1043: writing smithi093153278-87 from 1856570 to 2431848 tid 2 2023-08-27T20:52:39.823 INFO:tasks.rados.rados.0.smithi093.stdout:1043: writing smithi093153278-87 from 3185369 to 3496021 tid 3 2023-08-27T20:52:39.824 INFO:tasks.rados.rados.0.smithi093.stdout:1044: read oid 51 snap -1 2023-08-27T20:52:39.824 INFO:tasks.rados.rados.0.smithi093.stdout:1044: expect (ObjNum 727 snap 0 seq_num 727) 2023-08-27T20:52:39.824 INFO:tasks.rados.rados.0.smithi093.stdout:1045: write oid 485 current snap is 0 2023-08-27T20:52:39.824 INFO:tasks.rados.rados.0.smithi093.stdout:1040: expect (ObjNum 126 snap 0 seq_num 126) 2023-08-27T20:52:39.888 INFO:tasks.rados.rados.0.smithi093.stdout:1039: expect (ObjNum 267 snap 0 seq_num 267) 2023-08-27T20:52:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3909216325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:39 smithi136 ceph-mon[113531]: pgmap v74: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 9.1 MiB/s wr, 99 op/s 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:1042: finishing write tid 1 to smithi093153278-253 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:1042: finishing write tid 2 to smithi093153278-253 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:1042: finishing write tid 3 to smithi093153278-253 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:1042: finishing write tid 4 to smithi093153278-253 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:1042: finishing write tid 5 to smithi093153278-253 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 146 (ObjNum 802 snap 0 seq_num 802) dirty exists 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:1042: left oid 253 (ObjNum 802 snap 0 seq_num 802) 2023-08-27T20:52:39.965 INFO:tasks.rados.rados.0.smithi093.stdout:1043: finishing write tid 1 to smithi093153278-87 2023-08-27T20:52:39.966 INFO:tasks.rados.rados.0.smithi093.stdout:1043: finishing write tid 2 to smithi093153278-87 2023-08-27T20:52:39.966 INFO:tasks.rados.rados.0.smithi093.stdout:1043: finishing write tid 3 to smithi093153278-87 2023-08-27T20:52:39.966 INFO:tasks.rados.rados.0.smithi093.stdout:1043: finishing write tid 4 to smithi093153278-87 2023-08-27T20:52:39.966 INFO:tasks.rados.rados.0.smithi093.stdout:1043: finishing write tid 5 to smithi093153278-87 2023-08-27T20:52:39.966 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 153 (ObjNum 803 snap 0 seq_num 803) dirty exists 2023-08-27T20:52:39.966 INFO:tasks.rados.rados.0.smithi093.stdout:1043: left oid 87 (ObjNum 803 snap 0 seq_num 803) 2023-08-27T20:52:39.966 INFO:tasks.rados.rados.0.smithi093.stdout:1044: expect (ObjNum 727 snap 0 seq_num 727) 2023-08-27T20:52:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3909216325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:39 smithi093 ceph-mon[111335]: pgmap v74: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 9.1 MiB/s wr, 99 op/s 2023-08-27T20:52:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3909216325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:39 smithi093 ceph-mon[118902]: pgmap v74: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 9.1 MiB/s wr, 99 op/s 2023-08-27T20:52:40.052 INFO:tasks.rados.rados.0.smithi093.stdout:1045: seq_num 804 ranges {712828=717261,1915052=718936,3212792=300036} 2023-08-27T20:52:40.059 INFO:tasks.rados.rados.0.smithi093.stdout:1045: writing smithi093153278-485 from 712828 to 1430089 tid 1 2023-08-27T20:52:40.066 INFO:tasks.rados.rados.0.smithi093.stdout:1045: writing smithi093153278-485 from 1915052 to 2633988 tid 2 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1045: writing smithi093153278-485 from 3212792 to 3512828 tid 3 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1039: done (5 left) 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1040: done (4 left) 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1042: done (3 left) 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1043: done (2 left) 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1044: done (1 left) 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1046: write oid 354 current snap is 0 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1045: finishing write tid 1 to smithi093153278-485 2023-08-27T20:52:40.072 INFO:tasks.rados.rados.0.smithi093.stdout:1045: finishing write tid 2 to smithi093153278-485 2023-08-27T20:52:40.073 INFO:tasks.rados.rados.0.smithi093.stdout:1046: seq_num 805 ranges {729635=792510,1973534=578947,3240215=289420} 2023-08-27T20:52:40.080 INFO:tasks.rados.rados.0.smithi093.stdout:1046: writing smithi093153278-354 from 729635 to 1522145 tid 1 2023-08-27T20:52:40.086 INFO:tasks.rados.rados.0.smithi093.stdout:1046: writing smithi093153278-354 from 1973534 to 2552481 tid 2 2023-08-27T20:52:40.091 INFO:tasks.rados.rados.0.smithi093.stdout:1046: writing smithi093153278-354 from 3240215 to 3529635 tid 3 2023-08-27T20:52:40.091 INFO:tasks.rados.rados.0.smithi093.stdout:1047: write oid 358 current snap is 0 2023-08-27T20:52:40.091 INFO:tasks.rados.rados.0.smithi093.stdout:1045: finishing write tid 3 to smithi093153278-485 2023-08-27T20:52:40.092 INFO:tasks.rados.rados.0.smithi093.stdout:1045: finishing write tid 4 to smithi093153278-485 2023-08-27T20:52:40.092 INFO:tasks.rados.rados.0.smithi093.stdout:1045: finishing write tid 5 to smithi093153278-485 2023-08-27T20:52:40.092 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 485 v 160 (ObjNum 804 snap 0 seq_num 804) dirty exists 2023-08-27T20:52:40.092 INFO:tasks.rados.rados.0.smithi093.stdout:1045: left oid 485 (ObjNum 804 snap 0 seq_num 804) 2023-08-27T20:52:40.092 INFO:tasks.rados.rados.0.smithi093.stdout:1046: finishing write tid 1 to smithi093153278-354 2023-08-27T20:52:40.092 INFO:tasks.rados.rados.0.smithi093.stdout:1046: finishing write tid 2 to smithi093153278-354 2023-08-27T20:52:40.092 INFO:tasks.rados.rados.0.smithi093.stdout:1047: seq_num 806 ranges {746442=584112,1748369=722605,2983991=562451} 2023-08-27T20:52:40.098 INFO:tasks.rados.rados.0.smithi093.stdout:1047: writing smithi093153278-358 from 746442 to 1330554 tid 1 2023-08-27T20:52:40.105 INFO:tasks.rados.rados.0.smithi093.stdout:1047: writing smithi093153278-358 from 1748369 to 2470974 tid 2 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1047: writing smithi093153278-358 from 2983991 to 3546442 tid 3 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1045: done (2 left) 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1048: read oid 474 snap -1 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1048: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1049: read oid 32 snap -1 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1049: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1050: write oid 265 current snap is 0 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1046: finishing write tid 3 to smithi093153278-354 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1046: finishing write tid 4 to smithi093153278-354 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:1046: finishing write tid 5 to smithi093153278-354 2023-08-27T20:52:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 354 v 186 (ObjNum 805 snap 0 seq_num 805) dirty exists 2023-08-27T20:52:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:1046: left oid 354 (ObjNum 805 snap 0 seq_num 805) 2023-08-27T20:52:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:1047: finishing write tid 1 to smithi093153278-358 2023-08-27T20:52:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:1047: finishing write tid 2 to smithi093153278-358 2023-08-27T20:52:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:1050: seq_num 807 ranges {763249=659361,2090498=582616,3295061=268188} 2023-08-27T20:52:40.119 INFO:tasks.rados.rados.0.smithi093.stdout:1050: writing smithi093153278-265 from 763249 to 1422610 tid 1 2023-08-27T20:52:40.126 INFO:tasks.rados.rados.0.smithi093.stdout:1050: writing smithi093153278-265 from 2090498 to 2673114 tid 2 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1050: writing smithi093153278-265 from 3295061 to 3563249 tid 3 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1046: done (4 left) 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1051: read oid 333 snap -1 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1051: expect (ObjNum 332 snap 0 seq_num 332) 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1052: write oid 384 current snap is 0 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1047: finishing write tid 3 to smithi093153278-358 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1047: finishing write tid 4 to smithi093153278-358 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1047: finishing write tid 5 to smithi093153278-358 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 358 v 168 (ObjNum 806 snap 0 seq_num 806) dirty exists 2023-08-27T20:52:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:1047: left oid 358 (ObjNum 806 snap 0 seq_num 806) 2023-08-27T20:52:40.133 INFO:tasks.rados.rados.0.smithi093.stdout:1049: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:52:40.307 INFO:tasks.rados.rados.0.smithi093.stdout:1048: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:52:40.399 INFO:tasks.rados.rados.0.smithi093.stdout:1050: finishing write tid 1 to smithi093153278-265 2023-08-27T20:52:40.399 INFO:tasks.rados.rados.0.smithi093.stdout:1050: finishing write tid 2 to smithi093153278-265 2023-08-27T20:52:40.399 INFO:tasks.rados.rados.0.smithi093.stdout:1050: finishing write tid 3 to smithi093153278-265 2023-08-27T20:52:40.400 INFO:tasks.rados.rados.0.smithi093.stdout:1050: finishing write tid 4 to smithi093153278-265 2023-08-27T20:52:40.400 INFO:tasks.rados.rados.0.smithi093.stdout:1050: finishing write tid 5 to smithi093153278-265 2023-08-27T20:52:40.400 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 265 v 187 (ObjNum 807 snap 0 seq_num 807) dirty exists 2023-08-27T20:52:40.400 INFO:tasks.rados.rados.0.smithi093.stdout:1050: left oid 265 (ObjNum 807 snap 0 seq_num 807) 2023-08-27T20:52:40.400 INFO:tasks.rados.rados.0.smithi093.stdout:1051: expect (ObjNum 332 snap 0 seq_num 332) 2023-08-27T20:52:40.512 INFO:tasks.rados.rados.0.smithi093.stdout:1052: seq_num 808 ranges {780056=734610,2148980=726274,3322484=257572} 2023-08-27T20:52:40.520 INFO:tasks.rados.rados.0.smithi093.stdout:1052: writing smithi093153278-384 from 780056 to 1514666 tid 1 2023-08-27T20:52:40.527 INFO:tasks.rados.rados.0.smithi093.stdout:1052: writing smithi093153278-384 from 2148980 to 2875254 tid 2 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1052: writing smithi093153278-384 from 3322484 to 3580056 tid 3 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1047: done (5 left) 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1048: done (4 left) 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1049: done (3 left) 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1050: done (2 left) 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1051: done (1 left) 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1053: write oid 202 current snap is 0 2023-08-27T20:52:40.532 INFO:tasks.rados.rados.0.smithi093.stdout:1053: seq_num 809 ranges {796863=409859,1807462=586285,3066260=530603} 2023-08-27T20:52:40.538 INFO:tasks.rados.rados.0.smithi093.stdout:1053: writing smithi093153278-202 from 796863 to 1206722 tid 1 2023-08-27T20:52:40.545 INFO:tasks.rados.rados.0.smithi093.stdout:1053: writing smithi093153278-202 from 1807462 to 2393747 tid 2 2023-08-27T20:52:40.551 INFO:tasks.rados.rados.0.smithi093.stdout:1053: writing smithi093153278-202 from 3066260 to 3596863 tid 3 2023-08-27T20:52:40.552 INFO:tasks.rados.rados.0.smithi093.stdout:1054: write oid 117 current snap is 0 2023-08-27T20:52:40.552 INFO:tasks.rados.rados.0.smithi093.stdout:1052: finishing write tid 1 to smithi093153278-384 2023-08-27T20:52:40.552 INFO:tasks.rados.rados.0.smithi093.stdout:1054: seq_num 810 ranges {413670=485108,1465944=729943,2977330=524801} 2023-08-27T20:52:40.556 INFO:tasks.rados.rados.0.smithi093.stdout:1054: writing smithi093153278-117 from 413670 to 898778 tid 1 2023-08-27T20:52:40.563 INFO:tasks.rados.rados.0.smithi093.stdout:1054: writing smithi093153278-117 from 1465944 to 2195887 tid 2 2023-08-27T20:52:40.571 INFO:tasks.rados.rados.0.smithi093.stdout:1054: writing smithi093153278-117 from 2977330 to 3502131 tid 3 2023-08-27T20:52:40.571 INFO:tasks.rados.rados.0.smithi093.stdout:1055: write oid 249 current snap is 0 2023-08-27T20:52:40.571 INFO:tasks.rados.rados.0.smithi093.stdout:1055: seq_num 811 ranges {430477=560357,1408073=473601,2488400=736073} 2023-08-27T20:52:40.576 INFO:tasks.rados.rados.0.smithi093.stdout:1055: writing smithi093153278-249 from 430477 to 990834 tid 1 2023-08-27T20:52:40.581 INFO:tasks.rados.rados.0.smithi093.stdout:1055: writing smithi093153278-249 from 1408073 to 1881674 tid 2 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1055: writing smithi093153278-249 from 2488400 to 3224473 tid 3 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1056: read oid 310 snap -1 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1056: expect (ObjNum 708 snap 0 seq_num 708) 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1057: read oid 150 snap -1 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1057: expect (ObjNum 149 snap 0 seq_num 149) 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1058: write oid 322 current snap is 0 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1052: finishing write tid 2 to smithi093153278-384 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1052: finishing write tid 3 to smithi093153278-384 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1052: finishing write tid 4 to smithi093153278-384 2023-08-27T20:52:40.589 INFO:tasks.rados.rados.0.smithi093.stdout:1058: seq_num 812 ranges {447284=635606,1866555=733612,3315823=331461} 2023-08-27T20:52:40.595 INFO:tasks.rados.rados.0.smithi093.stdout:1058: writing smithi093153278-322 from 447284 to 1082890 tid 1 2023-08-27T20:52:40.604 INFO:tasks.rados.rados.0.smithi093.stdout:1058: writing smithi093153278-322 from 1866555 to 2600167 tid 2 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:1058: writing smithi093153278-322 from 3315823 to 3647284 tid 3 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:1059: write oid 70 current snap is 0 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:1052: finishing write tid 5 to smithi093153278-384 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 384 v 172 (ObjNum 808 snap 0 seq_num 808) dirty exists 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:1052: left oid 384 (ObjNum 808 snap 0 seq_num 808) 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:1053: finishing write tid 1 to smithi093153278-202 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:1053: finishing write tid 2 to smithi093153278-202 2023-08-27T20:52:40.610 INFO:tasks.rados.rados.0.smithi093.stdout:1059: seq_num 813 ranges {464091=710855,1925037=477270,2943246=474970} 2023-08-27T20:52:40.616 INFO:tasks.rados.rados.0.smithi093.stdout:1059: writing smithi093153278-70 from 464091 to 1174946 tid 1 2023-08-27T20:52:40.623 INFO:tasks.rados.rados.0.smithi093.stdout:1059: writing smithi093153278-70 from 1925037 to 2402307 tid 2 2023-08-27T20:52:40.629 INFO:tasks.rados.rados.0.smithi093.stdout:1059: writing smithi093153278-70 from 2943246 to 3418216 tid 3 2023-08-27T20:52:40.629 INFO:tasks.rados.rados.0.smithi093.stdout:1052: done (7 left) 2023-08-27T20:52:40.629 INFO:tasks.rados.rados.0.smithi093.stdout:1060: delete oid 264 current snap is 0 2023-08-27T20:52:40.629 INFO:tasks.rados.rados.0.smithi093.stdout:1053: finishing write tid 3 to smithi093153278-202 2023-08-27T20:52:40.629 INFO:tasks.rados.rados.0.smithi093.stdout:1053: finishing write tid 4 to smithi093153278-202 2023-08-27T20:52:40.629 INFO:tasks.rados.rados.0.smithi093.stdout:1053: finishing write tid 5 to smithi093153278-202 2023-08-27T20:52:40.629 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 202 v 190 (ObjNum 809 snap 0 seq_num 809) dirty exists 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1053: left oid 202 (ObjNum 809 snap 0 seq_num 809) 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1054: finishing write tid 1 to smithi093153278-117 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1054: finishing write tid 2 to smithi093153278-117 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1054: finishing write tid 3 to smithi093153278-117 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1054: finishing write tid 4 to smithi093153278-117 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1054: finishing write tid 5 to smithi093153278-117 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 117 v 180 (ObjNum 810 snap 0 seq_num 810) dirty exists 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1054: left oid 117 (ObjNum 810 snap 0 seq_num 810) 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1055: finishing write tid 1 to smithi093153278-249 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1055: finishing write tid 2 to smithi093153278-249 2023-08-27T20:52:40.630 INFO:tasks.rados.rados.0.smithi093.stdout:1057: expect (ObjNum 149 snap 0 seq_num 149) 2023-08-27T20:52:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:40] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:52:40.706 INFO:tasks.rados.rados.0.smithi093.stdout:1055: finishing write tid 3 to smithi093153278-249 2023-08-27T20:52:40.706 INFO:tasks.rados.rados.0.smithi093.stdout:1055: finishing write tid 4 to smithi093153278-249 2023-08-27T20:52:40.707 INFO:tasks.rados.rados.0.smithi093.stdout:1055: finishing write tid 5 to smithi093153278-249 2023-08-27T20:52:40.707 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 249 v 184 (ObjNum 811 snap 0 seq_num 811) dirty exists 2023-08-27T20:52:40.707 INFO:tasks.rados.rados.0.smithi093.stdout:1055: left oid 249 (ObjNum 811 snap 0 seq_num 811) 2023-08-27T20:52:40.707 INFO:tasks.rados.rados.0.smithi093.stdout:1056: expect (ObjNum 708 snap 0 seq_num 708) 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1058: finishing write tid 1 to smithi093153278-322 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1058: finishing write tid 2 to smithi093153278-322 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1053: done (7 left) 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1054: done (6 left) 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1055: done (5 left) 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1056: done (4 left) 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1057: done (3 left) 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1060: done (2 left) 2023-08-27T20:52:40.876 INFO:tasks.rados.rados.0.smithi093.stdout:1061: write oid 151 current snap is 0 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1058: finishing write tid 3 to smithi093153278-322 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1058: finishing write tid 4 to smithi093153278-322 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1058: finishing write tid 5 to smithi093153278-322 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 322 v 150 (ObjNum 812 snap 0 seq_num 812) dirty exists 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1058: left oid 322 (ObjNum 812 snap 0 seq_num 812) 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1059: finishing write tid 1 to smithi093153278-70 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1059: finishing write tid 2 to smithi093153278-70 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1059: finishing write tid 3 to smithi093153278-70 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1059: finishing write tid 4 to smithi093153278-70 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1059: finishing write tid 5 to smithi093153278-70 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 70 v 164 (ObjNum 813 snap 0 seq_num 813) dirty exists 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1059: left oid 70 (ObjNum 813 snap 0 seq_num 813) 2023-08-27T20:52:40.877 INFO:tasks.rados.rados.0.smithi093.stdout:1061: seq_num 815 ranges {497705=577706,1642001=480939,2598092=497514} 2023-08-27T20:52:40.882 INFO:tasks.rados.rados.0.smithi093.stdout:1061: writing smithi093153278-151 from 497705 to 1075411 tid 1 2023-08-27T20:52:40.887 INFO:tasks.rados.rados.0.smithi093.stdout:1061: writing smithi093153278-151 from 1642001 to 2122940 tid 2 2023-08-27T20:52:40.893 INFO:tasks.rados.rados.0.smithi093.stdout:1061: writing smithi093153278-151 from 2598092 to 3095606 tid 3 2023-08-27T20:52:40.893 INFO:tasks.rados.rados.0.smithi093.stdout:1058: done (2 left) 2023-08-27T20:52:40.893 INFO:tasks.rados.rados.0.smithi093.stdout:1059: done (1 left) 2023-08-27T20:52:40.893 INFO:tasks.rados.rados.0.smithi093.stdout:1062: write oid 195 current snap is 0 2023-08-27T20:52:40.893 INFO:tasks.rados.rados.0.smithi093.stdout:1061: finishing write tid 1 to smithi093153278-151 2023-08-27T20:52:40.894 INFO:tasks.rados.rados.0.smithi093.stdout:1062: seq_num 816 ranges {514512=652955,1700483=740950,3025515=688997} 2023-08-27T20:52:40.899 INFO:tasks.rados.rados.0.smithi093.stdout:1062: writing smithi093153278-195 from 514512 to 1167467 tid 1 2023-08-27T20:52:40.906 INFO:tasks.rados.rados.0.smithi093.stdout:1062: writing smithi093153278-195 from 1700483 to 2441433 tid 2 2023-08-27T20:52:40.914 INFO:tasks.rados.rados.0.smithi093.stdout:1062: writing smithi093153278-195 from 3025515 to 3714512 tid 3 2023-08-27T20:52:40.914 INFO:tasks.rados.rados.0.smithi093.stdout:1063: read oid 58 snap -1 2023-08-27T20:52:40.914 INFO:tasks.rados.rados.0.smithi093.stdout:1063: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:52:40.914 INFO:tasks.rados.rados.0.smithi093.stdout:1064: write oid 339 current snap is 0 2023-08-27T20:52:40.914 INFO:tasks.rados.rados.0.smithi093.stdout:1061: finishing write tid 2 to smithi093153278-151 2023-08-27T20:52:40.914 INFO:tasks.rados.rados.0.smithi093.stdout:1064: seq_num 817 ranges {531319=728204,1758965=484608,2652938=520058} 2023-08-27T20:52:40.921 INFO:tasks.rados.rados.0.smithi093.stdout:1064: writing smithi093153278-339 from 531319 to 1259523 tid 1 2023-08-27T20:52:40.926 INFO:tasks.rados.rados.0.smithi093.stdout:1064: writing smithi093153278-339 from 1758965 to 2243573 tid 2 2023-08-27T20:52:40.931 INFO:tasks.rados.rados.0.smithi093.stdout:1064: writing smithi093153278-339 from 2652938 to 3172996 tid 3 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1065: write oid 143 current snap is 0 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1061: finishing write tid 3 to smithi093153278-151 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1061: finishing write tid 4 to smithi093153278-151 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1061: finishing write tid 5 to smithi093153278-151 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 151 v 168 (ObjNum 815 snap 0 seq_num 815) dirty exists 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1061: left oid 151 (ObjNum 815 snap 0 seq_num 815) 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1062: finishing write tid 1 to smithi093153278-195 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1062: finishing write tid 2 to smithi093153278-195 2023-08-27T20:52:40.932 INFO:tasks.rados.rados.0.smithi093.stdout:1065: seq_num 818 ranges {548126=403453,1417447=744619,2680361=447683,3680450=67676} 2023-08-27T20:52:40.937 INFO:tasks.rados.rados.0.smithi093.stdout:1065: writing smithi093153278-143 from 548126 to 951579 tid 1 2023-08-27T20:52:40.943 INFO:tasks.rados.rados.0.smithi093.stdout:1065: writing smithi093153278-143 from 1417447 to 2162066 tid 2 2023-08-27T20:52:40.949 INFO:tasks.rados.rados.0.smithi093.stdout:1065: writing smithi093153278-143 from 2680361 to 3128044 tid 3 2023-08-27T20:52:40.952 INFO:tasks.rados.rados.0.smithi093.stdout:1065: writing smithi093153278-143 from 3680450 to 3748126 tid 4 2023-08-27T20:52:40.953 INFO:tasks.rados.rados.0.smithi093.stdout:1061: done (4 left) 2023-08-27T20:52:40.953 INFO:tasks.rados.rados.0.smithi093.stdout:1066: read oid 245 snap -1 2023-08-27T20:52:40.953 INFO:tasks.rados.rados.0.smithi093.stdout:1066: expect (ObjNum 784 snap 0 seq_num 784) 2023-08-27T20:52:40.953 INFO:tasks.rados.rados.0.smithi093.stdout:1067: read oid 107 snap -1 2023-08-27T20:52:40.953 INFO:tasks.rados.rados.0.smithi093.stdout:1067: expect (ObjNum 529 snap 0 seq_num 529) 2023-08-27T20:52:40.953 INFO:tasks.rados.rados.0.smithi093.stdout:1068: write oid 481 current snap is 0 2023-08-27T20:52:40.953 INFO:tasks.rados.rados.0.smithi093.stdout:1068: seq_num 819 ranges {564933=478702,1759576=488277,2991431=658955} 2023-08-27T20:52:40.958 INFO:tasks.rados.rados.0.smithi093.stdout:1068: writing smithi093153278-481 from 564933 to 1043635 tid 1 2023-08-27T20:52:40.965 INFO:tasks.rados.rados.0.smithi093.stdout:1068: writing smithi093153278-481 from 1759576 to 2247853 tid 2 2023-08-27T20:52:40.974 INFO:tasks.rados.rados.0.smithi093.stdout:1068: writing smithi093153278-481 from 2991431 to 3650386 tid 3 2023-08-27T20:52:40.974 INFO:tasks.rados.rados.0.smithi093.stdout:1069: read oid 29 snap -1 2023-08-27T20:52:40.974 INFO:tasks.rados.rados.0.smithi093.stdout:1069: expect (ObjNum 28 snap 0 seq_num 28) 2023-08-27T20:52:40.974 INFO:tasks.rados.rados.0.smithi093.stdout:1070: read oid 160 snap -1 2023-08-27T20:52:40.974 INFO:tasks.rados.rados.0.smithi093.stdout:1070: expect (ObjNum 159 snap 0 seq_num 159) 2023-08-27T20:52:40.974 INFO:tasks.rados.rados.0.smithi093.stdout:1071: read oid 299 snap -1 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1071: expect (ObjNum 773 snap 0 seq_num 773) 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1072: read oid 30 snap -1 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1072: expect (ObjNum 29 snap 0 seq_num 29) 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1073: write oid 249 current snap is 0 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1062: finishing write tid 3 to smithi093153278-195 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1062: finishing write tid 4 to smithi093153278-195 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1062: finishing write tid 5 to smithi093153278-195 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 138 (ObjNum 816 snap 0 seq_num 816) dirty exists 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1062: left oid 195 (ObjNum 816 snap 0 seq_num 816) 2023-08-27T20:52:40.975 INFO:tasks.rados.rados.0.smithi093.stdout:1063: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:52:41.108 INFO:tasks.rados.rados.0.smithi093.stdout:1064: finishing write tid 1 to smithi093153278-339 2023-08-27T20:52:41.108 INFO:tasks.rados.rados.0.smithi093.stdout:1064: finishing write tid 2 to smithi093153278-339 2023-08-27T20:52:41.108 INFO:tasks.rados.rados.0.smithi093.stdout:1073: seq_num 820 ranges {581740=553951,1818058=748288,3018854=470227} 2023-08-27T20:52:41.114 INFO:tasks.rados.rados.0.smithi093.stdout:1073: writing smithi093153278-249 from 581740 to 1135691 tid 1 2023-08-27T20:52:41.122 INFO:tasks.rados.rados.0.smithi093.stdout:1073: writing smithi093153278-249 from 1818058 to 2566346 tid 2 2023-08-27T20:52:41.127 INFO:tasks.rados.rados.0.smithi093.stdout:1073: writing smithi093153278-249 from 3018854 to 3489081 tid 3 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1062: done (11 left) 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1063: done (10 left) 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1074: write oid 123 current snap is 0 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1064: finishing write tid 3 to smithi093153278-339 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1064: finishing write tid 4 to smithi093153278-339 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1064: finishing write tid 5 to smithi093153278-339 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 339 v 184 (ObjNum 817 snap 0 seq_num 817) dirty exists 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1064: left oid 339 (ObjNum 817 snap 0 seq_num 817) 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1065: finishing write tid 1 to smithi093153278-143 2023-08-27T20:52:41.128 INFO:tasks.rados.rados.0.smithi093.stdout:1074: seq_num 821 ranges {598547=629200,1876540=491946,3046277=681499} 2023-08-27T20:52:41.134 INFO:tasks.rados.rados.0.smithi093.stdout:1074: writing smithi093153278-123 from 598547 to 1227747 tid 1 2023-08-27T20:52:41.140 INFO:tasks.rados.rados.0.smithi093.stdout:1074: writing smithi093153278-123 from 1876540 to 2368486 tid 2 2023-08-27T20:52:41.148 INFO:tasks.rados.rados.0.smithi093.stdout:1074: writing smithi093153278-123 from 3046277 to 3727776 tid 3 2023-08-27T20:52:41.148 INFO:tasks.rados.rados.0.smithi093.stdout:1064: done (10 left) 2023-08-27T20:52:41.148 INFO:tasks.rados.rados.0.smithi093.stdout:1075: write oid 280 current snap is 0 2023-08-27T20:52:41.148 INFO:tasks.rados.rados.0.smithi093.stdout:1065: finishing write tid 2 to smithi093153278-143 2023-08-27T20:52:41.148 INFO:tasks.rados.rados.0.smithi093.stdout:1065: finishing write tid 3 to smithi093153278-143 2023-08-27T20:52:41.149 INFO:tasks.rados.rados.0.smithi093.stdout:1075: seq_num 822 ranges {615354=420802,1651375=751957,3190053=492771} 2023-08-27T20:52:41.154 INFO:tasks.rados.rados.0.smithi093.stdout:1075: writing smithi093153278-280 from 615354 to 1036156 tid 1 2023-08-27T20:52:41.161 INFO:tasks.rados.rados.0.smithi093.stdout:1075: writing smithi093153278-280 from 1651375 to 2403332 tid 2 2023-08-27T20:52:41.169 INFO:tasks.rados.rados.0.smithi093.stdout:1075: writing smithi093153278-280 from 3190053 to 3682824 tid 3 2023-08-27T20:52:41.169 INFO:tasks.rados.rados.0.smithi093.stdout:1076: read oid 275 snap -1 2023-08-27T20:52:41.169 INFO:tasks.rados.rados.0.smithi093.stdout:1076: expect (ObjNum 274 snap 0 seq_num 274) 2023-08-27T20:52:41.169 INFO:tasks.rados.rados.0.smithi093.stdout:1077: write oid 284 current snap is 0 2023-08-27T20:52:41.169 INFO:tasks.rados.rados.0.smithi093.stdout:1065: finishing write tid 4 to smithi093153278-143 2023-08-27T20:52:41.169 INFO:tasks.rados.rados.0.smithi093.stdout:1065: finishing write tid 5 to smithi093153278-143 2023-08-27T20:52:41.169 INFO:tasks.rados.rados.0.smithi093.stdout:1077: seq_num 823 ranges {632161=496051,1593504=495615,2701123=420396,3611645=220516} 2023-08-27T20:52:41.175 INFO:tasks.rados.rados.0.smithi093.stdout:1077: writing smithi093153278-284 from 632161 to 1128212 tid 1 2023-08-27T20:52:41.180 INFO:tasks.rados.rados.0.smithi093.stdout:1077: writing smithi093153278-284 from 1593504 to 2089119 tid 2 2023-08-27T20:52:41.186 INFO:tasks.rados.rados.0.smithi093.stdout:1077: writing smithi093153278-284 from 2701123 to 3121519 tid 3 2023-08-27T20:52:41.190 INFO:tasks.rados.rados.0.smithi093.stdout:1077: writing smithi093153278-284 from 3611645 to 3832161 tid 4 2023-08-27T20:52:41.190 INFO:tasks.rados.rados.0.smithi093.stdout:1078: write oid 412 current snap is 0 2023-08-27T20:52:41.190 INFO:tasks.rados.rados.0.smithi093.stdout:1065: finishing write tid 6 to smithi093153278-143 2023-08-27T20:52:41.190 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 143 v 155 (ObjNum 818 snap 0 seq_num 818) dirty exists 2023-08-27T20:52:41.190 INFO:tasks.rados.rados.0.smithi093.stdout:1065: left oid 143 (ObjNum 818 snap 0 seq_num 818) 2023-08-27T20:52:41.190 INFO:tasks.rados.rados.0.smithi093.stdout:1066: expect (ObjNum 784 snap 0 seq_num 784) 2023-08-27T20:52:41.370 INFO:tasks.rados.rados.0.smithi093.stdout:1067: expect (ObjNum 529 snap 0 seq_num 529) 2023-08-27T20:52:41.502 INFO:tasks.rados.rados.0.smithi093.stdout:1068: finishing write tid 1 to smithi093153278-481 2023-08-27T20:52:41.503 INFO:tasks.rados.rados.0.smithi093.stdout:1068: finishing write tid 2 to smithi093153278-481 2023-08-27T20:52:41.503 INFO:tasks.rados.rados.0.smithi093.stdout:1069: expect (ObjNum 28 snap 0 seq_num 28) 2023-08-27T20:52:41.579 INFO:tasks.rados.rados.0.smithi093.stdout:1070: expect (ObjNum 159 snap 0 seq_num 159) 2023-08-27T20:52:41.663 INFO:tasks.rados.rados.0.smithi093.stdout:1068: finishing write tid 3 to smithi093153278-481 2023-08-27T20:52:41.663 INFO:tasks.rados.rados.0.smithi093.stdout:1068: finishing write tid 4 to smithi093153278-481 2023-08-27T20:52:41.663 INFO:tasks.rados.rados.0.smithi093.stdout:1068: finishing write tid 5 to smithi093153278-481 2023-08-27T20:52:41.663 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 481 v 172 (ObjNum 819 snap 0 seq_num 819) dirty exists 2023-08-27T20:52:41.663 INFO:tasks.rados.rados.0.smithi093.stdout:1068: left oid 481 (ObjNum 819 snap 0 seq_num 819) 2023-08-27T20:52:41.663 INFO:tasks.rados.rados.0.smithi093.stdout:1071: expect (ObjNum 773 snap 0 seq_num 773) 2023-08-27T20:52:41.790 INFO:tasks.rados.rados.0.smithi093.stdout:1072: expect (ObjNum 29 snap 0 seq_num 29) 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1073: finishing write tid 1 to smithi093153278-249 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1073: finishing write tid 2 to smithi093153278-249 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1073: finishing write tid 3 to smithi093153278-249 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1073: finishing write tid 4 to smithi093153278-249 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1073: finishing write tid 5 to smithi093153278-249 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 249 v 188 (ObjNum 820 snap 0 seq_num 820) dirty exists 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1073: left oid 249 (ObjNum 820 snap 0 seq_num 820) 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1074: finishing write tid 1 to smithi093153278-123 2023-08-27T20:52:41.869 INFO:tasks.rados.rados.0.smithi093.stdout:1078: seq_num 824 ranges {648968=571300,1651986=639273,2728546=631668} 2023-08-27T20:52:41.875 INFO:tasks.rados.rados.0.smithi093.stdout:1078: writing smithi093153278-412 from 648968 to 1220268 tid 1 2023-08-27T20:52:41.881 INFO:tasks.rados.rados.0.smithi093.stdout:1078: writing smithi093153278-412 from 1651986 to 2291259 tid 2 2023-08-27T20:52:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:1078: writing smithi093153278-412 from 2728546 to 3360214 tid 3 2023-08-27T20:52:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:1065: done (13 left) 2023-08-27T20:52:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:1066: done (12 left) 2023-08-27T20:52:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:1067: done (11 left) 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1068: done (10 left) 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1069: done (9 left) 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1070: done (8 left) 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1071: done (7 left) 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1072: done (6 left) 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1073: done (5 left) 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1079: delete oid 346 current snap is 0 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1074: finishing write tid 2 to smithi093153278-123 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1074: finishing write tid 3 to smithi093153278-123 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1074: finishing write tid 4 to smithi093153278-123 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1074: finishing write tid 5 to smithi093153278-123 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 123 v 160 (ObjNum 821 snap 0 seq_num 821) dirty exists 2023-08-27T20:52:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:1074: left oid 123 (ObjNum 821 snap 0 seq_num 821) 2023-08-27T20:52:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:1075: finishing write tid 1 to smithi093153278-280 2023-08-27T20:52:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:1075: finishing write tid 2 to smithi093153278-280 2023-08-27T20:52:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:1076: expect (ObjNum 274 snap 0 seq_num 274) 2023-08-27T20:52:41.968 INFO:tasks.rados.rados.0.smithi093.stdout:1075: finishing write tid 3 to smithi093153278-280 2023-08-27T20:52:41.968 INFO:tasks.rados.rados.0.smithi093.stdout:1075: finishing write tid 4 to smithi093153278-280 2023-08-27T20:52:41.968 INFO:tasks.rados.rados.0.smithi093.stdout:1075: finishing write tid 5 to smithi093153278-280 2023-08-27T20:52:41.968 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 280 v 142 (ObjNum 822 snap 0 seq_num 822) dirty exists 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1075: left oid 280 (ObjNum 822 snap 0 seq_num 822) 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1077: finishing write tid 1 to smithi093153278-284 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1077: finishing write tid 2 to smithi093153278-284 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1077: finishing write tid 3 to smithi093153278-284 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1077: finishing write tid 4 to smithi093153278-284 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1077: finishing write tid 5 to smithi093153278-284 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1074: done (5 left) 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1075: done (4 left) 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1076: done (3 left) 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1079: done (2 left) 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1080: write oid 91 current snap is 0 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:1077: finishing write tid 6 to smithi093153278-284 2023-08-27T20:52:41.969 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 284 v 177 (ObjNum 823 snap 0 seq_num 823) dirty exists 2023-08-27T20:52:41.970 INFO:tasks.rados.rados.0.smithi093.stdout:1077: left oid 284 (ObjNum 823 snap 0 seq_num 823) 2023-08-27T20:52:41.970 INFO:tasks.rados.rados.0.smithi093.stdout:1078: finishing write tid 1 to smithi093153278-412 2023-08-27T20:52:41.970 INFO:tasks.rados.rados.0.smithi093.stdout:1080: seq_num 826 ranges {682582=721798,2168950=642942,3583392=299190} 2023-08-27T20:52:41.975 INFO:tasks.rados.rados.0.smithi093.stdout:1080: writing smithi093153278-91 from 682582 to 1404380 tid 1 2023-08-27T20:52:41.983 INFO:tasks.rados.rados.0.smithi093.stdout:1080: writing smithi093153278-91 from 2168950 to 2811892 tid 2 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1080: writing smithi093153278-91 from 3583392 to 3882582 tid 3 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1077: done (2 left) 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1081: write oid 477 current snap is 0 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1078: finishing write tid 2 to smithi093153278-412 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1078: finishing write tid 3 to smithi093153278-412 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1078: finishing write tid 4 to smithi093153278-412 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1078: finishing write tid 5 to smithi093153278-412 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 412 v 173 (ObjNum 824 snap 0 seq_num 824) dirty exists 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1078: left oid 412 (ObjNum 824 snap 0 seq_num 824) 2023-08-27T20:52:41.990 INFO:tasks.rados.rados.0.smithi093.stdout:1080: finishing write tid 1 to smithi093153278-91 2023-08-27T20:52:41.991 INFO:tasks.rados.rados.0.smithi093.stdout:1080: finishing write tid 2 to smithi093153278-91 2023-08-27T20:52:41.991 INFO:tasks.rados.rados.0.smithi093.stdout:1081: seq_num 827 ranges {699389=797047,2111079=502953,3094462=581837} 2023-08-27T20:52:41.997 INFO:tasks.rados.rados.0.smithi093.stdout:1081: writing smithi093153278-477 from 699389 to 1496436 tid 1 2023-08-27T20:52:42.003 INFO:tasks.rados.rados.0.smithi093.stdout:1081: writing smithi093153278-477 from 2111079 to 2614032 tid 2 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1081: writing smithi093153278-477 from 3094462 to 3676299 tid 3 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1078: done (2 left) 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1082: delete oid 434 current snap is 0 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1080: finishing write tid 3 to smithi093153278-91 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1080: finishing write tid 4 to smithi093153278-91 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1080: finishing write tid 5 to smithi093153278-91 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 91 v 199 (ObjNum 826 snap 0 seq_num 826) dirty exists 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1080: left oid 91 (ObjNum 826 snap 0 seq_num 826) 2023-08-27T20:52:42.010 INFO:tasks.rados.rados.0.smithi093.stdout:1081: finishing write tid 1 to smithi093153278-477 2023-08-27T20:52:42.011 INFO:tasks.rados.rados.0.smithi093.stdout:1081: finishing write tid 2 to smithi093153278-477 2023-08-27T20:52:42.011 INFO:tasks.rados.rados.0.smithi093.stdout:1080: done (2 left) 2023-08-27T20:52:42.011 INFO:tasks.rados.rados.0.smithi093.stdout:1082: done (1 left) 2023-08-27T20:52:42.011 INFO:tasks.rados.rados.0.smithi093.stdout:1083: write oid 491 current snap is 0 2023-08-27T20:52:42.012 INFO:tasks.rados.rados.0.smithi093.stdout:1083: seq_num 829 ranges {733003=663898,1944396=506622,2865661=604381} 2023-08-27T20:52:42.018 INFO:tasks.rados.rados.0.smithi093.stdout:1083: writing smithi093153278-491 from 733003 to 1396901 tid 1 2023-08-27T20:52:42.025 INFO:tasks.rados.rados.0.smithi093.stdout:1083: writing smithi093153278-491 from 1944396 to 2451018 tid 2 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1083: writing smithi093153278-491 from 2865661 to 3470042 tid 3 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1084: read oid 231 snap -1 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1084: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1085: read oid 163 snap -1 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1085: expect (ObjNum 162 snap 0 seq_num 162) 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1086: write oid 182 current snap is 0 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1081: finishing write tid 3 to smithi093153278-477 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1081: finishing write tid 4 to smithi093153278-477 2023-08-27T20:52:42.031 INFO:tasks.rados.rados.0.smithi093.stdout:1086: seq_num 830 ranges {749810=739147,2002878=650280,3293084=415653} 2023-08-27T20:52:42.039 INFO:tasks.rados.rados.0.smithi093.stdout:1086: writing smithi093153278-182 from 749810 to 1488957 tid 1 2023-08-27T20:52:42.046 INFO:tasks.rados.rados.0.smithi093.stdout:1086: writing smithi093153278-182 from 2002878 to 2653158 tid 2 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:1086: writing smithi093153278-182 from 3293084 to 3708737 tid 3 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:1087: write oid 459 current snap is 0 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:1081: finishing write tid 5 to smithi093153278-477 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 477 v 203 (ObjNum 827 snap 0 seq_num 827) dirty exists 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:1081: left oid 477 (ObjNum 827 snap 0 seq_num 827) 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:1083: finishing write tid 1 to smithi093153278-491 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:1083: finishing write tid 2 to smithi093153278-491 2023-08-27T20:52:42.052 INFO:tasks.rados.rados.0.smithi093.stdout:1087: seq_num 831 ranges {766617=414396,1945007=510291,3204154=626925} 2023-08-27T20:52:42.058 INFO:tasks.rados.rados.0.smithi093.stdout:1087: writing smithi093153278-459 from 766617 to 1181013 tid 1 2023-08-27T20:52:42.065 INFO:tasks.rados.rados.0.smithi093.stdout:1087: writing smithi093153278-459 from 1945007 to 2455298 tid 2 2023-08-27T20:52:42.073 INFO:tasks.rados.rados.0.smithi093.stdout:1087: writing smithi093153278-459 from 3204154 to 3831079 tid 3 2023-08-27T20:52:42.073 INFO:tasks.rados.rados.0.smithi093.stdout:1081: done (5 left) 2023-08-27T20:52:42.073 INFO:tasks.rados.rados.0.smithi093.stdout:1088: read oid 420 snap -1 2023-08-27T20:52:42.073 INFO:tasks.rados.rados.0.smithi093.stdout:1088: expect (ObjNum 419 snap 0 seq_num 419) 2023-08-27T20:52:42.073 INFO:tasks.rados.rados.0.smithi093.stdout:1089: write oid 326 current snap is 0 2023-08-27T20:52:42.074 INFO:tasks.rados.rados.0.smithi093.stdout:1083: finishing write tid 3 to smithi093153278-491 2023-08-27T20:52:42.074 INFO:tasks.rados.rados.0.smithi093.stdout:1083: finishing write tid 4 to smithi093153278-491 2023-08-27T20:52:42.074 INFO:tasks.rados.rados.0.smithi093.stdout:1089: seq_num 832 ranges {783424=489645,2003489=653949,3231577=554550} 2023-08-27T20:52:42.080 INFO:tasks.rados.rados.0.smithi093.stdout:1089: writing smithi093153278-326 from 783424 to 1273069 tid 1 2023-08-27T20:52:42.088 INFO:tasks.rados.rados.0.smithi093.stdout:1089: writing smithi093153278-326 from 2003489 to 2657438 tid 2 2023-08-27T20:52:42.094 INFO:tasks.rados.rados.0.smithi093.stdout:1089: writing smithi093153278-326 from 3231577 to 3786127 tid 3 2023-08-27T20:52:42.094 INFO:tasks.rados.rados.0.smithi093.stdout:1090: write oid 350 current snap is 0 2023-08-27T20:52:42.095 INFO:tasks.rados.rados.0.smithi093.stdout:1083: finishing write tid 5 to smithi093153278-491 2023-08-27T20:52:42.095 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 491 v 159 (ObjNum 829 snap 0 seq_num 829) dirty exists 2023-08-27T20:52:42.095 INFO:tasks.rados.rados.0.smithi093.stdout:1083: left oid 491 (ObjNum 829 snap 0 seq_num 829) 2023-08-27T20:52:42.095 INFO:tasks.rados.rados.0.smithi093.stdout:1085: expect (ObjNum 162 snap 0 seq_num 162) 2023-08-27T20:52:42.178 INFO:tasks.rados.rados.0.smithi093.stdout:1084: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:52:42.298 INFO:tasks.rados.rados.0.smithi093.stdout:1086: finishing write tid 1 to smithi093153278-182 2023-08-27T20:52:42.299 INFO:tasks.rados.rados.0.smithi093.stdout:1086: finishing write tid 2 to smithi093153278-182 2023-08-27T20:52:42.299 INFO:tasks.rados.rados.0.smithi093.stdout:1090: seq_num 833 ranges {400231=564894,1661971=338260} 2023-08-27T20:52:42.303 INFO:tasks.rados.rados.0.smithi093.stdout:1090: writing smithi093153278-350 from 400231 to 965125 tid 1 2023-08-27T20:52:42.310 INFO:tasks.rados.rados.0.smithi093.stdout:1090: writing smithi093153278-350 from 1661971 to 2000231 tid 2 2023-08-27T20:52:42.310 INFO:tasks.rados.rados.0.smithi093.stdout:1083: done (7 left) 2023-08-27T20:52:42.310 INFO:tasks.rados.rados.0.smithi093.stdout:1084: done (6 left) 2023-08-27T20:52:42.310 INFO:tasks.rados.rados.0.smithi093.stdout:1085: done (5 left) 2023-08-27T20:52:42.310 INFO:tasks.rados.rados.0.smithi093.stdout:1091: read oid 119 snap -1 2023-08-27T20:52:42.311 INFO:tasks.rados.rados.0.smithi093.stdout:1091: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:52:42.311 INFO:tasks.rados.rados.0.smithi093.stdout:1092: write oid 355 current snap is 0 2023-08-27T20:52:42.311 INFO:tasks.rados.rados.0.smithi093.stdout:1086: finishing write tid 3 to smithi093153278-182 2023-08-27T20:52:42.311 INFO:tasks.rados.rados.0.smithi093.stdout:1086: finishing write tid 4 to smithi093153278-182 2023-08-27T20:52:42.311 INFO:tasks.rados.rados.0.smithi093.stdout:1092: seq_num 834 ranges {417038=640143,1720453=296585} 2023-08-27T20:52:42.315 INFO:tasks.rados.rados.0.smithi093.stdout:1092: writing smithi093153278-355 from 417038 to 1057181 tid 1 2023-08-27T20:52:42.320 INFO:tasks.rados.rados.0.smithi093.stdout:1092: writing smithi093153278-355 from 1720453 to 2017038 tid 2 2023-08-27T20:52:42.320 INFO:tasks.rados.rados.0.smithi093.stdout:1093: delete oid 52 current snap is 0 2023-08-27T20:52:42.320 INFO:tasks.rados.rados.0.smithi093.stdout:1086: finishing write tid 5 to smithi093153278-182 2023-08-27T20:52:42.321 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 182 v 163 (ObjNum 830 snap 0 seq_num 830) dirty exists 2023-08-27T20:52:42.321 INFO:tasks.rados.rados.0.smithi093.stdout:1086: left oid 182 (ObjNum 830 snap 0 seq_num 830) 2023-08-27T20:52:42.321 INFO:tasks.rados.rados.0.smithi093.stdout:1087: finishing write tid 1 to smithi093153278-459 2023-08-27T20:52:42.321 INFO:tasks.rados.rados.0.smithi093.stdout:1087: finishing write tid 2 to smithi093153278-459 2023-08-27T20:52:42.321 INFO:tasks.rados.rados.0.smithi093.stdout:1088: expect (ObjNum 419 snap 0 seq_num 419) 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:1087: finishing write tid 3 to smithi093153278-459 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:1086: done (7 left) 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:1088: done (6 left) 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:1093: done (5 left) 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:1094: write oid 285 current snap is 0 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:1087: finishing write tid 4 to smithi093153278-459 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:1087: finishing write tid 5 to smithi093153278-459 2023-08-27T20:52:42.416 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 459 v 188 (ObjNum 831 snap 0 seq_num 831) dirty exists 2023-08-27T20:52:42.417 INFO:tasks.rados.rados.0.smithi093.stdout:1087: left oid 459 (ObjNum 831 snap 0 seq_num 831) 2023-08-27T20:52:42.417 INFO:tasks.rados.rados.0.smithi093.stdout:1089: finishing write tid 1 to smithi093153278-326 2023-08-27T20:52:42.417 INFO:tasks.rados.rados.0.smithi093.stdout:1094: seq_num 836 ranges {450652=790641,1721064=329588} 2023-08-27T20:52:42.422 INFO:tasks.rados.rados.0.smithi093.stdout:1094: writing smithi093153278-285 from 450652 to 1241293 tid 1 2023-08-27T20:52:42.427 INFO:tasks.rados.rados.0.smithi093.stdout:1094: writing smithi093153278-285 from 1721064 to 2050652 tid 2 2023-08-27T20:52:42.427 INFO:tasks.rados.rados.0.smithi093.stdout:1087: done (5 left) 2023-08-27T20:52:42.427 INFO:tasks.rados.rados.0.smithi093.stdout:1095: read oid 500 snap -1 2023-08-27T20:52:42.427 INFO:tasks.rados.rados.0.smithi093.stdout:1095: expect (ObjNum 641 snap 0 seq_num 641) 2023-08-27T20:52:42.428 INFO:tasks.rados.rados.0.smithi093.stdout:1096: write oid 332 current snap is 0 2023-08-27T20:52:42.428 INFO:tasks.rados.rados.0.smithi093.stdout:1089: finishing write tid 2 to smithi093153278-326 2023-08-27T20:52:42.428 INFO:tasks.rados.rados.0.smithi093.stdout:1089: finishing write tid 3 to smithi093153278-326 2023-08-27T20:52:42.428 INFO:tasks.rados.rados.0.smithi093.stdout:1096: seq_num 837 ranges {467459=582243,1495899=404945} 2023-08-27T20:52:42.433 INFO:tasks.rados.rados.0.smithi093.stdout:1096: writing smithi093153278-332 from 467459 to 1049702 tid 1 2023-08-27T20:52:42.438 INFO:tasks.rados.rados.0.smithi093.stdout:1096: writing smithi093153278-332 from 1495899 to 1900844 tid 2 2023-08-27T20:52:42.438 INFO:tasks.rados.rados.0.smithi093.stdout:1097: read oid 152 snap -1 2023-08-27T20:52:42.438 INFO:tasks.rados.rados.0.smithi093.stdout:1097: expect (ObjNum 728 snap 0 seq_num 728) 2023-08-27T20:52:42.438 INFO:tasks.rados.rados.0.smithi093.stdout:1098: write oid 143 current snap is 0 2023-08-27T20:52:42.438 INFO:tasks.rados.rados.0.smithi093.stdout:1089: finishing write tid 4 to smithi093153278-326 2023-08-27T20:52:42.438 INFO:tasks.rados.rados.0.smithi093.stdout:1098: seq_num 838 ranges {484266=657492,1554381=529885} 2023-08-27T20:52:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:42 smithi136 ceph-mon[113531]: pgmap v75: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 9.1 MiB/s wr, 99 op/s 2023-08-27T20:52:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:42 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2375329990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:42.444 INFO:tasks.rados.rados.0.smithi093.stdout:1098: writing smithi093153278-143 from 484266 to 1141758 tid 1 2023-08-27T20:52:42.450 INFO:tasks.rados.rados.0.smithi093.stdout:1098: writing smithi093153278-143 from 1554381 to 2084266 tid 2 2023-08-27T20:52:42.450 INFO:tasks.rados.rados.0.smithi093.stdout:1099: read oid 245 snap -1 2023-08-27T20:52:42.450 INFO:tasks.rados.rados.0.smithi093.stdout:1099: expect (ObjNum 784 snap 0 seq_num 784) 2023-08-27T20:52:42.450 INFO:tasks.rados.rados.0.smithi093.stdout:1100: read oid 392 snap -1 2023-08-27T20:52:42.450 INFO:tasks.rados.rados.0.smithi093.stdout:1100: expect (ObjNum 391 snap 0 seq_num 391) 2023-08-27T20:52:42.450 INFO:tasks.rados.rados.0.smithi093.stdout:1101: write oid 117 current snap is 0 2023-08-27T20:52:42.451 INFO:tasks.rados.rados.0.smithi093.stdout:1089: finishing write tid 5 to smithi093153278-326 2023-08-27T20:52:42.451 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 163 (ObjNum 832 snap 0 seq_num 832) dirty exists 2023-08-27T20:52:42.451 INFO:tasks.rados.rados.0.smithi093.stdout:1089: left oid 326 (ObjNum 832 snap 0 seq_num 832) 2023-08-27T20:52:42.451 INFO:tasks.rados.rados.0.smithi093.stdout:1090: finishing write tid 1 to smithi093153278-350 2023-08-27T20:52:42.451 INFO:tasks.rados.rados.0.smithi093.stdout:1090: finishing write tid 2 to smithi093153278-350 2023-08-27T20:52:42.451 INFO:tasks.rados.rados.0.smithi093.stdout:1090: finishing write tid 3 to smithi093153278-350 2023-08-27T20:52:42.451 INFO:tasks.rados.rados.0.smithi093.stdout:1101: seq_num 839 ranges {501073=732741,1896510=204563} 2023-08-27T20:52:42.457 INFO:tasks.rados.rados.0.smithi093.stdout:1101: writing smithi093153278-117 from 501073 to 1233814 tid 1 2023-08-27T20:52:42.462 INFO:tasks.rados.rados.0.smithi093.stdout:1101: writing smithi093153278-117 from 1896510 to 2101073 tid 2 2023-08-27T20:52:42.462 INFO:tasks.rados.rados.0.smithi093.stdout:1089: done (11 left) 2023-08-27T20:52:42.462 INFO:tasks.rados.rados.0.smithi093.stdout:1102: read oid 121 snap -1 2023-08-27T20:52:42.462 INFO:tasks.rados.rados.0.smithi093.stdout:1102: expect (ObjNum 120 snap 0 seq_num 120) 2023-08-27T20:52:42.462 INFO:tasks.rados.rados.0.smithi093.stdout:1103: read oid 53 snap -1 2023-08-27T20:52:42.462 INFO:tasks.rados.rados.0.smithi093.stdout:1103: expect (ObjNum 52 snap 0 seq_num 52) 2023-08-27T20:52:42.462 INFO:tasks.rados.rados.0.smithi093.stdout:1104: write oid 442 current snap is 0 2023-08-27T20:52:42.463 INFO:tasks.rados.rados.0.smithi093.stdout:1090: finishing write tid 4 to smithi093153278-350 2023-08-27T20:52:42.463 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 350 v 145 (ObjNum 833 snap 0 seq_num 833) dirty exists 2023-08-27T20:52:42.463 INFO:tasks.rados.rados.0.smithi093.stdout:1090: left oid 350 (ObjNum 833 snap 0 seq_num 833) 2023-08-27T20:52:42.463 INFO:tasks.rados.rados.0.smithi093.stdout:1091: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:52:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:42 smithi093 ceph-mon[118902]: pgmap v75: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 9.1 MiB/s wr, 99 op/s 2023-08-27T20:52:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:42 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2375329990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:42 smithi093 ceph-mon[111335]: pgmap v75: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 70 MiB/s rd, 9.1 MiB/s wr, 99 op/s 2023-08-27T20:52:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:42 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2375329990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:42.587 INFO:tasks.rados.rados.0.smithi093.stdout:1092: finishing write tid 1 to smithi093153278-355 2023-08-27T20:52:42.587 INFO:tasks.rados.rados.0.smithi093.stdout:1092: finishing write tid 2 to smithi093153278-355 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:1092: finishing write tid 3 to smithi093153278-355 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:1092: finishing write tid 4 to smithi093153278-355 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 355 v 148 (ObjNum 834 snap 0 seq_num 834) dirty exists 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:1092: left oid 355 (ObjNum 834 snap 0 seq_num 834) 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:1094: finishing write tid 1 to smithi093153278-285 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:1094: finishing write tid 2 to smithi093153278-285 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:1094: finishing write tid 3 to smithi093153278-285 2023-08-27T20:52:42.588 INFO:tasks.rados.rados.0.smithi093.stdout:1104: seq_num 840 ranges {517880=407990,1554992=562888} 2023-08-27T20:52:42.592 INFO:tasks.rados.rados.0.smithi093.stdout:1104: writing smithi093153278-442 from 517880 to 925870 tid 1 2023-08-27T20:52:42.599 INFO:tasks.rados.rados.0.smithi093.stdout:1104: writing smithi093153278-442 from 1554992 to 2117880 tid 2 2023-08-27T20:52:42.599 INFO:tasks.rados.rados.0.smithi093.stdout:1090: done (13 left) 2023-08-27T20:52:42.599 INFO:tasks.rados.rados.0.smithi093.stdout:1091: done (12 left) 2023-08-27T20:52:42.599 INFO:tasks.rados.rados.0.smithi093.stdout:1092: done (11 left) 2023-08-27T20:52:42.599 INFO:tasks.rados.rados.0.smithi093.stdout:1105: read oid 82 snap -1 2023-08-27T20:52:42.599 INFO:tasks.rados.rados.0.smithi093.stdout:1105: expect (ObjNum 738 snap 0 seq_num 738) 2023-08-27T20:52:42.600 INFO:tasks.rados.rados.0.smithi093.stdout:1106: write oid 279 current snap is 0 2023-08-27T20:52:42.600 INFO:tasks.rados.rados.0.smithi093.stdout:1094: finishing write tid 4 to smithi093153278-285 2023-08-27T20:52:42.600 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 285 v 191 (ObjNum 836 snap 0 seq_num 836) dirty exists 2023-08-27T20:52:42.600 INFO:tasks.rados.rados.0.smithi093.stdout:1094: left oid 285 (ObjNum 836 snap 0 seq_num 836) 2023-08-27T20:52:42.600 INFO:tasks.rados.rados.0.smithi093.stdout:1095: expect (ObjNum 641 snap 0 seq_num 641) 2023-08-27T20:52:42.743 INFO:tasks.rados.rados.0.smithi093.stdout:1096: finishing write tid 1 to smithi093153278-332 2023-08-27T20:52:42.743 INFO:tasks.rados.rados.0.smithi093.stdout:1096: finishing write tid 2 to smithi093153278-332 2023-08-27T20:52:42.743 INFO:tasks.rados.rados.0.smithi093.stdout:1096: finishing write tid 3 to smithi093153278-332 2023-08-27T20:52:42.743 INFO:tasks.rados.rados.0.smithi093.stdout:1096: finishing write tid 4 to smithi093153278-332 2023-08-27T20:52:42.743 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 332 v 175 (ObjNum 837 snap 0 seq_num 837) dirty exists 2023-08-27T20:52:42.743 INFO:tasks.rados.rados.0.smithi093.stdout:1096: left oid 332 (ObjNum 837 snap 0 seq_num 837) 2023-08-27T20:52:42.743 INFO:tasks.rados.rados.0.smithi093.stdout:1097: expect (ObjNum 728 snap 0 seq_num 728) 2023-08-27T20:52:42.829 INFO:tasks.rados.rados.0.smithi093.stdout:1098: finishing write tid 1 to smithi093153278-143 2023-08-27T20:52:42.829 INFO:tasks.rados.rados.0.smithi093.stdout:1106: seq_num 841 ranges {534687=483239,1613474=412283} 2023-08-27T20:52:42.834 INFO:tasks.rados.rados.0.smithi093.stdout:1106: writing smithi093153278-279 from 534687 to 1017926 tid 1 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1106: writing smithi093153278-279 from 1613474 to 2025757 tid 2 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1094: done (12 left) 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1095: done (11 left) 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1096: done (10 left) 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1097: done (9 left) 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1107: read oid 326 snap -1 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1107: expect (ObjNum 832 snap 0 seq_num 832) 2023-08-27T20:52:42.840 INFO:tasks.rados.rados.0.smithi093.stdout:1108: write oid 223 current snap is 0 2023-08-27T20:52:42.841 INFO:tasks.rados.rados.0.smithi093.stdout:1108: seq_num 842 ranges {551494=558488,1671956=479538} 2023-08-27T20:52:42.846 INFO:tasks.rados.rados.0.smithi093.stdout:1108: writing smithi093153278-223 from 551494 to 1109982 tid 1 2023-08-27T20:52:42.852 INFO:tasks.rados.rados.0.smithi093.stdout:1108: writing smithi093153278-223 from 1671956 to 2151494 tid 2 2023-08-27T20:52:42.852 INFO:tasks.rados.rados.0.smithi093.stdout:1109: read oid 175 snap -1 2023-08-27T20:52:42.852 INFO:tasks.rados.rados.0.smithi093.stdout:1109: expect (ObjNum 678 snap 0 seq_num 678) 2023-08-27T20:52:42.852 INFO:tasks.rados.rados.0.smithi093.stdout:1110: read oid 403 snap -1 2023-08-27T20:52:42.852 INFO:tasks.rados.rados.0.smithi093.stdout:1110: expect (ObjNum 659 snap 0 seq_num 659) 2023-08-27T20:52:42.852 INFO:tasks.rados.rados.0.smithi093.stdout:1111: read oid 128 snap -1 2023-08-27T20:52:42.852 INFO:tasks.rados.rados.0.smithi093.stdout:1111: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:52:42.853 INFO:tasks.rados.rados.0.smithi093.stdout:1099: expect (ObjNum 784 snap 0 seq_num 784) 2023-08-27T20:52:43.027 INFO:tasks.rados.rados.0.smithi093.stdout:1098: finishing write tid 2 to smithi093153278-143 2023-08-27T20:52:43.027 INFO:tasks.rados.rados.0.smithi093.stdout:1098: finishing write tid 3 to smithi093153278-143 2023-08-27T20:52:43.027 INFO:tasks.rados.rados.0.smithi093.stdout:1098: finishing write tid 4 to smithi093153278-143 2023-08-27T20:52:43.027 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 143 v 166 (ObjNum 838 snap 0 seq_num 838) dirty exists 2023-08-27T20:52:43.028 INFO:tasks.rados.rados.0.smithi093.stdout:1098: left oid 143 (ObjNum 838 snap 0 seq_num 838) 2023-08-27T20:52:43.028 INFO:tasks.rados.rados.0.smithi093.stdout:1100: expect (ObjNum 391 snap 0 seq_num 391) 2023-08-27T20:52:43.107 INFO:tasks.rados.rados.0.smithi093.stdout:1101: finishing write tid 1 to smithi093153278-117 2023-08-27T20:52:43.108 INFO:tasks.rados.rados.0.smithi093.stdout:1101: finishing write tid 2 to smithi093153278-117 2023-08-27T20:52:43.108 INFO:tasks.rados.rados.0.smithi093.stdout:1101: finishing write tid 3 to smithi093153278-117 2023-08-27T20:52:43.108 INFO:tasks.rados.rados.0.smithi093.stdout:1101: finishing write tid 4 to smithi093153278-117 2023-08-27T20:52:43.108 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 117 v 194 (ObjNum 839 snap 0 seq_num 839) dirty exists 2023-08-27T20:52:43.108 INFO:tasks.rados.rados.0.smithi093.stdout:1101: left oid 117 (ObjNum 839 snap 0 seq_num 839) 2023-08-27T20:52:43.108 INFO:tasks.rados.rados.0.smithi093.stdout:1102: expect (ObjNum 120 snap 0 seq_num 120) 2023-08-27T20:52:43.167 INFO:tasks.rados.rados.0.smithi093.stdout:1103: expect (ObjNum 52 snap 0 seq_num 52) 2023-08-27T20:52:43.256 INFO:tasks.rados.rados.0.smithi093.stdout:1104: finishing write tid 1 to smithi093153278-442 2023-08-27T20:52:43.256 INFO:tasks.rados.rados.0.smithi093.stdout:1104: finishing write tid 2 to smithi093153278-442 2023-08-27T20:52:43.256 INFO:tasks.rados.rados.0.smithi093.stdout:1105: expect (ObjNum 738 snap 0 seq_num 738) 2023-08-27T20:52:43.362 INFO:tasks.rados.rados.0.smithi093.stdout:1104: finishing write tid 3 to smithi093153278-442 2023-08-27T20:52:43.362 INFO:tasks.rados.rados.0.smithi093.stdout:1098: done (13 left) 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1099: done (12 left) 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1100: done (11 left) 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1101: done (10 left) 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1102: done (9 left) 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1103: done (8 left) 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1105: done (7 left) 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1112: write oid 248 current snap is 0 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:1104: finishing write tid 4 to smithi093153278-442 2023-08-27T20:52:43.363 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 442 v 190 (ObjNum 840 snap 0 seq_num 840) dirty exists 2023-08-27T20:52:43.364 INFO:tasks.rados.rados.0.smithi093.stdout:1104: left oid 442 (ObjNum 840 snap 0 seq_num 840) 2023-08-27T20:52:43.364 INFO:tasks.rados.rados.0.smithi093.stdout:1106: finishing write tid 1 to smithi093153278-279 2023-08-27T20:52:43.364 INFO:tasks.rados.rados.0.smithi093.stdout:1112: seq_num 843 ranges {568301=633737,1614085=415952} 2023-08-27T20:52:43.368 INFO:tasks.rados.rados.0.smithi093.stdout:1112: writing smithi093153278-248 from 568301 to 1202038 tid 1 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1112: writing smithi093153278-248 from 1614085 to 2030037 tid 2 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1104: done (7 left) 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1113: read oid 179 snap -1 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1113: expect (ObjNum 178 snap 0 seq_num 178) 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1114: read oid 49 snap -1 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1114: expect (ObjNum 747 snap 0 seq_num 747) 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1115: read oid 45 snap -1 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1115: expect (ObjNum 44 snap 0 seq_num 44) 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1116: read oid 446 snap -1 2023-08-27T20:52:43.374 INFO:tasks.rados.rados.0.smithi093.stdout:1116: expect (ObjNum 445 snap 0 seq_num 445) 2023-08-27T20:52:43.375 INFO:tasks.rados.rados.0.smithi093.stdout:1117: write oid 131 current snap is 0 2023-08-27T20:52:43.375 INFO:tasks.rados.rados.0.smithi093.stdout:1106: finishing write tid 2 to smithi093153278-279 2023-08-27T20:52:43.375 INFO:tasks.rados.rados.0.smithi093.stdout:1106: finishing write tid 3 to smithi093153278-279 2023-08-27T20:52:43.375 INFO:tasks.rados.rados.0.smithi093.stdout:1106: finishing write tid 4 to smithi093153278-279 2023-08-27T20:52:43.375 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 279 v 193 (ObjNum 841 snap 0 seq_num 841) dirty exists 2023-08-27T20:52:43.375 INFO:tasks.rados.rados.0.smithi093.stdout:1106: left oid 279 (ObjNum 841 snap 0 seq_num 841) 2023-08-27T20:52:43.375 INFO:tasks.rados.rados.0.smithi093.stdout:1107: expect (ObjNum 832 snap 0 seq_num 832) 2023-08-27T20:52:43.595 INFO:tasks.rados.rados.0.smithi093.stdout:1108: finishing write tid 1 to smithi093153278-223 2023-08-27T20:52:43.595 INFO:tasks.rados.rados.0.smithi093.stdout:1117: seq_num 844 ranges {585108=425339,1788920=396188} 2023-08-27T20:52:43.600 INFO:tasks.rados.rados.0.smithi093.stdout:1117: writing smithi093153278-131 from 585108 to 1010447 tid 1 2023-08-27T20:52:43.607 INFO:tasks.rados.rados.0.smithi093.stdout:1117: writing smithi093153278-131 from 1788920 to 2185108 tid 2 2023-08-27T20:52:43.608 INFO:tasks.rados.rados.0.smithi093.stdout:1106: done (11 left) 2023-08-27T20:52:43.608 INFO:tasks.rados.rados.0.smithi093.stdout:1107: done (10 left) 2023-08-27T20:52:43.608 INFO:tasks.rados.rados.0.smithi093.stdout:1118: write oid 138 current snap is 0 2023-08-27T20:52:43.608 INFO:tasks.rados.rados.0.smithi093.stdout:1108: finishing write tid 2 to smithi093153278-223 2023-08-27T20:52:43.608 INFO:tasks.rados.rados.0.smithi093.stdout:1118: seq_num 845 ranges {601915=500588,1847402=354513} 2023-08-27T20:52:43.613 INFO:tasks.rados.rados.0.smithi093.stdout:1118: writing smithi093153278-138 from 601915 to 1102503 tid 1 2023-08-27T20:52:43.619 INFO:tasks.rados.rados.0.smithi093.stdout:1118: writing smithi093153278-138 from 1847402 to 2201915 tid 2 2023-08-27T20:52:43.619 INFO:tasks.rados.rados.0.smithi093.stdout:1119: write oid 301 current snap is 0 2023-08-27T20:52:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:1108: finishing write tid 3 to smithi093153278-223 2023-08-27T20:52:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:1108: finishing write tid 4 to smithi093153278-223 2023-08-27T20:52:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 223 v 120 (ObjNum 842 snap 0 seq_num 842) dirty exists 2023-08-27T20:52:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:1108: left oid 223 (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:52:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:1119: seq_num 846 ranges {618722=575837,1905884=312838} 2023-08-27T20:52:43.626 INFO:tasks.rados.rados.0.smithi093.stdout:1119: writing smithi093153278-301 from 618722 to 1194559 tid 1 2023-08-27T20:52:43.632 INFO:tasks.rados.rados.0.smithi093.stdout:1119: writing smithi093153278-301 from 1905884 to 2218722 tid 2 2023-08-27T20:52:43.632 INFO:tasks.rados.rados.0.smithi093.stdout:1108: done (11 left) 2023-08-27T20:52:43.632 INFO:tasks.rados.rados.0.smithi093.stdout:1120: write oid 445 current snap is 0 2023-08-27T20:52:43.632 INFO:tasks.rados.rados.0.smithi093.stdout:1120: seq_num 847 ranges {635529=651086,1848013=387516} 2023-08-27T20:52:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:1120: writing smithi093153278-445 from 635529 to 1286615 tid 1 2023-08-27T20:52:43.644 INFO:tasks.rados.rados.0.smithi093.stdout:1120: writing smithi093153278-445 from 1848013 to 2235529 tid 2 2023-08-27T20:52:43.644 INFO:tasks.rados.rados.0.smithi093.stdout:1121: read oid 114 snap -1 2023-08-27T20:52:43.644 INFO:tasks.rados.rados.0.smithi093.stdout:1121: expect (ObjNum 644 snap 0 seq_num 644) 2023-08-27T20:52:43.644 INFO:tasks.rados.rados.0.smithi093.stdout:1122: write oid 7 current snap is 0 2023-08-27T20:52:43.644 INFO:tasks.rados.rados.0.smithi093.stdout:1122: seq_num 848 ranges {652336=726335,1906495=345841} 2023-08-27T20:52:43.652 INFO:tasks.rados.rados.0.smithi093.stdout:1122: writing smithi093153278-7 from 652336 to 1378671 tid 1 2023-08-27T20:52:43.657 INFO:tasks.rados.rados.0.smithi093.stdout:1122: writing smithi093153278-7 from 1906495 to 2252336 tid 2 2023-08-27T20:52:43.657 INFO:tasks.rados.rados.0.smithi093.stdout:1123: write oid 418 current snap is 0 2023-08-27T20:52:43.657 INFO:tasks.rados.rados.0.smithi093.stdout:1111: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:52:43.736 INFO:tasks.rados.rados.0.smithi093.stdout:1109: expect (ObjNum 678 snap 0 seq_num 678) 2023-08-27T20:52:43.883 INFO:tasks.rados.rados.0.smithi093.stdout:1110: expect (ObjNum 659 snap 0 seq_num 659) 2023-08-27T20:52:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:1112: finishing write tid 1 to smithi093153278-248 2023-08-27T20:52:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:1123: seq_num 849 ranges {669143=401584,1564977=426959} 2023-08-27T20:52:44.025 INFO:tasks.rados.rados.0.smithi093.stdout:1123: writing smithi093153278-418 from 669143 to 1070727 tid 1 2023-08-27T20:52:44.030 INFO:tasks.rados.rados.0.smithi093.stdout:1123: writing smithi093153278-418 from 1564977 to 1991936 tid 2 2023-08-27T20:52:44.031 INFO:tasks.rados.rados.0.smithi093.stdout:1109: done (14 left) 2023-08-27T20:52:44.031 INFO:tasks.rados.rados.0.smithi093.stdout:1110: done (13 left) 2023-08-27T20:52:44.031 INFO:tasks.rados.rados.0.smithi093.stdout:1111: done (12 left) 2023-08-27T20:52:44.031 INFO:tasks.rados.rados.0.smithi093.stdout:1124: write oid 339 current snap is 0 2023-08-27T20:52:44.031 INFO:tasks.rados.rados.0.smithi093.stdout:1113: expect (ObjNum 178 snap 0 seq_num 178) 2023-08-27T20:52:44.126 INFO:tasks.rados.rados.0.smithi093.stdout:1112: finishing write tid 2 to smithi093153278-248 2023-08-27T20:52:44.127 INFO:tasks.rados.rados.0.smithi093.stdout:1112: finishing write tid 3 to smithi093153278-248 2023-08-27T20:52:44.127 INFO:tasks.rados.rados.0.smithi093.stdout:1112: finishing write tid 4 to smithi093153278-248 2023-08-27T20:52:44.127 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 248 v 123 (ObjNum 843 snap 0 seq_num 843) dirty exists 2023-08-27T20:52:44.127 INFO:tasks.rados.rados.0.smithi093.stdout:1112: left oid 248 (ObjNum 843 snap 0 seq_num 843) 2023-08-27T20:52:44.127 INFO:tasks.rados.rados.0.smithi093.stdout:1114: expect (ObjNum 747 snap 0 seq_num 747) 2023-08-27T20:52:44.241 INFO:tasks.rados.rados.0.smithi093.stdout:1116: expect (ObjNum 445 snap 0 seq_num 445) 2023-08-27T20:52:44.348 INFO:tasks.rados.rados.0.smithi093.stdout:1115: expect (ObjNum 44 snap 0 seq_num 44) 2023-08-27T20:52:44.430 INFO:tasks.rados.rados.0.smithi093.stdout:1117: finishing write tid 1 to smithi093153278-131 2023-08-27T20:52:44.431 INFO:tasks.rados.rados.0.smithi093.stdout:1117: finishing write tid 2 to smithi093153278-131 2023-08-27T20:52:44.431 INFO:tasks.rados.rados.0.smithi093.stdout:1117: finishing write tid 3 to smithi093153278-131 2023-08-27T20:52:44.431 INFO:tasks.rados.rados.0.smithi093.stdout:1117: finishing write tid 4 to smithi093153278-131 2023-08-27T20:52:44.431 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 131 v 178 (ObjNum 844 snap 0 seq_num 844) dirty exists 2023-08-27T20:52:44.431 INFO:tasks.rados.rados.0.smithi093.stdout:1117: left oid 131 (ObjNum 844 snap 0 seq_num 844) 2023-08-27T20:52:44.431 INFO:tasks.rados.rados.0.smithi093.stdout:1118: finishing write tid 1 to smithi093153278-138 2023-08-27T20:52:44.431 INFO:tasks.rados.rados.0.smithi093.stdout:1124: seq_num 850 ranges {685950=476833,1623459=570617} 2023-08-27T20:52:44.436 INFO:tasks.rados.rados.0.smithi093.stdout:1124: writing smithi093153278-339 from 685950 to 1162783 tid 1 2023-08-27T20:52:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:44 smithi136 ceph-mon[113531]: pgmap v76: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 15 MiB/s wr, 147 op/s 2023-08-27T20:52:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1258199327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:44.442 INFO:tasks.rados.rados.0.smithi093.stdout:1124: writing smithi093153278-339 from 1623459 to 2194076 tid 2 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1112: done (12 left) 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1113: done (11 left) 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1114: done (10 left) 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1115: done (9 left) 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1116: done (8 left) 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1117: done (7 left) 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1125: write oid 496 current snap is 0 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1118: finishing write tid 2 to smithi093153278-138 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1118: finishing write tid 3 to smithi093153278-138 2023-08-27T20:52:44.443 INFO:tasks.rados.rados.0.smithi093.stdout:1118: finishing write tid 4 to smithi093153278-138 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 138 v 196 (ObjNum 845 snap 0 seq_num 845) dirty exists 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1118: left oid 138 (ObjNum 845 snap 0 seq_num 845) 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1119: finishing write tid 1 to smithi093153278-301 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1119: finishing write tid 2 to smithi093153278-301 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1119: finishing write tid 3 to smithi093153278-301 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1119: finishing write tid 4 to smithi093153278-301 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 301 v 193 (ObjNum 846 snap 0 seq_num 846) dirty exists 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1119: left oid 301 (ObjNum 846 snap 0 seq_num 846) 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1120: finishing write tid 1 to smithi093153278-445 2023-08-27T20:52:44.444 INFO:tasks.rados.rados.0.smithi093.stdout:1125: seq_num 851 ranges {702757=552082,1681941=430628} 2023-08-27T20:52:44.449 INFO:tasks.rados.rados.0.smithi093.stdout:1125: writing smithi093153278-496 from 702757 to 1254839 tid 1 2023-08-27T20:52:44.454 INFO:tasks.rados.rados.0.smithi093.stdout:1125: writing smithi093153278-496 from 1681941 to 2112569 tid 2 2023-08-27T20:52:44.454 INFO:tasks.rados.rados.0.smithi093.stdout:1118: done (7 left) 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:1119: done (6 left) 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:1126: write oid 203 current snap is 0 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:1120: finishing write tid 2 to smithi093153278-445 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:1120: finishing write tid 3 to smithi093153278-445 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:1120: finishing write tid 4 to smithi093153278-445 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 445 v 238 (ObjNum 847 snap 0 seq_num 847) dirty exists 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:1120: left oid 445 (ObjNum 847 snap 0 seq_num 847) 2023-08-27T20:52:44.455 INFO:tasks.rados.rados.0.smithi093.stdout:1121: expect (ObjNum 644 snap 0 seq_num 644) 2023-08-27T20:52:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:44 smithi093 ceph-mon[111335]: pgmap v76: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 15 MiB/s wr, 147 op/s 2023-08-27T20:52:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1258199327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:44 smithi093 ceph-mon[118902]: pgmap v76: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 101 MiB/s rd, 15 MiB/s wr, 147 op/s 2023-08-27T20:52:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1258199327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:44.576 INFO:tasks.rados.rados.0.smithi093.stdout:1122: finishing write tid 1 to smithi093153278-7 2023-08-27T20:52:44.576 INFO:tasks.rados.rados.0.smithi093.stdout:1122: finishing write tid 2 to smithi093153278-7 2023-08-27T20:52:44.576 INFO:tasks.rados.rados.0.smithi093.stdout:1122: finishing write tid 3 to smithi093153278-7 2023-08-27T20:52:44.576 INFO:tasks.rados.rados.0.smithi093.stdout:1122: finishing write tid 4 to smithi093153278-7 2023-08-27T20:52:44.577 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 7 v 151 (ObjNum 848 snap 0 seq_num 848) dirty exists 2023-08-27T20:52:44.577 INFO:tasks.rados.rados.0.smithi093.stdout:1122: left oid 7 (ObjNum 848 snap 0 seq_num 848) 2023-08-27T20:52:44.577 INFO:tasks.rados.rados.0.smithi093.stdout:1126: seq_num 852 ranges {719564=743684,2140423=179141} 2023-08-27T20:52:44.584 INFO:tasks.rados.rados.0.smithi093.stdout:1126: writing smithi093153278-203 from 719564 to 1463248 tid 1 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1126: writing smithi093153278-203 from 2140423 to 2319564 tid 2 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1120: done (6 left) 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1121: done (5 left) 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1122: done (4 left) 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1127: read oid 43 snap -1 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1127: expect (ObjNum 42 snap 0 seq_num 42) 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1128: write oid 453 current snap is 0 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1123: finishing write tid 1 to smithi093153278-418 2023-08-27T20:52:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:1123: finishing write tid 2 to smithi093153278-418 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1123: finishing write tid 3 to smithi093153278-418 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1123: finishing write tid 4 to smithi093153278-418 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 418 v 197 (ObjNum 849 snap 0 seq_num 849) dirty exists 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1123: left oid 418 (ObjNum 849 snap 0 seq_num 849) 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1124: finishing write tid 1 to smithi093153278-339 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1124: finishing write tid 2 to smithi093153278-339 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1124: finishing write tid 3 to smithi093153278-339 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1124: finishing write tid 4 to smithi093153278-339 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 339 v 191 (ObjNum 850 snap 0 seq_num 850) dirty exists 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1124: left oid 339 (ObjNum 850 snap 0 seq_num 850) 2023-08-27T20:52:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:1125: finishing write tid 1 to smithi093153278-496 2023-08-27T20:52:44.592 INFO:tasks.rados.rados.0.smithi093.stdout:1125: finishing write tid 2 to smithi093153278-496 2023-08-27T20:52:44.592 INFO:tasks.rados.rados.0.smithi093.stdout:1125: finishing write tid 3 to smithi093153278-496 2023-08-27T20:52:44.592 INFO:tasks.rados.rados.0.smithi093.stdout:1125: finishing write tid 4 to smithi093153278-496 2023-08-27T20:52:44.592 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 496 v 176 (ObjNum 851 snap 0 seq_num 851) dirty exists 2023-08-27T20:52:44.592 INFO:tasks.rados.rados.0.smithi093.stdout:1125: left oid 496 (ObjNum 851 snap 0 seq_num 851) 2023-08-27T20:52:44.592 INFO:tasks.rados.rados.0.smithi093.stdout:1128: seq_num 853 ranges {736371=418933,1798905=434297} 2023-08-27T20:52:44.595 INFO:tasks.rados.rados.0.smithi093.stdout:1128: writing smithi093153278-453 from 736371 to 1155304 tid 1 2023-08-27T20:52:44.602 INFO:tasks.rados.rados.0.smithi093.stdout:1128: writing smithi093153278-453 from 1798905 to 2233202 tid 2 2023-08-27T20:52:44.602 INFO:tasks.rados.rados.0.smithi093.stdout:1123: done (5 left) 2023-08-27T20:52:44.602 INFO:tasks.rados.rados.0.smithi093.stdout:1124: done (4 left) 2023-08-27T20:52:44.602 INFO:tasks.rados.rados.0.smithi093.stdout:1125: done (3 left) 2023-08-27T20:52:44.602 INFO:tasks.rados.rados.0.smithi093.stdout:1129: write oid 445 current snap is 0 2023-08-27T20:52:44.602 INFO:tasks.rados.rados.0.smithi093.stdout:1126: finishing write tid 1 to smithi093153278-203 2023-08-27T20:52:44.602 INFO:tasks.rados.rados.0.smithi093.stdout:1129: seq_num 854 ranges {753178=494182,1857387=495791} 2023-08-27T20:52:44.608 INFO:tasks.rados.rados.0.smithi093.stdout:1129: writing smithi093153278-445 from 753178 to 1247360 tid 1 2023-08-27T20:52:44.614 INFO:tasks.rados.rados.0.smithi093.stdout:1129: writing smithi093153278-445 from 1857387 to 2353178 tid 2 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1130: read oid 26 snap -1 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1130: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1131: read oid 229 snap -1 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1131: expect (ObjNum 751 snap 0 seq_num 751) 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1132: read oid 316 snap -1 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1132: expect (ObjNum 608 snap 0 seq_num 608) 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1133: write oid 255 current snap is 0 2023-08-27T20:52:44.615 INFO:tasks.rados.rados.0.smithi093.stdout:1133: seq_num 855 ranges {769985=569431,1915869=437966} 2023-08-27T20:52:44.622 INFO:tasks.rados.rados.0.smithi093.stdout:1133: writing smithi093153278-255 from 769985 to 1339416 tid 1 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:1133: writing smithi093153278-255 from 1915869 to 2353835 tid 2 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:1134: delete oid 284 current snap is 0 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:1126: finishing write tid 2 to smithi093153278-203 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:1126: finishing write tid 3 to smithi093153278-203 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:1126: finishing write tid 4 to smithi093153278-203 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 203 v 166 (ObjNum 852 snap 0 seq_num 852) dirty exists 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:1126: left oid 203 (ObjNum 852 snap 0 seq_num 852) 2023-08-27T20:52:44.628 INFO:tasks.rados.rados.0.smithi093.stdout:1127: expect (ObjNum 42 snap 0 seq_num 42) 2023-08-27T20:52:44.712 INFO:tasks.rados.rados.0.smithi093.stdout:1128: finishing write tid 1 to smithi093153278-453 2023-08-27T20:52:44.712 INFO:tasks.rados.rados.0.smithi093.stdout:1128: finishing write tid 2 to smithi093153278-453 2023-08-27T20:52:44.712 INFO:tasks.rados.rados.0.smithi093.stdout:1128: finishing write tid 3 to smithi093153278-453 2023-08-27T20:52:44.712 INFO:tasks.rados.rados.0.smithi093.stdout:1128: finishing write tid 4 to smithi093153278-453 2023-08-27T20:52:44.713 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 453 v 180 (ObjNum 853 snap 0 seq_num 853) dirty exists 2023-08-27T20:52:44.713 INFO:tasks.rados.rados.0.smithi093.stdout:1128: left oid 453 (ObjNum 853 snap 0 seq_num 853) 2023-08-27T20:52:44.713 INFO:tasks.rados.rados.0.smithi093.stdout:1129: finishing write tid 1 to smithi093153278-445 2023-08-27T20:52:44.713 INFO:tasks.rados.rados.0.smithi093.stdout:1130: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:44.786 INFO:tasks.rados.rados.0.smithi093.stdout:1129: finishing write tid 2 to smithi093153278-445 2023-08-27T20:52:44.786 INFO:tasks.rados.rados.0.smithi093.stdout:1129: finishing write tid 3 to smithi093153278-445 2023-08-27T20:52:44.786 INFO:tasks.rados.rados.0.smithi093.stdout:1129: finishing write tid 4 to smithi093153278-445 2023-08-27T20:52:44.786 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 445 v 241 (ObjNum 854 snap 0 seq_num 854) dirty exists 2023-08-27T20:52:44.786 INFO:tasks.rados.rados.0.smithi093.stdout:1129: left oid 445 (ObjNum 854 snap 0 seq_num 854) 2023-08-27T20:52:44.786 INFO:tasks.rados.rados.0.smithi093.stdout:1131: expect (ObjNum 751 snap 0 seq_num 751) 2023-08-27T20:52:44.887 INFO:tasks.rados.rados.0.smithi093.stdout:1132: expect (ObjNum 608 snap 0 seq_num 608) 2023-08-27T20:52:44.978 INFO:tasks.rados.rados.0.smithi093.stdout:1133: finishing write tid 1 to smithi093153278-255 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1133: finishing write tid 2 to smithi093153278-255 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1133: finishing write tid 3 to smithi093153278-255 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1133: finishing write tid 4 to smithi093153278-255 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 255 v 244 (ObjNum 855 snap 0 seq_num 855) dirty exists 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1133: left oid 255 (ObjNum 855 snap 0 seq_num 855) 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1126: done (8 left) 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1127: done (7 left) 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1128: done (6 left) 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1129: done (5 left) 2023-08-27T20:52:44.979 INFO:tasks.rados.rados.0.smithi093.stdout:1130: done (4 left) 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1131: done (3 left) 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1132: done (2 left) 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1133: done (1 left) 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1134: done (0 left) 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1135: read oid 349 snap -1 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1135: expect (ObjNum 669 snap 0 seq_num 669) 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1136: write oid 244 current snap is 0 2023-08-27T20:52:44.980 INFO:tasks.rados.rados.0.smithi093.stdout:1136: seq_num 857 ranges {403599=719929,1916480=441635} 2023-08-27T20:52:44.984 INFO:tasks.rados.rados.0.smithi093.stdout:1136: writing smithi093153278-244 from 403599 to 1123528 tid 1 2023-08-27T20:52:44.991 INFO:tasks.rados.rados.0.smithi093.stdout:1136: writing smithi093153278-244 from 1916480 to 2358115 tid 2 2023-08-27T20:52:44.991 INFO:tasks.rados.rados.0.smithi093.stdout:1137: write oid 213 current snap is 0 2023-08-27T20:52:44.991 INFO:tasks.rados.rados.0.smithi093.stdout:1137: seq_num 858 ranges {420406=795178,1974962=445444} 2023-08-27T20:52:44.998 INFO:tasks.rados.rados.0.smithi093.stdout:1137: writing smithi093153278-213 from 420406 to 1215584 tid 1 2023-08-27T20:52:45.004 INFO:tasks.rados.rados.0.smithi093.stdout:1137: writing smithi093153278-213 from 1974962 to 2420406 tid 2 2023-08-27T20:52:45.005 INFO:tasks.rados.rados.0.smithi093.stdout:1138: write oid 246 current snap is 0 2023-08-27T20:52:45.005 INFO:tasks.rados.rados.0.smithi093.stdout:1138: seq_num 859 ranges {437213=470427,1633444=728951} 2023-08-27T20:52:45.009 INFO:tasks.rados.rados.0.smithi093.stdout:1138: writing smithi093153278-246 from 437213 to 907640 tid 1 2023-08-27T20:52:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:1138: writing smithi093153278-246 from 1633444 to 2362395 tid 2 2023-08-27T20:52:45.018 INFO:tasks.rados.rados.0.smithi093.stdout:1139: write oid 53 current snap is 0 2023-08-27T20:52:45.018 INFO:tasks.rados.rados.0.smithi093.stdout:1135: expect (ObjNum 669 snap 0 seq_num 669) 2023-08-27T20:52:45.143 INFO:tasks.rados.rados.0.smithi093.stdout:1136: finishing write tid 1 to smithi093153278-244 2023-08-27T20:52:45.144 INFO:tasks.rados.rados.0.smithi093.stdout:1139: seq_num 860 ranges {454020=662029,1691926=588962} 2023-08-27T20:52:45.149 INFO:tasks.rados.rados.0.smithi093.stdout:1139: writing smithi093153278-53 from 454020 to 1116049 tid 1 2023-08-27T20:52:45.156 INFO:tasks.rados.rados.0.smithi093.stdout:1139: writing smithi093153278-53 from 1691926 to 2280888 tid 2 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1135: done (4 left) 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1140: write oid 317 current snap is 0 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1136: finishing write tid 2 to smithi093153278-244 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1136: finishing write tid 3 to smithi093153278-244 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1136: finishing write tid 4 to smithi093153278-244 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 244 v 126 (ObjNum 857 snap 0 seq_num 857) dirty exists 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1136: left oid 244 (ObjNum 857 snap 0 seq_num 857) 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1137: finishing write tid 1 to smithi093153278-213 2023-08-27T20:52:45.157 INFO:tasks.rados.rados.0.smithi093.stdout:1137: finishing write tid 2 to smithi093153278-213 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:1137: finishing write tid 3 to smithi093153278-213 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:1137: finishing write tid 4 to smithi093153278-213 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 213 v 169 (ObjNum 858 snap 0 seq_num 858) dirty exists 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:1137: left oid 213 (ObjNum 858 snap 0 seq_num 858) 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:1138: finishing write tid 1 to smithi093153278-246 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:1138: finishing write tid 2 to smithi093153278-246 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:1138: finishing write tid 3 to smithi093153278-246 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:1138: finishing write tid 4 to smithi093153278-246 2023-08-27T20:52:45.158 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 246 v 158 (ObjNum 859 snap 0 seq_num 859) dirty exists 2023-08-27T20:52:45.159 INFO:tasks.rados.rados.0.smithi093.stdout:1138: left oid 246 (ObjNum 859 snap 0 seq_num 859) 2023-08-27T20:52:45.159 INFO:tasks.rados.rados.0.smithi093.stdout:1140: seq_num 861 ranges {470827=737278,1750408=720419} 2023-08-27T20:52:45.162 INFO:tasks.rados.rados.0.smithi093.stdout:1140: writing smithi093153278-317 from 470827 to 1208105 tid 1 2023-08-27T20:52:45.169 INFO:tasks.rados.rados.0.smithi093.stdout:1140: writing smithi093153278-317 from 1750408 to 2470827 tid 2 2023-08-27T20:52:45.169 INFO:tasks.rados.rados.0.smithi093.stdout:1136: done (4 left) 2023-08-27T20:52:45.170 INFO:tasks.rados.rados.0.smithi093.stdout:1137: done (3 left) 2023-08-27T20:52:45.170 INFO:tasks.rados.rados.0.smithi093.stdout:1138: done (2 left) 2023-08-27T20:52:45.170 INFO:tasks.rados.rados.0.smithi093.stdout:1141: write oid 340 current snap is 0 2023-08-27T20:52:45.170 INFO:tasks.rados.rados.0.smithi093.stdout:1139: finishing write tid 1 to smithi093153278-53 2023-08-27T20:52:45.170 INFO:tasks.rados.rados.0.smithi093.stdout:1141: seq_num 862 ranges {487634=412527,1408890=592631} 2023-08-27T20:52:45.174 INFO:tasks.rados.rados.0.smithi093.stdout:1141: writing smithi093153278-340 from 487634 to 900161 tid 1 2023-08-27T20:52:45.180 INFO:tasks.rados.rados.0.smithi093.stdout:1141: writing smithi093153278-340 from 1408890 to 2001521 tid 2 2023-08-27T20:52:45.181 INFO:tasks.rados.rados.0.smithi093.stdout:1142: write oid 207 current snap is 0 2023-08-27T20:52:45.181 INFO:tasks.rados.rados.0.smithi093.stdout:1142: seq_num 863 ranges {504441=487776,1467372=736289} 2023-08-27T20:52:45.185 INFO:tasks.rados.rados.0.smithi093.stdout:1142: writing smithi093153278-207 from 504441 to 992217 tid 1 2023-08-27T20:52:45.193 INFO:tasks.rados.rados.0.smithi093.stdout:1142: writing smithi093153278-207 from 1467372 to 2203661 tid 2 2023-08-27T20:52:45.193 INFO:tasks.rados.rados.0.smithi093.stdout:1143: read oid 272 snap -1 2023-08-27T20:52:45.193 INFO:tasks.rados.rados.0.smithi093.stdout:1143: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:45.193 INFO:tasks.rados.rados.0.smithi093.stdout:1144: read oid 211 snap -1 2023-08-27T20:52:45.193 INFO:tasks.rados.rados.0.smithi093.stdout:1144: expect (ObjNum 210 snap 0 seq_num 210) 2023-08-27T20:52:45.193 INFO:tasks.rados.rados.0.smithi093.stdout:1145: write oid 258 current snap is 0 2023-08-27T20:52:45.193 INFO:tasks.rados.rados.0.smithi093.stdout:1139: finishing write tid 2 to smithi093153278-53 2023-08-27T20:52:45.194 INFO:tasks.rados.rados.0.smithi093.stdout:1139: finishing write tid 3 to smithi093153278-53 2023-08-27T20:52:45.194 INFO:tasks.rados.rados.0.smithi093.stdout:1139: finishing write tid 4 to smithi093153278-53 2023-08-27T20:52:45.194 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 53 v 206 (ObjNum 860 snap 0 seq_num 860) dirty exists 2023-08-27T20:52:45.194 INFO:tasks.rados.rados.0.smithi093.stdout:1139: left oid 53 (ObjNum 860 snap 0 seq_num 860) 2023-08-27T20:52:45.194 INFO:tasks.rados.rados.0.smithi093.stdout:1145: seq_num 864 ranges {521248=563025,1809501=596300} 2023-08-27T20:52:45.199 INFO:tasks.rados.rados.0.smithi093.stdout:1145: writing smithi093153278-258 from 521248 to 1084273 tid 1 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1145: writing smithi093153278-258 from 1809501 to 2405801 tid 2 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1139: done (6 left) 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1146: read oid 97 snap -1 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1146: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1147: delete oid 132 current snap is 0 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1140: finishing write tid 1 to smithi093153278-317 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1140: finishing write tid 2 to smithi093153278-317 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1140: finishing write tid 3 to smithi093153278-317 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1140: finishing write tid 4 to smithi093153278-317 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 317 v 176 (ObjNum 861 snap 0 seq_num 861) dirty exists 2023-08-27T20:52:45.207 INFO:tasks.rados.rados.0.smithi093.stdout:1140: left oid 317 (ObjNum 861 snap 0 seq_num 861) 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1141: finishing write tid 1 to smithi093153278-340 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1141: finishing write tid 2 to smithi093153278-340 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1141: finishing write tid 3 to smithi093153278-340 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1141: finishing write tid 4 to smithi093153278-340 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 340 v 161 (ObjNum 862 snap 0 seq_num 862) dirty exists 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1141: left oid 340 (ObjNum 862 snap 0 seq_num 862) 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1142: finishing write tid 1 to smithi093153278-207 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1142: finishing write tid 2 to smithi093153278-207 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1142: finishing write tid 3 to smithi093153278-207 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1142: finishing write tid 4 to smithi093153278-207 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 207 v 169 (ObjNum 863 snap 0 seq_num 863) dirty exists 2023-08-27T20:52:45.208 INFO:tasks.rados.rados.0.smithi093.stdout:1142: left oid 207 (ObjNum 863 snap 0 seq_num 863) 2023-08-27T20:52:45.209 INFO:tasks.rados.rados.0.smithi093.stdout:1144: expect (ObjNum 210 snap 0 seq_num 210) 2023-08-27T20:52:45.318 INFO:tasks.rados.rados.0.smithi093.stdout:1143: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:52:45.401 INFO:tasks.rados.rados.0.smithi093.stdout:1145: finishing write tid 1 to smithi093153278-258 2023-08-27T20:52:45.401 INFO:tasks.rados.rados.0.smithi093.stdout:1145: finishing write tid 2 to smithi093153278-258 2023-08-27T20:52:45.401 INFO:tasks.rados.rados.0.smithi093.stdout:1145: finishing write tid 3 to smithi093153278-258 2023-08-27T20:52:45.401 INFO:tasks.rados.rados.0.smithi093.stdout:1140: done (7 left) 2023-08-27T20:52:45.402 INFO:tasks.rados.rados.0.smithi093.stdout:1141: done (6 left) 2023-08-27T20:52:45.402 INFO:tasks.rados.rados.0.smithi093.stdout:1142: done (5 left) 2023-08-27T20:52:45.402 INFO:tasks.rados.rados.0.smithi093.stdout:1143: done (4 left) 2023-08-27T20:52:45.402 INFO:tasks.rados.rados.0.smithi093.stdout:1144: done (3 left) 2023-08-27T20:52:45.403 INFO:tasks.rados.rados.0.smithi093.stdout:1147: done (2 left) 2023-08-27T20:52:45.403 INFO:tasks.rados.rados.0.smithi093.stdout:1148: read oid 337 snap -1 2023-08-27T20:52:45.403 INFO:tasks.rados.rados.0.smithi093.stdout:1148: expect (ObjNum 787 snap 0 seq_num 787) 2023-08-27T20:52:45.403 INFO:tasks.rados.rados.0.smithi093.stdout:1149: read oid 101 snap -1 2023-08-27T20:52:45.404 INFO:tasks.rados.rados.0.smithi093.stdout:1149: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:52:45.404 INFO:tasks.rados.rados.0.smithi093.stdout:1150: write oid 492 current snap is 0 2023-08-27T20:52:45.404 INFO:tasks.rados.rados.0.smithi093.stdout:1145: finishing write tid 4 to smithi093153278-258 2023-08-27T20:52:45.404 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 258 v 194 (ObjNum 864 snap 0 seq_num 864) dirty exists 2023-08-27T20:52:45.404 INFO:tasks.rados.rados.0.smithi093.stdout:1145: left oid 258 (ObjNum 864 snap 0 seq_num 864) 2023-08-27T20:52:45.404 INFO:tasks.rados.rados.0.smithi093.stdout:1150: seq_num 866 ranges {554862=713523,1926465=599969} 2023-08-27T20:52:45.406 INFO:tasks.rados.rados.0.smithi093.stdout:1150: writing smithi093153278-492 from 554862 to 1268385 tid 1 2023-08-27T20:52:45.414 INFO:tasks.rados.rados.0.smithi093.stdout:1150: writing smithi093153278-492 from 1926465 to 2526434 tid 2 2023-08-27T20:52:45.414 INFO:tasks.rados.rados.0.smithi093.stdout:1145: done (4 left) 2023-08-27T20:52:45.414 INFO:tasks.rados.rados.0.smithi093.stdout:1151: read oid 458 snap -1 2023-08-27T20:52:45.414 INFO:tasks.rados.rados.0.smithi093.stdout:1151: expect (ObjNum 457 snap 0 seq_num 457) 2023-08-27T20:52:45.414 INFO:tasks.rados.rados.0.smithi093.stdout:1152: write oid 322 current snap is 0 2023-08-27T20:52:45.415 INFO:tasks.rados.rados.0.smithi093.stdout:1146: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:52:45.526 INFO:tasks.rados.rados.0.smithi093.stdout:1148: expect (ObjNum 787 snap 0 seq_num 787) 2023-08-27T20:52:45.683 INFO:tasks.rados.rados.0.smithi093.stdout:1149: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:52:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:52:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:45] "GET /metrics HTTP/1.1" 200 242475 "" "Prometheus/2.43.0" 2023-08-27T20:52:45.798 INFO:tasks.rados.rados.0.smithi093.stdout:1150: finishing write tid 1 to smithi093153278-492 2023-08-27T20:52:45.799 INFO:tasks.rados.rados.0.smithi093.stdout:1150: finishing write tid 2 to smithi093153278-492 2023-08-27T20:52:45.799 INFO:tasks.rados.rados.0.smithi093.stdout:1152: seq_num 867 ranges {571669=505125,1701300=743627} 2023-08-27T20:52:45.804 INFO:tasks.rados.rados.0.smithi093.stdout:1152: writing smithi093153278-322 from 571669 to 1076794 tid 1 2023-08-27T20:52:45.812 INFO:tasks.rados.rados.0.smithi093.stdout:1152: writing smithi093153278-322 from 1701300 to 2444927 tid 2 2023-08-27T20:52:45.812 INFO:tasks.rados.rados.0.smithi093.stdout:1146: done (5 left) 2023-08-27T20:52:45.812 INFO:tasks.rados.rados.0.smithi093.stdout:1148: done (4 left) 2023-08-27T20:52:45.812 INFO:tasks.rados.rados.0.smithi093.stdout:1149: done (3 left) 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:1153: read oid 17 snap -1 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:1153: expect (ObjNum 16 snap 0 seq_num 16) 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:1154: write oid 65 current snap is 0 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:1150: finishing write tid 3 to smithi093153278-492 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:1150: finishing write tid 4 to smithi093153278-492 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 492 v 197 (ObjNum 866 snap 0 seq_num 866) dirty exists 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:1150: left oid 492 (ObjNum 866 snap 0 seq_num 866) 2023-08-27T20:52:45.813 INFO:tasks.rados.rados.0.smithi093.stdout:1151: expect (ObjNum 457 snap 0 seq_num 457) 2023-08-27T20:52:45.928 INFO:tasks.rados.rados.0.smithi093.stdout:1152: finishing write tid 1 to smithi093153278-322 2023-08-27T20:52:45.928 INFO:tasks.rados.rados.0.smithi093.stdout:1153: expect (ObjNum 16 snap 0 seq_num 16) 2023-08-27T20:52:46.001 INFO:tasks.rados.rados.0.smithi093.stdout:1152: finishing write tid 2 to smithi093153278-322 2023-08-27T20:52:46.001 INFO:tasks.rados.rados.0.smithi093.stdout:1152: finishing write tid 3 to smithi093153278-322 2023-08-27T20:52:46.001 INFO:tasks.rados.rados.0.smithi093.stdout:1152: finishing write tid 4 to smithi093153278-322 2023-08-27T20:52:46.002 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 322 v 172 (ObjNum 867 snap 0 seq_num 867) dirty exists 2023-08-27T20:52:46.002 INFO:tasks.rados.rados.0.smithi093.stdout:1152: left oid 322 (ObjNum 867 snap 0 seq_num 867) 2023-08-27T20:52:46.002 INFO:tasks.rados.rados.0.smithi093.stdout:1154: seq_num 868 ranges {588476=580374,1643429=603638} 2023-08-27T20:52:46.007 INFO:tasks.rados.rados.0.smithi093.stdout:1154: writing smithi093153278-65 from 588476 to 1168850 tid 1 2023-08-27T20:52:46.014 INFO:tasks.rados.rados.0.smithi093.stdout:1154: writing smithi093153278-65 from 1643429 to 2247067 tid 2 2023-08-27T20:52:46.014 INFO:tasks.rados.rados.0.smithi093.stdout:1150: done (4 left) 2023-08-27T20:52:46.014 INFO:tasks.rados.rados.0.smithi093.stdout:1151: done (3 left) 2023-08-27T20:52:46.014 INFO:tasks.rados.rados.0.smithi093.stdout:1152: done (2 left) 2023-08-27T20:52:46.014 INFO:tasks.rados.rados.0.smithi093.stdout:1153: done (1 left) 2023-08-27T20:52:46.015 INFO:tasks.rados.rados.0.smithi093.stdout:1155: write oid 433 current snap is 0 2023-08-27T20:52:46.015 INFO:tasks.rados.rados.0.smithi093.stdout:1154: finishing write tid 1 to smithi093153278-65 2023-08-27T20:52:46.015 INFO:tasks.rados.rados.0.smithi093.stdout:1155: seq_num 869 ranges {605283=655623,1701911=747296} 2023-08-27T20:52:46.021 INFO:tasks.rados.rados.0.smithi093.stdout:1155: writing smithi093153278-433 from 605283 to 1260906 tid 1 2023-08-27T20:52:46.027 INFO:tasks.rados.rados.0.smithi093.stdout:1155: writing smithi093153278-433 from 1701911 to 2449207 tid 2 2023-08-27T20:52:46.028 INFO:tasks.rados.rados.0.smithi093.stdout:1156: write oid 470 current snap is 0 2023-08-27T20:52:46.028 INFO:tasks.rados.rados.0.smithi093.stdout:1154: finishing write tid 2 to smithi093153278-65 2023-08-27T20:52:46.028 INFO:tasks.rados.rados.0.smithi093.stdout:1156: seq_num 870 ranges {622090=730872,1760393=607307} 2023-08-27T20:52:46.035 INFO:tasks.rados.rados.0.smithi093.stdout:1156: writing smithi093153278-470 from 622090 to 1352962 tid 1 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:1156: writing smithi093153278-470 from 1760393 to 2367700 tid 2 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:1157: write oid 146 current snap is 0 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:1154: finishing write tid 3 to smithi093153278-65 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:1154: finishing write tid 4 to smithi093153278-65 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 65 v 184 (ObjNum 868 snap 0 seq_num 868) dirty exists 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:1154: left oid 65 (ObjNum 868 snap 0 seq_num 868) 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:1155: finishing write tid 1 to smithi093153278-433 2023-08-27T20:52:46.041 INFO:tasks.rados.rados.0.smithi093.stdout:1157: seq_num 871 ranges {638897=406121,1818875=750965} 2023-08-27T20:52:46.046 INFO:tasks.rados.rados.0.smithi093.stdout:1157: writing smithi093153278-146 from 638897 to 1045018 tid 1 2023-08-27T20:52:46.055 INFO:tasks.rados.rados.0.smithi093.stdout:1157: writing smithi093153278-146 from 1818875 to 2569840 tid 2 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1154: done (3 left) 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1158: read oid 446 snap -1 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1158: expect (ObjNum 445 snap 0 seq_num 445) 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1159: read oid 271 snap -1 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1159: expect (ObjNum 270 snap 0 seq_num 270) 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1160: read oid 396 snap -1 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1160: expect (ObjNum 395 snap 0 seq_num 395) 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1161: write oid 392 current snap is 0 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1155: finishing write tid 2 to smithi093153278-433 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1155: finishing write tid 3 to smithi093153278-433 2023-08-27T20:52:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:1155: finishing write tid 4 to smithi093153278-433 2023-08-27T20:52:46.057 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 433 v 196 (ObjNum 869 snap 0 seq_num 869) dirty exists 2023-08-27T20:52:46.057 INFO:tasks.rados.rados.0.smithi093.stdout:1155: left oid 433 (ObjNum 869 snap 0 seq_num 869) 2023-08-27T20:52:46.057 INFO:tasks.rados.rados.0.smithi093.stdout:1156: finishing write tid 1 to smithi093153278-470 2023-08-27T20:52:46.057 INFO:tasks.rados.rados.0.smithi093.stdout:1161: seq_num 872 ranges {655704=481370,1761004=494623} 2023-08-27T20:52:46.062 INFO:tasks.rados.rados.0.smithi093.stdout:1161: writing smithi093153278-392 from 655704 to 1137074 tid 1 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1161: writing smithi093153278-392 from 1761004 to 2255627 tid 2 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1155: done (6 left) 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1162: read oid 339 snap -1 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1162: expect (ObjNum 850 snap 0 seq_num 850) 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1163: read oid 24 snap -1 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1163: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1164: read oid 250 snap -1 2023-08-27T20:52:46.069 INFO:tasks.rados.rados.0.smithi093.stdout:1164: expect (ObjNum 508 snap 0 seq_num 508) 2023-08-27T20:52:46.070 INFO:tasks.rados.rados.0.smithi093.stdout:1165: write oid 451 current snap is 0 2023-08-27T20:52:46.070 INFO:tasks.rados.rados.0.smithi093.stdout:1156: finishing write tid 2 to smithi093153278-470 2023-08-27T20:52:46.070 INFO:tasks.rados.rados.0.smithi093.stdout:1156: finishing write tid 3 to smithi093153278-470 2023-08-27T20:52:46.070 INFO:tasks.rados.rados.0.smithi093.stdout:1156: finishing write tid 4 to smithi093153278-470 2023-08-27T20:52:46.070 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 470 v 247 (ObjNum 870 snap 0 seq_num 870) dirty exists 2023-08-27T20:52:46.070 INFO:tasks.rados.rados.0.smithi093.stdout:1156: left oid 470 (ObjNum 870 snap 0 seq_num 870) 2023-08-27T20:52:46.070 INFO:tasks.rados.rados.0.smithi093.stdout:1165: seq_num 873 ranges {672511=556619,1819486=754634} 2023-08-27T20:52:46.075 INFO:tasks.rados.rados.0.smithi093.stdout:1165: writing smithi093153278-451 from 672511 to 1229130 tid 1 2023-08-27T20:52:46.083 INFO:tasks.rados.rados.0.smithi093.stdout:1165: writing smithi093153278-451 from 1819486 to 2574120 tid 2 2023-08-27T20:52:46.083 INFO:tasks.rados.rados.0.smithi093.stdout:1156: done (9 left) 2023-08-27T20:52:46.083 INFO:tasks.rados.rados.0.smithi093.stdout:1166: delete oid 496 current snap is 0 2023-08-27T20:52:46.084 INFO:tasks.rados.rados.0.smithi093.stdout:1157: finishing write tid 1 to smithi093153278-146 2023-08-27T20:52:46.084 INFO:tasks.rados.rados.0.smithi093.stdout:1157: finishing write tid 2 to smithi093153278-146 2023-08-27T20:52:46.084 INFO:tasks.rados.rados.0.smithi093.stdout:1157: finishing write tid 3 to smithi093153278-146 2023-08-27T20:52:46.084 INFO:tasks.rados.rados.0.smithi093.stdout:1157: finishing write tid 4 to smithi093153278-146 2023-08-27T20:52:46.084 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 146 v 179 (ObjNum 871 snap 0 seq_num 871) dirty exists 2023-08-27T20:52:46.084 INFO:tasks.rados.rados.0.smithi093.stdout:1157: left oid 146 (ObjNum 871 snap 0 seq_num 871) 2023-08-27T20:52:46.084 INFO:tasks.rados.rados.0.smithi093.stdout:1159: expect (ObjNum 270 snap 0 seq_num 270) 2023-08-27T20:52:46.161 INFO:tasks.rados.rados.0.smithi093.stdout:1160: expect (ObjNum 395 snap 0 seq_num 395) 2023-08-27T20:52:46.245 INFO:tasks.rados.rados.0.smithi093.stdout:1158: expect (ObjNum 445 snap 0 seq_num 445) 2023-08-27T20:52:46.352 INFO:tasks.rados.rados.0.smithi093.stdout:1161: finishing write tid 1 to smithi093153278-392 2023-08-27T20:52:46.352 INFO:tasks.rados.rados.0.smithi093.stdout:1161: finishing write tid 2 to smithi093153278-392 2023-08-27T20:52:46.352 INFO:tasks.rados.rados.0.smithi093.stdout:1161: finishing write tid 3 to smithi093153278-392 2023-08-27T20:52:46.352 INFO:tasks.rados.rados.0.smithi093.stdout:1161: finishing write tid 4 to smithi093153278-392 2023-08-27T20:52:46.352 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 392 v 175 (ObjNum 872 snap 0 seq_num 872) dirty exists 2023-08-27T20:52:46.352 INFO:tasks.rados.rados.0.smithi093.stdout:1161: left oid 392 (ObjNum 872 snap 0 seq_num 872) 2023-08-27T20:52:46.352 INFO:tasks.rados.rados.0.smithi093.stdout:1164: expect (ObjNum 508 snap 0 seq_num 508) 2023-08-27T20:52:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:46 smithi136 ceph-mon[113531]: pgmap v77: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 68 MiB/s rd, 10 MiB/s wr, 100 op/s 2023-08-27T20:52:46.462 INFO:tasks.rados.rados.0.smithi093.stdout:1163: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:52:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:46 smithi093 ceph-mon[111335]: pgmap v77: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 68 MiB/s rd, 10 MiB/s wr, 100 op/s 2023-08-27T20:52:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:46 smithi093 ceph-mon[118902]: pgmap v77: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 68 MiB/s rd, 10 MiB/s wr, 100 op/s 2023-08-27T20:52:46.538 INFO:tasks.rados.rados.0.smithi093.stdout:1162: expect (ObjNum 850 snap 0 seq_num 850) 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1165: finishing write tid 1 to smithi093153278-451 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1165: finishing write tid 2 to smithi093153278-451 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1165: finishing write tid 3 to smithi093153278-451 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1165: finishing write tid 4 to smithi093153278-451 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 451 v 200 (ObjNum 873 snap 0 seq_num 873) dirty exists 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1165: left oid 451 (ObjNum 873 snap 0 seq_num 873) 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1157: done (9 left) 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1158: done (8 left) 2023-08-27T20:52:46.682 INFO:tasks.rados.rados.0.smithi093.stdout:1159: done (7 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1160: done (6 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1161: done (5 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1162: done (4 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1163: done (3 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1164: done (2 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1165: done (1 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1166: done (0 left) 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1167: write oid 500 current snap is 0 2023-08-27T20:52:46.683 INFO:tasks.rados.rados.0.smithi093.stdout:1167: seq_num 875 ranges {706125=423470,1652803=758303} 2023-08-27T20:52:46.687 INFO:tasks.rados.rados.0.smithi093.stdout:1167: writing smithi093153278-500 from 706125 to 1129595 tid 1 2023-08-27T20:52:46.695 INFO:tasks.rados.rados.0.smithi093.stdout:1167: writing smithi093153278-500 from 1652803 to 2411106 tid 2 2023-08-27T20:52:46.695 INFO:tasks.rados.rados.0.smithi093.stdout:1168: read oid 264 snap -1 2023-08-27T20:52:46.695 INFO:tasks.rados.rados.0.smithi093.stdout:1168: expect deleted 2023-08-27T20:52:46.695 INFO:tasks.rados.rados.0.smithi093.stdout:1169: write oid 133 current snap is 0 2023-08-27T20:52:46.695 INFO:tasks.rados.rados.0.smithi093.stdout:1167: finishing write tid 1 to smithi093153278-500 2023-08-27T20:52:46.695 INFO:tasks.rados.rados.0.smithi093.stdout:1169: seq_num 876 ranges {722932=498719,1994932=501961} 2023-08-27T20:52:46.701 INFO:tasks.rados.rados.0.smithi093.stdout:1169: writing smithi093153278-133 from 722932 to 1221651 tid 1 2023-08-27T20:52:46.708 INFO:tasks.rados.rados.0.smithi093.stdout:1169: writing smithi093153278-133 from 1994932 to 2496893 tid 2 2023-08-27T20:52:46.709 INFO:tasks.rados.rados.0.smithi093.stdout:1170: read oid 400 snap -1 2023-08-27T20:52:46.709 INFO:tasks.rados.rados.0.smithi093.stdout:1170: expect (ObjNum 399 snap 0 seq_num 399) 2023-08-27T20:52:46.709 INFO:tasks.rados.rados.0.smithi093.stdout:1171: write oid 135 current snap is 0 2023-08-27T20:52:46.709 INFO:tasks.rados.rados.0.smithi093.stdout:1167: finishing write tid 2 to smithi093153278-500 2023-08-27T20:52:46.709 INFO:tasks.rados.rados.0.smithi093.stdout:1167: finishing write tid 3 to smithi093153278-500 2023-08-27T20:52:46.709 INFO:tasks.rados.rados.0.smithi093.stdout:1171: seq_num 877 ranges {739739=573968,2053414=686325} 2023-08-27T20:52:46.716 INFO:tasks.rados.rados.0.smithi093.stdout:1171: writing smithi093153278-135 from 739739 to 1313707 tid 1 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1171: writing smithi093153278-135 from 2053414 to 2739739 tid 2 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1168: done (4 left) 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1172: read oid 198 snap -1 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1172: expect (ObjNum 713 snap 0 seq_num 713) 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1173: read oid 26 snap -1 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1173: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1174: read oid 175 snap -1 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1174: expect (ObjNum 678 snap 0 seq_num 678) 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1175: write oid 77 current snap is 0 2023-08-27T20:52:46.725 INFO:tasks.rados.rados.0.smithi093.stdout:1167: finishing write tid 4 to smithi093153278-500 2023-08-27T20:52:46.726 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 500 v 209 (ObjNum 875 snap 0 seq_num 875) dirty exists 2023-08-27T20:52:46.726 INFO:tasks.rados.rados.0.smithi093.stdout:1167: left oid 500 (ObjNum 875 snap 0 seq_num 875) 2023-08-27T20:52:46.726 INFO:tasks.rados.rados.0.smithi093.stdout:1169: finishing write tid 1 to smithi093153278-133 2023-08-27T20:52:46.726 INFO:tasks.rados.rados.0.smithi093.stdout:1175: seq_num 878 ranges {756546=649217,2111896=505630} 2023-08-27T20:52:46.732 INFO:tasks.rados.rados.0.smithi093.stdout:1175: writing smithi093153278-77 from 756546 to 1405763 tid 1 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1175: writing smithi093153278-77 from 2111896 to 2617526 tid 2 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1167: done (7 left) 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1176: read oid 125 snap -1 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1176: expect (ObjNum 124 snap 0 seq_num 124) 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1177: write oid 289 current snap is 0 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1169: finishing write tid 2 to smithi093153278-133 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1169: finishing write tid 3 to smithi093153278-133 2023-08-27T20:52:46.740 INFO:tasks.rados.rados.0.smithi093.stdout:1169: finishing write tid 4 to smithi093153278-133 2023-08-27T20:52:46.741 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 133 v 199 (ObjNum 876 snap 0 seq_num 876) dirty exists 2023-08-27T20:52:46.741 INFO:tasks.rados.rados.0.smithi093.stdout:1169: left oid 133 (ObjNum 876 snap 0 seq_num 876) 2023-08-27T20:52:46.741 INFO:tasks.rados.rados.0.smithi093.stdout:1170: expect (ObjNum 399 snap 0 seq_num 399) 2023-08-27T20:52:46.825 INFO:tasks.rados.rados.0.smithi093.stdout:1171: finishing write tid 1 to smithi093153278-135 2023-08-27T20:52:46.825 INFO:tasks.rados.rados.0.smithi093.stdout:1177: seq_num 879 ranges {773353=724466,2170378=602975} 2023-08-27T20:52:46.833 INFO:tasks.rados.rados.0.smithi093.stdout:1177: writing smithi093153278-289 from 773353 to 1497819 tid 1 2023-08-27T20:52:46.840 INFO:tasks.rados.rados.0.smithi093.stdout:1177: writing smithi093153278-289 from 2170378 to 2773353 tid 2 2023-08-27T20:52:46.840 INFO:tasks.rados.rados.0.smithi093.stdout:1169: done (8 left) 2023-08-27T20:52:46.841 INFO:tasks.rados.rados.0.smithi093.stdout:1170: done (7 left) 2023-08-27T20:52:46.841 INFO:tasks.rados.rados.0.smithi093.stdout:1178: write oid 63 current snap is 0 2023-08-27T20:52:46.841 INFO:tasks.rados.rados.0.smithi093.stdout:1178: seq_num 880 ranges {790160=799715,2112507=509299} 2023-08-27T20:52:46.849 INFO:tasks.rados.rados.0.smithi093.stdout:1178: writing smithi093153278-63 from 790160 to 1589875 tid 1 2023-08-27T20:52:46.855 INFO:tasks.rados.rados.0.smithi093.stdout:1178: writing smithi093153278-63 from 2112507 to 2621806 tid 2 2023-08-27T20:52:46.855 INFO:tasks.rados.rados.0.smithi093.stdout:1179: write oid 420 current snap is 0 2023-08-27T20:52:46.855 INFO:tasks.rados.rados.0.smithi093.stdout:1173: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:46.929 INFO:tasks.rados.rados.0.smithi093.stdout:1174: expect (ObjNum 678 snap 0 seq_num 678) 2023-08-27T20:52:47.080 INFO:tasks.rados.rados.0.smithi093.stdout:1171: finishing write tid 2 to smithi093153278-135 2023-08-27T20:52:47.080 INFO:tasks.rados.rados.0.smithi093.stdout:1171: finishing write tid 3 to smithi093153278-135 2023-08-27T20:52:47.080 INFO:tasks.rados.rados.0.smithi093.stdout:1179: seq_num 881 ranges {406967=474964,1370989=769310,2677186=129781} 2023-08-27T20:52:47.084 INFO:tasks.rados.rados.0.smithi093.stdout:1179: writing smithi093153278-420 from 406967 to 881931 tid 1 2023-08-27T20:52:47.091 INFO:tasks.rados.rados.0.smithi093.stdout:1179: writing smithi093153278-420 from 1370989 to 2140299 tid 2 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1179: writing smithi093153278-420 from 2677186 to 2806967 tid 3 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1173: done (8 left) 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1174: done (7 left) 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1180: read oid 349 snap -1 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1180: expect (ObjNum 669 snap 0 seq_num 669) 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1181: read oid 320 snap -1 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1181: expect (ObjNum 517 snap 0 seq_num 517) 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1182: read oid 204 snap -1 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1182: expect (ObjNum 798 snap 0 seq_num 798) 2023-08-27T20:52:47.096 INFO:tasks.rados.rados.0.smithi093.stdout:1183: write oid 130 current snap is 0 2023-08-27T20:52:47.097 INFO:tasks.rados.rados.0.smithi093.stdout:1183: seq_num 882 ranges {423774=666566,1545824=512968,2704609=119165} 2023-08-27T20:52:47.101 INFO:tasks.rados.rados.0.smithi093.stdout:1183: writing smithi093153278-130 from 423774 to 1090340 tid 1 2023-08-27T20:52:47.107 INFO:tasks.rados.rados.0.smithi093.stdout:1183: writing smithi093153278-130 from 1545824 to 2058792 tid 2 2023-08-27T20:52:47.112 INFO:tasks.rados.rados.0.smithi093.stdout:1183: writing smithi093153278-130 from 2704609 to 2823774 tid 3 2023-08-27T20:52:47.112 INFO:tasks.rados.rados.0.smithi093.stdout:1184: read oid 275 snap -1 2023-08-27T20:52:47.112 INFO:tasks.rados.rados.0.smithi093.stdout:1184: expect (ObjNum 274 snap 0 seq_num 274) 2023-08-27T20:52:47.112 INFO:tasks.rados.rados.0.smithi093.stdout:1185: read oid 325 snap -1 2023-08-27T20:52:47.112 INFO:tasks.rados.rados.0.smithi093.stdout:1185: expect (ObjNum 324 snap 0 seq_num 324) 2023-08-27T20:52:47.112 INFO:tasks.rados.rados.0.smithi093.stdout:1186: write oid 60 current snap is 0 2023-08-27T20:52:47.113 INFO:tasks.rados.rados.0.smithi093.stdout:1171: finishing write tid 4 to smithi093153278-135 2023-08-27T20:52:47.113 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 135 v 164 (ObjNum 877 snap 0 seq_num 877) dirty exists 2023-08-27T20:52:47.113 INFO:tasks.rados.rados.0.smithi093.stdout:1171: left oid 135 (ObjNum 877 snap 0 seq_num 877) 2023-08-27T20:52:47.113 INFO:tasks.rados.rados.0.smithi093.stdout:1172: expect (ObjNum 713 snap 0 seq_num 713) 2023-08-27T20:52:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:1175: finishing write tid 1 to smithi093153278-77 2023-08-27T20:52:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:1175: finishing write tid 2 to smithi093153278-77 2023-08-27T20:52:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:1175: finishing write tid 3 to smithi093153278-77 2023-08-27T20:52:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:1175: finishing write tid 4 to smithi093153278-77 2023-08-27T20:52:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 77 v 203 (ObjNum 878 snap 0 seq_num 878) dirty exists 2023-08-27T20:52:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:1175: left oid 77 (ObjNum 878 snap 0 seq_num 878) 2023-08-27T20:52:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:1176: expect (ObjNum 124 snap 0 seq_num 124) 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:1177: finishing write tid 1 to smithi093153278-289 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:1177: finishing write tid 2 to smithi093153278-289 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:1177: finishing write tid 3 to smithi093153278-289 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:1177: finishing write tid 4 to smithi093153278-289 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 289 v 178 (ObjNum 879 snap 0 seq_num 879) dirty exists 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:1177: left oid 289 (ObjNum 879 snap 0 seq_num 879) 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:1178: finishing write tid 1 to smithi093153278-63 2023-08-27T20:52:47.351 INFO:tasks.rados.rados.0.smithi093.stdout:1178: finishing write tid 2 to smithi093153278-63 2023-08-27T20:52:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:1178: finishing write tid 3 to smithi093153278-63 2023-08-27T20:52:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:1178: finishing write tid 4 to smithi093153278-63 2023-08-27T20:52:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 63 v 167 (ObjNum 880 snap 0 seq_num 880) dirty exists 2023-08-27T20:52:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:1178: left oid 63 (ObjNum 880 snap 0 seq_num 880) 2023-08-27T20:52:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:1179: finishing write tid 1 to smithi093153278-420 2023-08-27T20:52:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:1179: finishing write tid 2 to smithi093153278-420 2023-08-27T20:52:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:1180: expect (ObjNum 669 snap 0 seq_num 669) 2023-08-27T20:52:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1547905865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:47.475 INFO:tasks.rados.rados.0.smithi093.stdout:1181: expect (ObjNum 517 snap 0 seq_num 517) 2023-08-27T20:52:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1547905865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1547905865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:47.577 INFO:tasks.rados.rados.0.smithi093.stdout:1179: finishing write tid 3 to smithi093153278-420 2023-08-27T20:52:47.577 INFO:tasks.rados.rados.0.smithi093.stdout:1179: finishing write tid 4 to smithi093153278-420 2023-08-27T20:52:47.577 INFO:tasks.rados.rados.0.smithi093.stdout:1179: finishing write tid 5 to smithi093153278-420 2023-08-27T20:52:47.578 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 420 v 200 (ObjNum 881 snap 0 seq_num 881) dirty exists 2023-08-27T20:52:47.578 INFO:tasks.rados.rados.0.smithi093.stdout:1179: left oid 420 (ObjNum 881 snap 0 seq_num 881) 2023-08-27T20:52:47.578 INFO:tasks.rados.rados.0.smithi093.stdout:1182: expect (ObjNum 798 snap 0 seq_num 798) 2023-08-27T20:52:47.765 INFO:tasks.rados.rados.0.smithi093.stdout:1183: finishing write tid 1 to smithi093153278-130 2023-08-27T20:52:47.765 INFO:tasks.rados.rados.0.smithi093.stdout:1183: finishing write tid 2 to smithi093153278-130 2023-08-27T20:52:47.765 INFO:tasks.rados.rados.0.smithi093.stdout:1183: finishing write tid 3 to smithi093153278-130 2023-08-27T20:52:47.765 INFO:tasks.rados.rados.0.smithi093.stdout:1184: expect (ObjNum 274 snap 0 seq_num 274) 2023-08-27T20:52:47.845 INFO:tasks.rados.rados.0.smithi093.stdout:1183: finishing write tid 4 to smithi093153278-130 2023-08-27T20:52:47.845 INFO:tasks.rados.rados.0.smithi093.stdout:1183: finishing write tid 5 to smithi093153278-130 2023-08-27T20:52:47.846 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 130 v 204 (ObjNum 882 snap 0 seq_num 882) dirty exists 2023-08-27T20:52:47.846 INFO:tasks.rados.rados.0.smithi093.stdout:1183: left oid 130 (ObjNum 882 snap 0 seq_num 882) 2023-08-27T20:52:47.846 INFO:tasks.rados.rados.0.smithi093.stdout:1185: expect (ObjNum 324 snap 0 seq_num 324) 2023-08-27T20:52:47.950 INFO:tasks.rados.rados.0.smithi093.stdout:1186: seq_num 883 ranges {440581=741815,1604306=656626,2732032=108549} 2023-08-27T20:52:47.956 INFO:tasks.rados.rados.0.smithi093.stdout:1186: writing smithi093153278-60 from 440581 to 1182396 tid 1 2023-08-27T20:52:47.962 INFO:tasks.rados.rados.0.smithi093.stdout:1186: writing smithi093153278-60 from 1604306 to 2260932 tid 2 2023-08-27T20:52:47.967 INFO:tasks.rados.rados.0.smithi093.stdout:1186: writing smithi093153278-60 from 2732032 to 2840581 tid 3 2023-08-27T20:52:47.967 INFO:tasks.rados.rados.0.smithi093.stdout:1171: done (13 left) 2023-08-27T20:52:47.967 INFO:tasks.rados.rados.0.smithi093.stdout:1172: done (12 left) 2023-08-27T20:52:47.967 INFO:tasks.rados.rados.0.smithi093.stdout:1175: done (11 left) 2023-08-27T20:52:47.967 INFO:tasks.rados.rados.0.smithi093.stdout:1176: done (10 left) 2023-08-27T20:52:47.967 INFO:tasks.rados.rados.0.smithi093.stdout:1177: done (9 left) 2023-08-27T20:52:47.967 INFO:tasks.rados.rados.0.smithi093.stdout:1178: done (8 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1179: done (7 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1180: done (6 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1181: done (5 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1182: done (4 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1183: done (3 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1184: done (2 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1185: done (1 left) 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1187: write oid 345 current snap is 0 2023-08-27T20:52:47.968 INFO:tasks.rados.rados.0.smithi093.stdout:1186: finishing write tid 1 to smithi093153278-60 2023-08-27T20:52:47.969 INFO:tasks.rados.rados.0.smithi093.stdout:1186: finishing write tid 2 to smithi093153278-60 2023-08-27T20:52:47.969 INFO:tasks.rados.rados.0.smithi093.stdout:1187: seq_num 884 ranges {457388=417064,1546435=516637,2643102=214286} 2023-08-27T20:52:47.970 INFO:tasks.rados.rados.0.smithi093.stdout:1187: writing smithi093153278-345 from 457388 to 874452 tid 1 2023-08-27T20:52:47.977 INFO:tasks.rados.rados.0.smithi093.stdout:1187: writing smithi093153278-345 from 1546435 to 2063072 tid 2 2023-08-27T20:52:47.982 INFO:tasks.rados.rados.0.smithi093.stdout:1187: writing smithi093153278-345 from 2643102 to 2857388 tid 3 2023-08-27T20:52:47.982 INFO:tasks.rados.rados.0.smithi093.stdout:1188: write oid 85 current snap is 0 2023-08-27T20:52:47.982 INFO:tasks.rados.rados.0.smithi093.stdout:1186: finishing write tid 3 to smithi093153278-60 2023-08-27T20:52:47.982 INFO:tasks.rados.rados.0.smithi093.stdout:1186: finishing write tid 4 to smithi093153278-60 2023-08-27T20:52:47.982 INFO:tasks.rados.rados.0.smithi093.stdout:1186: finishing write tid 5 to smithi093153278-60 2023-08-27T20:52:47.982 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 60 v 171 (ObjNum 883 snap 0 seq_num 883) dirty exists 2023-08-27T20:52:47.983 INFO:tasks.rados.rados.0.smithi093.stdout:1186: left oid 60 (ObjNum 883 snap 0 seq_num 883) 2023-08-27T20:52:47.983 INFO:tasks.rados.rados.0.smithi093.stdout:1187: finishing write tid 1 to smithi093153278-345 2023-08-27T20:52:47.983 INFO:tasks.rados.rados.0.smithi093.stdout:1187: finishing write tid 2 to smithi093153278-345 2023-08-27T20:52:47.983 INFO:tasks.rados.rados.0.smithi093.stdout:1188: seq_num 885 ranges {474195=492313,1604917=660295,2670525=203670} 2023-08-27T20:52:47.987 INFO:tasks.rados.rados.0.smithi093.stdout:1188: writing smithi093153278-85 from 474195 to 966508 tid 1 2023-08-27T20:52:47.994 INFO:tasks.rados.rados.0.smithi093.stdout:1188: writing smithi093153278-85 from 1604917 to 2265212 tid 2 2023-08-27T20:52:47.998 INFO:tasks.rados.rados.0.smithi093.stdout:1188: writing smithi093153278-85 from 2670525 to 2874195 tid 3 2023-08-27T20:52:47.998 INFO:tasks.rados.rados.0.smithi093.stdout:1186: done (2 left) 2023-08-27T20:52:47.998 INFO:tasks.rados.rados.0.smithi093.stdout:1189: write oid 72 current snap is 0 2023-08-27T20:52:47.998 INFO:tasks.rados.rados.0.smithi093.stdout:1187: finishing write tid 3 to smithi093153278-345 2023-08-27T20:52:47.998 INFO:tasks.rados.rados.0.smithi093.stdout:1187: finishing write tid 4 to smithi093153278-345 2023-08-27T20:52:47.998 INFO:tasks.rados.rados.0.smithi093.stdout:1187: finishing write tid 5 to smithi093153278-345 2023-08-27T20:52:47.998 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 345 v 130 (ObjNum 884 snap 0 seq_num 884) dirty exists 2023-08-27T20:52:47.999 INFO:tasks.rados.rados.0.smithi093.stdout:1187: left oid 345 (ObjNum 884 snap 0 seq_num 884) 2023-08-27T20:52:47.999 INFO:tasks.rados.rados.0.smithi093.stdout:1188: finishing write tid 1 to smithi093153278-85 2023-08-27T20:52:47.999 INFO:tasks.rados.rados.0.smithi093.stdout:1189: seq_num 886 ranges {491002=567562,1663399=520306,2814301=76701} 2023-08-27T20:52:48.003 INFO:tasks.rados.rados.0.smithi093.stdout:1189: writing smithi093153278-72 from 491002 to 1058564 tid 1 2023-08-27T20:52:48.009 INFO:tasks.rados.rados.0.smithi093.stdout:1189: writing smithi093153278-72 from 1663399 to 2183705 tid 2 2023-08-27T20:52:48.014 INFO:tasks.rados.rados.0.smithi093.stdout:1189: writing smithi093153278-72 from 2814301 to 2891002 tid 3 2023-08-27T20:52:48.014 INFO:tasks.rados.rados.0.smithi093.stdout:1187: done (2 left) 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:1190: read oid 5 snap -1 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:1190: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:1191: delete oid 446 current snap is 0 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:1188: finishing write tid 2 to smithi093153278-85 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:1188: finishing write tid 3 to smithi093153278-85 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:1188: finishing write tid 4 to smithi093153278-85 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:1188: finishing write tid 5 to smithi093153278-85 2023-08-27T20:52:48.015 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 85 v 203 (ObjNum 885 snap 0 seq_num 885) dirty exists 2023-08-27T20:52:48.016 INFO:tasks.rados.rados.0.smithi093.stdout:1188: left oid 85 (ObjNum 885 snap 0 seq_num 885) 2023-08-27T20:52:48.016 INFO:tasks.rados.rados.0.smithi093.stdout:1189: finishing write tid 1 to smithi093153278-72 2023-08-27T20:52:48.016 INFO:tasks.rados.rados.0.smithi093.stdout:1189: finishing write tid 2 to smithi093153278-72 2023-08-27T20:52:48.017 INFO:tasks.rados.rados.0.smithi093.stdout:1188: done (3 left) 2023-08-27T20:52:48.017 INFO:tasks.rados.rados.0.smithi093.stdout:1191: done (2 left) 2023-08-27T20:52:48.017 INFO:tasks.rados.rados.0.smithi093.stdout:1192: write oid 62 current snap is 0 2023-08-27T20:52:48.017 INFO:tasks.rados.rados.0.smithi093.stdout:1192: seq_num 888 ranges {524616=718060,1664010=523975,2752794=171822} 2023-08-27T20:52:48.024 INFO:tasks.rados.rados.0.smithi093.stdout:1192: writing smithi093153278-62 from 524616 to 1242676 tid 1 2023-08-27T20:52:48.029 INFO:tasks.rados.rados.0.smithi093.stdout:1192: writing smithi093153278-62 from 1664010 to 2187985 tid 2 2023-08-27T20:52:48.033 INFO:tasks.rados.rados.0.smithi093.stdout:1192: writing smithi093153278-62 from 2752794 to 2924616 tid 3 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1189: finishing write tid 3 to smithi093153278-72 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1189: finishing write tid 4 to smithi093153278-72 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1193: read oid 250 snap -1 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1193: expect (ObjNum 508 snap 0 seq_num 508) 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1194: delete oid 445 current snap is 0 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1189: finishing write tid 5 to smithi093153278-72 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 72 v 201 (ObjNum 886 snap 0 seq_num 886) dirty exists 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1189: left oid 72 (ObjNum 886 snap 0 seq_num 886) 2023-08-27T20:52:48.034 INFO:tasks.rados.rados.0.smithi093.stdout:1190: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:52:48.128 INFO:tasks.rados.rados.0.smithi093.stdout:1192: finishing write tid 1 to smithi093153278-62 2023-08-27T20:52:48.128 INFO:tasks.rados.rados.0.smithi093.stdout:1192: finishing write tid 2 to smithi093153278-62 2023-08-27T20:52:48.128 INFO:tasks.rados.rados.0.smithi093.stdout:1192: finishing write tid 3 to smithi093153278-62 2023-08-27T20:52:48.128 INFO:tasks.rados.rados.0.smithi093.stdout:1193: expect (ObjNum 508 snap 0 seq_num 508) 2023-08-27T20:52:48.244 INFO:tasks.rados.rados.0.smithi093.stdout:1192: finishing write tid 4 to smithi093153278-62 2023-08-27T20:52:48.244 INFO:tasks.rados.rados.0.smithi093.stdout:1192: finishing write tid 5 to smithi093153278-62 2023-08-27T20:52:48.244 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 62 v 181 (ObjNum 888 snap 0 seq_num 888) dirty exists 2023-08-27T20:52:48.244 INFO:tasks.rados.rados.0.smithi093.stdout:1192: left oid 62 (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:52:48.245 INFO:tasks.rados.rados.0.smithi093.stdout:1189: done (4 left) 2023-08-27T20:52:48.245 INFO:tasks.rados.rados.0.smithi093.stdout:1190: done (3 left) 2023-08-27T20:52:48.245 INFO:tasks.rados.rados.0.smithi093.stdout:1192: done (2 left) 2023-08-27T20:52:48.245 INFO:tasks.rados.rados.0.smithi093.stdout:1193: done (1 left) 2023-08-27T20:52:48.245 INFO:tasks.rados.rados.0.smithi093.stdout:1194: done (0 left) 2023-08-27T20:52:48.245 INFO:tasks.rados.rados.0.smithi093.stdout:1195: write oid 317 current snap is 0 2023-08-27T20:52:48.245 INFO:tasks.rados.rados.0.smithi093.stdout:1195: seq_num 890 ranges {558230=584911,1897327=527644,2923993=34237} 2023-08-27T20:52:48.250 INFO:tasks.rados.rados.0.smithi093.stdout:1195: writing smithi093153278-317 from 558230 to 1143141 tid 1 2023-08-27T20:52:48.257 INFO:tasks.rados.rados.0.smithi093.stdout:1195: writing smithi093153278-317 from 1897327 to 2424971 tid 2 2023-08-27T20:52:48.260 INFO:tasks.rados.rados.0.smithi093.stdout:1195: writing smithi093153278-317 from 2923993 to 2958230 tid 3 2023-08-27T20:52:48.260 INFO:tasks.rados.rados.0.smithi093.stdout:1196: delete oid 383 current snap is 0 2023-08-27T20:52:48.260 INFO:tasks.rados.rados.0.smithi093.stdout:1195: finishing write tid 1 to smithi093153278-317 2023-08-27T20:52:48.261 INFO:tasks.rados.rados.0.smithi093.stdout:1195: finishing write tid 2 to smithi093153278-317 2023-08-27T20:52:48.262 INFO:tasks.rados.rados.0.smithi093.stdout:1196: done (1 left) 2023-08-27T20:52:48.262 INFO:tasks.rados.rados.0.smithi093.stdout:1197: write oid 429 current snap is 0 2023-08-27T20:52:48.262 INFO:tasks.rados.rados.0.smithi093.stdout:1197: seq_num 892 ranges {591844=735409,1897938=531313,2862486=129358} 2023-08-27T20:52:48.269 INFO:tasks.rados.rados.0.smithi093.stdout:1197: writing smithi093153278-429 from 591844 to 1327253 tid 1 2023-08-27T20:52:48.275 INFO:tasks.rados.rados.0.smithi093.stdout:1197: writing smithi093153278-429 from 1897938 to 2429251 tid 2 2023-08-27T20:52:48.279 INFO:tasks.rados.rados.0.smithi093.stdout:1197: writing smithi093153278-429 from 2862486 to 2991844 tid 3 2023-08-27T20:52:48.279 INFO:tasks.rados.rados.0.smithi093.stdout:1198: delete oid 187 current snap is 0 2023-08-27T20:52:48.279 INFO:tasks.rados.rados.0.smithi093.stdout:1195: finishing write tid 3 to smithi093153278-317 2023-08-27T20:52:48.279 INFO:tasks.rados.rados.0.smithi093.stdout:1195: finishing write tid 4 to smithi093153278-317 2023-08-27T20:52:48.279 INFO:tasks.rados.rados.0.smithi093.stdout:1195: finishing write tid 5 to smithi093153278-317 2023-08-27T20:52:48.279 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 317 v 183 (ObjNum 890 snap 0 seq_num 890) dirty exists 2023-08-27T20:52:48.279 INFO:tasks.rados.rados.0.smithi093.stdout:1195: left oid 317 (ObjNum 890 snap 0 seq_num 890) 2023-08-27T20:52:48.280 INFO:tasks.rados.rados.0.smithi093.stdout:1195: done (2 left) 2023-08-27T20:52:48.280 INFO:tasks.rados.rados.0.smithi093.stdout:1198: done (1 left) 2023-08-27T20:52:48.280 INFO:tasks.rados.rados.0.smithi093.stdout:1199: write oid 127 current snap is 0 2023-08-27T20:52:48.280 INFO:tasks.rados.rados.0.smithi093.stdout:1197: finishing write tid 1 to smithi093153278-429 2023-08-27T20:52:48.280 INFO:tasks.rados.rados.0.smithi093.stdout:1199: seq_num 894 ranges {625458=485907,1614902=534982,2917332=108126} 2023-08-27T20:52:48.286 INFO:tasks.rados.rados.0.smithi093.stdout:1199: writing smithi093153278-127 from 625458 to 1111365 tid 1 2023-08-27T20:52:48.292 INFO:tasks.rados.rados.0.smithi093.stdout:1199: writing smithi093153278-127 from 1614902 to 2149884 tid 2 2023-08-27T20:52:48.297 INFO:tasks.rados.rados.0.smithi093.stdout:1199: writing smithi093153278-127 from 2917332 to 3025458 tid 3 2023-08-27T20:52:48.297 INFO:tasks.rados.rados.0.smithi093.stdout:1200: write oid 484 current snap is 0 2023-08-27T20:52:48.297 INFO:tasks.rados.rados.0.smithi093.stdout:1197: finishing write tid 2 to smithi093153278-429 2023-08-27T20:52:48.297 INFO:tasks.rados.rados.0.smithi093.stdout:1200: seq_num 895 ranges {642265=561156,1673384=678640,2828402=213863} 2023-08-27T20:52:48.304 INFO:tasks.rados.rados.0.smithi093.stdout:1200: writing smithi093153278-484 from 642265 to 1203421 tid 1 2023-08-27T20:52:48.310 INFO:tasks.rados.rados.0.smithi093.stdout:1200: writing smithi093153278-484 from 1673384 to 2352024 tid 2 2023-08-27T20:52:48.314 INFO:tasks.rados.rados.0.smithi093.stdout:1200: writing smithi093153278-484 from 2828402 to 3042265 tid 3 2023-08-27T20:52:48.314 INFO:tasks.rados.rados.0.smithi093.stdout:1201: write oid 124 current snap is 0 2023-08-27T20:52:48.314 INFO:tasks.rados.rados.0.smithi093.stdout:1197: finishing write tid 3 to smithi093153278-429 2023-08-27T20:52:48.314 INFO:tasks.rados.rados.0.smithi093.stdout:1197: finishing write tid 4 to smithi093153278-429 2023-08-27T20:52:48.315 INFO:tasks.rados.rados.0.smithi093.stdout:1197: finishing write tid 5 to smithi093153278-429 2023-08-27T20:52:48.315 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 429 v 208 (ObjNum 892 snap 0 seq_num 892) dirty exists 2023-08-27T20:52:48.315 INFO:tasks.rados.rados.0.smithi093.stdout:1197: left oid 429 (ObjNum 892 snap 0 seq_num 892) 2023-08-27T20:52:48.315 INFO:tasks.rados.rados.0.smithi093.stdout:1199: finishing write tid 1 to smithi093153278-127 2023-08-27T20:52:48.315 INFO:tasks.rados.rados.0.smithi093.stdout:1201: seq_num 896 ranges {659072=636405,1731866=422298,2855825=203247} 2023-08-27T20:52:48.321 INFO:tasks.rados.rados.0.smithi093.stdout:1201: writing smithi093153278-124 from 659072 to 1295477 tid 1 2023-08-27T20:52:48.326 INFO:tasks.rados.rados.0.smithi093.stdout:1201: writing smithi093153278-124 from 1731866 to 2154164 tid 2 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1201: writing smithi093153278-124 from 2855825 to 3059072 tid 3 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1197: done (3 left) 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1202: read oid 182 snap -1 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1202: expect (ObjNum 830 snap 0 seq_num 830) 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1203: write oid 349 current snap is 0 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1199: finishing write tid 2 to smithi093153278-127 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1199: finishing write tid 3 to smithi093153278-127 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1199: finishing write tid 4 to smithi093153278-127 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:1199: finishing write tid 5 to smithi093153278-127 2023-08-27T20:52:48.332 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 127 v 212 (ObjNum 894 snap 0 seq_num 894) dirty exists 2023-08-27T20:52:48.333 INFO:tasks.rados.rados.0.smithi093.stdout:1199: left oid 127 (ObjNum 894 snap 0 seq_num 894) 2023-08-27T20:52:48.333 INFO:tasks.rados.rados.0.smithi093.stdout:1200: finishing write tid 1 to smithi093153278-484 2023-08-27T20:52:48.333 INFO:tasks.rados.rados.0.smithi093.stdout:1203: seq_num 897 ranges {675879=711654,2073995=682309} 2023-08-27T20:52:48.339 INFO:tasks.rados.rados.0.smithi093.stdout:1203: writing smithi093153278-349 from 675879 to 1387533 tid 1 2023-08-27T20:52:48.347 INFO:tasks.rados.rados.0.smithi093.stdout:1203: writing smithi093153278-349 from 2073995 to 2756304 tid 2 2023-08-27T20:52:48.347 INFO:tasks.rados.rados.0.smithi093.stdout:1199: done (4 left) 2023-08-27T20:52:48.347 INFO:tasks.rados.rados.0.smithi093.stdout:1204: read oid 26 snap -1 2023-08-27T20:52:48.347 INFO:tasks.rados.rados.0.smithi093.stdout:1204: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:48.347 INFO:tasks.rados.rados.0.smithi093.stdout:1205: write oid 195 current snap is 0 2023-08-27T20:52:48.348 INFO:tasks.rados.rados.0.smithi093.stdout:1200: finishing write tid 2 to smithi093153278-484 2023-08-27T20:52:48.348 INFO:tasks.rados.rados.0.smithi093.stdout:1205: seq_num 898 ranges {692686=503256,1848830=425967,2910671=182015} 2023-08-27T20:52:48.353 INFO:tasks.rados.rados.0.smithi093.stdout:1205: writing smithi093153278-195 from 692686 to 1195942 tid 1 2023-08-27T20:52:48.359 INFO:tasks.rados.rados.0.smithi093.stdout:1205: writing smithi093153278-195 from 1848830 to 2274797 tid 2 2023-08-27T20:52:48.364 INFO:tasks.rados.rados.0.smithi093.stdout:1205: writing smithi093153278-195 from 2910671 to 3092686 tid 3 2023-08-27T20:52:48.364 INFO:tasks.rados.rados.0.smithi093.stdout:1206: write oid 70 current snap is 0 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1200: finishing write tid 3 to smithi093153278-484 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1200: finishing write tid 4 to smithi093153278-484 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1200: finishing write tid 5 to smithi093153278-484 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 484 v 252 (ObjNum 895 snap 0 seq_num 895) dirty exists 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1200: left oid 484 (ObjNum 895 snap 0 seq_num 895) 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1201: finishing write tid 1 to smithi093153278-124 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1201: finishing write tid 2 to smithi093153278-124 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1201: finishing write tid 3 to smithi093153278-124 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1201: finishing write tid 4 to smithi093153278-124 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1201: finishing write tid 5 to smithi093153278-124 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 124 v 189 (ObjNum 896 snap 0 seq_num 896) dirty exists 2023-08-27T20:52:48.365 INFO:tasks.rados.rados.0.smithi093.stdout:1201: left oid 124 (ObjNum 896 snap 0 seq_num 896) 2023-08-27T20:52:48.366 INFO:tasks.rados.rados.0.smithi093.stdout:1202: expect (ObjNum 830 snap 0 seq_num 830) 2023-08-27T20:52:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:48 smithi136 ceph-mon[113531]: pgmap v78: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 MiB/s rd, 15 MiB/s wr, 152 op/s 2023-08-27T20:52:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:48 smithi093 ceph-mon[111335]: pgmap v78: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 MiB/s rd, 15 MiB/s wr, 152 op/s 2023-08-27T20:52:48.618 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:48 smithi093 ceph-mon[118902]: pgmap v78: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 102 MiB/s rd, 15 MiB/s wr, 152 op/s 2023-08-27T20:52:48.618 INFO:tasks.rados.rados.0.smithi093.stdout:1203: finishing write tid 1 to smithi093153278-349 2023-08-27T20:52:48.618 INFO:tasks.rados.rados.0.smithi093.stdout:1204: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:1203: finishing write tid 2 to smithi093153278-349 2023-08-27T20:52:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:1203: finishing write tid 3 to smithi093153278-349 2023-08-27T20:52:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:1203: finishing write tid 4 to smithi093153278-349 2023-08-27T20:52:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 349 v 213 (ObjNum 897 snap 0 seq_num 897) dirty exists 2023-08-27T20:52:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:1203: left oid 349 (ObjNum 897 snap 0 seq_num 897) 2023-08-27T20:52:48.623 INFO:tasks.rados.rados.0.smithi093.stdout:1205: finishing write tid 1 to smithi093153278-195 2023-08-27T20:52:48.623 INFO:tasks.rados.rados.0.smithi093.stdout:1205: finishing write tid 2 to smithi093153278-195 2023-08-27T20:52:48.623 INFO:tasks.rados.rados.0.smithi093.stdout:1206: seq_num 899 ranges {709493=578505,1907312=685978} 2023-08-27T20:52:48.629 INFO:tasks.rados.rados.0.smithi093.stdout:1206: writing smithi093153278-70 from 709493 to 1287998 tid 1 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1206: writing smithi093153278-70 from 1907312 to 2593290 tid 2 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1200: done (6 left) 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1201: done (5 left) 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1202: done (4 left) 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1203: done (3 left) 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1204: done (2 left) 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1207: read oid 93 snap -1 2023-08-27T20:52:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:1207: expect (ObjNum 683 snap 0 seq_num 683) 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1208: read oid 326 snap -1 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1208: expect (ObjNum 832 snap 0 seq_num 832) 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1209: read oid 238 snap -1 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1209: expect (ObjNum 237 snap 0 seq_num 237) 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1210: write oid 341 current snap is 0 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1205: finishing write tid 3 to smithi093153278-195 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1205: finishing write tid 4 to smithi093153278-195 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1205: finishing write tid 5 to smithi093153278-195 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 155 (ObjNum 898 snap 0 seq_num 898) dirty exists 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1205: left oid 195 (ObjNum 898 snap 0 seq_num 898) 2023-08-27T20:52:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:1206: finishing write tid 1 to smithi093153278-70 2023-08-27T20:52:48.639 INFO:tasks.rados.rados.0.smithi093.stdout:1210: seq_num 900 ranges {726300=653754,1965794=429636,2965517=160783} 2023-08-27T20:52:48.645 INFO:tasks.rados.rados.0.smithi093.stdout:1210: writing smithi093153278-341 from 726300 to 1380054 tid 1 2023-08-27T20:52:48.650 INFO:tasks.rados.rados.0.smithi093.stdout:1210: writing smithi093153278-341 from 1965794 to 2395430 tid 2 2023-08-27T20:52:48.655 INFO:tasks.rados.rados.0.smithi093.stdout:1210: writing smithi093153278-341 from 2965517 to 3126300 tid 3 2023-08-27T20:52:48.655 INFO:tasks.rados.rados.0.smithi093.stdout:1205: done (5 left) 2023-08-27T20:52:48.655 INFO:tasks.rados.rados.0.smithi093.stdout:1211: write oid 148 current snap is 0 2023-08-27T20:52:48.655 INFO:tasks.rados.rados.0.smithi093.stdout:1211: seq_num 901 ranges {743107=729003,1907923=689647} 2023-08-27T20:52:48.663 INFO:tasks.rados.rados.0.smithi093.stdout:1211: writing smithi093153278-148 from 743107 to 1472110 tid 1 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1211: writing smithi093153278-148 from 1907923 to 2597570 tid 2 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1212: read oid 218 snap -1 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1212: expect (ObjNum 526 snap 0 seq_num 526) 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1213: read oid 333 snap -1 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1213: expect (ObjNum 332 snap 0 seq_num 332) 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1214: write oid 355 current snap is 0 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1206: finishing write tid 2 to smithi093153278-70 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1206: finishing write tid 3 to smithi093153278-70 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1206: finishing write tid 4 to smithi093153278-70 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 70 v 192 (ObjNum 899 snap 0 seq_num 899) dirty exists 2023-08-27T20:52:48.670 INFO:tasks.rados.rados.0.smithi093.stdout:1206: left oid 70 (ObjNum 899 snap 0 seq_num 899) 2023-08-27T20:52:48.671 INFO:tasks.rados.rados.0.smithi093.stdout:1214: seq_num 902 ranges {759914=404252,1566405=433305,2504010=655904} 2023-08-27T20:52:48.676 INFO:tasks.rados.rados.0.smithi093.stdout:1214: writing smithi093153278-355 from 759914 to 1164166 tid 1 2023-08-27T20:52:48.680 INFO:tasks.rados.rados.0.smithi093.stdout:1214: writing smithi093153278-355 from 1566405 to 1999710 tid 2 2023-08-27T20:52:48.687 INFO:tasks.rados.rados.0.smithi093.stdout:1214: writing smithi093153278-355 from 2504010 to 3159914 tid 3 2023-08-27T20:52:48.687 INFO:tasks.rados.rados.0.smithi093.stdout:1206: done (8 left) 2023-08-27T20:52:48.687 INFO:tasks.rados.rados.0.smithi093.stdout:1215: delete oid 107 current snap is 0 2023-08-27T20:52:48.687 INFO:tasks.rados.rados.0.smithi093.stdout:1207: expect (ObjNum 683 snap 0 seq_num 683) 2023-08-27T20:52:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:1209: expect (ObjNum 237 snap 0 seq_num 237) 2023-08-27T20:52:48.960 INFO:tasks.rados.rados.0.smithi093.stdout:1208: expect (ObjNum 832 snap 0 seq_num 832) 2023-08-27T20:52:49.166 INFO:tasks.rados.rados.0.smithi093.stdout:1210: finishing write tid 1 to smithi093153278-341 2023-08-27T20:52:49.166 INFO:tasks.rados.rados.0.smithi093.stdout:1210: finishing write tid 2 to smithi093153278-341 2023-08-27T20:52:49.166 INFO:tasks.rados.rados.0.smithi093.stdout:1210: finishing write tid 3 to smithi093153278-341 2023-08-27T20:52:49.166 INFO:tasks.rados.rados.0.smithi093.stdout:1210: finishing write tid 4 to smithi093153278-341 2023-08-27T20:52:49.166 INFO:tasks.rados.rados.0.smithi093.stdout:1210: finishing write tid 5 to smithi093153278-341 2023-08-27T20:52:49.166 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 341 v 185 (ObjNum 900 snap 0 seq_num 900) dirty exists 2023-08-27T20:52:49.167 INFO:tasks.rados.rados.0.smithi093.stdout:1210: left oid 341 (ObjNum 900 snap 0 seq_num 900) 2023-08-27T20:52:49.167 INFO:tasks.rados.rados.0.smithi093.stdout:1211: finishing write tid 1 to smithi093153278-148 2023-08-27T20:52:49.167 INFO:tasks.rados.rados.0.smithi093.stdout:1207: done (8 left) 2023-08-27T20:52:49.167 INFO:tasks.rados.rados.0.smithi093.stdout:1208: done (7 left) 2023-08-27T20:52:49.168 INFO:tasks.rados.rados.0.smithi093.stdout:1209: done (6 left) 2023-08-27T20:52:49.168 INFO:tasks.rados.rados.0.smithi093.stdout:1210: done (5 left) 2023-08-27T20:52:49.168 INFO:tasks.rados.rados.0.smithi093.stdout:1215: done (4 left) 2023-08-27T20:52:49.168 INFO:tasks.rados.rados.0.smithi093.stdout:1216: write oid 492 current snap is 0 2023-08-27T20:52:49.168 INFO:tasks.rados.rados.0.smithi093.stdout:1216: seq_num 904 ranges {793528=554750,2083369=436974,2958856=234672} 2023-08-27T20:52:49.174 INFO:tasks.rados.rados.0.smithi093.stdout:1216: writing smithi093153278-492 from 793528 to 1348278 tid 1 2023-08-27T20:52:49.180 INFO:tasks.rados.rados.0.smithi093.stdout:1216: writing smithi093153278-492 from 2083369 to 2520343 tid 2 2023-08-27T20:52:49.184 INFO:tasks.rados.rados.0.smithi093.stdout:1216: writing smithi093153278-492 from 2958856 to 3193528 tid 3 2023-08-27T20:52:49.184 INFO:tasks.rados.rados.0.smithi093.stdout:1217: read oid 351 snap -1 2023-08-27T20:52:49.184 INFO:tasks.rados.rados.0.smithi093.stdout:1217: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:52:49.184 INFO:tasks.rados.rados.0.smithi093.stdout:1218: delete oid 458 current snap is 0 2023-08-27T20:52:49.185 INFO:tasks.rados.rados.0.smithi093.stdout:1212: expect (ObjNum 526 snap 0 seq_num 526) 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1211: finishing write tid 2 to smithi093153278-148 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1211: finishing write tid 3 to smithi093153278-148 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1211: finishing write tid 4 to smithi093153278-148 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 148 v 174 (ObjNum 901 snap 0 seq_num 901) dirty exists 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1211: left oid 148 (ObjNum 901 snap 0 seq_num 901) 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1214: finishing write tid 1 to smithi093153278-355 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1211: done (6 left) 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1212: done (5 left) 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1218: done (4 left) 2023-08-27T20:52:49.299 INFO:tasks.rados.rados.0.smithi093.stdout:1219: read oid 412 snap -1 2023-08-27T20:52:49.300 INFO:tasks.rados.rados.0.smithi093.stdout:1219: expect (ObjNum 824 snap 0 seq_num 824) 2023-08-27T20:52:49.300 INFO:tasks.rados.rados.0.smithi093.stdout:1220: read oid 433 snap -1 2023-08-27T20:52:49.300 INFO:tasks.rados.rados.0.smithi093.stdout:1220: expect (ObjNum 869 snap 0 seq_num 869) 2023-08-27T20:52:49.300 INFO:tasks.rados.rados.0.smithi093.stdout:1221: write oid 198 current snap is 0 2023-08-27T20:52:49.300 INFO:tasks.rados.rados.0.smithi093.stdout:1213: expect (ObjNum 332 snap 0 seq_num 332) 2023-08-27T20:52:49.408 INFO:tasks.rados.rados.0.smithi093.stdout:1214: finishing write tid 2 to smithi093153278-355 2023-08-27T20:52:49.409 INFO:tasks.rados.rados.0.smithi093.stdout:1214: finishing write tid 3 to smithi093153278-355 2023-08-27T20:52:49.409 INFO:tasks.rados.rados.0.smithi093.stdout:1221: seq_num 906 ranges {427142=421601,1400333=440643,2613702=450326} 2023-08-27T20:52:49.413 INFO:tasks.rados.rados.0.smithi093.stdout:1221: writing smithi093153278-198 from 427142 to 848743 tid 1 2023-08-27T20:52:49.418 INFO:tasks.rados.rados.0.smithi093.stdout:1221: writing smithi093153278-198 from 1400333 to 1840976 tid 2 2023-08-27T20:52:49.426 INFO:tasks.rados.rados.0.smithi093.stdout:1221: writing smithi093153278-198 from 2613702 to 3064028 tid 3 2023-08-27T20:52:49.426 INFO:tasks.rados.rados.0.smithi093.stdout:1213: done (6 left) 2023-08-27T20:52:49.426 INFO:tasks.rados.rados.0.smithi093.stdout:1222: read oid 264 snap -1 2023-08-27T20:52:49.426 INFO:tasks.rados.rados.0.smithi093.stdout:1222: expect deleted 2023-08-27T20:52:49.426 INFO:tasks.rados.rados.0.smithi093.stdout:1223: read oid 17 snap -1 2023-08-27T20:52:49.426 INFO:tasks.rados.rados.0.smithi093.stdout:1223: expect (ObjNum 16 snap 0 seq_num 16) 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1224: delete oid 144 current snap is 0 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1214: finishing write tid 4 to smithi093153278-355 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1214: finishing write tid 5 to smithi093153278-355 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 355 v 159 (ObjNum 902 snap 0 seq_num 902) dirty exists 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1214: left oid 355 (ObjNum 902 snap 0 seq_num 902) 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1216: finishing write tid 1 to smithi093153278-492 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1216: finishing write tid 2 to smithi093153278-492 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1216: finishing write tid 3 to smithi093153278-492 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1216: finishing write tid 4 to smithi093153278-492 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:1216: finishing write tid 5 to smithi093153278-492 2023-08-27T20:52:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 492 v 208 (ObjNum 904 snap 0 seq_num 904) dirty exists 2023-08-27T20:52:49.428 INFO:tasks.rados.rados.0.smithi093.stdout:1216: left oid 492 (ObjNum 904 snap 0 seq_num 904) 2023-08-27T20:52:49.428 INFO:tasks.rados.rados.0.smithi093.stdout:1217: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:52:49.549 INFO:tasks.rados.rados.0.smithi093.stdout:1220: expect (ObjNum 869 snap 0 seq_num 869) 2023-08-27T20:52:49.663 INFO:tasks.rados.rados.0.smithi093.stdout:1219: expect (ObjNum 824 snap 0 seq_num 824) 2023-08-27T20:52:49.779 INFO:tasks.rados.rados.0.smithi093.stdout:1221: finishing write tid 1 to smithi093153278-198 2023-08-27T20:52:49.780 INFO:tasks.rados.rados.0.smithi093.stdout:1221: finishing write tid 2 to smithi093153278-198 2023-08-27T20:52:49.780 INFO:tasks.rados.rados.0.smithi093.stdout:1223: expect (ObjNum 16 snap 0 seq_num 16) 2023-08-27T20:52:49.850 INFO:tasks.rados.rados.0.smithi093.stdout:1221: finishing write tid 3 to smithi093153278-198 2023-08-27T20:52:49.850 INFO:tasks.rados.rados.0.smithi093.stdout:1221: finishing write tid 4 to smithi093153278-198 2023-08-27T20:52:49.850 INFO:tasks.rados.rados.0.smithi093.stdout:1221: finishing write tid 5 to smithi093153278-198 2023-08-27T20:52:49.850 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 198 v 256 (ObjNum 906 snap 0 seq_num 906) dirty exists 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1221: left oid 198 (ObjNum 906 snap 0 seq_num 906) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1214: done (8 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1216: done (7 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1217: done (6 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1219: done (5 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1220: done (4 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1221: done (3 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1222: done (2 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1223: done (1 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1224: done (0 left) 2023-08-27T20:52:49.851 INFO:tasks.rados.rados.0.smithi093.stdout:1225: read oid 103 snap -1 2023-08-27T20:52:49.852 INFO:tasks.rados.rados.0.smithi093.stdout:1225: expect (ObjNum 102 snap 0 seq_num 102) 2023-08-27T20:52:49.852 INFO:tasks.rados.rados.0.smithi093.stdout:1226: read oid 182 snap -1 2023-08-27T20:52:49.852 INFO:tasks.rados.rados.0.smithi093.stdout:1226: expect (ObjNum 830 snap 0 seq_num 830) 2023-08-27T20:52:49.852 INFO:tasks.rados.rados.0.smithi093.stdout:1227: read oid 465 snap -1 2023-08-27T20:52:49.852 INFO:tasks.rados.rados.0.smithi093.stdout:1227: expect (ObjNum 789 snap 0 seq_num 789) 2023-08-27T20:52:49.852 INFO:tasks.rados.rados.0.smithi093.stdout:1228: delete oid 57 current snap is 0 2023-08-27T20:52:49.853 INFO:tasks.rados.rados.0.smithi093.stdout:1228: done (3 left) 2023-08-27T20:52:49.853 INFO:tasks.rados.rados.0.smithi093.stdout:1229: read oid 347 snap -1 2023-08-27T20:52:49.853 INFO:tasks.rados.rados.0.smithi093.stdout:1229: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:52:49.853 INFO:tasks.rados.rados.0.smithi093.stdout:1230: write oid 68 current snap is 0 2023-08-27T20:52:49.853 INFO:tasks.rados.rados.0.smithi093.stdout:1230: seq_num 909 ranges {477563=647348,1859426=587970,2979618=297945} 2023-08-27T20:52:49.858 INFO:tasks.rados.rados.0.smithi093.stdout:1230: writing smithi093153278-68 from 477563 to 1124911 tid 1 2023-08-27T20:52:49.865 INFO:tasks.rados.rados.0.smithi093.stdout:1230: writing smithi093153278-68 from 1859426 to 2447396 tid 2 2023-08-27T20:52:49.870 INFO:tasks.rados.rados.0.smithi093.stdout:1230: writing smithi093153278-68 from 2979618 to 3277563 tid 3 2023-08-27T20:52:49.870 INFO:tasks.rados.rados.0.smithi093.stdout:1231: read oid 50 snap -1 2023-08-27T20:52:49.871 INFO:tasks.rados.rados.0.smithi093.stdout:1231: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:52:49.871 INFO:tasks.rados.rados.0.smithi093.stdout:1232: write oid 380 current snap is 0 2023-08-27T20:52:49.871 INFO:tasks.rados.rados.0.smithi093.stdout:1232: seq_num 910 ranges {494370=722597,1917908=447981,3007041=287329} 2023-08-27T20:52:49.877 INFO:tasks.rados.rados.0.smithi093.stdout:1232: writing smithi093153278-380 from 494370 to 1216967 tid 1 2023-08-27T20:52:49.883 INFO:tasks.rados.rados.0.smithi093.stdout:1232: writing smithi093153278-380 from 1917908 to 2365889 tid 2 2023-08-27T20:52:49.890 INFO:tasks.rados.rados.0.smithi093.stdout:1232: writing smithi093153278-380 from 3007041 to 3294370 tid 3 2023-08-27T20:52:49.890 INFO:tasks.rados.rados.0.smithi093.stdout:1233: write oid 146 current snap is 0 2023-08-27T20:52:49.890 INFO:tasks.rados.rados.0.smithi093.stdout:1233: seq_num 911 ranges {511177=797846,1976390=591639,3034464=276713} 2023-08-27T20:52:49.895 INFO:tasks.rados.rados.0.smithi093.stdout:1233: writing smithi093153278-146 from 511177 to 1309023 tid 1 2023-08-27T20:52:49.902 INFO:tasks.rados.rados.0.smithi093.stdout:1233: writing smithi093153278-146 from 1976390 to 2568029 tid 2 2023-08-27T20:52:49.907 INFO:tasks.rados.rados.0.smithi093.stdout:1233: writing smithi093153278-146 from 3034464 to 3311177 tid 3 2023-08-27T20:52:49.907 INFO:tasks.rados.rados.0.smithi093.stdout:1234: write oid 25 current snap is 0 2023-08-27T20:52:49.907 INFO:tasks.rados.rados.0.smithi093.stdout:1234: seq_num 912 ranges {527984=473095,1634872=451650,2661887=634311} 2023-08-27T20:52:49.912 INFO:tasks.rados.rados.0.smithi093.stdout:1234: writing smithi093153278-25 from 527984 to 1001079 tid 1 2023-08-27T20:52:49.918 INFO:tasks.rados.rados.0.smithi093.stdout:1234: writing smithi093153278-25 from 1634872 to 2086522 tid 2 2023-08-27T20:52:49.924 INFO:tasks.rados.rados.0.smithi093.stdout:1234: writing smithi093153278-25 from 2661887 to 3296198 tid 3 2023-08-27T20:52:49.925 INFO:tasks.rados.rados.0.smithi093.stdout:1235: write oid 17 current snap is 0 2023-08-27T20:52:49.925 INFO:tasks.rados.rados.0.smithi093.stdout:1235: seq_num 913 ranges {544791=664697,1693354=595308,2689310=445583} 2023-08-27T20:52:49.931 INFO:tasks.rados.rados.0.smithi093.stdout:1235: writing smithi093153278-17 from 544791 to 1209488 tid 1 2023-08-27T20:52:49.937 INFO:tasks.rados.rados.0.smithi093.stdout:1235: writing smithi093153278-17 from 1693354 to 2288662 tid 2 2023-08-27T20:52:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4013196298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:49 smithi136 ceph-mon[113531]: pgmap v79: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 10 MiB/s wr, 99 op/s 2023-08-27T20:52:49.942 INFO:tasks.rados.rados.0.smithi093.stdout:1235: writing smithi093153278-17 from 2689310 to 3134893 tid 3 2023-08-27T20:52:49.942 INFO:tasks.rados.rados.0.smithi093.stdout:1236: read oid 125 snap -1 2023-08-27T20:52:49.942 INFO:tasks.rados.rados.0.smithi093.stdout:1236: expect (ObjNum 124 snap 0 seq_num 124) 2023-08-27T20:52:49.942 INFO:tasks.rados.rados.0.smithi093.stdout:1237: write oid 378 current snap is 0 2023-08-27T20:52:49.942 INFO:tasks.rados.rados.0.smithi093.stdout:1226: expect (ObjNum 830 snap 0 seq_num 830) 2023-08-27T20:52:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4013196298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:49 smithi093 ceph-mon[111335]: pgmap v79: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 10 MiB/s wr, 99 op/s 2023-08-27T20:52:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4013196298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:49 smithi093 ceph-mon[118902]: pgmap v79: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 10 MiB/s wr, 99 op/s 2023-08-27T20:52:50.120 INFO:tasks.rados.rados.0.smithi093.stdout:1227: expect (ObjNum 789 snap 0 seq_num 789) 2023-08-27T20:52:50.270 INFO:tasks.rados.rados.0.smithi093.stdout:1225: expect (ObjNum 102 snap 0 seq_num 102) 2023-08-27T20:52:50.384 INFO:tasks.rados.rados.0.smithi093.stdout:1229: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:52:50.506 INFO:tasks.rados.rados.0.smithi093.stdout:1230: finishing write tid 1 to smithi093153278-68 2023-08-27T20:52:50.506 INFO:tasks.rados.rados.0.smithi093.stdout:1230: finishing write tid 2 to smithi093153278-68 2023-08-27T20:52:50.506 INFO:tasks.rados.rados.0.smithi093.stdout:1237: seq_num 914 ranges {561598=739946,1751836=455319,2716733=644865} 2023-08-27T20:52:50.513 INFO:tasks.rados.rados.0.smithi093.stdout:1237: writing smithi093153278-378 from 561598 to 1301544 tid 1 2023-08-27T20:52:50.518 INFO:tasks.rados.rados.0.smithi093.stdout:1237: writing smithi093153278-378 from 1751836 to 2207155 tid 2 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1237: writing smithi093153278-378 from 2716733 to 3361598 tid 3 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1225: done (11 left) 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1226: done (10 left) 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1227: done (9 left) 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1229: done (8 left) 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1238: write oid 192 current snap is 0 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1230: finishing write tid 3 to smithi093153278-68 2023-08-27T20:52:50.525 INFO:tasks.rados.rados.0.smithi093.stdout:1238: seq_num 915 ranges {578405=415195,1410318=598977,2744156=584480} 2023-08-27T20:52:50.530 INFO:tasks.rados.rados.0.smithi093.stdout:1238: writing smithi093153278-192 from 578405 to 993600 tid 1 2023-08-27T20:52:50.535 INFO:tasks.rados.rados.0.smithi093.stdout:1238: writing smithi093153278-192 from 1410318 to 2009295 tid 2 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:1238: writing smithi093153278-192 from 2744156 to 3328636 tid 3 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:1239: read oid 223 snap -1 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:1239: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:1240: write oid 478 current snap is 0 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:1230: finishing write tid 4 to smithi093153278-68 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:1230: finishing write tid 5 to smithi093153278-68 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 68 v 182 (ObjNum 909 snap 0 seq_num 909) dirty exists 2023-08-27T20:52:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:1230: left oid 68 (ObjNum 909 snap 0 seq_num 909) 2023-08-27T20:52:50.544 INFO:tasks.rados.rados.0.smithi093.stdout:1231: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:52:50.631 INFO:tasks.rados.rados.0.smithi093.stdout:1232: finishing write tid 1 to smithi093153278-380 2023-08-27T20:52:50.632 INFO:tasks.rados.rados.0.smithi093.stdout:1232: finishing write tid 2 to smithi093153278-380 2023-08-27T20:52:50.632 INFO:tasks.rados.rados.0.smithi093.stdout:1232: finishing write tid 3 to smithi093153278-380 2023-08-27T20:52:50.632 INFO:tasks.rados.rados.0.smithi093.stdout:1240: seq_num 916 ranges {595212=490444,1868800=458988,2887932=507280} 2023-08-27T20:52:50.637 INFO:tasks.rados.rados.0.smithi093.stdout:1240: writing smithi093153278-478 from 595212 to 1085656 tid 1 2023-08-27T20:52:50.644 INFO:tasks.rados.rados.0.smithi093.stdout:1240: writing smithi093153278-478 from 1868800 to 2327788 tid 2 2023-08-27T20:52:50.652 INFO:tasks.rados.rados.0.smithi093.stdout:1240: writing smithi093153278-478 from 2887932 to 3395212 tid 3 2023-08-27T20:52:50.652 INFO:tasks.rados.rados.0.smithi093.stdout:1230: done (10 left) 2023-08-27T20:52:50.652 INFO:tasks.rados.rados.0.smithi093.stdout:1231: done (9 left) 2023-08-27T20:52:50.653 INFO:tasks.rados.rados.0.smithi093.stdout:1241: read oid 298 snap -1 2023-08-27T20:52:50.653 INFO:tasks.rados.rados.0.smithi093.stdout:1241: expect (ObjNum 297 snap 0 seq_num 297) 2023-08-27T20:52:50.653 INFO:tasks.rados.rados.0.smithi093.stdout:1242: write oid 253 current snap is 0 2023-08-27T20:52:50.653 INFO:tasks.rados.rados.0.smithi093.stdout:1232: finishing write tid 4 to smithi093153278-380 2023-08-27T20:52:50.653 INFO:tasks.rados.rados.0.smithi093.stdout:1232: finishing write tid 5 to smithi093153278-380 2023-08-27T20:52:50.653 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 380 v 206 (ObjNum 910 snap 0 seq_num 910) dirty exists 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:1232: left oid 380 (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:1233: finishing write tid 1 to smithi093153278-146 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:1233: finishing write tid 2 to smithi093153278-146 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:1233: finishing write tid 3 to smithi093153278-146 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:1233: finishing write tid 4 to smithi093153278-146 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:1233: finishing write tid 5 to smithi093153278-146 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 146 v 187 (ObjNum 911 snap 0 seq_num 911) dirty exists 2023-08-27T20:52:50.654 INFO:tasks.rados.rados.0.smithi093.stdout:1233: left oid 146 (ObjNum 911 snap 0 seq_num 911) 2023-08-27T20:52:50.655 INFO:tasks.rados.rados.0.smithi093.stdout:1234: finishing write tid 1 to smithi093153278-25 2023-08-27T20:52:50.655 INFO:tasks.rados.rados.0.smithi093.stdout:1234: finishing write tid 2 to smithi093153278-25 2023-08-27T20:52:50.655 INFO:tasks.rados.rados.0.smithi093.stdout:1234: finishing write tid 3 to smithi093153278-25 2023-08-27T20:52:50.655 INFO:tasks.rados.rados.0.smithi093.stdout:1234: finishing write tid 4 to smithi093153278-25 2023-08-27T20:52:50.655 INFO:tasks.rados.rados.0.smithi093.stdout:1234: finishing write tid 5 to smithi093153278-25 2023-08-27T20:52:50.655 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 25 v 190 (ObjNum 912 snap 0 seq_num 912) dirty exists 2023-08-27T20:52:50.656 INFO:tasks.rados.rados.0.smithi093.stdout:1234: left oid 25 (ObjNum 912 snap 0 seq_num 912) 2023-08-27T20:52:50.656 INFO:tasks.rados.rados.0.smithi093.stdout:1235: finishing write tid 1 to smithi093153278-17 2023-08-27T20:52:50.656 INFO:tasks.rados.rados.0.smithi093.stdout:1235: finishing write tid 2 to smithi093153278-17 2023-08-27T20:52:50.656 INFO:tasks.rados.rados.0.smithi093.stdout:1235: finishing write tid 3 to smithi093153278-17 2023-08-27T20:52:50.656 INFO:tasks.rados.rados.0.smithi093.stdout:1235: finishing write tid 4 to smithi093153278-17 2023-08-27T20:52:50.656 INFO:tasks.rados.rados.0.smithi093.stdout:1235: finishing write tid 5 to smithi093153278-17 2023-08-27T20:52:50.656 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 17 v 166 (ObjNum 913 snap 0 seq_num 913) dirty exists 2023-08-27T20:52:50.657 INFO:tasks.rados.rados.0.smithi093.stdout:1235: left oid 17 (ObjNum 913 snap 0 seq_num 913) 2023-08-27T20:52:50.657 INFO:tasks.rados.rados.0.smithi093.stdout:1242: seq_num 917 ranges {612019=565693,1810929=602646,3082649=329370} 2023-08-27T20:52:50.657 INFO:tasks.rados.rados.0.smithi093.stdout:1242: writing smithi093153278-253 from 612019 to 1177712 tid 1 2023-08-27T20:52:50.664 INFO:tasks.rados.rados.0.smithi093.stdout:1242: writing smithi093153278-253 from 1810929 to 2413575 tid 2 2023-08-27T20:52:50.670 INFO:tasks.rados.rados.0.smithi093.stdout:1242: writing smithi093153278-253 from 3082649 to 3412019 tid 3 2023-08-27T20:52:50.670 INFO:tasks.rados.rados.0.smithi093.stdout:1232: done (10 left) 2023-08-27T20:52:50.670 INFO:tasks.rados.rados.0.smithi093.stdout:1233: done (9 left) 2023-08-27T20:52:50.670 INFO:tasks.rados.rados.0.smithi093.stdout:1234: done (8 left) 2023-08-27T20:52:50.670 INFO:tasks.rados.rados.0.smithi093.stdout:1235: done (7 left) 2023-08-27T20:52:50.670 INFO:tasks.rados.rados.0.smithi093.stdout:1243: write oid 415 current snap is 0 2023-08-27T20:52:50.671 INFO:tasks.rados.rados.0.smithi093.stdout:1236: expect (ObjNum 124 snap 0 seq_num 124) 2023-08-27T20:52:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[115431]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:50] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:52:50.735 INFO:tasks.rados.rados.0.smithi093.stdout:1237: finishing write tid 1 to smithi093153278-378 2023-08-27T20:52:50.735 INFO:tasks.rados.rados.0.smithi093.stdout:1237: finishing write tid 2 to smithi093153278-378 2023-08-27T20:52:50.735 INFO:tasks.rados.rados.0.smithi093.stdout:1237: finishing write tid 3 to smithi093153278-378 2023-08-27T20:52:50.735 INFO:tasks.rados.rados.0.smithi093.stdout:1237: finishing write tid 4 to smithi093153278-378 2023-08-27T20:52:50.735 INFO:tasks.rados.rados.0.smithi093.stdout:1237: finishing write tid 5 to smithi093153278-378 2023-08-27T20:52:50.736 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 378 v 186 (ObjNum 914 snap 0 seq_num 914) dirty exists 2023-08-27T20:52:50.736 INFO:tasks.rados.rados.0.smithi093.stdout:1237: left oid 378 (ObjNum 914 snap 0 seq_num 914) 2023-08-27T20:52:50.736 INFO:tasks.rados.rados.0.smithi093.stdout:1238: finishing write tid 1 to smithi093153278-192 2023-08-27T20:52:50.736 INFO:tasks.rados.rados.0.smithi093.stdout:1243: seq_num 918 ranges {628826=640942,1869411=462657,2826425=534649} 2023-08-27T20:52:50.741 INFO:tasks.rados.rados.0.smithi093.stdout:1243: writing smithi093153278-415 from 628826 to 1269768 tid 1 2023-08-27T20:52:50.747 INFO:tasks.rados.rados.0.smithi093.stdout:1243: writing smithi093153278-415 from 1869411 to 2332068 tid 2 2023-08-27T20:52:50.753 INFO:tasks.rados.rados.0.smithi093.stdout:1243: writing smithi093153278-415 from 2826425 to 3361074 tid 3 2023-08-27T20:52:50.754 INFO:tasks.rados.rados.0.smithi093.stdout:1236: done (7 left) 2023-08-27T20:52:50.754 INFO:tasks.rados.rados.0.smithi093.stdout:1237: done (6 left) 2023-08-27T20:52:50.754 INFO:tasks.rados.rados.0.smithi093.stdout:1244: write oid 108 current snap is 0 2023-08-27T20:52:50.754 INFO:tasks.rados.rados.0.smithi093.stdout:1238: finishing write tid 2 to smithi093153278-192 2023-08-27T20:52:50.754 INFO:tasks.rados.rados.0.smithi093.stdout:1238: finishing write tid 3 to smithi093153278-192 2023-08-27T20:52:50.754 INFO:tasks.rados.rados.0.smithi093.stdout:1244: seq_num 919 ranges {645633=716191,1927893=606315,3137495=308138} 2023-08-27T20:52:50.761 INFO:tasks.rados.rados.0.smithi093.stdout:1244: writing smithi093153278-108 from 645633 to 1361824 tid 1 2023-08-27T20:52:50.767 INFO:tasks.rados.rados.0.smithi093.stdout:1244: writing smithi093153278-108 from 1927893 to 2534208 tid 2 2023-08-27T20:52:50.773 INFO:tasks.rados.rados.0.smithi093.stdout:1244: writing smithi093153278-108 from 3137495 to 3445633 tid 3 2023-08-27T20:52:50.773 INFO:tasks.rados.rados.0.smithi093.stdout:1245: read oid 47 snap -1 2023-08-27T20:52:50.773 INFO:tasks.rados.rados.0.smithi093.stdout:1245: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:52:50.773 INFO:tasks.rados.rados.0.smithi093.stdout:1246: read oid 426 snap -1 2023-08-27T20:52:50.773 INFO:tasks.rados.rados.0.smithi093.stdout:1246: expect (ObjNum 596 snap 0 seq_num 596) 2023-08-27T20:52:50.774 INFO:tasks.rados.rados.0.smithi093.stdout:1247: write oid 452 current snap is 0 2023-08-27T20:52:50.774 INFO:tasks.rados.rados.0.smithi093.stdout:1238: finishing write tid 4 to smithi093153278-192 2023-08-27T20:52:50.774 INFO:tasks.rados.rados.0.smithi093.stdout:1238: finishing write tid 5 to smithi093153278-192 2023-08-27T20:52:50.774 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 192 v 178 (ObjNum 915 snap 0 seq_num 915) dirty exists 2023-08-27T20:52:50.774 INFO:tasks.rados.rados.0.smithi093.stdout:1238: left oid 192 (ObjNum 915 snap 0 seq_num 915) 2023-08-27T20:52:50.774 INFO:tasks.rados.rados.0.smithi093.stdout:1239: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:52:50.866 INFO:tasks.rados.rados.0.smithi093.stdout:1240: finishing write tid 1 to smithi093153278-478 2023-08-27T20:52:50.867 INFO:tasks.rados.rados.0.smithi093.stdout:1240: finishing write tid 2 to smithi093153278-478 2023-08-27T20:52:50.867 INFO:tasks.rados.rados.0.smithi093.stdout:1240: finishing write tid 3 to smithi093153278-478 2023-08-27T20:52:50.867 INFO:tasks.rados.rados.0.smithi093.stdout:1240: finishing write tid 4 to smithi093153278-478 2023-08-27T20:52:50.867 INFO:tasks.rados.rados.0.smithi093.stdout:1240: finishing write tid 5 to smithi093153278-478 2023-08-27T20:52:50.867 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 210 (ObjNum 916 snap 0 seq_num 916) dirty exists 2023-08-27T20:52:50.867 INFO:tasks.rados.rados.0.smithi093.stdout:1240: left oid 478 (ObjNum 916 snap 0 seq_num 916) 2023-08-27T20:52:50.867 INFO:tasks.rados.rados.0.smithi093.stdout:1247: seq_num 920 ranges {662440=507793,1702728=749973,2881271=557193} 2023-08-27T20:52:50.872 INFO:tasks.rados.rados.0.smithi093.stdout:1247: writing smithi093153278-452 from 662440 to 1170233 tid 1 2023-08-27T20:52:50.880 INFO:tasks.rados.rados.0.smithi093.stdout:1247: writing smithi093153278-452 from 1702728 to 2452701 tid 2 2023-08-27T20:52:50.885 INFO:tasks.rados.rados.0.smithi093.stdout:1247: writing smithi093153278-452 from 2881271 to 3438464 tid 3 2023-08-27T20:52:50.886 INFO:tasks.rados.rados.0.smithi093.stdout:1238: done (9 left) 2023-08-27T20:52:50.886 INFO:tasks.rados.rados.0.smithi093.stdout:1239: done (8 left) 2023-08-27T20:52:50.886 INFO:tasks.rados.rados.0.smithi093.stdout:1240: done (7 left) 2023-08-27T20:52:50.886 INFO:tasks.rados.rados.0.smithi093.stdout:1248: write oid 51 current snap is 0 2023-08-27T20:52:50.886 INFO:tasks.rados.rados.0.smithi093.stdout:1248: seq_num 921 ranges {679247=583042,2044857=609984,3192341=286906} 2023-08-27T20:52:50.892 INFO:tasks.rados.rados.0.smithi093.stdout:1248: writing smithi093153278-51 from 679247 to 1262289 tid 1 2023-08-27T20:52:50.900 INFO:tasks.rados.rados.0.smithi093.stdout:1248: writing smithi093153278-51 from 2044857 to 2654841 tid 2 2023-08-27T20:52:50.905 INFO:tasks.rados.rados.0.smithi093.stdout:1248: writing smithi093153278-51 from 3192341 to 3479247 tid 3 2023-08-27T20:52:50.905 INFO:tasks.rados.rados.0.smithi093.stdout:1249: write oid 383 current snap is 0 2023-08-27T20:52:50.905 INFO:tasks.rados.rados.0.smithi093.stdout:1241: expect (ObjNum 297 snap 0 seq_num 297) 2023-08-27T20:52:51.004 INFO:tasks.rados.rados.0.smithi093.stdout:1242: finishing write tid 1 to smithi093153278-253 2023-08-27T20:52:51.005 INFO:tasks.rados.rados.0.smithi093.stdout:1242: finishing write tid 2 to smithi093153278-253 2023-08-27T20:52:51.005 INFO:tasks.rados.rados.0.smithi093.stdout:1242: finishing write tid 3 to smithi093153278-253 2023-08-27T20:52:51.005 INFO:tasks.rados.rados.0.smithi093.stdout:1242: finishing write tid 4 to smithi093153278-253 2023-08-27T20:52:51.005 INFO:tasks.rados.rados.0.smithi093.stdout:1242: finishing write tid 5 to smithi093153278-253 2023-08-27T20:52:51.005 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 190 (ObjNum 917 snap 0 seq_num 917) dirty exists 2023-08-27T20:52:51.005 INFO:tasks.rados.rados.0.smithi093.stdout:1242: left oid 253 (ObjNum 917 snap 0 seq_num 917) 2023-08-27T20:52:51.005 INFO:tasks.rados.rados.0.smithi093.stdout:1249: seq_num 922 ranges {696054=658291,2103339=753642} 2023-08-27T20:52:51.011 INFO:tasks.rados.rados.0.smithi093.stdout:1249: writing smithi093153278-383 from 696054 to 1354345 tid 1 2023-08-27T20:52:51.020 INFO:tasks.rados.rados.0.smithi093.stdout:1249: writing smithi093153278-383 from 2103339 to 2856981 tid 2 2023-08-27T20:52:51.020 INFO:tasks.rados.rados.0.smithi093.stdout:1241: done (8 left) 2023-08-27T20:52:51.020 INFO:tasks.rados.rados.0.smithi093.stdout:1242: done (7 left) 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1250: read oid 33 snap -1 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1250: expect (ObjNum 32 snap 0 seq_num 32) 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1251: read oid 487 snap -1 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1251: expect deleted 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1252: read oid 373 snap -1 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1252: expect deleted 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1253: write oid 381 current snap is 0 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1243: finishing write tid 1 to smithi093153278-415 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1243: finishing write tid 2 to smithi093153278-415 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1243: finishing write tid 3 to smithi093153278-415 2023-08-27T20:52:51.021 INFO:tasks.rados.rados.0.smithi093.stdout:1243: finishing write tid 4 to smithi093153278-415 2023-08-27T20:52:51.022 INFO:tasks.rados.rados.0.smithi093.stdout:1253: seq_num 923 ranges {712861=733540,2161821=613653,3247187=265674} 2023-08-27T20:52:51.029 INFO:tasks.rados.rados.0.smithi093.stdout:1253: writing smithi093153278-381 from 712861 to 1446401 tid 1 2023-08-27T20:52:51.036 INFO:tasks.rados.rados.0.smithi093.stdout:1253: writing smithi093153278-381 from 2161821 to 2775474 tid 2 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:1253: writing smithi093153278-381 from 3247187 to 3512861 tid 3 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:1254: write oid 160 current snap is 0 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:1243: finishing write tid 5 to smithi093153278-415 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 415 v 216 (ObjNum 918 snap 0 seq_num 918) dirty exists 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:1243: left oid 415 (ObjNum 918 snap 0 seq_num 918) 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:1244: finishing write tid 1 to smithi093153278-108 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:1244: finishing write tid 2 to smithi093153278-108 2023-08-27T20:52:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:1254: seq_num 924 ranges {729668=408789,1820303=757311,3274610=255058} 2023-08-27T20:52:51.047 INFO:tasks.rados.rados.0.smithi093.stdout:1254: writing smithi093153278-160 from 729668 to 1138457 tid 1 2023-08-27T20:52:51.054 INFO:tasks.rados.rados.0.smithi093.stdout:1254: writing smithi093153278-160 from 1820303 to 2577614 tid 2 2023-08-27T20:52:51.060 INFO:tasks.rados.rados.0.smithi093.stdout:1254: writing smithi093153278-160 from 3274610 to 3529668 tid 3 2023-08-27T20:52:51.060 INFO:tasks.rados.rados.0.smithi093.stdout:1243: done (11 left) 2023-08-27T20:52:51.060 INFO:tasks.rados.rados.0.smithi093.stdout:1255: write oid 421 current snap is 0 2023-08-27T20:52:51.060 INFO:tasks.rados.rados.0.smithi093.stdout:1255: seq_num 925 ranges {746475=484038,1762432=617322,2785680=529906} 2023-08-27T20:52:51.067 INFO:tasks.rados.rados.0.smithi093.stdout:1255: writing smithi093153278-421 from 746475 to 1230513 tid 1 2023-08-27T20:52:51.073 INFO:tasks.rados.rados.0.smithi093.stdout:1255: writing smithi093153278-421 from 1762432 to 2379754 tid 2 2023-08-27T20:52:51.079 INFO:tasks.rados.rados.0.smithi093.stdout:1255: writing smithi093153278-421 from 2785680 to 3315586 tid 3 2023-08-27T20:52:51.079 INFO:tasks.rados.rados.0.smithi093.stdout:1256: read oid 147 snap -1 2023-08-27T20:52:51.079 INFO:tasks.rados.rados.0.smithi093.stdout:1256: expect (ObjNum 146 snap 0 seq_num 146) 2023-08-27T20:52:51.079 INFO:tasks.rados.rados.0.smithi093.stdout:1257: write oid 218 current snap is 0 2023-08-27T20:52:51.079 INFO:tasks.rados.rados.0.smithi093.stdout:1257: seq_num 926 ranges {763282=559287,1820914=760980,3213103=350179} 2023-08-27T20:52:51.086 INFO:tasks.rados.rados.0.smithi093.stdout:1257: writing smithi093153278-218 from 763282 to 1322569 tid 1 2023-08-27T20:52:51.093 INFO:tasks.rados.rados.0.smithi093.stdout:1257: writing smithi093153278-218 from 1820914 to 2581894 tid 2 2023-08-27T20:52:51.099 INFO:tasks.rados.rados.0.smithi093.stdout:1257: writing smithi093153278-218 from 3213103 to 3563282 tid 3 2023-08-27T20:52:51.099 INFO:tasks.rados.rados.0.smithi093.stdout:1258: read oid 343 snap -1 2023-08-27T20:52:51.099 INFO:tasks.rados.rados.0.smithi093.stdout:1258: expect (ObjNum 577 snap 0 seq_num 577) 2023-08-27T20:52:51.099 INFO:tasks.rados.rados.0.smithi093.stdout:1259: read oid 481 snap -1 2023-08-27T20:52:51.100 INFO:tasks.rados.rados.0.smithi093.stdout:1259: expect (ObjNum 819 snap 0 seq_num 819) 2023-08-27T20:52:51.100 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:52:51.100 INFO:tasks.rados.rados.0.smithi093.stdout:1244: finishing write tid 3 to smithi093153278-108 2023-08-27T20:52:51.100 INFO:tasks.rados.rados.0.smithi093.stdout:1244: finishing write tid 4 to smithi093153278-108 2023-08-27T20:52:51.100 INFO:tasks.rados.rados.0.smithi093.stdout:1246: expect (ObjNum 596 snap 0 seq_num 596) 2023-08-27T20:52:51.182 INFO:tasks.rados.rados.0.smithi093.stdout:1244: finishing write tid 5 to smithi093153278-108 2023-08-27T20:52:51.182 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 108 v 217 (ObjNum 919 snap 0 seq_num 919) dirty exists 2023-08-27T20:52:51.182 INFO:tasks.rados.rados.0.smithi093.stdout:1244: left oid 108 (ObjNum 919 snap 0 seq_num 919) 2023-08-27T20:52:51.182 INFO:tasks.rados.rados.0.smithi093.stdout:1245: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:52:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2763372493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2763372493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:51.268 INFO:tasks.rados.rados.0.smithi093.stdout:1247: finishing write tid 1 to smithi093153278-452 2023-08-27T20:52:51.268 INFO:tasks.rados.rados.0.smithi093.stdout:1247: finishing write tid 2 to smithi093153278-452 2023-08-27T20:52:51.268 INFO:tasks.rados.rados.0.smithi093.stdout:1247: finishing write tid 3 to smithi093153278-452 2023-08-27T20:52:51.268 INFO:tasks.rados.rados.0.smithi093.stdout:1247: finishing write tid 4 to smithi093153278-452 2023-08-27T20:52:51.268 INFO:tasks.rados.rados.0.smithi093.stdout:1247: finishing write tid 5 to smithi093153278-452 2023-08-27T20:52:51.268 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 452 v 214 (ObjNum 920 snap 0 seq_num 920) dirty exists 2023-08-27T20:52:51.268 INFO:tasks.rados.rados.0.smithi093.stdout:1247: left oid 452 (ObjNum 920 snap 0 seq_num 920) 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1248: finishing write tid 1 to smithi093153278-51 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1244: done (15 left) 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1245: done (14 left) 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1246: done (13 left) 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1247: done (12 left) 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1260: write oid 259 current snap is 0 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1248: finishing write tid 2 to smithi093153278-51 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1248: finishing write tid 3 to smithi093153278-51 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1248: finishing write tid 4 to smithi093153278-51 2023-08-27T20:52:51.269 INFO:tasks.rados.rados.0.smithi093.stdout:1260: seq_num 927 ranges {780089=634536,1879396=620991,3240526=339563} 2023-08-27T20:52:51.275 INFO:tasks.rados.rados.0.smithi093.stdout:1260: writing smithi093153278-259 from 780089 to 1414625 tid 1 2023-08-27T20:52:51.281 INFO:tasks.rados.rados.0.smithi093.stdout:1260: writing smithi093153278-259 from 1879396 to 2500387 tid 2 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1260: writing smithi093153278-259 from 3240526 to 3580089 tid 3 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1261: read oid 305 snap -1 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1261: expect (ObjNum 525 snap 0 seq_num 525) 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1262: write oid 478 current snap is 0 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1248: finishing write tid 5 to smithi093153278-51 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 51 v 182 (ObjNum 921 snap 0 seq_num 921) dirty exists 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1248: left oid 51 (ObjNum 921 snap 0 seq_num 921) 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1249: finishing write tid 1 to smithi093153278-383 2023-08-27T20:52:51.288 INFO:tasks.rados.rados.0.smithi093.stdout:1262: seq_num 928 ranges {796896=426138,1654231=764649,2984302=480075} 2023-08-27T20:52:51.294 INFO:tasks.rados.rados.0.smithi093.stdout:1262: writing smithi093153278-478 from 796896 to 1223034 tid 1 2023-08-27T20:52:51.301 INFO:tasks.rados.rados.0.smithi093.stdout:1262: writing smithi093153278-478 from 1654231 to 2418880 tid 2 2023-08-27T20:52:51.307 INFO:tasks.rados.rados.0.smithi093.stdout:1262: writing smithi093153278-478 from 2984302 to 3464377 tid 3 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:1248: done (14 left) 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:1263: delete oid 287 current snap is 0 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:1249: finishing write tid 2 to smithi093153278-383 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:1249: finishing write tid 3 to smithi093153278-383 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:1249: finishing write tid 4 to smithi093153278-383 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 383 v 220 (ObjNum 922 snap 0 seq_num 922) dirty exists 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:1249: left oid 383 (ObjNum 922 snap 0 seq_num 922) 2023-08-27T20:52:51.308 INFO:tasks.rados.rados.0.smithi093.stdout:1250: expect (ObjNum 32 snap 0 seq_num 32) 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:1253: finishing write tid 1 to smithi093153278-381 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:1253: finishing write tid 2 to smithi093153278-381 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:1253: finishing write tid 3 to smithi093153278-381 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:1253: finishing write tid 4 to smithi093153278-381 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:1253: finishing write tid 5 to smithi093153278-381 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 381 v 194 (ObjNum 923 snap 0 seq_num 923) dirty exists 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:1253: left oid 381 (ObjNum 923 snap 0 seq_num 923) 2023-08-27T20:52:51.387 INFO:tasks.rados.rados.0.smithi093.stdout:1249: done (14 left) 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1250: done (13 left) 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1251: done (12 left) 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1252: done (11 left) 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1253: done (10 left) 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1263: done (9 left) 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1264: write oid 341 current snap is 0 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1254: finishing write tid 1 to smithi093153278-160 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1254: finishing write tid 2 to smithi093153278-160 2023-08-27T20:52:51.388 INFO:tasks.rados.rados.0.smithi093.stdout:1264: seq_num 930 ranges {430510=576636,1654842=768318,2922795=502619} 2023-08-27T20:52:51.392 INFO:tasks.rados.rados.0.smithi093.stdout:1264: writing smithi093153278-341 from 430510 to 1007146 tid 1 2023-08-27T20:52:51.399 INFO:tasks.rados.rados.0.smithi093.stdout:1264: writing smithi093153278-341 from 1654842 to 2423160 tid 2 2023-08-27T20:52:51.406 INFO:tasks.rados.rados.0.smithi093.stdout:1264: writing smithi093153278-341 from 2922795 to 3425414 tid 3 2023-08-27T20:52:51.406 INFO:tasks.rados.rados.0.smithi093.stdout:1265: write oid 308 current snap is 0 2023-08-27T20:52:51.406 INFO:tasks.rados.rados.0.smithi093.stdout:1254: finishing write tid 3 to smithi093153278-160 2023-08-27T20:52:51.406 INFO:tasks.rados.rados.0.smithi093.stdout:1254: finishing write tid 4 to smithi093153278-160 2023-08-27T20:52:51.407 INFO:tasks.rados.rados.0.smithi093.stdout:1254: finishing write tid 5 to smithi093153278-160 2023-08-27T20:52:51.407 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 160 v 207 (ObjNum 924 snap 0 seq_num 924) dirty exists 2023-08-27T20:52:51.407 INFO:tasks.rados.rados.0.smithi093.stdout:1254: left oid 160 (ObjNum 924 snap 0 seq_num 924) 2023-08-27T20:52:51.407 INFO:tasks.rados.rados.0.smithi093.stdout:1255: finishing write tid 1 to smithi093153278-421 2023-08-27T20:52:51.407 INFO:tasks.rados.rados.0.smithi093.stdout:1255: finishing write tid 2 to smithi093153278-421 2023-08-27T20:52:51.407 INFO:tasks.rados.rados.0.smithi093.stdout:1255: finishing write tid 3 to smithi093153278-421 2023-08-27T20:52:51.407 INFO:tasks.rados.rados.0.smithi093.stdout:1255: finishing write tid 4 to smithi093153278-421 2023-08-27T20:52:51.408 INFO:tasks.rados.rados.0.smithi093.stdout:1255: finishing write tid 5 to smithi093153278-421 2023-08-27T20:52:51.408 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 421 v 211 (ObjNum 925 snap 0 seq_num 925) dirty exists 2023-08-27T20:52:51.408 INFO:tasks.rados.rados.0.smithi093.stdout:1255: left oid 421 (ObjNum 925 snap 0 seq_num 925) 2023-08-27T20:52:51.408 INFO:tasks.rados.rados.0.smithi093.stdout:1265: seq_num 931 ranges {447317=651885,1713324=628329,3066571=580746} 2023-08-27T20:52:51.411 INFO:tasks.rados.rados.0.smithi093.stdout:1265: writing smithi093153278-308 from 447317 to 1099202 tid 1 2023-08-27T20:52:51.418 INFO:tasks.rados.rados.0.smithi093.stdout:1265: writing smithi093153278-308 from 1713324 to 2341653 tid 2 2023-08-27T20:52:51.425 INFO:tasks.rados.rados.0.smithi093.stdout:1265: writing smithi093153278-308 from 3066571 to 3647317 tid 3 2023-08-27T20:52:51.425 INFO:tasks.rados.rados.0.smithi093.stdout:1254: done (10 left) 2023-08-27T20:52:51.425 INFO:tasks.rados.rados.0.smithi093.stdout:1255: done (9 left) 2023-08-27T20:52:51.425 INFO:tasks.rados.rados.0.smithi093.stdout:1266: read oid 142 snap -1 2023-08-27T20:52:51.426 INFO:tasks.rados.rados.0.smithi093.stdout:1266: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:52:51.426 INFO:tasks.rados.rados.0.smithi093.stdout:1267: delete oid 253 current snap is 0 2023-08-27T20:52:51.426 INFO:tasks.rados.rados.0.smithi093.stdout:1256: expect (ObjNum 146 snap 0 seq_num 146) 2023-08-27T20:52:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2763372493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:51.497 INFO:tasks.rados.rados.0.smithi093.stdout:1257: finishing write tid 1 to smithi093153278-218 2023-08-27T20:52:51.497 INFO:tasks.rados.rados.0.smithi093.stdout:1257: finishing write tid 2 to smithi093153278-218 2023-08-27T20:52:51.497 INFO:tasks.rados.rados.0.smithi093.stdout:1256: done (10 left) 2023-08-27T20:52:51.497 INFO:tasks.rados.rados.0.smithi093.stdout:1267: done (9 left) 2023-08-27T20:52:51.497 INFO:tasks.rados.rados.0.smithi093.stdout:1268: read oid 396 snap -1 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:1268: expect (ObjNum 395 snap 0 seq_num 395) 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:1269: write oid 326 current snap is 0 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:1257: finishing write tid 3 to smithi093153278-218 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:1257: finishing write tid 4 to smithi093153278-218 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:1257: finishing write tid 5 to smithi093153278-218 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 218 v 182 (ObjNum 926 snap 0 seq_num 926) dirty exists 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:1257: left oid 218 (ObjNum 926 snap 0 seq_num 926) 2023-08-27T20:52:51.498 INFO:tasks.rados.rados.0.smithi093.stdout:1258: expect (ObjNum 577 snap 0 seq_num 577) 2023-08-27T20:52:51.654 INFO:tasks.rados.rados.0.smithi093.stdout:1259: expect (ObjNum 819 snap 0 seq_num 819) 2023-08-27T20:52:51.824 INFO:tasks.rados.rados.0.smithi093.stdout:1260: finishing write tid 1 to smithi093153278-259 2023-08-27T20:52:51.825 INFO:tasks.rados.rados.0.smithi093.stdout:1260: finishing write tid 2 to smithi093153278-259 2023-08-27T20:52:51.825 INFO:tasks.rados.rados.0.smithi093.stdout:1260: finishing write tid 3 to smithi093153278-259 2023-08-27T20:52:51.825 INFO:tasks.rados.rados.0.smithi093.stdout:1260: finishing write tid 4 to smithi093153278-259 2023-08-27T20:52:51.825 INFO:tasks.rados.rados.0.smithi093.stdout:1260: finishing write tid 5 to smithi093153278-259 2023-08-27T20:52:51.825 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 259 v 173 (ObjNum 927 snap 0 seq_num 927) dirty exists 2023-08-27T20:52:51.825 INFO:tasks.rados.rados.0.smithi093.stdout:1260: left oid 259 (ObjNum 927 snap 0 seq_num 927) 2023-08-27T20:52:51.825 INFO:tasks.rados.rados.0.smithi093.stdout:1261: expect (ObjNum 525 snap 0 seq_num 525) 2023-08-27T20:52:51.944 INFO:tasks.rados.rados.0.smithi093.stdout:1262: finishing write tid 1 to smithi093153278-478 2023-08-27T20:52:51.944 INFO:tasks.rados.rados.0.smithi093.stdout:1262: finishing write tid 2 to smithi093153278-478 2023-08-27T20:52:51.944 INFO:tasks.rados.rados.0.smithi093.stdout:1269: seq_num 933 ranges {480931=402383,1313935=515645,2488711=452788,3443230=237701} 2023-08-27T20:52:51.948 INFO:tasks.rados.rados.0.smithi093.stdout:1269: writing smithi093153278-326 from 480931 to 883314 tid 1 2023-08-27T20:52:51.954 INFO:tasks.rados.rados.0.smithi093.stdout:1269: writing smithi093153278-326 from 1313935 to 1829580 tid 2 2023-08-27T20:52:51.960 INFO:tasks.rados.rados.0.smithi093.stdout:1269: writing smithi093153278-326 from 2488711 to 2941499 tid 3 2023-08-27T20:52:51.965 INFO:tasks.rados.rados.0.smithi093.stdout:1269: writing smithi093153278-326 from 3443230 to 3680931 tid 4 2023-08-27T20:52:51.965 INFO:tasks.rados.rados.0.smithi093.stdout:1257: done (10 left) 2023-08-27T20:52:51.965 INFO:tasks.rados.rados.0.smithi093.stdout:1258: done (9 left) 2023-08-27T20:52:51.965 INFO:tasks.rados.rados.0.smithi093.stdout:1259: done (8 left) 2023-08-27T20:52:51.965 INFO:tasks.rados.rados.0.smithi093.stdout:1260: done (7 left) 2023-08-27T20:52:51.965 INFO:tasks.rados.rados.0.smithi093.stdout:1261: done (6 left) 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1270: write oid 186 current snap is 0 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1262: finishing write tid 3 to smithi093153278-478 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1262: finishing write tid 4 to smithi093153278-478 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1262: finishing write tid 5 to smithi093153278-478 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 218 (ObjNum 928 snap 0 seq_num 928) dirty exists 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1262: left oid 478 (ObjNum 928 snap 0 seq_num 928) 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1264: finishing write tid 1 to smithi093153278-341 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1264: finishing write tid 2 to smithi093153278-341 2023-08-27T20:52:51.966 INFO:tasks.rados.rados.0.smithi093.stdout:1270: seq_num 934 ranges {497738=477632,1772417=775656,3316134=381604} 2023-08-27T20:52:51.970 INFO:tasks.rados.rados.0.smithi093.stdout:1270: writing smithi093153278-186 from 497738 to 975370 tid 1 2023-08-27T20:52:51.979 INFO:tasks.rados.rados.0.smithi093.stdout:1270: writing smithi093153278-186 from 1772417 to 2548073 tid 2 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1270: writing smithi093153278-186 from 3316134 to 3697738 tid 3 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1262: done (6 left) 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1271: read oid 248 snap -1 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1271: expect (ObjNum 843 snap 0 seq_num 843) 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1272: write oid 305 current snap is 0 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1264: finishing write tid 3 to smithi093153278-341 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1264: finishing write tid 4 to smithi093153278-341 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:1264: finishing write tid 5 to smithi093153278-341 2023-08-27T20:52:51.986 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 341 v 198 (ObjNum 930 snap 0 seq_num 930) dirty exists 2023-08-27T20:52:51.987 INFO:tasks.rados.rados.0.smithi093.stdout:1264: left oid 341 (ObjNum 930 snap 0 seq_num 930) 2023-08-27T20:52:51.987 INFO:tasks.rados.rados.0.smithi093.stdout:1265: finishing write tid 1 to smithi093153278-308 2023-08-27T20:52:51.987 INFO:tasks.rados.rados.0.smithi093.stdout:1265: finishing write tid 2 to smithi093153278-308 2023-08-27T20:52:51.987 INFO:tasks.rados.rados.0.smithi093.stdout:1272: seq_num 935 ranges {514545=552881,1830899=519314,2943557=475332} 2023-08-27T20:52:51.991 INFO:tasks.rados.rados.0.smithi093.stdout:1272: writing smithi093153278-305 from 514545 to 1067426 tid 1 2023-08-27T20:52:51.998 INFO:tasks.rados.rados.0.smithi093.stdout:1272: writing smithi093153278-305 from 1830899 to 2350213 tid 2 2023-08-27T20:52:52.005 INFO:tasks.rados.rados.0.smithi093.stdout:1272: writing smithi093153278-305 from 2943557 to 3418889 tid 3 2023-08-27T20:52:52.005 INFO:tasks.rados.rados.0.smithi093.stdout:1264: done (7 left) 2023-08-27T20:52:52.005 INFO:tasks.rados.rados.0.smithi093.stdout:1273: write oid 276 current snap is 0 2023-08-27T20:52:52.005 INFO:tasks.rados.rados.0.smithi093.stdout:1265: finishing write tid 3 to smithi093153278-308 2023-08-27T20:52:52.005 INFO:tasks.rados.rados.0.smithi093.stdout:1265: finishing write tid 4 to smithi093153278-308 2023-08-27T20:52:52.006 INFO:tasks.rados.rados.0.smithi093.stdout:1265: finishing write tid 5 to smithi093153278-308 2023-08-27T20:52:52.006 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 308 v 191 (ObjNum 931 snap 0 seq_num 931) dirty exists 2023-08-27T20:52:52.006 INFO:tasks.rados.rados.0.smithi093.stdout:1265: left oid 308 (ObjNum 931 snap 0 seq_num 931) 2023-08-27T20:52:52.006 INFO:tasks.rados.rados.0.smithi093.stdout:1266: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:52:52.082 INFO:tasks.rados.rados.0.smithi093.stdout:1268: expect (ObjNum 395 snap 0 seq_num 395) 2023-08-27T20:52:52.165 INFO:tasks.rados.rados.0.smithi093.stdout:1269: finishing write tid 1 to smithi093153278-326 2023-08-27T20:52:52.165 INFO:tasks.rados.rados.0.smithi093.stdout:1269: finishing write tid 2 to smithi093153278-326 2023-08-27T20:52:52.165 INFO:tasks.rados.rados.0.smithi093.stdout:1273: seq_num 936 ranges {531352=744483,2005734=779325,3487333=244019} 2023-08-27T20:52:52.171 INFO:tasks.rados.rados.0.smithi093.stdout:1273: writing smithi093153278-276 from 531352 to 1275835 tid 1 2023-08-27T20:52:52.180 INFO:tasks.rados.rados.0.smithi093.stdout:1273: writing smithi093153278-276 from 2005734 to 2785059 tid 2 2023-08-27T20:52:52.185 INFO:tasks.rados.rados.0.smithi093.stdout:1273: writing smithi093153278-276 from 3487333 to 3731352 tid 3 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1265: done (7 left) 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1266: done (6 left) 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1268: done (5 left) 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1274: read oid 151 snap -1 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1274: expect (ObjNum 815 snap 0 seq_num 815) 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1275: read oid 477 snap -1 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1275: expect (ObjNum 827 snap 0 seq_num 827) 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1276: read oid 109 snap -1 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1276: expect (ObjNum 108 snap 0 seq_num 108) 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1277: read oid 298 snap -1 2023-08-27T20:52:52.186 INFO:tasks.rados.rados.0.smithi093.stdout:1277: expect (ObjNum 297 snap 0 seq_num 297) 2023-08-27T20:52:52.187 INFO:tasks.rados.rados.0.smithi093.stdout:1278: read oid 91 snap -1 2023-08-27T20:52:52.187 INFO:tasks.rados.rados.0.smithi093.stdout:1278: expect (ObjNum 826 snap 0 seq_num 826) 2023-08-27T20:52:52.187 INFO:tasks.rados.rados.0.smithi093.stdout:1279: write oid 383 current snap is 0 2023-08-27T20:52:52.187 INFO:tasks.rados.rados.0.smithi093.stdout:1279: seq_num 937 ranges {548159=419732,1664216=522983,2714756=614229} 2023-08-27T20:52:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:1279: writing smithi093153278-383 from 548159 to 967891 tid 1 2023-08-27T20:52:52.199 INFO:tasks.rados.rados.0.smithi093.stdout:1279: writing smithi093153278-383 from 1664216 to 2187199 tid 2 2023-08-27T20:52:52.206 INFO:tasks.rados.rados.0.smithi093.stdout:1279: writing smithi093153278-383 from 2714756 to 3328985 tid 3 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1280: read oid 113 snap -1 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1280: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1281: write oid 378 current snap is 0 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1269: finishing write tid 3 to smithi093153278-326 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1269: finishing write tid 4 to smithi093153278-326 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1269: finishing write tid 5 to smithi093153278-326 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1269: finishing write tid 6 to smithi093153278-326 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 178 (ObjNum 933 snap 0 seq_num 933) dirty exists 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1269: left oid 326 (ObjNum 933 snap 0 seq_num 933) 2023-08-27T20:52:52.207 INFO:tasks.rados.rados.0.smithi093.stdout:1270: finishing write tid 1 to smithi093153278-186 2023-08-27T20:52:52.208 INFO:tasks.rados.rados.0.smithi093.stdout:1281: seq_num 938 ranges {564966=494981,1606345=782994,3025826=425501} 2023-08-27T20:52:52.212 INFO:tasks.rados.rados.0.smithi093.stdout:1281: writing smithi093153278-378 from 564966 to 1059947 tid 1 2023-08-27T20:52:52.220 INFO:tasks.rados.rados.0.smithi093.stdout:1281: writing smithi093153278-378 from 1606345 to 2389339 tid 2 2023-08-27T20:52:52.227 INFO:tasks.rados.rados.0.smithi093.stdout:1281: writing smithi093153278-378 from 3025826 to 3451327 tid 3 2023-08-27T20:52:52.227 INFO:tasks.rados.rados.0.smithi093.stdout:1269: done (12 left) 2023-08-27T20:52:52.227 INFO:tasks.rados.rados.0.smithi093.stdout:1282: write oid 154 current snap is 0 2023-08-27T20:52:52.227 INFO:tasks.rados.rados.0.smithi093.stdout:1270: finishing write tid 2 to smithi093153278-186 2023-08-27T20:52:52.227 INFO:tasks.rados.rados.0.smithi093.stdout:1270: finishing write tid 3 to smithi093153278-186 2023-08-27T20:52:52.227 INFO:tasks.rados.rados.0.smithi093.stdout:1282: seq_num 939 ranges {581773=570230,1664827=526652,2653249=636773} 2023-08-27T20:52:52.233 INFO:tasks.rados.rados.0.smithi093.stdout:1282: writing smithi093153278-154 from 581773 to 1152003 tid 1 2023-08-27T20:52:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:51 smithi093 ceph-mon[111335]: pgmap v80: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 10 MiB/s wr, 99 op/s 2023-08-27T20:52:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:51 smithi093 ceph-mon[118902]: pgmap v80: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 10 MiB/s wr, 99 op/s 2023-08-27T20:52:52.239 INFO:tasks.rados.rados.0.smithi093.stdout:1282: writing smithi093153278-154 from 1664827 to 2191479 tid 2 2023-08-27T20:52:52.245 INFO:tasks.rados.rados.0.smithi093.stdout:1282: writing smithi093153278-154 from 2653249 to 3290022 tid 3 2023-08-27T20:52:52.245 INFO:tasks.rados.rados.0.smithi093.stdout:1283: delete oid 439 current snap is 0 2023-08-27T20:52:52.245 INFO:tasks.rados.rados.0.smithi093.stdout:1270: finishing write tid 4 to smithi093153278-186 2023-08-27T20:52:52.245 INFO:tasks.rados.rados.0.smithi093.stdout:1270: finishing write tid 5 to smithi093153278-186 2023-08-27T20:52:52.246 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 186 v 224 (ObjNum 934 snap 0 seq_num 934) dirty exists 2023-08-27T20:52:52.246 INFO:tasks.rados.rados.0.smithi093.stdout:1270: left oid 186 (ObjNum 934 snap 0 seq_num 934) 2023-08-27T20:52:52.246 INFO:tasks.rados.rados.0.smithi093.stdout:1271: expect (ObjNum 843 snap 0 seq_num 843) 2023-08-27T20:52:52.343 INFO:tasks.rados.rados.0.smithi093.stdout:1272: finishing write tid 1 to smithi093153278-305 2023-08-27T20:52:52.344 INFO:tasks.rados.rados.0.smithi093.stdout:1272: finishing write tid 2 to smithi093153278-305 2023-08-27T20:52:52.344 INFO:tasks.rados.rados.0.smithi093.stdout:1272: finishing write tid 3 to smithi093153278-305 2023-08-27T20:52:52.344 INFO:tasks.rados.rados.0.smithi093.stdout:1272: finishing write tid 4 to smithi093153278-305 2023-08-27T20:52:52.344 INFO:tasks.rados.rados.0.smithi093.stdout:1272: finishing write tid 5 to smithi093153278-305 2023-08-27T20:52:52.344 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 305 v 182 (ObjNum 935 snap 0 seq_num 935) dirty exists 2023-08-27T20:52:52.345 INFO:tasks.rados.rados.0.smithi093.stdout:1272: left oid 305 (ObjNum 935 snap 0 seq_num 935) 2023-08-27T20:52:52.345 INFO:tasks.rados.rados.0.smithi093.stdout:1273: finishing write tid 1 to smithi093153278-276 2023-08-27T20:52:52.345 INFO:tasks.rados.rados.0.smithi093.stdout:1273: finishing write tid 2 to smithi093153278-276 2023-08-27T20:52:52.345 INFO:tasks.rados.rados.0.smithi093.stdout:1270: done (13 left) 2023-08-27T20:52:52.345 INFO:tasks.rados.rados.0.smithi093.stdout:1271: done (12 left) 2023-08-27T20:52:52.345 INFO:tasks.rados.rados.0.smithi093.stdout:1272: done (11 left) 2023-08-27T20:52:52.345 INFO:tasks.rados.rados.0.smithi093.stdout:1283: done (10 left) 2023-08-27T20:52:52.346 INFO:tasks.rados.rados.0.smithi093.stdout:1284: write oid 187 current snap is 0 2023-08-27T20:52:52.346 INFO:tasks.rados.rados.0.smithi093.stdout:1273: finishing write tid 3 to smithi093153278-276 2023-08-27T20:52:52.346 INFO:tasks.rados.rados.0.smithi093.stdout:1273: finishing write tid 4 to smithi093153278-276 2023-08-27T20:52:52.346 INFO:tasks.rados.rados.0.smithi093.stdout:1273: finishing write tid 5 to smithi093153278-276 2023-08-27T20:52:52.346 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 276 v 195 (ObjNum 936 snap 0 seq_num 936) dirty exists 2023-08-27T20:52:52.346 INFO:tasks.rados.rados.0.smithi093.stdout:1273: left oid 276 (ObjNum 936 snap 0 seq_num 936) 2023-08-27T20:52:52.347 INFO:tasks.rados.rados.0.smithi093.stdout:1284: seq_num 941 ranges {615387=720728,1781791=530321,3108095=707292} 2023-08-27T20:52:52.350 INFO:tasks.rados.rados.0.smithi093.stdout:1284: writing smithi093153278-187 from 615387 to 1336115 tid 1 2023-08-27T20:52:52.356 INFO:tasks.rados.rados.0.smithi093.stdout:1284: writing smithi093153278-187 from 1781791 to 2312112 tid 2 2023-08-27T20:52:52.364 INFO:tasks.rados.rados.0.smithi093.stdout:1284: writing smithi093153278-187 from 3108095 to 3815387 tid 3 2023-08-27T20:52:52.364 INFO:tasks.rados.rados.0.smithi093.stdout:1273: done (10 left) 2023-08-27T20:52:52.364 INFO:tasks.rados.rados.0.smithi093.stdout:1285: write oid 441 current snap is 0 2023-08-27T20:52:52.364 INFO:tasks.rados.rados.0.smithi093.stdout:1275: expect (ObjNum 827 snap 0 seq_num 827) 2023-08-27T20:52:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:51 smithi136 ceph-mon[113531]: pgmap v80: 89 pgs: 89 active+clean; 1.3 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 65 MiB/s rd, 10 MiB/s wr, 99 op/s 2023-08-27T20:52:52.591 INFO:tasks.rados.rados.0.smithi093.stdout:1276: expect (ObjNum 108 snap 0 seq_num 108) 2023-08-27T20:52:52.709 INFO:tasks.rados.rados.0.smithi093.stdout:1278: expect (ObjNum 826 snap 0 seq_num 826) 2023-08-27T20:52:52.890 INFO:tasks.rados.rados.0.smithi093.stdout:1279: finishing write tid 1 to smithi093153278-383 2023-08-27T20:52:52.890 INFO:tasks.rados.rados.0.smithi093.stdout:1279: finishing write tid 2 to smithi093153278-383 2023-08-27T20:52:52.890 INFO:tasks.rados.rados.0.smithi093.stdout:1274: expect (ObjNum 815 snap 0 seq_num 815) 2023-08-27T20:52:53.068 INFO:tasks.rados.rados.0.smithi093.stdout:1277: expect (ObjNum 297 snap 0 seq_num 297) 2023-08-27T20:52:53.170 INFO:tasks.rados.rados.0.smithi093.stdout:1279: finishing write tid 3 to smithi093153278-383 2023-08-27T20:52:53.172 INFO:tasks.rados.rados.0.smithi093.stdout:1279: finishing write tid 4 to smithi093153278-383 2023-08-27T20:52:53.172 INFO:tasks.rados.rados.0.smithi093.stdout:1280: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:52:53.300 INFO:tasks.rados.rados.0.smithi093.stdout:1279: finishing write tid 5 to smithi093153278-383 2023-08-27T20:52:53.300 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 383 v 228 (ObjNum 937 snap 0 seq_num 937) dirty exists 2023-08-27T20:52:53.300 INFO:tasks.rados.rados.0.smithi093.stdout:1279: left oid 383 (ObjNum 937 snap 0 seq_num 937) 2023-08-27T20:52:53.300 INFO:tasks.rados.rados.0.smithi093.stdout:1281: finishing write tid 1 to smithi093153278-378 2023-08-27T20:52:53.300 INFO:tasks.rados.rados.0.smithi093.stdout:1281: finishing write tid 2 to smithi093153278-378 2023-08-27T20:52:53.300 INFO:tasks.rados.rados.0.smithi093.stdout:1281: finishing write tid 3 to smithi093153278-378 2023-08-27T20:52:53.300 INFO:tasks.rados.rados.0.smithi093.stdout:1281: finishing write tid 4 to smithi093153278-378 2023-08-27T20:52:53.301 INFO:tasks.rados.rados.0.smithi093.stdout:1281: finishing write tid 5 to smithi093153278-378 2023-08-27T20:52:53.301 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 378 v 195 (ObjNum 938 snap 0 seq_num 938) dirty exists 2023-08-27T20:52:53.301 INFO:tasks.rados.rados.0.smithi093.stdout:1281: left oid 378 (ObjNum 938 snap 0 seq_num 938) 2023-08-27T20:52:53.301 INFO:tasks.rados.rados.0.smithi093.stdout:1282: finishing write tid 1 to smithi093153278-154 2023-08-27T20:52:53.301 INFO:tasks.rados.rados.0.smithi093.stdout:1285: seq_num 942 ranges {632194=795977,2123920=790332,3419165=413029} 2023-08-27T20:52:53.309 INFO:tasks.rados.rados.0.smithi093.stdout:1285: writing smithi093153278-441 from 632194 to 1428171 tid 1 2023-08-27T20:52:53.317 INFO:tasks.rados.rados.0.smithi093.stdout:1285: writing smithi093153278-441 from 2123920 to 2914252 tid 2 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1285: writing smithi093153278-441 from 3419165 to 3832194 tid 3 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1274: done (10 left) 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1275: done (9 left) 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1276: done (8 left) 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1277: done (7 left) 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1278: done (6 left) 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1279: done (5 left) 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1280: done (4 left) 2023-08-27T20:52:53.323 INFO:tasks.rados.rados.0.smithi093.stdout:1281: done (3 left) 2023-08-27T20:52:53.324 INFO:tasks.rados.rados.0.smithi093.stdout:1286: write oid 84 current snap is 0 2023-08-27T20:52:53.324 INFO:tasks.rados.rados.0.smithi093.stdout:1282: finishing write tid 2 to smithi093153278-154 2023-08-27T20:52:53.324 INFO:tasks.rados.rados.0.smithi093.stdout:1286: seq_num 943 ranges {649001=587579,1898755=533990,3162941=686060} 2023-08-27T20:52:53.329 INFO:tasks.rados.rados.0.smithi093.stdout:1286: writing smithi093153278-84 from 649001 to 1236580 tid 1 2023-08-27T20:52:53.335 INFO:tasks.rados.rados.0.smithi093.stdout:1286: writing smithi093153278-84 from 1898755 to 2432745 tid 2 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:1286: writing smithi093153278-84 from 3162941 to 3849001 tid 3 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:1287: write oid 72 current snap is 0 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:1282: finishing write tid 3 to smithi093153278-154 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:1282: finishing write tid 4 to smithi093153278-154 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:1282: finishing write tid 5 to smithi093153278-154 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 154 v 202 (ObjNum 939 snap 0 seq_num 939) dirty exists 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:1282: left oid 154 (ObjNum 939 snap 0 seq_num 939) 2023-08-27T20:52:53.344 INFO:tasks.rados.rados.0.smithi093.stdout:1287: seq_num 944 ranges {665808=662828,1957237=677648,3074011=609486} 2023-08-27T20:52:53.351 INFO:tasks.rados.rados.0.smithi093.stdout:1287: writing smithi093153278-72 from 665808 to 1328636 tid 1 2023-08-27T20:52:53.358 INFO:tasks.rados.rados.0.smithi093.stdout:1287: writing smithi093153278-72 from 1957237 to 2634885 tid 2 2023-08-27T20:52:53.364 INFO:tasks.rados.rados.0.smithi093.stdout:1287: writing smithi093153278-72 from 3074011 to 3683497 tid 3 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1282: done (4 left) 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1288: read oid 388 snap -1 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1288: expect (ObjNum 570 snap 0 seq_num 570) 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1289: read oid 183 snap -1 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1289: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1290: read oid 331 snap -1 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1290: expect (ObjNum 757 snap 0 seq_num 757) 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1291: write oid 257 current snap is 0 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1284: finishing write tid 1 to smithi093153278-187 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1284: finishing write tid 2 to smithi093153278-187 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1284: finishing write tid 3 to smithi093153278-187 2023-08-27T20:52:53.365 INFO:tasks.rados.rados.0.smithi093.stdout:1284: finishing write tid 4 to smithi093153278-187 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1284: finishing write tid 5 to smithi093153278-187 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 187 v 171 (ObjNum 941 snap 0 seq_num 941) dirty exists 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1284: left oid 187 (ObjNum 941 snap 0 seq_num 941) 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1285: finishing write tid 1 to smithi093153278-441 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1285: finishing write tid 2 to smithi093153278-441 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1285: finishing write tid 3 to smithi093153278-441 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1285: finishing write tid 4 to smithi093153278-441 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1285: finishing write tid 5 to smithi093153278-441 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 213 (ObjNum 942 snap 0 seq_num 942) dirty exists 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1285: left oid 441 (ObjNum 942 snap 0 seq_num 942) 2023-08-27T20:52:53.366 INFO:tasks.rados.rados.0.smithi093.stdout:1291: seq_num 945 ranges {682615=738077,2015719=537659,3217787=420758} 2023-08-27T20:52:53.372 INFO:tasks.rados.rados.0.smithi093.stdout:1291: writing smithi093153278-257 from 682615 to 1420692 tid 1 2023-08-27T20:52:53.378 INFO:tasks.rados.rados.0.smithi093.stdout:1291: writing smithi093153278-257 from 2015719 to 2553378 tid 2 2023-08-27T20:52:53.384 INFO:tasks.rados.rados.0.smithi093.stdout:1291: writing smithi093153278-257 from 3217787 to 3638545 tid 3 2023-08-27T20:52:53.384 INFO:tasks.rados.rados.0.smithi093.stdout:1284: done (7 left) 2023-08-27T20:52:53.385 INFO:tasks.rados.rados.0.smithi093.stdout:1285: done (6 left) 2023-08-27T20:52:53.385 INFO:tasks.rados.rados.0.smithi093.stdout:1292: write oid 230 current snap is 0 2023-08-27T20:52:53.385 INFO:tasks.rados.rados.0.smithi093.stdout:1286: finishing write tid 1 to smithi093153278-84 2023-08-27T20:52:53.385 INFO:tasks.rados.rados.0.smithi093.stdout:1292: seq_num 946 ranges {699422=413326,1557848=681317,3012504=748383} 2023-08-27T20:52:53.390 INFO:tasks.rados.rados.0.smithi093.stdout:1292: writing smithi093153278-230 from 699422 to 1112748 tid 1 2023-08-27T20:52:53.396 INFO:tasks.rados.rados.0.smithi093.stdout:1292: writing smithi093153278-230 from 1557848 to 2239165 tid 2 2023-08-27T20:52:53.405 INFO:tasks.rados.rados.0.smithi093.stdout:1292: writing smithi093153278-230 from 3012504 to 3760887 tid 3 2023-08-27T20:52:53.405 INFO:tasks.rados.rados.0.smithi093.stdout:1293: write oid 105 current snap is 0 2023-08-27T20:52:53.405 INFO:tasks.rados.rados.0.smithi093.stdout:1286: finishing write tid 2 to smithi093153278-84 2023-08-27T20:52:53.405 INFO:tasks.rados.rados.0.smithi093.stdout:1293: seq_num 947 ranges {716229=488575,1616330=541328,2756280=559655} 2023-08-27T20:52:53.411 INFO:tasks.rados.rados.0.smithi093.stdout:1293: writing smithi093153278-105 from 716229 to 1204804 tid 1 2023-08-27T20:52:53.417 INFO:tasks.rados.rados.0.smithi093.stdout:1293: writing smithi093153278-105 from 1616330 to 2157658 tid 2 2023-08-27T20:52:53.424 INFO:tasks.rados.rados.0.smithi093.stdout:1293: writing smithi093153278-105 from 2756280 to 3315935 tid 3 2023-08-27T20:52:53.424 INFO:tasks.rados.rados.0.smithi093.stdout:1294: read oid 171 snap -1 2023-08-27T20:52:53.424 INFO:tasks.rados.rados.0.smithi093.stdout:1294: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:52:53.424 INFO:tasks.rados.rados.0.smithi093.stdout:1295: read oid 385 snap -1 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:1295: expect (ObjNum 384 snap 0 seq_num 384) 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:1296: write oid 246 current snap is 0 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:1286: finishing write tid 3 to smithi093153278-84 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:1286: finishing write tid 4 to smithi093153278-84 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:1286: finishing write tid 5 to smithi093153278-84 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 84 v 222 (ObjNum 943 snap 0 seq_num 943) dirty exists 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:1286: left oid 84 (ObjNum 943 snap 0 seq_num 943) 2023-08-27T20:52:53.425 INFO:tasks.rados.rados.0.smithi093.stdout:1296: seq_num 948 ranges {733036=563824,2074812=684986,3183703=749333} 2023-08-27T20:52:53.430 INFO:tasks.rados.rados.0.smithi093.stdout:1296: writing smithi093153278-246 from 733036 to 1296860 tid 1 2023-08-27T20:52:53.438 INFO:tasks.rados.rados.0.smithi093.stdout:1296: writing smithi093153278-246 from 2074812 to 2759798 tid 2 2023-08-27T20:52:53.445 INFO:tasks.rados.rados.0.smithi093.stdout:1296: writing smithi093153278-246 from 3183703 to 3933036 tid 3 2023-08-27T20:52:53.445 INFO:tasks.rados.rados.0.smithi093.stdout:1286: done (10 left) 2023-08-27T20:52:53.446 INFO:tasks.rados.rados.0.smithi093.stdout:1297: write oid 425 current snap is 0 2023-08-27T20:52:53.446 INFO:tasks.rados.rados.0.smithi093.stdout:1287: finishing write tid 1 to smithi093153278-72 2023-08-27T20:52:53.446 INFO:tasks.rados.rados.0.smithi093.stdout:1287: finishing write tid 2 to smithi093153278-72 2023-08-27T20:52:53.446 INFO:tasks.rados.rados.0.smithi093.stdout:1297: seq_num 949 ranges {749843=639073,2133294=544997,3211126=582199} 2023-08-27T20:52:53.452 INFO:tasks.rados.rados.0.smithi093.stdout:1297: writing smithi093153278-425 from 749843 to 1388916 tid 1 2023-08-27T20:52:53.460 INFO:tasks.rados.rados.0.smithi093.stdout:1297: writing smithi093153278-425 from 2133294 to 2678291 tid 2 2023-08-27T20:52:53.466 INFO:tasks.rados.rados.0.smithi093.stdout:1297: writing smithi093153278-425 from 3211126 to 3793325 tid 3 2023-08-27T20:52:53.466 INFO:tasks.rados.rados.0.smithi093.stdout:1298: read oid 62 snap -1 2023-08-27T20:52:53.466 INFO:tasks.rados.rados.0.smithi093.stdout:1298: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:52:53.466 INFO:tasks.rados.rados.0.smithi093.stdout:1299: delete oid 300 current snap is 0 2023-08-27T20:52:53.466 INFO:tasks.rados.rados.0.smithi093.stdout:1290: expect (ObjNum 757 snap 0 seq_num 757) 2023-08-27T20:52:53.589 INFO:tasks.rados.rados.0.smithi093.stdout:1289: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:52:53.684 INFO:tasks.rados.rados.0.smithi093.stdout:1291: finishing write tid 1 to smithi093153278-257 2023-08-27T20:52:53.684 INFO:tasks.rados.rados.0.smithi093.stdout:1287: finishing write tid 3 to smithi093153278-72 2023-08-27T20:52:53.684 INFO:tasks.rados.rados.0.smithi093.stdout:1287: finishing write tid 4 to smithi093153278-72 2023-08-27T20:52:53.684 INFO:tasks.rados.rados.0.smithi093.stdout:1287: finishing write tid 5 to smithi093153278-72 2023-08-27T20:52:53.684 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 72 v 226 (ObjNum 944 snap 0 seq_num 944) dirty exists 2023-08-27T20:52:53.684 INFO:tasks.rados.rados.0.smithi093.stdout:1287: left oid 72 (ObjNum 944 snap 0 seq_num 944) 2023-08-27T20:52:53.684 INFO:tasks.rados.rados.0.smithi093.stdout:1288: expect (ObjNum 570 snap 0 seq_num 570) 2023-08-27T20:52:53.829 INFO:tasks.rados.rados.0.smithi093.stdout:1291: finishing write tid 2 to smithi093153278-257 2023-08-27T20:52:53.829 INFO:tasks.rados.rados.0.smithi093.stdout:1291: finishing write tid 3 to smithi093153278-257 2023-08-27T20:52:53.829 INFO:tasks.rados.rados.0.smithi093.stdout:1291: finishing write tid 4 to smithi093153278-257 2023-08-27T20:52:53.829 INFO:tasks.rados.rados.0.smithi093.stdout:1291: finishing write tid 5 to smithi093153278-257 2023-08-27T20:52:53.829 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 257 v 260 (ObjNum 945 snap 0 seq_num 945) dirty exists 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1291: left oid 257 (ObjNum 945 snap 0 seq_num 945) 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1292: finishing write tid 1 to smithi093153278-230 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1292: finishing write tid 2 to smithi093153278-230 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1287: done (12 left) 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1288: done (11 left) 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1289: done (10 left) 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1290: done (9 left) 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1291: done (8 left) 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1299: done (7 left) 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1300: delete oid 429 current snap is 0 2023-08-27T20:52:53.830 INFO:tasks.rados.rados.0.smithi093.stdout:1292: finishing write tid 3 to smithi093153278-230 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1292: finishing write tid 4 to smithi093153278-230 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1292: finishing write tid 5 to smithi093153278-230 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 206 (ObjNum 946 snap 0 seq_num 946) dirty exists 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1292: left oid 230 (ObjNum 946 snap 0 seq_num 946) 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1293: finishing write tid 1 to smithi093153278-105 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1293: finishing write tid 2 to smithi093153278-105 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1293: finishing write tid 3 to smithi093153278-105 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1293: finishing write tid 4 to smithi093153278-105 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1293: finishing write tid 5 to smithi093153278-105 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 105 v 175 (ObjNum 947 snap 0 seq_num 947) dirty exists 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1293: left oid 105 (ObjNum 947 snap 0 seq_num 947) 2023-08-27T20:52:53.831 INFO:tasks.rados.rados.0.smithi093.stdout:1294: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:52:53.917 INFO:tasks.rados.rados.0.smithi093.stdout:1296: finishing write tid 1 to smithi093153278-246 2023-08-27T20:52:53.918 INFO:tasks.rados.rados.0.smithi093.stdout:1295: expect (ObjNum 384 snap 0 seq_num 384) 2023-08-27T20:52:53.993 INFO:tasks.rados.rados.0.smithi093.stdout:1296: finishing write tid 2 to smithi093153278-246 2023-08-27T20:52:53.993 INFO:tasks.rados.rados.0.smithi093.stdout:1296: finishing write tid 3 to smithi093153278-246 2023-08-27T20:52:53.993 INFO:tasks.rados.rados.0.smithi093.stdout:1296: finishing write tid 4 to smithi093153278-246 2023-08-27T20:52:53.993 INFO:tasks.rados.rados.0.smithi093.stdout:1296: finishing write tid 5 to smithi093153278-246 2023-08-27T20:52:53.993 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 246 v 186 (ObjNum 948 snap 0 seq_num 948) dirty exists 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1296: left oid 246 (ObjNum 948 snap 0 seq_num 948) 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1297: finishing write tid 1 to smithi093153278-425 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1297: finishing write tid 2 to smithi093153278-425 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1292: done (7 left) 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1293: done (6 left) 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1294: done (5 left) 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1295: done (4 left) 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1296: done (3 left) 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1300: done (2 left) 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1301: write oid 205 current snap is 0 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1297: finishing write tid 3 to smithi093153278-425 2023-08-27T20:52:53.994 INFO:tasks.rados.rados.0.smithi093.stdout:1297: finishing write tid 4 to smithi093153278-425 2023-08-27T20:52:53.995 INFO:tasks.rados.rados.0.smithi093.stdout:1297: finishing write tid 5 to smithi093153278-425 2023-08-27T20:52:53.995 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 425 v 210 (ObjNum 949 snap 0 seq_num 949) dirty exists 2023-08-27T20:52:53.995 INFO:tasks.rados.rados.0.smithi093.stdout:1297: left oid 425 (ObjNum 949 snap 0 seq_num 949) 2023-08-27T20:52:53.995 INFO:tasks.rados.rados.0.smithi093.stdout:1298: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:52:54.199 INFO:tasks.rados.rados.0.smithi093.stdout:1301: seq_num 952 ranges {400264=581173,1508740=491524} 2023-08-27T20:52:54.204 INFO:tasks.rados.rados.0.smithi093.stdout:1301: writing smithi093153278-205 from 400264 to 981437 tid 1 2023-08-27T20:52:54.210 INFO:tasks.rados.rados.0.smithi093.stdout:1301: writing smithi093153278-205 from 1508740 to 2000264 tid 2 2023-08-27T20:52:54.210 INFO:tasks.rados.rados.0.smithi093.stdout:1297: done (2 left) 2023-08-27T20:52:54.210 INFO:tasks.rados.rados.0.smithi093.stdout:1298: done (1 left) 2023-08-27T20:52:54.211 INFO:tasks.rados.rados.0.smithi093.stdout:1302: write oid 499 current snap is 0 2023-08-27T20:52:54.211 INFO:tasks.rados.rados.0.smithi093.stdout:1301: finishing write tid 1 to smithi093153278-205 2023-08-27T20:52:54.211 INFO:tasks.rados.rados.0.smithi093.stdout:1302: seq_num 953 ranges {417071=656422,1567222=449849} 2023-08-27T20:52:54.216 INFO:tasks.rados.rados.0.smithi093.stdout:1302: writing smithi093153278-499 from 417071 to 1073493 tid 1 2023-08-27T20:52:54.221 INFO:tasks.rados.rados.0.smithi093.stdout:1302: writing smithi093153278-499 from 1567222 to 2017071 tid 2 2023-08-27T20:52:54.221 INFO:tasks.rados.rados.0.smithi093.stdout:1303: write oid 196 current snap is 0 2023-08-27T20:52:54.221 INFO:tasks.rados.rados.0.smithi093.stdout:1303: seq_num 954 ranges {433878=731671,1909351=124527} 2023-08-27T20:52:54.227 INFO:tasks.rados.rados.0.smithi093.stdout:1303: writing smithi093153278-196 from 433878 to 1165549 tid 1 2023-08-27T20:52:54.232 INFO:tasks.rados.rados.0.smithi093.stdout:1303: writing smithi093153278-196 from 1909351 to 2033878 tid 2 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1304: read oid 409 snap -1 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1304: expect (ObjNum 794 snap 0 seq_num 794) 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1305: read oid 430 snap -1 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1305: expect (ObjNum 766 snap 0 seq_num 766) 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1306: write oid 23 current snap is 0 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1301: finishing write tid 2 to smithi093153278-205 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1301: finishing write tid 3 to smithi093153278-205 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1301: finishing write tid 4 to smithi093153278-205 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 205 v 133 (ObjNum 952 snap 0 seq_num 952) dirty exists 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1301: left oid 205 (ObjNum 952 snap 0 seq_num 952) 2023-08-27T20:52:54.233 INFO:tasks.rados.rados.0.smithi093.stdout:1302: finishing write tid 1 to smithi093153278-499 2023-08-27T20:52:54.234 INFO:tasks.rados.rados.0.smithi093.stdout:1306: seq_num 955 ranges {450685=406920,1567833=482852} 2023-08-27T20:52:54.237 INFO:tasks.rados.rados.0.smithi093.stdout:1306: writing smithi093153278-23 from 450685 to 857605 tid 1 2023-08-27T20:52:54.244 INFO:tasks.rados.rados.0.smithi093.stdout:1306: writing smithi093153278-23 from 1567833 to 2050685 tid 2 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1301: done (5 left) 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1307: read oid 226 snap -1 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1307: expect (ObjNum 539 snap 0 seq_num 539) 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1308: write oid 325 current snap is 0 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1302: finishing write tid 2 to smithi093153278-499 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1302: finishing write tid 3 to smithi093153278-499 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1302: finishing write tid 4 to smithi093153278-499 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 499 v 163 (ObjNum 953 snap 0 seq_num 953) dirty exists 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1302: left oid 499 (ObjNum 953 snap 0 seq_num 953) 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1303: finishing write tid 1 to smithi093153278-196 2023-08-27T20:52:54.245 INFO:tasks.rados.rados.0.smithi093.stdout:1308: seq_num 956 ranges {467492=482169,1626315=441177} 2023-08-27T20:52:54.249 INFO:tasks.rados.rados.0.smithi093.stdout:1308: writing smithi093153278-325 from 467492 to 949661 tid 1 2023-08-27T20:52:54.256 INFO:tasks.rados.rados.0.smithi093.stdout:1308: writing smithi093153278-325 from 1626315 to 2067492 tid 2 2023-08-27T20:52:54.256 INFO:tasks.rados.rados.0.smithi093.stdout:1302: done (6 left) 2023-08-27T20:52:54.256 INFO:tasks.rados.rados.0.smithi093.stdout:1309: read oid 65 snap -1 2023-08-27T20:52:54.256 INFO:tasks.rados.rados.0.smithi093.stdout:1309: expect (ObjNum 868 snap 0 seq_num 868) 2023-08-27T20:52:54.256 INFO:tasks.rados.rados.0.smithi093.stdout:1310: write oid 173 current snap is 0 2023-08-27T20:52:54.256 INFO:tasks.rados.rados.0.smithi093.stdout:1305: expect (ObjNum 766 snap 0 seq_num 766) 2023-08-27T20:52:54.380 INFO:tasks.rados.rados.0.smithi093.stdout:1303: finishing write tid 2 to smithi093153278-196 2023-08-27T20:52:54.380 INFO:tasks.rados.rados.0.smithi093.stdout:1303: finishing write tid 3 to smithi093153278-196 2023-08-27T20:52:54.380 INFO:tasks.rados.rados.0.smithi093.stdout:1310: seq_num 957 ranges {484299=557418,1684797=399502} 2023-08-27T20:52:54.385 INFO:tasks.rados.rados.0.smithi093.stdout:1310: writing smithi093153278-173 from 484299 to 1041717 tid 1 2023-08-27T20:52:54.391 INFO:tasks.rados.rados.0.smithi093.stdout:1310: writing smithi093153278-173 from 1684797 to 2084299 tid 2 2023-08-27T20:52:54.391 INFO:tasks.rados.rados.0.smithi093.stdout:1305: done (7 left) 2023-08-27T20:52:54.391 INFO:tasks.rados.rados.0.smithi093.stdout:1311: write oid 314 current snap is 0 2023-08-27T20:52:54.391 INFO:tasks.rados.rados.0.smithi093.stdout:1303: finishing write tid 4 to smithi093153278-196 2023-08-27T20:52:54.391 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 196 v 198 (ObjNum 954 snap 0 seq_num 954) dirty exists 2023-08-27T20:52:54.391 INFO:tasks.rados.rados.0.smithi093.stdout:1303: left oid 196 (ObjNum 954 snap 0 seq_num 954) 2023-08-27T20:52:54.391 INFO:tasks.rados.rados.0.smithi093.stdout:1306: finishing write tid 1 to smithi093153278-23 2023-08-27T20:52:54.392 INFO:tasks.rados.rados.0.smithi093.stdout:1311: seq_num 958 ranges {501106=749020,1743279=357827} 2023-08-27T20:52:54.398 INFO:tasks.rados.rados.0.smithi093.stdout:1311: writing smithi093153278-314 from 501106 to 1250126 tid 1 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1311: writing smithi093153278-314 from 1743279 to 2101106 tid 2 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1303: done (7 left) 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1312: read oid 169 snap -1 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1312: expect (ObjNum 582 snap 0 seq_num 582) 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1313: read oid 277 snap -1 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1313: expect (ObjNum 276 snap 0 seq_num 276) 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1314: delete oid 196 current snap is 0 2023-08-27T20:52:54.403 INFO:tasks.rados.rados.0.smithi093.stdout:1304: expect (ObjNum 794 snap 0 seq_num 794) 2023-08-27T20:52:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:54 smithi136 ceph-mon[113531]: pgmap v81: 89 pgs: 89 active+clean; 1.2 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 MiB/s rd, 15 MiB/s wr, 146 op/s 2023-08-27T20:52:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/592434943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:54 smithi093 ceph-mon[111335]: pgmap v81: 89 pgs: 89 active+clean; 1.2 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 MiB/s rd, 15 MiB/s wr, 146 op/s 2023-08-27T20:52:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/592434943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:54 smithi093 ceph-mon[118902]: pgmap v81: 89 pgs: 89 active+clean; 1.2 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 96 MiB/s rd, 15 MiB/s wr, 146 op/s 2023-08-27T20:52:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/592434943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:54.537 INFO:tasks.rados.rados.0.smithi093.stdout:1306: finishing write tid 2 to smithi093153278-23 2023-08-27T20:52:54.537 INFO:tasks.rados.rados.0.smithi093.stdout:1306: finishing write tid 3 to smithi093153278-23 2023-08-27T20:52:54.537 INFO:tasks.rados.rados.0.smithi093.stdout:1306: finishing write tid 4 to smithi093153278-23 2023-08-27T20:52:54.538 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 23 v 231 (ObjNum 955 snap 0 seq_num 955) dirty exists 2023-08-27T20:52:54.538 INFO:tasks.rados.rados.0.smithi093.stdout:1306: left oid 23 (ObjNum 955 snap 0 seq_num 955) 2023-08-27T20:52:54.538 INFO:tasks.rados.rados.0.smithi093.stdout:1307: expect (ObjNum 539 snap 0 seq_num 539) 2023-08-27T20:52:54.658 INFO:tasks.rados.rados.0.smithi093.stdout:1308: finishing write tid 1 to smithi093153278-325 2023-08-27T20:52:54.659 INFO:tasks.rados.rados.0.smithi093.stdout:1308: finishing write tid 2 to smithi093153278-325 2023-08-27T20:52:54.659 INFO:tasks.rados.rados.0.smithi093.stdout:1308: finishing write tid 3 to smithi093153278-325 2023-08-27T20:52:54.659 INFO:tasks.rados.rados.0.smithi093.stdout:1308: finishing write tid 4 to smithi093153278-325 2023-08-27T20:52:54.659 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 325 v 220 (ObjNum 956 snap 0 seq_num 956) dirty exists 2023-08-27T20:52:54.659 INFO:tasks.rados.rados.0.smithi093.stdout:1308: left oid 325 (ObjNum 956 snap 0 seq_num 956) 2023-08-27T20:52:54.659 INFO:tasks.rados.rados.0.smithi093.stdout:1309: expect (ObjNum 868 snap 0 seq_num 868) 2023-08-27T20:52:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:1310: finishing write tid 1 to smithi093153278-173 2023-08-27T20:52:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:1310: finishing write tid 2 to smithi093153278-173 2023-08-27T20:52:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:1310: finishing write tid 3 to smithi093153278-173 2023-08-27T20:52:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:1310: finishing write tid 4 to smithi093153278-173 2023-08-27T20:52:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 173 v 178 (ObjNum 957 snap 0 seq_num 957) dirty exists 2023-08-27T20:52:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:1310: left oid 173 (ObjNum 957 snap 0 seq_num 957) 2023-08-27T20:52:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:1311: finishing write tid 1 to smithi093153278-314 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1304: done (9 left) 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1306: done (8 left) 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1307: done (7 left) 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1308: done (6 left) 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1309: done (5 left) 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1310: done (4 left) 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1314: done (3 left) 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1315: write oid 371 current snap is 0 2023-08-27T20:52:54.774 INFO:tasks.rados.rados.0.smithi093.stdout:1313: expect (ObjNum 276 snap 0 seq_num 276) 2023-08-27T20:52:54.853 INFO:tasks.rados.rados.0.smithi093.stdout:1311: finishing write tid 2 to smithi093153278-314 2023-08-27T20:52:54.853 INFO:tasks.rados.rados.0.smithi093.stdout:1311: finishing write tid 3 to smithi093153278-314 2023-08-27T20:52:54.854 INFO:tasks.rados.rados.0.smithi093.stdout:1312: expect (ObjNum 582 snap 0 seq_num 582) 2023-08-27T20:52:55.018 INFO:tasks.rados.rados.0.smithi093.stdout:1311: finishing write tid 4 to smithi093153278-314 2023-08-27T20:52:55.018 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 314 v 234 (ObjNum 958 snap 0 seq_num 958) dirty exists 2023-08-27T20:52:55.018 INFO:tasks.rados.rados.0.smithi093.stdout:1311: left oid 314 (ObjNum 958 snap 0 seq_num 958) 2023-08-27T20:52:55.018 INFO:tasks.rados.rados.0.smithi093.stdout:1315: seq_num 960 ranges {534720=499518,1460243=674477} 2023-08-27T20:52:55.023 INFO:tasks.rados.rados.0.smithi093.stdout:1315: writing smithi093153278-371 from 534720 to 1034238 tid 1 2023-08-27T20:52:55.030 INFO:tasks.rados.rados.0.smithi093.stdout:1315: writing smithi093153278-371 from 1460243 to 2134720 tid 2 2023-08-27T20:52:55.030 INFO:tasks.rados.rados.0.smithi093.stdout:1311: done (3 left) 2023-08-27T20:52:55.030 INFO:tasks.rados.rados.0.smithi093.stdout:1312: done (2 left) 2023-08-27T20:52:55.030 INFO:tasks.rados.rados.0.smithi093.stdout:1313: done (1 left) 2023-08-27T20:52:55.030 INFO:tasks.rados.rados.0.smithi093.stdout:1316: read oid 34 snap -1 2023-08-27T20:52:55.030 INFO:tasks.rados.rados.0.smithi093.stdout:1316: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:52:55.031 INFO:tasks.rados.rados.0.smithi093.stdout:1317: write oid 219 current snap is 0 2023-08-27T20:52:55.031 INFO:tasks.rados.rados.0.smithi093.stdout:1315: finishing write tid 1 to smithi093153278-371 2023-08-27T20:52:55.031 INFO:tasks.rados.rados.0.smithi093.stdout:1317: seq_num 961 ranges {551527=574767,1918725=232802} 2023-08-27T20:52:55.036 INFO:tasks.rados.rados.0.smithi093.stdout:1317: writing smithi093153278-219 from 551527 to 1126294 tid 1 2023-08-27T20:52:55.042 INFO:tasks.rados.rados.0.smithi093.stdout:1317: writing smithi093153278-219 from 1918725 to 2151527 tid 2 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1318: read oid 396 snap -1 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1318: expect (ObjNum 395 snap 0 seq_num 395) 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1319: write oid 158 current snap is 0 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1315: finishing write tid 2 to smithi093153278-371 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1315: finishing write tid 3 to smithi093153278-371 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1315: finishing write tid 4 to smithi093153278-371 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 371 v 189 (ObjNum 960 snap 0 seq_num 960) dirty exists 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1315: left oid 371 (ObjNum 960 snap 0 seq_num 960) 2023-08-27T20:52:55.043 INFO:tasks.rados.rados.0.smithi093.stdout:1316: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:52:55.125 INFO:tasks.rados.rados.0.smithi093.stdout:1317: finishing write tid 1 to smithi093153278-219 2023-08-27T20:52:55.125 INFO:tasks.rados.rados.0.smithi093.stdout:1317: finishing write tid 2 to smithi093153278-219 2023-08-27T20:52:55.125 INFO:tasks.rados.rados.0.smithi093.stdout:1318: expect (ObjNum 395 snap 0 seq_num 395) 2023-08-27T20:52:55.208 INFO:tasks.rados.rados.0.smithi093.stdout:1317: finishing write tid 3 to smithi093153278-219 2023-08-27T20:52:55.209 INFO:tasks.rados.rados.0.smithi093.stdout:1317: finishing write tid 4 to smithi093153278-219 2023-08-27T20:52:55.210 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 219 v 181 (ObjNum 961 snap 0 seq_num 961) dirty exists 2023-08-27T20:52:55.210 INFO:tasks.rados.rados.0.smithi093.stdout:1317: left oid 219 (ObjNum 961 snap 0 seq_num 961) 2023-08-27T20:52:55.210 INFO:tasks.rados.rados.0.smithi093.stdout:1319: seq_num 962 ranges {568334=650016,1860854=307480} 2023-08-27T20:52:55.214 INFO:tasks.rados.rados.0.smithi093.stdout:1319: writing smithi093153278-158 from 568334 to 1218350 tid 1 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1319: writing smithi093153278-158 from 1860854 to 2168334 tid 2 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1315: done (4 left) 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1316: done (3 left) 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1317: done (2 left) 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1318: done (1 left) 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1320: read oid 100 snap -1 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1320: expect (ObjNum 99 snap 0 seq_num 99) 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1321: read oid 412 snap -1 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1321: expect (ObjNum 824 snap 0 seq_num 824) 2023-08-27T20:52:55.222 INFO:tasks.rados.rados.0.smithi093.stdout:1322: read oid 401 snap -1 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1322: expect deleted 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1323: read oid 135 snap -1 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1323: expect (ObjNum 877 snap 0 seq_num 877) 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1319: finishing write tid 1 to smithi093153278-158 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1324: read oid 75 snap -1 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1324: expect (ObjNum 688 snap 0 seq_num 688) 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1325: read oid 50 snap -1 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1325: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1326: read oid 321 snap -1 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1326: expect (ObjNum 320 snap 0 seq_num 320) 2023-08-27T20:52:55.223 INFO:tasks.rados.rados.0.smithi093.stdout:1327: read oid 196 snap -1 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1327: expect deleted 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1328: read oid 203 snap -1 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1328: expect (ObjNum 852 snap 0 seq_num 852) 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1329: read oid 256 snap -1 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1329: expect (ObjNum 745 snap 0 seq_num 745) 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1330: read oid 229 snap -1 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1330: expect (ObjNum 751 snap 0 seq_num 751) 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1331: write oid 40 current snap is 0 2023-08-27T20:52:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:1331: seq_num 963 ranges {585141=725265,1919336=265805} 2023-08-27T20:52:55.229 INFO:tasks.rados.rados.0.smithi093.stdout:1331: writing smithi093153278-40 from 585141 to 1310406 tid 1 2023-08-27T20:52:55.234 INFO:tasks.rados.rados.0.smithi093.stdout:1331: writing smithi093153278-40 from 1919336 to 2185141 tid 2 2023-08-27T20:52:55.234 INFO:tasks.rados.rados.0.smithi093.stdout:1332: write oid 2 current snap is 0 2023-08-27T20:52:55.235 INFO:tasks.rados.rados.0.smithi093.stdout:1332: seq_num 964 ranges {601948=400514,1577818=624130} 2023-08-27T20:52:55.240 INFO:tasks.rados.rados.0.smithi093.stdout:1332: writing smithi093153278-2 from 601948 to 1002462 tid 1 2023-08-27T20:52:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:1332: writing smithi093153278-2 from 1577818 to 2201948 tid 2 2023-08-27T20:52:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:1333: delete oid 10 current snap is 0 2023-08-27T20:52:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:1324: expect (ObjNum 688 snap 0 seq_num 688) 2023-08-27T20:52:55.362 INFO:tasks.rados.rados.0.smithi093.stdout:1328: expect (ObjNum 852 snap 0 seq_num 852) 2023-08-27T20:52:55.471 INFO:tasks.rados.rados.0.smithi093.stdout:1330: expect (ObjNum 751 snap 0 seq_num 751) 2023-08-27T20:52:55.577 INFO:tasks.rados.rados.0.smithi093.stdout:1329: expect (ObjNum 745 snap 0 seq_num 745) 2023-08-27T20:52:55.691 INFO:tasks.rados.rados.0.smithi093.stdout:1326: expect (ObjNum 320 snap 0 seq_num 320) 2023-08-27T20:52:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:52:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:52:55] "GET /metrics HTTP/1.1" 200 242475 "" "Prometheus/2.43.0" 2023-08-27T20:52:55.796 INFO:tasks.rados.rados.0.smithi093.stdout:1331: finishing write tid 1 to smithi093153278-40 2023-08-27T20:52:55.796 INFO:tasks.rados.rados.0.smithi093.stdout:1331: finishing write tid 2 to smithi093153278-40 2023-08-27T20:52:55.796 INFO:tasks.rados.rados.0.smithi093.stdout:1331: finishing write tid 3 to smithi093153278-40 2023-08-27T20:52:55.796 INFO:tasks.rados.rados.0.smithi093.stdout:1331: finishing write tid 4 to smithi093153278-40 2023-08-27T20:52:55.797 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 40 v 237 (ObjNum 963 snap 0 seq_num 963) dirty exists 2023-08-27T20:52:55.797 INFO:tasks.rados.rados.0.smithi093.stdout:1331: left oid 40 (ObjNum 963 snap 0 seq_num 963) 2023-08-27T20:52:55.797 INFO:tasks.rados.rados.0.smithi093.stdout:1320: expect (ObjNum 99 snap 0 seq_num 99) 2023-08-27T20:52:55.907 INFO:tasks.rados.rados.0.smithi093.stdout:1332: finishing write tid 1 to smithi093153278-2 2023-08-27T20:52:55.907 INFO:tasks.rados.rados.0.smithi093.stdout:1321: expect (ObjNum 824 snap 0 seq_num 824) 2023-08-27T20:52:56.024 INFO:tasks.rados.rados.0.smithi093.stdout:1323: expect (ObjNum 877 snap 0 seq_num 877) 2023-08-27T20:52:56.137 INFO:tasks.rados.rados.0.smithi093.stdout:1319: finishing write tid 2 to smithi093153278-158 2023-08-27T20:52:56.137 INFO:tasks.rados.rados.0.smithi093.stdout:1319: finishing write tid 3 to smithi093153278-158 2023-08-27T20:52:56.137 INFO:tasks.rados.rados.0.smithi093.stdout:1319: finishing write tid 4 to smithi093153278-158 2023-08-27T20:52:56.137 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 158 v 216 (ObjNum 962 snap 0 seq_num 962) dirty exists 2023-08-27T20:52:56.137 INFO:tasks.rados.rados.0.smithi093.stdout:1319: left oid 158 (ObjNum 962 snap 0 seq_num 962) 2023-08-27T20:52:56.137 INFO:tasks.rados.rados.0.smithi093.stdout:1319: done (14 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1320: done (13 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1321: done (12 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1322: done (11 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1323: done (10 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1324: done (9 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1326: done (8 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1328: done (7 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1329: done (6 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1330: done (5 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1331: done (4 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1333: done (3 left) 2023-08-27T20:52:56.138 INFO:tasks.rados.rados.0.smithi093.stdout:1334: write oid 434 current snap is 0 2023-08-27T20:52:56.139 INFO:tasks.rados.rados.0.smithi093.stdout:1332: finishing write tid 2 to smithi093153278-2 2023-08-27T20:52:56.139 INFO:tasks.rados.rados.0.smithi093.stdout:1334: seq_num 966 ranges {635562=667365,2094782=140780} 2023-08-27T20:52:56.143 INFO:tasks.rados.rados.0.smithi093.stdout:1334: writing smithi093153278-434 from 635562 to 1302927 tid 1 2023-08-27T20:52:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:1334: writing smithi093153278-434 from 2094782 to 2235562 tid 2 2023-08-27T20:52:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:1335: write oid 195 current snap is 0 2023-08-27T20:52:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:1332: finishing write tid 3 to smithi093153278-2 2023-08-27T20:52:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:1332: finishing write tid 4 to smithi093153278-2 2023-08-27T20:52:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 2 v 223 (ObjNum 964 snap 0 seq_num 964) dirty exists 2023-08-27T20:52:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:1332: left oid 2 (ObjNum 964 snap 0 seq_num 964) 2023-08-27T20:52:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:1325: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:52:56.233 INFO:tasks.rados.rados.0.smithi093.stdout:1334: finishing write tid 1 to smithi093153278-434 2023-08-27T20:52:56.233 INFO:tasks.rados.rados.0.smithi093.stdout:1334: finishing write tid 2 to smithi093153278-434 2023-08-27T20:52:56.233 INFO:tasks.rados.rados.0.smithi093.stdout:1334: finishing write tid 3 to smithi093153278-434 2023-08-27T20:52:56.233 INFO:tasks.rados.rados.0.smithi093.stdout:1334: finishing write tid 4 to smithi093153278-434 2023-08-27T20:52:56.233 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 192 (ObjNum 966 snap 0 seq_num 966) dirty exists 2023-08-27T20:52:56.233 INFO:tasks.rados.rados.0.smithi093.stdout:1334: left oid 434 (ObjNum 966 snap 0 seq_num 966) 2023-08-27T20:52:56.234 INFO:tasks.rados.rados.0.smithi093.stdout:1335: seq_num 967 ranges {652369=742614,2153264=99105} 2023-08-27T20:52:56.240 INFO:tasks.rados.rados.0.smithi093.stdout:1335: writing smithi093153278-195 from 652369 to 1394983 tid 1 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1335: writing smithi093153278-195 from 2153264 to 2252369 tid 2 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1325: done (4 left) 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1327: done (3 left) 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1332: done (2 left) 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1334: done (1 left) 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1336: read oid 142 snap -1 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1336: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:52:56.245 INFO:tasks.rados.rados.0.smithi093.stdout:1337: write oid 488 current snap is 0 2023-08-27T20:52:56.246 INFO:tasks.rados.rados.0.smithi093.stdout:1335: finishing write tid 1 to smithi093153278-195 2023-08-27T20:52:56.246 INFO:tasks.rados.rados.0.smithi093.stdout:1337: seq_num 968 ranges {669176=417863,1811746=457430} 2023-08-27T20:52:56.250 INFO:tasks.rados.rados.0.smithi093.stdout:1337: writing smithi093153278-488 from 669176 to 1087039 tid 1 2023-08-27T20:52:56.257 INFO:tasks.rados.rados.0.smithi093.stdout:1337: writing smithi093153278-488 from 1811746 to 2269176 tid 2 2023-08-27T20:52:56.257 INFO:tasks.rados.rados.0.smithi093.stdout:1338: write oid 99 current snap is 0 2023-08-27T20:52:56.257 INFO:tasks.rados.rados.0.smithi093.stdout:1338: seq_num 969 ranges {685983=493112,1870228=415755} 2023-08-27T20:52:56.264 INFO:tasks.rados.rados.0.smithi093.stdout:1338: writing smithi093153278-99 from 685983 to 1179095 tid 1 2023-08-27T20:52:56.270 INFO:tasks.rados.rados.0.smithi093.stdout:1338: writing smithi093153278-99 from 1870228 to 2285983 tid 2 2023-08-27T20:52:56.270 INFO:tasks.rados.rados.0.smithi093.stdout:1339: read oid 498 snap -1 2023-08-27T20:52:56.270 INFO:tasks.rados.rados.0.smithi093.stdout:1339: expect (ObjNum 755 snap 0 seq_num 755) 2023-08-27T20:52:56.270 INFO:tasks.rados.rados.0.smithi093.stdout:1340: write oid 55 current snap is 0 2023-08-27T20:52:56.271 INFO:tasks.rados.rados.0.smithi093.stdout:1335: finishing write tid 2 to smithi093153278-195 2023-08-27T20:52:56.271 INFO:tasks.rados.rados.0.smithi093.stdout:1336: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:52:56.348 INFO:tasks.rados.rados.0.smithi093.stdout:1335: finishing write tid 3 to smithi093153278-195 2023-08-27T20:52:56.348 INFO:tasks.rados.rados.0.smithi093.stdout:1335: finishing write tid 4 to smithi093153278-195 2023-08-27T20:52:56.348 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 166 (ObjNum 967 snap 0 seq_num 967) dirty exists 2023-08-27T20:52:56.348 INFO:tasks.rados.rados.0.smithi093.stdout:1335: left oid 195 (ObjNum 967 snap 0 seq_num 967) 2023-08-27T20:52:56.348 INFO:tasks.rados.rados.0.smithi093.stdout:1337: finishing write tid 1 to smithi093153278-488 2023-08-27T20:52:56.348 INFO:tasks.rados.rados.0.smithi093.stdout:1337: finishing write tid 2 to smithi093153278-488 2023-08-27T20:52:56.348 INFO:tasks.rados.rados.0.smithi093.stdout:1340: seq_num 970 ranges {702790=568361,1812357=490433} 2023-08-27T20:52:56.354 INFO:tasks.rados.rados.0.smithi093.stdout:1340: writing smithi093153278-55 from 702790 to 1271151 tid 1 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1340: writing smithi093153278-55 from 1812357 to 2302790 tid 2 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1335: done (5 left) 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1336: done (4 left) 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1341: read oid 167 snap -1 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1341: expect (ObjNum 740 snap 0 seq_num 740) 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1342: write oid 210 current snap is 0 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1337: finishing write tid 3 to smithi093153278-488 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1337: finishing write tid 4 to smithi093153278-488 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 488 v 214 (ObjNum 968 snap 0 seq_num 968) dirty exists 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1337: left oid 488 (ObjNum 968 snap 0 seq_num 968) 2023-08-27T20:52:56.361 INFO:tasks.rados.rados.0.smithi093.stdout:1338: finishing write tid 1 to smithi093153278-99 2023-08-27T20:52:56.362 INFO:tasks.rados.rados.0.smithi093.stdout:1342: seq_num 971 ranges {719597=643610,1870839=448758} 2023-08-27T20:52:56.368 INFO:tasks.rados.rados.0.smithi093.stdout:1342: writing smithi093153278-210 from 719597 to 1363207 tid 1 2023-08-27T20:52:56.374 INFO:tasks.rados.rados.0.smithi093.stdout:1342: writing smithi093153278-210 from 1870839 to 2319597 tid 2 2023-08-27T20:52:56.374 INFO:tasks.rados.rados.0.smithi093.stdout:1337: done (5 left) 2023-08-27T20:52:56.374 INFO:tasks.rados.rados.0.smithi093.stdout:1343: read oid 260 snap -1 2023-08-27T20:52:56.374 INFO:tasks.rados.rados.0.smithi093.stdout:1343: expect (ObjNum 259 snap 0 seq_num 259) 2023-08-27T20:52:56.374 INFO:tasks.rados.rados.0.smithi093.stdout:1344: write oid 374 current snap is 0 2023-08-27T20:52:56.374 INFO:tasks.rados.rados.0.smithi093.stdout:1338: finishing write tid 2 to smithi093153278-99 2023-08-27T20:52:56.374 INFO:tasks.rados.rados.0.smithi093.stdout:1344: seq_num 972 ranges {736404=718859,1929321=407083} 2023-08-27T20:52:56.381 INFO:tasks.rados.rados.0.smithi093.stdout:1344: writing smithi093153278-374 from 736404 to 1455263 tid 1 2023-08-27T20:52:56.387 INFO:tasks.rados.rados.0.smithi093.stdout:1344: writing smithi093153278-374 from 1929321 to 2336404 tid 2 2023-08-27T20:52:56.387 INFO:tasks.rados.rados.0.smithi093.stdout:1345: write oid 228 current snap is 0 2023-08-27T20:52:56.387 INFO:tasks.rados.rados.0.smithi093.stdout:1338: finishing write tid 3 to smithi093153278-99 2023-08-27T20:52:56.387 INFO:tasks.rados.rados.0.smithi093.stdout:1345: seq_num 973 ranges {753211=794108,1987803=365408} 2023-08-27T20:52:56.395 INFO:tasks.rados.rados.0.smithi093.stdout:1345: writing smithi093153278-228 from 753211 to 1547319 tid 1 2023-08-27T20:52:56.400 INFO:tasks.rados.rados.0.smithi093.stdout:1345: writing smithi093153278-228 from 1987803 to 2353211 tid 2 2023-08-27T20:52:56.400 INFO:tasks.rados.rados.0.smithi093.stdout:1346: read oid 292 snap -1 2023-08-27T20:52:56.400 INFO:tasks.rados.rados.0.smithi093.stdout:1346: expect (ObjNum 291 snap 0 seq_num 291) 2023-08-27T20:52:56.400 INFO:tasks.rados.rados.0.smithi093.stdout:1347: read oid 73 snap -1 2023-08-27T20:52:56.400 INFO:tasks.rados.rados.0.smithi093.stdout:1347: expect (ObjNum 72 snap 0 seq_num 72) 2023-08-27T20:52:56.400 INFO:tasks.rados.rados.0.smithi093.stdout:1348: write oid 205 current snap is 0 2023-08-27T20:52:56.400 INFO:tasks.rados.rados.0.smithi093.stdout:1338: finishing write tid 4 to smithi093153278-99 2023-08-27T20:52:56.401 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 99 v 198 (ObjNum 969 snap 0 seq_num 969) dirty exists 2023-08-27T20:52:56.401 INFO:tasks.rados.rados.0.smithi093.stdout:1338: left oid 99 (ObjNum 969 snap 0 seq_num 969) 2023-08-27T20:52:56.401 INFO:tasks.rados.rados.0.smithi093.stdout:1339: expect (ObjNum 755 snap 0 seq_num 755) 2023-08-27T20:52:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:56 smithi136 ceph-mon[113531]: pgmap v82: 89 pgs: 89 active+clean; 1.2 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 9.7 MiB/s wr, 98 op/s 2023-08-27T20:52:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/961812332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:56 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:56 smithi093 ceph-mon[111335]: pgmap v82: 89 pgs: 89 active+clean; 1.2 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 9.7 MiB/s wr, 98 op/s 2023-08-27T20:52:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/961812332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:56 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:56 smithi093 ceph-mon[118902]: pgmap v82: 89 pgs: 89 active+clean; 1.2 GiB data, 2.1 GiB used, 713 GiB / 715 GiB avail; 64 MiB/s rd, 9.7 MiB/s wr, 98 op/s 2023-08-27T20:52:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/961812332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:56 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:52:56.515 INFO:tasks.rados.rados.0.smithi093.stdout:1340: finishing write tid 1 to smithi093153278-55 2023-08-27T20:52:56.516 INFO:tasks.rados.rados.0.smithi093.stdout:1340: finishing write tid 2 to smithi093153278-55 2023-08-27T20:52:56.516 INFO:tasks.rados.rados.0.smithi093.stdout:1348: seq_num 974 ranges {770018=585710,2046285=323733} 2023-08-27T20:52:56.523 INFO:tasks.rados.rados.0.smithi093.stdout:1348: writing smithi093153278-205 from 770018 to 1355728 tid 1 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1348: writing smithi093153278-205 from 2046285 to 2370018 tid 2 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1338: done (10 left) 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1339: done (9 left) 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1349: read oid 147 snap -1 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1349: expect (ObjNum 146 snap 0 seq_num 146) 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1350: write oid 230 current snap is 0 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1340: finishing write tid 3 to smithi093153278-55 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1340: finishing write tid 4 to smithi093153278-55 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 55 v 196 (ObjNum 970 snap 0 seq_num 970) dirty exists 2023-08-27T20:52:56.529 INFO:tasks.rados.rados.0.smithi093.stdout:1340: left oid 55 (ObjNum 970 snap 0 seq_num 970) 2023-08-27T20:52:56.530 INFO:tasks.rados.rados.0.smithi093.stdout:1341: expect (ObjNum 740 snap 0 seq_num 740) 2023-08-27T20:52:56.670 INFO:tasks.rados.rados.0.smithi093.stdout:1342: finishing write tid 1 to smithi093153278-210 2023-08-27T20:52:56.670 INFO:tasks.rados.rados.0.smithi093.stdout:1343: expect (ObjNum 259 snap 0 seq_num 259) 2023-08-27T20:52:56.743 INFO:tasks.rados.rados.0.smithi093.stdout:1342: finishing write tid 2 to smithi093153278-210 2023-08-27T20:52:56.743 INFO:tasks.rados.rados.0.smithi093.stdout:1342: finishing write tid 3 to smithi093153278-210 2023-08-27T20:52:56.743 INFO:tasks.rados.rados.0.smithi093.stdout:1342: finishing write tid 4 to smithi093153278-210 2023-08-27T20:52:56.743 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 210 v 263 (ObjNum 971 snap 0 seq_num 971) dirty exists 2023-08-27T20:52:56.744 INFO:tasks.rados.rados.0.smithi093.stdout:1342: left oid 210 (ObjNum 971 snap 0 seq_num 971) 2023-08-27T20:52:56.744 INFO:tasks.rados.rados.0.smithi093.stdout:1344: finishing write tid 1 to smithi093153278-374 2023-08-27T20:52:56.744 INFO:tasks.rados.rados.0.smithi093.stdout:1344: finishing write tid 2 to smithi093153278-374 2023-08-27T20:52:56.744 INFO:tasks.rados.rados.0.smithi093.stdout:1344: finishing write tid 3 to smithi093153278-374 2023-08-27T20:52:56.744 INFO:tasks.rados.rados.0.smithi093.stdout:1350: seq_num 975 ranges {786825=660959,2104767=282058} 2023-08-27T20:52:56.750 INFO:tasks.rados.rados.0.smithi093.stdout:1350: writing smithi093153278-230 from 786825 to 1447784 tid 1 2023-08-27T20:52:56.756 INFO:tasks.rados.rados.0.smithi093.stdout:1350: writing smithi093153278-230 from 2104767 to 2386825 tid 2 2023-08-27T20:52:56.756 INFO:tasks.rados.rados.0.smithi093.stdout:1340: done (10 left) 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1341: done (9 left) 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1342: done (8 left) 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1343: done (7 left) 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1351: read oid 216 snap -1 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1351: expect deleted 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1352: read oid 276 snap -1 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1352: expect (ObjNum 936 snap 0 seq_num 936) 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1353: write oid 7 current snap is 0 2023-08-27T20:52:56.757 INFO:tasks.rados.rados.0.smithi093.stdout:1344: finishing write tid 4 to smithi093153278-374 2023-08-27T20:52:56.758 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 374 v 219 (ObjNum 972 snap 0 seq_num 972) dirty exists 2023-08-27T20:52:56.758 INFO:tasks.rados.rados.0.smithi093.stdout:1344: left oid 374 (ObjNum 972 snap 0 seq_num 972) 2023-08-27T20:52:56.758 INFO:tasks.rados.rados.0.smithi093.stdout:1345: finishing write tid 1 to smithi093153278-228 2023-08-27T20:52:56.758 INFO:tasks.rados.rados.0.smithi093.stdout:1345: finishing write tid 2 to smithi093153278-228 2023-08-27T20:52:56.758 INFO:tasks.rados.rados.0.smithi093.stdout:1347: expect (ObjNum 72 snap 0 seq_num 72) 2023-08-27T20:52:56.857 INFO:tasks.rados.rados.0.smithi093.stdout:1345: finishing write tid 3 to smithi093153278-228 2023-08-27T20:52:56.857 INFO:tasks.rados.rados.0.smithi093.stdout:1346: expect (ObjNum 291 snap 0 seq_num 291) 2023-08-27T20:52:56.948 INFO:tasks.rados.rados.0.smithi093.stdout:1345: finishing write tid 4 to smithi093153278-228 2023-08-27T20:52:56.948 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 228 v 169 (ObjNum 973 snap 0 seq_num 973) dirty exists 2023-08-27T20:52:56.948 INFO:tasks.rados.rados.0.smithi093.stdout:1345: left oid 228 (ObjNum 973 snap 0 seq_num 973) 2023-08-27T20:52:56.949 INFO:tasks.rados.rados.0.smithi093.stdout:1353: seq_num 976 ranges {403632=736208,1763249=619999} 2023-08-27T20:52:56.954 INFO:tasks.rados.rados.0.smithi093.stdout:1353: writing smithi093153278-7 from 403632 to 1139840 tid 1 2023-08-27T20:52:56.961 INFO:tasks.rados.rados.0.smithi093.stdout:1353: writing smithi093153278-7 from 1763249 to 2383248 tid 2 2023-08-27T20:52:56.961 INFO:tasks.rados.rados.0.smithi093.stdout:1344: done (9 left) 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1345: done (8 left) 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1346: done (7 left) 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1347: done (6 left) 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1354: write oid 232 current snap is 0 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1348: finishing write tid 1 to smithi093153278-205 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1348: finishing write tid 2 to smithi093153278-205 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1348: finishing write tid 3 to smithi093153278-205 2023-08-27T20:52:56.962 INFO:tasks.rados.rados.0.smithi093.stdout:1354: seq_num 977 ranges {420439=411457,1421731=480010} 2023-08-27T20:52:56.966 INFO:tasks.rados.rados.0.smithi093.stdout:1354: writing smithi093153278-232 from 420439 to 831896 tid 1 2023-08-27T20:52:56.972 INFO:tasks.rados.rados.0.smithi093.stdout:1354: writing smithi093153278-232 from 1421731 to 1901741 tid 2 2023-08-27T20:52:56.972 INFO:tasks.rados.rados.0.smithi093.stdout:1348: finishing write tid 4 to smithi093153278-205 2023-08-27T20:52:56.972 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 205 v 136 (ObjNum 974 snap 0 seq_num 974) dirty exists 2023-08-27T20:52:56.972 INFO:tasks.rados.rados.0.smithi093.stdout:1348: left oid 205 (ObjNum 974 snap 0 seq_num 974) 2023-08-27T20:52:56.972 INFO:tasks.rados.rados.0.smithi093.stdout:1349: expect (ObjNum 146 snap 0 seq_num 146) 2023-08-27T20:52:57.045 INFO:tasks.rados.rados.0.smithi093.stdout:1350: finishing write tid 1 to smithi093153278-230 2023-08-27T20:52:57.045 INFO:tasks.rados.rados.0.smithi093.stdout:1355: read oid 308 snap -1 2023-08-27T20:52:57.045 INFO:tasks.rados.rados.0.smithi093.stdout:1355: expect (ObjNum 931 snap 0 seq_num 931) 2023-08-27T20:52:57.045 INFO:tasks.rados.rados.0.smithi093.stdout:1348: done (7 left) 2023-08-27T20:52:57.045 INFO:tasks.rados.rados.0.smithi093.stdout:1349: done (6 left) 2023-08-27T20:52:57.045 INFO:tasks.rados.rados.0.smithi093.stdout:1356: read oid 305 snap -1 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1356: expect (ObjNum 935 snap 0 seq_num 935) 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1357: read oid 259 snap -1 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1357: expect (ObjNum 927 snap 0 seq_num 927) 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1358: write oid 321 current snap is 0 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1350: finishing write tid 2 to smithi093153278-230 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1350: finishing write tid 3 to smithi093153278-230 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1350: finishing write tid 4 to smithi093153278-230 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 213 (ObjNum 975 snap 0 seq_num 975) dirty exists 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1350: left oid 230 (ObjNum 975 snap 0 seq_num 975) 2023-08-27T20:52:57.046 INFO:tasks.rados.rados.0.smithi093.stdout:1358: seq_num 978 ranges {437246=486706,1363860=623668} 2023-08-27T20:52:57.050 INFO:tasks.rados.rados.0.smithi093.stdout:1358: writing smithi093153278-321 from 437246 to 923952 tid 1 2023-08-27T20:52:57.057 INFO:tasks.rados.rados.0.smithi093.stdout:1358: writing smithi093153278-321 from 1363860 to 1987528 tid 2 2023-08-27T20:52:57.057 INFO:tasks.rados.rados.0.smithi093.stdout:1350: done (8 left) 2023-08-27T20:52:57.057 INFO:tasks.rados.rados.0.smithi093.stdout:1351: done (7 left) 2023-08-27T20:52:57.057 INFO:tasks.rados.rados.0.smithi093.stdout:1359: delete oid 292 current snap is 0 2023-08-27T20:52:57.057 INFO:tasks.rados.rados.0.smithi093.stdout:1352: expect (ObjNum 936 snap 0 seq_num 936) 2023-08-27T20:52:57.214 INFO:tasks.rados.rados.0.smithi093.stdout:1353: finishing write tid 1 to smithi093153278-7 2023-08-27T20:52:57.214 INFO:tasks.rados.rados.0.smithi093.stdout:1353: finishing write tid 2 to smithi093153278-7 2023-08-27T20:52:57.214 INFO:tasks.rados.rados.0.smithi093.stdout:1353: finishing write tid 3 to smithi093153278-7 2023-08-27T20:52:57.214 INFO:tasks.rados.rados.0.smithi093.stdout:1353: finishing write tid 4 to smithi093153278-7 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 7 v 172 (ObjNum 976 snap 0 seq_num 976) dirty exists 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1353: left oid 7 (ObjNum 976 snap 0 seq_num 976) 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1354: finishing write tid 1 to smithi093153278-232 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1354: finishing write tid 2 to smithi093153278-232 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1354: finishing write tid 3 to smithi093153278-232 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1354: finishing write tid 4 to smithi093153278-232 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 232 v 201 (ObjNum 977 snap 0 seq_num 977) dirty exists 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1354: left oid 232 (ObjNum 977 snap 0 seq_num 977) 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1352: done (7 left) 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1353: done (6 left) 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1354: done (5 left) 2023-08-27T20:52:57.215 INFO:tasks.rados.rados.0.smithi093.stdout:1359: done (4 left) 2023-08-27T20:52:57.216 INFO:tasks.rados.rados.0.smithi093.stdout:1360: read oid 35 snap -1 2023-08-27T20:52:57.216 INFO:tasks.rados.rados.0.smithi093.stdout:1360: expect (ObjNum 653 snap 0 seq_num 653) 2023-08-27T20:52:57.216 INFO:tasks.rados.rados.0.smithi093.stdout:1361: write oid 363 current snap is 0 2023-08-27T20:52:57.216 INFO:tasks.rados.rados.0.smithi093.stdout:1356: expect (ObjNum 935 snap 0 seq_num 935) 2023-08-27T20:52:57.415 INFO:tasks.rados.rados.0.smithi093.stdout:1358: finishing write tid 1 to smithi093153278-321 2023-08-27T20:52:57.415 INFO:tasks.rados.rados.0.smithi093.stdout:1357: expect (ObjNum 927 snap 0 seq_num 927) 2023-08-27T20:52:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:1358: finishing write tid 2 to smithi093153278-321 2023-08-27T20:52:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:1358: finishing write tid 3 to smithi093153278-321 2023-08-27T20:52:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:1358: finishing write tid 4 to smithi093153278-321 2023-08-27T20:52:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 321 v 217 (ObjNum 978 snap 0 seq_num 978) dirty exists 2023-08-27T20:52:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:1358: left oid 321 (ObjNum 978 snap 0 seq_num 978) 2023-08-27T20:52:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:1355: expect (ObjNum 931 snap 0 seq_num 931) 2023-08-27T20:52:57.768 INFO:tasks.rados.rados.0.smithi093.stdout:1360: expect (ObjNum 653 snap 0 seq_num 653) 2023-08-27T20:52:57.909 INFO:tasks.rados.rados.0.smithi093.stdout:1361: seq_num 980 ranges {470860=637204,1880824=590036} 2023-08-27T20:52:57.915 INFO:tasks.rados.rados.0.smithi093.stdout:1361: writing smithi093153278-363 from 470860 to 1108064 tid 1 2023-08-27T20:52:57.923 INFO:tasks.rados.rados.0.smithi093.stdout:1361: writing smithi093153278-363 from 1880824 to 2470860 tid 2 2023-08-27T20:52:57.923 INFO:tasks.rados.rados.0.smithi093.stdout:1355: done (5 left) 2023-08-27T20:52:57.923 INFO:tasks.rados.rados.0.smithi093.stdout:1356: done (4 left) 2023-08-27T20:52:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:1357: done (3 left) 2023-08-27T20:52:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:1358: done (2 left) 2023-08-27T20:52:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:1360: done (1 left) 2023-08-27T20:52:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:1362: write oid 213 current snap is 0 2023-08-27T20:52:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:1361: finishing write tid 1 to smithi093153278-363 2023-08-27T20:52:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:1362: seq_num 981 ranges {487667=428806,1655659=770995} 2023-08-27T20:52:57.927 INFO:tasks.rados.rados.0.smithi093.stdout:1362: writing smithi093153278-213 from 487667 to 916473 tid 1 2023-08-27T20:52:57.936 INFO:tasks.rados.rados.0.smithi093.stdout:1362: writing smithi093153278-213 from 1655659 to 2426654 tid 2 2023-08-27T20:52:57.936 INFO:tasks.rados.rados.0.smithi093.stdout:1363: read oid 233 snap -1 2023-08-27T20:52:57.936 INFO:tasks.rados.rados.0.smithi093.stdout:1363: expect deleted 2023-08-27T20:52:57.936 INFO:tasks.rados.rados.0.smithi093.stdout:1364: read oid 348 snap -1 2023-08-27T20:52:57.936 INFO:tasks.rados.rados.0.smithi093.stdout:1364: expect (ObjNum 647 snap 0 seq_num 647) 2023-08-27T20:52:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:1365: read oid 370 snap -1 2023-08-27T20:52:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:1365: expect (ObjNum 369 snap 0 seq_num 369) 2023-08-27T20:52:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:1366: read oid 407 snap -1 2023-08-27T20:52:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:1366: expect (ObjNum 406 snap 0 seq_num 406) 2023-08-27T20:52:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:1367: write oid 168 current snap is 0 2023-08-27T20:52:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:1367: seq_num 982 ranges {504474=504055,1714141=631006} 2023-08-27T20:52:57.942 INFO:tasks.rados.rados.0.smithi093.stdout:1367: writing smithi093153278-168 from 504474 to 1008529 tid 1 2023-08-27T20:52:57.950 INFO:tasks.rados.rados.0.smithi093.stdout:1367: writing smithi093153278-168 from 1714141 to 2345147 tid 2 2023-08-27T20:52:57.950 INFO:tasks.rados.rados.0.smithi093.stdout:1368: delete oid 406 current snap is 0 2023-08-27T20:52:57.950 INFO:tasks.rados.rados.0.smithi093.stdout:1361: finishing write tid 2 to smithi093153278-363 2023-08-27T20:52:57.950 INFO:tasks.rados.rados.0.smithi093.stdout:1361: finishing write tid 3 to smithi093153278-363 2023-08-27T20:52:57.950 INFO:tasks.rados.rados.0.smithi093.stdout:1361: finishing write tid 4 to smithi093153278-363 2023-08-27T20:52:57.950 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 266 (ObjNum 980 snap 0 seq_num 980) dirty exists 2023-08-27T20:52:57.950 INFO:tasks.rados.rados.0.smithi093.stdout:1361: left oid 363 (ObjNum 980 snap 0 seq_num 980) 2023-08-27T20:52:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:1362: finishing write tid 1 to smithi093153278-213 2023-08-27T20:52:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:1362: finishing write tid 2 to smithi093153278-213 2023-08-27T20:52:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:1362: finishing write tid 3 to smithi093153278-213 2023-08-27T20:52:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:1362: finishing write tid 4 to smithi093153278-213 2023-08-27T20:52:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 213 v 186 (ObjNum 981 snap 0 seq_num 981) dirty exists 2023-08-27T20:52:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:1362: left oid 213 (ObjNum 981 snap 0 seq_num 981) 2023-08-27T20:52:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:1365: expect (ObjNum 369 snap 0 seq_num 369) 2023-08-27T20:52:58.022 INFO:tasks.rados.rados.0.smithi093.stdout:1364: expect (ObjNum 647 snap 0 seq_num 647) 2023-08-27T20:52:58.150 INFO:tasks.rados.rados.0.smithi093.stdout:1367: finishing write tid 1 to smithi093153278-168 2023-08-27T20:52:58.150 INFO:tasks.rados.rados.0.smithi093.stdout:1366: expect (ObjNum 406 snap 0 seq_num 406) 2023-08-27T20:52:58.235 INFO:tasks.rados.rados.0.smithi093.stdout:1367: finishing write tid 2 to smithi093153278-168 2023-08-27T20:52:58.235 INFO:tasks.rados.rados.0.smithi093.stdout:1367: finishing write tid 3 to smithi093153278-168 2023-08-27T20:52:58.235 INFO:tasks.rados.rados.0.smithi093.stdout:1367: finishing write tid 4 to smithi093153278-168 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 168 v 216 (ObjNum 982 snap 0 seq_num 982) dirty exists 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1367: left oid 168 (ObjNum 982 snap 0 seq_num 982) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1361: done (7 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1362: done (6 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1363: done (5 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1364: done (4 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1365: done (3 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1366: done (2 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1367: done (1 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1368: done (0 left) 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1369: write oid 472 current snap is 0 2023-08-27T20:52:58.236 INFO:tasks.rados.rados.0.smithi093.stdout:1369: seq_num 984 ranges {538088=654553,1714752=634675} 2023-08-27T20:52:58.241 INFO:tasks.rados.rados.0.smithi093.stdout:1369: writing smithi093153278-472 from 538088 to 1192641 tid 1 2023-08-27T20:52:58.248 INFO:tasks.rados.rados.0.smithi093.stdout:1369: writing smithi093153278-472 from 1714752 to 2349427 tid 2 2023-08-27T20:52:58.248 INFO:tasks.rados.rados.0.smithi093.stdout:1370: write oid 41 current snap is 0 2023-08-27T20:52:58.248 INFO:tasks.rados.rados.0.smithi093.stdout:1369: finishing write tid 1 to smithi093153278-472 2023-08-27T20:52:58.248 INFO:tasks.rados.rados.0.smithi093.stdout:1370: seq_num 985 ranges {554895=729802,1773234=778333} 2023-08-27T20:52:58.255 INFO:tasks.rados.rados.0.smithi093.stdout:1370: writing smithi093153278-41 from 554895 to 1284697 tid 1 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:1370: writing smithi093153278-41 from 1773234 to 2551567 tid 2 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:1371: read oid 373 snap -1 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:1371: expect deleted 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:1372: write oid 219 current snap is 0 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:1369: finishing write tid 2 to smithi093153278-472 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:1369: finishing write tid 3 to smithi093153278-472 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:1369: finishing write tid 4 to smithi093153278-472 2023-08-27T20:52:58.262 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 472 v 204 (ObjNum 984 snap 0 seq_num 984) dirty exists 2023-08-27T20:52:58.263 INFO:tasks.rados.rados.0.smithi093.stdout:1369: left oid 472 (ObjNum 984 snap 0 seq_num 984) 2023-08-27T20:52:58.263 INFO:tasks.rados.rados.0.smithi093.stdout:1370: finishing write tid 1 to smithi093153278-41 2023-08-27T20:52:58.263 INFO:tasks.rados.rados.0.smithi093.stdout:1372: seq_num 986 ranges {571702=405051,1431716=638344,2560365=11337} 2023-08-27T20:52:58.267 INFO:tasks.rados.rados.0.smithi093.stdout:1372: writing smithi093153278-219 from 571702 to 976753 tid 1 2023-08-27T20:52:58.273 INFO:tasks.rados.rados.0.smithi093.stdout:1372: writing smithi093153278-219 from 1431716 to 2070060 tid 2 2023-08-27T20:52:58.276 INFO:tasks.rados.rados.0.smithi093.stdout:1372: writing smithi093153278-219 from 2560365 to 2571702 tid 3 2023-08-27T20:52:58.276 INFO:tasks.rados.rados.0.smithi093.stdout:1369: done (3 left) 2023-08-27T20:52:58.276 INFO:tasks.rados.rados.0.smithi093.stdout:1373: read oid 82 snap -1 2023-08-27T20:52:58.276 INFO:tasks.rados.rados.0.smithi093.stdout:1373: expect (ObjNum 738 snap 0 seq_num 738) 2023-08-27T20:52:58.277 INFO:tasks.rados.rados.0.smithi093.stdout:1374: write oid 391 current snap is 0 2023-08-27T20:52:58.277 INFO:tasks.rados.rados.0.smithi093.stdout:1374: seq_num 987 ranges {588509=480300,1773845=782002} 2023-08-27T20:52:58.282 INFO:tasks.rados.rados.0.smithi093.stdout:1374: writing smithi093153278-391 from 588509 to 1068809 tid 1 2023-08-27T20:52:58.290 INFO:tasks.rados.rados.0.smithi093.stdout:1374: writing smithi093153278-391 from 1773845 to 2555847 tid 2 2023-08-27T20:52:58.291 INFO:tasks.rados.rados.0.smithi093.stdout:1375: write oid 393 current snap is 0 2023-08-27T20:52:58.291 INFO:tasks.rados.rados.0.smithi093.stdout:1375: seq_num 988 ranges {605316=555549,1832327=642013} 2023-08-27T20:52:58.297 INFO:tasks.rados.rados.0.smithi093.stdout:1375: writing smithi093153278-393 from 605316 to 1160865 tid 1 2023-08-27T20:52:58.304 INFO:tasks.rados.rados.0.smithi093.stdout:1375: writing smithi093153278-393 from 1832327 to 2474340 tid 2 2023-08-27T20:52:58.304 INFO:tasks.rados.rados.0.smithi093.stdout:1371: done (5 left) 2023-08-27T20:52:58.304 INFO:tasks.rados.rados.0.smithi093.stdout:1376: write oid 345 current snap is 0 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1370: finishing write tid 2 to smithi093153278-41 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1370: finishing write tid 3 to smithi093153278-41 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1370: finishing write tid 4 to smithi093153278-41 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 41 v 269 (ObjNum 985 snap 0 seq_num 985) dirty exists 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1370: left oid 41 (ObjNum 985 snap 0 seq_num 985) 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1372: finishing write tid 1 to smithi093153278-219 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1372: finishing write tid 2 to smithi093153278-219 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1372: finishing write tid 3 to smithi093153278-219 2023-08-27T20:52:58.305 INFO:tasks.rados.rados.0.smithi093.stdout:1376: seq_num 989 ranges {622123=747151,2007162=614961} 2023-08-27T20:52:58.308 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:58 smithi136 ceph-mon[113531]: pgmap v83: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 97 MiB/s rd, 14 MiB/s wr, 142 op/s 2023-08-27T20:52:58.308 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:58 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:58.309 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:58 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:52:58.309 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:58 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:52:58.309 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:58 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:52:58.309 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:58 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:52:58.311 INFO:tasks.rados.rados.0.smithi093.stdout:1376: writing smithi093153278-345 from 622123 to 1369274 tid 1 2023-08-27T20:52:58.318 INFO:tasks.rados.rados.0.smithi093.stdout:1376: writing smithi093153278-345 from 2007162 to 2622123 tid 2 2023-08-27T20:52:58.319 INFO:tasks.rados.rados.0.smithi093.stdout:1370: done (5 left) 2023-08-27T20:52:58.319 INFO:tasks.rados.rados.0.smithi093.stdout:1377: write oid 40 current snap is 0 2023-08-27T20:52:58.319 INFO:tasks.rados.rados.0.smithi093.stdout:1372: finishing write tid 4 to smithi093153278-219 2023-08-27T20:52:58.319 INFO:tasks.rados.rados.0.smithi093.stdout:1377: seq_num 990 ranges {638930=422400,1665644=645682} 2023-08-27T20:52:58.324 INFO:tasks.rados.rados.0.smithi093.stdout:1377: writing smithi093153278-40 from 638930 to 1061330 tid 1 2023-08-27T20:52:58.331 INFO:tasks.rados.rados.0.smithi093.stdout:1377: writing smithi093153278-40 from 1665644 to 2311326 tid 2 2023-08-27T20:52:58.331 INFO:tasks.rados.rados.0.smithi093.stdout:1378: read oid 111 snap -1 2023-08-27T20:52:58.332 INFO:tasks.rados.rados.0.smithi093.stdout:1378: expect (ObjNum 705 snap 0 seq_num 705) 2023-08-27T20:52:58.332 INFO:tasks.rados.rados.0.smithi093.stdout:1379: write oid 222 current snap is 0 2023-08-27T20:52:58.332 INFO:tasks.rados.rados.0.smithi093.stdout:1372: finishing write tid 5 to smithi093153278-219 2023-08-27T20:52:58.332 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 219 v 185 (ObjNum 986 snap 0 seq_num 986) dirty exists 2023-08-27T20:52:58.332 INFO:tasks.rados.rados.0.smithi093.stdout:1372: left oid 219 (ObjNum 986 snap 0 seq_num 986) 2023-08-27T20:52:58.332 INFO:tasks.rados.rados.0.smithi093.stdout:1373: expect (ObjNum 738 snap 0 seq_num 738) 2023-08-27T20:52:58.434 INFO:tasks.rados.rados.0.smithi093.stdout:1374: finishing write tid 1 to smithi093153278-391 2023-08-27T20:52:58.434 INFO:tasks.rados.rados.0.smithi093.stdout:1374: finishing write tid 2 to smithi093153278-391 2023-08-27T20:52:58.434 INFO:tasks.rados.rados.0.smithi093.stdout:1374: finishing write tid 3 to smithi093153278-391 2023-08-27T20:52:58.435 INFO:tasks.rados.rados.0.smithi093.stdout:1374: finishing write tid 4 to smithi093153278-391 2023-08-27T20:52:58.435 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 391 v 226 (ObjNum 987 snap 0 seq_num 987) dirty exists 2023-08-27T20:52:58.435 INFO:tasks.rados.rados.0.smithi093.stdout:1374: left oid 391 (ObjNum 987 snap 0 seq_num 987) 2023-08-27T20:52:58.435 INFO:tasks.rados.rados.0.smithi093.stdout:1375: finishing write tid 1 to smithi093153278-393 2023-08-27T20:52:58.435 INFO:tasks.rados.rados.0.smithi093.stdout:1379: seq_num 991 ranges {655737=497649,1607773=789340} 2023-08-27T20:52:58.440 INFO:tasks.rados.rados.0.smithi093.stdout:1379: writing smithi093153278-222 from 655737 to 1153386 tid 1 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1379: writing smithi093153278-222 from 1607773 to 2397113 tid 2 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1372: done (7 left) 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1373: done (6 left) 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1374: done (5 left) 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1380: write oid 84 current snap is 0 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1375: finishing write tid 2 to smithi093153278-393 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1375: finishing write tid 3 to smithi093153278-393 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1375: finishing write tid 4 to smithi093153278-393 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 393 v 139 (ObjNum 988 snap 0 seq_num 988) dirty exists 2023-08-27T20:52:58.448 INFO:tasks.rados.rados.0.smithi093.stdout:1375: left oid 393 (ObjNum 988 snap 0 seq_num 988) 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1376: finishing write tid 1 to smithi093153278-345 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1376: finishing write tid 2 to smithi093153278-345 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1376: finishing write tid 3 to smithi093153278-345 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1376: finishing write tid 4 to smithi093153278-345 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 345 v 142 (ObjNum 989 snap 0 seq_num 989) dirty exists 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1376: left oid 345 (ObjNum 989 snap 0 seq_num 989) 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1377: finishing write tid 1 to smithi093153278-40 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1377: finishing write tid 2 to smithi093153278-40 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1377: finishing write tid 3 to smithi093153278-40 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1377: finishing write tid 4 to smithi093153278-40 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 40 v 240 (ObjNum 990 snap 0 seq_num 990) dirty exists 2023-08-27T20:52:58.449 INFO:tasks.rados.rados.0.smithi093.stdout:1377: left oid 40 (ObjNum 990 snap 0 seq_num 990) 2023-08-27T20:52:58.450 INFO:tasks.rados.rados.0.smithi093.stdout:1380: seq_num 992 ranges {672544=572898,1666255=532998} 2023-08-27T20:52:58.454 INFO:tasks.rados.rados.0.smithi093.stdout:1380: writing smithi093153278-84 from 672544 to 1245442 tid 1 2023-08-27T20:52:58.459 INFO:tasks.rados.rados.0.smithi093.stdout:1380: writing smithi093153278-84 from 1666255 to 2199253 tid 2 2023-08-27T20:52:58.459 INFO:tasks.rados.rados.0.smithi093.stdout:1375: done (5 left) 2023-08-27T20:52:58.460 INFO:tasks.rados.rados.0.smithi093.stdout:1376: done (4 left) 2023-08-27T20:52:58.460 INFO:tasks.rados.rados.0.smithi093.stdout:1377: done (3 left) 2023-08-27T20:52:58.460 INFO:tasks.rados.rados.0.smithi093.stdout:1381: read oid 295 snap -1 2023-08-27T20:52:58.460 INFO:tasks.rados.rados.0.smithi093.stdout:1381: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:52:58.460 INFO:tasks.rados.rados.0.smithi093.stdout:1382: write oid 155 current snap is 0 2023-08-27T20:52:58.460 INFO:tasks.rados.rados.0.smithi093.stdout:1378: expect (ObjNum 705 snap 0 seq_num 705) 2023-08-27T20:52:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[111335]: pgmap v83: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 97 MiB/s rd, 14 MiB/s wr, 142 op/s 2023-08-27T20:52:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:52:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:52:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:52:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:52:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[118902]: pgmap v83: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 97 MiB/s rd, 14 MiB/s wr, 142 op/s 2023-08-27T20:52:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:52:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:52:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "auth get-or-create", "entity": "mgr.x", "caps": ["mon", "profile mgr", "osd", "allow *", "mds", "allow *"]}]: dispatch 2023-08-27T20:52:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr services"}]: dispatch 2023-08-27T20:52:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:58 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:52:58.632 INFO:tasks.rados.rados.0.smithi093.stdout:1379: finishing write tid 1 to smithi093153278-222 2023-08-27T20:52:58.632 INFO:tasks.rados.rados.0.smithi093.stdout:1379: finishing write tid 2 to smithi093153278-222 2023-08-27T20:52:58.632 INFO:tasks.rados.rados.0.smithi093.stdout:1379: finishing write tid 3 to smithi093153278-222 2023-08-27T20:52:58.632 INFO:tasks.rados.rados.0.smithi093.stdout:1379: finishing write tid 4 to smithi093153278-222 2023-08-27T20:52:58.632 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 222 v 229 (ObjNum 991 snap 0 seq_num 991) dirty exists 2023-08-27T20:52:58.633 INFO:tasks.rados.rados.0.smithi093.stdout:1379: left oid 222 (ObjNum 991 snap 0 seq_num 991) 2023-08-27T20:52:58.633 INFO:tasks.rados.rados.0.smithi093.stdout:1380: finishing write tid 1 to smithi093153278-84 2023-08-27T20:52:58.633 INFO:tasks.rados.rados.0.smithi093.stdout:1382: seq_num 993 ranges {689351=648147,2124737=564614} 2023-08-27T20:52:58.639 INFO:tasks.rados.rados.0.smithi093.stdout:1382: writing smithi093153278-155 from 689351 to 1337498 tid 1 2023-08-27T20:52:58.647 INFO:tasks.rados.rados.0.smithi093.stdout:1382: writing smithi093153278-155 from 2124737 to 2689351 tid 2 2023-08-27T20:52:58.647 INFO:tasks.rados.rados.0.smithi093.stdout:1378: done (4 left) 2023-08-27T20:52:58.647 INFO:tasks.rados.rados.0.smithi093.stdout:1379: done (3 left) 2023-08-27T20:52:58.647 INFO:tasks.rados.rados.0.smithi093.stdout:1383: delete oid 68 current snap is 0 2023-08-27T20:52:58.648 INFO:tasks.rados.rados.0.smithi093.stdout:1380: finishing write tid 2 to smithi093153278-84 2023-08-27T20:52:58.648 INFO:tasks.rados.rados.0.smithi093.stdout:1380: finishing write tid 3 to smithi093153278-84 2023-08-27T20:52:58.648 INFO:tasks.rados.rados.0.smithi093.stdout:1380: finishing write tid 4 to smithi093153278-84 2023-08-27T20:52:58.648 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 84 v 229 (ObjNum 992 snap 0 seq_num 992) dirty exists 2023-08-27T20:52:58.648 INFO:tasks.rados.rados.0.smithi093.stdout:1380: left oid 84 (ObjNum 992 snap 0 seq_num 992) 2023-08-27T20:52:58.648 INFO:tasks.rados.rados.0.smithi093.stdout:1381: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:52:58.739 INFO:tasks.rados.rados.0.smithi093.stdout:1382: finishing write tid 1 to smithi093153278-155 2023-08-27T20:52:58.739 INFO:tasks.rados.rados.0.smithi093.stdout:1382: finishing write tid 2 to smithi093153278-155 2023-08-27T20:52:58.740 INFO:tasks.rados.rados.0.smithi093.stdout:1382: finishing write tid 3 to smithi093153278-155 2023-08-27T20:52:58.740 INFO:tasks.rados.rados.0.smithi093.stdout:1382: finishing write tid 4 to smithi093153278-155 2023-08-27T20:52:58.740 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 155 v 207 (ObjNum 993 snap 0 seq_num 993) dirty exists 2023-08-27T20:52:58.740 INFO:tasks.rados.rados.0.smithi093.stdout:1382: left oid 155 (ObjNum 993 snap 0 seq_num 993) 2023-08-27T20:52:58.740 INFO:tasks.rados.rados.0.smithi093.stdout:1380: done (3 left) 2023-08-27T20:52:58.740 INFO:tasks.rados.rados.0.smithi093.stdout:1381: done (2 left) 2023-08-27T20:52:58.741 INFO:tasks.rados.rados.0.smithi093.stdout:1382: done (1 left) 2023-08-27T20:52:58.741 INFO:tasks.rados.rados.0.smithi093.stdout:1383: done (0 left) 2023-08-27T20:52:58.741 INFO:tasks.rados.rados.0.smithi093.stdout:1384: read oid 251 snap -1 2023-08-27T20:52:58.741 INFO:tasks.rados.rados.0.smithi093.stdout:1384: expect (ObjNum 250 snap 0 seq_num 250) 2023-08-27T20:52:58.741 INFO:tasks.rados.rados.0.smithi093.stdout:1385: write oid 388 current snap is 0 2023-08-27T20:52:58.741 INFO:tasks.rados.rados.0.smithi093.stdout:1385: seq_num 995 ranges {722965=798645,2125348=597617} 2023-08-27T20:52:58.747 INFO:tasks.rados.rados.0.smithi093.stdout:1385: writing smithi093153278-388 from 722965 to 1521610 tid 1 2023-08-27T20:52:58.754 INFO:tasks.rados.rados.0.smithi093.stdout:1385: writing smithi093153278-388 from 2125348 to 2722965 tid 2 2023-08-27T20:52:58.754 INFO:tasks.rados.rados.0.smithi093.stdout:1386: read oid 384 snap -1 2023-08-27T20:52:58.754 INFO:tasks.rados.rados.0.smithi093.stdout:1386: expect (ObjNum 808 snap 0 seq_num 808) 2023-08-27T20:52:58.754 INFO:tasks.rados.rados.0.smithi093.stdout:1387: write oid 281 current snap is 0 2023-08-27T20:52:58.754 INFO:tasks.rados.rados.0.smithi093.stdout:1387: seq_num 996 ranges {739772=590247,1900183=540336} 2023-08-27T20:52:58.761 INFO:tasks.rados.rados.0.smithi093.stdout:1387: writing smithi093153278-281 from 739772 to 1330019 tid 1 2023-08-27T20:52:58.768 INFO:tasks.rados.rados.0.smithi093.stdout:1387: writing smithi093153278-281 from 1900183 to 2440519 tid 2 2023-08-27T20:52:58.768 INFO:tasks.rados.rados.0.smithi093.stdout:1388: read oid 33 snap -1 2023-08-27T20:52:58.768 INFO:tasks.rados.rados.0.smithi093.stdout:1388: expect (ObjNum 32 snap 0 seq_num 32) 2023-08-27T20:52:58.768 INFO:tasks.rados.rados.0.smithi093.stdout:1389: read oid 324 snap -1 2023-08-27T20:52:58.768 INFO:tasks.rados.rados.0.smithi093.stdout:1389: expect (ObjNum 646 snap 0 seq_num 646) 2023-08-27T20:52:58.769 INFO:tasks.rados.rados.0.smithi093.stdout:1390: read oid 119 snap -1 2023-08-27T20:52:58.769 INFO:tasks.rados.rados.0.smithi093.stdout:1390: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:52:58.769 INFO:tasks.rados.rados.0.smithi093.stdout:1391: read oid 151 snap -1 2023-08-27T20:52:58.769 INFO:tasks.rados.rados.0.smithi093.stdout:1391: expect (ObjNum 815 snap 0 seq_num 815) 2023-08-27T20:52:58.769 INFO:tasks.rados.rados.0.smithi093.stdout:1392: read oid 130 snap -1 2023-08-27T20:52:58.769 INFO:tasks.rados.rados.0.smithi093.stdout:1392: expect (ObjNum 882 snap 0 seq_num 882) 2023-08-27T20:52:58.769 INFO:tasks.rados.rados.0.smithi093.stdout:1393: write oid 78 current snap is 0 2023-08-27T20:52:58.770 INFO:tasks.rados.rados.0.smithi093.stdout:1384: expect (ObjNum 250 snap 0 seq_num 250) 2023-08-27T20:52:58.841 INFO:tasks.rados.rados.0.smithi093.stdout:1385: finishing write tid 1 to smithi093153278-388 2023-08-27T20:52:58.841 INFO:tasks.rados.rados.0.smithi093.stdout:1393: seq_num 997 ranges {756579=665496,1958665=400347} 2023-08-27T20:52:58.848 INFO:tasks.rados.rados.0.smithi093.stdout:1393: writing smithi093153278-78 from 756579 to 1422075 tid 1 2023-08-27T20:52:58.854 INFO:tasks.rados.rados.0.smithi093.stdout:1393: writing smithi093153278-78 from 1958665 to 2359012 tid 2 2023-08-27T20:52:58.854 INFO:tasks.rados.rados.0.smithi093.stdout:1384: done (9 left) 2023-08-27T20:52:58.854 INFO:tasks.rados.rados.0.smithi093.stdout:1394: read oid 26 snap -1 2023-08-27T20:52:58.854 INFO:tasks.rados.rados.0.smithi093.stdout:1394: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:58.854 INFO:tasks.rados.rados.0.smithi093.stdout:1395: write oid 266 current snap is 0 2023-08-27T20:52:58.854 INFO:tasks.rados.rados.0.smithi093.stdout:1385: finishing write tid 2 to smithi093153278-388 2023-08-27T20:52:58.854 INFO:tasks.rados.rados.0.smithi093.stdout:1395: seq_num 998 ranges {773386=740745,2017147=544005} 2023-08-27T20:52:58.862 INFO:tasks.rados.rados.0.smithi093.stdout:1395: writing smithi093153278-266 from 773386 to 1514131 tid 1 2023-08-27T20:52:58.868 INFO:tasks.rados.rados.0.smithi093.stdout:1395: writing smithi093153278-266 from 2017147 to 2561152 tid 2 2023-08-27T20:52:58.868 INFO:tasks.rados.rados.0.smithi093.stdout:1396: read oid 149 snap -1 2023-08-27T20:52:58.868 INFO:tasks.rados.rados.0.smithi093.stdout:1396: expect (ObjNum 790 snap 0 seq_num 790) 2023-08-27T20:52:58.868 INFO:tasks.rados.rados.0.smithi093.stdout:1397: read oid 177 snap -1 2023-08-27T20:52:58.868 INFO:tasks.rados.rados.0.smithi093.stdout:1397: expect (ObjNum 176 snap 0 seq_num 176) 2023-08-27T20:52:58.868 INFO:tasks.rados.rados.0.smithi093.stdout:1398: delete oid 30 current snap is 0 2023-08-27T20:52:58.868 INFO:tasks.rados.rados.0.smithi093.stdout:1385: finishing write tid 3 to smithi093153278-388 2023-08-27T20:52:58.869 INFO:tasks.rados.rados.0.smithi093.stdout:1385: finishing write tid 4 to smithi093153278-388 2023-08-27T20:52:58.869 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 388 v 232 (ObjNum 995 snap 0 seq_num 995) dirty exists 2023-08-27T20:52:58.869 INFO:tasks.rados.rados.0.smithi093.stdout:1385: left oid 388 (ObjNum 995 snap 0 seq_num 995) 2023-08-27T20:52:58.869 INFO:tasks.rados.rados.0.smithi093.stdout:1386: expect (ObjNum 808 snap 0 seq_num 808) 2023-08-27T20:52:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:58 smithi136 systemd[1]: Stopping Ceph mgr.x for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:52:59.015 INFO:tasks.rados.rados.0.smithi093.stdout:1387: finishing write tid 1 to smithi093153278-281 2023-08-27T20:52:59.015 INFO:tasks.rados.rados.0.smithi093.stdout:1389: expect (ObjNum 646 snap 0 seq_num 646) 2023-08-27T20:52:59.141 INFO:tasks.rados.rados.0.smithi093.stdout:1387: finishing write tid 2 to smithi093153278-281 2023-08-27T20:52:59.141 INFO:tasks.rados.rados.0.smithi093.stdout:1387: finishing write tid 3 to smithi093153278-281 2023-08-27T20:52:59.141 INFO:tasks.rados.rados.0.smithi093.stdout:1387: finishing write tid 4 to smithi093153278-281 2023-08-27T20:52:59.141 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 281 v 243 (ObjNum 996 snap 0 seq_num 996) dirty exists 2023-08-27T20:52:59.141 INFO:tasks.rados.rados.0.smithi093.stdout:1387: left oid 281 (ObjNum 996 snap 0 seq_num 996) 2023-08-27T20:52:59.142 INFO:tasks.rados.rados.0.smithi093.stdout:1392: expect (ObjNum 882 snap 0 seq_num 882) 2023-08-27T20:52:59.258 INFO:tasks.rados.rados.0.smithi093.stdout:1388: expect (ObjNum 32 snap 0 seq_num 32) 2023-08-27T20:52:59.337 INFO:tasks.rados.rados.0.smithi093.stdout:1390: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:52:59.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:59 smithi136 ceph-mon[113531]: Upgrade: Updating mgr.x 2023-08-27T20:52:59.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:59 smithi136 ceph-mon[113531]: Deploying daemon mgr.x on smithi136 2023-08-27T20:52:59.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:52:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1198668448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:59.343 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:58 smithi136 bash[140443]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x 2023-08-27T20:52:59.344 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 systemd[1]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.x.service: Main process exited, code=exited, status=143/n/a 2023-08-27T20:52:59.344 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 systemd[1]: ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315@mgr.x.service: Failed with result 'exit-code'. 2023-08-27T20:52:59.344 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 systemd[1]: Stopped Ceph mgr.x for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:52:59.344 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 systemd[1]: Starting Ceph mgr.x for b79a52c0-451a-11ee-817b-1b7a5f1ba315... 2023-08-27T20:52:59.462 INFO:tasks.rados.rados.0.smithi093.stdout:1391: expect (ObjNum 815 snap 0 seq_num 815) 2023-08-27T20:52:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:59 smithi093 ceph-mon[111335]: Upgrade: Updating mgr.x 2023-08-27T20:52:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:59 smithi093 ceph-mon[111335]: Deploying daemon mgr.x on smithi136 2023-08-27T20:52:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:52:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1198668448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:59 smithi093 ceph-mon[118902]: Upgrade: Updating mgr.x 2023-08-27T20:52:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:59 smithi093 ceph-mon[118902]: Deploying daemon mgr.x on smithi136 2023-08-27T20:52:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:52:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1198668448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:52:59.621 INFO:tasks.rados.rados.0.smithi093.stdout:1393: finishing write tid 1 to smithi093153278-78 2023-08-27T20:52:59.621 INFO:tasks.rados.rados.0.smithi093.stdout:1394: expect (ObjNum 25 snap 0 seq_num 25) 2023-08-27T20:52:59.663 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 ceph-mgr[140601]: -- 172.21.15.136:0/2891905749 <== mon.1 v2:172.21.15.136:3300/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x561a564a11e0 con 0x561a56477000 2023-08-27T20:52:59.697 INFO:tasks.rados.rados.0.smithi093.stdout:1393: finishing write tid 2 to smithi093153278-78 2023-08-27T20:52:59.697 INFO:tasks.rados.rados.0.smithi093.stdout:1393: finishing write tid 3 to smithi093153278-78 2023-08-27T20:52:59.697 INFO:tasks.rados.rados.0.smithi093.stdout:1393: finishing write tid 4 to smithi093153278-78 2023-08-27T20:52:59.697 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 195 (ObjNum 997 snap 0 seq_num 997) dirty exists 2023-08-27T20:52:59.697 INFO:tasks.rados.rados.0.smithi093.stdout:1393: left oid 78 (ObjNum 997 snap 0 seq_num 997) 2023-08-27T20:52:59.697 INFO:tasks.rados.rados.0.smithi093.stdout:1395: finishing write tid 1 to smithi093153278-266 2023-08-27T20:52:59.697 INFO:tasks.rados.rados.0.smithi093.stdout:1395: finishing write tid 2 to smithi093153278-266 2023-08-27T20:52:59.698 INFO:tasks.rados.rados.0.smithi093.stdout:1395: finishing write tid 3 to smithi093153278-266 2023-08-27T20:52:59.698 INFO:tasks.rados.rados.0.smithi093.stdout:1395: finishing write tid 4 to smithi093153278-266 2023-08-27T20:52:59.698 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 266 v 186 (ObjNum 998 snap 0 seq_num 998) dirty exists 2023-08-27T20:52:59.698 INFO:tasks.rados.rados.0.smithi093.stdout:1395: left oid 266 (ObjNum 998 snap 0 seq_num 998) 2023-08-27T20:52:59.698 INFO:tasks.rados.rados.0.smithi093.stdout:1396: expect (ObjNum 790 snap 0 seq_num 790) 2023-08-27T20:52:59.886 INFO:tasks.rados.rados.0.smithi093.stdout:1397: expect (ObjNum 176 snap 0 seq_num 176) 2023-08-27T20:52:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 bash[140583]: b62a489d313ac45775bca95f1b3b10f16ea882943c62eca5a2f4cd43a1cbca21 2023-08-27T20:52:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 systemd[1]: Started Ceph mgr.x for b79a52c0-451a-11ee-817b-1b7a5f1ba315. 2023-08-27T20:52:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:52:59.794+0000 7fa0defff200 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:52:59.984 INFO:tasks.rados.rados.0.smithi093.stdout:1385: done (13 left) 2023-08-27T20:52:59.984 INFO:tasks.rados.rados.0.smithi093.stdout:1386: done (12 left) 2023-08-27T20:52:59.984 INFO:tasks.rados.rados.0.smithi093.stdout:1387: done (11 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1388: done (10 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1389: done (9 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1390: done (8 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1391: done (7 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1392: done (6 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1393: done (5 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1394: done (4 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1395: done (3 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1396: done (2 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1397: done (1 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1398: done (0 left) 2023-08-27T20:52:59.985 INFO:tasks.rados.rados.0.smithi093.stdout:1399: delete oid 373 current snap is 0 2023-08-27T20:52:59.986 INFO:tasks.rados.rados.0.smithi093.stdout:1399: done (0 left) 2023-08-27T20:52:59.986 INFO:tasks.rados.rados.0.smithi093.stdout:1400: write oid 377 current snap is 0 2023-08-27T20:52:59.986 INFO:tasks.rados.rados.0.smithi093.stdout:1400: seq_num 1001 ranges {423807=566492,1676240=407685,2739004=84803} 2023-08-27T20:52:59.991 INFO:tasks.rados.rados.0.smithi093.stdout:1400: writing smithi093153278-377 from 423807 to 990299 tid 1 2023-08-27T20:52:59.997 INFO:tasks.rados.rados.0.smithi093.stdout:1400: writing smithi093153278-377 from 1676240 to 2083925 tid 2 2023-08-27T20:53:00.001 INFO:tasks.rados.rados.0.smithi093.stdout:1400: writing smithi093153278-377 from 2739004 to 2823807 tid 3 2023-08-27T20:53:00.001 INFO:tasks.rados.rados.0.smithi093.stdout:1401: write oid 440 current snap is 0 2023-08-27T20:53:00.001 INFO:tasks.rados.rados.0.smithi093.stdout:1400: finishing write tid 1 to smithi093153278-377 2023-08-27T20:53:00.001 INFO:tasks.rados.rados.0.smithi093.stdout:1401: seq_num 1002 ranges {440614=641741,1734722=551343} 2023-08-27T20:53:00.007 INFO:tasks.rados.rados.0.smithi093.stdout:1401: writing smithi093153278-440 from 440614 to 1082355 tid 1 2023-08-27T20:53:00.014 INFO:tasks.rados.rados.0.smithi093.stdout:1401: writing smithi093153278-440 from 1734722 to 2286065 tid 2 2023-08-27T20:53:00.014 INFO:tasks.rados.rados.0.smithi093.stdout:1402: write oid 253 current snap is 0 2023-08-27T20:53:00.014 INFO:tasks.rados.rados.0.smithi093.stdout:1400: finishing write tid 2 to smithi093153278-377 2023-08-27T20:53:00.014 INFO:tasks.rados.rados.0.smithi093.stdout:1402: seq_num 1003 ranges {457421=716990,1676851=411354,2677497=179924} 2023-08-27T20:53:00.021 INFO:tasks.rados.rados.0.smithi093.stdout:1402: writing smithi093153278-253 from 457421 to 1174411 tid 1 2023-08-27T20:53:00.025 INFO:tasks.rados.rados.0.smithi093.stdout:1402: writing smithi093153278-253 from 1676851 to 2088205 tid 2 2023-08-27T20:53:00.030 INFO:tasks.rados.rados.0.smithi093.stdout:1402: writing smithi093153278-253 from 2677497 to 2857421 tid 3 2023-08-27T20:53:00.030 INFO:tasks.rados.rados.0.smithi093.stdout:1403: read oid 104 snap -1 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1403: expect (ObjNum 714 snap 0 seq_num 714) 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1404: read oid 279 snap -1 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1404: expect (ObjNum 841 snap 0 seq_num 841) 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1405: read oid 221 snap -1 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1405: expect (ObjNum 663 snap 0 seq_num 663) 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1406: write oid 444 current snap is 0 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1400: finishing write tid 3 to smithi093153278-377 2023-08-27T20:53:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:1406: seq_num 1004 ranges {474228=508592,1451686=555012,2704920=169308} 2023-08-27T20:53:00.036 INFO:tasks.rados.rados.0.smithi093.stdout:1406: writing smithi093153278-444 from 474228 to 982820 tid 1 2023-08-27T20:53:00.042 INFO:tasks.rados.rados.0.smithi093.stdout:1406: writing smithi093153278-444 from 1451686 to 2006698 tid 2 2023-08-27T20:53:00.047 INFO:tasks.rados.rados.0.smithi093.stdout:1406: writing smithi093153278-444 from 2704920 to 2874228 tid 3 2023-08-27T20:53:00.047 INFO:tasks.rados.rados.0.smithi093.stdout:1407: write oid 271 current snap is 0 2023-08-27T20:53:00.047 INFO:tasks.rados.rados.0.smithi093.stdout:1400: finishing write tid 4 to smithi093153278-377 2023-08-27T20:53:00.047 INFO:tasks.rados.rados.0.smithi093.stdout:1400: finishing write tid 5 to smithi093153278-377 2023-08-27T20:53:00.047 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 377 v 212 (ObjNum 1001 snap 0 seq_num 1001) dirty exists 2023-08-27T20:53:00.047 INFO:tasks.rados.rados.0.smithi093.stdout:1400: left oid 377 (ObjNum 1001 snap 0 seq_num 1001) 2023-08-27T20:53:00.048 INFO:tasks.rados.rados.0.smithi093.stdout:1407: seq_num 1005 ranges {491035=583841,1510168=698670,2732343=158692} 2023-08-27T20:53:00.052 INFO:tasks.rados.rados.0.smithi093.stdout:1407: writing smithi093153278-271 from 491035 to 1074876 tid 1 2023-08-27T20:53:00.059 INFO:tasks.rados.rados.0.smithi093.stdout:1407: writing smithi093153278-271 from 1510168 to 2208838 tid 2 2023-08-27T20:53:00.063 INFO:tasks.rados.rados.0.smithi093.stdout:1407: writing smithi093153278-271 from 2732343 to 2891035 tid 3 2023-08-27T20:53:00.063 INFO:tasks.rados.rados.0.smithi093.stdout:1400: done (7 left) 2023-08-27T20:53:00.064 INFO:tasks.rados.rados.0.smithi093.stdout:1408: write oid 126 current snap is 0 2023-08-27T20:53:00.064 INFO:tasks.rados.rados.0.smithi093.stdout:1401: finishing write tid 1 to smithi093153278-440 2023-08-27T20:53:00.064 INFO:tasks.rados.rados.0.smithi093.stdout:1408: seq_num 1006 ranges {507842=659090,1568650=558681,2759766=148076} 2023-08-27T20:53:00.069 INFO:tasks.rados.rados.0.smithi093.stdout:1408: writing smithi093153278-126 from 507842 to 1166932 tid 1 2023-08-27T20:53:00.075 INFO:tasks.rados.rados.0.smithi093.stdout:1408: writing smithi093153278-126 from 1568650 to 2127331 tid 2 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:1408: writing smithi093153278-126 from 2759766 to 2907842 tid 3 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:1409: read oid 278 snap -1 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:1409: expect (ObjNum 734 snap 0 seq_num 734) 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:1410: write oid 35 current snap is 0 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:1401: finishing write tid 2 to smithi093153278-440 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:1401: finishing write tid 3 to smithi093153278-440 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:1401: finishing write tid 4 to smithi093153278-440 2023-08-27T20:53:00.080 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 440 v 220 (ObjNum 1002 snap 0 seq_num 1002) dirty exists 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:1401: left oid 440 (ObjNum 1002 snap 0 seq_num 1002) 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:1402: finishing write tid 1 to smithi093153278-253 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:1402: finishing write tid 2 to smithi093153278-253 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:1402: finishing write tid 3 to smithi093153278-253 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:1402: finishing write tid 4 to smithi093153278-253 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:1402: finishing write tid 5 to smithi093153278-253 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 216 (ObjNum 1003 snap 0 seq_num 1003) dirty exists 2023-08-27T20:53:00.081 INFO:tasks.rados.rados.0.smithi093.stdout:1402: left oid 253 (ObjNum 1003 snap 0 seq_num 1003) 2023-08-27T20:53:00.082 INFO:tasks.rados.rados.0.smithi093.stdout:1403: expect (ObjNum 714 snap 0 seq_num 714) 2023-08-27T20:53:00.161 INFO:tasks.rados.rados.0.smithi093.stdout:1404: expect (ObjNum 841 snap 0 seq_num 841) 2023-08-27T20:53:00.249 INFO:tasks.rados.rados.0.smithi093.stdout:1405: expect (ObjNum 663 snap 0 seq_num 663) 2023-08-27T20:53:00.354 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:52:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:52:59.970+0000 7fa0defff200 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:53:00.355 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:00 smithi136 ceph-mon[113531]: pgmap v84: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.0 MiB/s wr, 91 op/s 2023-08-27T20:53:00.355 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:00 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:00.355 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:00 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:00.355 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:00 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:00.381 INFO:tasks.rados.rados.0.smithi093.stdout:1406: finishing write tid 1 to smithi093153278-444 2023-08-27T20:53:00.381 INFO:tasks.rados.rados.0.smithi093.stdout:1406: finishing write tid 2 to smithi093153278-444 2023-08-27T20:53:00.382 INFO:tasks.rados.rados.0.smithi093.stdout:1406: finishing write tid 3 to smithi093153278-444 2023-08-27T20:53:00.382 INFO:tasks.rados.rados.0.smithi093.stdout:1406: finishing write tid 4 to smithi093153278-444 2023-08-27T20:53:00.382 INFO:tasks.rados.rados.0.smithi093.stdout:1406: finishing write tid 5 to smithi093153278-444 2023-08-27T20:53:00.382 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 444 v 220 (ObjNum 1004 snap 0 seq_num 1004) dirty exists 2023-08-27T20:53:00.382 INFO:tasks.rados.rados.0.smithi093.stdout:1406: left oid 444 (ObjNum 1004 snap 0 seq_num 1004) 2023-08-27T20:53:00.383 INFO:tasks.rados.rados.0.smithi093.stdout:1407: finishing write tid 1 to smithi093153278-271 2023-08-27T20:53:00.383 INFO:tasks.rados.rados.0.smithi093.stdout:1407: finishing write tid 2 to smithi093153278-271 2023-08-27T20:53:00.383 INFO:tasks.rados.rados.0.smithi093.stdout:1410: seq_num 1007 ranges {524649=734339,1910779=702339} 2023-08-27T20:53:00.388 INFO:tasks.rados.rados.0.smithi093.stdout:1410: writing smithi093153278-35 from 524649 to 1258988 tid 1 2023-08-27T20:53:00.396 INFO:tasks.rados.rados.0.smithi093.stdout:1410: writing smithi093153278-35 from 1910779 to 2613118 tid 2 2023-08-27T20:53:00.397 INFO:tasks.rados.rados.0.smithi093.stdout:1401: done (9 left) 2023-08-27T20:53:00.397 INFO:tasks.rados.rados.0.smithi093.stdout:1402: done (8 left) 2023-08-27T20:53:00.397 INFO:tasks.rados.rados.0.smithi093.stdout:1403: done (7 left) 2023-08-27T20:53:00.397 INFO:tasks.rados.rados.0.smithi093.stdout:1404: done (6 left) 2023-08-27T20:53:00.397 INFO:tasks.rados.rados.0.smithi093.stdout:1405: done (5 left) 2023-08-27T20:53:00.398 INFO:tasks.rados.rados.0.smithi093.stdout:1406: done (4 left) 2023-08-27T20:53:00.398 INFO:tasks.rados.rados.0.smithi093.stdout:1411: read oid 439 snap -1 2023-08-27T20:53:00.398 INFO:tasks.rados.rados.0.smithi093.stdout:1411: expect deleted 2023-08-27T20:53:00.398 INFO:tasks.rados.rados.0.smithi093.stdout:1412: write oid 323 current snap is 0 2023-08-27T20:53:00.398 INFO:tasks.rados.rados.0.smithi093.stdout:1407: finishing write tid 3 to smithi093153278-271 2023-08-27T20:53:00.398 INFO:tasks.rados.rados.0.smithi093.stdout:1412: seq_num 1008 ranges {541456=409588,1569261=562350,2814612=126844} 2023-08-27T20:53:00.401 INFO:tasks.rados.rados.0.smithi093.stdout:1412: writing smithi093153278-323 from 541456 to 951044 tid 1 2023-08-27T20:53:00.407 INFO:tasks.rados.rados.0.smithi093.stdout:1412: writing smithi093153278-323 from 1569261 to 2131611 tid 2 2023-08-27T20:53:00.412 INFO:tasks.rados.rados.0.smithi093.stdout:1412: writing smithi093153278-323 from 2814612 to 2941456 tid 3 2023-08-27T20:53:00.412 INFO:tasks.rados.rados.0.smithi093.stdout:1413: write oid 426 current snap is 0 2023-08-27T20:53:00.413 INFO:tasks.rados.rados.0.smithi093.stdout:1413: seq_num 1009 ranges {558263=484837,1627743=706008} 2023-08-27T20:53:00.418 INFO:tasks.rados.rados.0.smithi093.stdout:1413: writing smithi093153278-426 from 558263 to 1043100 tid 1 2023-08-27T20:53:00.426 INFO:tasks.rados.rados.0.smithi093.stdout:1413: writing smithi093153278-426 from 1627743 to 2333751 tid 2 2023-08-27T20:53:00.426 INFO:tasks.rados.rados.0.smithi093.stdout:1414: read oid 285 snap -1 2023-08-27T20:53:00.426 INFO:tasks.rados.rados.0.smithi093.stdout:1414: expect (ObjNum 836 snap 0 seq_num 836) 2023-08-27T20:53:00.426 INFO:tasks.rados.rados.0.smithi093.stdout:1415: write oid 112 current snap is 0 2023-08-27T20:53:00.427 INFO:tasks.rados.rados.0.smithi093.stdout:1407: finishing write tid 4 to smithi093153278-271 2023-08-27T20:53:00.427 INFO:tasks.rados.rados.0.smithi093.stdout:1415: seq_num 1010 ranges {575070=560086,1686225=566019,2869458=105612} 2023-08-27T20:53:00.432 INFO:tasks.rados.rados.0.smithi093.stdout:1415: writing smithi093153278-112 from 575070 to 1135156 tid 1 2023-08-27T20:53:00.438 INFO:tasks.rados.rados.0.smithi093.stdout:1415: writing smithi093153278-112 from 1686225 to 2252244 tid 2 2023-08-27T20:53:00.444 INFO:tasks.rados.rados.0.smithi093.stdout:1415: writing smithi093153278-112 from 2869458 to 2975070 tid 3 2023-08-27T20:53:00.444 INFO:tasks.rados.rados.0.smithi093.stdout:1416: read oid 142 snap -1 2023-08-27T20:53:00.444 INFO:tasks.rados.rados.0.smithi093.stdout:1416: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:00.444 INFO:tasks.rados.rados.0.smithi093.stdout:1417: read oid 494 snap -1 2023-08-27T20:53:00.444 INFO:tasks.rados.rados.0.smithi093.stdout:1417: expect (ObjNum 493 snap 0 seq_num 493) 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1418: read oid 111 snap -1 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1418: expect (ObjNum 705 snap 0 seq_num 705) 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1419: read oid 114 snap -1 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1419: expect (ObjNum 644 snap 0 seq_num 644) 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1420: write oid 470 current snap is 0 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1407: finishing write tid 5 to smithi093153278-271 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 271 v 190 (ObjNum 1005 snap 0 seq_num 1005) dirty exists 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1407: left oid 271 (ObjNum 1005 snap 0 seq_num 1005) 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1408: finishing write tid 1 to smithi093153278-126 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1408: finishing write tid 2 to smithi093153278-126 2023-08-27T20:53:00.445 INFO:tasks.rados.rados.0.smithi093.stdout:1408: finishing write tid 3 to smithi093153278-126 2023-08-27T20:53:00.446 INFO:tasks.rados.rados.0.smithi093.stdout:1408: finishing write tid 4 to smithi093153278-126 2023-08-27T20:53:00.446 INFO:tasks.rados.rados.0.smithi093.stdout:1408: finishing write tid 5 to smithi093153278-126 2023-08-27T20:53:00.446 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 203 (ObjNum 1006 snap 0 seq_num 1006) dirty exists 2023-08-27T20:53:00.446 INFO:tasks.rados.rados.0.smithi093.stdout:1408: left oid 126 (ObjNum 1006 snap 0 seq_num 1006) 2023-08-27T20:53:00.446 INFO:tasks.rados.rados.0.smithi093.stdout:1420: seq_num 1011 ranges {591877=635335,1628354=709677} 2023-08-27T20:53:00.450 INFO:tasks.rados.rados.0.smithi093.stdout:1420: writing smithi093153278-470 from 591877 to 1227212 tid 1 2023-08-27T20:53:00.457 INFO:tasks.rados.rados.0.smithi093.stdout:1420: writing smithi093153278-470 from 1628354 to 2338031 tid 2 2023-08-27T20:53:00.457 INFO:tasks.rados.rados.0.smithi093.stdout:1407: done (13 left) 2023-08-27T20:53:00.457 INFO:tasks.rados.rados.0.smithi093.stdout:1408: done (12 left) 2023-08-27T20:53:00.457 INFO:tasks.rados.rados.0.smithi093.stdout:1421: read oid 183 snap -1 2023-08-27T20:53:00.457 INFO:tasks.rados.rados.0.smithi093.stdout:1421: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:53:00.457 INFO:tasks.rados.rados.0.smithi093.stdout:1422: delete oid 210 current snap is 0 2023-08-27T20:53:00.458 INFO:tasks.rados.rados.0.smithi093.stdout:1409: expect (ObjNum 734 snap 0 seq_num 734) 2023-08-27T20:53:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[111335]: pgmap v84: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.0 MiB/s wr, 91 op/s 2023-08-27T20:53:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[118902]: pgmap v84: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.0 MiB/s wr, 91 op/s 2023-08-27T20:53:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:00 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:00.575 INFO:tasks.rados.rados.0.smithi093.stdout:1410: finishing write tid 1 to smithi093153278-35 2023-08-27T20:53:00.575 INFO:tasks.rados.rados.0.smithi093.stdout:1410: finishing write tid 2 to smithi093153278-35 2023-08-27T20:53:00.575 INFO:tasks.rados.rados.0.smithi093.stdout:1410: finishing write tid 3 to smithi093153278-35 2023-08-27T20:53:00.575 INFO:tasks.rados.rados.0.smithi093.stdout:1410: finishing write tid 4 to smithi093153278-35 2023-08-27T20:53:00.575 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 35 v 189 (ObjNum 1007 snap 0 seq_num 1007) dirty exists 2023-08-27T20:53:00.575 INFO:tasks.rados.rados.0.smithi093.stdout:1410: left oid 35 (ObjNum 1007 snap 0 seq_num 1007) 2023-08-27T20:53:00.575 INFO:tasks.rados.rados.0.smithi093.stdout:1412: finishing write tid 1 to smithi093153278-323 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1412: finishing write tid 2 to smithi093153278-323 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1412: finishing write tid 3 to smithi093153278-323 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1409: done (13 left) 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1410: done (12 left) 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1411: done (11 left) 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1422: done (10 left) 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1423: read oid 247 snap -1 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1423: expect (ObjNum 246 snap 0 seq_num 246) 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1424: write oid 331 current snap is 0 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1412: finishing write tid 4 to smithi093153278-323 2023-08-27T20:53:00.576 INFO:tasks.rados.rados.0.smithi093.stdout:1424: seq_num 1013 ranges {625491=502186,1861671=713346} 2023-08-27T20:53:00.580 INFO:tasks.rados.rados.0.smithi093.stdout:1424: writing smithi093153278-331 from 625491 to 1127677 tid 1 2023-08-27T20:53:00.588 INFO:tasks.rados.rados.0.smithi093.stdout:1424: writing smithi093153278-331 from 1861671 to 2575017 tid 2 2023-08-27T20:53:00.589 INFO:tasks.rados.rados.0.smithi093.stdout:1425: write oid 300 current snap is 0 2023-08-27T20:53:00.589 INFO:tasks.rados.rados.0.smithi093.stdout:1412: finishing write tid 5 to smithi093153278-323 2023-08-27T20:53:00.589 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 323 v 224 (ObjNum 1008 snap 0 seq_num 1008) dirty exists 2023-08-27T20:53:00.589 INFO:tasks.rados.rados.0.smithi093.stdout:1412: left oid 323 (ObjNum 1008 snap 0 seq_num 1008) 2023-08-27T20:53:00.589 INFO:tasks.rados.rados.0.smithi093.stdout:1413: finishing write tid 1 to smithi093153278-426 2023-08-27T20:53:00.589 INFO:tasks.rados.rados.0.smithi093.stdout:1425: seq_num 1014 ranges {642298=577435,1920153=573357,2979150=63148} 2023-08-27T20:53:00.595 INFO:tasks.rados.rados.0.smithi093.stdout:1425: writing smithi093153278-300 from 642298 to 1219733 tid 1 2023-08-27T20:53:00.602 INFO:tasks.rados.rados.0.smithi093.stdout:1425: writing smithi093153278-300 from 1920153 to 2493510 tid 2 2023-08-27T20:53:00.605 INFO:tasks.rados.rados.0.smithi093.stdout:1425: writing smithi093153278-300 from 2979150 to 3042298 tid 3 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:1412: done (12 left) 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:1426: write oid 147 current snap is 0 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:1413: finishing write tid 2 to smithi093153278-426 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:1413: finishing write tid 3 to smithi093153278-426 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:1413: finishing write tid 4 to smithi093153278-426 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 426 v 227 (ObjNum 1009 snap 0 seq_num 1009) dirty exists 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:1413: left oid 426 (ObjNum 1009 snap 0 seq_num 1009) 2023-08-27T20:53:00.606 INFO:tasks.rados.rados.0.smithi093.stdout:1426: seq_num 1015 ranges {659105=652684,1862282=717015} 2023-08-27T20:53:00.612 INFO:tasks.rados.rados.0.smithi093.stdout:1426: writing smithi093153278-147 from 659105 to 1311789 tid 1 2023-08-27T20:53:00.619 INFO:tasks.rados.rados.0.smithi093.stdout:1426: writing smithi093153278-147 from 1862282 to 2579297 tid 2 2023-08-27T20:53:00.620 INFO:tasks.rados.rados.0.smithi093.stdout:1413: done (12 left) 2023-08-27T20:53:00.620 INFO:tasks.rados.rados.0.smithi093.stdout:1427: write oid 453 current snap is 0 2023-08-27T20:53:00.620 INFO:tasks.rados.rados.0.smithi093.stdout:1414: expect (ObjNum 836 snap 0 seq_num 836) 2023-08-27T20:53:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:00.353+0000 7fa0defff200 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:53:00.703 INFO:tasks.rados.rados.0.smithi093.stdout:1415: finishing write tid 1 to smithi093153278-112 2023-08-27T20:53:00.703 INFO:tasks.rados.rados.0.smithi093.stdout:1415: finishing write tid 2 to smithi093153278-112 2023-08-27T20:53:00.703 INFO:tasks.rados.rados.0.smithi093.stdout:1417: expect (ObjNum 493 snap 0 seq_num 493) 2023-08-27T20:53:00.776 INFO:tasks.rados.rados.0.smithi093.stdout:1415: finishing write tid 3 to smithi093153278-112 2023-08-27T20:53:00.776 INFO:tasks.rados.rados.0.smithi093.stdout:1418: expect (ObjNum 705 snap 0 seq_num 705) 2023-08-27T20:53:00.956 INFO:tasks.rados.rados.0.smithi093.stdout:1416: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:01.036 INFO:tasks.rados.rados.0.smithi093.stdout:1415: finishing write tid 4 to smithi093153278-112 2023-08-27T20:53:01.036 INFO:tasks.rados.rados.0.smithi093.stdout:1419: expect (ObjNum 644 snap 0 seq_num 644) 2023-08-27T20:53:01.060 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:00.906+0000 7fa0defff200 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:53:01.161 INFO:tasks.rados.rados.0.smithi093.stdout:1415: finishing write tid 5 to smithi093153278-112 2023-08-27T20:53:01.161 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 112 v 201 (ObjNum 1010 snap 0 seq_num 1010) dirty exists 2023-08-27T20:53:01.161 INFO:tasks.rados.rados.0.smithi093.stdout:1415: left oid 112 (ObjNum 1010 snap 0 seq_num 1010) 2023-08-27T20:53:01.161 INFO:tasks.rados.rados.0.smithi093.stdout:1420: finishing write tid 1 to smithi093153278-470 2023-08-27T20:53:01.161 INFO:tasks.rados.rados.0.smithi093.stdout:1420: finishing write tid 2 to smithi093153278-470 2023-08-27T20:53:01.162 INFO:tasks.rados.rados.0.smithi093.stdout:1427: seq_num 1016 ranges {675912=727933,1920764=460673,2801290=274622} 2023-08-27T20:53:01.168 INFO:tasks.rados.rados.0.smithi093.stdout:1427: writing smithi093153278-453 from 675912 to 1403845 tid 1 2023-08-27T20:53:01.173 INFO:tasks.rados.rados.0.smithi093.stdout:1427: writing smithi093153278-453 from 1920764 to 2381437 tid 2 2023-08-27T20:53:01.178 INFO:tasks.rados.rados.0.smithi093.stdout:1427: writing smithi093153278-453 from 2801290 to 3075912 tid 3 2023-08-27T20:53:01.178 INFO:tasks.rados.rados.0.smithi093.stdout:1414: done (12 left) 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1415: done (11 left) 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1416: done (10 left) 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1417: done (9 left) 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1418: done (8 left) 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1419: done (7 left) 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1428: read oid 409 snap -1 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1428: expect (ObjNum 794 snap 0 seq_num 794) 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1429: write oid 396 current snap is 0 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1420: finishing write tid 3 to smithi093153278-470 2023-08-27T20:53:01.179 INFO:tasks.rados.rados.0.smithi093.stdout:1420: finishing write tid 4 to smithi093153278-470 2023-08-27T20:53:01.180 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 470 v 272 (ObjNum 1011 snap 0 seq_num 1011) dirty exists 2023-08-27T20:53:01.180 INFO:tasks.rados.rados.0.smithi093.stdout:1420: left oid 470 (ObjNum 1011 snap 0 seq_num 1011) 2023-08-27T20:53:01.180 INFO:tasks.rados.rados.0.smithi093.stdout:1421: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:53:01.274 INFO:tasks.rados.rados.0.smithi093.stdout:1423: expect (ObjNum 246 snap 0 seq_num 246) 2023-08-27T20:53:01.341 INFO:tasks.rados.rados.0.smithi093.stdout:1424: finishing write tid 1 to smithi093153278-331 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1424: finishing write tid 2 to smithi093153278-331 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1424: finishing write tid 3 to smithi093153278-331 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1424: finishing write tid 4 to smithi093153278-331 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 331 v 276 (ObjNum 1013 snap 0 seq_num 1013) dirty exists 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1424: left oid 331 (ObjNum 1013 snap 0 seq_num 1013) 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1425: finishing write tid 1 to smithi093153278-300 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1425: finishing write tid 2 to smithi093153278-300 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1425: finishing write tid 3 to smithi093153278-300 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1425: finishing write tid 4 to smithi093153278-300 2023-08-27T20:53:01.342 INFO:tasks.rados.rados.0.smithi093.stdout:1425: finishing write tid 5 to smithi093153278-300 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 300 v 176 (ObjNum 1014 snap 0 seq_num 1014) dirty exists 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1425: left oid 300 (ObjNum 1014 snap 0 seq_num 1014) 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1426: finishing write tid 1 to smithi093153278-147 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1426: finishing write tid 2 to smithi093153278-147 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1426: finishing write tid 3 to smithi093153278-147 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1426: finishing write tid 4 to smithi093153278-147 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 147 v 198 (ObjNum 1015 snap 0 seq_num 1015) dirty exists 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1426: left oid 147 (ObjNum 1015 snap 0 seq_num 1015) 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1427: finishing write tid 1 to smithi093153278-453 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1427: finishing write tid 2 to smithi093153278-453 2023-08-27T20:53:01.343 INFO:tasks.rados.rados.0.smithi093.stdout:1427: finishing write tid 3 to smithi093153278-453 2023-08-27T20:53:01.344 INFO:tasks.rados.rados.0.smithi093.stdout:1427: finishing write tid 4 to smithi093153278-453 2023-08-27T20:53:01.344 INFO:tasks.rados.rados.0.smithi093.stdout:1427: finishing write tid 5 to smithi093153278-453 2023-08-27T20:53:01.344 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 453 v 205 (ObjNum 1016 snap 0 seq_num 1016) dirty exists 2023-08-27T20:53:01.344 INFO:tasks.rados.rados.0.smithi093.stdout:1427: left oid 453 (ObjNum 1016 snap 0 seq_num 1016) 2023-08-27T20:53:01.344 INFO:tasks.rados.rados.0.smithi093.stdout:1429: seq_num 1017 ranges {692719=403182,1579246=720684,2828713=264006} 2023-08-27T20:53:01.347 INFO:tasks.rados.rados.0.smithi093.stdout:1429: writing smithi093153278-396 from 692719 to 1095901 tid 1 2023-08-27T20:53:01.353 INFO:tasks.rados.rados.0.smithi093.stdout:1429: writing smithi093153278-396 from 1579246 to 2299930 tid 2 2023-08-27T20:53:01.358 INFO:tasks.rados.rados.0.smithi093.stdout:1429: writing smithi093153278-396 from 2828713 to 3092719 tid 3 2023-08-27T20:53:01.358 INFO:tasks.rados.rados.0.smithi093.stdout:1420: done (8 left) 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1421: done (7 left) 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1423: done (6 left) 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1424: done (5 left) 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1425: done (4 left) 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1426: done (3 left) 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1427: done (2 left) 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1430: read oid 106 snap -1 2023-08-27T20:53:01.359 INFO:tasks.rados.rados.0.smithi093.stdout:1430: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:53:01.360 INFO:tasks.rados.rados.0.smithi093.stdout:1431: write oid 314 current snap is 0 2023-08-27T20:53:01.360 INFO:tasks.rados.rados.0.smithi093.stdout:1428: expect (ObjNum 794 snap 0 seq_num 794) 2023-08-27T20:53:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3154276338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:01.161+0000 7fa0defff200 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:53:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3154276338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3154276338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:01.492 INFO:tasks.rados.rados.0.smithi093.stdout:1429: finishing write tid 1 to smithi093153278-396 2023-08-27T20:53:01.493 INFO:tasks.rados.rados.0.smithi093.stdout:1429: finishing write tid 2 to smithi093153278-396 2023-08-27T20:53:01.493 INFO:tasks.rados.rados.0.smithi093.stdout:1429: finishing write tid 3 to smithi093153278-396 2023-08-27T20:53:01.493 INFO:tasks.rados.rados.0.smithi093.stdout:1429: finishing write tid 4 to smithi093153278-396 2023-08-27T20:53:01.493 INFO:tasks.rados.rados.0.smithi093.stdout:1429: finishing write tid 5 to smithi093153278-396 2023-08-27T20:53:01.493 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 396 v 221 (ObjNum 1017 snap 0 seq_num 1017) dirty exists 2023-08-27T20:53:01.493 INFO:tasks.rados.rados.0.smithi093.stdout:1429: left oid 396 (ObjNum 1017 snap 0 seq_num 1017) 2023-08-27T20:53:01.493 INFO:tasks.rados.rados.0.smithi093.stdout:1430: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:53:01.625 INFO:tasks.rados.rados.0.smithi093.stdout:1431: seq_num 1018 ranges {709526=478431,1637728=464342,2856136=253390} 2023-08-27T20:53:01.631 INFO:tasks.rados.rados.0.smithi093.stdout:1431: writing smithi093153278-314 from 709526 to 1187957 tid 1 2023-08-27T20:53:01.636 INFO:tasks.rados.rados.0.smithi093.stdout:1431: writing smithi093153278-314 from 1637728 to 2102070 tid 2 2023-08-27T20:53:01.642 INFO:tasks.rados.rados.0.smithi093.stdout:1431: writing smithi093153278-314 from 2856136 to 3109526 tid 3 2023-08-27T20:53:01.642 INFO:tasks.rados.rados.0.smithi093.stdout:1428: done (3 left) 2023-08-27T20:53:01.643 INFO:tasks.rados.rados.0.smithi093.stdout:1429: done (2 left) 2023-08-27T20:53:01.643 INFO:tasks.rados.rados.0.smithi093.stdout:1430: done (1 left) 2023-08-27T20:53:01.643 INFO:tasks.rados.rados.0.smithi093.stdout:1432: delete oid 217 current snap is 0 2023-08-27T20:53:01.643 INFO:tasks.rados.rados.0.smithi093.stdout:1431: finishing write tid 1 to smithi093153278-314 2023-08-27T20:53:01.643 INFO:tasks.rados.rados.0.smithi093.stdout:1431: finishing write tid 2 to smithi093153278-314 2023-08-27T20:53:01.643 INFO:tasks.rados.rados.0.smithi093.stdout:1432: done (1 left) 2023-08-27T20:53:01.644 INFO:tasks.rados.rados.0.smithi093.stdout:1433: write oid 491 current snap is 0 2023-08-27T20:53:01.644 INFO:tasks.rados.rados.0.smithi093.stdout:1433: seq_num 1020 ranges {743140=745282,2154692=468011} 2023-08-27T20:53:01.651 INFO:tasks.rados.rados.0.smithi093.stdout:1433: writing smithi093153278-491 from 743140 to 1488422 tid 1 2023-08-27T20:53:01.658 INFO:tasks.rados.rados.0.smithi093.stdout:1433: writing smithi093153278-491 from 2154692 to 2622703 tid 2 2023-08-27T20:53:01.658 INFO:tasks.rados.rados.0.smithi093.stdout:1434: delete oid 387 current snap is 0 2023-08-27T20:53:01.658 INFO:tasks.rados.rados.0.smithi093.stdout:1431: finishing write tid 3 to smithi093153278-314 2023-08-27T20:53:01.658 INFO:tasks.rados.rados.0.smithi093.stdout:1431: finishing write tid 4 to smithi093153278-314 2023-08-27T20:53:01.659 INFO:tasks.rados.rados.0.smithi093.stdout:1431: finishing write tid 5 to smithi093153278-314 2023-08-27T20:53:01.659 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 314 v 247 (ObjNum 1018 snap 0 seq_num 1018) dirty exists 2023-08-27T20:53:01.659 INFO:tasks.rados.rados.0.smithi093.stdout:1431: left oid 314 (ObjNum 1018 snap 0 seq_num 1018) 2023-08-27T20:53:01.659 INFO:tasks.rados.rados.0.smithi093.stdout:1433: finishing write tid 1 to smithi093153278-491 2023-08-27T20:53:01.660 INFO:tasks.rados.rados.0.smithi093.stdout:1431: done (2 left) 2023-08-27T20:53:01.660 INFO:tasks.rados.rados.0.smithi093.stdout:1434: done (1 left) 2023-08-27T20:53:01.660 INFO:tasks.rados.rados.0.smithi093.stdout:1435: read oid 416 snap -1 2023-08-27T20:53:01.660 INFO:tasks.rados.rados.0.smithi093.stdout:1435: expect (ObjNum 415 snap 0 seq_num 415) 2023-08-27T20:53:01.660 INFO:tasks.rados.rados.0.smithi093.stdout:1436: write oid 63 current snap is 0 2023-08-27T20:53:01.660 INFO:tasks.rados.rados.0.smithi093.stdout:1436: seq_num 1022 ranges {776754=495780,1871656=471680,2965828=210926} 2023-08-27T20:53:01.667 INFO:tasks.rados.rados.0.smithi093.stdout:1436: writing smithi093153278-63 from 776754 to 1272534 tid 1 2023-08-27T20:53:01.673 INFO:tasks.rados.rados.0.smithi093.stdout:1436: writing smithi093153278-63 from 1871656 to 2343336 tid 2 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1436: writing smithi093153278-63 from 2965828 to 3176754 tid 3 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1437: write oid 262 current snap is 0 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1433: finishing write tid 2 to smithi093153278-491 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1433: finishing write tid 3 to smithi093153278-491 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1433: finishing write tid 4 to smithi093153278-491 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 491 v 230 (ObjNum 1020 snap 0 seq_num 1020) dirty exists 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1433: left oid 491 (ObjNum 1020 snap 0 seq_num 1020) 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1436: finishing write tid 1 to smithi093153278-63 2023-08-27T20:53:01.678 INFO:tasks.rados.rados.0.smithi093.stdout:1435: expect (ObjNum 415 snap 0 seq_num 415) 2023-08-27T20:53:01.773 INFO:tasks.rados.rados.0.smithi093.stdout:1436: finishing write tid 2 to smithi093153278-63 2023-08-27T20:53:01.774 INFO:tasks.rados.rados.0.smithi093.stdout:1436: finishing write tid 3 to smithi093153278-63 2023-08-27T20:53:01.774 INFO:tasks.rados.rados.0.smithi093.stdout:1436: finishing write tid 4 to smithi093153278-63 2023-08-27T20:53:01.774 INFO:tasks.rados.rados.0.smithi093.stdout:1437: seq_num 1023 ranges {793561=571029,1813785=731691,2993251=200310} 2023-08-27T20:53:01.782 INFO:tasks.rados.rados.0.smithi093.stdout:1437: writing smithi093153278-262 from 793561 to 1364590 tid 1 2023-08-27T20:53:01.789 INFO:tasks.rados.rados.0.smithi093.stdout:1437: writing smithi093153278-262 from 1813785 to 2545476 tid 2 2023-08-27T20:53:01.793 INFO:tasks.rados.rados.0.smithi093.stdout:1437: writing smithi093153278-262 from 2993251 to 3193561 tid 3 2023-08-27T20:53:01.793 INFO:tasks.rados.rados.0.smithi093.stdout:1433: done (3 left) 2023-08-27T20:53:01.793 INFO:tasks.rados.rados.0.smithi093.stdout:1435: done (2 left) 2023-08-27T20:53:01.793 INFO:tasks.rados.rados.0.smithi093.stdout:1438: write oid 238 current snap is 0 2023-08-27T20:53:01.793 INFO:tasks.rados.rados.0.smithi093.stdout:1436: finishing write tid 5 to smithi093153278-63 2023-08-27T20:53:01.794 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 63 v 202 (ObjNum 1022 snap 0 seq_num 1022) dirty exists 2023-08-27T20:53:01.794 INFO:tasks.rados.rados.0.smithi093.stdout:1436: left oid 63 (ObjNum 1022 snap 0 seq_num 1022) 2023-08-27T20:53:01.794 INFO:tasks.rados.rados.0.smithi093.stdout:1437: finishing write tid 1 to smithi093153278-262 2023-08-27T20:53:01.794 INFO:tasks.rados.rados.0.smithi093.stdout:1438: seq_num 1024 ranges {410368=646278,1472267=475349,2504321=405600} 2023-08-27T20:53:01.798 INFO:tasks.rados.rados.0.smithi093.stdout:1438: writing smithi093153278-238 from 410368 to 1056646 tid 1 2023-08-27T20:53:01.804 INFO:tasks.rados.rados.0.smithi093.stdout:1438: writing smithi093153278-238 from 1472267 to 1947616 tid 2 2023-08-27T20:53:01.809 INFO:tasks.rados.rados.0.smithi093.stdout:1438: writing smithi093153278-238 from 2504321 to 2909921 tid 3 2023-08-27T20:53:01.809 INFO:tasks.rados.rados.0.smithi093.stdout:1436: done (2 left) 2023-08-27T20:53:01.809 INFO:tasks.rados.rados.0.smithi093.stdout:1439: write oid 321 current snap is 0 2023-08-27T20:53:01.810 INFO:tasks.rados.rados.0.smithi093.stdout:1437: finishing write tid 2 to smithi093153278-262 2023-08-27T20:53:01.810 INFO:tasks.rados.rados.0.smithi093.stdout:1439: seq_num 1025 ranges {427175=721527,1930749=735360} 2023-08-27T20:53:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:1439: writing smithi093153278-321 from 427175 to 1148702 tid 1 2023-08-27T20:53:01.824 INFO:tasks.rados.rados.0.smithi093.stdout:1439: writing smithi093153278-321 from 1930749 to 2666109 tid 2 2023-08-27T20:53:01.825 INFO:tasks.rados.rados.0.smithi093.stdout:1440: write oid 494 current snap is 0 2023-08-27T20:53:01.825 INFO:tasks.rados.rados.0.smithi093.stdout:1437: finishing write tid 3 to smithi093153278-262 2023-08-27T20:53:01.825 INFO:tasks.rados.rados.0.smithi093.stdout:1440: seq_num 1026 ranges {443982=796776,1989231=479018,2959167=284815} 2023-08-27T20:53:01.831 INFO:tasks.rados.rados.0.smithi093.stdout:1440: writing smithi093153278-494 from 443982 to 1240758 tid 1 2023-08-27T20:53:01.838 INFO:tasks.rados.rados.0.smithi093.stdout:1440: writing smithi093153278-494 from 1989231 to 2468249 tid 2 2023-08-27T20:53:01.843 INFO:tasks.rados.rados.0.smithi093.stdout:1440: writing smithi093153278-494 from 2959167 to 3243982 tid 3 2023-08-27T20:53:01.843 INFO:tasks.rados.rados.0.smithi093.stdout:1441: read oid 110 snap -1 2023-08-27T20:53:01.843 INFO:tasks.rados.rados.0.smithi093.stdout:1441: expect (ObjNum 109 snap 0 seq_num 109) 2023-08-27T20:53:01.844 INFO:tasks.rados.rados.0.smithi093.stdout:1442: read oid 332 snap -1 2023-08-27T20:53:01.844 INFO:tasks.rados.rados.0.smithi093.stdout:1442: expect (ObjNum 837 snap 0 seq_num 837) 2023-08-27T20:53:01.844 INFO:tasks.rados.rados.0.smithi093.stdout:1443: write oid 364 current snap is 0 2023-08-27T20:53:01.844 INFO:tasks.rados.rados.0.smithi093.stdout:1437: finishing write tid 4 to smithi093153278-262 2023-08-27T20:53:01.844 INFO:tasks.rados.rados.0.smithi093.stdout:1443: seq_num 1027 ranges {460789=588378,1764066=739029,3219296=41493} 2023-08-27T20:53:01.849 INFO:tasks.rados.rados.0.smithi093.stdout:1443: writing smithi093153278-364 from 460789 to 1049167 tid 1 2023-08-27T20:53:01.857 INFO:tasks.rados.rados.0.smithi093.stdout:1443: writing smithi093153278-364 from 1764066 to 2503095 tid 2 2023-08-27T20:53:01.861 INFO:tasks.rados.rados.0.smithi093.stdout:1443: writing smithi093153278-364 from 3219296 to 3260789 tid 3 2023-08-27T20:53:01.862 INFO:tasks.rados.rados.0.smithi093.stdout:1444: write oid 163 current snap is 0 2023-08-27T20:53:01.862 INFO:tasks.rados.rados.0.smithi093.stdout:1437: finishing write tid 5 to smithi093153278-262 2023-08-27T20:53:01.862 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 262 v 226 (ObjNum 1023 snap 0 seq_num 1023) dirty exists 2023-08-27T20:53:01.862 INFO:tasks.rados.rados.0.smithi093.stdout:1437: left oid 262 (ObjNum 1023 snap 0 seq_num 1023) 2023-08-27T20:53:01.862 INFO:tasks.rados.rados.0.smithi093.stdout:1438: finishing write tid 1 to smithi093153278-238 2023-08-27T20:53:01.862 INFO:tasks.rados.rados.0.smithi093.stdout:1438: finishing write tid 2 to smithi093153278-238 2023-08-27T20:53:01.862 INFO:tasks.rados.rados.0.smithi093.stdout:1444: seq_num 1028 ranges {477596=663627,1706195=482687,2614013=567041} 2023-08-27T20:53:01.867 INFO:tasks.rados.rados.0.smithi093.stdout:1444: writing smithi093153278-163 from 477596 to 1141223 tid 1 2023-08-27T20:53:01.873 INFO:tasks.rados.rados.0.smithi093.stdout:1444: writing smithi093153278-163 from 1706195 to 2188882 tid 2 2023-08-27T20:53:01.880 INFO:tasks.rados.rados.0.smithi093.stdout:1444: writing smithi093153278-163 from 2614013 to 3181054 tid 3 2023-08-27T20:53:01.880 INFO:tasks.rados.rados.0.smithi093.stdout:1437: done (7 left) 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:1445: read oid 253 snap -1 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:1445: expect (ObjNum 1003 snap 0 seq_num 1003) 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:1446: delete oid 258 current snap is 0 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:1438: finishing write tid 3 to smithi093153278-238 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:1438: finishing write tid 4 to smithi093153278-238 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:1438: finishing write tid 5 to smithi093153278-238 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 238 v 280 (ObjNum 1024 snap 0 seq_num 1024) dirty exists 2023-08-27T20:53:01.881 INFO:tasks.rados.rados.0.smithi093.stdout:1438: left oid 238 (ObjNum 1024 snap 0 seq_num 1024) 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:1439: finishing write tid 1 to smithi093153278-321 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:1439: finishing write tid 2 to smithi093153278-321 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:1439: finishing write tid 3 to smithi093153278-321 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:1439: finishing write tid 4 to smithi093153278-321 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 321 v 229 (ObjNum 1025 snap 0 seq_num 1025) dirty exists 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:1439: left oid 321 (ObjNum 1025 snap 0 seq_num 1025) 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:1440: finishing write tid 1 to smithi093153278-494 2023-08-27T20:53:01.882 INFO:tasks.rados.rados.0.smithi093.stdout:1440: finishing write tid 2 to smithi093153278-494 2023-08-27T20:53:01.883 INFO:tasks.rados.rados.0.smithi093.stdout:1440: finishing write tid 3 to smithi093153278-494 2023-08-27T20:53:01.883 INFO:tasks.rados.rados.0.smithi093.stdout:1440: finishing write tid 4 to smithi093153278-494 2023-08-27T20:53:01.883 INFO:tasks.rados.rados.0.smithi093.stdout:1440: finishing write tid 5 to smithi093153278-494 2023-08-27T20:53:01.883 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 494 v 194 (ObjNum 1026 snap 0 seq_num 1026) dirty exists 2023-08-27T20:53:01.883 INFO:tasks.rados.rados.0.smithi093.stdout:1440: left oid 494 (ObjNum 1026 snap 0 seq_num 1026) 2023-08-27T20:53:01.883 INFO:tasks.rados.rados.0.smithi093.stdout:1442: expect (ObjNum 837 snap 0 seq_num 837) 2023-08-27T20:53:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:01.627+0000 7fa0defff200 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:53:01.967 INFO:tasks.rados.rados.0.smithi093.stdout:1443: finishing write tid 1 to smithi093153278-364 2023-08-27T20:53:01.967 INFO:tasks.rados.rados.0.smithi093.stdout:1441: expect (ObjNum 109 snap 0 seq_num 109) 2023-08-27T20:53:02.096 INFO:tasks.rados.rados.0.smithi093.stdout:1443: finishing write tid 2 to smithi093153278-364 2023-08-27T20:53:02.096 INFO:tasks.rados.rados.0.smithi093.stdout:1438: done (8 left) 2023-08-27T20:53:02.096 INFO:tasks.rados.rados.0.smithi093.stdout:1439: done (7 left) 2023-08-27T20:53:02.096 INFO:tasks.rados.rados.0.smithi093.stdout:1440: done (6 left) 2023-08-27T20:53:02.096 INFO:tasks.rados.rados.0.smithi093.stdout:1441: done (5 left) 2023-08-27T20:53:02.096 INFO:tasks.rados.rados.0.smithi093.stdout:1442: done (4 left) 2023-08-27T20:53:02.096 INFO:tasks.rados.rados.0.smithi093.stdout:1446: done (3 left) 2023-08-27T20:53:02.097 INFO:tasks.rados.rados.0.smithi093.stdout:1447: read oid 167 snap -1 2023-08-27T20:53:02.097 INFO:tasks.rados.rados.0.smithi093.stdout:1447: expect (ObjNum 740 snap 0 seq_num 740) 2023-08-27T20:53:02.097 INFO:tasks.rados.rados.0.smithi093.stdout:1448: write oid 155 current snap is 0 2023-08-27T20:53:02.097 INFO:tasks.rados.rados.0.smithi093.stdout:1443: finishing write tid 3 to smithi093153278-364 2023-08-27T20:53:02.097 INFO:tasks.rados.rados.0.smithi093.stdout:1443: finishing write tid 4 to smithi093153278-364 2023-08-27T20:53:02.097 INFO:tasks.rados.rados.0.smithi093.stdout:1448: seq_num 1030 ranges {511210=414125,1423159=486356,2668859=589585} 2023-08-27T20:53:02.101 INFO:tasks.rados.rados.0.smithi093.stdout:1448: writing smithi093153278-155 from 511210 to 925335 tid 1 2023-08-27T20:53:02.106 INFO:tasks.rados.rados.0.smithi093.stdout:1448: writing smithi093153278-155 from 1423159 to 1909515 tid 2 2023-08-27T20:53:02.114 INFO:tasks.rados.rados.0.smithi093.stdout:1448: writing smithi093153278-155 from 2668859 to 3258444 tid 3 2023-08-27T20:53:02.114 INFO:tasks.rados.rados.0.smithi093.stdout:1449: write oid 395 current snap is 0 2023-08-27T20:53:02.114 INFO:tasks.rados.rados.0.smithi093.stdout:1449: seq_num 1031 ranges {528017=489374,1481641=630014,2696282=400857} 2023-08-27T20:53:02.119 INFO:tasks.rados.rados.0.smithi093.stdout:1449: writing smithi093153278-395 from 528017 to 1017391 tid 1 2023-08-27T20:53:02.126 INFO:tasks.rados.rados.0.smithi093.stdout:1449: writing smithi093153278-395 from 1481641 to 2111655 tid 2 2023-08-27T20:53:02.131 INFO:tasks.rados.rados.0.smithi093.stdout:1449: writing smithi093153278-395 from 2696282 to 3097139 tid 3 2023-08-27T20:53:02.132 INFO:tasks.rados.rados.0.smithi093.stdout:1450: write oid 260 current snap is 0 2023-08-27T20:53:02.132 INFO:tasks.rados.rados.0.smithi093.stdout:1443: finishing write tid 5 to smithi093153278-364 2023-08-27T20:53:02.132 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 364 v 198 (ObjNum 1027 snap 0 seq_num 1027) dirty exists 2023-08-27T20:53:02.132 INFO:tasks.rados.rados.0.smithi093.stdout:1443: left oid 364 (ObjNum 1027 snap 0 seq_num 1027) 2023-08-27T20:53:02.132 INFO:tasks.rados.rados.0.smithi093.stdout:1444: finishing write tid 1 to smithi093153278-163 2023-08-27T20:53:02.132 INFO:tasks.rados.rados.0.smithi093.stdout:1444: finishing write tid 2 to smithi093153278-163 2023-08-27T20:53:02.132 INFO:tasks.rados.rados.0.smithi093.stdout:1450: seq_num 1032 ranges {544824=564623,1823770=490025,3007352=337472} 2023-08-27T20:53:02.137 INFO:tasks.rados.rados.0.smithi093.stdout:1450: writing smithi093153278-260 from 544824 to 1109447 tid 1 2023-08-27T20:53:02.144 INFO:tasks.rados.rados.0.smithi093.stdout:1450: writing smithi093153278-260 from 1823770 to 2313795 tid 2 2023-08-27T20:53:02.150 INFO:tasks.rados.rados.0.smithi093.stdout:1450: writing smithi093153278-260 from 3007352 to 3344824 tid 3 2023-08-27T20:53:02.150 INFO:tasks.rados.rados.0.smithi093.stdout:1443: done (6 left) 2023-08-27T20:53:02.150 INFO:tasks.rados.rados.0.smithi093.stdout:1451: write oid 187 current snap is 0 2023-08-27T20:53:02.150 INFO:tasks.rados.rados.0.smithi093.stdout:1451: seq_num 1033 ranges {561631=639872,1882252=633683,3034775=326856} 2023-08-27T20:53:02.156 INFO:tasks.rados.rados.0.smithi093.stdout:1451: writing smithi093153278-187 from 561631 to 1201503 tid 1 2023-08-27T20:53:02.164 INFO:tasks.rados.rados.0.smithi093.stdout:1451: writing smithi093153278-187 from 1882252 to 2515935 tid 2 2023-08-27T20:53:02.168 INFO:tasks.rados.rados.0.smithi093.stdout:1451: writing smithi093153278-187 from 3034775 to 3361631 tid 3 2023-08-27T20:53:02.169 INFO:tasks.rados.rados.0.smithi093.stdout:1452: delete oid 33 current snap is 0 2023-08-27T20:53:02.169 INFO:tasks.rados.rados.0.smithi093.stdout:1445: expect (ObjNum 1003 snap 0 seq_num 1003) 2023-08-27T20:53:02.263 INFO:tasks.rados.rados.0.smithi093.stdout:1444: finishing write tid 3 to smithi093153278-163 2023-08-27T20:53:02.263 INFO:tasks.rados.rados.0.smithi093.stdout:1444: finishing write tid 4 to smithi093153278-163 2023-08-27T20:53:02.263 INFO:tasks.rados.rados.0.smithi093.stdout:1444: finishing write tid 5 to smithi093153278-163 2023-08-27T20:53:02.263 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 163 v 189 (ObjNum 1028 snap 0 seq_num 1028) dirty exists 2023-08-27T20:53:02.263 INFO:tasks.rados.rados.0.smithi093.stdout:1444: left oid 163 (ObjNum 1028 snap 0 seq_num 1028) 2023-08-27T20:53:02.263 INFO:tasks.rados.rados.0.smithi093.stdout:1444: done (7 left) 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1445: done (6 left) 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1452: done (5 left) 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1453: read oid 195 snap -1 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1453: expect (ObjNum 967 snap 0 seq_num 967) 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1454: read oid 300 snap -1 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1454: expect (ObjNum 1014 snap 0 seq_num 1014) 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1455: read oid 53 snap -1 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1455: expect (ObjNum 860 snap 0 seq_num 860) 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1456: read oid 386 snap -1 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1456: expect deleted 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1457: read oid 183 snap -1 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1457: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:53:02.264 INFO:tasks.rados.rados.0.smithi093.stdout:1458: write oid 219 current snap is 0 2023-08-27T20:53:02.265 INFO:tasks.rados.rados.0.smithi093.stdout:1447: expect (ObjNum 740 snap 0 seq_num 740) 2023-08-27T20:53:02.407 INFO:tasks.rados.rados.0.smithi093.stdout:1448: finishing write tid 1 to smithi093153278-155 2023-08-27T20:53:02.407 INFO:tasks.rados.rados.0.smithi093.stdout:1448: finishing write tid 2 to smithi093153278-155 2023-08-27T20:53:02.408 INFO:tasks.rados.rados.0.smithi093.stdout:1448: finishing write tid 3 to smithi093153278-155 2023-08-27T20:53:02.408 INFO:tasks.rados.rados.0.smithi093.stdout:1448: finishing write tid 4 to smithi093153278-155 2023-08-27T20:53:02.408 INFO:tasks.rados.rados.0.smithi093.stdout:1448: finishing write tid 5 to smithi093153278-155 2023-08-27T20:53:02.408 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 155 v 234 (ObjNum 1030 snap 0 seq_num 1030) dirty exists 2023-08-27T20:53:02.408 INFO:tasks.rados.rados.0.smithi093.stdout:1448: left oid 155 (ObjNum 1030 snap 0 seq_num 1030) 2023-08-27T20:53:02.408 INFO:tasks.rados.rados.0.smithi093.stdout:1449: finishing write tid 1 to smithi093153278-395 2023-08-27T20:53:02.408 INFO:tasks.rados.rados.0.smithi093.stdout:1458: seq_num 1035 ranges {595245=506723,1715569=637352,2805974=562298} 2023-08-27T20:53:02.413 INFO:tasks.rados.rados.0.smithi093.stdout:1458: writing smithi093153278-219 from 595245 to 1101968 tid 1 2023-08-27T20:53:02.420 INFO:tasks.rados.rados.0.smithi093.stdout:1458: writing smithi093153278-219 from 1715569 to 2352921 tid 2 2023-08-27T20:53:02.426 INFO:tasks.rados.rados.0.smithi093.stdout:1458: writing smithi093153278-219 from 2805974 to 3368272 tid 3 2023-08-27T20:53:02.426 INFO:tasks.rados.rados.0.smithi093.stdout:1447: done (10 left) 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1448: done (9 left) 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1459: read oid 354 snap -1 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1459: expect (ObjNum 805 snap 0 seq_num 805) 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1460: delete oid 264 current snap is 0 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1449: finishing write tid 2 to smithi093153278-395 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1449: finishing write tid 3 to smithi093153278-395 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1449: finishing write tid 4 to smithi093153278-395 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1449: finishing write tid 5 to smithi093153278-395 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 284 (ObjNum 1031 snap 0 seq_num 1031) dirty exists 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1449: left oid 395 (ObjNum 1031 snap 0 seq_num 1031) 2023-08-27T20:53:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:1450: finishing write tid 1 to smithi093153278-260 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1450: finishing write tid 2 to smithi093153278-260 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1450: finishing write tid 3 to smithi093153278-260 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1450: finishing write tid 4 to smithi093153278-260 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1450: finishing write tid 5 to smithi093153278-260 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 260 v 234 (ObjNum 1032 snap 0 seq_num 1032) dirty exists 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1450: left oid 260 (ObjNum 1032 snap 0 seq_num 1032) 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1451: finishing write tid 1 to smithi093153278-187 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1451: finishing write tid 2 to smithi093153278-187 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1451: finishing write tid 3 to smithi093153278-187 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1451: finishing write tid 4 to smithi093153278-187 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1451: finishing write tid 5 to smithi093153278-187 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 187 v 193 (ObjNum 1033 snap 0 seq_num 1033) dirty exists 2023-08-27T20:53:02.428 INFO:tasks.rados.rados.0.smithi093.stdout:1451: left oid 187 (ObjNum 1033 snap 0 seq_num 1033) 2023-08-27T20:53:02.429 INFO:tasks.rados.rados.0.smithi093.stdout:1455: expect (ObjNum 860 snap 0 seq_num 860) 2023-08-27T20:53:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:02 smithi136 ceph-mon[113531]: pgmap v85: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.0 MiB/s wr, 91 op/s 2023-08-27T20:53:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:02 smithi093 ceph-mon[111335]: pgmap v85: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.0 MiB/s wr, 91 op/s 2023-08-27T20:53:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:02 smithi093 ceph-mon[118902]: pgmap v85: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.0 MiB/s wr, 91 op/s 2023-08-27T20:53:02.538 INFO:tasks.rados.rados.0.smithi093.stdout:1453: expect (ObjNum 967 snap 0 seq_num 967) 2023-08-27T20:53:02.735 INFO:tasks.rados.rados.0.smithi093.stdout:1457: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:53:02.833 INFO:tasks.rados.rados.0.smithi093.stdout:1454: expect (ObjNum 1014 snap 0 seq_num 1014) 2023-08-27T20:53:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:1449: done (10 left) 2023-08-27T20:53:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:1450: done (9 left) 2023-08-27T20:53:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:1451: done (8 left) 2023-08-27T20:53:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:1453: done (7 left) 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1454: done (6 left) 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1455: done (5 left) 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1456: done (4 left) 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1457: done (3 left) 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1460: done (2 left) 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1461: write oid 402 current snap is 0 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1458: finishing write tid 1 to smithi093153278-219 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1458: finishing write tid 2 to smithi093153278-219 2023-08-27T20:53:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:1461: seq_num 1037 ranges {628859=657221,1716180=641021,3144467=284392} 2023-08-27T20:53:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:02.443+0000 7fa0defff200 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:53:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:02.538+0000 7fa0defff200 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:53:02.940 INFO:tasks.rados.rados.0.smithi093.stdout:1461: writing smithi093153278-402 from 628859 to 1286080 tid 1 2023-08-27T20:53:02.947 INFO:tasks.rados.rados.0.smithi093.stdout:1461: writing smithi093153278-402 from 1716180 to 2357201 tid 2 2023-08-27T20:53:02.953 INFO:tasks.rados.rados.0.smithi093.stdout:1461: writing smithi093153278-402 from 3144467 to 3428859 tid 3 2023-08-27T20:53:02.953 INFO:tasks.rados.rados.0.smithi093.stdout:1462: delete oid 472 current snap is 0 2023-08-27T20:53:02.953 INFO:tasks.rados.rados.0.smithi093.stdout:1458: finishing write tid 3 to smithi093153278-219 2023-08-27T20:53:02.953 INFO:tasks.rados.rados.0.smithi093.stdout:1458: finishing write tid 4 to smithi093153278-219 2023-08-27T20:53:02.954 INFO:tasks.rados.rados.0.smithi093.stdout:1458: finishing write tid 5 to smithi093153278-219 2023-08-27T20:53:02.954 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 219 v 197 (ObjNum 1035 snap 0 seq_num 1035) dirty exists 2023-08-27T20:53:02.954 INFO:tasks.rados.rados.0.smithi093.stdout:1458: left oid 219 (ObjNum 1035 snap 0 seq_num 1035) 2023-08-27T20:53:02.954 INFO:tasks.rados.rados.0.smithi093.stdout:1459: expect (ObjNum 805 snap 0 seq_num 805) 2023-08-27T20:53:03.095 INFO:tasks.rados.rados.0.smithi093.stdout:1461: finishing write tid 1 to smithi093153278-402 2023-08-27T20:53:03.095 INFO:tasks.rados.rados.0.smithi093.stdout:1461: finishing write tid 2 to smithi093153278-402 2023-08-27T20:53:03.095 INFO:tasks.rados.rados.0.smithi093.stdout:1461: finishing write tid 3 to smithi093153278-402 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1461: finishing write tid 4 to smithi093153278-402 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1461: finishing write tid 5 to smithi093153278-402 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 402 v 207 (ObjNum 1037 snap 0 seq_num 1037) dirty exists 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1461: left oid 402 (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1458: done (3 left) 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1459: done (2 left) 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1461: done (1 left) 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1462: done (0 left) 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1463: read oid 173 snap -1 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1463: expect (ObjNum 957 snap 0 seq_num 957) 2023-08-27T20:53:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:1464: write oid 376 current snap is 0 2023-08-27T20:53:03.097 INFO:tasks.rados.rados.0.smithi093.stdout:1464: seq_num 1039 ranges {662473=407719,1833144=644690,3199313=263160} 2023-08-27T20:53:03.100 INFO:tasks.rados.rados.0.smithi093.stdout:1464: writing smithi093153278-376 from 662473 to 1070192 tid 1 2023-08-27T20:53:03.108 INFO:tasks.rados.rados.0.smithi093.stdout:1464: writing smithi093153278-376 from 1833144 to 2477834 tid 2 2023-08-27T20:53:03.114 INFO:tasks.rados.rados.0.smithi093.stdout:1464: writing smithi093153278-376 from 3199313 to 3462473 tid 3 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1465: read oid 488 snap -1 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1465: expect (ObjNum 968 snap 0 seq_num 968) 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1466: read oid 48 snap -1 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1466: expect (ObjNum 725 snap 0 seq_num 725) 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1467: read oid 131 snap -1 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1467: expect (ObjNum 844 snap 0 seq_num 844) 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1468: read oid 176 snap -1 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1468: expect (ObjNum 175 snap 0 seq_num 175) 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1469: write oid 421 current snap is 0 2023-08-27T20:53:03.115 INFO:tasks.rados.rados.0.smithi093.stdout:1469: seq_num 1040 ranges {679280=482968,1775273=788348,3110383=368897} 2023-08-27T20:53:03.121 INFO:tasks.rados.rados.0.smithi093.stdout:1469: writing smithi093153278-421 from 679280 to 1162248 tid 1 2023-08-27T20:53:03.128 INFO:tasks.rados.rados.0.smithi093.stdout:1469: writing smithi093153278-421 from 1775273 to 2563621 tid 2 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1469: writing smithi093153278-421 from 3110383 to 3479280 tid 3 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1470: read oid 440 snap -1 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1470: expect (ObjNum 1002 snap 0 seq_num 1002) 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1471: read oid 348 snap -1 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1471: expect (ObjNum 647 snap 0 seq_num 647) 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1472: read oid 82 snap -1 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1472: expect (ObjNum 738 snap 0 seq_num 738) 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1473: write oid 166 current snap is 0 2023-08-27T20:53:03.135 INFO:tasks.rados.rados.0.smithi093.stdout:1463: expect (ObjNum 957 snap 0 seq_num 957) 2023-08-27T20:53:03.225 INFO:tasks.rados.rados.0.smithi093.stdout:1464: finishing write tid 1 to smithi093153278-376 2023-08-27T20:53:03.225 INFO:tasks.rados.rados.0.smithi093.stdout:1464: finishing write tid 2 to smithi093153278-376 2023-08-27T20:53:03.225 INFO:tasks.rados.rados.0.smithi093.stdout:1473: seq_num 1041 ranges {696087=558217,1833755=648359,3137806=358281} 2023-08-27T20:53:03.231 INFO:tasks.rados.rados.0.smithi093.stdout:1473: writing smithi093153278-166 from 696087 to 1254304 tid 1 2023-08-27T20:53:03.238 INFO:tasks.rados.rados.0.smithi093.stdout:1473: writing smithi093153278-166 from 1833755 to 2482114 tid 2 2023-08-27T20:53:03.244 INFO:tasks.rados.rados.0.smithi093.stdout:1473: writing smithi093153278-166 from 3137806 to 3496087 tid 3 2023-08-27T20:53:03.244 INFO:tasks.rados.rados.0.smithi093.stdout:1463: done (10 left) 2023-08-27T20:53:03.244 INFO:tasks.rados.rados.0.smithi093.stdout:1474: read oid 122 snap -1 2023-08-27T20:53:03.244 INFO:tasks.rados.rados.0.smithi093.stdout:1474: expect (ObjNum 605 snap 0 seq_num 605) 2023-08-27T20:53:03.244 INFO:tasks.rados.rados.0.smithi093.stdout:1475: read oid 183 snap -1 2023-08-27T20:53:03.244 INFO:tasks.rados.rados.0.smithi093.stdout:1475: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:53:03.245 INFO:tasks.rados.rados.0.smithi093.stdout:1476: read oid 50 snap -1 2023-08-27T20:53:03.245 INFO:tasks.rados.rados.0.smithi093.stdout:1476: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:53:03.245 INFO:tasks.rados.rados.0.smithi093.stdout:1477: write oid 115 current snap is 0 2023-08-27T20:53:03.245 INFO:tasks.rados.rados.0.smithi093.stdout:1477: seq_num 1042 ranges {712894=749819,2008590=792017,3281582=231312} 2023-08-27T20:53:03.252 INFO:tasks.rados.rados.0.smithi093.stdout:1477: writing smithi093153278-115 from 712894 to 1462713 tid 1 2023-08-27T20:53:03.259 INFO:tasks.rados.rados.0.smithi093.stdout:1477: writing smithi093153278-115 from 2008590 to 2800607 tid 2 2023-08-27T20:53:03.263 INFO:tasks.rados.rados.0.smithi093.stdout:1477: writing smithi093153278-115 from 3281582 to 3512894 tid 3 2023-08-27T20:53:03.263 INFO:tasks.rados.rados.0.smithi093.stdout:1478: write oid 446 current snap is 0 2023-08-27T20:53:03.263 INFO:tasks.rados.rados.0.smithi093.stdout:1478: seq_num 1043 ranges {729701=425068,1667072=652028,2909005=485180} 2023-08-27T20:53:03.270 INFO:tasks.rados.rados.0.smithi093.stdout:1478: writing smithi093153278-446 from 729701 to 1154769 tid 1 2023-08-27T20:53:03.276 INFO:tasks.rados.rados.0.smithi093.stdout:1478: writing smithi093153278-446 from 1667072 to 2319100 tid 2 2023-08-27T20:53:03.282 INFO:tasks.rados.rados.0.smithi093.stdout:1478: writing smithi093153278-446 from 2909005 to 3394185 tid 3 2023-08-27T20:53:03.282 INFO:tasks.rados.rados.0.smithi093.stdout:1479: read oid 27 snap -1 2023-08-27T20:53:03.282 INFO:tasks.rados.rados.0.smithi093.stdout:1479: expect (ObjNum 26 snap 0 seq_num 26) 2023-08-27T20:53:03.283 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:53:03.283 INFO:tasks.rados.rados.0.smithi093.stdout:1465: expect (ObjNum 968 snap 0 seq_num 968) 2023-08-27T20:53:03.380 INFO:tasks.rados.rados.0.smithi093.stdout:1466: expect (ObjNum 725 snap 0 seq_num 725) 2023-08-27T20:53:03.463 INFO:tasks.rados.rados.0.smithi093.stdout:1467: expect (ObjNum 844 snap 0 seq_num 844) 2023-08-27T20:53:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2092214033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2092214033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:03 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.498 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.499 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.499 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2092214033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:03.499 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.499 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:03.499 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:03.209+0000 7fa0defff200 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:53:03.500 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:03.327+0000 7fa0defff200 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:53:03.500 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:03.423+0000 7fa0defff200 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:53:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:1464: finishing write tid 3 to smithi093153278-376 2023-08-27T20:53:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:1464: finishing write tid 4 to smithi093153278-376 2023-08-27T20:53:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:1464: finishing write tid 5 to smithi093153278-376 2023-08-27T20:53:03.551 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 376 v 288 (ObjNum 1039 snap 0 seq_num 1039) dirty exists 2023-08-27T20:53:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:1464: left oid 376 (ObjNum 1039 snap 0 seq_num 1039) 2023-08-27T20:53:03.552 INFO:tasks.rados.rados.0.smithi093.stdout:1468: expect (ObjNum 175 snap 0 seq_num 175) 2023-08-27T20:53:03.671 INFO:tasks.rados.rados.0.smithi093.stdout:1469: finishing write tid 1 to smithi093153278-421 2023-08-27T20:53:03.671 INFO:tasks.rados.rados.0.smithi093.stdout:1469: finishing write tid 2 to smithi093153278-421 2023-08-27T20:53:03.672 INFO:tasks.rados.rados.0.smithi093.stdout:1471: expect (ObjNum 647 snap 0 seq_num 647) 2023-08-27T20:53:03.766 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:03.526+0000 7fa0defff200 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:1469: finishing write tid 3 to smithi093153278-421 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:1469: finishing write tid 4 to smithi093153278-421 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:1469: finishing write tid 5 to smithi093153278-421 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 421 v 233 (ObjNum 1040 snap 0 seq_num 1040) dirty exists 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:1469: left oid 421 (ObjNum 1040 snap 0 seq_num 1040) 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:1464: done (15 left) 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:1465: done (14 left) 2023-08-27T20:53:03.798 INFO:tasks.rados.rados.0.smithi093.stdout:1466: done (13 left) 2023-08-27T20:53:03.799 INFO:tasks.rados.rados.0.smithi093.stdout:1467: done (12 left) 2023-08-27T20:53:03.799 INFO:tasks.rados.rados.0.smithi093.stdout:1468: done (11 left) 2023-08-27T20:53:03.799 INFO:tasks.rados.rados.0.smithi093.stdout:1469: done (10 left) 2023-08-27T20:53:03.799 INFO:tasks.rados.rados.0.smithi093.stdout:1471: done (9 left) 2023-08-27T20:53:03.799 INFO:tasks.rados.rados.0.smithi093.stdout:1480: write oid 484 current snap is 0 2023-08-27T20:53:03.799 INFO:tasks.rados.rados.0.smithi093.stdout:1480: seq_num 1044 ranges {746508=500317,2009201=795686,3220075=326433} 2023-08-27T20:53:03.804 INFO:tasks.rados.rados.0.smithi093.stdout:1480: writing smithi093153278-484 from 746508 to 1246825 tid 1 2023-08-27T20:53:03.812 INFO:tasks.rados.rados.0.smithi093.stdout:1480: writing smithi093153278-484 from 2009201 to 2804887 tid 2 2023-08-27T20:53:03.817 INFO:tasks.rados.rados.0.smithi093.stdout:1480: writing smithi093153278-484 from 3220075 to 3546508 tid 3 2023-08-27T20:53:03.817 INFO:tasks.rados.rados.0.smithi093.stdout:1481: read oid 76 snap -1 2023-08-27T20:53:03.817 INFO:tasks.rados.rados.0.smithi093.stdout:1481: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:53:03.817 INFO:tasks.rados.rados.0.smithi093.stdout:1482: read oid 496 snap -1 2023-08-27T20:53:03.817 INFO:tasks.rados.rados.0.smithi093.stdout:1482: expect deleted 2023-08-27T20:53:03.817 INFO:tasks.rados.rados.0.smithi093.stdout:1470: expect (ObjNum 1002 snap 0 seq_num 1002) 2023-08-27T20:53:03.951 INFO:tasks.rados.rados.0.smithi093.stdout:1472: expect (ObjNum 738 snap 0 seq_num 738) 2023-08-27T20:53:04.026 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:03.765+0000 7fa0defff200 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:53:04.026 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:03.870+0000 7fa0defff200 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:53:04.061 INFO:tasks.rados.rados.0.smithi093.stdout:1473: finishing write tid 1 to smithi093153278-166 2023-08-27T20:53:04.061 INFO:tasks.rados.rados.0.smithi093.stdout:1473: finishing write tid 2 to smithi093153278-166 2023-08-27T20:53:04.061 INFO:tasks.rados.rados.0.smithi093.stdout:1473: finishing write tid 3 to smithi093153278-166 2023-08-27T20:53:04.061 INFO:tasks.rados.rados.0.smithi093.stdout:1473: finishing write tid 4 to smithi093153278-166 2023-08-27T20:53:04.061 INFO:tasks.rados.rados.0.smithi093.stdout:1473: finishing write tid 5 to smithi093153278-166 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 166 v 193 (ObjNum 1041 snap 0 seq_num 1041) dirty exists 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1473: left oid 166 (ObjNum 1041 snap 0 seq_num 1041) 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1483: read oid 251 snap -1 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1483: expect (ObjNum 250 snap 0 seq_num 250) 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1470: done (12 left) 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1472: done (11 left) 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1473: done (10 left) 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1484: delete oid 375 current snap is 0 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1477: finishing write tid 1 to smithi093153278-115 2023-08-27T20:53:04.062 INFO:tasks.rados.rados.0.smithi093.stdout:1474: expect (ObjNum 605 snap 0 seq_num 605) 2023-08-27T20:53:04.153 INFO:tasks.rados.rados.0.smithi093.stdout:1475: expect (ObjNum 182 snap 0 seq_num 182) 2023-08-27T20:53:04.253 INFO:tasks.rados.rados.0.smithi093.stdout:1477: finishing write tid 2 to smithi093153278-115 2023-08-27T20:53:04.253 INFO:tasks.rados.rados.0.smithi093.stdout:1477: finishing write tid 3 to smithi093153278-115 2023-08-27T20:53:04.253 INFO:tasks.rados.rados.0.smithi093.stdout:1474: done (10 left) 2023-08-27T20:53:04.253 INFO:tasks.rados.rados.0.smithi093.stdout:1475: done (9 left) 2023-08-27T20:53:04.253 INFO:tasks.rados.rados.0.smithi093.stdout:1484: done (8 left) 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:1485: read oid 284 snap -1 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:1485: expect deleted 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:1486: write oid 133 current snap is 0 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:1477: finishing write tid 4 to smithi093153278-115 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:1477: finishing write tid 5 to smithi093153278-115 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 115 v 202 (ObjNum 1042 snap 0 seq_num 1042) dirty exists 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:1477: left oid 115 (ObjNum 1042 snap 0 seq_num 1042) 2023-08-27T20:53:04.254 INFO:tasks.rados.rados.0.smithi093.stdout:1476: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:53:04.341 INFO:tasks.rados.rados.0.smithi093.stdout:1478: finishing write tid 1 to smithi093153278-446 2023-08-27T20:53:04.341 INFO:tasks.rados.rados.0.smithi093.stdout:1478: finishing write tid 2 to smithi093153278-446 2023-08-27T20:53:04.341 INFO:tasks.rados.rados.0.smithi093.stdout:1478: finishing write tid 3 to smithi093153278-446 2023-08-27T20:53:04.341 INFO:tasks.rados.rados.0.smithi093.stdout:1478: finishing write tid 4 to smithi093153278-446 2023-08-27T20:53:04.341 INFO:tasks.rados.rados.0.smithi093.stdout:1478: finishing write tid 5 to smithi093153278-446 2023-08-27T20:53:04.341 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 446 v 210 (ObjNum 1043 snap 0 seq_num 1043) dirty exists 2023-08-27T20:53:04.341 INFO:tasks.rados.rados.0.smithi093.stdout:1478: left oid 446 (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:53:04.342 INFO:tasks.rados.rados.0.smithi093.stdout:1479: expect (ObjNum 26 snap 0 seq_num 26) 2023-08-27T20:53:04.360 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:04 smithi136 ceph-mon[113531]: pgmap v86: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 94 MiB/s rd, 13 MiB/s wr, 137 op/s 2023-08-27T20:53:04.360 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:04 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:04.360 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:04 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:04.360 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:04.104+0000 7fa0defff200 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:53:04.361 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:04.207+0000 7fa0defff200 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:1480: finishing write tid 1 to smithi093153278-484 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:1480: finishing write tid 2 to smithi093153278-484 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:1480: finishing write tid 3 to smithi093153278-484 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:1480: finishing write tid 4 to smithi093153278-484 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:1480: finishing write tid 5 to smithi093153278-484 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 484 v 292 (ObjNum 1044 snap 0 seq_num 1044) dirty exists 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:1480: left oid 484 (ObjNum 1044 snap 0 seq_num 1044) 2023-08-27T20:53:04.418 INFO:tasks.rados.rados.0.smithi093.stdout:1481: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:53:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:04 smithi093 ceph-mon[111335]: pgmap v86: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 94 MiB/s rd, 13 MiB/s wr, 137 op/s 2023-08-27T20:53:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:04 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:04 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:04 smithi093 ceph-mon[118902]: pgmap v86: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail; 94 MiB/s rd, 13 MiB/s wr, 137 op/s 2023-08-27T20:53:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:04 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:04 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:04.520 INFO:tasks.rados.rados.0.smithi093.stdout:1483: expect (ObjNum 250 snap 0 seq_num 250) 2023-08-27T20:53:04.591 INFO:tasks.rados.rados.0.smithi093.stdout:1486: seq_num 1046 ranges {780122=650815,2126165=799355} 2023-08-27T20:53:04.599 INFO:tasks.rados.rados.0.smithi093.stdout:1486: writing smithi093153278-133 from 780122 to 1430937 tid 1 2023-08-27T20:53:04.607 INFO:tasks.rados.rados.0.smithi093.stdout:1486: writing smithi093153278-133 from 2126165 to 2925520 tid 2 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1476: done (9 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1477: done (8 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1478: done (7 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1479: done (6 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1480: done (5 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1481: done (4 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1482: done (3 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1483: done (2 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1485: done (1 left) 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1487: read oid 479 snap -1 2023-08-27T20:53:04.608 INFO:tasks.rados.rados.0.smithi093.stdout:1487: expect (ObjNum 478 snap 0 seq_num 478) 2023-08-27T20:53:04.609 INFO:tasks.rados.rados.0.smithi093.stdout:1488: delete oid 453 current snap is 0 2023-08-27T20:53:04.609 INFO:tasks.rados.rados.0.smithi093.stdout:1486: finishing write tid 1 to smithi093153278-133 2023-08-27T20:53:04.609 INFO:tasks.rados.rados.0.smithi093.stdout:1488: done (2 left) 2023-08-27T20:53:04.609 INFO:tasks.rados.rados.0.smithi093.stdout:1489: read oid 271 snap -1 2023-08-27T20:53:04.610 INFO:tasks.rados.rados.0.smithi093.stdout:1489: expect (ObjNum 1005 snap 0 seq_num 1005) 2023-08-27T20:53:04.610 INFO:tasks.rados.rados.0.smithi093.stdout:1490: read oid 5 snap -1 2023-08-27T20:53:04.610 INFO:tasks.rados.rados.0.smithi093.stdout:1490: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:53:04.610 INFO:tasks.rados.rados.0.smithi093.stdout:1491: write oid 15 current snap is 0 2023-08-27T20:53:04.610 INFO:tasks.rados.rados.0.smithi093.stdout:1491: seq_num 1048 ranges {413736=401313,1326776=403024,2413414=457893,3438716=175020} 2023-08-27T20:53:04.613 INFO:tasks.rados.rados.0.smithi093.stdout:1491: writing smithi093153278-15 from 413736 to 815049 tid 1 2023-08-27T20:53:04.619 INFO:tasks.rados.rados.0.smithi093.stdout:1491: writing smithi093153278-15 from 1326776 to 1729800 tid 2 2023-08-27T20:53:04.625 INFO:tasks.rados.rados.0.smithi093.stdout:1491: writing smithi093153278-15 from 2413414 to 2871307 tid 3 2023-08-27T20:53:04.629 INFO:tasks.rados.rados.0.smithi093.stdout:1491: writing smithi093153278-15 from 3438716 to 3613736 tid 4 2023-08-27T20:53:04.629 INFO:tasks.rados.rados.0.smithi093.stdout:1492: read oid 448 snap -1 2023-08-27T20:53:04.629 INFO:tasks.rados.rados.0.smithi093.stdout:1492: expect (ObjNum 569 snap 0 seq_num 569) 2023-08-27T20:53:04.630 INFO:tasks.rados.rados.0.smithi093.stdout:1493: read oid 355 snap -1 2023-08-27T20:53:04.630 INFO:tasks.rados.rados.0.smithi093.stdout:1493: expect (ObjNum 902 snap 0 seq_num 902) 2023-08-27T20:53:04.630 INFO:tasks.rados.rados.0.smithi093.stdout:1494: delete oid 124 current snap is 0 2023-08-27T20:53:04.630 INFO:tasks.rados.rados.0.smithi093.stdout:1487: expect (ObjNum 478 snap 0 seq_num 478) 2023-08-27T20:53:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:04.406+0000 7fa0defff200 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:53:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:04.504+0000 7fa0defff200 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:53:04.695 INFO:tasks.rados.rados.0.smithi093.stdout:1486: finishing write tid 2 to smithi093153278-133 2023-08-27T20:53:04.695 INFO:tasks.rados.rados.0.smithi093.stdout:1486: finishing write tid 3 to smithi093153278-133 2023-08-27T20:53:04.695 INFO:tasks.rados.rados.0.smithi093.stdout:1486: finishing write tid 4 to smithi093153278-133 2023-08-27T20:53:04.695 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 133 v 236 (ObjNum 1046 snap 0 seq_num 1046) dirty exists 2023-08-27T20:53:04.695 INFO:tasks.rados.rados.0.smithi093.stdout:1486: left oid 133 (ObjNum 1046 snap 0 seq_num 1046) 2023-08-27T20:53:04.695 INFO:tasks.rados.rados.0.smithi093.stdout:1489: expect (ObjNum 1005 snap 0 seq_num 1005) 2023-08-27T20:53:04.815 INFO:tasks.rados.rados.0.smithi093.stdout:1491: finishing write tid 1 to smithi093153278-15 2023-08-27T20:53:04.816 INFO:tasks.rados.rados.0.smithi093.stdout:1490: expect (ObjNum 648 snap 0 seq_num 648) 2023-08-27T20:53:04.907 INFO:tasks.rados.rados.0.smithi093.stdout:1491: finishing write tid 2 to smithi093153278-15 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1491: finishing write tid 3 to smithi093153278-15 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1486: done (7 left) 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1487: done (6 left) 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1489: done (5 left) 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1490: done (4 left) 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1494: done (3 left) 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1495: write oid 393 current snap is 0 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1491: finishing write tid 4 to smithi093153278-15 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1491: finishing write tid 5 to smithi093153278-15 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:1491: finishing write tid 6 to smithi093153278-15 2023-08-27T20:53:04.908 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 15 v 241 (ObjNum 1048 snap 0 seq_num 1048) dirty exists 2023-08-27T20:53:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:1491: left oid 15 (ObjNum 1048 snap 0 seq_num 1048) 2023-08-27T20:53:04.909 INFO:tasks.rados.rados.0.smithi093.stdout:1492: expect (ObjNum 569 snap 0 seq_num 569) 2023-08-27T20:53:05.097 INFO:tasks.rados.rados.0.smithi093.stdout:1493: expect (ObjNum 902 snap 0 seq_num 902) 2023-08-27T20:53:05.260 INFO:tasks.rados.rados.0.smithi093.stdout:1495: seq_num 1050 ranges {447350=668164,1560093=406693,2584613=480437} 2023-08-27T20:53:05.267 INFO:tasks.rados.rados.0.smithi093.stdout:1495: writing smithi093153278-393 from 447350 to 1115514 tid 1 2023-08-27T20:53:05.272 INFO:tasks.rados.rados.0.smithi093.stdout:1495: writing smithi093153278-393 from 1560093 to 1966786 tid 2 2023-08-27T20:53:05.280 INFO:tasks.rados.rados.0.smithi093.stdout:1495: writing smithi093153278-393 from 2584613 to 3065050 tid 3 2023-08-27T20:53:05.280 INFO:tasks.rados.rados.0.smithi093.stdout:1491: done (3 left) 2023-08-27T20:53:05.280 INFO:tasks.rados.rados.0.smithi093.stdout:1492: done (2 left) 2023-08-27T20:53:05.280 INFO:tasks.rados.rados.0.smithi093.stdout:1493: done (1 left) 2023-08-27T20:53:05.280 INFO:tasks.rados.rados.0.smithi093.stdout:1496: write oid 162 current snap is 0 2023-08-27T20:53:05.280 INFO:tasks.rados.rados.0.smithi093.stdout:1495: finishing write tid 1 to smithi093153278-393 2023-08-27T20:53:05.281 INFO:tasks.rados.rados.0.smithi093.stdout:1496: seq_num 1051 ranges {464157=743413,1618575=666704,3012036=408062} 2023-08-27T20:53:05.288 INFO:tasks.rados.rados.0.smithi093.stdout:1496: writing smithi093153278-162 from 464157 to 1207570 tid 1 2023-08-27T20:53:05.294 INFO:tasks.rados.rados.0.smithi093.stdout:1496: writing smithi093153278-162 from 1618575 to 2285279 tid 2 2023-08-27T20:53:05.300 INFO:tasks.rados.rados.0.smithi093.stdout:1496: writing smithi093153278-162 from 3012036 to 3420098 tid 3 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1497: write oid 132 current snap is 0 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1495: finishing write tid 2 to smithi093153278-393 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1495: finishing write tid 3 to smithi093153278-393 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1495: finishing write tid 4 to smithi093153278-393 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1495: finishing write tid 5 to smithi093153278-393 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 393 v 146 (ObjNum 1050 snap 0 seq_num 1050) dirty exists 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1495: left oid 393 (ObjNum 1050 snap 0 seq_num 1050) 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1496: finishing write tid 1 to smithi093153278-162 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1496: finishing write tid 2 to smithi093153278-162 2023-08-27T20:53:05.301 INFO:tasks.rados.rados.0.smithi093.stdout:1497: seq_num 1052 ranges {480964=418662,1560704=410362,2523106=619334} 2023-08-27T20:53:05.305 INFO:tasks.rados.rados.0.smithi093.stdout:1497: writing smithi093153278-132 from 480964 to 899626 tid 1 2023-08-27T20:53:05.311 INFO:tasks.rados.rados.0.smithi093.stdout:1497: writing smithi093153278-132 from 1560704 to 1971066 tid 2 2023-08-27T20:53:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:1497: writing smithi093153278-132 from 2523106 to 3142440 tid 3 2023-08-27T20:53:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:1495: done (2 left) 2023-08-27T20:53:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:1498: delete oid 303 current snap is 0 2023-08-27T20:53:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:1496: finishing write tid 3 to smithi093153278-162 2023-08-27T20:53:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:1496: finishing write tid 4 to smithi093153278-162 2023-08-27T20:53:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:1496: finishing write tid 5 to smithi093153278-162 2023-08-27T20:53:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 162 v 180 (ObjNum 1051 snap 0 seq_num 1051) dirty exists 2023-08-27T20:53:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:1496: left oid 162 (ObjNum 1051 snap 0 seq_num 1051) 2023-08-27T20:53:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:1497: finishing write tid 1 to smithi093153278-132 2023-08-27T20:53:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:1497: finishing write tid 2 to smithi093153278-132 2023-08-27T20:53:05.321 INFO:tasks.rados.rados.0.smithi093.stdout:1496: done (2 left) 2023-08-27T20:53:05.322 INFO:tasks.rados.rados.0.smithi093.stdout:1498: done (1 left) 2023-08-27T20:53:05.322 INFO:tasks.rados.rados.0.smithi093.stdout:1499: read oid 171 snap -1 2023-08-27T20:53:05.322 INFO:tasks.rados.rados.0.smithi093.stdout:1499: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:53:05.322 INFO:tasks.rados.rados.0.smithi093.stdout:1500: write oid 35 current snap is 0 2023-08-27T20:53:05.322 INFO:tasks.rados.rados.0.smithi093.stdout:1500: seq_num 1054 ranges {514578=569160,1677668=414031,2577952=641878} 2023-08-27T20:53:05.327 INFO:tasks.rados.rados.0.smithi093.stdout:1500: writing smithi093153278-35 from 514578 to 1083738 tid 1 2023-08-27T20:53:05.333 INFO:tasks.rados.rados.0.smithi093.stdout:1500: writing smithi093153278-35 from 1677668 to 2091699 tid 2 2023-08-27T20:53:05.339 INFO:tasks.rados.rados.0.smithi093.stdout:1500: writing smithi093153278-35 from 2577952 to 3219830 tid 3 2023-08-27T20:53:05.340 INFO:tasks.rados.rados.0.smithi093.stdout:1501: delete oid 104 current snap is 0 2023-08-27T20:53:05.340 INFO:tasks.rados.rados.0.smithi093.stdout:1497: finishing write tid 3 to smithi093153278-132 2023-08-27T20:53:05.340 INFO:tasks.rados.rados.0.smithi093.stdout:1497: finishing write tid 4 to smithi093153278-132 2023-08-27T20:53:05.340 INFO:tasks.rados.rados.0.smithi093.stdout:1497: finishing write tid 5 to smithi093153278-132 2023-08-27T20:53:05.340 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 132 v 201 (ObjNum 1052 snap 0 seq_num 1052) dirty exists 2023-08-27T20:53:05.340 INFO:tasks.rados.rados.0.smithi093.stdout:1497: left oid 132 (ObjNum 1052 snap 0 seq_num 1052) 2023-08-27T20:53:05.340 INFO:tasks.rados.rados.0.smithi093.stdout:1499: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:53:05.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:05.323+0000 7fa0defff200 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:53:05.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:05.089+0000 7f1937891700 -1 mgr handle_mgr_map I was active but no longer am 2023-08-27T20:53:05.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setuser ceph since I am not root 2023-08-27T20:53:05.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ignoring --setgroup ceph since I am not root 2023-08-27T20:53:05.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mgr[111564]: -- 172.21.15.93:0/2054039033 <== mon.2 v2:172.21.15.93:3301/0 4 ==== auth_reply(proto 2 0 (0) Success) v1 ==== 194+0+0 (secure 0 0 0) 0x55f45bf0c9c0 con 0x55f45bee2c00 2023-08-27T20:53:05.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:05.279+0000 7f6d2fa7c1c0 -1 mgr[py] Module balancer has missing NOTIFY_TYPES member 2023-08-27T20:53:05.453 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:05.452+0000 7f6d2fa7c1c0 -1 mgr[py] Module diskprediction_local has missing NOTIFY_TYPES member 2023-08-27T20:53:05.455 INFO:tasks.rados.rados.0.smithi093.stdout:1500: finishing write tid 1 to smithi093153278-35 2023-08-27T20:53:05.456 INFO:tasks.rados.rados.0.smithi093.stdout:1500: finishing write tid 2 to smithi093153278-35 2023-08-27T20:53:05.456 INFO:tasks.rados.rados.0.smithi093.stdout:1497: done (3 left) 2023-08-27T20:53:05.456 INFO:tasks.rados.rados.0.smithi093.stdout:1499: done (2 left) 2023-08-27T20:53:05.459 INFO:tasks.rados.rados.0.smithi093.stdout:1501: done (1 left) 2023-08-27T20:53:05.459 INFO:tasks.rados.rados.0.smithi093.stdout:1502: write oid 274 current snap is 0 2023-08-27T20:53:05.459 INFO:tasks.rados.rados.0.smithi093.stdout:1502: seq_num 1056 ranges {548192=719658,1678279=417700,2516445=780775} 2023-08-27T20:53:05.463 INFO:tasks.rados.rados.0.smithi093.stdout:1502: writing smithi093153278-274 from 548192 to 1267850 tid 1 2023-08-27T20:53:05.467 INFO:tasks.rados.rados.0.smithi093.stdout:1502: writing smithi093153278-274 from 1678279 to 2095979 tid 2 2023-08-27T20:53:05.474 INFO:tasks.rados.rados.0.smithi093.stdout:1502: writing smithi093153278-274 from 2516445 to 3297220 tid 3 2023-08-27T20:53:05.475 INFO:tasks.rados.rados.0.smithi093.stdout:1503: read oid 88 snap -1 2023-08-27T20:53:05.475 INFO:tasks.rados.rados.0.smithi093.stdout:1503: expect (ObjNum 603 snap 0 seq_num 603) 2023-08-27T20:53:05.475 INFO:tasks.rados.rados.0.smithi093.stdout:1504: write oid 489 current snap is 0 2023-08-27T20:53:05.475 INFO:tasks.rados.rados.0.smithi093.stdout:1504: seq_num 1057 ranges {564999=511260,1853114=561358,3060221=592047} 2023-08-27T20:53:05.480 INFO:tasks.rados.rados.0.smithi093.stdout:1504: writing smithi093153278-489 from 564999 to 1076259 tid 1 2023-08-27T20:53:05.488 INFO:tasks.rados.rados.0.smithi093.stdout:1504: writing smithi093153278-489 from 1853114 to 2414472 tid 2 2023-08-27T20:53:05.495 INFO:tasks.rados.rados.0.smithi093.stdout:1504: writing smithi093153278-489 from 3060221 to 3652268 tid 3 2023-08-27T20:53:05.495 INFO:tasks.rados.rados.0.smithi093.stdout:1505: read oid 74 snap -1 2023-08-27T20:53:05.495 INFO:tasks.rados.rados.0.smithi093.stdout:1505: expect (ObjNum 597 snap 0 seq_num 597) 2023-08-27T20:53:05.495 INFO:tasks.rados.rados.0.smithi093.stdout:1506: write oid 394 current snap is 0 2023-08-27T20:53:05.496 INFO:tasks.rados.rados.0.smithi093.stdout:1506: seq_num 1058 ranges {581806=586509,1911596=421369,3087644=403319} 2023-08-27T20:53:05.501 INFO:tasks.rados.rados.0.smithi093.stdout:1506: writing smithi093153278-394 from 581806 to 1168315 tid 1 2023-08-27T20:53:05.508 INFO:tasks.rados.rados.0.smithi093.stdout:1506: writing smithi093153278-394 from 1911596 to 2332965 tid 2 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1506: writing smithi093153278-394 from 3087644 to 3490963 tid 3 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1507: delete oid 397 current snap is 0 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1500: finishing write tid 3 to smithi093153278-35 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1500: finishing write tid 4 to smithi093153278-35 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1500: finishing write tid 5 to smithi093153278-35 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 35 v 197 (ObjNum 1054 snap 0 seq_num 1054) dirty exists 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1500: left oid 35 (ObjNum 1054 snap 0 seq_num 1054) 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1502: finishing write tid 1 to smithi093153278-274 2023-08-27T20:53:05.515 INFO:tasks.rados.rados.0.smithi093.stdout:1502: finishing write tid 2 to smithi093153278-274 2023-08-27T20:53:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:1502: finishing write tid 3 to smithi093153278-274 2023-08-27T20:53:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:1502: finishing write tid 4 to smithi093153278-274 2023-08-27T20:53:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:1502: finishing write tid 5 to smithi093153278-274 2023-08-27T20:53:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 274 v 239 (ObjNum 1056 snap 0 seq_num 1056) dirty exists 2023-08-27T20:53:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:1502: left oid 274 (ObjNum 1056 snap 0 seq_num 1056) 2023-08-27T20:53:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:1503: expect (ObjNum 603 snap 0 seq_num 603) 2023-08-27T20:53:05.601 INFO:tasks.rados.rados.0.smithi093.stdout:1504: finishing write tid 1 to smithi093153278-489 2023-08-27T20:53:05.601 INFO:tasks.rados.rados.0.smithi093.stdout:1504: finishing write tid 2 to smithi093153278-489 2023-08-27T20:53:05.601 INFO:tasks.rados.rados.0.smithi093.stdout:1500: done (6 left) 2023-08-27T20:53:05.601 INFO:tasks.rados.rados.0.smithi093.stdout:1502: done (5 left) 2023-08-27T20:53:05.601 INFO:tasks.rados.rados.0.smithi093.stdout:1503: done (4 left) 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:1507: done (3 left) 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:1508: write oid 401 current snap is 0 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:1504: finishing write tid 3 to smithi093153278-489 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:1504: finishing write tid 4 to smithi093153278-489 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:1504: finishing write tid 5 to smithi093153278-489 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 489 v 205 (ObjNum 1057 snap 0 seq_num 1057) dirty exists 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:1504: left oid 489 (ObjNum 1057 snap 0 seq_num 1057) 2023-08-27T20:53:05.602 INFO:tasks.rados.rados.0.smithi093.stdout:1505: expect (ObjNum 597 snap 0 seq_num 597) 2023-08-27T20:53:05.691 INFO:tasks.rados.rados.0.smithi093.stdout:1506: finishing write tid 1 to smithi093153278-394 2023-08-27T20:53:05.691 INFO:tasks.rados.rados.0.smithi093.stdout:1506: finishing write tid 2 to smithi093153278-394 2023-08-27T20:53:05.691 INFO:tasks.rados.rados.0.smithi093.stdout:1506: finishing write tid 3 to smithi093153278-394 2023-08-27T20:53:05.691 INFO:tasks.rados.rados.0.smithi093.stdout:1508: seq_num 1060 ranges {615420=737007,1912207=425038,3026137=542216} 2023-08-27T20:53:05.699 INFO:tasks.rados.rados.0.smithi093.stdout:1508: writing smithi093153278-401 from 615420 to 1352427 tid 1 2023-08-27T20:53:05.706 INFO:tasks.rados.rados.0.smithi093.stdout:1508: writing smithi093153278-401 from 1912207 to 2337245 tid 2 2023-08-27T20:53:05.714 INFO:tasks.rados.rados.0.smithi093.stdout:1508: writing smithi093153278-401 from 3026137 to 3568353 tid 3 2023-08-27T20:53:05.714 INFO:tasks.rados.rados.0.smithi093.stdout:1504: done (3 left) 2023-08-27T20:53:05.714 INFO:tasks.rados.rados.0.smithi093.stdout:1505: done (2 left) 2023-08-27T20:53:05.714 INFO:tasks.rados.rados.0.smithi093.stdout:1509: read oid 34 snap -1 2023-08-27T20:53:05.714 INFO:tasks.rados.rados.0.smithi093.stdout:1509: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:05.714 INFO:tasks.rados.rados.0.smithi093.stdout:1510: delete oid 371 current snap is 0 2023-08-27T20:53:05.715 INFO:tasks.rados.rados.0.smithi093.stdout:1506: finishing write tid 4 to smithi093153278-394 2023-08-27T20:53:05.715 INFO:tasks.rados.rados.0.smithi093.stdout:1506: finishing write tid 5 to smithi093153278-394 2023-08-27T20:53:05.715 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 394 v 237 (ObjNum 1058 snap 0 seq_num 1058) dirty exists 2023-08-27T20:53:05.715 INFO:tasks.rados.rados.0.smithi093.stdout:1506: left oid 394 (ObjNum 1058 snap 0 seq_num 1058) 2023-08-27T20:53:05.715 INFO:tasks.rados.rados.0.smithi093.stdout:1508: finishing write tid 1 to smithi093153278-401 2023-08-27T20:53:05.715 INFO:tasks.rados.rados.0.smithi093.stdout:1508: finishing write tid 2 to smithi093153278-401 2023-08-27T20:53:05.716 INFO:tasks.rados.rados.0.smithi093.stdout:1506: done (3 left) 2023-08-27T20:53:05.716 INFO:tasks.rados.rados.0.smithi093.stdout:1510: done (2 left) 2023-08-27T20:53:05.716 INFO:tasks.rados.rados.0.smithi093.stdout:1511: write oid 235 current snap is 0 2023-08-27T20:53:05.716 INFO:tasks.rados.rados.0.smithi093.stdout:1511: seq_num 1062 ranges {649034=487505,1629171=428707,2680983=564760,3798768=50266} 2023-08-27T20:53:05.722 INFO:tasks.rados.rados.0.smithi093.stdout:1511: writing smithi093153278-235 from 649034 to 1136539 tid 1 2023-08-27T20:53:05.727 INFO:tasks.rados.rados.0.smithi093.stdout:1511: writing smithi093153278-235 from 1629171 to 2057878 tid 2 2023-08-27T20:53:05.734 INFO:tasks.rados.rados.0.smithi093.stdout:1511: writing smithi093153278-235 from 2680983 to 3245743 tid 3 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:1511: writing smithi093153278-235 from 3798768 to 3849034 tid 4 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:1512: write oid 28 current snap is 0 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:1508: finishing write tid 3 to smithi093153278-401 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:1508: finishing write tid 4 to smithi093153278-401 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:1508: finishing write tid 5 to smithi093153278-401 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 401 v 150 (ObjNum 1060 snap 0 seq_num 1060) dirty exists 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:1508: left oid 401 (ObjNum 1060 snap 0 seq_num 1060) 2023-08-27T20:53:05.738 INFO:tasks.rados.rados.0.smithi093.stdout:1509: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:05.822 INFO:tasks.rados.rados.0.smithi093.stdout:1511: finishing write tid 1 to smithi093153278-235 2023-08-27T20:53:05.822 INFO:tasks.rados.rados.0.smithi093.stdout:1511: finishing write tid 2 to smithi093153278-235 2023-08-27T20:53:05.822 INFO:tasks.rados.rados.0.smithi093.stdout:1512: seq_num 1063 ranges {665841=562754,1687653=572365,2708406=776032} 2023-08-27T20:53:05.829 INFO:tasks.rados.rados.0.smithi093.stdout:1512: writing smithi093153278-28 from 665841 to 1228595 tid 1 2023-08-27T20:53:05.834 INFO:tasks.rados.rados.0.smithi093.stdout:1512: writing smithi093153278-28 from 1687653 to 2260018 tid 2 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1512: writing smithi093153278-28 from 2708406 to 3484438 tid 3 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1508: done (3 left) 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1509: done (2 left) 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1513: read oid 152 snap -1 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1513: expect (ObjNum 728 snap 0 seq_num 728) 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1514: write oid 451 current snap is 0 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1511: finishing write tid 3 to smithi093153278-235 2023-08-27T20:53:05.842 INFO:tasks.rados.rados.0.smithi093.stdout:1514: seq_num 1064 ranges {682648=638003,2029782=432376,3019476=587304} 2023-08-27T20:53:05.848 INFO:tasks.rados.rados.0.smithi093.stdout:1514: writing smithi093153278-451 from 682648 to 1320651 tid 1 2023-08-27T20:53:05.855 INFO:tasks.rados.rados.0.smithi093.stdout:1514: writing smithi093153278-451 from 2029782 to 2462158 tid 2 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1514: writing smithi093153278-451 from 3019476 to 3606780 tid 3 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1515: read oid 112 snap -1 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1515: expect (ObjNum 1010 snap 0 seq_num 1010) 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1516: read oid 477 snap -1 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1516: expect (ObjNum 827 snap 0 seq_num 827) 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1517: read oid 23 snap -1 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1517: expect (ObjNum 955 snap 0 seq_num 955) 2023-08-27T20:53:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:1518: write oid 401 current snap is 0 2023-08-27T20:53:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:1511: finishing write tid 4 to smithi093153278-235 2023-08-27T20:53:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:1511: finishing write tid 5 to smithi093153278-235 2023-08-27T20:53:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:1518: seq_num 1065 ranges {699455=429605,1804617=576034,3163252=514929} 2023-08-27T20:53:05.867 INFO:tasks.rados.rados.0.smithi093.stdout:1518: writing smithi093153278-401 from 699455 to 1129060 tid 1 2023-08-27T20:53:05.874 INFO:tasks.rados.rados.0.smithi093.stdout:1518: writing smithi093153278-401 from 1804617 to 2380651 tid 2 2023-08-27T20:53:05.882 INFO:tasks.rados.rados.0.smithi093.stdout:1518: writing smithi093153278-401 from 3163252 to 3678181 tid 3 2023-08-27T20:53:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:1519: read oid 429 snap -1 2023-08-27T20:53:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:1519: expect deleted 2023-08-27T20:53:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:1520: write oid 481 current snap is 0 2023-08-27T20:53:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:1511: finishing write tid 6 to smithi093153278-235 2023-08-27T20:53:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 235 v 217 (ObjNum 1062 snap 0 seq_num 1062) dirty exists 2023-08-27T20:53:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:1511: left oid 235 (ObjNum 1062 snap 0 seq_num 1062) 2023-08-27T20:53:05.884 INFO:tasks.rados.rados.0.smithi093.stdout:1512: finishing write tid 1 to smithi093153278-28 2023-08-27T20:53:05.884 INFO:tasks.rados.rados.0.smithi093.stdout:1512: finishing write tid 2 to smithi093153278-28 2023-08-27T20:53:05.884 INFO:tasks.rados.rados.0.smithi093.stdout:1512: finishing write tid 3 to smithi093153278-28 2023-08-27T20:53:05.884 INFO:tasks.rados.rados.0.smithi093.stdout:1512: finishing write tid 4 to smithi093153278-28 2023-08-27T20:53:05.884 INFO:tasks.rados.rados.0.smithi093.stdout:1512: finishing write tid 5 to smithi093153278-28 2023-08-27T20:53:05.884 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 28 v 184 (ObjNum 1063 snap 0 seq_num 1063) dirty exists 2023-08-27T20:53:05.884 INFO:tasks.rados.rados.0.smithi093.stdout:1512: left oid 28 (ObjNum 1063 snap 0 seq_num 1063) 2023-08-27T20:53:05.885 INFO:tasks.rados.rados.0.smithi093.stdout:1513: expect (ObjNum 728 snap 0 seq_num 728) 2023-08-27T20:53:05.931 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:05.529+0000 7fa0defff200 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:53:05.931 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:05.716+0000 7fa0defff200 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:53:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:1514: finishing write tid 1 to smithi093153278-451 2023-08-27T20:53:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:1514: finishing write tid 2 to smithi093153278-451 2023-08-27T20:53:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:1514: finishing write tid 3 to smithi093153278-451 2023-08-27T20:53:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:1514: finishing write tid 4 to smithi093153278-451 2023-08-27T20:53:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:1520: seq_num 1066 ranges {716262=504854,1863099=719692,3074322=726201} 2023-08-27T20:53:05.980 INFO:tasks.rados.rados.0.smithi093.stdout:1520: writing smithi093153278-481 from 716262 to 1221116 tid 1 2023-08-27T20:53:05.988 INFO:tasks.rados.rados.0.smithi093.stdout:1520: writing smithi093153278-481 from 1863099 to 2582791 tid 2 2023-08-27T20:53:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:1520: writing smithi093153278-481 from 3074322 to 3800523 tid 3 2023-08-27T20:53:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:1511: done (9 left) 2023-08-27T20:53:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:1512: done (8 left) 2023-08-27T20:53:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:1513: done (7 left) 2023-08-27T20:53:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:1521: read oid 390 snap -1 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:1521: expect (ObjNum 742 snap 0 seq_num 742) 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:1522: read oid 97 snap -1 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:1522: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:1523: write oid 275 current snap is 0 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:1514: finishing write tid 5 to smithi093153278-451 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 451 v 224 (ObjNum 1064 snap 0 seq_num 1064) dirty exists 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:1514: left oid 451 (ObjNum 1064 snap 0 seq_num 1064) 2023-08-27T20:53:05.996 INFO:tasks.rados.rados.0.smithi093.stdout:1516: expect (ObjNum 827 snap 0 seq_num 827) 2023-08-27T20:53:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: osdmap e71: 8 total, 8 up, 8 in 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: mgrmap e20: x(active, starting, since 0.00598791s) 2023-08-27T20:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1072053410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:06.231 INFO:tasks.rados.rados.0.smithi093.stdout:1515: expect (ObjNum 1010 snap 0 seq_num 1010) 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:05.782+0000 7f6d2fa7c1c0 -1 mgr[py] Module crash has missing NOTIFY_TYPES member 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T20:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: osdmap e71: 8 total, 8 up, 8 in 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: mgrmap e20: x(active, starting, since 0.00598791s) 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1072053410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' 2023-08-27T20:53:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 172.21.15.93:0/1163886584' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T20:53:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd=[{"prefix": "mgr fail", "who": "y"}]: dispatch 2023-08-27T20:53:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: osdmap e71: 8 total, 8 up, 8 in 2023-08-27T20:53:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='mgr.14460 ' entity='mgr.y' cmd='[{"prefix": "mgr fail", "who": "y"}]': finished 2023-08-27T20:53:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: mgrmap e20: x(active, starting, since 0.00598791s) 2023-08-27T20:53:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1072053410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:06.385 INFO:tasks.rados.rados.0.smithi093.stdout:1518: finishing write tid 1 to smithi093153278-401 2023-08-27T20:53:06.385 INFO:tasks.rados.rados.0.smithi093.stdout:1517: expect (ObjNum 955 snap 0 seq_num 955) 2023-08-27T20:53:06.466 INFO:tasks.rados.rados.0.smithi093.stdout:1518: finishing write tid 2 to smithi093153278-401 2023-08-27T20:53:06.467 INFO:tasks.rados.rados.0.smithi093.stdout:1523: seq_num 1067 ranges {733069=580103,1921581=579703,3218098=537473} 2023-08-27T20:53:06.473 INFO:tasks.rados.rados.0.smithi093.stdout:1523: writing smithi093153278-275 from 733069 to 1313172 tid 1 2023-08-27T20:53:06.482 INFO:tasks.rados.rados.0.smithi093.stdout:1523: writing smithi093153278-275 from 1921581 to 2501284 tid 2 2023-08-27T20:53:06.487 INFO:tasks.rados.rados.0.smithi093.stdout:1523: writing smithi093153278-275 from 3218098 to 3755571 tid 3 2023-08-27T20:53:06.487 INFO:tasks.rados.rados.0.smithi093.stdout:1514: done (9 left) 2023-08-27T20:53:06.487 INFO:tasks.rados.rados.0.smithi093.stdout:1515: done (8 left) 2023-08-27T20:53:06.488 INFO:tasks.rados.rados.0.smithi093.stdout:1516: done (7 left) 2023-08-27T20:53:06.488 INFO:tasks.rados.rados.0.smithi093.stdout:1517: done (6 left) 2023-08-27T20:53:06.488 INFO:tasks.rados.rados.0.smithi093.stdout:1524: write oid 407 current snap is 0 2023-08-27T20:53:06.488 INFO:tasks.rados.rados.0.smithi093.stdout:1524: seq_num 1068 ranges {749876=655352,1863710=723361,3012815=748745} 2023-08-27T20:53:06.495 INFO:tasks.rados.rados.0.smithi093.stdout:1524: writing smithi093153278-407 from 749876 to 1405228 tid 1 2023-08-27T20:53:06.501 INFO:tasks.rados.rados.0.smithi093.stdout:1524: writing smithi093153278-407 from 1863710 to 2587071 tid 2 2023-08-27T20:53:06.508 INFO:tasks.rados.rados.0.smithi093.stdout:1524: writing smithi093153278-407 from 3012815 to 3761560 tid 3 2023-08-27T20:53:06.508 INFO:tasks.rados.rados.0.smithi093.stdout:1525: write oid 93 current snap is 0 2023-08-27T20:53:06.508 INFO:tasks.rados.rados.0.smithi093.stdout:1525: seq_num 1069 ranges {766683=730601,1922192=583372,3156591=676370} 2023-08-27T20:53:06.516 INFO:tasks.rados.rados.0.smithi093.stdout:1525: writing smithi093153278-93 from 766683 to 1497284 tid 1 2023-08-27T20:53:06.522 INFO:tasks.rados.rados.0.smithi093.stdout:1525: writing smithi093153278-93 from 1922192 to 2505564 tid 2 2023-08-27T20:53:06.529 INFO:tasks.rados.rados.0.smithi093.stdout:1525: writing smithi093153278-93 from 3156591 to 3832961 tid 3 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:1526: write oid 443 current snap is 0 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:1518: finishing write tid 3 to smithi093153278-401 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:1518: finishing write tid 4 to smithi093153278-401 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:1518: finishing write tid 5 to smithi093153278-401 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 401 v 154 (ObjNum 1065 snap 0 seq_num 1065) dirty exists 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:1518: left oid 401 (ObjNum 1065 snap 0 seq_num 1065) 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:1520: finishing write tid 1 to smithi093153278-481 2023-08-27T20:53:06.530 INFO:tasks.rados.rados.0.smithi093.stdout:1526: seq_num 1070 ranges {783490=405850,1980674=727030,3184014=487642} 2023-08-27T20:53:06.536 INFO:tasks.rados.rados.0.smithi093.stdout:1526: writing smithi093153278-443 from 783490 to 1189340 tid 1 2023-08-27T20:53:06.544 INFO:tasks.rados.rados.0.smithi093.stdout:1526: writing smithi093153278-443 from 1980674 to 2707704 tid 2 2023-08-27T20:53:06.550 INFO:tasks.rados.rados.0.smithi093.stdout:1526: writing smithi093153278-443 from 3184014 to 3671656 tid 3 2023-08-27T20:53:06.550 INFO:tasks.rados.rados.0.smithi093.stdout:1518: done (8 left) 2023-08-27T20:53:06.550 INFO:tasks.rados.rados.0.smithi093.stdout:1519: done (7 left) 2023-08-27T20:53:06.551 INFO:tasks.rados.rados.0.smithi093.stdout:1527: read oid 30 snap -1 2023-08-27T20:53:06.551 INFO:tasks.rados.rados.0.smithi093.stdout:1527: expect deleted 2023-08-27T20:53:06.551 INFO:tasks.rados.rados.0.smithi093.stdout:1528: write oid 412 current snap is 0 2023-08-27T20:53:06.551 INFO:tasks.rados.rados.0.smithi093.stdout:1520: finishing write tid 2 to smithi093153278-481 2023-08-27T20:53:06.551 INFO:tasks.rados.rados.0.smithi093.stdout:1528: seq_num 1071 ranges {400297=481099,1639156=361141} 2023-08-27T20:53:06.555 INFO:tasks.rados.rados.0.smithi093.stdout:1528: writing smithi093153278-412 from 400297 to 881396 tid 1 2023-08-27T20:53:06.561 INFO:tasks.rados.rados.0.smithi093.stdout:1528: writing smithi093153278-412 from 1639156 to 2000297 tid 2 2023-08-27T20:53:06.561 INFO:tasks.rados.rados.0.smithi093.stdout:1529: write oid 13 current snap is 0 2023-08-27T20:53:06.561 INFO:tasks.rados.rados.0.smithi093.stdout:1529: seq_num 1072 ranges {417104=672701,1813991=203113} 2023-08-27T20:53:06.567 INFO:tasks.rados.rados.0.smithi093.stdout:1529: writing smithi093153278-13 from 417104 to 1089805 tid 1 2023-08-27T20:53:06.572 INFO:tasks.rados.rados.0.smithi093.stdout:1529: writing smithi093153278-13 from 1813991 to 2017104 tid 2 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1530: read oid 56 snap -1 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1530: expect (ObjNum 55 snap 0 seq_num 55) 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1531: write oid 108 current snap is 0 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1520: finishing write tid 3 to smithi093153278-481 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1520: finishing write tid 4 to smithi093153278-481 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1520: finishing write tid 5 to smithi093153278-481 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 481 v 221 (ObjNum 1066 snap 0 seq_num 1066) dirty exists 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1520: left oid 481 (ObjNum 1066 snap 0 seq_num 1066) 2023-08-27T20:53:06.573 INFO:tasks.rados.rados.0.smithi093.stdout:1521: expect (ObjNum 742 snap 0 seq_num 742) 2023-08-27T20:53:06.675 INFO:tasks.rados.rados.0.smithi093.stdout:1522: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:53:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:06.403+0000 7fa0defff200 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:53:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:06.595+0000 7fa0defff200 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:53:06.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:06.323+0000 7f6d2fa7c1c0 -1 mgr[py] Module rook has missing NOTIFY_TYPES member 2023-08-27T20:53:06.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:06.498+0000 7f6d2fa7c1c0 -1 mgr[py] Module rgw has missing NOTIFY_TYPES member 2023-08-27T20:53:06.791 INFO:tasks.rados.rados.0.smithi093.stdout:1523: finishing write tid 1 to smithi093153278-275 2023-08-27T20:53:06.792 INFO:tasks.rados.rados.0.smithi093.stdout:1531: seq_num 1073 ranges {433911=747950,1756120=277791} 2023-08-27T20:53:06.798 INFO:tasks.rados.rados.0.smithi093.stdout:1531: writing smithi093153278-108 from 433911 to 1181861 tid 1 2023-08-27T20:53:06.803 INFO:tasks.rados.rados.0.smithi093.stdout:1531: writing smithi093153278-108 from 1756120 to 2033911 tid 2 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1520: done (11 left) 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1521: done (10 left) 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1522: done (9 left) 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1532: read oid 469 snap -1 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1532: expect (ObjNum 468 snap 0 seq_num 468) 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1533: read oid 24 snap -1 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1533: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1534: write oid 381 current snap is 0 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1523: finishing write tid 2 to smithi093153278-275 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1523: finishing write tid 3 to smithi093153278-275 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1523: finishing write tid 4 to smithi093153278-275 2023-08-27T20:53:06.804 INFO:tasks.rados.rados.0.smithi093.stdout:1523: finishing write tid 5 to smithi093153278-275 2023-08-27T20:53:06.805 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 275 v 209 (ObjNum 1067 snap 0 seq_num 1067) dirty exists 2023-08-27T20:53:06.805 INFO:tasks.rados.rados.0.smithi093.stdout:1523: left oid 275 (ObjNum 1067 snap 0 seq_num 1067) 2023-08-27T20:53:06.805 INFO:tasks.rados.rados.0.smithi093.stdout:1524: finishing write tid 1 to smithi093153278-407 2023-08-27T20:53:06.805 INFO:tasks.rados.rados.0.smithi093.stdout:1524: finishing write tid 2 to smithi093153278-407 2023-08-27T20:53:06.805 INFO:tasks.rados.rados.0.smithi093.stdout:1534: seq_num 1074 ranges {450718=423199,1414602=636116} 2023-08-27T20:53:06.808 INFO:tasks.rados.rados.0.smithi093.stdout:1534: writing smithi093153278-381 from 450718 to 873917 tid 1 2023-08-27T20:53:06.815 INFO:tasks.rados.rados.0.smithi093.stdout:1534: writing smithi093153278-381 from 1414602 to 2050718 tid 2 2023-08-27T20:53:06.815 INFO:tasks.rados.rados.0.smithi093.stdout:1523: done (11 left) 2023-08-27T20:53:06.816 INFO:tasks.rados.rados.0.smithi093.stdout:1535: write oid 388 current snap is 0 2023-08-27T20:53:06.816 INFO:tasks.rados.rados.0.smithi093.stdout:1524: finishing write tid 3 to smithi093153278-407 2023-08-27T20:53:06.816 INFO:tasks.rados.rados.0.smithi093.stdout:1524: finishing write tid 4 to smithi093153278-407 2023-08-27T20:53:06.816 INFO:tasks.rados.rados.0.smithi093.stdout:1535: seq_num 1075 ranges {467525=498448,1473084=594379} 2023-08-27T20:53:06.820 INFO:tasks.rados.rados.0.smithi093.stdout:1535: writing smithi093153278-388 from 467525 to 965973 tid 1 2023-08-27T20:53:06.827 INFO:tasks.rados.rados.0.smithi093.stdout:1535: writing smithi093153278-388 from 1473084 to 2067463 tid 2 2023-08-27T20:53:06.827 INFO:tasks.rados.rados.0.smithi093.stdout:1536: write oid 247 current snap is 0 2023-08-27T20:53:06.827 INFO:tasks.rados.rados.0.smithi093.stdout:1524: finishing write tid 5 to smithi093153278-407 2023-08-27T20:53:06.827 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 407 v 228 (ObjNum 1068 snap 0 seq_num 1068) dirty exists 2023-08-27T20:53:06.827 INFO:tasks.rados.rados.0.smithi093.stdout:1524: left oid 407 (ObjNum 1068 snap 0 seq_num 1068) 2023-08-27T20:53:06.828 INFO:tasks.rados.rados.0.smithi093.stdout:1525: finishing write tid 1 to smithi093153278-93 2023-08-27T20:53:06.828 INFO:tasks.rados.rados.0.smithi093.stdout:1525: finishing write tid 2 to smithi093153278-93 2023-08-27T20:53:06.828 INFO:tasks.rados.rados.0.smithi093.stdout:1536: seq_num 1076 ranges {484332=573697,1531566=552766} 2023-08-27T20:53:06.833 INFO:tasks.rados.rados.0.smithi093.stdout:1536: writing smithi093153278-247 from 484332 to 1058029 tid 1 2023-08-27T20:53:06.839 INFO:tasks.rados.rados.0.smithi093.stdout:1536: writing smithi093153278-247 from 1531566 to 2084332 tid 2 2023-08-27T20:53:06.839 INFO:tasks.rados.rados.0.smithi093.stdout:1524: done (12 left) 2023-08-27T20:53:06.839 INFO:tasks.rados.rados.0.smithi093.stdout:1537: write oid 354 current snap is 0 2023-08-27T20:53:06.839 INFO:tasks.rados.rados.0.smithi093.stdout:1525: finishing write tid 3 to smithi093153278-93 2023-08-27T20:53:06.839 INFO:tasks.rados.rados.0.smithi093.stdout:1537: seq_num 1077 ranges {501139=648946,1873695=227444} 2023-08-27T20:53:06.845 INFO:tasks.rados.rados.0.smithi093.stdout:1537: writing smithi093153278-354 from 501139 to 1150085 tid 1 2023-08-27T20:53:06.851 INFO:tasks.rados.rados.0.smithi093.stdout:1537: writing smithi093153278-354 from 1873695 to 2101139 tid 2 2023-08-27T20:53:06.851 INFO:tasks.rados.rados.0.smithi093.stdout:1538: write oid 189 current snap is 0 2023-08-27T20:53:06.851 INFO:tasks.rados.rados.0.smithi093.stdout:1525: finishing write tid 4 to smithi093153278-93 2023-08-27T20:53:06.851 INFO:tasks.rados.rados.0.smithi093.stdout:1525: finishing write tid 5 to smithi093153278-93 2023-08-27T20:53:06.851 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 93 v 202 (ObjNum 1069 snap 0 seq_num 1069) dirty exists 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1525: left oid 93 (ObjNum 1069 snap 0 seq_num 1069) 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1526: finishing write tid 1 to smithi093153278-443 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1526: finishing write tid 2 to smithi093153278-443 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1526: finishing write tid 3 to smithi093153278-443 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1526: finishing write tid 4 to smithi093153278-443 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1526: finishing write tid 5 to smithi093153278-443 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 239 (ObjNum 1070 snap 0 seq_num 1070) dirty exists 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1526: left oid 443 (ObjNum 1070 snap 0 seq_num 1070) 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1528: finishing write tid 1 to smithi093153278-412 2023-08-27T20:53:06.852 INFO:tasks.rados.rados.0.smithi093.stdout:1538: seq_num 1078 ranges {517946=724195,1932177=185769} 2023-08-27T20:53:06.857 INFO:tasks.rados.rados.0.smithi093.stdout:1538: writing smithi093153278-189 from 517946 to 1242141 tid 1 2023-08-27T20:53:06.863 INFO:tasks.rados.rados.0.smithi093.stdout:1538: writing smithi093153278-189 from 1932177 to 2117946 tid 2 2023-08-27T20:53:06.863 INFO:tasks.rados.rados.0.smithi093.stdout:1525: done (13 left) 2023-08-27T20:53:06.863 INFO:tasks.rados.rados.0.smithi093.stdout:1526: done (12 left) 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1527: done (11 left) 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1539: read oid 402 snap -1 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1539: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1528: finishing write tid 2 to smithi093153278-412 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1528: finishing write tid 3 to smithi093153278-412 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1528: finishing write tid 4 to smithi093153278-412 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 412 v 210 (ObjNum 1071 snap 0 seq_num 1071) dirty exists 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1528: left oid 412 (ObjNum 1071 snap 0 seq_num 1071) 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1529: finishing write tid 1 to smithi093153278-13 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1529: finishing write tid 2 to smithi093153278-13 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1540: read oid 205 snap -1 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1540: expect (ObjNum 974 snap 0 seq_num 974) 2023-08-27T20:53:06.864 INFO:tasks.rados.rados.0.smithi093.stdout:1528: done (12 left) 2023-08-27T20:53:06.865 INFO:tasks.rados.rados.0.smithi093.stdout:1541: write oid 372 current snap is 0 2023-08-27T20:53:06.865 INFO:tasks.rados.rados.0.smithi093.stdout:1529: finishing write tid 3 to smithi093153278-13 2023-08-27T20:53:06.865 INFO:tasks.rados.rados.0.smithi093.stdout:1529: finishing write tid 4 to smithi093153278-13 2023-08-27T20:53:06.865 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 13 v 187 (ObjNum 1072 snap 0 seq_num 1072) dirty exists 2023-08-27T20:53:06.865 INFO:tasks.rados.rados.0.smithi093.stdout:1529: left oid 13 (ObjNum 1072 snap 0 seq_num 1072) 2023-08-27T20:53:06.865 INFO:tasks.rados.rados.0.smithi093.stdout:1530: expect (ObjNum 55 snap 0 seq_num 55) 2023-08-27T20:53:06.961 INFO:tasks.rados.rados.0.smithi093.stdout:1541: seq_num 1079 ranges {534753=799444,1990659=144094} 2023-08-27T20:53:06.969 INFO:tasks.rados.rados.0.smithi093.stdout:1541: writing smithi093153278-372 from 534753 to 1334197 tid 1 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1541: writing smithi093153278-372 from 1990659 to 2134753 tid 2 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1529: done (12 left) 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1530: done (11 left) 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1542: read oid 330 snap -1 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1542: expect (ObjNum 733 snap 0 seq_num 733) 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1543: write oid 185 current snap is 0 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1531: finishing write tid 1 to smithi093153278-108 2023-08-27T20:53:06.974 INFO:tasks.rados.rados.0.smithi093.stdout:1533: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:53:07.035 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:06.783+0000 7fa0defff200 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:53:07.035 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:06.914+0000 7fa0defff200 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:53:07.035 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: [27/Aug/2023:20:53:06] ENGINE Bus STARTING 2023-08-27T20:53:07.035 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: CherryPy Checker: 2023-08-27T20:53:07.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: Active manager daemon x restarted 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: Activating manager daemon x 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: osdmap e72: 8 total, 8 up, 8 in 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: mgrmap e21: x(active, starting, since 0.00722235s) 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:53:07.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:53:07.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:06 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:53:07.052 INFO:tasks.rados.rados.0.smithi093.stdout:1531: finishing write tid 2 to smithi093153278-108 2023-08-27T20:53:07.052 INFO:tasks.rados.rados.0.smithi093.stdout:1531: finishing write tid 3 to smithi093153278-108 2023-08-27T20:53:07.052 INFO:tasks.rados.rados.0.smithi093.stdout:1531: finishing write tid 4 to smithi093153278-108 2023-08-27T20:53:07.053 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 108 v 250 (ObjNum 1073 snap 0 seq_num 1073) dirty exists 2023-08-27T20:53:07.053 INFO:tasks.rados.rados.0.smithi093.stdout:1531: left oid 108 (ObjNum 1073 snap 0 seq_num 1073) 2023-08-27T20:53:07.053 INFO:tasks.rados.rados.0.smithi093.stdout:1534: finishing write tid 1 to smithi093153278-381 2023-08-27T20:53:07.053 INFO:tasks.rados.rados.0.smithi093.stdout:1543: seq_num 1080 ranges {551560=591046,1765494=386066} 2023-08-27T20:53:07.058 INFO:tasks.rados.rados.0.smithi093.stdout:1543: writing smithi093153278-185 from 551560 to 1142606 tid 1 2023-08-27T20:53:07.064 INFO:tasks.rados.rados.0.smithi093.stdout:1543: writing smithi093153278-185 from 1765494 to 2151560 tid 2 2023-08-27T20:53:07.065 INFO:tasks.rados.rados.0.smithi093.stdout:1531: done (12 left) 2023-08-27T20:53:07.065 INFO:tasks.rados.rados.0.smithi093.stdout:1533: done (11 left) 2023-08-27T20:53:07.065 INFO:tasks.rados.rados.0.smithi093.stdout:1544: read oid 244 snap -1 2023-08-27T20:53:07.065 INFO:tasks.rados.rados.0.smithi093.stdout:1544: expect (ObjNum 857 snap 0 seq_num 857) 2023-08-27T20:53:07.065 INFO:tasks.rados.rados.0.smithi093.stdout:1545: write oid 496 current snap is 0 2023-08-27T20:53:07.065 INFO:tasks.rados.rados.0.smithi093.stdout:1532: expect (ObjNum 468 snap 0 seq_num 468) 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:1534: finishing write tid 2 to smithi093153278-381 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:1534: finishing write tid 3 to smithi093153278-381 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:1534: finishing write tid 4 to smithi093153278-381 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 381 v 224 (ObjNum 1074 snap 0 seq_num 1074) dirty exists 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:1534: left oid 381 (ObjNum 1074 snap 0 seq_num 1074) 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:1535: finishing write tid 1 to smithi093153278-388 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:1535: finishing write tid 2 to smithi093153278-388 2023-08-27T20:53:07.195 INFO:tasks.rados.rados.0.smithi093.stdout:1535: finishing write tid 3 to smithi093153278-388 2023-08-27T20:53:07.196 INFO:tasks.rados.rados.0.smithi093.stdout:1545: seq_num 1081 ranges {568367=666295,1707623=460744} 2023-08-27T20:53:07.201 INFO:tasks.rados.rados.0.smithi093.stdout:1545: writing smithi093153278-496 from 568367 to 1234662 tid 1 2023-08-27T20:53:07.207 INFO:tasks.rados.rados.0.smithi093.stdout:1545: writing smithi093153278-496 from 1707623 to 2168367 tid 2 2023-08-27T20:53:07.207 INFO:tasks.rados.rados.0.smithi093.stdout:1532: done (12 left) 2023-08-27T20:53:07.207 INFO:tasks.rados.rados.0.smithi093.stdout:1534: done (11 left) 2023-08-27T20:53:07.207 INFO:tasks.rados.rados.0.smithi093.stdout:1546: write oid 484 current snap is 0 2023-08-27T20:53:07.207 INFO:tasks.rados.rados.0.smithi093.stdout:1535: finishing write tid 4 to smithi093153278-388 2023-08-27T20:53:07.207 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 388 v 240 (ObjNum 1075 snap 0 seq_num 1075) dirty exists 2023-08-27T20:53:07.207 INFO:tasks.rados.rados.0.smithi093.stdout:1535: left oid 388 (ObjNum 1075 snap 0 seq_num 1075) 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:1536: finishing write tid 1 to smithi093153278-247 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:1536: finishing write tid 2 to smithi093153278-247 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:1536: finishing write tid 3 to smithi093153278-247 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:1536: finishing write tid 4 to smithi093153278-247 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 247 v 207 (ObjNum 1076 snap 0 seq_num 1076) dirty exists 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:1536: left oid 247 (ObjNum 1076 snap 0 seq_num 1076) 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:1537: finishing write tid 1 to smithi093153278-354 2023-08-27T20:53:07.208 INFO:tasks.rados.rados.0.smithi093.stdout:1546: seq_num 1082 ranges {585174=741544,1766105=419069} 2023-08-27T20:53:07.214 INFO:tasks.rados.rados.0.smithi093.stdout:1546: writing smithi093153278-484 from 585174 to 1326718 tid 1 2023-08-27T20:53:07.219 INFO:tasks.rados.rados.0.smithi093.stdout:1546: writing smithi093153278-484 from 1766105 to 2185174 tid 2 2023-08-27T20:53:07.219 INFO:tasks.rados.rados.0.smithi093.stdout:1535: done (11 left) 2023-08-27T20:53:07.219 INFO:tasks.rados.rados.0.smithi093.stdout:1536: done (10 left) 2023-08-27T20:53:07.219 INFO:tasks.rados.rados.0.smithi093.stdout:1547: write oid 363 current snap is 0 2023-08-27T20:53:07.220 INFO:tasks.rados.rados.0.smithi093.stdout:1537: finishing write tid 2 to smithi093153278-354 2023-08-27T20:53:07.220 INFO:tasks.rados.rados.0.smithi093.stdout:1537: finishing write tid 3 to smithi093153278-354 2023-08-27T20:53:07.220 INFO:tasks.rados.rados.0.smithi093.stdout:1537: finishing write tid 4 to smithi093153278-354 2023-08-27T20:53:07.220 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 354 v 245 (ObjNum 1077 snap 0 seq_num 1077) dirty exists 2023-08-27T20:53:07.220 INFO:tasks.rados.rados.0.smithi093.stdout:1537: left oid 354 (ObjNum 1077 snap 0 seq_num 1077) 2023-08-27T20:53:07.220 INFO:tasks.rados.rados.0.smithi093.stdout:1538: finishing write tid 1 to smithi093153278-189 2023-08-27T20:53:07.220 INFO:tasks.rados.rados.0.smithi093.stdout:1547: seq_num 1083 ranges {601981=416793,1424587=492702} 2023-08-27T20:53:07.224 INFO:tasks.rados.rados.0.smithi093.stdout:1547: writing smithi093153278-363 from 601981 to 1018774 tid 1 2023-08-27T20:53:07.230 INFO:tasks.rados.rados.0.smithi093.stdout:1547: writing smithi093153278-363 from 1424587 to 1917289 tid 2 2023-08-27T20:53:07.230 INFO:tasks.rados.rados.0.smithi093.stdout:1537: done (10 left) 2023-08-27T20:53:07.230 INFO:tasks.rados.rados.0.smithi093.stdout:1548: read oid 222 snap -1 2023-08-27T20:53:07.230 INFO:tasks.rados.rados.0.smithi093.stdout:1548: expect (ObjNum 991 snap 0 seq_num 991) 2023-08-27T20:53:07.230 INFO:tasks.rados.rados.0.smithi093.stdout:1549: write oid 37 current snap is 0 2023-08-27T20:53:07.230 INFO:tasks.rados.rados.0.smithi093.stdout:1538: finishing write tid 2 to smithi093153278-189 2023-08-27T20:53:07.230 INFO:tasks.rados.rados.0.smithi093.stdout:1538: finishing write tid 3 to smithi093153278-189 2023-08-27T20:53:07.231 INFO:tasks.rados.rados.0.smithi093.stdout:1538: finishing write tid 4 to smithi093153278-189 2023-08-27T20:53:07.231 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 189 v 231 (ObjNum 1078 snap 0 seq_num 1078) dirty exists 2023-08-27T20:53:07.231 INFO:tasks.rados.rados.0.smithi093.stdout:1538: left oid 189 (ObjNum 1078 snap 0 seq_num 1078) 2023-08-27T20:53:07.231 INFO:tasks.rados.rados.0.smithi093.stdout:1549: seq_num 1084 ranges {618788=492042,1883069=335719} 2023-08-27T20:53:07.236 INFO:tasks.rados.rados.0.smithi093.stdout:1549: writing smithi093153278-37 from 618788 to 1110830 tid 1 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: Active manager daemon x restarted 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: Activating manager daemon x 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: osdmap e72: 8 total, 8 up, 8 in 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: mgrmap e21: x(active, starting, since 0.00722235s) 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:53:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:53:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:06.921+0000 7f6d2fa7c1c0 -1 mgr[py] Module alerts has missing NOTIFY_TYPES member 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: Active manager daemon x restarted 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: Activating manager daemon x 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/crt"}]: dispatch 2023-08-27T20:53:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: osdmap e72: 8 total, 8 up, 8 in 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: mgrmap e21: x(active, starting, since 0.00722235s) 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/x/key"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mgr metadata", "who": "x", "id": "x"}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:53:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:53:07.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:53:07.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:53:07.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:53:07.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:53:07.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:53:07.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:53:07.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:06 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:53:07.244 INFO:tasks.rados.rados.0.smithi093.stdout:1549: writing smithi093153278-37 from 1883069 to 2218788 tid 2 2023-08-27T20:53:07.244 INFO:tasks.rados.rados.0.smithi093.stdout:1538: done (11 left) 2023-08-27T20:53:07.244 INFO:tasks.rados.rados.0.smithi093.stdout:1550: write oid 242 current snap is 0 2023-08-27T20:53:07.244 INFO:tasks.rados.rados.0.smithi093.stdout:1540: expect (ObjNum 974 snap 0 seq_num 974) 2023-08-27T20:53:07.364 INFO:tasks.rados.rados.0.smithi093.stdout:1539: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:53:07.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: The Application mounted at '' has an empty config. 2023-08-27T20:53:07.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T20:53:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: [27/Aug/2023:20:53:07] ENGINE Serving on http://:::9283 2023-08-27T20:53:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: [27/Aug/2023:20:53:07] ENGINE Bus STARTED 2023-08-27T20:53:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: [27/Aug/2023:20:53:07] ENGINE Bus STOPPING 2023-08-27T20:53:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: [27/Aug/2023:20:53:07] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T20:53:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: [27/Aug/2023:20:53:07] ENGINE Bus STOPPED 2023-08-27T20:53:07.535 INFO:tasks.rados.rados.0.smithi093.stdout:1541: finishing write tid 1 to smithi093153278-372 2023-08-27T20:53:07.535 INFO:tasks.rados.rados.0.smithi093.stdout:1541: finishing write tid 2 to smithi093153278-372 2023-08-27T20:53:07.535 INFO:tasks.rados.rados.0.smithi093.stdout:1541: finishing write tid 3 to smithi093153278-372 2023-08-27T20:53:07.535 INFO:tasks.rados.rados.0.smithi093.stdout:1550: seq_num 1085 ranges {635595=567291,1825198=410397} 2023-08-27T20:53:07.541 INFO:tasks.rados.rados.0.smithi093.stdout:1550: writing smithi093153278-242 from 635595 to 1202886 tid 1 2023-08-27T20:53:07.547 INFO:tasks.rados.rados.0.smithi093.stdout:1550: writing smithi093153278-242 from 1825198 to 2235595 tid 2 2023-08-27T20:53:07.547 INFO:tasks.rados.rados.0.smithi093.stdout:1539: done (11 left) 2023-08-27T20:53:07.547 INFO:tasks.rados.rados.0.smithi093.stdout:1540: done (10 left) 2023-08-27T20:53:07.548 INFO:tasks.rados.rados.0.smithi093.stdout:1551: read oid 390 snap -1 2023-08-27T20:53:07.548 INFO:tasks.rados.rados.0.smithi093.stdout:1551: expect (ObjNum 742 snap 0 seq_num 742) 2023-08-27T20:53:07.548 INFO:tasks.rados.rados.0.smithi093.stdout:1552: write oid 198 current snap is 0 2023-08-27T20:53:07.548 INFO:tasks.rados.rados.0.smithi093.stdout:1542: expect (ObjNum 733 snap 0 seq_num 733) 2023-08-27T20:53:07.621 INFO:tasks.rados.rados.0.smithi093.stdout:1541: finishing write tid 4 to smithi093153278-372 2023-08-27T20:53:07.621 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 372 v 205 (ObjNum 1079 snap 0 seq_num 1079) dirty exists 2023-08-27T20:53:07.621 INFO:tasks.rados.rados.0.smithi093.stdout:1541: left oid 372 (ObjNum 1079 snap 0 seq_num 1079) 2023-08-27T20:53:07.621 INFO:tasks.rados.rados.0.smithi093.stdout:1543: finishing write tid 1 to smithi093153278-185 2023-08-27T20:53:07.621 INFO:tasks.rados.rados.0.smithi093.stdout:1543: finishing write tid 2 to smithi093153278-185 2023-08-27T20:53:07.621 INFO:tasks.rados.rados.0.smithi093.stdout:1543: finishing write tid 3 to smithi093153278-185 2023-08-27T20:53:07.622 INFO:tasks.rados.rados.0.smithi093.stdout:1543: finishing write tid 4 to smithi093153278-185 2023-08-27T20:53:07.622 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 185 v 227 (ObjNum 1080 snap 0 seq_num 1080) dirty exists 2023-08-27T20:53:07.622 INFO:tasks.rados.rados.0.smithi093.stdout:1543: left oid 185 (ObjNum 1080 snap 0 seq_num 1080) 2023-08-27T20:53:07.622 INFO:tasks.rados.rados.0.smithi093.stdout:1544: expect (ObjNum 857 snap 0 seq_num 857) 2023-08-27T20:53:07.762 INFO:tasks.rados.rados.0.smithi093.stdout:1545: finishing write tid 1 to smithi093153278-496 2023-08-27T20:53:07.762 INFO:tasks.rados.rados.0.smithi093.stdout:1545: finishing write tid 2 to smithi093153278-496 2023-08-27T20:53:07.762 INFO:tasks.rados.rados.0.smithi093.stdout:1545: finishing write tid 3 to smithi093153278-496 2023-08-27T20:53:07.762 INFO:tasks.rados.rados.0.smithi093.stdout:1545: finishing write tid 4 to smithi093153278-496 2023-08-27T20:53:07.762 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 496 v 230 (ObjNum 1081 snap 0 seq_num 1081) dirty exists 2023-08-27T20:53:07.762 INFO:tasks.rados.rados.0.smithi093.stdout:1545: left oid 496 (ObjNum 1081 snap 0 seq_num 1081) 2023-08-27T20:53:07.763 INFO:tasks.rados.rados.0.smithi093.stdout:1546: finishing write tid 1 to smithi093153278-484 2023-08-27T20:53:07.763 INFO:tasks.rados.rados.0.smithi093.stdout:1552: seq_num 1086 ranges {652402=642540,1883680=368722} 2023-08-27T20:53:07.769 INFO:tasks.rados.rados.0.smithi093.stdout:1552: writing smithi093153278-198 from 652402 to 1294942 tid 1 2023-08-27T20:53:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:1552: writing smithi093153278-198 from 1883680 to 2252402 tid 2 2023-08-27T20:53:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:1541: done (11 left) 2023-08-27T20:53:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:1542: done (10 left) 2023-08-27T20:53:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:1543: done (9 left) 2023-08-27T20:53:07.775 INFO:tasks.rados.rados.0.smithi093.stdout:1544: done (8 left) 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1545: done (7 left) 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1553: write oid 278 current snap is 0 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1546: finishing write tid 2 to smithi093153278-484 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1546: finishing write tid 3 to smithi093153278-484 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1546: finishing write tid 4 to smithi093153278-484 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 484 v 295 (ObjNum 1082 snap 0 seq_num 1082) dirty exists 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1546: left oid 484 (ObjNum 1082 snap 0 seq_num 1082) 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1547: finishing write tid 1 to smithi093153278-363 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1547: finishing write tid 2 to smithi093153278-363 2023-08-27T20:53:07.776 INFO:tasks.rados.rados.0.smithi093.stdout:1553: seq_num 1087 ranges {669209=717789,1942162=327047} 2023-08-27T20:53:07.782 INFO:tasks.rados.rados.0.smithi093.stdout:1553: writing smithi093153278-278 from 669209 to 1386998 tid 1 2023-08-27T20:53:07.787 INFO:tasks.rados.rados.0.smithi093.stdout:1553: writing smithi093153278-278 from 1942162 to 2269209 tid 2 2023-08-27T20:53:07.787 INFO:tasks.rados.rados.0.smithi093.stdout:1546: done (7 left) 2023-08-27T20:53:07.787 INFO:tasks.rados.rados.0.smithi093.stdout:1554: write oid 336 current snap is 0 2023-08-27T20:53:07.787 INFO:tasks.rados.rados.0.smithi093.stdout:1547: finishing write tid 3 to smithi093153278-363 2023-08-27T20:53:07.788 INFO:tasks.rados.rados.0.smithi093.stdout:1547: finishing write tid 4 to smithi093153278-363 2023-08-27T20:53:07.788 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 298 (ObjNum 1083 snap 0 seq_num 1083) dirty exists 2023-08-27T20:53:07.788 INFO:tasks.rados.rados.0.smithi093.stdout:1547: left oid 363 (ObjNum 1083 snap 0 seq_num 1083) 2023-08-27T20:53:07.788 INFO:tasks.rados.rados.0.smithi093.stdout:1548: expect (ObjNum 991 snap 0 seq_num 991) 2023-08-27T20:53:07.872 INFO:tasks.rados.rados.0.smithi093.stdout:1549: finishing write tid 1 to smithi093153278-37 2023-08-27T20:53:07.872 INFO:tasks.rados.rados.0.smithi093.stdout:1549: finishing write tid 2 to smithi093153278-37 2023-08-27T20:53:07.872 INFO:tasks.rados.rados.0.smithi093.stdout:1549: finishing write tid 3 to smithi093153278-37 2023-08-27T20:53:07.872 INFO:tasks.rados.rados.0.smithi093.stdout:1549: finishing write tid 4 to smithi093153278-37 2023-08-27T20:53:07.872 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 37 v 301 (ObjNum 1084 snap 0 seq_num 1084) dirty exists 2023-08-27T20:53:07.872 INFO:tasks.rados.rados.0.smithi093.stdout:1549: left oid 37 (ObjNum 1084 snap 0 seq_num 1084) 2023-08-27T20:53:07.872 INFO:tasks.rados.rados.0.smithi093.stdout:1554: seq_num 1088 ranges {686016=509391,1716997=569019} 2023-08-27T20:53:07.878 INFO:tasks.rados.rados.0.smithi093.stdout:1554: writing smithi093153278-336 from 686016 to 1195407 tid 1 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1554: writing smithi093153278-336 from 1716997 to 2286016 tid 2 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1547: done (7 left) 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1548: done (6 left) 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1549: done (5 left) 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1555: read oid 6 snap -1 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1555: expect (ObjNum 5 snap 0 seq_num 5) 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1556: write oid 195 current snap is 0 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1550: finishing write tid 1 to smithi093153278-242 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1550: finishing write tid 2 to smithi093153278-242 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1550: finishing write tid 3 to smithi093153278-242 2023-08-27T20:53:07.885 INFO:tasks.rados.rados.0.smithi093.stdout:1550: finishing write tid 4 to smithi093153278-242 2023-08-27T20:53:07.886 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 242 v 234 (ObjNum 1085 snap 0 seq_num 1085) dirty exists 2023-08-27T20:53:07.886 INFO:tasks.rados.rados.0.smithi093.stdout:1550: left oid 242 (ObjNum 1085 snap 0 seq_num 1085) 2023-08-27T20:53:07.886 INFO:tasks.rados.rados.0.smithi093.stdout:1556: seq_num 1089 ranges {702823=584640,2059126=243697} 2023-08-27T20:53:07.891 INFO:tasks.rados.rados.0.smithi093.stdout:1556: writing smithi093153278-195 from 702823 to 1287463 tid 1 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1556: writing smithi093153278-195 from 2059126 to 2302823 tid 2 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1550: done (6 left) 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1557: read oid 221 snap -1 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1557: expect (ObjNum 663 snap 0 seq_num 663) 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1558: read oid 81 snap -1 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1558: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1559: read oid 338 snap -1 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1559: expect (ObjNum 337 snap 0 seq_num 337) 2023-08-27T20:53:07.898 INFO:tasks.rados.rados.0.smithi093.stdout:1560: read oid 347 snap -1 2023-08-27T20:53:07.899 INFO:tasks.rados.rados.0.smithi093.stdout:1560: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:53:07.899 INFO:tasks.rados.rados.0.smithi093.stdout:1561: read oid 418 snap -1 2023-08-27T20:53:07.899 INFO:tasks.rados.rados.0.smithi093.stdout:1561: expect (ObjNum 849 snap 0 seq_num 849) 2023-08-27T20:53:07.899 INFO:tasks.rados.rados.0.smithi093.stdout:1562: read oid 330 snap -1 2023-08-27T20:53:07.899 INFO:tasks.rados.rados.0.smithi093.stdout:1562: expect (ObjNum 733 snap 0 seq_num 733) 2023-08-27T20:53:07.899 INFO:tasks.rados.rados.0.smithi093.stdout:1563: write oid 395 current snap is 0 2023-08-27T20:53:07.899 INFO:tasks.rados.rados.0.smithi093.stdout:1563: seq_num 1090 ranges {719630=659889,2117608=202022} 2023-08-27T20:53:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:1563: writing smithi093153278-395 from 719630 to 1379519 tid 1 2023-08-27T20:53:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:1563: writing smithi093153278-395 from 2117608 to 2319630 tid 2 2023-08-27T20:53:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:1564: write oid 110 current snap is 0 2023-08-27T20:53:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:1551: expect (ObjNum 742 snap 0 seq_num 742) 2023-08-27T20:53:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:53:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:53:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[111335]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:53:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/324406655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:53:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:53:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[118902]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:53:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/324406655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:07.990 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:07.738+0000 7f6d2fa7c1c0 -1 mgr[py] Module progress has missing NOTIFY_TYPES member 2023-08-27T20:53:07.990 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:07 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:07.829+0000 7f6d2fa7c1c0 -1 mgr[py] Module telegraf has missing NOTIFY_TYPES member 2023-08-27T20:53:08.015 INFO:tasks.rados.rados.0.smithi093.stdout:1552: finishing write tid 1 to smithi093153278-198 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1552: finishing write tid 2 to smithi093153278-198 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1552: finishing write tid 3 to smithi093153278-198 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1552: finishing write tid 4 to smithi093153278-198 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 198 v 304 (ObjNum 1086 snap 0 seq_num 1086) dirty exists 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1552: left oid 198 (ObjNum 1086 snap 0 seq_num 1086) 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1553: finishing write tid 1 to smithi093153278-278 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1553: finishing write tid 2 to smithi093153278-278 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1553: finishing write tid 3 to smithi093153278-278 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1553: finishing write tid 4 to smithi093153278-278 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 278 v 307 (ObjNum 1087 snap 0 seq_num 1087) dirty exists 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1553: left oid 278 (ObjNum 1087 snap 0 seq_num 1087) 2023-08-27T20:53:08.016 INFO:tasks.rados.rados.0.smithi093.stdout:1564: seq_num 1091 ranges {736437=735138,2176090=160347} 2023-08-27T20:53:08.023 INFO:tasks.rados.rados.0.smithi093.stdout:1564: writing smithi093153278-110 from 736437 to 1471575 tid 1 2023-08-27T20:53:08.028 INFO:tasks.rados.rados.0.smithi093.stdout:1564: writing smithi093153278-110 from 2176090 to 2336437 tid 2 2023-08-27T20:53:08.028 INFO:tasks.rados.rados.0.smithi093.stdout:1551: done (13 left) 2023-08-27T20:53:08.028 INFO:tasks.rados.rados.0.smithi093.stdout:1552: done (12 left) 2023-08-27T20:53:08.029 INFO:tasks.rados.rados.0.smithi093.stdout:1553: done (11 left) 2023-08-27T20:53:08.029 INFO:tasks.rados.rados.0.smithi093.stdout:1565: write oid 156 current snap is 0 2023-08-27T20:53:08.029 INFO:tasks.rados.rados.0.smithi093.stdout:1554: finishing write tid 1 to smithi093153278-336 2023-08-27T20:53:08.029 INFO:tasks.rados.rados.0.smithi093.stdout:1565: seq_num 1092 ranges {753244=410387,1834572=518672} 2023-08-27T20:53:08.034 INFO:tasks.rados.rados.0.smithi093.stdout:1565: writing smithi093153278-156 from 753244 to 1163631 tid 1 2023-08-27T20:53:08.041 INFO:tasks.rados.rados.0.smithi093.stdout:1565: writing smithi093153278-156 from 1834572 to 2353244 tid 2 2023-08-27T20:53:08.041 INFO:tasks.rados.rados.0.smithi093.stdout:1566: write oid 200 current snap is 0 2023-08-27T20:53:08.041 INFO:tasks.rados.rados.0.smithi093.stdout:1554: finishing write tid 2 to smithi093153278-336 2023-08-27T20:53:08.041 INFO:tasks.rados.rados.0.smithi093.stdout:1554: finishing write tid 3 to smithi093153278-336 2023-08-27T20:53:08.041 INFO:tasks.rados.rados.0.smithi093.stdout:1554: finishing write tid 4 to smithi093153278-336 2023-08-27T20:53:08.042 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 336 v 237 (ObjNum 1088 snap 0 seq_num 1088) dirty exists 2023-08-27T20:53:08.042 INFO:tasks.rados.rados.0.smithi093.stdout:1554: left oid 336 (ObjNum 1088 snap 0 seq_num 1088) 2023-08-27T20:53:08.042 INFO:tasks.rados.rados.0.smithi093.stdout:1555: expect (ObjNum 5 snap 0 seq_num 5) 2023-08-27T20:53:08.109 INFO:tasks.rados.rados.0.smithi093.stdout:1556: finishing write tid 1 to smithi093153278-195 2023-08-27T20:53:08.110 INFO:tasks.rados.rados.0.smithi093.stdout:1566: seq_num 1093 ranges {770051=485636,1776701=511047} 2023-08-27T20:53:08.116 INFO:tasks.rados.rados.0.smithi093.stdout:1566: writing smithi093153278-200 from 770051 to 1255687 tid 1 2023-08-27T20:53:08.122 INFO:tasks.rados.rados.0.smithi093.stdout:1566: writing smithi093153278-200 from 1776701 to 2287748 tid 2 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1554: done (12 left) 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1555: done (11 left) 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1567: write oid 204 current snap is 0 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1556: finishing write tid 2 to smithi093153278-195 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1556: finishing write tid 3 to smithi093153278-195 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1556: finishing write tid 4 to smithi093153278-195 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 190 (ObjNum 1089 snap 0 seq_num 1089) dirty exists 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1556: left oid 195 (ObjNum 1089 snap 0 seq_num 1089) 2023-08-27T20:53:08.123 INFO:tasks.rados.rados.0.smithi093.stdout:1562: expect (ObjNum 733 snap 0 seq_num 733) 2023-08-27T20:53:08.196 INFO:tasks.rados.rados.0.smithi093.stdout:1561: expect (ObjNum 849 snap 0 seq_num 849) 2023-08-27T20:53:08.271 INFO:tasks.rados.rados.0.smithi093.stdout:1557: expect (ObjNum 663 snap 0 seq_num 663) 2023-08-27T20:53:08.403 INFO:tasks.rados.rados.0.smithi093.stdout:1559: expect (ObjNum 337 snap 0 seq_num 337) 2023-08-27T20:53:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:53:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:53:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-mon[113531]: from='mgr.14985 172.21.15.136:0/4217653437' entity='mgr.x' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:53:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/324406655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:08.521 INFO:tasks.rados.rados.0.smithi093.stdout:1560: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:53:08.646 INFO:tasks.rados.rados.0.smithi093.stdout:1558: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:53:08.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:08.465+0000 7f6d2fa7c1c0 -1 mgr[py] Module telemetry has missing NOTIFY_TYPES member 2023-08-27T20:53:08.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:08.552+0000 7f6d2fa7c1c0 -1 mgr[py] Module iostat has missing NOTIFY_TYPES member 2023-08-27T20:53:08.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:08.642+0000 7f6d2fa7c1c0 -1 mgr[py] Module zabbix has missing NOTIFY_TYPES member 2023-08-27T20:53:08.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:08.737+0000 7f6d2fa7c1c0 -1 mgr[py] Module status has missing NOTIFY_TYPES member 2023-08-27T20:53:08.868 INFO:tasks.rados.rados.0.smithi093.stdout:1563: finishing write tid 1 to smithi093153278-395 2023-08-27T20:53:08.868 INFO:tasks.rados.rados.0.smithi093.stdout:1563: finishing write tid 2 to smithi093153278-395 2023-08-27T20:53:08.869 INFO:tasks.rados.rados.0.smithi093.stdout:1563: finishing write tid 3 to smithi093153278-395 2023-08-27T20:53:08.869 INFO:tasks.rados.rados.0.smithi093.stdout:1567: seq_num 1094 ranges {786858=560885,1835183=551675} 2023-08-27T20:53:08.876 INFO:tasks.rados.rados.0.smithi093.stdout:1567: writing smithi093153278-204 from 786858 to 1347743 tid 1 2023-08-27T20:53:08.882 INFO:tasks.rados.rados.0.smithi093.stdout:1567: writing smithi093153278-204 from 1835183 to 2386858 tid 2 2023-08-27T20:53:08.882 INFO:tasks.rados.rados.0.smithi093.stdout:1556: done (11 left) 2023-08-27T20:53:08.882 INFO:tasks.rados.rados.0.smithi093.stdout:1557: done (10 left) 2023-08-27T20:53:08.882 INFO:tasks.rados.rados.0.smithi093.stdout:1558: done (9 left) 2023-08-27T20:53:08.882 INFO:tasks.rados.rados.0.smithi093.stdout:1559: done (8 left) 2023-08-27T20:53:08.882 INFO:tasks.rados.rados.0.smithi093.stdout:1560: done (7 left) 2023-08-27T20:53:08.883 INFO:tasks.rados.rados.0.smithi093.stdout:1561: done (6 left) 2023-08-27T20:53:08.883 INFO:tasks.rados.rados.0.smithi093.stdout:1562: done (5 left) 2023-08-27T20:53:08.883 INFO:tasks.rados.rados.0.smithi093.stdout:1568: write oid 419 current snap is 0 2023-08-27T20:53:08.883 INFO:tasks.rados.rados.0.smithi093.stdout:1563: finishing write tid 4 to smithi093153278-395 2023-08-27T20:53:08.883 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 310 (ObjNum 1090 snap 0 seq_num 1090) dirty exists 2023-08-27T20:53:08.883 INFO:tasks.rados.rados.0.smithi093.stdout:1563: left oid 395 (ObjNum 1090 snap 0 seq_num 1090) 2023-08-27T20:53:08.883 INFO:tasks.rados.rados.0.smithi093.stdout:1568: seq_num 1095 ranges {403665=752487,1610018=514716} 2023-08-27T20:53:08.888 INFO:tasks.rados.rados.0.smithi093.stdout:1568: writing smithi093153278-419 from 403665 to 1156152 tid 1 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1568: writing smithi093153278-419 from 1610018 to 2124734 tid 2 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1563: done (5 left) 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1569: read oid 222 snap -1 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1569: expect (ObjNum 991 snap 0 seq_num 991) 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1570: read oid 115 snap -1 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1570: expect (ObjNum 1042 snap 0 seq_num 1042) 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1571: write oid 487 current snap is 0 2023-08-27T20:53:08.894 INFO:tasks.rados.rados.0.smithi093.stdout:1564: finishing write tid 1 to smithi093153278-110 2023-08-27T20:53:08.895 INFO:tasks.rados.rados.0.smithi093.stdout:1571: seq_num 1096 ranges {420472=427736,1268500=658374,2347953=72519} 2023-08-27T20:53:08.899 INFO:tasks.rados.rados.0.smithi093.stdout:1571: writing smithi093153278-487 from 420472 to 848208 tid 1 2023-08-27T20:53:08.905 INFO:tasks.rados.rados.0.smithi093.stdout:1571: writing smithi093153278-487 from 1268500 to 1926874 tid 2 2023-08-27T20:53:08.908 INFO:tasks.rados.rados.0.smithi093.stdout:1571: writing smithi093153278-487 from 2347953 to 2420472 tid 3 2023-08-27T20:53:08.908 INFO:tasks.rados.rados.0.smithi093.stdout:1572: write oid 426 current snap is 0 2023-08-27T20:53:08.908 INFO:tasks.rados.rados.0.smithi093.stdout:1564: finishing write tid 2 to smithi093153278-110 2023-08-27T20:53:08.908 INFO:tasks.rados.rados.0.smithi093.stdout:1564: finishing write tid 3 to smithi093153278-110 2023-08-27T20:53:08.908 INFO:tasks.rados.rados.0.smithi093.stdout:1572: seq_num 1097 ranges {437279=502985,1610629=518385} 2023-08-27T20:53:08.913 INFO:tasks.rados.rados.0.smithi093.stdout:1572: writing smithi093153278-426 from 437279 to 940264 tid 1 2023-08-27T20:53:08.920 INFO:tasks.rados.rados.0.smithi093.stdout:1572: writing smithi093153278-426 from 1610629 to 2129014 tid 2 2023-08-27T20:53:08.920 INFO:tasks.rados.rados.0.smithi093.stdout:1573: read oid 123 snap -1 2023-08-27T20:53:08.920 INFO:tasks.rados.rados.0.smithi093.stdout:1573: expect (ObjNum 821 snap 0 seq_num 821) 2023-08-27T20:53:08.920 INFO:tasks.rados.rados.0.smithi093.stdout:1574: write oid 122 current snap is 0 2023-08-27T20:53:08.920 INFO:tasks.rados.rados.0.smithi093.stdout:1564: finishing write tid 4 to smithi093153278-110 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 110 v 242 (ObjNum 1091 snap 0 seq_num 1091) dirty exists 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1564: left oid 110 (ObjNum 1091 snap 0 seq_num 1091) 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1565: finishing write tid 1 to smithi093153278-156 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1565: finishing write tid 2 to smithi093153278-156 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1565: finishing write tid 3 to smithi093153278-156 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1565: finishing write tid 4 to smithi093153278-156 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 156 v 213 (ObjNum 1092 snap 0 seq_num 1092) dirty exists 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1565: left oid 156 (ObjNum 1092 snap 0 seq_num 1092) 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1566: finishing write tid 1 to smithi093153278-200 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1566: finishing write tid 2 to smithi093153278-200 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1566: finishing write tid 3 to smithi093153278-200 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:1566: finishing write tid 4 to smithi093153278-200 2023-08-27T20:53:08.921 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 200 v 253 (ObjNum 1093 snap 0 seq_num 1093) dirty exists 2023-08-27T20:53:08.922 INFO:tasks.rados.rados.0.smithi093.stdout:1566: left oid 200 (ObjNum 1093 snap 0 seq_num 1093) 2023-08-27T20:53:08.922 INFO:tasks.rados.rados.0.smithi093.stdout:1567: finishing write tid 1 to smithi093153278-204 2023-08-27T20:53:08.922 INFO:tasks.rados.rados.0.smithi093.stdout:1574: seq_num 1098 ranges {454086=578234,1669111=662043} 2023-08-27T20:53:08.925 INFO:tasks.rados.rados.0.smithi093.stdout:1574: writing smithi093153278-122 from 454086 to 1032320 tid 1 2023-08-27T20:53:08.933 INFO:tasks.rados.rados.0.smithi093.stdout:1574: writing smithi093153278-122 from 1669111 to 2331154 tid 2 2023-08-27T20:53:08.933 INFO:tasks.rados.rados.0.smithi093.stdout:1564: done (10 left) 2023-08-27T20:53:08.933 INFO:tasks.rados.rados.0.smithi093.stdout:1565: done (9 left) 2023-08-27T20:53:08.933 INFO:tasks.rados.rados.0.smithi093.stdout:1566: done (8 left) 2023-08-27T20:53:08.934 INFO:tasks.rados.rados.0.smithi093.stdout:1575: read oid 15 snap -1 2023-08-27T20:53:08.934 INFO:tasks.rados.rados.0.smithi093.stdout:1575: expect (ObjNum 1048 snap 0 seq_num 1048) 2023-08-27T20:53:08.934 INFO:tasks.rados.rados.0.smithi093.stdout:1576: write oid 71 current snap is 0 2023-08-27T20:53:08.934 INFO:tasks.rados.rados.0.smithi093.stdout:1567: finishing write tid 2 to smithi093153278-204 2023-08-27T20:53:08.934 INFO:tasks.rados.rados.0.smithi093.stdout:1576: seq_num 1099 ranges {470893=653483,1727593=522054} 2023-08-27T20:53:08.940 INFO:tasks.rados.rados.0.smithi093.stdout:1576: writing smithi093153278-71 from 470893 to 1124376 tid 1 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1576: writing smithi093153278-71 from 1727593 to 2249647 tid 2 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1577: read oid 80 snap -1 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1577: expect (ObjNum 744 snap 0 seq_num 744) 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1578: read oid 10 snap -1 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1578: expect deleted 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1579: write oid 443 current snap is 0 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1567: finishing write tid 3 to smithi093153278-204 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:1567: finishing write tid 4 to smithi093153278-204 2023-08-27T20:53:08.947 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 204 v 245 (ObjNum 1094 snap 0 seq_num 1094) dirty exists 2023-08-27T20:53:08.948 INFO:tasks.rados.rados.0.smithi093.stdout:1567: left oid 204 (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:53:08.948 INFO:tasks.rados.rados.0.smithi093.stdout:1568: finishing write tid 1 to smithi093153278-419 2023-08-27T20:53:08.948 INFO:tasks.rados.rados.0.smithi093.stdout:1579: seq_num 1100 ranges {487700=728732,1786075=665712} 2023-08-27T20:53:08.953 INFO:tasks.rados.rados.0.smithi093.stdout:1579: writing smithi093153278-443 from 487700 to 1216432 tid 1 2023-08-27T20:53:08.960 INFO:tasks.rados.rados.0.smithi093.stdout:1579: writing smithi093153278-443 from 1786075 to 2451787 tid 2 2023-08-27T20:53:08.961 INFO:tasks.rados.rados.0.smithi093.stdout:1567: done (12 left) 2023-08-27T20:53:08.961 INFO:tasks.rados.rados.0.smithi093.stdout:1580: write oid 409 current snap is 0 2023-08-27T20:53:08.961 INFO:tasks.rados.rados.0.smithi093.stdout:1568: finishing write tid 2 to smithi093153278-419 2023-08-27T20:53:08.961 INFO:tasks.rados.rados.0.smithi093.stdout:1568: finishing write tid 3 to smithi093153278-419 2023-08-27T20:53:08.961 INFO:tasks.rados.rados.0.smithi093.stdout:1580: seq_num 1101 ranges {504507=403981,1328204=409370,2252362=252145} 2023-08-27T20:53:08.965 INFO:tasks.rados.rados.0.smithi093.stdout:1580: writing smithi093153278-409 from 504507 to 908488 tid 1 2023-08-27T20:53:08.970 INFO:tasks.rados.rados.0.smithi093.stdout:1580: writing smithi093153278-409 from 1328204 to 1737574 tid 2 2023-08-27T20:53:08.974 INFO:tasks.rados.rados.0.smithi093.stdout:1580: writing smithi093153278-409 from 2252362 to 2504507 tid 3 2023-08-27T20:53:08.975 INFO:tasks.rados.rados.0.smithi093.stdout:1581: write oid 203 current snap is 0 2023-08-27T20:53:08.975 INFO:tasks.rados.rados.0.smithi093.stdout:1568: finishing write tid 4 to smithi093153278-419 2023-08-27T20:53:08.975 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 419 v 256 (ObjNum 1095 snap 0 seq_num 1095) dirty exists 2023-08-27T20:53:08.975 INFO:tasks.rados.rados.0.smithi093.stdout:1568: left oid 419 (ObjNum 1095 snap 0 seq_num 1095) 2023-08-27T20:53:08.975 INFO:tasks.rados.rados.0.smithi093.stdout:1569: expect (ObjNum 991 snap 0 seq_num 991) 2023-08-27T20:53:08.988 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:08 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:08.979+0000 7f6d2fa7c1c0 -1 mgr[py] Module osd_perf_query has missing NOTIFY_TYPES member 2023-08-27T20:53:09.058 INFO:tasks.rados.rados.0.smithi093.stdout:1570: expect (ObjNum 1042 snap 0 seq_num 1042) 2023-08-27T20:53:09.262 INFO:tasks.rados.rados.0.smithi093.stdout:1571: finishing write tid 1 to smithi093153278-487 2023-08-27T20:53:09.262 INFO:tasks.rados.rados.0.smithi093.stdout:1571: finishing write tid 2 to smithi093153278-487 2023-08-27T20:53:09.262 INFO:tasks.rados.rados.0.smithi093.stdout:1571: finishing write tid 3 to smithi093153278-487 2023-08-27T20:53:09.262 INFO:tasks.rados.rados.0.smithi093.stdout:1571: finishing write tid 4 to smithi093153278-487 2023-08-27T20:53:09.263 INFO:tasks.rados.rados.0.smithi093.stdout:1571: finishing write tid 5 to smithi093153278-487 2023-08-27T20:53:09.263 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 487 v 243 (ObjNum 1096 snap 0 seq_num 1096) dirty exists 2023-08-27T20:53:09.263 INFO:tasks.rados.rados.0.smithi093.stdout:1571: left oid 487 (ObjNum 1096 snap 0 seq_num 1096) 2023-08-27T20:53:09.263 INFO:tasks.rados.rados.0.smithi093.stdout:1572: finishing write tid 1 to smithi093153278-426 2023-08-27T20:53:09.263 INFO:tasks.rados.rados.0.smithi093.stdout:1581: seq_num 1102 ranges {521314=479230,1786686=669381} 2023-08-27T20:53:09.267 INFO:tasks.rados.rados.0.smithi093.stdout:1581: writing smithi093153278-203 from 521314 to 1000544 tid 1 2023-08-27T20:53:09.276 INFO:tasks.rados.rados.0.smithi093.stdout:1581: writing smithi093153278-203 from 1786686 to 2456067 tid 2 2023-08-27T20:53:09.276 INFO:tasks.rados.rados.0.smithi093.stdout:1568: done (13 left) 2023-08-27T20:53:09.276 INFO:tasks.rados.rados.0.smithi093.stdout:1569: done (12 left) 2023-08-27T20:53:09.276 INFO:tasks.rados.rados.0.smithi093.stdout:1570: done (11 left) 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1571: done (10 left) 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1582: read oid 320 snap -1 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1582: expect (ObjNum 517 snap 0 seq_num 517) 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1583: read oid 275 snap -1 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1583: expect (ObjNum 1067 snap 0 seq_num 1067) 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1584: read oid 42 snap -1 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1584: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1585: read oid 482 snap -1 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1585: expect (ObjNum 481 snap 0 seq_num 481) 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1586: read oid 486 snap -1 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1586: expect (ObjNum 634 snap 0 seq_num 634) 2023-08-27T20:53:09.277 INFO:tasks.rados.rados.0.smithi093.stdout:1587: write oid 332 current snap is 0 2023-08-27T20:53:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:1572: finishing write tid 2 to smithi093153278-426 2023-08-27T20:53:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:1572: finishing write tid 3 to smithi093153278-426 2023-08-27T20:53:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:1572: finishing write tid 4 to smithi093153278-426 2023-08-27T20:53:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 426 v 246 (ObjNum 1097 snap 0 seq_num 1097) dirty exists 2023-08-27T20:53:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:1572: left oid 426 (ObjNum 1097 snap 0 seq_num 1097) 2023-08-27T20:53:09.278 INFO:tasks.rados.rados.0.smithi093.stdout:1573: expect (ObjNum 821 snap 0 seq_num 821) 2023-08-27T20:53:09.396 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:09 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:09.074+0000 7f6d2fa7c1c0 -1 mgr[py] Module selftest has missing NOTIFY_TYPES member 2023-08-27T20:53:09.396 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:09 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:09.299+0000 7f6d2fa7c1c0 -1 mgr[py] Module volumes has missing NOTIFY_TYPES member 2023-08-27T20:53:09.453 INFO:tasks.rados.rados.0.smithi093.stdout:1574: finishing write tid 1 to smithi093153278-122 2023-08-27T20:53:09.453 INFO:tasks.rados.rados.0.smithi093.stdout:1587: seq_num 1103 ranges {538121=670832,1961521=413039} 2023-08-27T20:53:09.460 INFO:tasks.rados.rados.0.smithi093.stdout:1587: writing smithi093153278-332 from 538121 to 1208953 tid 1 2023-08-27T20:53:09.467 INFO:tasks.rados.rados.0.smithi093.stdout:1587: writing smithi093153278-332 from 1961521 to 2374560 tid 2 2023-08-27T20:53:09.467 INFO:tasks.rados.rados.0.smithi093.stdout:1572: done (15 left) 2023-08-27T20:53:09.467 INFO:tasks.rados.rados.0.smithi093.stdout:1573: done (14 left) 2023-08-27T20:53:09.467 INFO:tasks.rados.rados.0.smithi093.stdout:1588: read oid 137 snap -1 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:1588: expect (ObjNum 786 snap 0 seq_num 786) 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:1589: write oid 4 current snap is 0 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:1574: finishing write tid 2 to smithi093153278-122 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:1574: finishing write tid 3 to smithi093153278-122 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:1574: finishing write tid 4 to smithi093153278-122 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 122 v 157 (ObjNum 1098 snap 0 seq_num 1098) dirty exists 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:1574: left oid 122 (ObjNum 1098 snap 0 seq_num 1098) 2023-08-27T20:53:09.468 INFO:tasks.rados.rados.0.smithi093.stdout:1575: expect (ObjNum 1048 snap 0 seq_num 1048) 2023-08-27T20:53:09.621 INFO:tasks.rados.rados.0.smithi093.stdout:1576: finishing write tid 1 to smithi093153278-71 2023-08-27T20:53:09.621 INFO:tasks.rados.rados.0.smithi093.stdout:1576: finishing write tid 2 to smithi093153278-71 2023-08-27T20:53:09.621 INFO:tasks.rados.rados.0.smithi093.stdout:1589: seq_num 1104 ranges {554928=746081,2020003=534925} 2023-08-27T20:53:09.628 INFO:tasks.rados.rados.0.smithi093.stdout:1589: writing smithi093153278-4 from 554928 to 1301009 tid 1 2023-08-27T20:53:09.635 INFO:tasks.rados.rados.0.smithi093.stdout:1589: writing smithi093153278-4 from 2020003 to 2554928 tid 2 2023-08-27T20:53:09.635 INFO:tasks.rados.rados.0.smithi093.stdout:1574: done (15 left) 2023-08-27T20:53:09.636 INFO:tasks.rados.rados.0.smithi093.stdout:1575: done (14 left) 2023-08-27T20:53:09.636 INFO:tasks.rados.rados.0.smithi093.stdout:1578: done (13 left) 2023-08-27T20:53:09.636 INFO:tasks.rados.rados.0.smithi093.stdout:1590: write oid 243 current snap is 0 2023-08-27T20:53:09.636 INFO:tasks.rados.rados.0.smithi093.stdout:1576: finishing write tid 3 to smithi093153278-71 2023-08-27T20:53:09.636 INFO:tasks.rados.rados.0.smithi093.stdout:1590: seq_num 1105 ranges {571735=421330,1562132=416708} 2023-08-27T20:53:09.640 INFO:tasks.rados.rados.0.smithi093.stdout:1590: writing smithi093153278-243 from 571735 to 993065 tid 1 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:1590: writing smithi093153278-243 from 1562132 to 1978840 tid 2 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:1591: read oid 321 snap -1 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:1591: expect (ObjNum 1025 snap 0 seq_num 1025) 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:1592: read oid 352 snap -1 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:1592: expect (ObjNum 351 snap 0 seq_num 351) 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:1576: finishing write tid 4 to smithi093153278-71 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 71 v 216 (ObjNum 1099 snap 0 seq_num 1099) dirty exists 2023-08-27T20:53:09.647 INFO:tasks.rados.rados.0.smithi093.stdout:1576: left oid 71 (ObjNum 1099 snap 0 seq_num 1099) 2023-08-27T20:53:09.648 INFO:tasks.rados.rados.0.smithi093.stdout:1577: expect (ObjNum 744 snap 0 seq_num 744) 2023-08-27T20:53:09.675 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:09 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:09.396+0000 7f6d2fa7c1c0 -1 mgr[py] Module pg_autoscaler has missing NOTIFY_TYPES member 2023-08-27T20:53:09.675 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:09 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:09.584+0000 7f6d2fa7c1c0 -1 mgr[py] Module snap_schedule has missing NOTIFY_TYPES member 2023-08-27T20:53:09.676 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:09 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:09.675+0000 7f6d2fa7c1c0 -1 mgr[py] Module osd_support has missing NOTIFY_TYPES member 2023-08-27T20:53:09.763 INFO:tasks.rados.rados.0.smithi093.stdout:1579: finishing write tid 1 to smithi093153278-443 2023-08-27T20:53:09.763 INFO:tasks.rados.rados.0.smithi093.stdout:1579: finishing write tid 2 to smithi093153278-443 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1579: finishing write tid 3 to smithi093153278-443 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1579: finishing write tid 4 to smithi093153278-443 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 248 (ObjNum 1100 snap 0 seq_num 1100) dirty exists 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1579: left oid 443 (ObjNum 1100 snap 0 seq_num 1100) 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1580: finishing write tid 1 to smithi093153278-409 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1576: done (15 left) 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1577: done (14 left) 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1579: done (13 left) 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1593: read oid 81 snap -1 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1593: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1594: read oid 277 snap -1 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1594: expect (ObjNum 276 snap 0 seq_num 276) 2023-08-27T20:53:09.764 INFO:tasks.rados.rados.0.smithi093.stdout:1595: read oid 99 snap -1 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1595: expect (ObjNum 969 snap 0 seq_num 969) 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1580: finishing write tid 2 to smithi093153278-409 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1580: finishing write tid 3 to smithi093153278-409 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1580: finishing write tid 4 to smithi093153278-409 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1580: finishing write tid 5 to smithi093153278-409 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 409 v 211 (ObjNum 1101 snap 0 seq_num 1101) dirty exists 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1580: left oid 409 (ObjNum 1101 snap 0 seq_num 1101) 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1581: finishing write tid 1 to smithi093153278-203 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1581: finishing write tid 2 to smithi093153278-203 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1581: finishing write tid 3 to smithi093153278-203 2023-08-27T20:53:09.765 INFO:tasks.rados.rados.0.smithi093.stdout:1580: done (15 left) 2023-08-27T20:53:09.766 INFO:tasks.rados.rados.0.smithi093.stdout:1596: write oid 69 current snap is 0 2023-08-27T20:53:09.766 INFO:tasks.rados.rados.0.smithi093.stdout:1581: finishing write tid 4 to smithi093153278-203 2023-08-27T20:53:09.766 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 203 v 208 (ObjNum 1102 snap 0 seq_num 1102) dirty exists 2023-08-27T20:53:09.766 INFO:tasks.rados.rados.0.smithi093.stdout:1581: left oid 203 (ObjNum 1102 snap 0 seq_num 1102) 2023-08-27T20:53:09.766 INFO:tasks.rados.rados.0.smithi093.stdout:1596: seq_num 1106 ranges {588542=496579,1620614=676719} 2023-08-27T20:53:09.769 INFO:tasks.rados.rados.0.smithi093.stdout:1596: writing smithi093153278-69 from 588542 to 1085121 tid 1 2023-08-27T20:53:09.776 INFO:tasks.rados.rados.0.smithi093.stdout:1596: writing smithi093153278-69 from 1620614 to 2297333 tid 2 2023-08-27T20:53:09.776 INFO:tasks.rados.rados.0.smithi093.stdout:1581: done (15 left) 2023-08-27T20:53:09.776 INFO:tasks.rados.rados.0.smithi093.stdout:1597: write oid 219 current snap is 0 2023-08-27T20:53:09.776 INFO:tasks.rados.rados.0.smithi093.stdout:1597: seq_num 1107 ranges {605349=571828,1679096=420377} 2023-08-27T20:53:09.782 INFO:tasks.rados.rados.0.smithi093.stdout:1597: writing smithi093153278-219 from 605349 to 1177177 tid 1 2023-08-27T20:53:09.787 INFO:tasks.rados.rados.0.smithi093.stdout:1597: writing smithi093153278-219 from 1679096 to 2099473 tid 2 2023-08-27T20:53:09.787 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:53:09.788 INFO:tasks.rados.rados.0.smithi093.stdout:1586: expect (ObjNum 634 snap 0 seq_num 634) 2023-08-27T20:53:09.947 INFO:tasks.rados.rados.0.smithi093.stdout:1583: expect (ObjNum 1067 snap 0 seq_num 1067) 2023-08-27T20:53:10.133 INFO:tasks.rados.rados.0.smithi093.stdout:1582: expect (ObjNum 517 snap 0 seq_num 517) 2023-08-27T20:53:10.239 INFO:tasks.rados.rados.0.smithi093.stdout:1584: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:53:10.325 INFO:tasks.rados.rados.0.smithi093.stdout:1585: expect (ObjNum 481 snap 0 seq_num 481) 2023-08-27T20:53:10.390 INFO:tasks.rados.rados.0.smithi093.stdout:1587: finishing write tid 1 to smithi093153278-332 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1587: finishing write tid 2 to smithi093153278-332 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1587: finishing write tid 3 to smithi093153278-332 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1582: done (15 left) 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1583: done (14 left) 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1584: done (13 left) 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1585: done (12 left) 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1586: done (11 left) 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1598: write oid 110 current snap is 0 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:1587: finishing write tid 4 to smithi093153278-332 2023-08-27T20:53:10.391 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 332 v 205 (ObjNum 1103 snap 0 seq_num 1103) dirty exists 2023-08-27T20:53:10.392 INFO:tasks.rados.rados.0.smithi093.stdout:1587: left oid 332 (ObjNum 1103 snap 0 seq_num 1103) 2023-08-27T20:53:10.392 INFO:tasks.rados.rados.0.smithi093.stdout:1588: expect (ObjNum 786 snap 0 seq_num 786) 2023-08-27T20:53:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2587854512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:10.466 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2587854512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:10.466 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2587854512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:10.529 INFO:tasks.rados.rados.0.smithi093.stdout:1589: finishing write tid 1 to smithi093153278-4 2023-08-27T20:53:10.530 INFO:tasks.rados.rados.0.smithi093.stdout:1589: finishing write tid 2 to smithi093153278-4 2023-08-27T20:53:10.530 INFO:tasks.rados.rados.0.smithi093.stdout:1598: seq_num 1108 ranges {622156=647077,1737578=680388} 2023-08-27T20:53:10.536 INFO:tasks.rados.rados.0.smithi093.stdout:1598: writing smithi093153278-110 from 622156 to 1269233 tid 1 2023-08-27T20:53:10.543 INFO:tasks.rados.rados.0.smithi093.stdout:1598: writing smithi093153278-110 from 1737578 to 2417966 tid 2 2023-08-27T20:53:10.543 INFO:tasks.rados.rados.0.smithi093.stdout:1587: done (11 left) 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:1588: done (10 left) 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:1599: write oid 478 current snap is 0 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:1589: finishing write tid 3 to smithi093153278-4 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:1589: finishing write tid 4 to smithi093153278-4 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 4 v 248 (ObjNum 1104 snap 0 seq_num 1104) dirty exists 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:1589: left oid 4 (ObjNum 1104 snap 0 seq_num 1104) 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:1590: finishing write tid 1 to smithi093153278-243 2023-08-27T20:53:10.544 INFO:tasks.rados.rados.0.smithi093.stdout:1592: expect (ObjNum 351 snap 0 seq_num 351) 2023-08-27T20:53:10.668 INFO:tasks.rados.rados.0.smithi093.stdout:1590: finishing write tid 2 to smithi093153278-243 2023-08-27T20:53:10.668 INFO:tasks.rados.rados.0.smithi093.stdout:1590: finishing write tid 3 to smithi093153278-243 2023-08-27T20:53:10.668 INFO:tasks.rados.rados.0.smithi093.stdout:1599: seq_num 1109 ranges {638963=722326,2079707=424046} 2023-08-27T20:53:10.675 INFO:tasks.rados.rados.0.smithi093.stdout:1599: writing smithi093153278-478 from 638963 to 1361289 tid 1 2023-08-27T20:53:10.682 INFO:tasks.rados.rados.0.smithi093.stdout:1599: writing smithi093153278-478 from 2079707 to 2503753 tid 2 2023-08-27T20:53:10.682 INFO:tasks.rados.rados.0.smithi093.stdout:1589: done (10 left) 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:1592: done (9 left) 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:1600: read oid 329 snap -1 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:1600: expect (ObjNum 328 snap 0 seq_num 328) 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:1601: write oid 491 current snap is 0 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:1590: finishing write tid 4 to smithi093153278-243 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 243 v 251 (ObjNum 1105 snap 0 seq_num 1105) dirty exists 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:1590: left oid 243 (ObjNum 1105 snap 0 seq_num 1105) 2023-08-27T20:53:10.683 INFO:tasks.rados.rados.0.smithi093.stdout:1591: expect (ObjNum 1025 snap 0 seq_num 1025) 2023-08-27T20:53:10.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:10.466+0000 7f6d2fa7c1c0 -1 mgr[py] Module influx has missing NOTIFY_TYPES member 2023-08-27T20:53:10.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:10.667+0000 7f6d2fa7c1c0 -1 mgr[py] Module nfs has missing NOTIFY_TYPES member 2023-08-27T20:53:10.822 INFO:tasks.rados.rados.0.smithi093.stdout:1593: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:53:11.044 INFO:tasks.rados.rados.0.smithi093.stdout:1594: expect (ObjNum 276 snap 0 seq_num 276) 2023-08-27T20:53:11.128 INFO:tasks.rados.rados.0.smithi093.stdout:1595: expect (ObjNum 969 snap 0 seq_num 969) 2023-08-27T20:53:11.219 INFO:tasks.rados.rados.0.smithi093.stdout:1596: finishing write tid 1 to smithi093153278-69 2023-08-27T20:53:11.219 INFO:tasks.rados.rados.0.smithi093.stdout:1596: finishing write tid 2 to smithi093153278-69 2023-08-27T20:53:11.219 INFO:tasks.rados.rados.0.smithi093.stdout:1596: finishing write tid 3 to smithi093153278-69 2023-08-27T20:53:11.219 INFO:tasks.rados.rados.0.smithi093.stdout:1596: finishing write tid 4 to smithi093153278-69 2023-08-27T20:53:11.219 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 69 v 251 (ObjNum 1106 snap 0 seq_num 1106) dirty exists 2023-08-27T20:53:11.220 INFO:tasks.rados.rados.0.smithi093.stdout:1596: left oid 69 (ObjNum 1106 snap 0 seq_num 1106) 2023-08-27T20:53:11.220 INFO:tasks.rados.rados.0.smithi093.stdout:1597: finishing write tid 1 to smithi093153278-219 2023-08-27T20:53:11.220 INFO:tasks.rados.rados.0.smithi093.stdout:1597: finishing write tid 2 to smithi093153278-219 2023-08-27T20:53:11.220 INFO:tasks.rados.rados.0.smithi093.stdout:1601: seq_num 1110 ranges {655770=513928,1854542=684057} 2023-08-27T20:53:11.225 INFO:tasks.rados.rados.0.smithi093.stdout:1601: writing smithi093153278-491 from 655770 to 1169698 tid 1 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1601: writing smithi093153278-491 from 1854542 to 2538599 tid 2 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1590: done (10 left) 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1591: done (9 left) 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1593: done (8 left) 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1594: done (7 left) 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1595: done (6 left) 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1596: done (5 left) 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1602: read oid 275 snap -1 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1602: expect (ObjNum 1067 snap 0 seq_num 1067) 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1603: write oid 11 current snap is 0 2023-08-27T20:53:11.234 INFO:tasks.rados.rados.0.smithi093.stdout:1597: finishing write tid 3 to smithi093153278-219 2023-08-27T20:53:11.235 INFO:tasks.rados.rados.0.smithi093.stdout:1597: finishing write tid 4 to smithi093153278-219 2023-08-27T20:53:11.235 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 219 v 212 (ObjNum 1107 snap 0 seq_num 1107) dirty exists 2023-08-27T20:53:11.235 INFO:tasks.rados.rados.0.smithi093.stdout:1597: left oid 219 (ObjNum 1107 snap 0 seq_num 1107) 2023-08-27T20:53:11.235 INFO:tasks.rados.rados.0.smithi093.stdout:1598: finishing write tid 1 to smithi093153278-110 2023-08-27T20:53:11.235 INFO:tasks.rados.rados.0.smithi093.stdout:1603: seq_num 1111 ranges {672577=589177,1913024=427715} 2023-08-27T20:53:11.239 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:10.853+0000 7f6d2fa7c1c0 -1 mgr[py] Module orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:53:11.240 INFO:tasks.rados.rados.0.smithi093.stdout:1603: writing smithi093153278-11 from 672577 to 1261754 tid 1 2023-08-27T20:53:11.247 INFO:tasks.rados.rados.0.smithi093.stdout:1603: writing smithi093153278-11 from 1913024 to 2340739 tid 2 2023-08-27T20:53:11.247 INFO:tasks.rados.rados.0.smithi093.stdout:1597: done (6 left) 2023-08-27T20:53:11.247 INFO:tasks.rados.rados.0.smithi093.stdout:1604: read oid 150 snap -1 2023-08-27T20:53:11.247 INFO:tasks.rados.rados.0.smithi093.stdout:1604: expect (ObjNum 149 snap 0 seq_num 149) 2023-08-27T20:53:11.247 INFO:tasks.rados.rados.0.smithi093.stdout:1605: read oid 233 snap -1 2023-08-27T20:53:11.248 INFO:tasks.rados.rados.0.smithi093.stdout:1605: expect deleted 2023-08-27T20:53:11.248 INFO:tasks.rados.rados.0.smithi093.stdout:1606: read oid 449 snap -1 2023-08-27T20:53:11.248 INFO:tasks.rados.rados.0.smithi093.stdout:1606: expect (ObjNum 538 snap 0 seq_num 538) 2023-08-27T20:53:11.248 INFO:tasks.rados.rados.0.smithi093.stdout:1607: delete oid 336 current snap is 0 2023-08-27T20:53:11.248 INFO:tasks.rados.rados.0.smithi093.stdout:1598: finishing write tid 2 to smithi093153278-110 2023-08-27T20:53:11.248 INFO:tasks.rados.rados.0.smithi093.stdout:1598: finishing write tid 3 to smithi093153278-110 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1598: finishing write tid 4 to smithi093153278-110 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 110 v 254 (ObjNum 1108 snap 0 seq_num 1108) dirty exists 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1598: left oid 110 (ObjNum 1108 snap 0 seq_num 1108) 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1599: finishing write tid 1 to smithi093153278-478 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1599: finishing write tid 2 to smithi093153278-478 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1599: finishing write tid 3 to smithi093153278-478 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1599: finishing write tid 4 to smithi093153278-478 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 243 (ObjNum 1109 snap 0 seq_num 1109) dirty exists 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1599: left oid 478 (ObjNum 1109 snap 0 seq_num 1109) 2023-08-27T20:53:11.249 INFO:tasks.rados.rados.0.smithi093.stdout:1600: expect (ObjNum 328 snap 0 seq_num 328) 2023-08-27T20:53:11.365 INFO:tasks.rados.rados.0.smithi093.stdout:1601: finishing write tid 1 to smithi093153278-491 2023-08-27T20:53:11.365 INFO:tasks.rados.rados.0.smithi093.stdout:1601: finishing write tid 2 to smithi093153278-491 2023-08-27T20:53:11.365 INFO:tasks.rados.rados.0.smithi093.stdout:1602: expect (ObjNum 1067 snap 0 seq_num 1067) 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1601: finishing write tid 3 to smithi093153278-491 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1601: finishing write tid 4 to smithi093153278-491 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 491 v 249 (ObjNum 1110 snap 0 seq_num 1110) dirty exists 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1601: left oid 491 (ObjNum 1110 snap 0 seq_num 1110) 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1603: finishing write tid 1 to smithi093153278-11 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1598: done (9 left) 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1599: done (8 left) 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1600: done (7 left) 2023-08-27T20:53:11.548 INFO:tasks.rados.rados.0.smithi093.stdout:1601: done (6 left) 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1602: done (5 left) 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1607: done (4 left) 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1608: read oid 261 snap -1 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1608: expect deleted 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1609: delete oid 185 current snap is 0 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1603: finishing write tid 2 to smithi093153278-11 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1603: finishing write tid 3 to smithi093153278-11 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1603: finishing write tid 4 to smithi093153278-11 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 11 v 211 (ObjNum 1111 snap 0 seq_num 1111) dirty exists 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1603: left oid 11 (ObjNum 1111 snap 0 seq_num 1111) 2023-08-27T20:53:11.549 INFO:tasks.rados.rados.0.smithi093.stdout:1604: expect (ObjNum 149 snap 0 seq_num 149) 2023-08-27T20:53:11.628 INFO:tasks.rados.rados.0.smithi093.stdout:1606: expect (ObjNum 538 snap 0 seq_num 538) 2023-08-27T20:53:11.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:11.479+0000 7f6d2fa7c1c0 -1 mgr[py] Module prometheus has missing NOTIFY_TYPES member 2023-08-27T20:53:11.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:11.668+0000 7f6d2fa7c1c0 -1 mgr[py] Module test_orchestrator has missing NOTIFY_TYPES member 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1603: done (5 left) 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1604: done (4 left) 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1605: done (3 left) 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1606: done (2 left) 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1608: done (1 left) 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1609: done (0 left) 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1610: read oid 121 snap -1 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1610: expect (ObjNum 120 snap 0 seq_num 120) 2023-08-27T20:53:11.772 INFO:tasks.rados.rados.0.smithi093.stdout:1611: delete oid 217 current snap is 0 2023-08-27T20:53:11.773 INFO:tasks.rados.rados.0.smithi093.stdout:1610: expect (ObjNum 120 snap 0 seq_num 120) 2023-08-27T20:53:11.835 INFO:tasks.rados.rados.0.smithi093.stdout:1610: done (1 left) 2023-08-27T20:53:11.836 INFO:tasks.rados.rados.0.smithi093.stdout:1611: done (0 left) 2023-08-27T20:53:11.836 INFO:tasks.rados.rados.0.smithi093.stdout:1612: read oid 140 snap -1 2023-08-27T20:53:11.836 INFO:tasks.rados.rados.0.smithi093.stdout:1612: expect (ObjNum 139 snap 0 seq_num 139) 2023-08-27T20:53:11.836 INFO:tasks.rados.rados.0.smithi093.stdout:1613: write oid 255 current snap is 0 2023-08-27T20:53:11.836 INFO:tasks.rados.rados.0.smithi093.stdout:1613: seq_num 1115 ranges {739805=490173,1630599=435053,2636284=103521} 2023-08-27T20:53:11.841 INFO:tasks.rados.rados.0.smithi093.stdout:1613: writing smithi093153278-255 from 739805 to 1229978 tid 1 2023-08-27T20:53:11.846 INFO:tasks.rados.rados.0.smithi093.stdout:1613: writing smithi093153278-255 from 1630599 to 2065652 tid 2 2023-08-27T20:53:11.850 INFO:tasks.rados.rados.0.smithi093.stdout:1613: writing smithi093153278-255 from 2636284 to 2739805 tid 3 2023-08-27T20:53:11.850 INFO:tasks.rados.rados.0.smithi093.stdout:1614: write oid 64 current snap is 0 2023-08-27T20:53:11.850 INFO:tasks.rados.rados.0.smithi093.stdout:1614: seq_num 1116 ranges {756612=565422,2089081=578711} 2023-08-27T20:53:11.857 INFO:tasks.rados.rados.0.smithi093.stdout:1614: writing smithi093153278-64 from 756612 to 1322034 tid 1 2023-08-27T20:53:11.865 INFO:tasks.rados.rados.0.smithi093.stdout:1614: writing smithi093153278-64 from 2089081 to 2667792 tid 2 2023-08-27T20:53:11.865 INFO:tasks.rados.rados.0.smithi093.stdout:1615: write oid 263 current snap is 0 2023-08-27T20:53:11.865 INFO:tasks.rados.rados.0.smithi093.stdout:1612: expect (ObjNum 139 snap 0 seq_num 139) 2023-08-27T20:53:11.954 INFO:tasks.rados.rados.0.smithi093.stdout:1613: finishing write tid 1 to smithi093153278-255 2023-08-27T20:53:11.954 INFO:tasks.rados.rados.0.smithi093.stdout:1613: finishing write tid 2 to smithi093153278-255 2023-08-27T20:53:11.954 INFO:tasks.rados.rados.0.smithi093.stdout:1615: seq_num 1117 ranges {773419=640671,2147563=438722} 2023-08-27T20:53:11.963 INFO:tasks.rados.rados.0.smithi093.stdout:1615: writing smithi093153278-263 from 773419 to 1414090 tid 1 2023-08-27T20:53:11.971 INFO:tasks.rados.rados.0.smithi093.stdout:1615: writing smithi093153278-263 from 2147563 to 2586285 tid 2 2023-08-27T20:53:11.971 INFO:tasks.rados.rados.0.smithi093.stdout:1612: done (3 left) 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1616: delete oid 46 current snap is 0 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1613: finishing write tid 3 to smithi093153278-255 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1613: finishing write tid 4 to smithi093153278-255 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1613: finishing write tid 5 to smithi093153278-255 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 255 v 314 (ObjNum 1115 snap 0 seq_num 1115) dirty exists 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1613: left oid 255 (ObjNum 1115 snap 0 seq_num 1115) 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1614: finishing write tid 1 to smithi093153278-64 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1614: finishing write tid 2 to smithi093153278-64 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1614: finishing write tid 3 to smithi093153278-64 2023-08-27T20:53:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:1614: finishing write tid 4 to smithi093153278-64 2023-08-27T20:53:11.973 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 64 v 160 (ObjNum 1116 snap 0 seq_num 1116) dirty exists 2023-08-27T20:53:11.973 INFO:tasks.rados.rados.0.smithi093.stdout:1614: left oid 64 (ObjNum 1116 snap 0 seq_num 1116) 2023-08-27T20:53:11.973 INFO:tasks.rados.rados.0.smithi093.stdout:1615: finishing write tid 1 to smithi093153278-263 2023-08-27T20:53:11.973 INFO:tasks.rados.rados.0.smithi093.stdout:1613: done (3 left) 2023-08-27T20:53:11.974 INFO:tasks.rados.rados.0.smithi093.stdout:1614: done (2 left) 2023-08-27T20:53:11.974 INFO:tasks.rados.rados.0.smithi093.stdout:1616: done (1 left) 2023-08-27T20:53:11.974 INFO:tasks.rados.rados.0.smithi093.stdout:1617: read oid 28 snap -1 2023-08-27T20:53:11.974 INFO:tasks.rados.rados.0.smithi093.stdout:1617: expect (ObjNum 1063 snap 0 seq_num 1063) 2023-08-27T20:53:11.974 INFO:tasks.rados.rados.0.smithi093.stdout:1618: write oid 315 current snap is 0 2023-08-27T20:53:11.974 INFO:tasks.rados.rados.0.smithi093.stdout:1618: seq_num 1119 ranges {407033=507522,1464527=442391,2345976=461057} 2023-08-27T20:53:11.978 INFO:tasks.rados.rados.0.smithi093.stdout:1618: writing smithi093153278-315 from 407033 to 914555 tid 1 2023-08-27T20:53:11.983 INFO:tasks.rados.rados.0.smithi093.stdout:1618: writing smithi093153278-315 from 1464527 to 1906918 tid 2 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:1618: writing smithi093153278-315 from 2345976 to 2807033 tid 3 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:1619: read oid 176 snap -1 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:1619: expect (ObjNum 175 snap 0 seq_num 175) 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:1620: write oid 419 current snap is 0 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:1615: finishing write tid 2 to smithi093153278-263 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:1615: finishing write tid 3 to smithi093153278-263 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:1615: finishing write tid 4 to smithi093153278-263 2023-08-27T20:53:11.989 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 263 v 259 (ObjNum 1117 snap 0 seq_num 1117) dirty exists 2023-08-27T20:53:11.990 INFO:tasks.rados.rados.0.smithi093.stdout:1615: left oid 263 (ObjNum 1117 snap 0 seq_num 1117) 2023-08-27T20:53:11.990 INFO:tasks.rados.rados.0.smithi093.stdout:1618: finishing write tid 1 to smithi093153278-315 2023-08-27T20:53:11.990 INFO:tasks.rados.rados.0.smithi093.stdout:1617: expect (ObjNum 1063 snap 0 seq_num 1063) 2023-08-27T20:53:12.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[111335]: Standby manager daemon y started 2023-08-27T20:53:12.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-27T20:53:12.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:53:12.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-27T20:53:12.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[111335]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:53:12.126 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[118902]: Standby manager daemon y started 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-mon[118902]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:11.861+0000 7f6d2fa7c1c0 -1 mgr[py] Module devicehealth has missing NOTIFY_TYPES member 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:12.007+0000 7f6d2fa7c1c0 -1 mgr[py] Module rbd_support has missing NOTIFY_TYPES member 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:12] ENGINE Bus STARTING 2023-08-27T20:53:12.127 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: CherryPy Checker: 2023-08-27T20:53:12.174 INFO:tasks.rados.rados.0.smithi093.stdout:1618: finishing write tid 2 to smithi093153278-315 2023-08-27T20:53:12.174 INFO:tasks.rados.rados.0.smithi093.stdout:1620: seq_num 1120 ranges {423840=582771,1523009=586049,2657046=166794} 2023-08-27T20:53:12.178 INFO:tasks.rados.rados.0.smithi093.stdout:1620: writing smithi093153278-419 from 423840 to 1006611 tid 1 2023-08-27T20:53:12.184 INFO:tasks.rados.rados.0.smithi093.stdout:1620: writing smithi093153278-419 from 1523009 to 2109058 tid 2 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1620: writing smithi093153278-419 from 2657046 to 2823840 tid 3 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1615: done (4 left) 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1617: done (3 left) 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1621: read oid 458 snap -1 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1621: expect deleted 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1622: write oid 221 current snap is 0 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1618: finishing write tid 3 to smithi093153278-315 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1618: finishing write tid 4 to smithi093153278-315 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1618: finishing write tid 5 to smithi093153278-315 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 247 (ObjNum 1119 snap 0 seq_num 1119) dirty exists 2023-08-27T20:53:12.189 INFO:tasks.rados.rados.0.smithi093.stdout:1618: left oid 315 (ObjNum 1119 snap 0 seq_num 1119) 2023-08-27T20:53:12.190 INFO:tasks.rados.rados.0.smithi093.stdout:1619: expect (ObjNum 175 snap 0 seq_num 175) 2023-08-27T20:53:12.287 INFO:tasks.rados.rados.0.smithi093.stdout:1622: seq_num 1121 ranges {440647=658020,1581491=446060,2800822=39825} 2023-08-27T20:53:12.293 INFO:tasks.rados.rados.0.smithi093.stdout:1622: writing smithi093153278-221 from 440647 to 1098667 tid 1 2023-08-27T20:53:12.298 INFO:tasks.rados.rados.0.smithi093.stdout:1622: writing smithi093153278-221 from 1581491 to 2027551 tid 2 2023-08-27T20:53:12.303 INFO:tasks.rados.rados.0.smithi093.stdout:1622: writing smithi093153278-221 from 2800822 to 2840647 tid 3 2023-08-27T20:53:12.303 INFO:tasks.rados.rados.0.smithi093.stdout:1618: done (4 left) 2023-08-27T20:53:12.303 INFO:tasks.rados.rados.0.smithi093.stdout:1619: done (3 left) 2023-08-27T20:53:12.303 INFO:tasks.rados.rados.0.smithi093.stdout:1623: write oid 453 current snap is 0 2023-08-27T20:53:12.303 INFO:tasks.rados.rados.0.smithi093.stdout:1620: finishing write tid 1 to smithi093153278-419 2023-08-27T20:53:12.303 INFO:tasks.rados.rados.0.smithi093.stdout:1623: seq_num 1122 ranges {457454=733269,1923620=589718} 2023-08-27T20:53:12.309 INFO:tasks.rados.rados.0.smithi093.stdout:1623: writing smithi093153278-453 from 457454 to 1190723 tid 1 2023-08-27T20:53:12.317 INFO:tasks.rados.rados.0.smithi093.stdout:1623: writing smithi093153278-453 from 1923620 to 2513338 tid 2 2023-08-27T20:53:12.317 INFO:tasks.rados.rados.0.smithi093.stdout:1624: write oid 169 current snap is 0 2023-08-27T20:53:12.317 INFO:tasks.rados.rados.0.smithi093.stdout:1620: finishing write tid 2 to smithi093153278-419 2023-08-27T20:53:12.317 INFO:tasks.rados.rados.0.smithi093.stdout:1620: finishing write tid 3 to smithi093153278-419 2023-08-27T20:53:12.317 INFO:tasks.rados.rados.0.smithi093.stdout:1620: finishing write tid 4 to smithi093153278-419 2023-08-27T20:53:12.317 INFO:tasks.rados.rados.0.smithi093.stdout:1620: finishing write tid 5 to smithi093153278-419 2023-08-27T20:53:12.317 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 419 v 263 (ObjNum 1120 snap 0 seq_num 1120) dirty exists 2023-08-27T20:53:12.318 INFO:tasks.rados.rados.0.smithi093.stdout:1620: left oid 419 (ObjNum 1120 snap 0 seq_num 1120) 2023-08-27T20:53:12.318 INFO:tasks.rados.rados.0.smithi093.stdout:1624: seq_num 1123 ranges {474261=408518,1582102=449729,2739315=134946} 2023-08-27T20:53:12.321 INFO:tasks.rados.rados.0.smithi093.stdout:1624: writing smithi093153278-169 from 474261 to 882779 tid 1 2023-08-27T20:53:12.328 INFO:tasks.rados.rados.0.smithi093.stdout:1624: writing smithi093153278-169 from 1582102 to 2031831 tid 2 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1624: writing smithi093153278-169 from 2739315 to 2874261 tid 3 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1620: done (4 left) 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1621: done (3 left) 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1625: read oid 500 snap -1 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1625: expect (ObjNum 875 snap 0 seq_num 875) 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1626: write oid 405 current snap is 0 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1622: finishing write tid 1 to smithi093153278-221 2023-08-27T20:53:12.333 INFO:tasks.rados.rados.0.smithi093.stdout:1626: seq_num 1124 ranges {491068=483767,1640584=593387,2650385=240683} 2023-08-27T20:53:12.338 INFO:tasks.rados.rados.0.smithi093.stdout:1626: writing smithi093153278-405 from 491068 to 974835 tid 1 2023-08-27T20:53:12.345 INFO:tasks.rados.rados.0.smithi093.stdout:1626: writing smithi093153278-405 from 1640584 to 2233971 tid 2 2023-08-27T20:53:12.349 INFO:tasks.rados.rados.0.smithi093.stdout:1626: writing smithi093153278-405 from 2650385 to 2891068 tid 3 2023-08-27T20:53:12.349 INFO:tasks.rados.rados.0.smithi093.stdout:1627: read oid 343 snap -1 2023-08-27T20:53:12.349 INFO:tasks.rados.rados.0.smithi093.stdout:1627: expect (ObjNum 577 snap 0 seq_num 577) 2023-08-27T20:53:12.349 INFO:tasks.rados.rados.0.smithi093.stdout:1628: write oid 438 current snap is 0 2023-08-27T20:53:12.349 INFO:tasks.rados.rados.0.smithi093.stdout:1622: finishing write tid 2 to smithi093153278-221 2023-08-27T20:53:12.350 INFO:tasks.rados.rados.0.smithi093.stdout:1628: seq_num 1125 ranges {507875=559016,1699066=737045} 2023-08-27T20:53:12.356 INFO:tasks.rados.rados.0.smithi093.stdout:1628: writing smithi093153278-438 from 507875 to 1066891 tid 1 2023-08-27T20:53:12.365 INFO:tasks.rados.rados.0.smithi093.stdout:1628: writing smithi093153278-438 from 1699066 to 2436111 tid 2 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:1629: write oid 86 current snap is 0 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:1622: finishing write tid 3 to smithi093153278-221 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:1622: finishing write tid 4 to smithi093153278-221 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:1622: finishing write tid 5 to smithi093153278-221 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 221 v 319 (ObjNum 1121 snap 0 seq_num 1121) dirty exists 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:1622: left oid 221 (ObjNum 1121 snap 0 seq_num 1121) 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:1623: finishing write tid 1 to smithi093153278-453 2023-08-27T20:53:12.366 INFO:tasks.rados.rados.0.smithi093.stdout:1629: seq_num 1126 ranges {524682=750618,1757548=597056} 2023-08-27T20:53:12.374 INFO:tasks.rados.rados.0.smithi093.stdout:1629: writing smithi093153278-86 from 524682 to 1275300 tid 1 2023-08-27T20:53:12.381 INFO:tasks.rados.rados.0.smithi093.stdout:1629: writing smithi093153278-86 from 1757548 to 2354604 tid 2 2023-08-27T20:53:12.381 INFO:tasks.rados.rados.0.smithi093.stdout:1622: done (7 left) 2023-08-27T20:53:12.381 INFO:tasks.rados.rados.0.smithi093.stdout:1630: read oid 470 snap -1 2023-08-27T20:53:12.381 INFO:tasks.rados.rados.0.smithi093.stdout:1630: expect (ObjNum 1011 snap 0 seq_num 1011) 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1631: read oid 328 snap -1 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1631: expect (ObjNum 764 snap 0 seq_num 764) 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1632: write oid 29 current snap is 0 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1623: finishing write tid 2 to smithi093153278-453 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1623: finishing write tid 3 to smithi093153278-453 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1623: finishing write tid 4 to smithi093153278-453 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 453 v 224 (ObjNum 1122 snap 0 seq_num 1122) dirty exists 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1623: left oid 453 (ObjNum 1122 snap 0 seq_num 1122) 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1624: finishing write tid 1 to smithi093153278-169 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1624: finishing write tid 2 to smithi093153278-169 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1624: finishing write tid 3 to smithi093153278-169 2023-08-27T20:53:12.382 INFO:tasks.rados.rados.0.smithi093.stdout:1624: finishing write tid 4 to smithi093153278-169 2023-08-27T20:53:12.383 INFO:tasks.rados.rados.0.smithi093.stdout:1624: finishing write tid 5 to smithi093153278-169 2023-08-27T20:53:12.383 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 169 v 235 (ObjNum 1123 snap 0 seq_num 1123) dirty exists 2023-08-27T20:53:12.383 INFO:tasks.rados.rados.0.smithi093.stdout:1624: left oid 169 (ObjNum 1123 snap 0 seq_num 1123) 2023-08-27T20:53:12.383 INFO:tasks.rados.rados.0.smithi093.stdout:1632: seq_num 1127 ranges {541489=425867,1416030=740714,2732654=208835} 2023-08-27T20:53:12.386 INFO:tasks.rados.rados.0.smithi093.stdout:1632: writing smithi093153278-29 from 541489 to 967356 tid 1 2023-08-27T20:53:12.393 INFO:tasks.rados.rados.0.smithi093.stdout:1632: writing smithi093153278-29 from 1416030 to 2156744 tid 2 2023-08-27T20:53:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:1632: writing smithi093153278-29 from 2732654 to 2941489 tid 3 2023-08-27T20:53:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:1623: done (9 left) 2023-08-27T20:53:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:1624: done (8 left) 2023-08-27T20:53:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:1633: write oid 202 current snap is 0 2023-08-27T20:53:12.398 INFO:tasks.rados.rados.0.smithi093.stdout:1633: seq_num 1128 ranges {558296=501116,1474512=600725,2760077=198219} 2023-08-27T20:53:12.403 INFO:tasks.rados.rados.0.smithi093.stdout:1633: writing smithi093153278-202 from 558296 to 1059412 tid 1 2023-08-27T20:53:12.408 INFO:tasks.rados.rados.0.smithi093.stdout:1633: writing smithi093153278-202 from 1474512 to 2075237 tid 2 2023-08-27T20:53:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:1633: writing smithi093153278-202 from 2760077 to 2958296 tid 3 2023-08-27T20:53:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:1634: read oid 14 snap -1 2023-08-27T20:53:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:1634: expect (ObjNum 13 snap 0 seq_num 13) 2023-08-27T20:53:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:1635: write oid 266 current snap is 0 2023-08-27T20:53:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:1625: expect (ObjNum 875 snap 0 seq_num 875) 2023-08-27T20:53:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:12 smithi136 ceph-mon[113531]: Standby manager daemon y started 2023-08-27T20:53:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/crt"}]: dispatch 2023-08-27T20:53:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/crt"}]: dispatch 2023-08-27T20:53:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/y/key"}]: dispatch 2023-08-27T20:53:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:12 smithi136 ceph-mon[113531]: from='mgr.? 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config-key get", "key": "mgr/dashboard/key"}]: dispatch 2023-08-27T20:53:12.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: The Application mounted at '' has an empty config. 2023-08-27T20:53:12.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:12.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:12] ENGINE Serving on http://:::9283 2023-08-27T20:53:12.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:12] ENGINE Bus STARTED 2023-08-27T20:53:12.578 INFO:tasks.rados.rados.0.smithi093.stdout:1626: finishing write tid 1 to smithi093153278-405 2023-08-27T20:53:12.579 INFO:tasks.rados.rados.0.smithi093.stdout:1626: finishing write tid 2 to smithi093153278-405 2023-08-27T20:53:12.579 INFO:tasks.rados.rados.0.smithi093.stdout:1626: finishing write tid 3 to smithi093153278-405 2023-08-27T20:53:12.579 INFO:tasks.rados.rados.0.smithi093.stdout:1626: finishing write tid 4 to smithi093153278-405 2023-08-27T20:53:12.579 INFO:tasks.rados.rados.0.smithi093.stdout:1626: finishing write tid 5 to smithi093153278-405 2023-08-27T20:53:12.580 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 405 v 253 (ObjNum 1124 snap 0 seq_num 1124) dirty exists 2023-08-27T20:53:12.580 INFO:tasks.rados.rados.0.smithi093.stdout:1626: left oid 405 (ObjNum 1124 snap 0 seq_num 1124) 2023-08-27T20:53:12.580 INFO:tasks.rados.rados.0.smithi093.stdout:1635: seq_num 1129 ranges {575103=576365,1932994=744383} 2023-08-27T20:53:12.584 INFO:tasks.rados.rados.0.smithi093.stdout:1635: writing smithi093153278-266 from 575103 to 1151468 tid 1 2023-08-27T20:53:12.593 INFO:tasks.rados.rados.0.smithi093.stdout:1635: writing smithi093153278-266 from 1932994 to 2677377 tid 2 2023-08-27T20:53:12.593 INFO:tasks.rados.rados.0.smithi093.stdout:1625: done (10 left) 2023-08-27T20:53:12.593 INFO:tasks.rados.rados.0.smithi093.stdout:1626: done (9 left) 2023-08-27T20:53:12.593 INFO:tasks.rados.rados.0.smithi093.stdout:1636: write oid 192 current snap is 0 2023-08-27T20:53:12.593 INFO:tasks.rados.rados.0.smithi093.stdout:1627: expect (ObjNum 577 snap 0 seq_num 577) 2023-08-27T20:53:12.751 INFO:tasks.rados.rados.0.smithi093.stdout:1628: finishing write tid 1 to smithi093153278-438 2023-08-27T20:53:12.751 INFO:tasks.rados.rados.0.smithi093.stdout:1628: finishing write tid 2 to smithi093153278-438 2023-08-27T20:53:12.751 INFO:tasks.rados.rados.0.smithi093.stdout:1628: finishing write tid 3 to smithi093153278-438 2023-08-27T20:53:12.751 INFO:tasks.rados.rados.0.smithi093.stdout:1628: finishing write tid 4 to smithi093153278-438 2023-08-27T20:53:12.751 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 438 v 241 (ObjNum 1125 snap 0 seq_num 1125) dirty exists 2023-08-27T20:53:12.751 INFO:tasks.rados.rados.0.smithi093.stdout:1628: left oid 438 (ObjNum 1125 snap 0 seq_num 1125) 2023-08-27T20:53:12.752 INFO:tasks.rados.rados.0.smithi093.stdout:1629: finishing write tid 1 to smithi093153278-86 2023-08-27T20:53:12.752 INFO:tasks.rados.rados.0.smithi093.stdout:1636: seq_num 1130 ranges {591910=651614,1875123=604394} 2023-08-27T20:53:12.757 INFO:tasks.rados.rados.0.smithi093.stdout:1636: writing smithi093153278-192 from 591910 to 1243524 tid 1 2023-08-27T20:53:12.764 INFO:tasks.rados.rados.0.smithi093.stdout:1636: writing smithi093153278-192 from 1875123 to 2479517 tid 2 2023-08-27T20:53:12.764 INFO:tasks.rados.rados.0.smithi093.stdout:1627: done (9 left) 2023-08-27T20:53:12.764 INFO:tasks.rados.rados.0.smithi093.stdout:1628: done (8 left) 2023-08-27T20:53:12.765 INFO:tasks.rados.rados.0.smithi093.stdout:1637: read oid 23 snap -1 2023-08-27T20:53:12.765 INFO:tasks.rados.rados.0.smithi093.stdout:1637: expect (ObjNum 955 snap 0 seq_num 955) 2023-08-27T20:53:12.765 INFO:tasks.rados.rados.0.smithi093.stdout:1638: write oid 436 current snap is 0 2023-08-27T20:53:12.765 INFO:tasks.rados.rados.0.smithi093.stdout:1629: finishing write tid 2 to smithi093153278-86 2023-08-27T20:53:12.765 INFO:tasks.rados.rados.0.smithi093.stdout:1629: finishing write tid 3 to smithi093153278-86 2023-08-27T20:53:12.765 INFO:tasks.rados.rados.0.smithi093.stdout:1638: seq_num 1131 ranges {608717=726863,1933605=748052} 2023-08-27T20:53:12.771 INFO:tasks.rados.rados.0.smithi093.stdout:1638: writing smithi093153278-436 from 608717 to 1335580 tid 1 2023-08-27T20:53:12.779 INFO:tasks.rados.rados.0.smithi093.stdout:1638: writing smithi093153278-436 from 1933605 to 2681657 tid 2 2023-08-27T20:53:12.779 INFO:tasks.rados.rados.0.smithi093.stdout:1639: write oid 377 current snap is 0 2023-08-27T20:53:12.779 INFO:tasks.rados.rados.0.smithi093.stdout:1629: finishing write tid 4 to smithi093153278-86 2023-08-27T20:53:12.779 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 86 v 219 (ObjNum 1126 snap 0 seq_num 1126) dirty exists 2023-08-27T20:53:12.779 INFO:tasks.rados.rados.0.smithi093.stdout:1629: left oid 86 (ObjNum 1126 snap 0 seq_num 1126) 2023-08-27T20:53:12.779 INFO:tasks.rados.rados.0.smithi093.stdout:1630: expect (ObjNum 1011 snap 0 seq_num 1011) 2023-08-27T20:53:12.944 INFO:tasks.rados.rados.0.smithi093.stdout:1632: finishing write tid 1 to smithi093153278-29 2023-08-27T20:53:12.944 INFO:tasks.rados.rados.0.smithi093.stdout:1631: expect (ObjNum 764 snap 0 seq_num 764) 2023-08-27T20:53:13.099 INFO:tasks.rados.rados.0.smithi093.stdout:1632: finishing write tid 2 to smithi093153278-29 2023-08-27T20:53:13.099 INFO:tasks.rados.rados.0.smithi093.stdout:1632: finishing write tid 3 to smithi093153278-29 2023-08-27T20:53:13.099 INFO:tasks.rados.rados.0.smithi093.stdout:1632: finishing write tid 4 to smithi093153278-29 2023-08-27T20:53:13.099 INFO:tasks.rados.rados.0.smithi093.stdout:1632: finishing write tid 5 to smithi093153278-29 2023-08-27T20:53:13.099 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 29 v 258 (ObjNum 1127 snap 0 seq_num 1127) dirty exists 2023-08-27T20:53:13.099 INFO:tasks.rados.rados.0.smithi093.stdout:1632: left oid 29 (ObjNum 1127 snap 0 seq_num 1127) 2023-08-27T20:53:13.099 INFO:tasks.rados.rados.0.smithi093.stdout:1633: finishing write tid 1 to smithi093153278-202 2023-08-27T20:53:13.100 INFO:tasks.rados.rados.0.smithi093.stdout:1633: finishing write tid 2 to smithi093153278-202 2023-08-27T20:53:13.100 INFO:tasks.rados.rados.0.smithi093.stdout:1633: finishing write tid 3 to smithi093153278-202 2023-08-27T20:53:13.100 INFO:tasks.rados.rados.0.smithi093.stdout:1639: seq_num 1132 ranges {625524=402112,1592087=608063,2869769=155755} 2023-08-27T20:53:13.104 INFO:tasks.rados.rados.0.smithi093.stdout:1639: writing smithi093153278-377 from 625524 to 1027636 tid 1 2023-08-27T20:53:13.110 INFO:tasks.rados.rados.0.smithi093.stdout:1639: writing smithi093153278-377 from 1592087 to 2200150 tid 2 2023-08-27T20:53:13.115 INFO:tasks.rados.rados.0.smithi093.stdout:1639: writing smithi093153278-377 from 2869769 to 3025524 tid 3 2023-08-27T20:53:13.115 INFO:tasks.rados.rados.0.smithi093.stdout:1629: done (10 left) 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:1630: done (9 left) 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:1631: done (8 left) 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:1632: done (7 left) 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:1640: write oid 301 current snap is 0 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:1633: finishing write tid 4 to smithi093153278-202 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:1633: finishing write tid 5 to smithi093153278-202 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 202 v 256 (ObjNum 1128 snap 0 seq_num 1128) dirty exists 2023-08-27T20:53:13.116 INFO:tasks.rados.rados.0.smithi093.stdout:1633: left oid 202 (ObjNum 1128 snap 0 seq_num 1128) 2023-08-27T20:53:13.117 INFO:tasks.rados.rados.0.smithi093.stdout:1640: seq_num 1133 ranges {642331=593714,1766922=751721} 2023-08-27T20:53:13.121 INFO:tasks.rados.rados.0.smithi093.stdout:1640: writing smithi093153278-301 from 642331 to 1236045 tid 1 2023-08-27T20:53:13.129 INFO:tasks.rados.rados.0.smithi093.stdout:1640: writing smithi093153278-301 from 1766922 to 2518643 tid 2 2023-08-27T20:53:13.129 INFO:tasks.rados.rados.0.smithi093.stdout:1633: done (7 left) 2023-08-27T20:53:13.129 INFO:tasks.rados.rados.0.smithi093.stdout:1641: write oid 100 current snap is 0 2023-08-27T20:53:13.129 INFO:tasks.rados.rados.0.smithi093.stdout:1634: expect (ObjNum 13 snap 0 seq_num 13) 2023-08-27T20:53:13.199 INFO:tasks.rados.rados.0.smithi093.stdout:1635: finishing write tid 1 to smithi093153278-266 2023-08-27T20:53:13.199 INFO:tasks.rados.rados.0.smithi093.stdout:1635: finishing write tid 2 to smithi093153278-266 2023-08-27T20:53:13.199 INFO:tasks.rados.rados.0.smithi093.stdout:1635: finishing write tid 3 to smithi093153278-266 2023-08-27T20:53:13.199 INFO:tasks.rados.rados.0.smithi093.stdout:1635: finishing write tid 4 to smithi093153278-266 2023-08-27T20:53:13.199 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 266 v 208 (ObjNum 1129 snap 0 seq_num 1129) dirty exists 2023-08-27T20:53:13.199 INFO:tasks.rados.rados.0.smithi093.stdout:1635: left oid 266 (ObjNum 1129 snap 0 seq_num 1129) 2023-08-27T20:53:13.199 INFO:tasks.rados.rados.0.smithi093.stdout:1636: finishing write tid 1 to smithi093153278-192 2023-08-27T20:53:13.200 INFO:tasks.rados.rados.0.smithi093.stdout:1641: seq_num 1134 ranges {659138=668963,2109051=611732} 2023-08-27T20:53:13.205 INFO:tasks.rados.rados.0.smithi093.stdout:1641: writing smithi093153278-100 from 659138 to 1328101 tid 1 2023-08-27T20:53:13.213 INFO:tasks.rados.rados.0.smithi093.stdout:1641: writing smithi093153278-100 from 2109051 to 2720783 tid 2 2023-08-27T20:53:13.213 INFO:tasks.rados.rados.0.smithi093.stdout:1634: done (7 left) 2023-08-27T20:53:13.213 INFO:tasks.rados.rados.0.smithi093.stdout:1635: done (6 left) 2023-08-27T20:53:13.214 INFO:tasks.rados.rados.0.smithi093.stdout:1642: delete oid 212 current snap is 0 2023-08-27T20:53:13.214 INFO:tasks.rados.rados.0.smithi093.stdout:1637: expect (ObjNum 955 snap 0 seq_num 955) 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1636: finishing write tid 2 to smithi093153278-192 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1636: finishing write tid 3 to smithi093153278-192 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1636: finishing write tid 4 to smithi093153278-192 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 192 v 214 (ObjNum 1130 snap 0 seq_num 1130) dirty exists 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1636: left oid 192 (ObjNum 1130 snap 0 seq_num 1130) 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1638: finishing write tid 1 to smithi093153278-436 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1638: finishing write tid 2 to smithi093153278-436 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1638: finishing write tid 3 to smithi093153278-436 2023-08-27T20:53:13.293 INFO:tasks.rados.rados.0.smithi093.stdout:1638: finishing write tid 4 to smithi093153278-436 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 436 v 163 (ObjNum 1131 snap 0 seq_num 1131) dirty exists 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1638: left oid 436 (ObjNum 1131 snap 0 seq_num 1131) 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1639: finishing write tid 1 to smithi093153278-377 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1639: finishing write tid 2 to smithi093153278-377 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1639: finishing write tid 3 to smithi093153278-377 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1639: finishing write tid 4 to smithi093153278-377 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1639: finishing write tid 5 to smithi093153278-377 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 377 v 257 (ObjNum 1132 snap 0 seq_num 1132) dirty exists 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1639: left oid 377 (ObjNum 1132 snap 0 seq_num 1132) 2023-08-27T20:53:13.294 INFO:tasks.rados.rados.0.smithi093.stdout:1640: finishing write tid 1 to smithi093153278-301 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1640: finishing write tid 2 to smithi093153278-301 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1640: finishing write tid 3 to smithi093153278-301 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1640: finishing write tid 4 to smithi093153278-301 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 301 v 261 (ObjNum 1133 snap 0 seq_num 1133) dirty exists 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1640: left oid 301 (ObjNum 1133 snap 0 seq_num 1133) 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1636: done (6 left) 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1637: done (5 left) 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1638: done (4 left) 2023-08-27T20:53:13.295 INFO:tasks.rados.rados.0.smithi093.stdout:1639: done (3 left) 2023-08-27T20:53:13.296 INFO:tasks.rados.rados.0.smithi093.stdout:1640: done (2 left) 2023-08-27T20:53:13.296 INFO:tasks.rados.rados.0.smithi093.stdout:1642: done (1 left) 2023-08-27T20:53:13.296 INFO:tasks.rados.rados.0.smithi093.stdout:1643: write oid 407 current snap is 0 2023-08-27T20:53:13.296 INFO:tasks.rados.rados.0.smithi093.stdout:1641: finishing write tid 1 to smithi093153278-100 2023-08-27T20:53:13.296 INFO:tasks.rados.rados.0.smithi093.stdout:1643: seq_num 1136 ranges {692752=419461,1826015=615401,2979461=113291} 2023-08-27T20:53:13.298 INFO:tasks.rados.rados.0.smithi093.stdout:1643: writing smithi093153278-407 from 692752 to 1112213 tid 1 2023-08-27T20:53:13.305 INFO:tasks.rados.rados.0.smithi093.stdout:1643: writing smithi093153278-407 from 1826015 to 2441416 tid 2 2023-08-27T20:53:13.309 INFO:tasks.rados.rados.0.smithi093.stdout:1643: writing smithi093153278-407 from 2979461 to 3092752 tid 3 2023-08-27T20:53:13.309 INFO:tasks.rados.rados.0.smithi093.stdout:1644: read oid 380 snap -1 2023-08-27T20:53:13.309 INFO:tasks.rados.rados.0.smithi093.stdout:1644: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:53:13.309 INFO:tasks.rados.rados.0.smithi093.stdout:1645: delete oid 304 current snap is 0 2023-08-27T20:53:13.309 INFO:tasks.rados.rados.0.smithi093.stdout:1641: finishing write tid 2 to smithi093153278-100 2023-08-27T20:53:13.310 INFO:tasks.rados.rados.0.smithi093.stdout:1641: finishing write tid 3 to smithi093153278-100 2023-08-27T20:53:13.310 INFO:tasks.rados.rados.0.smithi093.stdout:1641: finishing write tid 4 to smithi093153278-100 2023-08-27T20:53:13.310 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 100 v 227 (ObjNum 1134 snap 0 seq_num 1134) dirty exists 2023-08-27T20:53:13.310 INFO:tasks.rados.rados.0.smithi093.stdout:1641: left oid 100 (ObjNum 1134 snap 0 seq_num 1134) 2023-08-27T20:53:13.310 INFO:tasks.rados.rados.0.smithi093.stdout:1643: finishing write tid 1 to smithi093153278-407 2023-08-27T20:53:13.310 INFO:tasks.rados.rados.0.smithi093.stdout:1643: finishing write tid 2 to smithi093153278-407 2023-08-27T20:53:13.311 INFO:tasks.rados.rados.0.smithi093.stdout:1641: done (3 left) 2023-08-27T20:53:13.311 INFO:tasks.rados.rados.0.smithi093.stdout:1645: done (2 left) 2023-08-27T20:53:13.312 INFO:tasks.rados.rados.0.smithi093.stdout:1646: read oid 335 snap -1 2023-08-27T20:53:13.312 INFO:tasks.rados.rados.0.smithi093.stdout:1646: expect deleted 2023-08-27T20:53:13.312 INFO:tasks.rados.rados.0.smithi093.stdout:1647: write oid 165 current snap is 0 2023-08-27T20:53:13.312 INFO:tasks.rados.rados.0.smithi093.stdout:1643: finishing write tid 3 to smithi093153278-407 2023-08-27T20:53:13.312 INFO:tasks.rados.rados.0.smithi093.stdout:1647: seq_num 1138 ranges {726366=569959,1826626=502717,2801601=324765} 2023-08-27T20:53:13.318 INFO:tasks.rados.rados.0.smithi093.stdout:1647: writing smithi093153278-165 from 726366 to 1296325 tid 1 2023-08-27T20:53:13.324 INFO:tasks.rados.rados.0.smithi093.stdout:1647: writing smithi093153278-165 from 1826626 to 2329343 tid 2 2023-08-27T20:53:13.329 INFO:tasks.rados.rados.0.smithi093.stdout:1647: writing smithi093153278-165 from 2801601 to 3126366 tid 3 2023-08-27T20:53:13.329 INFO:tasks.rados.rados.0.smithi093.stdout:1648: write oid 284 current snap is 0 2023-08-27T20:53:13.329 INFO:tasks.rados.rados.0.smithi093.stdout:1643: finishing write tid 4 to smithi093153278-407 2023-08-27T20:53:13.329 INFO:tasks.rados.rados.0.smithi093.stdout:1648: seq_num 1139 ranges {743173=645208,1885108=762728} 2023-08-27T20:53:13.336 INFO:tasks.rados.rados.0.smithi093.stdout:1648: writing smithi093153278-284 from 743173 to 1388381 tid 1 2023-08-27T20:53:13.344 INFO:tasks.rados.rados.0.smithi093.stdout:1648: writing smithi093153278-284 from 1885108 to 2647836 tid 2 2023-08-27T20:53:13.344 INFO:tasks.rados.rados.0.smithi093.stdout:1649: read oid 170 snap -1 2023-08-27T20:53:13.344 INFO:tasks.rados.rados.0.smithi093.stdout:1649: expect (ObjNum 169 snap 0 seq_num 169) 2023-08-27T20:53:13.344 INFO:tasks.rados.rados.0.smithi093.stdout:1650: write oid 102 current snap is 0 2023-08-27T20:53:13.344 INFO:tasks.rados.rados.0.smithi093.stdout:1650: seq_num 1140 ranges {759980=720457,1943590=506386,2856447=303533} 2023-08-27T20:53:13.352 INFO:tasks.rados.rados.0.smithi093.stdout:1650: writing smithi093153278-102 from 759980 to 1480437 tid 1 2023-08-27T20:53:13.357 INFO:tasks.rados.rados.0.smithi093.stdout:1650: writing smithi093153278-102 from 1943590 to 2449976 tid 2 2023-08-27T20:53:13.361 INFO:tasks.rados.rados.0.smithi093.stdout:1650: writing smithi093153278-102 from 2856447 to 3159980 tid 3 2023-08-27T20:53:13.361 INFO:tasks.rados.rados.0.smithi093.stdout:1651: delete oid 63 current snap is 0 2023-08-27T20:53:13.361 INFO:tasks.rados.rados.0.smithi093.stdout:1643: finishing write tid 5 to smithi093153278-407 2023-08-27T20:53:13.361 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 407 v 245 (ObjNum 1136 snap 0 seq_num 1136) dirty exists 2023-08-27T20:53:13.361 INFO:tasks.rados.rados.0.smithi093.stdout:1643: left oid 407 (ObjNum 1136 snap 0 seq_num 1136) 2023-08-27T20:53:13.361 INFO:tasks.rados.rados.0.smithi093.stdout:1644: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:53:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:13 smithi136 ceph-mon[113531]: mgrmap e22: x(active, starting, since 5s), standbys: y 2023-08-27T20:53:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2437002960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:13 smithi093 ceph-mon[111335]: mgrmap e22: x(active, starting, since 5s), standbys: y 2023-08-27T20:53:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2437002960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:13 smithi093 ceph-mon[118902]: mgrmap e22: x(active, starting, since 5s), standbys: y 2023-08-27T20:53:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2437002960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:13.525 INFO:tasks.rados.rados.0.smithi093.stdout:1647: finishing write tid 1 to smithi093153278-165 2023-08-27T20:53:13.525 INFO:tasks.rados.rados.0.smithi093.stdout:1647: finishing write tid 2 to smithi093153278-165 2023-08-27T20:53:13.525 INFO:tasks.rados.rados.0.smithi093.stdout:1647: finishing write tid 3 to smithi093153278-165 2023-08-27T20:53:13.525 INFO:tasks.rados.rados.0.smithi093.stdout:1647: finishing write tid 4 to smithi093153278-165 2023-08-27T20:53:13.525 INFO:tasks.rados.rados.0.smithi093.stdout:1647: finishing write tid 5 to smithi093153278-165 2023-08-27T20:53:13.525 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 165 v 261 (ObjNum 1138 snap 0 seq_num 1138) dirty exists 2023-08-27T20:53:13.525 INFO:tasks.rados.rados.0.smithi093.stdout:1647: left oid 165 (ObjNum 1138 snap 0 seq_num 1138) 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1648: finishing write tid 1 to smithi093153278-284 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1643: done (7 left) 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1644: done (6 left) 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1646: done (5 left) 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1647: done (4 left) 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1651: done (3 left) 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1652: write oid 195 current snap is 0 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1648: finishing write tid 2 to smithi093153278-284 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1648: finishing write tid 3 to smithi093153278-284 2023-08-27T20:53:13.526 INFO:tasks.rados.rados.0.smithi093.stdout:1652: seq_num 1142 ranges {793594=587308,2060554=510055} 2023-08-27T20:53:13.532 INFO:tasks.rados.rados.0.smithi093.stdout:1652: writing smithi093153278-195 from 793594 to 1380902 tid 1 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1652: writing smithi093153278-195 from 2060554 to 2570609 tid 2 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1653: read oid 136 snap -1 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1653: expect (ObjNum 776 snap 0 seq_num 776) 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1654: read oid 187 snap -1 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1654: expect (ObjNum 1033 snap 0 seq_num 1033) 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1655: read oid 81 snap -1 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1655: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1656: write oid 230 current snap is 0 2023-08-27T20:53:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:1656: seq_num 1143 ranges {410401=662557,1719036=770066,2938716=271685} 2023-08-27T20:53:13.545 INFO:tasks.rados.rados.0.smithi093.stdout:1656: writing smithi093153278-230 from 410401 to 1072958 tid 1 2023-08-27T20:53:13.552 INFO:tasks.rados.rados.0.smithi093.stdout:1656: writing smithi093153278-230 from 1719036 to 2489102 tid 2 2023-08-27T20:53:13.557 INFO:tasks.rados.rados.0.smithi093.stdout:1656: writing smithi093153278-230 from 2938716 to 3210401 tid 3 2023-08-27T20:53:13.557 INFO:tasks.rados.rados.0.smithi093.stdout:1657: write oid 175 current snap is 0 2023-08-27T20:53:13.557 INFO:tasks.rados.rados.0.smithi093.stdout:1648: finishing write tid 4 to smithi093153278-284 2023-08-27T20:53:13.557 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 284 v 230 (ObjNum 1139 snap 0 seq_num 1139) dirty exists 2023-08-27T20:53:13.557 INFO:tasks.rados.rados.0.smithi093.stdout:1648: left oid 284 (ObjNum 1139 snap 0 seq_num 1139) 2023-08-27T20:53:13.557 INFO:tasks.rados.rados.0.smithi093.stdout:1649: expect (ObjNum 169 snap 0 seq_num 169) 2023-08-27T20:53:13.644 INFO:tasks.rados.rados.0.smithi093.stdout:1650: finishing write tid 1 to smithi093153278-102 2023-08-27T20:53:13.644 INFO:tasks.rados.rados.0.smithi093.stdout:1650: finishing write tid 2 to smithi093153278-102 2023-08-27T20:53:13.644 INFO:tasks.rados.rados.0.smithi093.stdout:1650: finishing write tid 3 to smithi093153278-102 2023-08-27T20:53:13.644 INFO:tasks.rados.rados.0.smithi093.stdout:1650: finishing write tid 4 to smithi093153278-102 2023-08-27T20:53:13.644 INFO:tasks.rados.rados.0.smithi093.stdout:1650: finishing write tid 5 to smithi093153278-102 2023-08-27T20:53:13.644 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 102 v 194 (ObjNum 1140 snap 0 seq_num 1140) dirty exists 2023-08-27T20:53:13.644 INFO:tasks.rados.rados.0.smithi093.stdout:1650: left oid 102 (ObjNum 1140 snap 0 seq_num 1140) 2023-08-27T20:53:13.645 INFO:tasks.rados.rados.0.smithi093.stdout:1657: seq_num 1144 ranges {427208=737806,1777518=513724,2966139=261069} 2023-08-27T20:53:13.650 INFO:tasks.rados.rados.0.smithi093.stdout:1657: writing smithi093153278-175 from 427208 to 1165014 tid 1 2023-08-27T20:53:13.656 INFO:tasks.rados.rados.0.smithi093.stdout:1657: writing smithi093153278-175 from 1777518 to 2291242 tid 2 2023-08-27T20:53:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:1657: writing smithi093153278-175 from 2966139 to 3227208 tid 3 2023-08-27T20:53:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:1648: done (8 left) 2023-08-27T20:53:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:1649: done (7 left) 2023-08-27T20:53:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:1650: done (6 left) 2023-08-27T20:53:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:1658: read oid 430 snap -1 2023-08-27T20:53:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:1658: expect (ObjNum 766 snap 0 seq_num 766) 2023-08-27T20:53:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:1659: write oid 49 current snap is 0 2023-08-27T20:53:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:1652: finishing write tid 1 to smithi093153278-195 2023-08-27T20:53:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:1655: expect (ObjNum 670 snap 0 seq_num 670) 2023-08-27T20:53:13.874 INFO:tasks.rados.rados.0.smithi093.stdout:1652: finishing write tid 2 to smithi093153278-195 2023-08-27T20:53:13.874 INFO:tasks.rados.rados.0.smithi093.stdout:1652: finishing write tid 3 to smithi093153278-195 2023-08-27T20:53:13.874 INFO:tasks.rados.rados.0.smithi093.stdout:1652: finishing write tid 4 to smithi093153278-195 2023-08-27T20:53:13.874 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 197 (ObjNum 1142 snap 0 seq_num 1142) dirty exists 2023-08-27T20:53:13.874 INFO:tasks.rados.rados.0.smithi093.stdout:1652: left oid 195 (ObjNum 1142 snap 0 seq_num 1142) 2023-08-27T20:53:13.875 INFO:tasks.rados.rados.0.smithi093.stdout:1654: expect (ObjNum 1033 snap 0 seq_num 1033) 2023-08-27T20:53:14.001 INFO:tasks.rados.rados.0.smithi093.stdout:1653: expect (ObjNum 776 snap 0 seq_num 776) 2023-08-27T20:53:14.132 INFO:tasks.rados.rados.0.smithi093.stdout:1656: finishing write tid 1 to smithi093153278-230 2023-08-27T20:53:14.132 INFO:tasks.rados.rados.0.smithi093.stdout:1656: finishing write tid 2 to smithi093153278-230 2023-08-27T20:53:14.132 INFO:tasks.rados.rados.0.smithi093.stdout:1656: finishing write tid 3 to smithi093153278-230 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1656: finishing write tid 4 to smithi093153278-230 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1656: finishing write tid 5 to smithi093153278-230 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 239 (ObjNum 1143 snap 0 seq_num 1143) dirty exists 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1656: left oid 230 (ObjNum 1143 snap 0 seq_num 1143) 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1657: finishing write tid 1 to smithi093153278-175 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1657: finishing write tid 2 to smithi093153278-175 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1657: finishing write tid 3 to smithi093153278-175 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1657: finishing write tid 4 to smithi093153278-175 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1657: finishing write tid 5 to smithi093153278-175 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 175 v 215 (ObjNum 1144 snap 0 seq_num 1144) dirty exists 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1657: left oid 175 (ObjNum 1144 snap 0 seq_num 1144) 2023-08-27T20:53:14.133 INFO:tasks.rados.rados.0.smithi093.stdout:1658: expect (ObjNum 766 snap 0 seq_num 766) 2023-08-27T20:53:14.252 INFO:tasks.rados.rados.0.smithi093.stdout:1659: seq_num 1145 ranges {444015=413055,1436000=773735,2709915=534100} 2023-08-27T20:53:14.257 INFO:tasks.rados.rados.0.smithi093.stdout:1659: writing smithi093153278-49 from 444015 to 857070 tid 1 2023-08-27T20:53:14.264 INFO:tasks.rados.rados.0.smithi093.stdout:1659: writing smithi093153278-49 from 1436000 to 2209735 tid 2 2023-08-27T20:53:14.270 INFO:tasks.rados.rados.0.smithi093.stdout:1659: writing smithi093153278-49 from 2709915 to 3244015 tid 3 2023-08-27T20:53:14.270 INFO:tasks.rados.rados.0.smithi093.stdout:1652: done (7 left) 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1653: done (6 left) 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1654: done (5 left) 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1655: done (4 left) 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1656: done (3 left) 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1657: done (2 left) 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1658: done (1 left) 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1660: write oid 129 current snap is 0 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1659: finishing write tid 1 to smithi093153278-49 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1659: finishing write tid 2 to smithi093153278-49 2023-08-27T20:53:14.271 INFO:tasks.rados.rados.0.smithi093.stdout:1660: seq_num 1146 ranges {460822=488304,1378129=517393,2504632=405962} 2023-08-27T20:53:14.275 INFO:tasks.rados.rados.0.smithi093.stdout:1660: writing smithi093153278-129 from 460822 to 949126 tid 1 2023-08-27T20:53:14.280 INFO:tasks.rados.rados.0.smithi093.stdout:1660: writing smithi093153278-129 from 1378129 to 1895522 tid 2 2023-08-27T20:53:14.286 INFO:tasks.rados.rados.0.smithi093.stdout:1660: writing smithi093153278-129 from 2504632 to 2910594 tid 3 2023-08-27T20:53:14.286 INFO:tasks.rados.rados.0.smithi093.stdout:1661: write oid 261 current snap is 0 2023-08-27T20:53:14.286 INFO:tasks.rados.rados.0.smithi093.stdout:1659: finishing write tid 3 to smithi093153278-49 2023-08-27T20:53:14.286 INFO:tasks.rados.rados.0.smithi093.stdout:1659: finishing write tid 4 to smithi093153278-49 2023-08-27T20:53:14.286 INFO:tasks.rados.rados.0.smithi093.stdout:1661: seq_num 1147 ranges {477629=563553,1836611=777404,3048408=229221} 2023-08-27T20:53:14.291 INFO:tasks.rados.rados.0.smithi093.stdout:1661: writing smithi093153278-261 from 477629 to 1041182 tid 1 2023-08-27T20:53:14.300 INFO:tasks.rados.rados.0.smithi093.stdout:1661: writing smithi093153278-261 from 1836611 to 2614015 tid 2 2023-08-27T20:53:14.304 INFO:tasks.rados.rados.0.smithi093.stdout:1661: writing smithi093153278-261 from 3048408 to 3277629 tid 3 2023-08-27T20:53:14.304 INFO:tasks.rados.rados.0.smithi093.stdout:1662: delete oid 392 current snap is 0 2023-08-27T20:53:14.304 INFO:tasks.rados.rados.0.smithi093.stdout:1659: finishing write tid 5 to smithi093153278-49 2023-08-27T20:53:14.304 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 49 v 167 (ObjNum 1145 snap 0 seq_num 1145) dirty exists 2023-08-27T20:53:14.304 INFO:tasks.rados.rados.0.smithi093.stdout:1659: left oid 49 (ObjNum 1145 snap 0 seq_num 1145) 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1660: finishing write tid 1 to smithi093153278-129 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1660: finishing write tid 2 to smithi093153278-129 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1660: finishing write tid 3 to smithi093153278-129 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1660: finishing write tid 4 to smithi093153278-129 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1660: finishing write tid 5 to smithi093153278-129 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 129 v 201 (ObjNum 1146 snap 0 seq_num 1146) dirty exists 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1660: left oid 129 (ObjNum 1146 snap 0 seq_num 1146) 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1661: finishing write tid 1 to smithi093153278-261 2023-08-27T20:53:14.305 INFO:tasks.rados.rados.0.smithi093.stdout:1661: finishing write tid 2 to smithi093153278-261 2023-08-27T20:53:14.306 INFO:tasks.rados.rados.0.smithi093.stdout:1661: finishing write tid 3 to smithi093153278-261 2023-08-27T20:53:14.306 INFO:tasks.rados.rados.0.smithi093.stdout:1661: finishing write tid 4 to smithi093153278-261 2023-08-27T20:53:14.306 INFO:tasks.rados.rados.0.smithi093.stdout:1661: finishing write tid 5 to smithi093153278-261 2023-08-27T20:53:14.306 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 261 v 261 (ObjNum 1147 snap 0 seq_num 1147) dirty exists 2023-08-27T20:53:14.306 INFO:tasks.rados.rados.0.smithi093.stdout:1661: left oid 261 (ObjNum 1147 snap 0 seq_num 1147) 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1659: done (3 left) 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1660: done (2 left) 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1661: done (1 left) 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1662: done (0 left) 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1663: read oid 432 snap -1 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1663: expect (ObjNum 431 snap 0 seq_num 431) 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1664: write oid 2 current snap is 0 2023-08-27T20:53:14.307 INFO:tasks.rados.rados.0.smithi093.stdout:1664: seq_num 1149 ranges {511243=430404,1669928=664720,3103254=207989} 2023-08-27T20:53:14.311 INFO:tasks.rados.rados.0.smithi093.stdout:1664: writing smithi093153278-2 from 511243 to 941647 tid 1 2023-08-27T20:53:14.319 INFO:tasks.rados.rados.0.smithi093.stdout:1664: writing smithi093153278-2 from 1669928 to 2334648 tid 2 2023-08-27T20:53:14.325 INFO:tasks.rados.rados.0.smithi093.stdout:1664: writing smithi093153278-2 from 3103254 to 3311243 tid 3 2023-08-27T20:53:14.325 INFO:tasks.rados.rados.0.smithi093.stdout:1665: read oid 331 snap -1 2023-08-27T20:53:14.325 INFO:tasks.rados.rados.0.smithi093.stdout:1665: expect (ObjNum 1013 snap 0 seq_num 1013) 2023-08-27T20:53:14.325 INFO:tasks.rados.rados.0.smithi093.stdout:1666: read oid 189 snap -1 2023-08-27T20:53:14.325 INFO:tasks.rados.rados.0.smithi093.stdout:1666: expect (ObjNum 1078 snap 0 seq_num 1078) 2023-08-27T20:53:14.325 INFO:tasks.rados.rados.0.smithi093.stdout:1667: write oid 330 current snap is 0 2023-08-27T20:53:14.325 INFO:tasks.rados.rados.0.smithi093.stdout:1667: seq_num 1150 ranges {528050=505653,1612057=524731,2614324=567403} 2023-08-27T20:53:14.331 INFO:tasks.rados.rados.0.smithi093.stdout:1667: writing smithi093153278-330 from 528050 to 1033703 tid 1 2023-08-27T20:53:14.337 INFO:tasks.rados.rados.0.smithi093.stdout:1667: writing smithi093153278-330 from 1612057 to 2136788 tid 2 2023-08-27T20:53:14.343 INFO:tasks.rados.rados.0.smithi093.stdout:1667: writing smithi093153278-330 from 2614324 to 3181727 tid 3 2023-08-27T20:53:14.343 INFO:tasks.rados.rados.0.smithi093.stdout:1668: write oid 155 current snap is 0 2023-08-27T20:53:14.343 INFO:tasks.rados.rados.0.smithi093.stdout:1663: expect (ObjNum 431 snap 0 seq_num 431) 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:1664: finishing write tid 1 to smithi093153278-2 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:1664: finishing write tid 2 to smithi093153278-2 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:1664: finishing write tid 3 to smithi093153278-2 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:1664: finishing write tid 4 to smithi093153278-2 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:1664: finishing write tid 5 to smithi093153278-2 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 2 v 265 (ObjNum 1149 snap 0 seq_num 1149) dirty exists 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:1664: left oid 2 (ObjNum 1149 snap 0 seq_num 1149) 2023-08-27T20:53:14.447 INFO:tasks.rados.rados.0.smithi093.stdout:1665: expect (ObjNum 1013 snap 0 seq_num 1013) 2023-08-27T20:53:14.625 INFO:tasks.rados.rados.0.smithi093.stdout:1666: expect (ObjNum 1078 snap 0 seq_num 1078) 2023-08-27T20:53:14.741 INFO:tasks.rados.rados.0.smithi093.stdout:1667: finishing write tid 1 to smithi093153278-330 2023-08-27T20:53:14.741 INFO:tasks.rados.rados.0.smithi093.stdout:1667: finishing write tid 2 to smithi093153278-330 2023-08-27T20:53:14.741 INFO:tasks.rados.rados.0.smithi093.stdout:1667: finishing write tid 3 to smithi093153278-330 2023-08-27T20:53:14.741 INFO:tasks.rados.rados.0.smithi093.stdout:1668: seq_num 1151 ranges {544857=580902,1670539=668389,3041747=303110} 2023-08-27T20:53:14.747 INFO:tasks.rados.rados.0.smithi093.stdout:1668: writing smithi093153278-155 from 544857 to 1125759 tid 1 2023-08-27T20:53:14.753 INFO:tasks.rados.rados.0.smithi093.stdout:1668: writing smithi093153278-155 from 1670539 to 2338928 tid 2 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1668: writing smithi093153278-155 from 3041747 to 3344857 tid 3 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1663: done (5 left) 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1664: done (4 left) 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1665: done (3 left) 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1666: done (2 left) 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1669: read oid 33 snap -1 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1669: expect deleted 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1670: delete oid 258 current snap is 0 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1667: finishing write tid 4 to smithi093153278-330 2023-08-27T20:53:14.760 INFO:tasks.rados.rados.0.smithi093.stdout:1667: finishing write tid 5 to smithi093153278-330 2023-08-27T20:53:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 330 v 213 (ObjNum 1150 snap 0 seq_num 1150) dirty exists 2023-08-27T20:53:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:1667: left oid 330 (ObjNum 1150 snap 0 seq_num 1150) 2023-08-27T20:53:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:1668: finishing write tid 1 to smithi093153278-155 2023-08-27T20:53:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:1668: finishing write tid 2 to smithi093153278-155 2023-08-27T20:53:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:1667: done (3 left) 2023-08-27T20:53:14.761 INFO:tasks.rados.rados.0.smithi093.stdout:1669: done (2 left) 2023-08-27T20:53:14.762 INFO:tasks.rados.rados.0.smithi093.stdout:1670: done (1 left) 2023-08-27T20:53:14.762 INFO:tasks.rados.rados.0.smithi093.stdout:1671: write oid 422 current snap is 0 2023-08-27T20:53:14.762 INFO:tasks.rados.rados.0.smithi093.stdout:1671: seq_num 1153 ranges {578471=731400,1787503=672058,3096593=281878} 2023-08-27T20:53:14.768 INFO:tasks.rados.rados.0.smithi093.stdout:1671: writing smithi093153278-422 from 578471 to 1309871 tid 1 2023-08-27T20:53:14.774 INFO:tasks.rados.rados.0.smithi093.stdout:1671: writing smithi093153278-422 from 1787503 to 2459561 tid 2 2023-08-27T20:53:14.779 INFO:tasks.rados.rados.0.smithi093.stdout:1671: writing smithi093153278-422 from 3096593 to 3378471 tid 3 2023-08-27T20:53:14.779 INFO:tasks.rados.rados.0.smithi093.stdout:1672: write oid 152 current snap is 0 2023-08-27T20:53:14.779 INFO:tasks.rados.rados.0.smithi093.stdout:1672: seq_num 1154 ranges {595278=406649,1729632=532069,3007663=387615} 2023-08-27T20:53:14.785 INFO:tasks.rados.rados.0.smithi093.stdout:1672: writing smithi093153278-152 from 595278 to 1001927 tid 1 2023-08-27T20:53:14.791 INFO:tasks.rados.rados.0.smithi093.stdout:1672: writing smithi093153278-152 from 1729632 to 2261701 tid 2 2023-08-27T20:53:14.798 INFO:tasks.rados.rados.0.smithi093.stdout:1672: writing smithi093153278-152 from 3007663 to 3395278 tid 3 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1673: read oid 182 snap -1 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1673: expect (ObjNum 830 snap 0 seq_num 830) 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1674: read oid 339 snap -1 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1674: expect (ObjNum 850 snap 0 seq_num 850) 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1675: read oid 427 snap -1 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1675: expect (ObjNum 426 snap 0 seq_num 426) 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1676: write oid 328 current snap is 0 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1668: finishing write tid 3 to smithi093153278-155 2023-08-27T20:53:14.799 INFO:tasks.rados.rados.0.smithi093.stdout:1676: seq_num 1155 ranges {612085=481898,1788114=675727,3035086=376999} 2023-08-27T20:53:14.804 INFO:tasks.rados.rados.0.smithi093.stdout:1676: writing smithi093153278-328 from 612085 to 1093983 tid 1 2023-08-27T20:53:14.812 INFO:tasks.rados.rados.0.smithi093.stdout:1676: writing smithi093153278-328 from 1788114 to 2463841 tid 2 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:1676: writing smithi093153278-328 from 3035086 to 3412085 tid 3 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:1677: delete oid 129 current snap is 0 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:1668: finishing write tid 4 to smithi093153278-155 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:1668: finishing write tid 5 to smithi093153278-155 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 155 v 266 (ObjNum 1151 snap 0 seq_num 1151) dirty exists 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:1668: left oid 155 (ObjNum 1151 snap 0 seq_num 1151) 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:1671: finishing write tid 1 to smithi093153278-422 2023-08-27T20:53:14.818 INFO:tasks.rados.rados.0.smithi093.stdout:1671: finishing write tid 2 to smithi093153278-422 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1671: finishing write tid 3 to smithi093153278-422 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1671: finishing write tid 4 to smithi093153278-422 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1671: finishing write tid 5 to smithi093153278-422 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 422 v 217 (ObjNum 1153 snap 0 seq_num 1153) dirty exists 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1671: left oid 422 (ObjNum 1153 snap 0 seq_num 1153) 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1672: finishing write tid 1 to smithi093153278-152 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1672: finishing write tid 2 to smithi093153278-152 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1672: finishing write tid 3 to smithi093153278-152 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1672: finishing write tid 4 to smithi093153278-152 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1672: finishing write tid 5 to smithi093153278-152 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 152 v 251 (ObjNum 1154 snap 0 seq_num 1154) dirty exists 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1672: left oid 152 (ObjNum 1154 snap 0 seq_num 1154) 2023-08-27T20:53:14.819 INFO:tasks.rados.rados.0.smithi093.stdout:1673: expect (ObjNum 830 snap 0 seq_num 830) 2023-08-27T20:53:15.016 INFO:tasks.rados.rados.0.smithi093.stdout:1674: expect (ObjNum 850 snap 0 seq_num 850) 2023-08-27T20:53:15.159 INFO:tasks.rados.rados.0.smithi093.stdout:1676: finishing write tid 1 to smithi093153278-328 2023-08-27T20:53:15.159 INFO:tasks.rados.rados.0.smithi093.stdout:1675: expect (ObjNum 426 snap 0 seq_num 426) 2023-08-27T20:53:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/882251134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/882251134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:15.246 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/882251134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:15.260 INFO:tasks.rados.rados.0.smithi093.stdout:1676: finishing write tid 2 to smithi093153278-328 2023-08-27T20:53:15.260 INFO:tasks.rados.rados.0.smithi093.stdout:1668: done (7 left) 2023-08-27T20:53:15.260 INFO:tasks.rados.rados.0.smithi093.stdout:1671: done (6 left) 2023-08-27T20:53:15.260 INFO:tasks.rados.rados.0.smithi093.stdout:1672: done (5 left) 2023-08-27T20:53:15.260 INFO:tasks.rados.rados.0.smithi093.stdout:1673: done (4 left) 2023-08-27T20:53:15.260 INFO:tasks.rados.rados.0.smithi093.stdout:1674: done (3 left) 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:1675: done (2 left) 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:1677: done (1 left) 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:1678: delete oid 481 current snap is 0 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:1676: finishing write tid 3 to smithi093153278-328 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:1676: finishing write tid 4 to smithi093153278-328 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:1676: finishing write tid 5 to smithi093153278-328 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 255 (ObjNum 1155 snap 0 seq_num 1155) dirty exists 2023-08-27T20:53:15.261 INFO:tasks.rados.rados.0.smithi093.stdout:1676: left oid 328 (ObjNum 1155 snap 0 seq_num 1155) 2023-08-27T20:53:15.262 INFO:tasks.rados.rados.0.smithi093.stdout:1676: done (1 left) 2023-08-27T20:53:15.262 INFO:tasks.rados.rados.0.smithi093.stdout:1678: done (0 left) 2023-08-27T20:53:15.262 INFO:tasks.rados.rados.0.smithi093.stdout:1679: write oid 49 current snap is 0 2023-08-27T20:53:15.262 INFO:tasks.rados.rados.0.smithi093.stdout:1679: seq_num 1158 ranges {662506=423998,1563560=539407,2717355=490285} 2023-08-27T20:53:15.267 INFO:tasks.rados.rados.0.smithi093.stdout:1679: writing smithi093153278-49 from 662506 to 1086504 tid 1 2023-08-27T20:53:15.273 INFO:tasks.rados.rados.0.smithi093.stdout:1679: writing smithi093153278-49 from 1563560 to 2102967 tid 2 2023-08-27T20:53:15.280 INFO:tasks.rados.rados.0.smithi093.stdout:1679: writing smithi093153278-49 from 2717355 to 3207640 tid 3 2023-08-27T20:53:15.280 INFO:tasks.rados.rados.0.smithi093.stdout:1680: read oid 223 snap -1 2023-08-27T20:53:15.280 INFO:tasks.rados.rados.0.smithi093.stdout:1680: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:53:15.280 INFO:tasks.rados.rados.0.smithi093.stdout:1681: write oid 352 current snap is 0 2023-08-27T20:53:15.280 INFO:tasks.rados.rados.0.smithi093.stdout:1679: finishing write tid 1 to smithi093153278-49 2023-08-27T20:53:15.280 INFO:tasks.rados.rados.0.smithi093.stdout:1681: seq_num 1159 ranges {679313=499247,1622042=683065,2744778=701557} 2023-08-27T20:53:15.286 INFO:tasks.rados.rados.0.smithi093.stdout:1681: writing smithi093153278-352 from 679313 to 1178560 tid 1 2023-08-27T20:53:15.292 INFO:tasks.rados.rados.0.smithi093.stdout:1681: writing smithi093153278-352 from 1622042 to 2305107 tid 2 2023-08-27T20:53:15.299 INFO:tasks.rados.rados.0.smithi093.stdout:1681: writing smithi093153278-352 from 2744778 to 3446335 tid 3 2023-08-27T20:53:15.299 INFO:tasks.rados.rados.0.smithi093.stdout:1682: write oid 121 current snap is 0 2023-08-27T20:53:15.299 INFO:tasks.rados.rados.0.smithi093.stdout:1679: finishing write tid 2 to smithi093153278-49 2023-08-27T20:53:15.299 INFO:tasks.rados.rados.0.smithi093.stdout:1682: seq_num 1160 ranges {696120=574496,1680524=543076,2888554=512829} 2023-08-27T20:53:15.306 INFO:tasks.rados.rados.0.smithi093.stdout:1682: writing smithi093153278-121 from 696120 to 1270616 tid 1 2023-08-27T20:53:15.311 INFO:tasks.rados.rados.0.smithi093.stdout:1682: writing smithi093153278-121 from 1680524 to 2223600 tid 2 2023-08-27T20:53:15.318 INFO:tasks.rados.rados.0.smithi093.stdout:1682: writing smithi093153278-121 from 2888554 to 3401383 tid 3 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:1683: write oid 385 current snap is 0 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:1679: finishing write tid 3 to smithi093153278-49 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:1679: finishing write tid 4 to smithi093153278-49 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:1679: finishing write tid 5 to smithi093153278-49 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 49 v 171 (ObjNum 1158 snap 0 seq_num 1158) dirty exists 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:1679: left oid 49 (ObjNum 1158 snap 0 seq_num 1158) 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:1681: finishing write tid 1 to smithi093153278-352 2023-08-27T20:53:15.319 INFO:tasks.rados.rados.0.smithi093.stdout:1680: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:53:15.409 INFO:tasks.rados.rados.0.smithi093.stdout:1681: finishing write tid 2 to smithi093153278-352 2023-08-27T20:53:15.409 INFO:tasks.rados.rados.0.smithi093.stdout:1681: finishing write tid 3 to smithi093153278-352 2023-08-27T20:53:15.409 INFO:tasks.rados.rados.0.smithi093.stdout:1681: finishing write tid 4 to smithi093153278-352 2023-08-27T20:53:15.409 INFO:tasks.rados.rados.0.smithi093.stdout:1681: finishing write tid 5 to smithi093153278-352 2023-08-27T20:53:15.409 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 352 v 269 (ObjNum 1159 snap 0 seq_num 1159) dirty exists 2023-08-27T20:53:15.410 INFO:tasks.rados.rados.0.smithi093.stdout:1681: left oid 352 (ObjNum 1159 snap 0 seq_num 1159) 2023-08-27T20:53:15.410 INFO:tasks.rados.rados.0.smithi093.stdout:1682: finishing write tid 1 to smithi093153278-121 2023-08-27T20:53:15.410 INFO:tasks.rados.rados.0.smithi093.stdout:1683: seq_num 1161 ranges {712927=649745,2139006=686734} 2023-08-27T20:53:15.416 INFO:tasks.rados.rados.0.smithi093.stdout:1683: writing smithi093153278-385 from 712927 to 1362672 tid 1 2023-08-27T20:53:15.425 INFO:tasks.rados.rados.0.smithi093.stdout:1683: writing smithi093153278-385 from 2139006 to 2825740 tid 2 2023-08-27T20:53:15.425 INFO:tasks.rados.rados.0.smithi093.stdout:1679: done (4 left) 2023-08-27T20:53:15.425 INFO:tasks.rados.rados.0.smithi093.stdout:1680: done (3 left) 2023-08-27T20:53:15.425 INFO:tasks.rados.rados.0.smithi093.stdout:1681: done (2 left) 2023-08-27T20:53:15.425 INFO:tasks.rados.rados.0.smithi093.stdout:1684: read oid 98 snap -1 2023-08-27T20:53:15.425 INFO:tasks.rados.rados.0.smithi093.stdout:1684: expect deleted 2023-08-27T20:53:15.425 INFO:tasks.rados.rados.0.smithi093.stdout:1685: write oid 88 current snap is 0 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:1682: finishing write tid 2 to smithi093153278-121 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:1682: finishing write tid 3 to smithi093153278-121 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:1682: finishing write tid 4 to smithi093153278-121 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:1682: finishing write tid 5 to smithi093153278-121 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 121 v 270 (ObjNum 1160 snap 0 seq_num 1160) dirty exists 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:1682: left oid 121 (ObjNum 1160 snap 0 seq_num 1160) 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:1683: finishing write tid 1 to smithi093153278-385 2023-08-27T20:53:15.426 INFO:tasks.rados.rados.0.smithi093.stdout:1685: seq_num 1162 ranges {729734=724994,2197488=430392,3227047=302687} 2023-08-27T20:53:15.432 INFO:tasks.rados.rados.0.smithi093.stdout:1685: writing smithi093153278-88 from 729734 to 1454728 tid 1 2023-08-27T20:53:15.438 INFO:tasks.rados.rados.0.smithi093.stdout:1685: writing smithi093153278-88 from 2197488 to 2627880 tid 2 2023-08-27T20:53:15.443 INFO:tasks.rados.rados.0.smithi093.stdout:1685: writing smithi093153278-88 from 3227047 to 3529734 tid 3 2023-08-27T20:53:15.443 INFO:tasks.rados.rados.0.smithi093.stdout:1682: done (3 left) 2023-08-27T20:53:15.443 INFO:tasks.rados.rados.0.smithi093.stdout:1686: write oid 183 current snap is 0 2023-08-27T20:53:15.443 INFO:tasks.rados.rados.0.smithi093.stdout:1686: seq_num 1163 ranges {746541=400243,1739617=690403,3138117=408424} 2023-08-27T20:53:15.449 INFO:tasks.rados.rados.0.smithi093.stdout:1686: writing smithi093153278-183 from 746541 to 1146784 tid 1 2023-08-27T20:53:15.456 INFO:tasks.rados.rados.0.smithi093.stdout:1686: writing smithi093153278-183 from 1739617 to 2430020 tid 2 2023-08-27T20:53:15.462 INFO:tasks.rados.rados.0.smithi093.stdout:1686: writing smithi093153278-183 from 3138117 to 3546541 tid 3 2023-08-27T20:53:15.462 INFO:tasks.rados.rados.0.smithi093.stdout:1687: read oid 399 snap -1 2023-08-27T20:53:15.463 INFO:tasks.rados.rados.0.smithi093.stdout:1687: expect (ObjNum 523 snap 0 seq_num 523) 2023-08-27T20:53:15.463 INFO:tasks.rados.rados.0.smithi093.stdout:1688: read oid 276 snap -1 2023-08-27T20:53:15.463 INFO:tasks.rados.rados.0.smithi093.stdout:1688: expect (ObjNum 936 snap 0 seq_num 936) 2023-08-27T20:53:15.463 INFO:tasks.rados.rados.0.smithi093.stdout:1689: write oid 344 current snap is 0 2023-08-27T20:53:15.463 INFO:tasks.rados.rados.0.smithi093.stdout:1689: seq_num 1164 ranges {763348=591845,1914452=434061,2881893=674270} 2023-08-27T20:53:15.470 INFO:tasks.rados.rados.0.smithi093.stdout:1689: writing smithi093153278-344 from 763348 to 1355193 tid 1 2023-08-27T20:53:15.475 INFO:tasks.rados.rados.0.smithi093.stdout:1689: writing smithi093153278-344 from 1914452 to 2348513 tid 2 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:1689: writing smithi093153278-344 from 2881893 to 3556163 tid 3 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:1690: read oid 108 snap -1 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:1690: expect (ObjNum 1073 snap 0 seq_num 1073) 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:1691: delete oid 411 current snap is 0 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:1683: finishing write tid 2 to smithi093153278-385 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:1683: finishing write tid 3 to smithi093153278-385 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:1683: finishing write tid 4 to smithi093153278-385 2023-08-27T20:53:15.483 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 385 v 258 (ObjNum 1161 snap 0 seq_num 1161) dirty exists 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1683: left oid 385 (ObjNum 1161 snap 0 seq_num 1161) 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1685: finishing write tid 1 to smithi093153278-88 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1685: finishing write tid 2 to smithi093153278-88 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1685: finishing write tid 3 to smithi093153278-88 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1685: finishing write tid 4 to smithi093153278-88 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1685: finishing write tid 5 to smithi093153278-88 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 265 (ObjNum 1162 snap 0 seq_num 1162) dirty exists 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1685: left oid 88 (ObjNum 1162 snap 0 seq_num 1162) 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1686: finishing write tid 1 to smithi093153278-183 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1686: finishing write tid 2 to smithi093153278-183 2023-08-27T20:53:15.484 INFO:tasks.rados.rados.0.smithi093.stdout:1687: expect (ObjNum 523 snap 0 seq_num 523) 2023-08-27T20:53:15.606 INFO:tasks.rados.rados.0.smithi093.stdout:1689: finishing write tid 1 to smithi093153278-344 2023-08-27T20:53:15.606 INFO:tasks.rados.rados.0.smithi093.stdout:1686: finishing write tid 3 to smithi093153278-183 2023-08-27T20:53:15.606 INFO:tasks.rados.rados.0.smithi093.stdout:1686: finishing write tid 4 to smithi093153278-183 2023-08-27T20:53:15.606 INFO:tasks.rados.rados.0.smithi093.stdout:1686: finishing write tid 5 to smithi093153278-183 2023-08-27T20:53:15.607 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 183 v 223 (ObjNum 1163 snap 0 seq_num 1163) dirty exists 2023-08-27T20:53:15.607 INFO:tasks.rados.rados.0.smithi093.stdout:1686: left oid 183 (ObjNum 1163 snap 0 seq_num 1163) 2023-08-27T20:53:15.607 INFO:tasks.rados.rados.0.smithi093.stdout:1688: expect (ObjNum 936 snap 0 seq_num 936) 2023-08-27T20:53:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:53:15] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:53:15.759 INFO:tasks.rados.rados.0.smithi093.stdout:1689: finishing write tid 2 to smithi093153278-344 2023-08-27T20:53:15.759 INFO:tasks.rados.rados.0.smithi093.stdout:1689: finishing write tid 3 to smithi093153278-344 2023-08-27T20:53:15.759 INFO:tasks.rados.rados.0.smithi093.stdout:1689: finishing write tid 4 to smithi093153278-344 2023-08-27T20:53:15.759 INFO:tasks.rados.rados.0.smithi093.stdout:1689: finishing write tid 5 to smithi093153278-344 2023-08-27T20:53:15.759 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 344 v 262 (ObjNum 1164 snap 0 seq_num 1164) dirty exists 2023-08-27T20:53:15.759 INFO:tasks.rados.rados.0.smithi093.stdout:1689: left oid 344 (ObjNum 1164 snap 0 seq_num 1164) 2023-08-27T20:53:15.759 INFO:tasks.rados.rados.0.smithi093.stdout:1690: expect (ObjNum 1073 snap 0 seq_num 1073) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1683: done (8 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1684: done (7 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1685: done (6 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1686: done (5 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1687: done (4 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1688: done (3 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1689: done (2 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1690: done (1 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1691: done (0 left) 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1692: read oid 384 snap -1 2023-08-27T20:53:15.865 INFO:tasks.rados.rados.0.smithi093.stdout:1692: expect (ObjNum 808 snap 0 seq_num 808) 2023-08-27T20:53:15.866 INFO:tasks.rados.rados.0.smithi093.stdout:1693: read oid 21 snap -1 2023-08-27T20:53:15.866 INFO:tasks.rados.rados.0.smithi093.stdout:1693: expect (ObjNum 601 snap 0 seq_num 601) 2023-08-27T20:53:15.866 INFO:tasks.rados.rados.0.smithi093.stdout:1694: write oid 148 current snap is 0 2023-08-27T20:53:15.866 INFO:tasks.rados.rados.0.smithi093.stdout:1694: seq_num 1166 ranges {796962=742343,2031416=437730,2936739=413167} 2023-08-27T20:53:15.872 INFO:tasks.rados.rados.0.smithi093.stdout:1694: writing smithi093153278-148 from 796962 to 1539305 tid 1 2023-08-27T20:53:15.878 INFO:tasks.rados.rados.0.smithi093.stdout:1694: writing smithi093153278-148 from 2031416 to 2469146 tid 2 2023-08-27T20:53:15.883 INFO:tasks.rados.rados.0.smithi093.stdout:1694: writing smithi093153278-148 from 2936739 to 3349906 tid 3 2023-08-27T20:53:15.883 INFO:tasks.rados.rados.0.smithi093.stdout:1695: read oid 47 snap -1 2023-08-27T20:53:15.883 INFO:tasks.rados.rados.0.smithi093.stdout:1695: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:53:15.883 INFO:tasks.rados.rados.0.smithi093.stdout:1696: read oid 42 snap -1 2023-08-27T20:53:15.883 INFO:tasks.rados.rados.0.smithi093.stdout:1696: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:53:15.884 INFO:tasks.rados.rados.0.smithi093.stdout:1697: delete oid 412 current snap is 0 2023-08-27T20:53:15.884 INFO:tasks.rados.rados.0.smithi093.stdout:1693: expect (ObjNum 601 snap 0 seq_num 601) 2023-08-27T20:53:15.965 INFO:tasks.rados.rados.0.smithi093.stdout:1692: expect (ObjNum 808 snap 0 seq_num 808) 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:1694: finishing write tid 1 to smithi093153278-148 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:1694: finishing write tid 2 to smithi093153278-148 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:1694: finishing write tid 3 to smithi093153278-148 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:1694: finishing write tid 4 to smithi093153278-148 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:1694: finishing write tid 5 to smithi093153278-148 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 148 v 219 (ObjNum 1166 snap 0 seq_num 1166) dirty exists 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:1694: left oid 148 (ObjNum 1166 snap 0 seq_num 1166) 2023-08-27T20:53:16.108 INFO:tasks.rados.rados.0.smithi093.stdout:1695: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:53:16.193 INFO:tasks.rados.rados.0.smithi093.stdout:1696: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:53:16.275 INFO:tasks.rados.rados.0.smithi093.stdout:1692: done (5 left) 2023-08-27T20:53:16.275 INFO:tasks.rados.rados.0.smithi093.stdout:1693: done (4 left) 2023-08-27T20:53:16.275 INFO:tasks.rados.rados.0.smithi093.stdout:1694: done (3 left) 2023-08-27T20:53:16.275 INFO:tasks.rados.rados.0.smithi093.stdout:1695: done (2 left) 2023-08-27T20:53:16.275 INFO:tasks.rados.rados.0.smithi093.stdout:1696: done (1 left) 2023-08-27T20:53:16.275 INFO:tasks.rados.rados.0.smithi093.stdout:1697: done (0 left) 2023-08-27T20:53:16.276 INFO:tasks.rados.rados.0.smithi093.stdout:1698: delete oid 451 current snap is 0 2023-08-27T20:53:16.277 INFO:tasks.rados.rados.0.smithi093.stdout:1698: done (0 left) 2023-08-27T20:53:16.277 INFO:tasks.rados.rados.0.smithi093.stdout:1699: read oid 122 snap -1 2023-08-27T20:53:16.277 INFO:tasks.rados.rados.0.smithi093.stdout:1699: expect (ObjNum 1098 snap 0 seq_num 1098) 2023-08-27T20:53:16.277 INFO:tasks.rados.rados.0.smithi093.stdout:1700: write oid 281 current snap is 0 2023-08-27T20:53:16.277 INFO:tasks.rados.rados.0.smithi093.stdout:1700: seq_num 1169 ranges {447383=568090,1690509=701410,2902655=646983} 2023-08-27T20:53:16.283 INFO:tasks.rados.rados.0.smithi093.stdout:1700: writing smithi093153278-281 from 447383 to 1015473 tid 1 2023-08-27T20:53:16.290 INFO:tasks.rados.rados.0.smithi093.stdout:1700: writing smithi093153278-281 from 1690509 to 2391919 tid 2 2023-08-27T20:53:16.297 INFO:tasks.rados.rados.0.smithi093.stdout:1700: writing smithi093153278-281 from 2902655 to 3549638 tid 3 2023-08-27T20:53:16.297 INFO:tasks.rados.rados.0.smithi093.stdout:1701: write oid 359 current snap is 0 2023-08-27T20:53:16.297 INFO:tasks.rados.rados.0.smithi093.stdout:1701: seq_num 1170 ranges {464190=643339,1748991=445068,2930078=574608} 2023-08-27T20:53:16.303 INFO:tasks.rados.rados.0.smithi093.stdout:1701: writing smithi093153278-359 from 464190 to 1107529 tid 1 2023-08-27T20:53:16.309 INFO:tasks.rados.rados.0.smithi093.stdout:1701: writing smithi093153278-359 from 1748991 to 2194059 tid 2 2023-08-27T20:53:16.316 INFO:tasks.rados.rados.0.smithi093.stdout:1701: writing smithi093153278-359 from 2930078 to 3504686 tid 3 2023-08-27T20:53:16.317 INFO:tasks.rados.rados.0.smithi093.stdout:1702: read oid 300 snap -1 2023-08-27T20:53:16.317 INFO:tasks.rados.rados.0.smithi093.stdout:1702: expect (ObjNum 1014 snap 0 seq_num 1014) 2023-08-27T20:53:16.317 INFO:tasks.rados.rados.0.smithi093.stdout:1703: write oid 110 current snap is 0 2023-08-27T20:53:16.317 INFO:tasks.rados.rados.0.smithi093.stdout:1703: seq_num 1171 ranges {480997=434941,1407473=705079,2557501=785880} 2023-08-27T20:53:16.321 INFO:tasks.rados.rados.0.smithi093.stdout:1703: writing smithi093153278-110 from 480997 to 915938 tid 1 2023-08-27T20:53:16.328 INFO:tasks.rados.rados.0.smithi093.stdout:1703: writing smithi093153278-110 from 1407473 to 2112552 tid 2 2023-08-27T20:53:16.335 INFO:tasks.rados.rados.0.smithi093.stdout:1703: writing smithi093153278-110 from 2557501 to 3343381 tid 3 2023-08-27T20:53:16.335 INFO:tasks.rados.rados.0.smithi093.stdout:1704: write oid 355 current snap is 0 2023-08-27T20:53:16.335 INFO:tasks.rados.rados.0.smithi093.stdout:1699: expect (ObjNum 1098 snap 0 seq_num 1098) 2023-08-27T20:53:16.463 INFO:tasks.rados.rados.0.smithi093.stdout:1700: finishing write tid 1 to smithi093153278-281 2023-08-27T20:53:16.463 INFO:tasks.rados.rados.0.smithi093.stdout:1700: finishing write tid 2 to smithi093153278-281 2023-08-27T20:53:16.463 INFO:tasks.rados.rados.0.smithi093.stdout:1700: finishing write tid 3 to smithi093153278-281 2023-08-27T20:53:16.463 INFO:tasks.rados.rados.0.smithi093.stdout:1700: finishing write tid 4 to smithi093153278-281 2023-08-27T20:53:16.463 INFO:tasks.rados.rados.0.smithi093.stdout:1700: finishing write tid 5 to smithi093153278-281 2023-08-27T20:53:16.464 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 281 v 267 (ObjNum 1169 snap 0 seq_num 1169) dirty exists 2023-08-27T20:53:16.464 INFO:tasks.rados.rados.0.smithi093.stdout:1700: left oid 281 (ObjNum 1169 snap 0 seq_num 1169) 2023-08-27T20:53:16.464 INFO:tasks.rados.rados.0.smithi093.stdout:1701: finishing write tid 1 to smithi093153278-359 2023-08-27T20:53:16.464 INFO:tasks.rados.rados.0.smithi093.stdout:1704: seq_num 1172 ranges {497804=510190,1465955=448737,2584924=597152} 2023-08-27T20:53:16.468 INFO:tasks.rados.rados.0.smithi093.stdout:1704: writing smithi093153278-355 from 497804 to 1007994 tid 1 2023-08-27T20:53:16.473 INFO:tasks.rados.rados.0.smithi093.stdout:1704: writing smithi093153278-355 from 1465955 to 1914692 tid 2 2023-08-27T20:53:16.480 INFO:tasks.rados.rados.0.smithi093.stdout:1704: writing smithi093153278-355 from 2584924 to 3182076 tid 3 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1699: done (5 left) 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1700: done (4 left) 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1705: read oid 241 snap -1 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1705: expect (ObjNum 240 snap 0 seq_num 240) 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1706: delete oid 246 current snap is 0 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1701: finishing write tid 2 to smithi093153278-359 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1701: finishing write tid 3 to smithi093153278-359 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1701: finishing write tid 4 to smithi093153278-359 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1701: finishing write tid 5 to smithi093153278-359 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 359 v 323 (ObjNum 1170 snap 0 seq_num 1170) dirty exists 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1701: left oid 359 (ObjNum 1170 snap 0 seq_num 1170) 2023-08-27T20:53:16.481 INFO:tasks.rados.rados.0.smithi093.stdout:1703: finishing write tid 1 to smithi093153278-110 2023-08-27T20:53:16.482 INFO:tasks.rados.rados.0.smithi093.stdout:1702: expect (ObjNum 1014 snap 0 seq_num 1014) 2023-08-27T20:53:16.576 INFO:tasks.rados.rados.0.smithi093.stdout:1703: finishing write tid 2 to smithi093153278-110 2023-08-27T20:53:16.576 INFO:tasks.rados.rados.0.smithi093.stdout:1703: finishing write tid 3 to smithi093153278-110 2023-08-27T20:53:16.576 INFO:tasks.rados.rados.0.smithi093.stdout:1703: finishing write tid 4 to smithi093153278-110 2023-08-27T20:53:16.576 INFO:tasks.rados.rados.0.smithi093.stdout:1703: finishing write tid 5 to smithi093153278-110 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 110 v 273 (ObjNum 1171 snap 0 seq_num 1171) dirty exists 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1703: left oid 110 (ObjNum 1171 snap 0 seq_num 1171) 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1701: done (5 left) 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1702: done (4 left) 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1703: done (3 left) 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1706: done (2 left) 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1707: delete oid 353 current snap is 0 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1704: finishing write tid 1 to smithi093153278-355 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1704: finishing write tid 2 to smithi093153278-355 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1704: finishing write tid 3 to smithi093153278-355 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1704: finishing write tid 4 to smithi093153278-355 2023-08-27T20:53:16.577 INFO:tasks.rados.rados.0.smithi093.stdout:1704: finishing write tid 5 to smithi093153278-355 2023-08-27T20:53:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 355 v 206 (ObjNum 1172 snap 0 seq_num 1172) dirty exists 2023-08-27T20:53:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:1704: left oid 355 (ObjNum 1172 snap 0 seq_num 1172) 2023-08-27T20:53:16.578 INFO:tasks.rados.rados.0.smithi093.stdout:1705: expect (ObjNum 240 snap 0 seq_num 240) 2023-08-27T20:53:16.639 INFO:tasks.rados.rados.0.smithi093.stdout:1704: done (2 left) 2023-08-27T20:53:16.639 INFO:tasks.rados.rados.0.smithi093.stdout:1705: done (1 left) 2023-08-27T20:53:16.639 INFO:tasks.rados.rados.0.smithi093.stdout:1707: done (0 left) 2023-08-27T20:53:16.639 INFO:tasks.rados.rados.0.smithi093.stdout:1708: write oid 102 current snap is 0 2023-08-27T20:53:16.639 INFO:tasks.rados.rados.0.smithi093.stdout:1708: seq_num 1175 ranges {548225=735937,1925048=596064,2950840=547321} 2023-08-27T20:53:16.645 INFO:tasks.rados.rados.0.smithi093.stdout:1708: writing smithi093153278-102 from 548225 to 1284162 tid 1 2023-08-27T20:53:16.652 INFO:tasks.rados.rados.0.smithi093.stdout:1708: writing smithi093153278-102 from 1925048 to 2521112 tid 2 2023-08-27T20:53:16.658 INFO:tasks.rados.rados.0.smithi093.stdout:1708: writing smithi093153278-102 from 2950840 to 3498161 tid 3 2023-08-27T20:53:16.658 INFO:tasks.rados.rados.0.smithi093.stdout:1709: delete oid 5 current snap is 0 2023-08-27T20:53:16.658 INFO:tasks.rados.rados.0.smithi093.stdout:1708: finishing write tid 1 to smithi093153278-102 2023-08-27T20:53:16.658 INFO:tasks.rados.rados.0.smithi093.stdout:1708: finishing write tid 2 to smithi093153278-102 2023-08-27T20:53:16.660 INFO:tasks.rados.rados.0.smithi093.stdout:1709: done (1 left) 2023-08-27T20:53:16.660 INFO:tasks.rados.rados.0.smithi093.stdout:1710: write oid 230 current snap is 0 2023-08-27T20:53:16.660 INFO:tasks.rados.rados.0.smithi093.stdout:1710: seq_num 1177 ranges {581839=486435,1642012=599733,3005686=569865} 2023-08-27T20:53:16.665 INFO:tasks.rados.rados.0.smithi093.stdout:1710: writing smithi093153278-230 from 581839 to 1068274 tid 1 2023-08-27T20:53:16.672 INFO:tasks.rados.rados.0.smithi093.stdout:1710: writing smithi093153278-230 from 1642012 to 2241745 tid 2 2023-08-27T20:53:16.679 INFO:tasks.rados.rados.0.smithi093.stdout:1710: writing smithi093153278-230 from 3005686 to 3575551 tid 3 2023-08-27T20:53:16.679 INFO:tasks.rados.rados.0.smithi093.stdout:1711: write oid 174 current snap is 0 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:1708: finishing write tid 3 to smithi093153278-102 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:1708: finishing write tid 4 to smithi093153278-102 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:1708: finishing write tid 5 to smithi093153278-102 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 102 v 210 (ObjNum 1175 snap 0 seq_num 1175) dirty exists 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:1708: left oid 102 (ObjNum 1175 snap 0 seq_num 1175) 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:1710: finishing write tid 1 to smithi093153278-230 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:1710: finishing write tid 2 to smithi093153278-230 2023-08-27T20:53:16.680 INFO:tasks.rados.rados.0.smithi093.stdout:1711: seq_num 1178 ranges {598646=561684,1700494=459744,2633109=781137} 2023-08-27T20:53:16.685 INFO:tasks.rados.rados.0.smithi093.stdout:1711: writing smithi093153278-174 from 598646 to 1160330 tid 1 2023-08-27T20:53:16.691 INFO:tasks.rados.rados.0.smithi093.stdout:1711: writing smithi093153278-174 from 1700494 to 2160238 tid 2 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1711: writing smithi093153278-174 from 2633109 to 3414246 tid 3 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1708: done (2 left) 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1712: write oid 484 current snap is 0 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1710: finishing write tid 3 to smithi093153278-230 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1710: finishing write tid 4 to smithi093153278-230 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1710: finishing write tid 5 to smithi093153278-230 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 243 (ObjNum 1177 snap 0 seq_num 1177) dirty exists 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1710: left oid 230 (ObjNum 1177 snap 0 seq_num 1177) 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1711: finishing write tid 1 to smithi093153278-174 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1711: finishing write tid 2 to smithi093153278-174 2023-08-27T20:53:16.699 INFO:tasks.rados.rados.0.smithi093.stdout:1712: seq_num 1179 ranges {615453=753286,2158976=603402,3460532=354921} 2023-08-27T20:53:16.706 INFO:tasks.rados.rados.0.smithi093.stdout:1712: writing smithi093153278-484 from 615453 to 1368739 tid 1 2023-08-27T20:53:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:1712: writing smithi093153278-484 from 2158976 to 2762378 tid 2 2023-08-27T20:53:16.722 INFO:tasks.rados.rados.0.smithi093.stdout:1712: writing smithi093153278-484 from 3460532 to 3815453 tid 3 2023-08-27T20:53:16.722 INFO:tasks.rados.rados.0.smithi093.stdout:1710: done (2 left) 2023-08-27T20:53:16.722 INFO:tasks.rados.rados.0.smithi093.stdout:1713: write oid 429 current snap is 0 2023-08-27T20:53:16.722 INFO:tasks.rados.rados.0.smithi093.stdout:1711: finishing write tid 3 to smithi093153278-174 2023-08-27T20:53:16.723 INFO:tasks.rados.rados.0.smithi093.stdout:1713: seq_num 1180 ranges {632260=428535,1817458=463413,2687955=520034,3709324=122936} 2023-08-27T20:53:16.725 INFO:tasks.rados.rados.0.smithi093.stdout:1713: writing smithi093153278-429 from 632260 to 1060795 tid 1 2023-08-27T20:53:16.732 INFO:tasks.rados.rados.0.smithi093.stdout:1713: writing smithi093153278-429 from 1817458 to 2280871 tid 2 2023-08-27T20:53:16.737 INFO:tasks.rados.rados.0.smithi093.stdout:1713: writing smithi093153278-429 from 2687955 to 3207989 tid 3 2023-08-27T20:53:16.741 INFO:tasks.rados.rados.0.smithi093.stdout:1713: writing smithi093153278-429 from 3709324 to 3832260 tid 4 2023-08-27T20:53:16.741 INFO:tasks.rados.rados.0.smithi093.stdout:1714: delete oid 21 current snap is 0 2023-08-27T20:53:16.741 INFO:tasks.rados.rados.0.smithi093.stdout:1711: finishing write tid 4 to smithi093153278-174 2023-08-27T20:53:16.741 INFO:tasks.rados.rados.0.smithi093.stdout:1711: finishing write tid 5 to smithi093153278-174 2023-08-27T20:53:16.741 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 174 v 235 (ObjNum 1178 snap 0 seq_num 1178) dirty exists 2023-08-27T20:53:16.741 INFO:tasks.rados.rados.0.smithi093.stdout:1711: left oid 174 (ObjNum 1178 snap 0 seq_num 1178) 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1712: finishing write tid 1 to smithi093153278-484 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1712: finishing write tid 2 to smithi093153278-484 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1712: finishing write tid 3 to smithi093153278-484 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1712: finishing write tid 4 to smithi093153278-484 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1712: finishing write tid 5 to smithi093153278-484 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 484 v 327 (ObjNum 1179 snap 0 seq_num 1179) dirty exists 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1712: left oid 484 (ObjNum 1179 snap 0 seq_num 1179) 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1713: finishing write tid 1 to smithi093153278-429 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1713: finishing write tid 2 to smithi093153278-429 2023-08-27T20:53:16.742 INFO:tasks.rados.rados.0.smithi093.stdout:1713: finishing write tid 3 to smithi093153278-429 2023-08-27T20:53:16.743 INFO:tasks.rados.rados.0.smithi093.stdout:1711: done (3 left) 2023-08-27T20:53:16.743 INFO:tasks.rados.rados.0.smithi093.stdout:1712: done (2 left) 2023-08-27T20:53:16.743 INFO:tasks.rados.rados.0.smithi093.stdout:1714: done (1 left) 2023-08-27T20:53:16.743 INFO:tasks.rados.rados.0.smithi093.stdout:1715: read oid 86 snap -1 2023-08-27T20:53:16.743 INFO:tasks.rados.rados.0.smithi093.stdout:1715: expect (ObjNum 1126 snap 0 seq_num 1126) 2023-08-27T20:53:16.744 INFO:tasks.rados.rados.0.smithi093.stdout:1716: write oid 250 current snap is 0 2023-08-27T20:53:16.744 INFO:tasks.rados.rados.0.smithi093.stdout:1716: seq_num 1182 ranges {665874=579033,1934422=467082,3142801=542578} 2023-08-27T20:53:16.749 INFO:tasks.rados.rados.0.smithi093.stdout:1716: writing smithi093153278-250 from 665874 to 1244907 tid 1 2023-08-27T20:53:16.755 INFO:tasks.rados.rados.0.smithi093.stdout:1716: writing smithi093153278-250 from 1934422 to 2401504 tid 2 2023-08-27T20:53:16.763 INFO:tasks.rados.rados.0.smithi093.stdout:1716: writing smithi093153278-250 from 3142801 to 3685379 tid 3 2023-08-27T20:53:16.763 INFO:tasks.rados.rados.0.smithi093.stdout:1717: read oid 101 snap -1 2023-08-27T20:53:16.763 INFO:tasks.rados.rados.0.smithi093.stdout:1717: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:53:16.763 INFO:tasks.rados.rados.0.smithi093.stdout:1718: write oid 372 current snap is 0 2023-08-27T20:53:16.763 INFO:tasks.rados.rados.0.smithi093.stdout:1718: seq_num 1183 ranges {682681=654282,1876551=610740,3053871=470203} 2023-08-27T20:53:16.770 INFO:tasks.rados.rados.0.smithi093.stdout:1718: writing smithi093153278-372 from 682681 to 1336963 tid 1 2023-08-27T20:53:16.776 INFO:tasks.rados.rados.0.smithi093.stdout:1718: writing smithi093153278-372 from 1876551 to 2487291 tid 2 2023-08-27T20:53:16.782 INFO:tasks.rados.rados.0.smithi093.stdout:1718: writing smithi093153278-372 from 3053871 to 3524074 tid 3 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1719: read oid 48 snap -1 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1719: expect (ObjNum 725 snap 0 seq_num 725) 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1720: write oid 87 current snap is 0 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1713: finishing write tid 4 to smithi093153278-429 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1713: finishing write tid 5 to smithi093153278-429 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1713: finishing write tid 6 to smithi093153278-429 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 429 v 278 (ObjNum 1180 snap 0 seq_num 1180) dirty exists 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1713: left oid 429 (ObjNum 1180 snap 0 seq_num 1180) 2023-08-27T20:53:16.783 INFO:tasks.rados.rados.0.smithi093.stdout:1720: seq_num 1184 ranges {699488=729531,1935033=470751,3081294=681475} 2023-08-27T20:53:16.790 INFO:tasks.rados.rados.0.smithi093.stdout:1720: writing smithi093153278-87 from 699488 to 1429019 tid 1 2023-08-27T20:53:16.795 INFO:tasks.rados.rados.0.smithi093.stdout:1720: writing smithi093153278-87 from 1935033 to 2405784 tid 2 2023-08-27T20:53:16.803 INFO:tasks.rados.rados.0.smithi093.stdout:1720: writing smithi093153278-87 from 3081294 to 3762769 tid 3 2023-08-27T20:53:16.803 INFO:tasks.rados.rados.0.smithi093.stdout:1713: done (6 left) 2023-08-27T20:53:16.803 INFO:tasks.rados.rados.0.smithi093.stdout:1721: read oid 169 snap -1 2023-08-27T20:53:16.803 INFO:tasks.rados.rados.0.smithi093.stdout:1721: expect (ObjNum 1123 snap 0 seq_num 1123) 2023-08-27T20:53:16.804 INFO:tasks.rados.rados.0.smithi093.stdout:1722: write oid 146 current snap is 0 2023-08-27T20:53:16.804 INFO:tasks.rados.rados.0.smithi093.stdout:1715: expect (ObjNum 1126 snap 0 seq_num 1126) 2023-08-27T20:53:16.967 INFO:tasks.rados.rados.0.smithi093.stdout:1716: finishing write tid 1 to smithi093153278-250 2023-08-27T20:53:16.967 INFO:tasks.rados.rados.0.smithi093.stdout:1716: finishing write tid 2 to smithi093153278-250 2023-08-27T20:53:16.967 INFO:tasks.rados.rados.0.smithi093.stdout:1716: finishing write tid 3 to smithi093153278-250 2023-08-27T20:53:16.967 INFO:tasks.rados.rados.0.smithi093.stdout:1722: seq_num 1185 ranges {716295=404780,1593515=614409,2708717=492747,3640519=275776} 2023-08-27T20:53:16.973 INFO:tasks.rados.rados.0.smithi093.stdout:1722: writing smithi093153278-146 from 716295 to 1121075 tid 1 2023-08-27T20:53:16.979 INFO:tasks.rados.rados.0.smithi093.stdout:1722: writing smithi093153278-146 from 1593515 to 2207924 tid 2 2023-08-27T20:53:16.984 INFO:tasks.rados.rados.0.smithi093.stdout:1722: writing smithi093153278-146 from 2708717 to 3201464 tid 3 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1722: writing smithi093153278-146 from 3640519 to 3916295 tid 4 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1715: done (7 left) 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1723: read oid 410 snap -1 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1723: expect deleted 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1724: read oid 347 snap -1 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1724: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1725: read oid 60 snap -1 2023-08-27T20:53:16.989 INFO:tasks.rados.rados.0.smithi093.stdout:1725: expect (ObjNum 883 snap 0 seq_num 883) 2023-08-27T20:53:16.990 INFO:tasks.rados.rados.0.smithi093.stdout:1726: write oid 393 current snap is 0 2023-08-27T20:53:16.990 INFO:tasks.rados.rados.0.smithi093.stdout:1716: finishing write tid 4 to smithi093153278-250 2023-08-27T20:53:16.990 INFO:tasks.rados.rados.0.smithi093.stdout:1716: finishing write tid 5 to smithi093153278-250 2023-08-27T20:53:16.990 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 250 v 331 (ObjNum 1182 snap 0 seq_num 1182) dirty exists 2023-08-27T20:53:16.990 INFO:tasks.rados.rados.0.smithi093.stdout:1716: left oid 250 (ObjNum 1182 snap 0 seq_num 1182) 2023-08-27T20:53:16.990 INFO:tasks.rados.rados.0.smithi093.stdout:1726: seq_num 1186 ranges {733102=596382,1768350=758067,3136140=704019} 2023-08-27T20:53:16.996 INFO:tasks.rados.rados.0.smithi093.stdout:1726: writing smithi093153278-393 from 733102 to 1329484 tid 1 2023-08-27T20:53:17.003 INFO:tasks.rados.rados.0.smithi093.stdout:1726: writing smithi093153278-393 from 1768350 to 2526417 tid 2 2023-08-27T20:53:17.010 INFO:tasks.rados.rados.0.smithi093.stdout:1726: writing smithi093153278-393 from 3136140 to 3840159 tid 3 2023-08-27T20:53:17.010 INFO:tasks.rados.rados.0.smithi093.stdout:1716: done (10 left) 2023-08-27T20:53:17.010 INFO:tasks.rados.rados.0.smithi093.stdout:1727: write oid 453 current snap is 0 2023-08-27T20:53:17.011 INFO:tasks.rados.rados.0.smithi093.stdout:1717: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:53:17.127 INFO:tasks.rados.rados.0.smithi093.stdout:1718: finishing write tid 1 to smithi093153278-372 2023-08-27T20:53:17.127 INFO:tasks.rados.rados.0.smithi093.stdout:1718: finishing write tid 2 to smithi093153278-372 2023-08-27T20:53:17.127 INFO:tasks.rados.rados.0.smithi093.stdout:1718: finishing write tid 3 to smithi093153278-372 2023-08-27T20:53:17.127 INFO:tasks.rados.rados.0.smithi093.stdout:1727: seq_num 1187 ranges {749909=671631,2110479=618078,3163563=515291} 2023-08-27T20:53:17.135 INFO:tasks.rados.rados.0.smithi093.stdout:1727: writing smithi093153278-453 from 749909 to 1421540 tid 1 2023-08-27T20:53:17.142 INFO:tasks.rados.rados.0.smithi093.stdout:1727: writing smithi093153278-453 from 2110479 to 2728557 tid 2 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:1727: writing smithi093153278-453 from 3163563 to 3678854 tid 3 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:1717: done (10 left) 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:1728: write oid 273 current snap is 0 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:1718: finishing write tid 4 to smithi093153278-372 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:1718: finishing write tid 5 to smithi093153278-372 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 372 v 219 (ObjNum 1183 snap 0 seq_num 1183) dirty exists 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:1718: left oid 372 (ObjNum 1183 snap 0 seq_num 1183) 2023-08-27T20:53:17.148 INFO:tasks.rados.rados.0.smithi093.stdout:1728: seq_num 1188 ranges {766716=746880,2168961=761736,3474633=442916} 2023-08-27T20:53:17.156 INFO:tasks.rados.rados.0.smithi093.stdout:1728: writing smithi093153278-273 from 766716 to 1513596 tid 1 2023-08-27T20:53:17.163 INFO:tasks.rados.rados.0.smithi093.stdout:1728: writing smithi093153278-273 from 2168961 to 2930697 tid 2 2023-08-27T20:53:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:1728: writing smithi093153278-273 from 3474633 to 3917549 tid 3 2023-08-27T20:53:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:1718: done (10 left) 2023-08-27T20:53:17.170 INFO:tasks.rados.rados.0.smithi093.stdout:1729: write oid 49 current snap is 0 2023-08-27T20:53:17.170 INFO:tasks.rados.rados.0.smithi093.stdout:1729: seq_num 1189 ranges {783523=422129,1827443=621747,3218409=654188} 2023-08-27T20:53:17.176 INFO:tasks.rados.rados.0.smithi093.stdout:1729: writing smithi093153278-49 from 783523 to 1205652 tid 1 2023-08-27T20:53:17.182 INFO:tasks.rados.rados.0.smithi093.stdout:1729: writing smithi093153278-49 from 1827443 to 2449190 tid 2 2023-08-27T20:53:17.190 INFO:tasks.rados.rados.0.smithi093.stdout:1729: writing smithi093153278-49 from 3218409 to 3872597 tid 3 2023-08-27T20:53:17.191 INFO:tasks.rados.rados.0.smithi093.stdout:1730: write oid 224 current snap is 0 2023-08-27T20:53:17.191 INFO:tasks.rados.rados.0.smithi093.stdout:1719: expect (ObjNum 725 snap 0 seq_num 725) 2023-08-27T20:53:17.277 INFO:tasks.rados.rados.0.smithi093.stdout:1720: finishing write tid 1 to smithi093153278-87 2023-08-27T20:53:17.277 INFO:tasks.rados.rados.0.smithi093.stdout:1730: seq_num 1190 ranges {400330=497378,1485925=514405} 2023-08-27T20:53:17.282 INFO:tasks.rados.rados.0.smithi093.stdout:1730: writing smithi093153278-224 from 400330 to 897708 tid 1 2023-08-27T20:53:17.288 INFO:tasks.rados.rados.0.smithi093.stdout:1730: writing smithi093153278-224 from 1485925 to 2000330 tid 2 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1719: done (11 left) 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1731: write oid 416 current snap is 0 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1720: finishing write tid 2 to smithi093153278-87 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1720: finishing write tid 3 to smithi093153278-87 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1720: finishing write tid 4 to smithi093153278-87 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1720: finishing write tid 5 to smithi093153278-87 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 224 (ObjNum 1184 snap 0 seq_num 1184) dirty exists 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1720: left oid 87 (ObjNum 1184 snap 0 seq_num 1184) 2023-08-27T20:53:17.289 INFO:tasks.rados.rados.0.smithi093.stdout:1731: seq_num 1191 ranges {417137=572627,1428054=589083} 2023-08-27T20:53:17.294 INFO:tasks.rados.rados.0.smithi093.stdout:1731: writing smithi093153278-416 from 417137 to 989764 tid 1 2023-08-27T20:53:17.300 INFO:tasks.rados.rados.0.smithi093.stdout:1731: writing smithi093153278-416 from 1428054 to 2017137 tid 2 2023-08-27T20:53:17.300 INFO:tasks.rados.rados.0.smithi093.stdout:1720: done (11 left) 2023-08-27T20:53:17.300 INFO:tasks.rados.rados.0.smithi093.stdout:1732: read oid 70 snap -1 2023-08-27T20:53:17.300 INFO:tasks.rados.rados.0.smithi093.stdout:1732: expect (ObjNum 899 snap 0 seq_num 899) 2023-08-27T20:53:17.300 INFO:tasks.rados.rados.0.smithi093.stdout:1733: read oid 457 snap -1 2023-08-27T20:53:17.300 INFO:tasks.rados.rados.0.smithi093.stdout:1733: expect (ObjNum 456 snap 0 seq_num 456) 2023-08-27T20:53:17.300 INFO:tasks.rados.rados.0.smithi093.stdout:1734: write oid 257 current snap is 0 2023-08-27T20:53:17.301 INFO:tasks.rados.rados.0.smithi093.stdout:1721: expect (ObjNum 1123 snap 0 seq_num 1123) 2023-08-27T20:53:17.446 INFO:tasks.rados.rados.0.smithi093.stdout:1722: finishing write tid 1 to smithi093153278-146 2023-08-27T20:53:17.447 INFO:tasks.rados.rados.0.smithi093.stdout:1722: finishing write tid 2 to smithi093153278-146 2023-08-27T20:53:17.447 INFO:tasks.rados.rados.0.smithi093.stdout:1722: finishing write tid 3 to smithi093153278-146 2023-08-27T20:53:17.447 INFO:tasks.rados.rados.0.smithi093.stdout:1724: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:53:17.564 INFO:tasks.rados.rados.0.smithi093.stdout:1722: finishing write tid 4 to smithi093153278-146 2023-08-27T20:53:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:1722: finishing write tid 5 to smithi093153278-146 2023-08-27T20:53:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:1722: finishing write tid 6 to smithi093153278-146 2023-08-27T20:53:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 146 v 230 (ObjNum 1185 snap 0 seq_num 1185) dirty exists 2023-08-27T20:53:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:1722: left oid 146 (ObjNum 1185 snap 0 seq_num 1185) 2023-08-27T20:53:17.565 INFO:tasks.rados.rados.0.smithi093.stdout:1725: expect (ObjNum 883 snap 0 seq_num 883) 2023-08-27T20:53:17.680 INFO:tasks.rados.rados.0.smithi093.stdout:1726: finishing write tid 1 to smithi093153278-393 2023-08-27T20:53:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:1726: finishing write tid 2 to smithi093153278-393 2023-08-27T20:53:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:1726: finishing write tid 3 to smithi093153278-393 2023-08-27T20:53:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:1726: finishing write tid 4 to smithi093153278-393 2023-08-27T20:53:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:1726: finishing write tid 5 to smithi093153278-393 2023-08-27T20:53:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 393 v 175 (ObjNum 1186 snap 0 seq_num 1186) dirty exists 2023-08-27T20:53:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:1726: left oid 393 (ObjNum 1186 snap 0 seq_num 1186) 2023-08-27T20:53:17.681 INFO:tasks.rados.rados.0.smithi093.stdout:1734: seq_num 1192 ranges {433944=647876,1486536=547408} 2023-08-27T20:53:17.686 INFO:tasks.rados.rados.0.smithi093.stdout:1734: writing smithi093153278-257 from 433944 to 1081820 tid 1 2023-08-27T20:53:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3100124071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:17.692 INFO:tasks.rados.rados.0.smithi093.stdout:1734: writing smithi093153278-257 from 1486536 to 2033944 tid 2 2023-08-27T20:53:17.692 INFO:tasks.rados.rados.0.smithi093.stdout:1721: done (13 left) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1722: done (12 left) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1723: done (11 left) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1724: done (10 left) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1725: done (9 left) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1726: done (8 left) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1735: read oid 31 snap -1 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1735: expect (ObjNum 760 snap 0 seq_num 760) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1736: read oid 175 snap -1 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1736: expect (ObjNum 1144 snap 0 seq_num 1144) 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1737: read oid 462 snap -1 2023-08-27T20:53:17.693 INFO:tasks.rados.rados.0.smithi093.stdout:1737: expect (ObjNum 581 snap 0 seq_num 581) 2023-08-27T20:53:17.694 INFO:tasks.rados.rados.0.smithi093.stdout:1738: read oid 58 snap -1 2023-08-27T20:53:17.694 INFO:tasks.rados.rados.0.smithi093.stdout:1738: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:53:17.694 INFO:tasks.rados.rados.0.smithi093.stdout:1739: write oid 419 current snap is 0 2023-08-27T20:53:17.694 INFO:tasks.rados.rados.0.smithi093.stdout:1739: seq_num 1193 ranges {450751=723125,1945018=105733} 2023-08-27T20:53:17.699 INFO:tasks.rados.rados.0.smithi093.stdout:1739: writing smithi093153278-419 from 450751 to 1173876 tid 1 2023-08-27T20:53:17.704 INFO:tasks.rados.rados.0.smithi093.stdout:1739: writing smithi093153278-419 from 1945018 to 2050751 tid 2 2023-08-27T20:53:17.704 INFO:tasks.rados.rados.0.smithi093.stdout:1740: write oid 496 current snap is 0 2023-08-27T20:53:17.704 INFO:tasks.rados.rados.0.smithi093.stdout:1727: finishing write tid 1 to smithi093153278-453 2023-08-27T20:53:17.704 INFO:tasks.rados.rados.0.smithi093.stdout:1740: seq_num 1194 ranges {467558=514727,1719853=347705} 2023-08-27T20:53:17.709 INFO:tasks.rados.rados.0.smithi093.stdout:1740: writing smithi093153278-496 from 467558 to 982285 tid 1 2023-08-27T20:53:17.716 INFO:tasks.rados.rados.0.smithi093.stdout:1740: writing smithi093153278-496 from 1719853 to 2067558 tid 2 2023-08-27T20:53:17.716 INFO:tasks.rados.rados.0.smithi093.stdout:1741: write oid 302 current snap is 0 2023-08-27T20:53:17.716 INFO:tasks.rados.rados.0.smithi093.stdout:1727: finishing write tid 2 to smithi093153278-453 2023-08-27T20:53:17.716 INFO:tasks.rados.rados.0.smithi093.stdout:1727: finishing write tid 3 to smithi093153278-453 2023-08-27T20:53:17.716 INFO:tasks.rados.rados.0.smithi093.stdout:1727: finishing write tid 4 to smithi093153278-453 2023-08-27T20:53:17.716 INFO:tasks.rados.rados.0.smithi093.stdout:1727: finishing write tid 5 to smithi093153278-453 2023-08-27T20:53:17.716 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 453 v 239 (ObjNum 1187 snap 0 seq_num 1187) dirty exists 2023-08-27T20:53:17.717 INFO:tasks.rados.rados.0.smithi093.stdout:1727: left oid 453 (ObjNum 1187 snap 0 seq_num 1187) 2023-08-27T20:53:17.717 INFO:tasks.rados.rados.0.smithi093.stdout:1728: finishing write tid 1 to smithi093153278-273 2023-08-27T20:53:17.717 INFO:tasks.rados.rados.0.smithi093.stdout:1728: finishing write tid 2 to smithi093153278-273 2023-08-27T20:53:17.717 INFO:tasks.rados.rados.0.smithi093.stdout:1728: finishing write tid 3 to smithi093153278-273 2023-08-27T20:53:17.717 INFO:tasks.rados.rados.0.smithi093.stdout:1728: finishing write tid 4 to smithi093153278-273 2023-08-27T20:53:17.717 INFO:tasks.rados.rados.0.smithi093.stdout:1741: seq_num 1195 ranges {484365=589976,1661982=422383} 2023-08-27T20:53:17.721 INFO:tasks.rados.rados.0.smithi093.stdout:1741: writing smithi093153278-302 from 484365 to 1074341 tid 1 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:1741: writing smithi093153278-302 from 1661982 to 2084365 tid 2 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:1727: done (14 left) 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:1742: delete oid 126 current snap is 0 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:1728: finishing write tid 5 to smithi093153278-273 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 273 v 335 (ObjNum 1188 snap 0 seq_num 1188) dirty exists 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:1728: left oid 273 (ObjNum 1188 snap 0 seq_num 1188) 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:1729: finishing write tid 1 to smithi093153278-49 2023-08-27T20:53:17.728 INFO:tasks.rados.rados.0.smithi093.stdout:1729: finishing write tid 2 to smithi093153278-49 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1729: finishing write tid 3 to smithi093153278-49 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1729: finishing write tid 4 to smithi093153278-49 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1729: finishing write tid 5 to smithi093153278-49 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 49 v 179 (ObjNum 1189 snap 0 seq_num 1189) dirty exists 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1729: left oid 49 (ObjNum 1189 snap 0 seq_num 1189) 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1730: finishing write tid 1 to smithi093153278-224 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1730: finishing write tid 2 to smithi093153278-224 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1730: finishing write tid 3 to smithi093153278-224 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1730: finishing write tid 4 to smithi093153278-224 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 224 v 213 (ObjNum 1190 snap 0 seq_num 1190) dirty exists 2023-08-27T20:53:17.729 INFO:tasks.rados.rados.0.smithi093.stdout:1730: left oid 224 (ObjNum 1190 snap 0 seq_num 1190) 2023-08-27T20:53:17.730 INFO:tasks.rados.rados.0.smithi093.stdout:1731: finishing write tid 1 to smithi093153278-416 2023-08-27T20:53:17.730 INFO:tasks.rados.rados.0.smithi093.stdout:1733: expect (ObjNum 456 snap 0 seq_num 456) 2023-08-27T20:53:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3100124071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3100124071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:17.835 INFO:tasks.rados.rados.0.smithi093.stdout:1731: finishing write tid 2 to smithi093153278-416 2023-08-27T20:53:17.835 INFO:tasks.rados.rados.0.smithi093.stdout:1731: finishing write tid 3 to smithi093153278-416 2023-08-27T20:53:17.835 INFO:tasks.rados.rados.0.smithi093.stdout:1731: finishing write tid 4 to smithi093153278-416 2023-08-27T20:53:17.835 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 416 v 242 (ObjNum 1191 snap 0 seq_num 1191) dirty exists 2023-08-27T20:53:17.835 INFO:tasks.rados.rados.0.smithi093.stdout:1731: left oid 416 (ObjNum 1191 snap 0 seq_num 1191) 2023-08-27T20:53:17.835 INFO:tasks.rados.rados.0.smithi093.stdout:1732: expect (ObjNum 899 snap 0 seq_num 899) 2023-08-27T20:53:18.013 INFO:tasks.rados.rados.0.smithi093.stdout:1734: finishing write tid 1 to smithi093153278-257 2023-08-27T20:53:18.014 INFO:tasks.rados.rados.0.smithi093.stdout:1734: finishing write tid 2 to smithi093153278-257 2023-08-27T20:53:18.014 INFO:tasks.rados.rados.0.smithi093.stdout:1734: finishing write tid 3 to smithi093153278-257 2023-08-27T20:53:18.014 INFO:tasks.rados.rados.0.smithi093.stdout:1734: finishing write tid 4 to smithi093153278-257 2023-08-27T20:53:18.014 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 257 v 338 (ObjNum 1192 snap 0 seq_num 1192) dirty exists 2023-08-27T20:53:18.014 INFO:tasks.rados.rados.0.smithi093.stdout:1734: left oid 257 (ObjNum 1192 snap 0 seq_num 1192) 2023-08-27T20:53:18.014 INFO:tasks.rados.rados.0.smithi093.stdout:1735: expect (ObjNum 760 snap 0 seq_num 760) 2023-08-27T20:53:18.100 INFO:tasks.rados.rados.0.smithi093.stdout:1728: done (14 left) 2023-08-27T20:53:18.100 INFO:tasks.rados.rados.0.smithi093.stdout:1729: done (13 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1730: done (12 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1731: done (11 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1732: done (10 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1733: done (9 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1734: done (8 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1735: done (7 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1742: done (6 left) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1743: read oid 363 snap -1 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1743: expect (ObjNum 1083 snap 0 seq_num 1083) 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1744: read oid 258 snap -1 2023-08-27T20:53:18.101 INFO:tasks.rados.rados.0.smithi093.stdout:1744: expect deleted 2023-08-27T20:53:18.102 INFO:tasks.rados.rados.0.smithi093.stdout:1745: read oid 424 snap -1 2023-08-27T20:53:18.102 INFO:tasks.rados.rados.0.smithi093.stdout:1745: expect (ObjNum 423 snap 0 seq_num 423) 2023-08-27T20:53:18.102 INFO:tasks.rados.rados.0.smithi093.stdout:1746: write oid 372 current snap is 0 2023-08-27T20:53:18.102 INFO:tasks.rados.rados.0.smithi093.stdout:1737: expect (ObjNum 581 snap 0 seq_num 581) 2023-08-27T20:53:18.269 INFO:tasks.rados.rados.0.smithi093.stdout:1736: expect (ObjNum 1144 snap 0 seq_num 1144) 2023-08-27T20:53:18.445 INFO:tasks.rados.rados.0.smithi093.stdout:1739: finishing write tid 1 to smithi093153278-419 2023-08-27T20:53:18.445 INFO:tasks.rados.rados.0.smithi093.stdout:1738: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:53:18.535 INFO:tasks.rados.rados.0.smithi093.stdout:1739: finishing write tid 2 to smithi093153278-419 2023-08-27T20:53:18.535 INFO:tasks.rados.rados.0.smithi093.stdout:1739: finishing write tid 3 to smithi093153278-419 2023-08-27T20:53:18.535 INFO:tasks.rados.rados.0.smithi093.stdout:1739: finishing write tid 4 to smithi093153278-419 2023-08-27T20:53:18.535 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 419 v 270 (ObjNum 1193 snap 0 seq_num 1193) dirty exists 2023-08-27T20:53:18.535 INFO:tasks.rados.rados.0.smithi093.stdout:1739: left oid 419 (ObjNum 1193 snap 0 seq_num 1193) 2023-08-27T20:53:18.536 INFO:tasks.rados.rados.0.smithi093.stdout:1740: finishing write tid 1 to smithi093153278-496 2023-08-27T20:53:18.536 INFO:tasks.rados.rados.0.smithi093.stdout:1746: seq_num 1197 ranges {517979=740474,1778946=339033} 2023-08-27T20:53:18.542 INFO:tasks.rados.rados.0.smithi093.stdout:1746: writing smithi093153278-372 from 517979 to 1258453 tid 1 2023-08-27T20:53:18.547 INFO:tasks.rados.rados.0.smithi093.stdout:1746: writing smithi093153278-372 from 1778946 to 2117979 tid 2 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1736: done (9 left) 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1737: done (8 left) 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1738: done (7 left) 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1739: done (6 left) 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1747: read oid 208 snap -1 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1747: expect deleted 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1748: write oid 277 current snap is 0 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1740: finishing write tid 2 to smithi093153278-496 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1740: finishing write tid 3 to smithi093153278-496 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1740: finishing write tid 4 to smithi093153278-496 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 496 v 246 (ObjNum 1194 snap 0 seq_num 1194) dirty exists 2023-08-27T20:53:18.548 INFO:tasks.rados.rados.0.smithi093.stdout:1740: left oid 496 (ObjNum 1194 snap 0 seq_num 1194) 2023-08-27T20:53:18.549 INFO:tasks.rados.rados.0.smithi093.stdout:1741: finishing write tid 1 to smithi093153278-302 2023-08-27T20:53:18.549 INFO:tasks.rados.rados.0.smithi093.stdout:1748: seq_num 1198 ranges {534786=415723,1437428=697358} 2023-08-27T20:53:18.552 INFO:tasks.rados.rados.0.smithi093.stdout:1748: writing smithi093153278-277 from 534786 to 950509 tid 1 2023-08-27T20:53:18.559 INFO:tasks.rados.rados.0.smithi093.stdout:1748: writing smithi093153278-277 from 1437428 to 2134786 tid 2 2023-08-27T20:53:18.559 INFO:tasks.rados.rados.0.smithi093.stdout:1740: done (7 left) 2023-08-27T20:53:18.559 INFO:tasks.rados.rados.0.smithi093.stdout:1749: write oid 487 current snap is 0 2023-08-27T20:53:18.559 INFO:tasks.rados.rados.0.smithi093.stdout:1741: finishing write tid 2 to smithi093153278-302 2023-08-27T20:53:18.559 INFO:tasks.rados.rados.0.smithi093.stdout:1741: finishing write tid 3 to smithi093153278-302 2023-08-27T20:53:18.559 INFO:tasks.rados.rados.0.smithi093.stdout:1749: seq_num 1199 ranges {551593=490972,1779557=372036} 2023-08-27T20:53:18.564 INFO:tasks.rados.rados.0.smithi093.stdout:1749: writing smithi093153278-487 from 551593 to 1042565 tid 1 2023-08-27T20:53:18.570 INFO:tasks.rados.rados.0.smithi093.stdout:1749: writing smithi093153278-487 from 1779557 to 2151593 tid 2 2023-08-27T20:53:18.571 INFO:tasks.rados.rados.0.smithi093.stdout:1750: write oid 9 current snap is 0 2023-08-27T20:53:18.571 INFO:tasks.rados.rados.0.smithi093.stdout:1741: finishing write tid 4 to smithi093153278-302 2023-08-27T20:53:18.571 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 302 v 273 (ObjNum 1195 snap 0 seq_num 1195) dirty exists 2023-08-27T20:53:18.571 INFO:tasks.rados.rados.0.smithi093.stdout:1741: left oid 302 (ObjNum 1195 snap 0 seq_num 1195) 2023-08-27T20:53:18.571 INFO:tasks.rados.rados.0.smithi093.stdout:1750: seq_num 1200 ranges {568400=566221,1838039=330361} 2023-08-27T20:53:18.576 INFO:tasks.rados.rados.0.smithi093.stdout:1750: writing smithi093153278-9 from 568400 to 1134621 tid 1 2023-08-27T20:53:18.582 INFO:tasks.rados.rados.0.smithi093.stdout:1750: writing smithi093153278-9 from 1838039 to 2168400 tid 2 2023-08-27T20:53:18.583 INFO:tasks.rados.rados.0.smithi093.stdout:1741: done (8 left) 2023-08-27T20:53:18.583 INFO:tasks.rados.rados.0.smithi093.stdout:1751: write oid 190 current snap is 0 2023-08-27T20:53:18.583 INFO:tasks.rados.rados.0.smithi093.stdout:1751: seq_num 1201 ranges {585207=641470,1896521=288686} 2023-08-27T20:53:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:1751: writing smithi093153278-190 from 585207 to 1226677 tid 1 2023-08-27T20:53:18.595 INFO:tasks.rados.rados.0.smithi093.stdout:1751: writing smithi093153278-190 from 1896521 to 2185207 tid 2 2023-08-27T20:53:18.595 INFO:tasks.rados.rados.0.smithi093.stdout:1752: read oid 405 snap -1 2023-08-27T20:53:18.595 INFO:tasks.rados.rados.0.smithi093.stdout:1752: expect (ObjNum 1124 snap 0 seq_num 1124) 2023-08-27T20:53:18.595 INFO:tasks.rados.rados.0.smithi093.stdout:1753: write oid 234 current snap is 0 2023-08-27T20:53:18.595 INFO:tasks.rados.rados.0.smithi093.stdout:1743: expect (ObjNum 1083 snap 0 seq_num 1083) 2023-08-27T20:53:18.709 INFO:tasks.rados.rados.0.smithi093.stdout:1745: expect (ObjNum 423 snap 0 seq_num 423) 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:1746: finishing write tid 1 to smithi093153278-372 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:1746: finishing write tid 2 to smithi093153278-372 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:1746: finishing write tid 3 to smithi093153278-372 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:1746: finishing write tid 4 to smithi093153278-372 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 372 v 222 (ObjNum 1197 snap 0 seq_num 1197) dirty exists 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:1746: left oid 372 (ObjNum 1197 snap 0 seq_num 1197) 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:1748: finishing write tid 1 to smithi093153278-277 2023-08-27T20:53:18.808 INFO:tasks.rados.rados.0.smithi093.stdout:1753: seq_num 1202 ranges {602014=433072,1671356=530658} 2023-08-27T20:53:18.813 INFO:tasks.rados.rados.0.smithi093.stdout:1753: writing smithi093153278-234 from 602014 to 1035086 tid 1 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1753: writing smithi093153278-234 from 1671356 to 2202014 tid 2 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1743: done (10 left) 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1744: done (9 left) 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1745: done (8 left) 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1746: done (7 left) 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1747: done (6 left) 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1754: read oid 32 snap -1 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1754: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1755: write oid 90 current snap is 0 2023-08-27T20:53:18.820 INFO:tasks.rados.rados.0.smithi093.stdout:1748: finishing write tid 2 to smithi093153278-277 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:1748: finishing write tid 3 to smithi093153278-277 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:1748: finishing write tid 4 to smithi093153278-277 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 277 v 281 (ObjNum 1198 snap 0 seq_num 1198) dirty exists 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:1748: left oid 277 (ObjNum 1198 snap 0 seq_num 1198) 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:1749: finishing write tid 1 to smithi093153278-487 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:1749: finishing write tid 2 to smithi093153278-487 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:1749: finishing write tid 3 to smithi093153278-487 2023-08-27T20:53:18.821 INFO:tasks.rados.rados.0.smithi093.stdout:1755: seq_num 1203 ranges {618821=508321,1613485=531077} 2023-08-27T20:53:18.825 INFO:tasks.rados.rados.0.smithi093.stdout:1755: writing smithi093153278-90 from 618821 to 1127142 tid 1 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:1755: writing smithi093153278-90 from 1613485 to 2144562 tid 2 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:1748: done (7 left) 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:1756: write oid 385 current snap is 0 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:1749: finishing write tid 4 to smithi093153278-487 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 487 v 276 (ObjNum 1199 snap 0 seq_num 1199) dirty exists 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:1749: left oid 487 (ObjNum 1199 snap 0 seq_num 1199) 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:1750: finishing write tid 1 to smithi093153278-9 2023-08-27T20:53:18.832 INFO:tasks.rados.rados.0.smithi093.stdout:1750: finishing write tid 2 to smithi093153278-9 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1750: finishing write tid 3 to smithi093153278-9 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1750: finishing write tid 4 to smithi093153278-9 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 9 v 284 (ObjNum 1200 snap 0 seq_num 1200) dirty exists 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1750: left oid 9 (ObjNum 1200 snap 0 seq_num 1200) 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1751: finishing write tid 1 to smithi093153278-190 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1751: finishing write tid 2 to smithi093153278-190 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1751: finishing write tid 3 to smithi093153278-190 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1751: finishing write tid 4 to smithi093153278-190 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 190 v 273 (ObjNum 1201 snap 0 seq_num 1201) dirty exists 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1751: left oid 190 (ObjNum 1201 snap 0 seq_num 1201) 2023-08-27T20:53:18.833 INFO:tasks.rados.rados.0.smithi093.stdout:1752: expect (ObjNum 1124 snap 0 seq_num 1124) 2023-08-27T20:53:18.952 INFO:tasks.rados.rados.0.smithi093.stdout:1753: finishing write tid 1 to smithi093153278-234 2023-08-27T20:53:18.952 INFO:tasks.rados.rados.0.smithi093.stdout:1753: finishing write tid 2 to smithi093153278-234 2023-08-27T20:53:18.953 INFO:tasks.rados.rados.0.smithi093.stdout:1753: finishing write tid 3 to smithi093153278-234 2023-08-27T20:53:18.953 INFO:tasks.rados.rados.0.smithi093.stdout:1753: finishing write tid 4 to smithi093153278-234 2023-08-27T20:53:18.953 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 234 v 234 (ObjNum 1202 snap 0 seq_num 1202) dirty exists 2023-08-27T20:53:18.953 INFO:tasks.rados.rados.0.smithi093.stdout:1753: left oid 234 (ObjNum 1202 snap 0 seq_num 1202) 2023-08-27T20:53:18.953 INFO:tasks.rados.rados.0.smithi093.stdout:1754: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:53:19.127 INFO:tasks.rados.rados.0.smithi093.stdout:1755: finishing write tid 1 to smithi093153278-90 2023-08-27T20:53:19.127 INFO:tasks.rados.rados.0.smithi093.stdout:1755: finishing write tid 2 to smithi093153278-90 2023-08-27T20:53:19.127 INFO:tasks.rados.rados.0.smithi093.stdout:1755: finishing write tid 3 to smithi093153278-90 2023-08-27T20:53:19.127 INFO:tasks.rados.rados.0.smithi093.stdout:1755: finishing write tid 4 to smithi093153278-90 2023-08-27T20:53:19.127 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 90 v 276 (ObjNum 1203 snap 0 seq_num 1203) dirty exists 2023-08-27T20:53:19.127 INFO:tasks.rados.rados.0.smithi093.stdout:1755: left oid 90 (ObjNum 1203 snap 0 seq_num 1203) 2023-08-27T20:53:19.127 INFO:tasks.rados.rados.0.smithi093.stdout:1756: seq_num 1204 ranges {635628=583570,1671967=563661} 2023-08-27T20:53:19.133 INFO:tasks.rados.rados.0.smithi093.stdout:1756: writing smithi093153278-385 from 635628 to 1219198 tid 1 2023-08-27T20:53:19.139 INFO:tasks.rados.rados.0.smithi093.stdout:1756: writing smithi093153278-385 from 1671967 to 2235628 tid 2 2023-08-27T20:53:19.139 INFO:tasks.rados.rados.0.smithi093.stdout:1749: done (7 left) 2023-08-27T20:53:19.139 INFO:tasks.rados.rados.0.smithi093.stdout:1750: done (6 left) 2023-08-27T20:53:19.139 INFO:tasks.rados.rados.0.smithi093.stdout:1751: done (5 left) 2023-08-27T20:53:19.139 INFO:tasks.rados.rados.0.smithi093.stdout:1752: done (4 left) 2023-08-27T20:53:19.139 INFO:tasks.rados.rados.0.smithi093.stdout:1753: done (3 left) 2023-08-27T20:53:19.139 INFO:tasks.rados.rados.0.smithi093.stdout:1754: done (2 left) 2023-08-27T20:53:19.140 INFO:tasks.rados.rados.0.smithi093.stdout:1755: done (1 left) 2023-08-27T20:53:19.140 INFO:tasks.rados.rados.0.smithi093.stdout:1757: write oid 45 current snap is 0 2023-08-27T20:53:19.140 INFO:tasks.rados.rados.0.smithi093.stdout:1756: finishing write tid 1 to smithi093153278-385 2023-08-27T20:53:19.140 INFO:tasks.rados.rados.0.smithi093.stdout:1757: seq_num 1205 ranges {652435=658819,1730449=521986} 2023-08-27T20:53:19.145 INFO:tasks.rados.rados.0.smithi093.stdout:1757: writing smithi093153278-45 from 652435 to 1311254 tid 1 2023-08-27T20:53:19.151 INFO:tasks.rados.rados.0.smithi093.stdout:1757: writing smithi093153278-45 from 1730449 to 2252435 tid 2 2023-08-27T20:53:19.151 INFO:tasks.rados.rados.0.smithi093.stdout:1758: read oid 42 snap -1 2023-08-27T20:53:19.151 INFO:tasks.rados.rados.0.smithi093.stdout:1758: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:53:19.151 INFO:tasks.rados.rados.0.smithi093.stdout:1759: read oid 63 snap -1 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1759: expect deleted 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1760: write oid 115 current snap is 0 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1756: finishing write tid 2 to smithi093153278-385 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1756: finishing write tid 3 to smithi093153278-385 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1756: finishing write tid 4 to smithi093153278-385 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 385 v 265 (ObjNum 1204 snap 0 seq_num 1204) dirty exists 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1756: left oid 385 (ObjNum 1204 snap 0 seq_num 1204) 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1757: finishing write tid 1 to smithi093153278-45 2023-08-27T20:53:19.152 INFO:tasks.rados.rados.0.smithi093.stdout:1760: seq_num 1206 ranges {669242=734068,2188931=80311} 2023-08-27T20:53:19.159 INFO:tasks.rados.rados.0.smithi093.stdout:1760: writing smithi093153278-115 from 669242 to 1403310 tid 1 2023-08-27T20:53:19.164 INFO:tasks.rados.rados.0.smithi093.stdout:1760: writing smithi093153278-115 from 2188931 to 2269242 tid 2 2023-08-27T20:53:19.164 INFO:tasks.rados.rados.0.smithi093.stdout:1756: done (4 left) 2023-08-27T20:53:19.164 INFO:tasks.rados.rados.0.smithi093.stdout:1761: write oid 406 current snap is 0 2023-08-27T20:53:19.164 INFO:tasks.rados.rados.0.smithi093.stdout:1757: finishing write tid 2 to smithi093153278-45 2023-08-27T20:53:19.164 INFO:tasks.rados.rados.0.smithi093.stdout:1761: seq_num 1207 ranges {686049=409317,1847413=438636} 2023-08-27T20:53:19.170 INFO:tasks.rados.rados.0.smithi093.stdout:1761: writing smithi093153278-406 from 686049 to 1095366 tid 1 2023-08-27T20:53:19.176 INFO:tasks.rados.rados.0.smithi093.stdout:1761: writing smithi093153278-406 from 1847413 to 2286049 tid 2 2023-08-27T20:53:19.177 INFO:tasks.rados.rados.0.smithi093.stdout:1762: delete oid 149 current snap is 0 2023-08-27T20:53:19.177 INFO:tasks.rados.rados.0.smithi093.stdout:1757: finishing write tid 3 to smithi093153278-45 2023-08-27T20:53:19.177 INFO:tasks.rados.rados.0.smithi093.stdout:1757: finishing write tid 4 to smithi093153278-45 2023-08-27T20:53:19.177 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 45 v 227 (ObjNum 1205 snap 0 seq_num 1205) dirty exists 2023-08-27T20:53:19.177 INFO:tasks.rados.rados.0.smithi093.stdout:1757: left oid 45 (ObjNum 1205 snap 0 seq_num 1205) 2023-08-27T20:53:19.177 INFO:tasks.rados.rados.0.smithi093.stdout:1758: expect (ObjNum 41 snap 0 seq_num 41) 2023-08-27T20:53:19.260 INFO:tasks.rados.rados.0.smithi093.stdout:1760: finishing write tid 1 to smithi093153278-115 2023-08-27T20:53:19.260 INFO:tasks.rados.rados.0.smithi093.stdout:1760: finishing write tid 2 to smithi093153278-115 2023-08-27T20:53:19.260 INFO:tasks.rados.rados.0.smithi093.stdout:1760: finishing write tid 3 to smithi093153278-115 2023-08-27T20:53:19.260 INFO:tasks.rados.rados.0.smithi093.stdout:1760: finishing write tid 4 to smithi093153278-115 2023-08-27T20:53:19.260 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 115 v 225 (ObjNum 1206 snap 0 seq_num 1206) dirty exists 2023-08-27T20:53:19.260 INFO:tasks.rados.rados.0.smithi093.stdout:1760: left oid 115 (ObjNum 1206 snap 0 seq_num 1206) 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1761: finishing write tid 1 to smithi093153278-406 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1761: finishing write tid 2 to smithi093153278-406 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1761: finishing write tid 3 to smithi093153278-406 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1761: finishing write tid 4 to smithi093153278-406 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 406 v 222 (ObjNum 1207 snap 0 seq_num 1207) dirty exists 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1761: left oid 406 (ObjNum 1207 snap 0 seq_num 1207) 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1757: done (5 left) 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1758: done (4 left) 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1759: done (3 left) 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1760: done (2 left) 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1761: done (1 left) 2023-08-27T20:53:19.261 INFO:tasks.rados.rados.0.smithi093.stdout:1762: done (0 left) 2023-08-27T20:53:19.262 INFO:tasks.rados.rados.0.smithi093.stdout:1763: read oid 142 snap -1 2023-08-27T20:53:19.262 INFO:tasks.rados.rados.0.smithi093.stdout:1763: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:19.262 INFO:tasks.rados.rados.0.smithi093.stdout:1764: write oid 23 current snap is 0 2023-08-27T20:53:19.262 INFO:tasks.rados.rados.0.smithi093.stdout:1764: seq_num 1209 ranges {719663=676168,1964377=355286} 2023-08-27T20:53:19.267 INFO:tasks.rados.rados.0.smithi093.stdout:1764: writing smithi093153278-23 from 719663 to 1395831 tid 1 2023-08-27T20:53:19.272 INFO:tasks.rados.rados.0.smithi093.stdout:1764: writing smithi093153278-23 from 1964377 to 2319663 tid 2 2023-08-27T20:53:19.272 INFO:tasks.rados.rados.0.smithi093.stdout:1765: write oid 87 current snap is 0 2023-08-27T20:53:19.272 INFO:tasks.rados.rados.0.smithi093.stdout:1765: seq_num 1210 ranges {736470=751417,2022859=313611} 2023-08-27T20:53:19.280 INFO:tasks.rados.rados.0.smithi093.stdout:1765: writing smithi093153278-87 from 736470 to 1487887 tid 1 2023-08-27T20:53:19.285 INFO:tasks.rados.rados.0.smithi093.stdout:1765: writing smithi093153278-87 from 2022859 to 2336470 tid 2 2023-08-27T20:53:19.285 INFO:tasks.rados.rados.0.smithi093.stdout:1766: delete oid 225 current snap is 0 2023-08-27T20:53:19.285 INFO:tasks.rados.rados.0.smithi093.stdout:1763: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:19.364 INFO:tasks.rados.rados.0.smithi093.stdout:1764: finishing write tid 1 to smithi093153278-23 2023-08-27T20:53:19.364 INFO:tasks.rados.rados.0.smithi093.stdout:1764: finishing write tid 2 to smithi093153278-23 2023-08-27T20:53:19.364 INFO:tasks.rados.rados.0.smithi093.stdout:1764: finishing write tid 3 to smithi093153278-23 2023-08-27T20:53:19.364 INFO:tasks.rados.rados.0.smithi093.stdout:1764: finishing write tid 4 to smithi093153278-23 2023-08-27T20:53:19.364 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 23 v 279 (ObjNum 1209 snap 0 seq_num 1209) dirty exists 2023-08-27T20:53:19.364 INFO:tasks.rados.rados.0.smithi093.stdout:1764: left oid 23 (ObjNum 1209 snap 0 seq_num 1209) 2023-08-27T20:53:19.364 INFO:tasks.rados.rados.0.smithi093.stdout:1765: finishing write tid 1 to smithi093153278-87 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1765: finishing write tid 2 to smithi093153278-87 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1765: finishing write tid 3 to smithi093153278-87 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1765: finishing write tid 4 to smithi093153278-87 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 230 (ObjNum 1210 snap 0 seq_num 1210) dirty exists 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1765: left oid 87 (ObjNum 1210 snap 0 seq_num 1210) 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1763: done (3 left) 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1764: done (2 left) 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1765: done (1 left) 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1766: done (0 left) 2023-08-27T20:53:19.365 INFO:tasks.rados.rados.0.smithi093.stdout:1767: delete oid 208 current snap is 0 2023-08-27T20:53:19.366 INFO:tasks.rados.rados.0.smithi093.stdout:1767: done (0 left) 2023-08-27T20:53:19.366 INFO:tasks.rados.rados.0.smithi093.stdout:1768: write oid 21 current snap is 0 2023-08-27T20:53:19.366 INFO:tasks.rados.rados.0.smithi093.stdout:1768: seq_num 1213 ranges {786891=577164,2081952=304939} 2023-08-27T20:53:19.372 INFO:tasks.rados.rados.0.smithi093.stdout:1768: writing smithi093153278-21 from 786891 to 1364055 tid 1 2023-08-27T20:53:19.379 INFO:tasks.rados.rados.0.smithi093.stdout:1768: writing smithi093153278-21 from 2081952 to 2386891 tid 2 2023-08-27T20:53:19.379 INFO:tasks.rados.rados.0.smithi093.stdout:1769: write oid 439 current snap is 0 2023-08-27T20:53:19.379 INFO:tasks.rados.rados.0.smithi093.stdout:1768: finishing write tid 1 to smithi093153278-21 2023-08-27T20:53:19.379 INFO:tasks.rados.rados.0.smithi093.stdout:1769: seq_num 1214 ranges {403698=652413,1740434=663264} 2023-08-27T20:53:19.384 INFO:tasks.rados.rados.0.smithi093.stdout:1769: writing smithi093153278-439 from 403698 to 1056111 tid 1 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1769: writing smithi093153278-439 from 1740434 to 2403698 tid 2 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1770: read oid 302 snap -1 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1770: expect (ObjNum 1195 snap 0 seq_num 1195) 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1771: read oid 240 snap -1 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1771: expect (ObjNum 557 snap 0 seq_num 557) 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1772: read oid 250 snap -1 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1772: expect (ObjNum 1182 snap 0 seq_num 1182) 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1773: read oid 311 snap -1 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1773: expect (ObjNum 310 snap 0 seq_num 310) 2023-08-27T20:53:19.393 INFO:tasks.rados.rados.0.smithi093.stdout:1774: read oid 221 snap -1 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1774: expect (ObjNum 1121 snap 0 seq_num 1121) 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1775: write oid 421 current snap is 0 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1768: finishing write tid 2 to smithi093153278-21 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1768: finishing write tid 3 to smithi093153278-21 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1768: finishing write tid 4 to smithi093153278-21 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 21 v 269 (ObjNum 1213 snap 0 seq_num 1213) dirty exists 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1768: left oid 21 (ObjNum 1213 snap 0 seq_num 1213) 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1769: finishing write tid 1 to smithi093153278-439 2023-08-27T20:53:19.394 INFO:tasks.rados.rados.0.smithi093.stdout:1775: seq_num 1215 ranges {420505=727662,1798916=553091} 2023-08-27T20:53:19.399 INFO:tasks.rados.rados.0.smithi093.stdout:1775: writing smithi093153278-421 from 420505 to 1148167 tid 1 2023-08-27T20:53:19.406 INFO:tasks.rados.rados.0.smithi093.stdout:1775: writing smithi093153278-421 from 1798916 to 2352007 tid 2 2023-08-27T20:53:19.407 INFO:tasks.rados.rados.0.smithi093.stdout:1768: done (7 left) 2023-08-27T20:53:19.407 INFO:tasks.rados.rados.0.smithi093.stdout:1776: write oid 274 current snap is 0 2023-08-27T20:53:19.407 INFO:tasks.rados.rados.0.smithi093.stdout:1776: seq_num 1216 ranges {437312=402911,1341045=696749} 2023-08-27T20:53:19.411 INFO:tasks.rados.rados.0.smithi093.stdout:1776: writing smithi093153278-274 from 437312 to 840223 tid 1 2023-08-27T20:53:19.418 INFO:tasks.rados.rados.0.smithi093.stdout:1776: writing smithi093153278-274 from 1341045 to 2037794 tid 2 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1777: read oid 83 snap -1 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1777: expect (ObjNum 82 snap 0 seq_num 82) 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1778: read oid 218 snap -1 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1778: expect (ObjNum 926 snap 0 seq_num 926) 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1779: read oid 15 snap -1 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1779: expect (ObjNum 1048 snap 0 seq_num 1048) 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1780: read oid 142 snap -1 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1780: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1781: write oid 20 current snap is 0 2023-08-27T20:53:19.419 INFO:tasks.rados.rados.0.smithi093.stdout:1781: seq_num 1217 ranges {454119=594513,1515880=556760} 2023-08-27T20:53:19.424 INFO:tasks.rados.rados.0.smithi093.stdout:1781: writing smithi093153278-20 from 454119 to 1048632 tid 1 2023-08-27T20:53:19.430 INFO:tasks.rados.rados.0.smithi093.stdout:1781: writing smithi093153278-20 from 1515880 to 2072640 tid 2 2023-08-27T20:53:19.430 INFO:tasks.rados.rados.0.smithi093.stdout:1782: write oid 139 current snap is 0 2023-08-27T20:53:19.431 INFO:tasks.rados.rados.0.smithi093.stdout:1770: expect (ObjNum 1195 snap 0 seq_num 1195) 2023-08-27T20:53:19.518 INFO:tasks.rados.rados.0.smithi093.stdout:1773: expect (ObjNum 310 snap 0 seq_num 310) 2023-08-27T20:53:19.620 INFO:tasks.rados.rados.0.smithi093.stdout:1769: finishing write tid 2 to smithi093153278-439 2023-08-27T20:53:19.620 INFO:tasks.rados.rados.0.smithi093.stdout:1782: seq_num 1218 ranges {470926=669762,1574362=700418} 2023-08-27T20:53:19.626 INFO:tasks.rados.rados.0.smithi093.stdout:1782: writing smithi093153278-139 from 470926 to 1140688 tid 1 2023-08-27T20:53:19.633 INFO:tasks.rados.rados.0.smithi093.stdout:1782: writing smithi093153278-139 from 1574362 to 2274780 tid 2 2023-08-27T20:53:19.633 INFO:tasks.rados.rados.0.smithi093.stdout:1770: done (13 left) 2023-08-27T20:53:19.633 INFO:tasks.rados.rados.0.smithi093.stdout:1773: done (12 left) 2023-08-27T20:53:19.634 INFO:tasks.rados.rados.0.smithi093.stdout:1783: write oid 123 current snap is 0 2023-08-27T20:53:19.634 INFO:tasks.rados.rados.0.smithi093.stdout:1769: finishing write tid 3 to smithi093153278-439 2023-08-27T20:53:19.634 INFO:tasks.rados.rados.0.smithi093.stdout:1769: finishing write tid 4 to smithi093153278-439 2023-08-27T20:53:19.634 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 439 v 245 (ObjNum 1214 snap 0 seq_num 1214) dirty exists 2023-08-27T20:53:19.634 INFO:tasks.rados.rados.0.smithi093.stdout:1769: left oid 439 (ObjNum 1214 snap 0 seq_num 1214) 2023-08-27T20:53:19.634 INFO:tasks.rados.rados.0.smithi093.stdout:1771: expect (ObjNum 557 snap 0 seq_num 557) 2023-08-27T20:53:19.793 INFO:tasks.rados.rados.0.smithi093.stdout:1775: finishing write tid 1 to smithi093153278-421 2023-08-27T20:53:19.793 INFO:tasks.rados.rados.0.smithi093.stdout:1772: expect (ObjNum 1182 snap 0 seq_num 1182) 2023-08-27T20:53:20.001 INFO:tasks.rados.rados.0.smithi093.stdout:1774: expect (ObjNum 1121 snap 0 seq_num 1121) 2023-08-27T20:53:20.122 INFO:tasks.rados.rados.0.smithi093.stdout:1775: finishing write tid 2 to smithi093153278-421 2023-08-27T20:53:20.122 INFO:tasks.rados.rados.0.smithi093.stdout:1775: finishing write tid 3 to smithi093153278-421 2023-08-27T20:53:20.122 INFO:tasks.rados.rados.0.smithi093.stdout:1775: finishing write tid 4 to smithi093153278-421 2023-08-27T20:53:20.122 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 421 v 272 (ObjNum 1215 snap 0 seq_num 1215) dirty exists 2023-08-27T20:53:20.122 INFO:tasks.rados.rados.0.smithi093.stdout:1775: left oid 421 (ObjNum 1215 snap 0 seq_num 1215) 2023-08-27T20:53:20.122 INFO:tasks.rados.rados.0.smithi093.stdout:1776: finishing write tid 1 to smithi093153278-274 2023-08-27T20:53:20.123 INFO:tasks.rados.rados.0.smithi093.stdout:1779: expect (ObjNum 1048 snap 0 seq_num 1048) 2023-08-27T20:53:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2905919123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2905919123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2905919123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:20.271 INFO:tasks.rados.rados.0.smithi093.stdout:1780: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:20.346 INFO:tasks.rados.rados.0.smithi093.stdout:1776: finishing write tid 2 to smithi093153278-274 2023-08-27T20:53:20.347 INFO:tasks.rados.rados.0.smithi093.stdout:1781: finishing write tid 1 to smithi093153278-20 2023-08-27T20:53:20.347 INFO:tasks.rados.rados.0.smithi093.stdout:1776: finishing write tid 3 to smithi093153278-274 2023-08-27T20:53:20.347 INFO:tasks.rados.rados.0.smithi093.stdout:1776: finishing write tid 4 to smithi093153278-274 2023-08-27T20:53:20.347 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 274 v 279 (ObjNum 1216 snap 0 seq_num 1216) dirty exists 2023-08-27T20:53:20.347 INFO:tasks.rados.rados.0.smithi093.stdout:1776: left oid 274 (ObjNum 1216 snap 0 seq_num 1216) 2023-08-27T20:53:20.347 INFO:tasks.rados.rados.0.smithi093.stdout:1777: expect (ObjNum 82 snap 0 seq_num 82) 2023-08-27T20:53:20.450 INFO:tasks.rados.rados.0.smithi093.stdout:1781: finishing write tid 2 to smithi093153278-20 2023-08-27T20:53:20.450 INFO:tasks.rados.rados.0.smithi093.stdout:1781: finishing write tid 3 to smithi093153278-20 2023-08-27T20:53:20.450 INFO:tasks.rados.rados.0.smithi093.stdout:1783: seq_num 1219 ranges {487733=745011,1632844=560429} 2023-08-27T20:53:20.456 INFO:tasks.rados.rados.0.smithi093.stdout:1783: writing smithi093153278-123 from 487733 to 1232744 tid 1 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1783: writing smithi093153278-123 from 1632844 to 2193273 tid 2 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1769: done (12 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1771: done (11 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1772: done (10 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1774: done (9 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1775: done (8 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1776: done (7 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1777: done (6 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1779: done (5 left) 2023-08-27T20:53:20.463 INFO:tasks.rados.rados.0.smithi093.stdout:1780: done (4 left) 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1784: read oid 89 snap -1 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1784: expect (ObjNum 631 snap 0 seq_num 631) 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1785: read oid 395 snap -1 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1785: expect (ObjNum 1090 snap 0 seq_num 1090) 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1786: read oid 496 snap -1 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1786: expect (ObjNum 1194 snap 0 seq_num 1194) 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1787: write oid 215 current snap is 0 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1781: finishing write tid 4 to smithi093153278-20 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 250 (ObjNum 1217 snap 0 seq_num 1217) dirty exists 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1781: left oid 20 (ObjNum 1217 snap 0 seq_num 1217) 2023-08-27T20:53:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:1778: expect (ObjNum 926 snap 0 seq_num 926) 2023-08-27T20:53:20.632 INFO:tasks.rados.rados.0.smithi093.stdout:1782: finishing write tid 1 to smithi093153278-139 2023-08-27T20:53:20.632 INFO:tasks.rados.rados.0.smithi093.stdout:1782: finishing write tid 2 to smithi093153278-139 2023-08-27T20:53:20.632 INFO:tasks.rados.rados.0.smithi093.stdout:1782: finishing write tid 3 to smithi093153278-139 2023-08-27T20:53:20.632 INFO:tasks.rados.rados.0.smithi093.stdout:1782: finishing write tid 4 to smithi093153278-139 2023-08-27T20:53:20.632 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 139 v 216 (ObjNum 1218 snap 0 seq_num 1218) dirty exists 2023-08-27T20:53:20.632 INFO:tasks.rados.rados.0.smithi093.stdout:1782: left oid 139 (ObjNum 1218 snap 0 seq_num 1218) 2023-08-27T20:53:20.633 INFO:tasks.rados.rados.0.smithi093.stdout:1787: seq_num 1220 ranges {504540=420260,1574973=704087} 2023-08-27T20:53:20.636 INFO:tasks.rados.rados.0.smithi093.stdout:1787: writing smithi093153278-215 from 504540 to 924800 tid 1 2023-08-27T20:53:20.644 INFO:tasks.rados.rados.0.smithi093.stdout:1787: writing smithi093153278-215 from 1574973 to 2279060 tid 2 2023-08-27T20:53:20.644 INFO:tasks.rados.rados.0.smithi093.stdout:1778: done (7 left) 2023-08-27T20:53:20.645 INFO:tasks.rados.rados.0.smithi093.stdout:1781: done (6 left) 2023-08-27T20:53:20.645 INFO:tasks.rados.rados.0.smithi093.stdout:1782: done (5 left) 2023-08-27T20:53:20.645 INFO:tasks.rados.rados.0.smithi093.stdout:1788: write oid 444 current snap is 0 2023-08-27T20:53:20.645 INFO:tasks.rados.rados.0.smithi093.stdout:1783: finishing write tid 1 to smithi093153278-123 2023-08-27T20:53:20.645 INFO:tasks.rados.rados.0.smithi093.stdout:1788: seq_num 1221 ranges {521347=495509,1633455=564098} 2023-08-27T20:53:20.650 INFO:tasks.rados.rados.0.smithi093.stdout:1788: writing smithi093153278-444 from 521347 to 1016856 tid 1 2023-08-27T20:53:20.657 INFO:tasks.rados.rados.0.smithi093.stdout:1788: writing smithi093153278-444 from 1633455 to 2197553 tid 2 2023-08-27T20:53:20.657 INFO:tasks.rados.rados.0.smithi093.stdout:1789: read oid 492 snap -1 2023-08-27T20:53:20.657 INFO:tasks.rados.rados.0.smithi093.stdout:1789: expect (ObjNum 904 snap 0 seq_num 904) 2023-08-27T20:53:20.657 INFO:tasks.rados.rados.0.smithi093.stdout:1790: read oid 67 snap -1 2023-08-27T20:53:20.657 INFO:tasks.rados.rados.0.smithi093.stdout:1790: expect (ObjNum 781 snap 0 seq_num 781) 2023-08-27T20:53:20.657 INFO:tasks.rados.rados.0.smithi093.stdout:1791: write oid 497 current snap is 0 2023-08-27T20:53:20.657 INFO:tasks.rados.rados.0.smithi093.stdout:1785: expect (ObjNum 1090 snap 0 seq_num 1090) 2023-08-27T20:53:20.810 INFO:tasks.rados.rados.0.smithi093.stdout:1783: finishing write tid 2 to smithi093153278-123 2023-08-27T20:53:20.810 INFO:tasks.rados.rados.0.smithi093.stdout:1783: finishing write tid 3 to smithi093153278-123 2023-08-27T20:53:20.810 INFO:tasks.rados.rados.0.smithi093.stdout:1783: finishing write tid 4 to smithi093153278-123 2023-08-27T20:53:20.810 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 123 v 219 (ObjNum 1219 snap 0 seq_num 1219) dirty exists 2023-08-27T20:53:20.810 INFO:tasks.rados.rados.0.smithi093.stdout:1783: left oid 123 (ObjNum 1219 snap 0 seq_num 1219) 2023-08-27T20:53:20.810 INFO:tasks.rados.rados.0.smithi093.stdout:1786: expect (ObjNum 1194 snap 0 seq_num 1194) 2023-08-27T20:53:20.896 INFO:tasks.rados.rados.0.smithi093.stdout:1784: expect (ObjNum 631 snap 0 seq_num 631) 2023-08-27T20:53:20.975 INFO:tasks.rados.rados.0.smithi093.stdout:1787: finishing write tid 1 to smithi093153278-215 2023-08-27T20:53:20.975 INFO:tasks.rados.rados.0.smithi093.stdout:1787: finishing write tid 2 to smithi093153278-215 2023-08-27T20:53:20.975 INFO:tasks.rados.rados.0.smithi093.stdout:1787: finishing write tid 3 to smithi093153278-215 2023-08-27T20:53:20.975 INFO:tasks.rados.rados.0.smithi093.stdout:1791: seq_num 1222 ranges {538154=570758,1691937=707756} 2023-08-27T20:53:20.981 INFO:tasks.rados.rados.0.smithi093.stdout:1791: writing smithi093153278-497 from 538154 to 1108912 tid 1 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:1791: writing smithi093153278-497 from 1691937 to 2399693 tid 2 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:1783: done (8 left) 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:1784: done (7 left) 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:1785: done (6 left) 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:1786: done (5 left) 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:1792: write oid 276 current snap is 0 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:1787: finishing write tid 4 to smithi093153278-215 2023-08-27T20:53:20.988 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 215 v 248 (ObjNum 1220 snap 0 seq_num 1220) dirty exists 2023-08-27T20:53:20.989 INFO:tasks.rados.rados.0.smithi093.stdout:1787: left oid 215 (ObjNum 1220 snap 0 seq_num 1220) 2023-08-27T20:53:20.989 INFO:tasks.rados.rados.0.smithi093.stdout:1788: finishing write tid 1 to smithi093153278-444 2023-08-27T20:53:20.989 INFO:tasks.rados.rados.0.smithi093.stdout:1792: seq_num 1223 ranges {554961=646007,1750419=451414} 2023-08-27T20:53:20.994 INFO:tasks.rados.rados.0.smithi093.stdout:1792: writing smithi093153278-276 from 554961 to 1200968 tid 1 2023-08-27T20:53:20.999 INFO:tasks.rados.rados.0.smithi093.stdout:1792: writing smithi093153278-276 from 1750419 to 2201833 tid 2 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1787: done (5 left) 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1793: read oid 353 snap -1 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1793: expect deleted 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1794: delete oid 266 current snap is 0 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1788: finishing write tid 2 to smithi093153278-444 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1788: finishing write tid 3 to smithi093153278-444 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1788: finishing write tid 4 to smithi093153278-444 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 444 v 282 (ObjNum 1221 snap 0 seq_num 1221) dirty exists 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1788: left oid 444 (ObjNum 1221 snap 0 seq_num 1221) 2023-08-27T20:53:21.000 INFO:tasks.rados.rados.0.smithi093.stdout:1790: expect (ObjNum 781 snap 0 seq_num 781) 2023-08-27T20:53:21.151 INFO:tasks.rados.rados.0.smithi093.stdout:1789: expect (ObjNum 904 snap 0 seq_num 904) 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:1791: finishing write tid 1 to smithi093153278-497 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:1791: finishing write tid 2 to smithi093153278-497 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:1791: finishing write tid 3 to smithi093153278-497 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:1791: finishing write tid 4 to smithi093153278-497 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 497 v 237 (ObjNum 1222 snap 0 seq_num 1222) dirty exists 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:1791: left oid 497 (ObjNum 1222 snap 0 seq_num 1222) 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:1792: finishing write tid 1 to smithi093153278-276 2023-08-27T20:53:21.350 INFO:tasks.rados.rados.0.smithi093.stdout:1788: done (6 left) 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1789: done (5 left) 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1790: done (4 left) 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1791: done (3 left) 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1794: done (2 left) 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1795: read oid 431 snap -1 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1795: expect (ObjNum 430 snap 0 seq_num 430) 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1796: write oid 426 current snap is 0 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1792: finishing write tid 2 to smithi093153278-276 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1792: finishing write tid 3 to smithi093153278-276 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:1792: finishing write tid 4 to smithi093153278-276 2023-08-27T20:53:21.351 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 276 v 240 (ObjNum 1223 snap 0 seq_num 1223) dirty exists 2023-08-27T20:53:21.352 INFO:tasks.rados.rados.0.smithi093.stdout:1792: left oid 276 (ObjNum 1223 snap 0 seq_num 1223) 2023-08-27T20:53:21.352 INFO:tasks.rados.rados.0.smithi093.stdout:1796: seq_num 1225 ranges {588575=512858,1867383=455083} 2023-08-27T20:53:21.355 INFO:tasks.rados.rados.0.smithi093.stdout:1796: writing smithi093153278-426 from 588575 to 1101433 tid 1 2023-08-27T20:53:21.363 INFO:tasks.rados.rados.0.smithi093.stdout:1796: writing smithi093153278-426 from 1867383 to 2322466 tid 2 2023-08-27T20:53:21.363 INFO:tasks.rados.rados.0.smithi093.stdout:1792: done (3 left) 2023-08-27T20:53:21.363 INFO:tasks.rados.rados.0.smithi093.stdout:1793: done (2 left) 2023-08-27T20:53:21.363 INFO:tasks.rados.rados.0.smithi093.stdout:1797: read oid 58 snap -1 2023-08-27T20:53:21.363 INFO:tasks.rados.rados.0.smithi093.stdout:1797: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:53:21.363 INFO:tasks.rados.rados.0.smithi093.stdout:1798: read oid 34 snap -1 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1798: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1799: read oid 164 snap -1 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1799: expect (ObjNum 645 snap 0 seq_num 645) 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1800: read oid 232 snap -1 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1800: expect (ObjNum 977 snap 0 seq_num 977) 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1801: read oid 133 snap -1 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1801: expect (ObjNum 1046 snap 0 seq_num 1046) 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1802: write oid 248 current snap is 0 2023-08-27T20:53:21.364 INFO:tasks.rados.rados.0.smithi093.stdout:1795: expect (ObjNum 430 snap 0 seq_num 430) 2023-08-27T20:53:21.461 INFO:tasks.rados.rados.0.smithi093.stdout:1796: finishing write tid 1 to smithi093153278-426 2023-08-27T20:53:21.461 INFO:tasks.rados.rados.0.smithi093.stdout:1797: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:53:21.549 INFO:tasks.rados.rados.0.smithi093.stdout:1801: expect (ObjNum 1046 snap 0 seq_num 1046) 2023-08-27T20:53:21.730 INFO:tasks.rados.rados.0.smithi093.stdout:1796: finishing write tid 2 to smithi093153278-426 2023-08-27T20:53:21.730 INFO:tasks.rados.rados.0.smithi093.stdout:1796: finishing write tid 3 to smithi093153278-426 2023-08-27T20:53:21.731 INFO:tasks.rados.rados.0.smithi093.stdout:1796: finishing write tid 4 to smithi093153278-426 2023-08-27T20:53:21.731 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 426 v 285 (ObjNum 1225 snap 0 seq_num 1225) dirty exists 2023-08-27T20:53:21.731 INFO:tasks.rados.rados.0.smithi093.stdout:1796: left oid 426 (ObjNum 1225 snap 0 seq_num 1225) 2023-08-27T20:53:21.731 INFO:tasks.rados.rados.0.smithi093.stdout:1800: expect (ObjNum 977 snap 0 seq_num 977) 2023-08-27T20:53:21.838 INFO:tasks.rados.rados.0.smithi093.stdout:1798: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:21.922 INFO:tasks.rados.rados.0.smithi093.stdout:1799: expect (ObjNum 645 snap 0 seq_num 645) 2023-08-27T20:53:22.054 INFO:tasks.rados.rados.0.smithi093.stdout:1802: seq_num 1226 ranges {605382=588107,1925865=679517} 2023-08-27T20:53:22.061 INFO:tasks.rados.rados.0.smithi093.stdout:1802: writing smithi093153278-248 from 605382 to 1193489 tid 1 2023-08-27T20:53:22.069 INFO:tasks.rados.rados.0.smithi093.stdout:1802: writing smithi093153278-248 from 1925865 to 2605382 tid 2 2023-08-27T20:53:22.069 INFO:tasks.rados.rados.0.smithi093.stdout:1795: done (7 left) 2023-08-27T20:53:22.069 INFO:tasks.rados.rados.0.smithi093.stdout:1796: done (6 left) 2023-08-27T20:53:22.069 INFO:tasks.rados.rados.0.smithi093.stdout:1797: done (5 left) 2023-08-27T20:53:22.069 INFO:tasks.rados.rados.0.smithi093.stdout:1798: done (4 left) 2023-08-27T20:53:22.069 INFO:tasks.rados.rados.0.smithi093.stdout:1799: done (3 left) 2023-08-27T20:53:22.070 INFO:tasks.rados.rados.0.smithi093.stdout:1800: done (2 left) 2023-08-27T20:53:22.070 INFO:tasks.rados.rados.0.smithi093.stdout:1801: done (1 left) 2023-08-27T20:53:22.070 INFO:tasks.rados.rados.0.smithi093.stdout:1803: write oid 95 current snap is 0 2023-08-27T20:53:22.070 INFO:tasks.rados.rados.0.smithi093.stdout:1802: finishing write tid 1 to smithi093153278-248 2023-08-27T20:53:22.070 INFO:tasks.rados.rados.0.smithi093.stdout:1803: seq_num 1227 ranges {622189=663356,1984347=458752} 2023-08-27T20:53:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:1803: writing smithi093153278-95 from 622189 to 1285545 tid 1 2023-08-27T20:53:22.082 INFO:tasks.rados.rados.0.smithi093.stdout:1803: writing smithi093153278-95 from 1984347 to 2443099 tid 2 2023-08-27T20:53:22.082 INFO:tasks.rados.rados.0.smithi093.stdout:1804: write oid 156 current snap is 0 2023-08-27T20:53:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:1804: seq_num 1228 ranges {638996=738605,1926476=712520} 2023-08-27T20:53:22.090 INFO:tasks.rados.rados.0.smithi093.stdout:1804: writing smithi093153278-156 from 638996 to 1377601 tid 1 2023-08-27T20:53:22.097 INFO:tasks.rados.rados.0.smithi093.stdout:1804: writing smithi093153278-156 from 1926476 to 2638996 tid 2 2023-08-27T20:53:22.097 INFO:tasks.rados.rados.0.smithi093.stdout:1805: write oid 36 current snap is 0 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1802: finishing write tid 2 to smithi093153278-248 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1802: finishing write tid 3 to smithi093153278-248 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1802: finishing write tid 4 to smithi093153278-248 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 248 v 182 (ObjNum 1226 snap 0 seq_num 1226) dirty exists 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1802: left oid 248 (ObjNum 1226 snap 0 seq_num 1226) 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1803: finishing write tid 1 to smithi093153278-95 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1803: finishing write tid 2 to smithi093153278-95 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1803: finishing write tid 3 to smithi093153278-95 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1803: finishing write tid 4 to smithi093153278-95 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 95 v 268 (ObjNum 1227 snap 0 seq_num 1227) dirty exists 2023-08-27T20:53:22.098 INFO:tasks.rados.rados.0.smithi093.stdout:1803: left oid 95 (ObjNum 1227 snap 0 seq_num 1227) 2023-08-27T20:53:22.099 INFO:tasks.rados.rados.0.smithi093.stdout:1804: finishing write tid 1 to smithi093153278-156 2023-08-27T20:53:22.099 INFO:tasks.rados.rados.0.smithi093.stdout:1805: seq_num 1229 ranges {655803=413854,1584958=462421} 2023-08-27T20:53:22.103 INFO:tasks.rados.rados.0.smithi093.stdout:1805: writing smithi093153278-36 from 655803 to 1069657 tid 1 2023-08-27T20:53:22.108 INFO:tasks.rados.rados.0.smithi093.stdout:1805: writing smithi093153278-36 from 1584958 to 2047379 tid 2 2023-08-27T20:53:22.108 INFO:tasks.rados.rados.0.smithi093.stdout:1802: done (3 left) 2023-08-27T20:53:22.109 INFO:tasks.rados.rados.0.smithi093.stdout:1803: done (2 left) 2023-08-27T20:53:22.109 INFO:tasks.rados.rados.0.smithi093.stdout:1806: write oid 179 current snap is 0 2023-08-27T20:53:22.109 INFO:tasks.rados.rados.0.smithi093.stdout:1804: finishing write tid 2 to smithi093153278-156 2023-08-27T20:53:22.109 INFO:tasks.rados.rados.0.smithi093.stdout:1804: finishing write tid 3 to smithi093153278-156 2023-08-27T20:53:22.109 INFO:tasks.rados.rados.0.smithi093.stdout:1806: seq_num 1230 ranges {672610=489103,1643440=722432} 2023-08-27T20:53:22.115 INFO:tasks.rados.rados.0.smithi093.stdout:1806: writing smithi093153278-179 from 672610 to 1161713 tid 1 2023-08-27T20:53:22.122 INFO:tasks.rados.rados.0.smithi093.stdout:1806: writing smithi093153278-179 from 1643440 to 2365872 tid 2 2023-08-27T20:53:22.122 INFO:tasks.rados.rados.0.smithi093.stdout:1807: write oid 450 current snap is 0 2023-08-27T20:53:22.122 INFO:tasks.rados.rados.0.smithi093.stdout:1804: finishing write tid 4 to smithi093153278-156 2023-08-27T20:53:22.122 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 156 v 243 (ObjNum 1228 snap 0 seq_num 1228) dirty exists 2023-08-27T20:53:22.123 INFO:tasks.rados.rados.0.smithi093.stdout:1804: left oid 156 (ObjNum 1228 snap 0 seq_num 1228) 2023-08-27T20:53:22.123 INFO:tasks.rados.rados.0.smithi093.stdout:1805: finishing write tid 1 to smithi093153278-36 2023-08-27T20:53:22.123 INFO:tasks.rados.rados.0.smithi093.stdout:1807: seq_num 1231 ranges {689417=564352,1701922=466090} 2023-08-27T20:53:22.129 INFO:tasks.rados.rados.0.smithi093.stdout:1807: writing smithi093153278-450 from 689417 to 1253769 tid 1 2023-08-27T20:53:22.134 INFO:tasks.rados.rados.0.smithi093.stdout:1807: writing smithi093153278-450 from 1701922 to 2168012 tid 2 2023-08-27T20:53:22.134 INFO:tasks.rados.rados.0.smithi093.stdout:1804: done (3 left) 2023-08-27T20:53:22.134 INFO:tasks.rados.rados.0.smithi093.stdout:1808: read oid 117 snap -1 2023-08-27T20:53:22.134 INFO:tasks.rados.rados.0.smithi093.stdout:1808: expect (ObjNum 839 snap 0 seq_num 839) 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:1809: write oid 110 current snap is 0 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:1805: finishing write tid 2 to smithi093153278-36 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:1805: finishing write tid 3 to smithi093153278-36 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:1805: finishing write tid 4 to smithi093153278-36 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 253 (ObjNum 1229 snap 0 seq_num 1229) dirty exists 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:1805: left oid 36 (ObjNum 1229 snap 0 seq_num 1229) 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:1806: finishing write tid 1 to smithi093153278-179 2023-08-27T20:53:22.135 INFO:tasks.rados.rados.0.smithi093.stdout:1809: seq_num 1232 ranges {706224=755954,2160404=545820} 2023-08-27T20:53:22.142 INFO:tasks.rados.rados.0.smithi093.stdout:1809: writing smithi093153278-110 from 706224 to 1462178 tid 1 2023-08-27T20:53:22.149 INFO:tasks.rados.rados.0.smithi093.stdout:1809: writing smithi093153278-110 from 2160404 to 2706224 tid 2 2023-08-27T20:53:22.149 INFO:tasks.rados.rados.0.smithi093.stdout:1805: done (4 left) 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:1810: write oid 473 current snap is 0 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:1806: finishing write tid 2 to smithi093153278-179 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:1806: finishing write tid 3 to smithi093153278-179 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:1806: finishing write tid 4 to smithi093153278-179 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 179 v 256 (ObjNum 1230 snap 0 seq_num 1230) dirty exists 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:1806: left oid 179 (ObjNum 1230 snap 0 seq_num 1230) 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:1807: finishing write tid 1 to smithi093153278-450 2023-08-27T20:53:22.150 INFO:tasks.rados.rados.0.smithi093.stdout:1810: seq_num 1233 ranges {723031=431203,1818886=469759} 2023-08-27T20:53:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:1810: writing smithi093153278-473 from 723031 to 1154234 tid 1 2023-08-27T20:53:22.162 INFO:tasks.rados.rados.0.smithi093.stdout:1810: writing smithi093153278-473 from 1818886 to 2288645 tid 2 2023-08-27T20:53:22.162 INFO:tasks.rados.rados.0.smithi093.stdout:1806: done (4 left) 2023-08-27T20:53:22.162 INFO:tasks.rados.rados.0.smithi093.stdout:1811: write oid 195 current snap is 0 2023-08-27T20:53:22.162 INFO:tasks.rados.rados.0.smithi093.stdout:1811: seq_num 1234 ranges {739838=506452,1877368=613417} 2023-08-27T20:53:22.168 INFO:tasks.rados.rados.0.smithi093.stdout:1811: writing smithi093153278-195 from 739838 to 1246290 tid 1 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1811: writing smithi093153278-195 from 1877368 to 2490785 tid 2 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1812: read oid 302 snap -1 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1812: expect (ObjNum 1195 snap 0 seq_num 1195) 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1813: write oid 120 current snap is 0 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1807: finishing write tid 2 to smithi093153278-450 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1807: finishing write tid 3 to smithi093153278-450 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1807: finishing write tid 4 to smithi093153278-450 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 450 v 228 (ObjNum 1231 snap 0 seq_num 1231) dirty exists 2023-08-27T20:53:22.176 INFO:tasks.rados.rados.0.smithi093.stdout:1807: left oid 450 (ObjNum 1231 snap 0 seq_num 1231) 2023-08-27T20:53:22.177 INFO:tasks.rados.rados.0.smithi093.stdout:1813: seq_num 1235 ranges {756645=581701,1935850=473428} 2023-08-27T20:53:22.182 INFO:tasks.rados.rados.0.smithi093.stdout:1813: writing smithi093153278-120 from 756645 to 1338346 tid 1 2023-08-27T20:53:22.189 INFO:tasks.rados.rados.0.smithi093.stdout:1813: writing smithi093153278-120 from 1935850 to 2409278 tid 2 2023-08-27T20:53:22.189 INFO:tasks.rados.rados.0.smithi093.stdout:1807: done (6 left) 2023-08-27T20:53:22.189 INFO:tasks.rados.rados.0.smithi093.stdout:1814: write oid 200 current snap is 0 2023-08-27T20:53:22.189 INFO:tasks.rados.rados.0.smithi093.stdout:1814: seq_num 1236 ranges {773452=656950,1877979=617086} 2023-08-27T20:53:22.196 INFO:tasks.rados.rados.0.smithi093.stdout:1814: writing smithi093153278-200 from 773452 to 1430402 tid 1 2023-08-27T20:53:22.202 INFO:tasks.rados.rados.0.smithi093.stdout:1814: writing smithi093153278-200 from 1877979 to 2495065 tid 2 2023-08-27T20:53:22.202 INFO:tasks.rados.rados.0.smithi093.stdout:1815: write oid 369 current snap is 0 2023-08-27T20:53:22.202 INFO:tasks.rados.rados.0.smithi093.stdout:1815: seq_num 1237 ranges {790259=732199,1936461=477097} 2023-08-27T20:53:22.210 INFO:tasks.rados.rados.0.smithi093.stdout:1815: writing smithi093153278-369 from 790259 to 1522458 tid 1 2023-08-27T20:53:22.215 INFO:tasks.rados.rados.0.smithi093.stdout:1815: writing smithi093153278-369 from 1936461 to 2413558 tid 2 2023-08-27T20:53:22.216 INFO:tasks.rados.rados.0.smithi093.stdout:1816: write oid 427 current snap is 0 2023-08-27T20:53:22.216 INFO:tasks.rados.rados.0.smithi093.stdout:1808: expect (ObjNum 839 snap 0 seq_num 839) 2023-08-27T20:53:22.304 INFO:tasks.rados.rados.0.smithi093.stdout:1809: finishing write tid 1 to smithi093153278-110 2023-08-27T20:53:22.304 INFO:tasks.rados.rados.0.smithi093.stdout:1809: finishing write tid 2 to smithi093153278-110 2023-08-27T20:53:22.304 INFO:tasks.rados.rados.0.smithi093.stdout:1816: seq_num 1238 ranges {407066=407448,1594943=620755} 2023-08-27T20:53:22.308 INFO:tasks.rados.rados.0.smithi093.stdout:1816: writing smithi093153278-427 from 407066 to 814514 tid 1 2023-08-27T20:53:22.315 INFO:tasks.rados.rados.0.smithi093.stdout:1816: writing smithi093153278-427 from 1594943 to 2215698 tid 2 2023-08-27T20:53:22.316 INFO:tasks.rados.rados.0.smithi093.stdout:1808: done (8 left) 2023-08-27T20:53:22.316 INFO:tasks.rados.rados.0.smithi093.stdout:1817: write oid 141 current snap is 0 2023-08-27T20:53:22.316 INFO:tasks.rados.rados.0.smithi093.stdout:1817: seq_num 1239 ranges {423873=482697,1653425=480766,2691441=132432} 2023-08-27T20:53:22.321 INFO:tasks.rados.rados.0.smithi093.stdout:1817: writing smithi093153278-141 from 423873 to 906570 tid 1 2023-08-27T20:53:22.327 INFO:tasks.rados.rados.0.smithi093.stdout:1817: writing smithi093153278-141 from 1653425 to 2134191 tid 2 2023-08-27T20:53:22.331 INFO:tasks.rados.rados.0.smithi093.stdout:1817: writing smithi093153278-141 from 2691441 to 2823873 tid 3 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1818: read oid 140 snap -1 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1818: expect (ObjNum 139 snap 0 seq_num 139) 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1819: read oid 260 snap -1 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1819: expect (ObjNum 1032 snap 0 seq_num 1032) 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1820: write oid 489 current snap is 0 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1809: finishing write tid 3 to smithi093153278-110 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1809: finishing write tid 4 to smithi093153278-110 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 110 v 287 (ObjNum 1232 snap 0 seq_num 1232) dirty exists 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1809: left oid 110 (ObjNum 1232 snap 0 seq_num 1232) 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1810: finishing write tid 1 to smithi093153278-473 2023-08-27T20:53:22.332 INFO:tasks.rados.rados.0.smithi093.stdout:1820: seq_num 1240 ranges {440680=674299,1711907=624424} 2023-08-27T20:53:22.338 INFO:tasks.rados.rados.0.smithi093.stdout:1820: writing smithi093153278-489 from 440680 to 1114979 tid 1 2023-08-27T20:53:22.345 INFO:tasks.rados.rados.0.smithi093.stdout:1820: writing smithi093153278-489 from 1711907 to 2336331 tid 2 2023-08-27T20:53:22.345 INFO:tasks.rados.rados.0.smithi093.stdout:1809: done (11 left) 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:1821: delete oid 388 current snap is 0 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:1810: finishing write tid 2 to smithi093153278-473 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:1810: finishing write tid 3 to smithi093153278-473 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:1810: finishing write tid 4 to smithi093153278-473 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 473 v 259 (ObjNum 1233 snap 0 seq_num 1233) dirty exists 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:1810: left oid 473 (ObjNum 1233 snap 0 seq_num 1233) 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:1811: finishing write tid 1 to smithi093153278-195 2023-08-27T20:53:22.346 INFO:tasks.rados.rados.0.smithi093.stdout:1812: expect (ObjNum 1195 snap 0 seq_num 1195) 2023-08-27T20:53:22.426 INFO:tasks.rados.rados.0.smithi093.stdout:1811: finishing write tid 2 to smithi093153278-195 2023-08-27T20:53:22.426 INFO:tasks.rados.rados.0.smithi093.stdout:1811: finishing write tid 3 to smithi093153278-195 2023-08-27T20:53:22.426 INFO:tasks.rados.rados.0.smithi093.stdout:1811: finishing write tid 4 to smithi093153278-195 2023-08-27T20:53:22.426 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 216 (ObjNum 1234 snap 0 seq_num 1234) dirty exists 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1811: left oid 195 (ObjNum 1234 snap 0 seq_num 1234) 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1810: done (11 left) 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1811: done (10 left) 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1812: done (9 left) 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1821: done (8 left) 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1822: write oid 312 current snap is 0 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1813: finishing write tid 1 to smithi093153278-120 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1813: finishing write tid 2 to smithi093153278-120 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1813: finishing write tid 3 to smithi093153278-120 2023-08-27T20:53:22.427 INFO:tasks.rados.rados.0.smithi093.stdout:1822: seq_num 1242 ranges {474294=424797,1428871=628093,2657357=216937} 2023-08-27T20:53:22.430 INFO:tasks.rados.rados.0.smithi093.stdout:1822: writing smithi093153278-312 from 474294 to 899091 tid 1 2023-08-27T20:53:22.437 INFO:tasks.rados.rados.0.smithi093.stdout:1822: writing smithi093153278-312 from 1428871 to 2056964 tid 2 2023-08-27T20:53:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2318104140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:22.441 INFO:tasks.rados.rados.0.smithi093.stdout:1822: writing smithi093153278-312 from 2657357 to 2874294 tid 3 2023-08-27T20:53:22.441 INFO:tasks.rados.rados.0.smithi093.stdout:1823: write oid 182 current snap is 0 2023-08-27T20:53:22.441 INFO:tasks.rados.rados.0.smithi093.stdout:1823: seq_num 1243 ranges {491101=500046,1487353=488104,2401133=489968} 2023-08-27T20:53:22.446 INFO:tasks.rados.rados.0.smithi093.stdout:1823: writing smithi093153278-182 from 491101 to 991147 tid 1 2023-08-27T20:53:22.452 INFO:tasks.rados.rados.0.smithi093.stdout:1823: writing smithi093153278-182 from 1487353 to 1975457 tid 2 2023-08-27T20:53:22.457 INFO:tasks.rados.rados.0.smithi093.stdout:1823: writing smithi093153278-182 from 2401133 to 2891101 tid 3 2023-08-27T20:53:22.457 INFO:tasks.rados.rados.0.smithi093.stdout:1824: write oid 40 current snap is 0 2023-08-27T20:53:22.457 INFO:tasks.rados.rados.0.smithi093.stdout:1813: finishing write tid 4 to smithi093153278-120 2023-08-27T20:53:22.457 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 120 v 282 (ObjNum 1235 snap 0 seq_num 1235) dirty exists 2023-08-27T20:53:22.457 INFO:tasks.rados.rados.0.smithi093.stdout:1813: left oid 120 (ObjNum 1235 snap 0 seq_num 1235) 2023-08-27T20:53:22.457 INFO:tasks.rados.rados.0.smithi093.stdout:1814: finishing write tid 1 to smithi093153278-200 2023-08-27T20:53:22.458 INFO:tasks.rados.rados.0.smithi093.stdout:1824: seq_num 1244 ranges {507908=575295,1829482=631762} 2023-08-27T20:53:22.463 INFO:tasks.rados.rados.0.smithi093.stdout:1824: writing smithi093153278-40 from 507908 to 1083203 tid 1 2023-08-27T20:53:22.471 INFO:tasks.rados.rados.0.smithi093.stdout:1824: writing smithi093153278-40 from 1829482 to 2461244 tid 2 2023-08-27T20:53:22.471 INFO:tasks.rados.rados.0.smithi093.stdout:1813: done (10 left) 2023-08-27T20:53:22.471 INFO:tasks.rados.rados.0.smithi093.stdout:1825: read oid 139 snap -1 2023-08-27T20:53:22.471 INFO:tasks.rados.rados.0.smithi093.stdout:1825: expect (ObjNum 1218 snap 0 seq_num 1218) 2023-08-27T20:53:22.471 INFO:tasks.rados.rados.0.smithi093.stdout:1826: read oid 222 snap -1 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1826: expect (ObjNum 991 snap 0 seq_num 991) 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1827: write oid 285 current snap is 0 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1814: finishing write tid 2 to smithi093153278-200 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1814: finishing write tid 3 to smithi093153278-200 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1814: finishing write tid 4 to smithi093153278-200 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 200 v 285 (ObjNum 1236 snap 0 seq_num 1236) dirty exists 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1814: left oid 200 (ObjNum 1236 snap 0 seq_num 1236) 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1815: finishing write tid 1 to smithi093153278-369 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1815: finishing write tid 2 to smithi093153278-369 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1815: finishing write tid 3 to smithi093153278-369 2023-08-27T20:53:22.472 INFO:tasks.rados.rados.0.smithi093.stdout:1827: seq_num 1245 ranges {524715=650544,1887964=491773} 2023-08-27T20:53:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:1827: writing smithi093153278-285 from 524715 to 1175259 tid 1 2023-08-27T20:53:22.484 INFO:tasks.rados.rados.0.smithi093.stdout:1827: writing smithi093153278-285 from 1887964 to 2379737 tid 2 2023-08-27T20:53:22.484 INFO:tasks.rados.rados.0.smithi093.stdout:1814: done (12 left) 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:1828: read oid 32 snap -1 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:1828: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:1829: write oid 482 current snap is 0 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:1815: finishing write tid 4 to smithi093153278-369 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 369 v 262 (ObjNum 1237 snap 0 seq_num 1237) dirty exists 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:1815: left oid 369 (ObjNum 1237 snap 0 seq_num 1237) 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:1816: finishing write tid 1 to smithi093153278-427 2023-08-27T20:53:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:1829: seq_num 1246 ranges {541522=725793,1946446=635431} 2023-08-27T20:53:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2318104140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2318104140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:22.491 INFO:tasks.rados.rados.0.smithi093.stdout:1829: writing smithi093153278-482 from 541522 to 1267315 tid 1 2023-08-27T20:53:22.499 INFO:tasks.rados.rados.0.smithi093.stdout:1829: writing smithi093153278-482 from 1946446 to 2581877 tid 2 2023-08-27T20:53:22.499 INFO:tasks.rados.rados.0.smithi093.stdout:1815: done (13 left) 2023-08-27T20:53:22.499 INFO:tasks.rados.rados.0.smithi093.stdout:1830: delete oid 354 current snap is 0 2023-08-27T20:53:22.499 INFO:tasks.rados.rados.0.smithi093.stdout:1816: finishing write tid 2 to smithi093153278-427 2023-08-27T20:53:22.499 INFO:tasks.rados.rados.0.smithi093.stdout:1816: finishing write tid 3 to smithi093153278-427 2023-08-27T20:53:22.499 INFO:tasks.rados.rados.0.smithi093.stdout:1816: finishing write tid 4 to smithi093153278-427 2023-08-27T20:53:22.499 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 427 v 233 (ObjNum 1238 snap 0 seq_num 1238) dirty exists 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1816: left oid 427 (ObjNum 1238 snap 0 seq_num 1238) 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1817: finishing write tid 1 to smithi093153278-141 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1817: finishing write tid 2 to smithi093153278-141 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1817: finishing write tid 3 to smithi093153278-141 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1817: finishing write tid 4 to smithi093153278-141 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1817: finishing write tid 5 to smithi093153278-141 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 141 v 223 (ObjNum 1239 snap 0 seq_num 1239) dirty exists 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1817: left oid 141 (ObjNum 1239 snap 0 seq_num 1239) 2023-08-27T20:53:22.500 INFO:tasks.rados.rados.0.smithi093.stdout:1818: expect (ObjNum 139 snap 0 seq_num 139) 2023-08-27T20:53:22.571 INFO:tasks.rados.rados.0.smithi093.stdout:1819: expect (ObjNum 1032 snap 0 seq_num 1032) 2023-08-27T20:53:22.707 INFO:tasks.rados.rados.0.smithi093.stdout:1820: finishing write tid 1 to smithi093153278-489 2023-08-27T20:53:22.707 INFO:tasks.rados.rados.0.smithi093.stdout:1820: finishing write tid 2 to smithi093153278-489 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1820: finishing write tid 3 to smithi093153278-489 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1820: finishing write tid 4 to smithi093153278-489 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 489 v 226 (ObjNum 1240 snap 0 seq_num 1240) dirty exists 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1820: left oid 489 (ObjNum 1240 snap 0 seq_num 1240) 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1822: finishing write tid 1 to smithi093153278-312 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1822: finishing write tid 2 to smithi093153278-312 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1822: finishing write tid 3 to smithi093153278-312 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1822: finishing write tid 4 to smithi093153278-312 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1822: finishing write tid 5 to smithi093153278-312 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 312 v 247 (ObjNum 1242 snap 0 seq_num 1242) dirty exists 2023-08-27T20:53:22.708 INFO:tasks.rados.rados.0.smithi093.stdout:1822: left oid 312 (ObjNum 1242 snap 0 seq_num 1242) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1823: finishing write tid 1 to smithi093153278-182 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1823: finishing write tid 2 to smithi093153278-182 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1823: finishing write tid 3 to smithi093153278-182 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1823: finishing write tid 4 to smithi093153278-182 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1816: done (13 left) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1817: done (12 left) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1818: done (11 left) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1819: done (10 left) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1820: done (9 left) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1822: done (8 left) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1830: done (7 left) 2023-08-27T20:53:22.709 INFO:tasks.rados.rados.0.smithi093.stdout:1831: write oid 151 current snap is 0 2023-08-27T20:53:22.710 INFO:tasks.rados.rados.0.smithi093.stdout:1823: finishing write tid 5 to smithi093153278-182 2023-08-27T20:53:22.710 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 182 v 289 (ObjNum 1243 snap 0 seq_num 1243) dirty exists 2023-08-27T20:53:22.710 INFO:tasks.rados.rados.0.smithi093.stdout:1823: left oid 182 (ObjNum 1243 snap 0 seq_num 1243) 2023-08-27T20:53:22.710 INFO:tasks.rados.rados.0.smithi093.stdout:1824: finishing write tid 1 to smithi093153278-40 2023-08-27T20:53:22.710 INFO:tasks.rados.rados.0.smithi093.stdout:1831: seq_num 1248 ranges {575136=592644,1663410=639100,2705542=269594} 2023-08-27T20:53:22.713 INFO:tasks.rados.rados.0.smithi093.stdout:1831: writing smithi093153278-151 from 575136 to 1167780 tid 1 2023-08-27T20:53:22.719 INFO:tasks.rados.rados.0.smithi093.stdout:1831: writing smithi093153278-151 from 1663410 to 2302510 tid 2 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1831: writing smithi093153278-151 from 2705542 to 2975136 tid 3 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1823: done (7 left) 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1832: read oid 466 snap -1 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1832: expect (ObjNum 783 snap 0 seq_num 783) 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1833: read oid 345 snap -1 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1833: expect (ObjNum 989 snap 0 seq_num 989) 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1834: read oid 324 snap -1 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1834: expect (ObjNum 646 snap 0 seq_num 646) 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1835: write oid 103 current snap is 0 2023-08-27T20:53:22.724 INFO:tasks.rados.rados.0.smithi093.stdout:1826: expect (ObjNum 991 snap 0 seq_num 991) 2023-08-27T20:53:22.805 INFO:tasks.rados.rados.0.smithi093.stdout:1824: finishing write tid 2 to smithi093153278-40 2023-08-27T20:53:22.805 INFO:tasks.rados.rados.0.smithi093.stdout:1824: finishing write tid 3 to smithi093153278-40 2023-08-27T20:53:22.806 INFO:tasks.rados.rados.0.smithi093.stdout:1824: finishing write tid 4 to smithi093153278-40 2023-08-27T20:53:22.806 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 40 v 288 (ObjNum 1244 snap 0 seq_num 1244) dirty exists 2023-08-27T20:53:22.806 INFO:tasks.rados.rados.0.smithi093.stdout:1824: left oid 40 (ObjNum 1244 snap 0 seq_num 1244) 2023-08-27T20:53:22.806 INFO:tasks.rados.rados.0.smithi093.stdout:1825: expect (ObjNum 1218 snap 0 seq_num 1218) 2023-08-27T20:53:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:1827: finishing write tid 1 to smithi093153278-285 2023-08-27T20:53:22.912 INFO:tasks.rados.rados.0.smithi093.stdout:1828: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:53:23.085 INFO:tasks.rados.rados.0.smithi093.stdout:1827: finishing write tid 2 to smithi093153278-285 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:1827: finishing write tid 3 to smithi093153278-285 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:1827: finishing write tid 4 to smithi093153278-285 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 285 v 272 (ObjNum 1245 snap 0 seq_num 1245) dirty exists 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:1827: left oid 285 (ObjNum 1245 snap 0 seq_num 1245) 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:1829: finishing write tid 1 to smithi093153278-482 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:1829: finishing write tid 2 to smithi093153278-482 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:1829: finishing write tid 3 to smithi093153278-482 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:1829: finishing write tid 4 to smithi093153278-482 2023-08-27T20:53:23.086 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 482 v 219 (ObjNum 1246 snap 0 seq_num 1246) dirty exists 2023-08-27T20:53:23.087 INFO:tasks.rados.rados.0.smithi093.stdout:1829: left oid 482 (ObjNum 1246 snap 0 seq_num 1246) 2023-08-27T20:53:23.087 INFO:tasks.rados.rados.0.smithi093.stdout:1831: finishing write tid 1 to smithi093153278-151 2023-08-27T20:53:23.087 INFO:tasks.rados.rados.0.smithi093.stdout:1835: seq_num 1249 ranges {591943=667893,1721892=782758} 2023-08-27T20:53:23.091 INFO:tasks.rados.rados.0.smithi093.stdout:1835: writing smithi093153278-103 from 591943 to 1259836 tid 1 2023-08-27T20:53:23.099 INFO:tasks.rados.rados.0.smithi093.stdout:1835: writing smithi093153278-103 from 1721892 to 2504650 tid 2 2023-08-27T20:53:23.099 INFO:tasks.rados.rados.0.smithi093.stdout:1824: done (10 left) 2023-08-27T20:53:23.099 INFO:tasks.rados.rados.0.smithi093.stdout:1825: done (9 left) 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1826: done (8 left) 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1827: done (7 left) 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1828: done (6 left) 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1829: done (5 left) 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1836: delete oid 292 current snap is 0 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1831: finishing write tid 2 to smithi093153278-151 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1831: finishing write tid 3 to smithi093153278-151 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1831: finishing write tid 4 to smithi093153278-151 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1831: finishing write tid 5 to smithi093153278-151 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 151 v 252 (ObjNum 1248 snap 0 seq_num 1248) dirty exists 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1831: left oid 151 (ObjNum 1248 snap 0 seq_num 1248) 2023-08-27T20:53:23.100 INFO:tasks.rados.rados.0.smithi093.stdout:1832: expect (ObjNum 783 snap 0 seq_num 783) 2023-08-27T20:53:23.262 INFO:tasks.rados.rados.0.smithi093.stdout:1834: expect (ObjNum 646 snap 0 seq_num 646) 2023-08-27T20:53:23.387 INFO:tasks.rados.rados.0.smithi093.stdout:1833: expect (ObjNum 989 snap 0 seq_num 989) 2023-08-27T20:53:23.488 INFO:tasks.rados.rados.0.smithi093.stdout:1835: finishing write tid 1 to smithi093153278-103 2023-08-27T20:53:23.488 INFO:tasks.rados.rados.0.smithi093.stdout:1835: finishing write tid 2 to smithi093153278-103 2023-08-27T20:53:23.488 INFO:tasks.rados.rados.0.smithi093.stdout:1835: finishing write tid 3 to smithi093153278-103 2023-08-27T20:53:23.488 INFO:tasks.rados.rados.0.smithi093.stdout:1835: finishing write tid 4 to smithi093153278-103 2023-08-27T20:53:23.489 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 103 v 231 (ObjNum 1249 snap 0 seq_num 1249) dirty exists 2023-08-27T20:53:23.489 INFO:tasks.rados.rados.0.smithi093.stdout:1835: left oid 103 (ObjNum 1249 snap 0 seq_num 1249) 2023-08-27T20:53:23.489 INFO:tasks.rados.rados.0.smithi093.stdout:1831: done (5 left) 2023-08-27T20:53:23.489 INFO:tasks.rados.rados.0.smithi093.stdout:1832: done (4 left) 2023-08-27T20:53:23.489 INFO:tasks.rados.rados.0.smithi093.stdout:1833: done (3 left) 2023-08-27T20:53:23.489 INFO:tasks.rados.rados.0.smithi093.stdout:1834: done (2 left) 2023-08-27T20:53:23.490 INFO:tasks.rados.rados.0.smithi093.stdout:1835: done (1 left) 2023-08-27T20:53:23.490 INFO:tasks.rados.rados.0.smithi093.stdout:1836: done (0 left) 2023-08-27T20:53:23.490 INFO:tasks.rados.rados.0.smithi093.stdout:1837: read oid 123 snap -1 2023-08-27T20:53:23.490 INFO:tasks.rados.rados.0.smithi093.stdout:1837: expect (ObjNum 1219 snap 0 seq_num 1219) 2023-08-27T20:53:23.490 INFO:tasks.rados.rados.0.smithi093.stdout:1838: write oid 349 current snap is 0 2023-08-27T20:53:23.490 INFO:tasks.rados.rados.0.smithi093.stdout:1838: seq_num 1251 ranges {625557=418391,1838856=786427} 2023-08-27T20:53:23.493 INFO:tasks.rados.rados.0.smithi093.stdout:1838: writing smithi093153278-349 from 625557 to 1043948 tid 1 2023-08-27T20:53:23.502 INFO:tasks.rados.rados.0.smithi093.stdout:1838: writing smithi093153278-349 from 1838856 to 2625283 tid 2 2023-08-27T20:53:23.502 INFO:tasks.rados.rados.0.smithi093.stdout:1839: write oid 169 current snap is 0 2023-08-27T20:53:23.502 INFO:tasks.rados.rados.0.smithi093.stdout:1839: seq_num 1252 ranges {642364=493640,1897338=646438} 2023-08-27T20:53:23.508 INFO:tasks.rados.rados.0.smithi093.stdout:1839: writing smithi093153278-169 from 642364 to 1136004 tid 1 2023-08-27T20:53:23.516 INFO:tasks.rados.rados.0.smithi093.stdout:1839: writing smithi093153278-169 from 1897338 to 2543776 tid 2 2023-08-27T20:53:23.516 INFO:tasks.rados.rados.0.smithi093.stdout:1840: write oid 143 current snap is 0 2023-08-27T20:53:23.516 INFO:tasks.rados.rados.0.smithi093.stdout:1837: expect (ObjNum 1219 snap 0 seq_num 1219) 2023-08-27T20:53:23.659 INFO:tasks.rados.rados.0.smithi093.stdout:1838: finishing write tid 1 to smithi093153278-349 2023-08-27T20:53:23.659 INFO:tasks.rados.rados.0.smithi093.stdout:1838: finishing write tid 2 to smithi093153278-349 2023-08-27T20:53:23.659 INFO:tasks.rados.rados.0.smithi093.stdout:1838: finishing write tid 3 to smithi093153278-349 2023-08-27T20:53:23.659 INFO:tasks.rados.rados.0.smithi093.stdout:1838: finishing write tid 4 to smithi093153278-349 2023-08-27T20:53:23.660 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 349 v 291 (ObjNum 1251 snap 0 seq_num 1251) dirty exists 2023-08-27T20:53:23.660 INFO:tasks.rados.rados.0.smithi093.stdout:1838: left oid 349 (ObjNum 1251 snap 0 seq_num 1251) 2023-08-27T20:53:23.660 INFO:tasks.rados.rados.0.smithi093.stdout:1839: finishing write tid 1 to smithi093153278-169 2023-08-27T20:53:23.660 INFO:tasks.rados.rados.0.smithi093.stdout:1840: seq_num 1253 ranges {659171=568889,1839467=790096} 2023-08-27T20:53:23.665 INFO:tasks.rados.rados.0.smithi093.stdout:1840: writing smithi093153278-143 from 659171 to 1228060 tid 1 2023-08-27T20:53:23.673 INFO:tasks.rados.rados.0.smithi093.stdout:1840: writing smithi093153278-143 from 1839467 to 2629563 tid 2 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1837: done (3 left) 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1838: done (2 left) 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1841: write oid 437 current snap is 0 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1839: finishing write tid 2 to smithi093153278-169 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1839: finishing write tid 3 to smithi093153278-169 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1839: finishing write tid 4 to smithi093153278-169 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 169 v 265 (ObjNum 1252 snap 0 seq_num 1252) dirty exists 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1839: left oid 169 (ObjNum 1252 snap 0 seq_num 1252) 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1840: finishing write tid 1 to smithi093153278-143 2023-08-27T20:53:23.674 INFO:tasks.rados.rados.0.smithi093.stdout:1841: seq_num 1254 ranges {675978=644138,1897949=650107} 2023-08-27T20:53:23.680 INFO:tasks.rados.rados.0.smithi093.stdout:1841: writing smithi093153278-437 from 675978 to 1320116 tid 1 2023-08-27T20:53:23.687 INFO:tasks.rados.rados.0.smithi093.stdout:1841: writing smithi093153278-437 from 1897949 to 2548056 tid 2 2023-08-27T20:53:23.687 INFO:tasks.rados.rados.0.smithi093.stdout:1839: done (2 left) 2023-08-27T20:53:23.688 INFO:tasks.rados.rados.0.smithi093.stdout:1842: write oid 449 current snap is 0 2023-08-27T20:53:23.688 INFO:tasks.rados.rados.0.smithi093.stdout:1840: finishing write tid 2 to smithi093153278-143 2023-08-27T20:53:23.688 INFO:tasks.rados.rados.0.smithi093.stdout:1840: finishing write tid 3 to smithi093153278-143 2023-08-27T20:53:23.688 INFO:tasks.rados.rados.0.smithi093.stdout:1840: finishing write tid 4 to smithi093153278-143 2023-08-27T20:53:23.688 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 143 v 292 (ObjNum 1253 snap 0 seq_num 1253) dirty exists 2023-08-27T20:53:23.688 INFO:tasks.rados.rados.0.smithi093.stdout:1840: left oid 143 (ObjNum 1253 snap 0 seq_num 1253) 2023-08-27T20:53:23.688 INFO:tasks.rados.rados.0.smithi093.stdout:1841: finishing write tid 1 to smithi093153278-437 2023-08-27T20:53:23.689 INFO:tasks.rados.rados.0.smithi093.stdout:1842: seq_num 1255 ranges {692785=435740,1672784=793765,2897503=195282} 2023-08-27T20:53:23.693 INFO:tasks.rados.rados.0.smithi093.stdout:1842: writing smithi093153278-449 from 692785 to 1128525 tid 1 2023-08-27T20:53:23.700 INFO:tasks.rados.rados.0.smithi093.stdout:1842: writing smithi093153278-449 from 1672784 to 2466549 tid 2 2023-08-27T20:53:23.704 INFO:tasks.rados.rados.0.smithi093.stdout:1842: writing smithi093153278-449 from 2897503 to 3092785 tid 3 2023-08-27T20:53:23.704 INFO:tasks.rados.rados.0.smithi093.stdout:1840: done (2 left) 2023-08-27T20:53:23.704 INFO:tasks.rados.rados.0.smithi093.stdout:1843: delete oid 408 current snap is 0 2023-08-27T20:53:23.704 INFO:tasks.rados.rados.0.smithi093.stdout:1841: finishing write tid 2 to smithi093153278-437 2023-08-27T20:53:23.704 INFO:tasks.rados.rados.0.smithi093.stdout:1841: finishing write tid 3 to smithi093153278-437 2023-08-27T20:53:23.705 INFO:tasks.rados.rados.0.smithi093.stdout:1841: finishing write tid 4 to smithi093153278-437 2023-08-27T20:53:23.705 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 437 v 276 (ObjNum 1254 snap 0 seq_num 1254) dirty exists 2023-08-27T20:53:23.705 INFO:tasks.rados.rados.0.smithi093.stdout:1841: left oid 437 (ObjNum 1254 snap 0 seq_num 1254) 2023-08-27T20:53:23.705 INFO:tasks.rados.rados.0.smithi093.stdout:1842: finishing write tid 1 to smithi093153278-449 2023-08-27T20:53:23.705 INFO:tasks.rados.rados.0.smithi093.stdout:1842: finishing write tid 2 to smithi093153278-449 2023-08-27T20:53:23.706 INFO:tasks.rados.rados.0.smithi093.stdout:1842: finishing write tid 3 to smithi093153278-449 2023-08-27T20:53:23.706 INFO:tasks.rados.rados.0.smithi093.stdout:1842: finishing write tid 4 to smithi093153278-449 2023-08-27T20:53:23.706 INFO:tasks.rados.rados.0.smithi093.stdout:1842: finishing write tid 5 to smithi093153278-449 2023-08-27T20:53:23.706 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 295 (ObjNum 1255 snap 0 seq_num 1255) dirty exists 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1842: left oid 449 (ObjNum 1255 snap 0 seq_num 1255) 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1841: done (2 left) 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1842: done (1 left) 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1843: done (0 left) 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1844: read oid 79 snap -1 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1844: expect (ObjNum 78 snap 0 seq_num 78) 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1845: read oid 284 snap -1 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1845: expect (ObjNum 1139 snap 0 seq_num 1139) 2023-08-27T20:53:23.707 INFO:tasks.rados.rados.0.smithi093.stdout:1846: delete oid 375 current snap is 0 2023-08-27T20:53:23.708 INFO:tasks.rados.rados.0.smithi093.stdout:1846: done (2 left) 2023-08-27T20:53:23.708 INFO:tasks.rados.rados.0.smithi093.stdout:1847: write oid 13 current snap is 0 2023-08-27T20:53:23.708 INFO:tasks.rados.rados.0.smithi093.stdout:1847: seq_num 1258 ranges {743206=661487,2131877=541092} 2023-08-27T20:53:23.715 INFO:tasks.rados.rados.0.smithi093.stdout:1847: writing smithi093153278-13 from 743206 to 1404693 tid 1 2023-08-27T20:53:23.722 INFO:tasks.rados.rados.0.smithi093.stdout:1847: writing smithi093153278-13 from 2131877 to 2672969 tid 2 2023-08-27T20:53:23.722 INFO:tasks.rados.rados.0.smithi093.stdout:1848: delete oid 147 current snap is 0 2023-08-27T20:53:23.722 INFO:tasks.rados.rados.0.smithi093.stdout:1844: expect (ObjNum 78 snap 0 seq_num 78) 2023-08-27T20:53:23.823 INFO:tasks.rados.rados.0.smithi093.stdout:1845: expect (ObjNum 1139 snap 0 seq_num 1139) 2023-08-27T20:53:23.919 INFO:tasks.rados.rados.0.smithi093.stdout:1847: finishing write tid 1 to smithi093153278-13 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1847: finishing write tid 2 to smithi093153278-13 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1847: finishing write tid 3 to smithi093153278-13 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1847: finishing write tid 4 to smithi093153278-13 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 13 v 222 (ObjNum 1258 snap 0 seq_num 1258) dirty exists 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1847: left oid 13 (ObjNum 1258 snap 0 seq_num 1258) 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1844: done (3 left) 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1845: done (2 left) 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1847: done (1 left) 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1848: done (0 left) 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1849: read oid 315 snap -1 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1849: expect (ObjNum 1119 snap 0 seq_num 1119) 2023-08-27T20:53:23.920 INFO:tasks.rados.rados.0.smithi093.stdout:1850: read oid 338 snap -1 2023-08-27T20:53:23.921 INFO:tasks.rados.rados.0.smithi093.stdout:1850: expect (ObjNum 337 snap 0 seq_num 337) 2023-08-27T20:53:23.921 INFO:tasks.rados.rados.0.smithi093.stdout:1851: delete oid 70 current snap is 0 2023-08-27T20:53:23.921 INFO:tasks.rados.rados.0.smithi093.stdout:1851: done (2 left) 2023-08-27T20:53:23.921 INFO:tasks.rados.rados.0.smithi093.stdout:1852: write oid 304 current snap is 0 2023-08-27T20:53:23.921 INFO:tasks.rados.rados.0.smithi093.stdout:1852: seq_num 1261 ranges {793627=487234,1790970=404772,2829335=364292} 2023-08-27T20:53:23.928 INFO:tasks.rados.rados.0.smithi093.stdout:1852: writing smithi093153278-304 from 793627 to 1280861 tid 1 2023-08-27T20:53:23.933 INFO:tasks.rados.rados.0.smithi093.stdout:1852: writing smithi093153278-304 from 1790970 to 2195742 tid 2 2023-08-27T20:53:23.939 INFO:tasks.rados.rados.0.smithi093.stdout:1852: writing smithi093153278-304 from 2829335 to 3193627 tid 3 2023-08-27T20:53:23.939 INFO:tasks.rados.rados.0.smithi093.stdout:1853: write oid 164 current snap is 0 2023-08-27T20:53:23.939 INFO:tasks.rados.rados.0.smithi093.stdout:1853: seq_num 1262 ranges {410434=678836,1565805=548430,2573111=637323} 2023-08-27T20:53:23.945 INFO:tasks.rados.rados.0.smithi093.stdout:1853: writing smithi093153278-164 from 410434 to 1089270 tid 1 2023-08-27T20:53:23.950 INFO:tasks.rados.rados.0.smithi093.stdout:1853: writing smithi093153278-164 from 1565805 to 2114235 tid 2 2023-08-27T20:53:23.957 INFO:tasks.rados.rados.0.smithi093.stdout:1853: writing smithi093153278-164 from 2573111 to 3210434 tid 3 2023-08-27T20:53:23.957 INFO:tasks.rados.rados.0.smithi093.stdout:1854: read oid 461 snap -1 2023-08-27T20:53:23.957 INFO:tasks.rados.rados.0.smithi093.stdout:1854: expect (ObjNum 704 snap 0 seq_num 704) 2023-08-27T20:53:23.957 INFO:tasks.rados.rados.0.smithi093.stdout:1855: read oid 275 snap -1 2023-08-27T20:53:23.957 INFO:tasks.rados.rados.0.smithi093.stdout:1855: expect (ObjNum 1067 snap 0 seq_num 1067) 2023-08-27T20:53:23.957 INFO:tasks.rados.rados.0.smithi093.stdout:1856: write oid 154 current snap is 0 2023-08-27T20:53:23.957 INFO:tasks.rados.rados.0.smithi093.stdout:1850: expect (ObjNum 337 snap 0 seq_num 337) 2023-08-27T20:53:24.069 INFO:tasks.rados.rados.0.smithi093.stdout:1849: expect (ObjNum 1119 snap 0 seq_num 1119) 2023-08-27T20:53:24.155 INFO:tasks.rados.rados.0.smithi093.stdout:1852: finishing write tid 1 to smithi093153278-304 2023-08-27T20:53:24.155 INFO:tasks.rados.rados.0.smithi093.stdout:1852: finishing write tid 2 to smithi093153278-304 2023-08-27T20:53:24.155 INFO:tasks.rados.rados.0.smithi093.stdout:1856: seq_num 1263 ranges {427241=754085,1624287=408441,2600534=549964} 2023-08-27T20:53:24.162 INFO:tasks.rados.rados.0.smithi093.stdout:1856: writing smithi093153278-154 from 427241 to 1181326 tid 1 2023-08-27T20:53:24.166 INFO:tasks.rados.rados.0.smithi093.stdout:1856: writing smithi093153278-154 from 1624287 to 2032728 tid 2 2023-08-27T20:53:24.173 INFO:tasks.rados.rados.0.smithi093.stdout:1856: writing smithi093153278-154 from 2600534 to 3150498 tid 3 2023-08-27T20:53:24.173 INFO:tasks.rados.rados.0.smithi093.stdout:1849: done (6 left) 2023-08-27T20:53:24.174 INFO:tasks.rados.rados.0.smithi093.stdout:1850: done (5 left) 2023-08-27T20:53:24.174 INFO:tasks.rados.rados.0.smithi093.stdout:1857: write oid 498 current snap is 0 2023-08-27T20:53:24.174 INFO:tasks.rados.rados.0.smithi093.stdout:1852: finishing write tid 3 to smithi093153278-304 2023-08-27T20:53:24.174 INFO:tasks.rados.rados.0.smithi093.stdout:1852: finishing write tid 4 to smithi093153278-304 2023-08-27T20:53:24.174 INFO:tasks.rados.rados.0.smithi093.stdout:1857: seq_num 1264 ranges {444048=429334,1282769=552099,2627957=616091} 2023-08-27T20:53:24.178 INFO:tasks.rados.rados.0.smithi093.stdout:1857: writing smithi093153278-498 from 444048 to 873382 tid 1 2023-08-27T20:53:24.183 INFO:tasks.rados.rados.0.smithi093.stdout:1857: writing smithi093153278-498 from 1282769 to 1834868 tid 2 2023-08-27T20:53:24.191 INFO:tasks.rados.rados.0.smithi093.stdout:1857: writing smithi093153278-498 from 2627957 to 3244048 tid 3 2023-08-27T20:53:24.191 INFO:tasks.rados.rados.0.smithi093.stdout:1858: write oid 222 current snap is 0 2023-08-27T20:53:24.191 INFO:tasks.rados.rados.0.smithi093.stdout:1852: finishing write tid 5 to smithi093153278-304 2023-08-27T20:53:24.191 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 304 v 280 (ObjNum 1261 snap 0 seq_num 1261) dirty exists 2023-08-27T20:53:24.191 INFO:tasks.rados.rados.0.smithi093.stdout:1852: left oid 304 (ObjNum 1261 snap 0 seq_num 1261) 2023-08-27T20:53:24.192 INFO:tasks.rados.rados.0.smithi093.stdout:1853: finishing write tid 1 to smithi093153278-164 2023-08-27T20:53:24.192 INFO:tasks.rados.rados.0.smithi093.stdout:1853: finishing write tid 2 to smithi093153278-164 2023-08-27T20:53:24.192 INFO:tasks.rados.rados.0.smithi093.stdout:1858: seq_num 1265 ranges {460855=504583,1624898=412110,2539027=572508} 2023-08-27T20:53:24.196 INFO:tasks.rados.rados.0.smithi093.stdout:1858: writing smithi093153278-222 from 460855 to 965438 tid 1 2023-08-27T20:53:24.202 INFO:tasks.rados.rados.0.smithi093.stdout:1858: writing smithi093153278-222 from 1624898 to 2037008 tid 2 2023-08-27T20:53:24.208 INFO:tasks.rados.rados.0.smithi093.stdout:1858: writing smithi093153278-222 from 2539027 to 3111535 tid 3 2023-08-27T20:53:24.208 INFO:tasks.rados.rados.0.smithi093.stdout:1852: done (6 left) 2023-08-27T20:53:24.208 INFO:tasks.rados.rados.0.smithi093.stdout:1859: write oid 479 current snap is 0 2023-08-27T20:53:24.209 INFO:tasks.rados.rados.0.smithi093.stdout:1859: seq_num 1266 ranges {477662=579832,1683380=555768,2966450=311212} 2023-08-27T20:53:24.214 INFO:tasks.rados.rados.0.smithi093.stdout:1859: writing smithi093153278-479 from 477662 to 1057494 tid 1 2023-08-27T20:53:24.220 INFO:tasks.rados.rados.0.smithi093.stdout:1859: writing smithi093153278-479 from 1683380 to 2239148 tid 2 2023-08-27T20:53:24.226 INFO:tasks.rados.rados.0.smithi093.stdout:1859: writing smithi093153278-479 from 2966450 to 3277662 tid 3 2023-08-27T20:53:24.227 INFO:tasks.rados.rados.0.smithi093.stdout:1860: read oid 13 snap -1 2023-08-27T20:53:24.227 INFO:tasks.rados.rados.0.smithi093.stdout:1860: expect (ObjNum 1258 snap 0 seq_num 1258) 2023-08-27T20:53:24.227 INFO:tasks.rados.rados.0.smithi093.stdout:1861: read oid 291 snap -1 2023-08-27T20:53:24.227 INFO:tasks.rados.rados.0.smithi093.stdout:1861: expect deleted 2023-08-27T20:53:24.227 INFO:tasks.rados.rados.0.smithi093.stdout:1862: write oid 78 current snap is 0 2023-08-27T20:53:24.227 INFO:tasks.rados.rados.0.smithi093.stdout:1855: expect (ObjNum 1067 snap 0 seq_num 1067) 2023-08-27T20:53:24.410 INFO:tasks.rados.rados.0.smithi093.stdout:1853: finishing write tid 3 to smithi093153278-164 2023-08-27T20:53:24.410 INFO:tasks.rados.rados.0.smithi093.stdout:1853: finishing write tid 4 to smithi093153278-164 2023-08-27T20:53:24.411 INFO:tasks.rados.rados.0.smithi093.stdout:1853: finishing write tid 5 to smithi093153278-164 2023-08-27T20:53:24.411 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 164 v 258 (ObjNum 1262 snap 0 seq_num 1262) dirty exists 2023-08-27T20:53:24.411 INFO:tasks.rados.rados.0.smithi093.stdout:1853: left oid 164 (ObjNum 1262 snap 0 seq_num 1262) 2023-08-27T20:53:24.411 INFO:tasks.rados.rados.0.smithi093.stdout:1854: expect (ObjNum 704 snap 0 seq_num 704) 2023-08-27T20:53:24.621 INFO:tasks.rados.rados.0.smithi093.stdout:1856: finishing write tid 1 to smithi093153278-154 2023-08-27T20:53:24.621 INFO:tasks.rados.rados.0.smithi093.stdout:1856: finishing write tid 2 to smithi093153278-154 2023-08-27T20:53:24.621 INFO:tasks.rados.rados.0.smithi093.stdout:1856: finishing write tid 3 to smithi093153278-154 2023-08-27T20:53:24.621 INFO:tasks.rados.rados.0.smithi093.stdout:1862: seq_num 1267 ranges {494469=655081,1741862=415779,2710226=584243} 2023-08-27T20:53:24.627 INFO:tasks.rados.rados.0.smithi093.stdout:1862: writing smithi093153278-78 from 494469 to 1149550 tid 1 2023-08-27T20:53:24.633 INFO:tasks.rados.rados.0.smithi093.stdout:1862: writing smithi093153278-78 from 1741862 to 2157641 tid 2 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1862: writing smithi093153278-78 from 2710226 to 3294469 tid 3 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1853: done (9 left) 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1854: done (8 left) 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1855: done (7 left) 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1863: read oid 113 snap -1 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1863: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1864: read oid 100 snap -1 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1864: expect (ObjNum 1134 snap 0 seq_num 1134) 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1865: read oid 481 snap -1 2023-08-27T20:53:24.641 INFO:tasks.rados.rados.0.smithi093.stdout:1865: expect deleted 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1866: read oid 8 snap -1 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1866: expect (ObjNum 7 snap 0 seq_num 7) 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1867: delete oid 426 current snap is 0 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1856: finishing write tid 4 to smithi093153278-154 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1856: finishing write tid 5 to smithi093153278-154 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 154 v 270 (ObjNum 1263 snap 0 seq_num 1263) dirty exists 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1856: left oid 154 (ObjNum 1263 snap 0 seq_num 1263) 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1857: finishing write tid 1 to smithi093153278-498 2023-08-27T20:53:24.642 INFO:tasks.rados.rados.0.smithi093.stdout:1857: finishing write tid 2 to smithi093153278-498 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1857: finishing write tid 3 to smithi093153278-498 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1857: finishing write tid 4 to smithi093153278-498 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1857: finishing write tid 5 to smithi093153278-498 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 498 v 186 (ObjNum 1264 snap 0 seq_num 1264) dirty exists 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1857: left oid 498 (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1858: finishing write tid 1 to smithi093153278-222 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1858: finishing write tid 2 to smithi093153278-222 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1858: finishing write tid 3 to smithi093153278-222 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1858: finishing write tid 4 to smithi093153278-222 2023-08-27T20:53:24.643 INFO:tasks.rados.rados.0.smithi093.stdout:1858: finishing write tid 5 to smithi093153278-222 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 222 v 291 (ObjNum 1265 snap 0 seq_num 1265) dirty exists 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1858: left oid 222 (ObjNum 1265 snap 0 seq_num 1265) 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1859: finishing write tid 1 to smithi093153278-479 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1859: finishing write tid 2 to smithi093153278-479 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1859: finishing write tid 3 to smithi093153278-479 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1859: finishing write tid 4 to smithi093153278-479 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1859: finishing write tid 5 to smithi093153278-479 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 479 v 227 (ObjNum 1266 snap 0 seq_num 1266) dirty exists 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1859: left oid 479 (ObjNum 1266 snap 0 seq_num 1266) 2023-08-27T20:53:24.644 INFO:tasks.rados.rados.0.smithi093.stdout:1860: expect (ObjNum 1258 snap 0 seq_num 1258) 2023-08-27T20:53:24.801 INFO:tasks.rados.rados.0.smithi093.stdout:1862: finishing write tid 1 to smithi093153278-78 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1862: finishing write tid 2 to smithi093153278-78 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1856: done (11 left) 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1857: done (10 left) 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1858: done (9 left) 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1859: done (8 left) 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1860: done (7 left) 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1861: done (6 left) 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1867: done (5 left) 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1868: write oid 371 current snap is 0 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1862: finishing write tid 3 to smithi093153278-78 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1862: finishing write tid 4 to smithi093153278-78 2023-08-27T20:53:24.802 INFO:tasks.rados.rados.0.smithi093.stdout:1862: finishing write tid 5 to smithi093153278-78 2023-08-27T20:53:24.803 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 238 (ObjNum 1267 snap 0 seq_num 1267) dirty exists 2023-08-27T20:53:24.803 INFO:tasks.rados.rados.0.smithi093.stdout:1862: left oid 78 (ObjNum 1267 snap 0 seq_num 1267) 2023-08-27T20:53:24.803 INFO:tasks.rados.rados.0.smithi093.stdout:1863: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:53:24.924 INFO:tasks.rados.rados.0.smithi093.stdout:1864: expect (ObjNum 1134 snap 0 seq_num 1134) 2023-08-27T20:53:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/980351443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/980351443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/980351443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:25.053 INFO:tasks.rados.rados.0.smithi093.stdout:1866: expect (ObjNum 7 snap 0 seq_num 7) 2023-08-27T20:53:25.119 INFO:tasks.rados.rados.0.smithi093.stdout:1868: seq_num 1269 ranges {528083=405579,1342473=419448,2248719=733949} 2023-08-27T20:53:25.124 INFO:tasks.rados.rados.0.smithi093.stdout:1868: writing smithi093153278-371 from 528083 to 933662 tid 1 2023-08-27T20:53:25.128 INFO:tasks.rados.rados.0.smithi093.stdout:1868: writing smithi093153278-371 from 1342473 to 1761921 tid 2 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1868: writing smithi093153278-371 from 2248719 to 2982668 tid 3 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1862: done (5 left) 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1863: done (4 left) 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1864: done (3 left) 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1865: done (2 left) 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1866: done (1 left) 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1869: write oid 386 current snap is 0 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1868: finishing write tid 1 to smithi093153278-371 2023-08-27T20:53:25.136 INFO:tasks.rados.rados.0.smithi093.stdout:1868: finishing write tid 2 to smithi093153278-371 2023-08-27T20:53:25.137 INFO:tasks.rados.rados.0.smithi093.stdout:1869: seq_num 1270 ranges {544890=597181,1917308=563106,3076142=268748} 2023-08-27T20:53:25.142 INFO:tasks.rados.rados.0.smithi093.stdout:1869: writing smithi093153278-386 from 544890 to 1142071 tid 1 2023-08-27T20:53:25.149 INFO:tasks.rados.rados.0.smithi093.stdout:1869: writing smithi093153278-386 from 1917308 to 2480414 tid 2 2023-08-27T20:53:25.154 INFO:tasks.rados.rados.0.smithi093.stdout:1869: writing smithi093153278-386 from 3076142 to 3344890 tid 3 2023-08-27T20:53:25.154 INFO:tasks.rados.rados.0.smithi093.stdout:1870: write oid 421 current snap is 0 2023-08-27T20:53:25.154 INFO:tasks.rados.rados.0.smithi093.stdout:1870: seq_num 1271 ranges {561697=672430,1975790=706764,3103565=258132} 2023-08-27T20:53:25.160 INFO:tasks.rados.rados.0.smithi093.stdout:1870: writing smithi093153278-421 from 561697 to 1234127 tid 1 2023-08-27T20:53:25.169 INFO:tasks.rados.rados.0.smithi093.stdout:1870: writing smithi093153278-421 from 1975790 to 2682554 tid 2 2023-08-27T20:53:25.172 INFO:tasks.rados.rados.0.smithi093.stdout:1870: writing smithi093153278-421 from 3103565 to 3361697 tid 3 2023-08-27T20:53:25.172 INFO:tasks.rados.rados.0.smithi093.stdout:1871: write oid 196 current snap is 0 2023-08-27T20:53:25.172 INFO:tasks.rados.rados.0.smithi093.stdout:1871: seq_num 1272 ranges {578504=747679,2034272=566775,3130988=247516} 2023-08-27T20:53:25.179 INFO:tasks.rados.rados.0.smithi093.stdout:1871: writing smithi093153278-196 from 578504 to 1326183 tid 1 2023-08-27T20:53:25.186 INFO:tasks.rados.rados.0.smithi093.stdout:1871: writing smithi093153278-196 from 2034272 to 2601047 tid 2 2023-08-27T20:53:25.191 INFO:tasks.rados.rados.0.smithi093.stdout:1871: writing smithi093153278-196 from 3130988 to 3378504 tid 3 2023-08-27T20:53:25.191 INFO:tasks.rados.rados.0.smithi093.stdout:1872: read oid 38 snap -1 2023-08-27T20:53:25.191 INFO:tasks.rados.rados.0.smithi093.stdout:1872: expect (ObjNum 37 snap 0 seq_num 37) 2023-08-27T20:53:25.191 INFO:tasks.rados.rados.0.smithi093.stdout:1873: read oid 240 snap -1 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1873: expect (ObjNum 557 snap 0 seq_num 557) 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1874: read oid 380 snap -1 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1874: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1875: read oid 35 snap -1 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1875: expect (ObjNum 1054 snap 0 seq_num 1054) 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1876: write oid 83 current snap is 0 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1868: finishing write tid 3 to smithi093153278-371 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1868: finishing write tid 4 to smithi093153278-371 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1868: finishing write tid 5 to smithi093153278-371 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 371 v 242 (ObjNum 1269 snap 0 seq_num 1269) dirty exists 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1868: left oid 371 (ObjNum 1269 snap 0 seq_num 1269) 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1869: finishing write tid 1 to smithi093153278-386 2023-08-27T20:53:25.192 INFO:tasks.rados.rados.0.smithi093.stdout:1876: seq_num 1273 ranges {595311=422928,1576401=710433,3042058=353253} 2023-08-27T20:53:25.196 INFO:tasks.rados.rados.0.smithi093.stdout:1876: writing smithi093153278-83 from 595311 to 1018239 tid 1 2023-08-27T20:53:25.204 INFO:tasks.rados.rados.0.smithi093.stdout:1876: writing smithi093153278-83 from 1576401 to 2286834 tid 2 2023-08-27T20:53:25.211 INFO:tasks.rados.rados.0.smithi093.stdout:1876: writing smithi093153278-83 from 3042058 to 3395311 tid 3 2023-08-27T20:53:25.211 INFO:tasks.rados.rados.0.smithi093.stdout:1868: done (8 left) 2023-08-27T20:53:25.211 INFO:tasks.rados.rados.0.smithi093.stdout:1877: write oid 469 current snap is 0 2023-08-27T20:53:25.211 INFO:tasks.rados.rados.0.smithi093.stdout:1869: finishing write tid 2 to smithi093153278-386 2023-08-27T20:53:25.211 INFO:tasks.rados.rados.0.smithi093.stdout:1877: seq_num 1274 ranges {612118=498177,1634883=570444,2669481=706662} 2023-08-27T20:53:25.217 INFO:tasks.rados.rados.0.smithi093.stdout:1877: writing smithi093153278-469 from 612118 to 1110295 tid 1 2023-08-27T20:53:25.223 INFO:tasks.rados.rados.0.smithi093.stdout:1877: writing smithi093153278-469 from 1634883 to 2205327 tid 2 2023-08-27T20:53:25.230 INFO:tasks.rados.rados.0.smithi093.stdout:1877: writing smithi093153278-469 from 2669481 to 3376143 tid 3 2023-08-27T20:53:25.230 INFO:tasks.rados.rados.0.smithi093.stdout:1878: read oid 398 snap -1 2023-08-27T20:53:25.230 INFO:tasks.rados.rados.0.smithi093.stdout:1878: expect (ObjNum 397 snap 0 seq_num 397) 2023-08-27T20:53:25.230 INFO:tasks.rados.rados.0.smithi093.stdout:1879: write oid 305 current snap is 0 2023-08-27T20:53:25.230 INFO:tasks.rados.rados.0.smithi093.stdout:1869: finishing write tid 3 to smithi093153278-386 2023-08-27T20:53:25.230 INFO:tasks.rados.rados.0.smithi093.stdout:1869: finishing write tid 4 to smithi093153278-386 2023-08-27T20:53:25.230 INFO:tasks.rados.rados.0.smithi093.stdout:1869: finishing write tid 5 to smithi093153278-386 2023-08-27T20:53:25.231 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 386 v 226 (ObjNum 1270 snap 0 seq_num 1270) dirty exists 2023-08-27T20:53:25.231 INFO:tasks.rados.rados.0.smithi093.stdout:1869: left oid 386 (ObjNum 1270 snap 0 seq_num 1270) 2023-08-27T20:53:25.231 INFO:tasks.rados.rados.0.smithi093.stdout:1870: finishing write tid 1 to smithi093153278-421 2023-08-27T20:53:25.231 INFO:tasks.rados.rados.0.smithi093.stdout:1870: finishing write tid 2 to smithi093153278-421 2023-08-27T20:53:25.231 INFO:tasks.rados.rados.0.smithi093.stdout:1879: seq_num 1275 ranges {628925=573426,1693365=714102,3096904=332021} 2023-08-27T20:53:25.237 INFO:tasks.rados.rados.0.smithi093.stdout:1879: writing smithi093153278-305 from 628925 to 1202351 tid 1 2023-08-27T20:53:25.244 INFO:tasks.rados.rados.0.smithi093.stdout:1879: writing smithi093153278-305 from 1693365 to 2407467 tid 2 2023-08-27T20:53:25.250 INFO:tasks.rados.rados.0.smithi093.stdout:1879: writing smithi093153278-305 from 3096904 to 3428925 tid 3 2023-08-27T20:53:25.250 INFO:tasks.rados.rados.0.smithi093.stdout:1869: done (10 left) 2023-08-27T20:53:25.250 INFO:tasks.rados.rados.0.smithi093.stdout:1880: read oid 457 snap -1 2023-08-27T20:53:25.250 INFO:tasks.rados.rados.0.smithi093.stdout:1880: expect (ObjNum 456 snap 0 seq_num 456) 2023-08-27T20:53:25.250 INFO:tasks.rados.rados.0.smithi093.stdout:1881: write oid 169 current snap is 0 2023-08-27T20:53:25.250 INFO:tasks.rados.rados.0.smithi093.stdout:1870: finishing write tid 3 to smithi093153278-421 2023-08-27T20:53:25.251 INFO:tasks.rados.rados.0.smithi093.stdout:1870: finishing write tid 4 to smithi093153278-421 2023-08-27T20:53:25.251 INFO:tasks.rados.rados.0.smithi093.stdout:1870: finishing write tid 5 to smithi093153278-421 2023-08-27T20:53:25.251 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 421 v 284 (ObjNum 1271 snap 0 seq_num 1271) dirty exists 2023-08-27T20:53:25.251 INFO:tasks.rados.rados.0.smithi093.stdout:1870: left oid 421 (ObjNum 1271 snap 0 seq_num 1271) 2023-08-27T20:53:25.251 INFO:tasks.rados.rados.0.smithi093.stdout:1871: finishing write tid 1 to smithi093153278-196 2023-08-27T20:53:25.251 INFO:tasks.rados.rados.0.smithi093.stdout:1871: finishing write tid 2 to smithi093153278-196 2023-08-27T20:53:25.251 INFO:tasks.rados.rados.0.smithi093.stdout:1881: seq_num 1276 ranges {645732=648675,1751847=574113,3124327=321405} 2023-08-27T20:53:25.257 INFO:tasks.rados.rados.0.smithi093.stdout:1881: writing smithi093153278-169 from 645732 to 1294407 tid 1 2023-08-27T20:53:25.263 INFO:tasks.rados.rados.0.smithi093.stdout:1881: writing smithi093153278-169 from 1751847 to 2325960 tid 2 2023-08-27T20:53:25.269 INFO:tasks.rados.rados.0.smithi093.stdout:1881: writing smithi093153278-169 from 3124327 to 3445732 tid 3 2023-08-27T20:53:25.270 INFO:tasks.rados.rados.0.smithi093.stdout:1870: done (11 left) 2023-08-27T20:53:25.270 INFO:tasks.rados.rados.0.smithi093.stdout:1882: read oid 46 snap -1 2023-08-27T20:53:25.270 INFO:tasks.rados.rados.0.smithi093.stdout:1882: expect deleted 2023-08-27T20:53:25.270 INFO:tasks.rados.rados.0.smithi093.stdout:1883: write oid 12 current snap is 0 2023-08-27T20:53:25.270 INFO:tasks.rados.rados.0.smithi093.stdout:1872: expect (ObjNum 37 snap 0 seq_num 37) 2023-08-27T20:53:25.350 INFO:tasks.rados.rados.0.smithi093.stdout:1871: finishing write tid 3 to smithi093153278-196 2023-08-27T20:53:25.350 INFO:tasks.rados.rados.0.smithi093.stdout:1871: finishing write tid 4 to smithi093153278-196 2023-08-27T20:53:25.350 INFO:tasks.rados.rados.0.smithi093.stdout:1871: finishing write tid 5 to smithi093153278-196 2023-08-27T20:53:25.350 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 196 v 251 (ObjNum 1272 snap 0 seq_num 1272) dirty exists 2023-08-27T20:53:25.350 INFO:tasks.rados.rados.0.smithi093.stdout:1871: left oid 196 (ObjNum 1272 snap 0 seq_num 1272) 2023-08-27T20:53:25.350 INFO:tasks.rados.rados.0.smithi093.stdout:1883: seq_num 1277 ranges {662539=723924,2093976=717771,3435397=27142} 2023-08-27T20:53:25.357 INFO:tasks.rados.rados.0.smithi093.stdout:1883: writing smithi093153278-12 from 662539 to 1386463 tid 1 2023-08-27T20:53:25.365 INFO:tasks.rados.rados.0.smithi093.stdout:1883: writing smithi093153278-12 from 2093976 to 2811747 tid 2 2023-08-27T20:53:25.369 INFO:tasks.rados.rados.0.smithi093.stdout:1883: writing smithi093153278-12 from 3435397 to 3462539 tid 3 2023-08-27T20:53:25.369 INFO:tasks.rados.rados.0.smithi093.stdout:1871: done (12 left) 2023-08-27T20:53:25.369 INFO:tasks.rados.rados.0.smithi093.stdout:1872: done (11 left) 2023-08-27T20:53:25.369 INFO:tasks.rados.rados.0.smithi093.stdout:1884: write oid 448 current snap is 0 2023-08-27T20:53:25.369 INFO:tasks.rados.rados.0.smithi093.stdout:1875: expect (ObjNum 1054 snap 0 seq_num 1054) 2023-08-27T20:53:25.525 INFO:tasks.rados.rados.0.smithi093.stdout:1876: finishing write tid 1 to smithi093153278-83 2023-08-27T20:53:25.525 INFO:tasks.rados.rados.0.smithi093.stdout:1874: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:53:25.686 INFO:tasks.rados.rados.0.smithi093.stdout:1873: expect (ObjNum 557 snap 0 seq_num 557) 2023-08-27T20:53:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:53:25] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:53:25.836 INFO:tasks.rados.rados.0.smithi093.stdout:1876: finishing write tid 2 to smithi093153278-83 2023-08-27T20:53:25.836 INFO:tasks.rados.rados.0.smithi093.stdout:1876: finishing write tid 3 to smithi093153278-83 2023-08-27T20:53:25.836 INFO:tasks.rados.rados.0.smithi093.stdout:1876: finishing write tid 4 to smithi093153278-83 2023-08-27T20:53:25.836 INFO:tasks.rados.rados.0.smithi093.stdout:1876: finishing write tid 5 to smithi093153278-83 2023-08-27T20:53:25.836 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 83 v 231 (ObjNum 1273 snap 0 seq_num 1273) dirty exists 2023-08-27T20:53:25.836 INFO:tasks.rados.rados.0.smithi093.stdout:1876: left oid 83 (ObjNum 1273 snap 0 seq_num 1273) 2023-08-27T20:53:25.837 INFO:tasks.rados.rados.0.smithi093.stdout:1877: finishing write tid 1 to smithi093153278-469 2023-08-27T20:53:25.837 INFO:tasks.rados.rados.0.smithi093.stdout:1877: finishing write tid 2 to smithi093153278-469 2023-08-27T20:53:25.837 INFO:tasks.rados.rados.0.smithi093.stdout:1884: seq_num 1278 ranges {679346=515526,1868811=577782,3179173=300173} 2023-08-27T20:53:25.842 INFO:tasks.rados.rados.0.smithi093.stdout:1884: writing smithi093153278-448 from 679346 to 1194872 tid 1 2023-08-27T20:53:25.849 INFO:tasks.rados.rados.0.smithi093.stdout:1884: writing smithi093153278-448 from 1868811 to 2446593 tid 2 2023-08-27T20:53:25.855 INFO:tasks.rados.rados.0.smithi093.stdout:1884: writing smithi093153278-448 from 3179173 to 3479346 tid 3 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1873: done (11 left) 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1874: done (10 left) 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1875: done (9 left) 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1876: done (8 left) 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1885: write oid 81 current snap is 0 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1877: finishing write tid 3 to smithi093153278-469 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1877: finishing write tid 4 to smithi093153278-469 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1877: finishing write tid 5 to smithi093153278-469 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 469 v 300 (ObjNum 1274 snap 0 seq_num 1274) dirty exists 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1877: left oid 469 (ObjNum 1274 snap 0 seq_num 1274) 2023-08-27T20:53:25.856 INFO:tasks.rados.rados.0.smithi093.stdout:1878: expect (ObjNum 397 snap 0 seq_num 397) 2023-08-27T20:53:25.936 INFO:tasks.rados.rados.0.smithi093.stdout:1879: finishing write tid 1 to smithi093153278-305 2023-08-27T20:53:25.936 INFO:tasks.rados.rados.0.smithi093.stdout:1879: finishing write tid 2 to smithi093153278-305 2023-08-27T20:53:25.936 INFO:tasks.rados.rados.0.smithi093.stdout:1880: expect (ObjNum 456 snap 0 seq_num 456) 2023-08-27T20:53:26.043 INFO:tasks.rados.rados.0.smithi093.stdout:1879: finishing write tid 3 to smithi093153278-305 2023-08-27T20:53:26.049 INFO:tasks.rados.rados.0.smithi093.stdout:1879: finishing write tid 4 to smithi093153278-305 2023-08-27T20:53:26.049 INFO:tasks.rados.rados.0.smithi093.stdout:1879: finishing write tid 5 to smithi093153278-305 2023-08-27T20:53:26.050 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 305 v 235 (ObjNum 1275 snap 0 seq_num 1275) dirty exists 2023-08-27T20:53:26.050 INFO:tasks.rados.rados.0.smithi093.stdout:1879: left oid 305 (ObjNum 1275 snap 0 seq_num 1275) 2023-08-27T20:53:26.050 INFO:tasks.rados.rados.0.smithi093.stdout:1885: seq_num 1279 ranges {696153=590775,1927293=721440,3206596=289557} 2023-08-27T20:53:26.050 INFO:tasks.rados.rados.0.smithi093.stdout:1885: writing smithi093153278-81 from 696153 to 1286928 tid 1 2023-08-27T20:53:26.058 INFO:tasks.rados.rados.0.smithi093.stdout:1885: writing smithi093153278-81 from 1927293 to 2648733 tid 2 2023-08-27T20:53:26.063 INFO:tasks.rados.rados.0.smithi093.stdout:1885: writing smithi093153278-81 from 3206596 to 3496153 tid 3 2023-08-27T20:53:26.063 INFO:tasks.rados.rados.0.smithi093.stdout:1877: done (8 left) 2023-08-27T20:53:26.063 INFO:tasks.rados.rados.0.smithi093.stdout:1878: done (7 left) 2023-08-27T20:53:26.063 INFO:tasks.rados.rados.0.smithi093.stdout:1879: done (6 left) 2023-08-27T20:53:26.063 INFO:tasks.rados.rados.0.smithi093.stdout:1880: done (5 left) 2023-08-27T20:53:26.063 INFO:tasks.rados.rados.0.smithi093.stdout:1886: delete oid 146 current snap is 0 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1881: finishing write tid 1 to smithi093153278-169 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1881: finishing write tid 2 to smithi093153278-169 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1881: finishing write tid 3 to smithi093153278-169 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1881: finishing write tid 4 to smithi093153278-169 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1881: finishing write tid 5 to smithi093153278-169 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 169 v 274 (ObjNum 1276 snap 0 seq_num 1276) dirty exists 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1881: left oid 169 (ObjNum 1276 snap 0 seq_num 1276) 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1883: finishing write tid 1 to smithi093153278-12 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1883: finishing write tid 2 to smithi093153278-12 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1883: finishing write tid 3 to smithi093153278-12 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1883: finishing write tid 4 to smithi093153278-12 2023-08-27T20:53:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:1883: finishing write tid 5 to smithi093153278-12 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 12 v 230 (ObjNum 1277 snap 0 seq_num 1277) dirty exists 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1883: left oid 12 (ObjNum 1277 snap 0 seq_num 1277) 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1884: finishing write tid 1 to smithi093153278-448 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1884: finishing write tid 2 to smithi093153278-448 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1884: finishing write tid 3 to smithi093153278-448 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1884: finishing write tid 4 to smithi093153278-448 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1884: finishing write tid 5 to smithi093153278-448 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 448 v 230 (ObjNum 1278 snap 0 seq_num 1278) dirty exists 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1884: left oid 448 (ObjNum 1278 snap 0 seq_num 1278) 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1885: finishing write tid 1 to smithi093153278-81 2023-08-27T20:53:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:1885: finishing write tid 2 to smithi093153278-81 2023-08-27T20:53:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:1881: done (5 left) 2023-08-27T20:53:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:1882: done (4 left) 2023-08-27T20:53:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:1883: done (3 left) 2023-08-27T20:53:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:1884: done (2 left) 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1886: done (1 left) 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1887: read oid 192 snap -1 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1887: expect (ObjNum 1130 snap 0 seq_num 1130) 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1888: read oid 334 snap -1 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1888: expect (ObjNum 756 snap 0 seq_num 756) 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1889: read oid 234 snap -1 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1889: expect (ObjNum 1202 snap 0 seq_num 1202) 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1890: write oid 314 current snap is 0 2023-08-27T20:53:26.067 INFO:tasks.rados.rados.0.smithi093.stdout:1890: seq_num 1281 ranges {729767=741273,1927904=725109,3145089=384678} 2023-08-27T20:53:26.073 INFO:tasks.rados.rados.0.smithi093.stdout:1890: writing smithi093153278-314 from 729767 to 1471040 tid 1 2023-08-27T20:53:26.080 INFO:tasks.rados.rados.0.smithi093.stdout:1890: writing smithi093153278-314 from 1927904 to 2653013 tid 2 2023-08-27T20:53:26.086 INFO:tasks.rados.rados.0.smithi093.stdout:1890: writing smithi093153278-314 from 3145089 to 3529767 tid 3 2023-08-27T20:53:26.086 INFO:tasks.rados.rados.0.smithi093.stdout:1891: read oid 3 snap -1 2023-08-27T20:53:26.086 INFO:tasks.rados.rados.0.smithi093.stdout:1891: expect (ObjNum 696 snap 0 seq_num 696) 2023-08-27T20:53:26.086 INFO:tasks.rados.rados.0.smithi093.stdout:1892: read oid 443 snap -1 2023-08-27T20:53:26.087 INFO:tasks.rados.rados.0.smithi093.stdout:1892: expect (ObjNum 1100 snap 0 seq_num 1100) 2023-08-27T20:53:26.087 INFO:tasks.rados.rados.0.smithi093.stdout:1893: read oid 266 snap -1 2023-08-27T20:53:26.087 INFO:tasks.rados.rados.0.smithi093.stdout:1893: expect deleted 2023-08-27T20:53:26.087 INFO:tasks.rados.rados.0.smithi093.stdout:1894: write oid 425 current snap is 0 2023-08-27T20:53:26.087 INFO:tasks.rados.rados.0.smithi093.stdout:1885: finishing write tid 3 to smithi093153278-81 2023-08-27T20:53:26.087 INFO:tasks.rados.rados.0.smithi093.stdout:1894: seq_num 1282 ranges {746574=416522,1586386=468767,2772512=629544} 2023-08-27T20:53:26.092 INFO:tasks.rados.rados.0.smithi093.stdout:1894: writing smithi093153278-425 from 746574 to 1163096 tid 1 2023-08-27T20:53:26.098 INFO:tasks.rados.rados.0.smithi093.stdout:1894: writing smithi093153278-425 from 1586386 to 2055153 tid 2 2023-08-27T20:53:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:1894: writing smithi093153278-425 from 2772512 to 3402056 tid 3 2023-08-27T20:53:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:1885: finishing write tid 4 to smithi093153278-81 2023-08-27T20:53:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:1885: finishing write tid 5 to smithi093153278-81 2023-08-27T20:53:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 343 (ObjNum 1279 snap 0 seq_num 1279) dirty exists 2023-08-27T20:53:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:1885: left oid 81 (ObjNum 1279 snap 0 seq_num 1279) 2023-08-27T20:53:26.106 INFO:tasks.rados.rados.0.smithi093.stdout:1888: expect (ObjNum 756 snap 0 seq_num 756) 2023-08-27T20:53:26.229 INFO:tasks.rados.rados.0.smithi093.stdout:1889: expect (ObjNum 1202 snap 0 seq_num 1202) 2023-08-27T20:53:26.334 INFO:tasks.rados.rados.0.smithi093.stdout:1887: expect (ObjNum 1130 snap 0 seq_num 1130) 2023-08-27T20:53:26.501 INFO:tasks.rados.rados.0.smithi093.stdout:1890: finishing write tid 1 to smithi093153278-314 2023-08-27T20:53:26.502 INFO:tasks.rados.rados.0.smithi093.stdout:1890: finishing write tid 2 to smithi093153278-314 2023-08-27T20:53:26.502 INFO:tasks.rados.rados.0.smithi093.stdout:1892: expect (ObjNum 1100 snap 0 seq_num 1100) 2023-08-27T20:53:26.642 INFO:tasks.rados.rados.0.smithi093.stdout:1890: finishing write tid 3 to smithi093153278-314 2023-08-27T20:53:26.642 INFO:tasks.rados.rados.0.smithi093.stdout:1890: finishing write tid 4 to smithi093153278-314 2023-08-27T20:53:26.642 INFO:tasks.rados.rados.0.smithi093.stdout:1890: finishing write tid 5 to smithi093153278-314 2023-08-27T20:53:26.642 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 314 v 304 (ObjNum 1281 snap 0 seq_num 1281) dirty exists 2023-08-27T20:53:26.642 INFO:tasks.rados.rados.0.smithi093.stdout:1890: left oid 314 (ObjNum 1281 snap 0 seq_num 1281) 2023-08-27T20:53:26.642 INFO:tasks.rados.rados.0.smithi093.stdout:1891: expect (ObjNum 696 snap 0 seq_num 696) 2023-08-27T20:53:26.812 INFO:tasks.rados.rados.0.smithi093.stdout:1894: finishing write tid 1 to smithi093153278-425 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1885: done (8 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1887: done (7 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1888: done (6 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1889: done (5 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1890: done (4 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1891: done (3 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1892: done (2 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1893: done (1 left) 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1895: write oid 169 current snap is 0 2023-08-27T20:53:26.813 INFO:tasks.rados.rados.0.smithi093.stdout:1894: finishing write tid 2 to smithi093153278-425 2023-08-27T20:53:26.814 INFO:tasks.rados.rados.0.smithi093.stdout:1894: finishing write tid 3 to smithi093153278-425 2023-08-27T20:53:26.814 INFO:tasks.rados.rados.0.smithi093.stdout:1895: seq_num 1283 ranges {763381=491771,2044868=728778,3199935=363446} 2023-08-27T20:53:26.819 INFO:tasks.rados.rados.0.smithi093.stdout:1895: writing smithi093153278-169 from 763381 to 1255152 tid 1 2023-08-27T20:53:26.827 INFO:tasks.rados.rados.0.smithi093.stdout:1895: writing smithi093153278-169 from 2044868 to 2773646 tid 2 2023-08-27T20:53:26.832 INFO:tasks.rados.rados.0.smithi093.stdout:1895: writing smithi093153278-169 from 3199935 to 3563381 tid 3 2023-08-27T20:53:26.832 INFO:tasks.rados.rados.0.smithi093.stdout:1896: read oid 97 snap -1 2023-08-27T20:53:26.832 INFO:tasks.rados.rados.0.smithi093.stdout:1896: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:53:26.832 INFO:tasks.rados.rados.0.smithi093.stdout:1897: write oid 457 current snap is 0 2023-08-27T20:53:26.833 INFO:tasks.rados.rados.0.smithi093.stdout:1894: finishing write tid 4 to smithi093153278-425 2023-08-27T20:53:26.833 INFO:tasks.rados.rados.0.smithi093.stdout:1894: finishing write tid 5 to smithi093153278-425 2023-08-27T20:53:26.833 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 425 v 278 (ObjNum 1282 snap 0 seq_num 1282) dirty exists 2023-08-27T20:53:26.833 INFO:tasks.rados.rados.0.smithi093.stdout:1894: left oid 425 (ObjNum 1282 snap 0 seq_num 1282) 2023-08-27T20:53:26.833 INFO:tasks.rados.rados.0.smithi093.stdout:1897: seq_num 1284 ranges {780188=567020,2103350=472436,3227358=352830} 2023-08-27T20:53:26.839 INFO:tasks.rados.rados.0.smithi093.stdout:1897: writing smithi093153278-457 from 780188 to 1347208 tid 1 2023-08-27T20:53:26.846 INFO:tasks.rados.rados.0.smithi093.stdout:1897: writing smithi093153278-457 from 2103350 to 2575786 tid 2 2023-08-27T20:53:26.852 INFO:tasks.rados.rados.0.smithi093.stdout:1897: writing smithi093153278-457 from 3227358 to 3580188 tid 3 2023-08-27T20:53:26.852 INFO:tasks.rados.rados.0.smithi093.stdout:1894: done (3 left) 2023-08-27T20:53:26.852 INFO:tasks.rados.rados.0.smithi093.stdout:1898: read oid 247 snap -1 2023-08-27T20:53:26.852 INFO:tasks.rados.rados.0.smithi093.stdout:1898: expect (ObjNum 1076 snap 0 seq_num 1076) 2023-08-27T20:53:26.853 INFO:tasks.rados.rados.0.smithi093.stdout:1899: read oid 272 snap -1 2023-08-27T20:53:26.853 INFO:tasks.rados.rados.0.smithi093.stdout:1899: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:53:26.853 INFO:tasks.rados.rados.0.smithi093.stdout:1900: write oid 403 current snap is 0 2023-08-27T20:53:26.853 INFO:tasks.rados.rados.0.smithi093.stdout:1895: finishing write tid 1 to smithi093153278-169 2023-08-27T20:53:26.853 INFO:tasks.rados.rados.0.smithi093.stdout:1895: finishing write tid 2 to smithi093153278-169 2023-08-27T20:53:26.853 INFO:tasks.rados.rados.0.smithi093.stdout:1900: seq_num 1285 ranges {796995=758622,2161832=732447} 2023-08-27T20:53:26.861 INFO:tasks.rados.rados.0.smithi093.stdout:1900: writing smithi093153278-403 from 796995 to 1555617 tid 1 2023-08-27T20:53:26.868 INFO:tasks.rados.rados.0.smithi093.stdout:1900: writing smithi093153278-403 from 2161832 to 2894279 tid 2 2023-08-27T20:53:26.868 INFO:tasks.rados.rados.0.smithi093.stdout:1901: write oid 424 current snap is 0 2023-08-27T20:53:26.869 INFO:tasks.rados.rados.0.smithi093.stdout:1895: finishing write tid 3 to smithi093153278-169 2023-08-27T20:53:26.869 INFO:tasks.rados.rados.0.smithi093.stdout:1895: finishing write tid 4 to smithi093153278-169 2023-08-27T20:53:26.869 INFO:tasks.rados.rados.0.smithi093.stdout:1901: seq_num 1286 ranges {413802=433871,1420314=476105,2482204=790985} 2023-08-27T20:53:26.873 INFO:tasks.rados.rados.0.smithi093.stdout:1901: writing smithi093153278-424 from 413802 to 847673 tid 1 2023-08-27T20:53:26.879 INFO:tasks.rados.rados.0.smithi093.stdout:1901: writing smithi093153278-424 from 1420314 to 1896419 tid 2 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1901: writing smithi093153278-424 from 2482204 to 3273189 tid 3 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1902: read oid 322 snap -1 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1902: expect (ObjNum 867 snap 0 seq_num 867) 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1903: read oid 116 snap -1 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1903: expect (ObjNum 115 snap 0 seq_num 115) 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1904: delete oid 333 current snap is 0 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1895: finishing write tid 5 to smithi093153278-169 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 169 v 282 (ObjNum 1283 snap 0 seq_num 1283) dirty exists 2023-08-27T20:53:26.887 INFO:tasks.rados.rados.0.smithi093.stdout:1895: left oid 169 (ObjNum 1283 snap 0 seq_num 1283) 2023-08-27T20:53:26.888 INFO:tasks.rados.rados.0.smithi093.stdout:1896: expect (ObjNum 96 snap 0 seq_num 96) 2023-08-27T20:53:27.002 INFO:tasks.rados.rados.0.smithi093.stdout:1897: finishing write tid 1 to smithi093153278-457 2023-08-27T20:53:27.002 INFO:tasks.rados.rados.0.smithi093.stdout:1897: finishing write tid 2 to smithi093153278-457 2023-08-27T20:53:27.002 INFO:tasks.rados.rados.0.smithi093.stdout:1897: finishing write tid 3 to smithi093153278-457 2023-08-27T20:53:27.002 INFO:tasks.rados.rados.0.smithi093.stdout:1899: expect (ObjNum 271 snap 0 seq_num 271) 2023-08-27T20:53:27.083 INFO:tasks.rados.rados.0.smithi093.stdout:1897: finishing write tid 4 to smithi093153278-457 2023-08-27T20:53:27.083 INFO:tasks.rados.rados.0.smithi093.stdout:1897: finishing write tid 5 to smithi093153278-457 2023-08-27T20:53:27.083 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 457 v 297 (ObjNum 1284 snap 0 seq_num 1284) dirty exists 2023-08-27T20:53:27.083 INFO:tasks.rados.rados.0.smithi093.stdout:1897: left oid 457 (ObjNum 1284 snap 0 seq_num 1284) 2023-08-27T20:53:27.083 INFO:tasks.rados.rados.0.smithi093.stdout:1898: expect (ObjNum 1076 snap 0 seq_num 1076) 2023-08-27T20:53:27.162 INFO:tasks.rados.rados.0.smithi093.stdout:1900: finishing write tid 1 to smithi093153278-403 2023-08-27T20:53:27.162 INFO:tasks.rados.rados.0.smithi093.stdout:1900: finishing write tid 2 to smithi093153278-403 2023-08-27T20:53:27.162 INFO:tasks.rados.rados.0.smithi093.stdout:1900: finishing write tid 3 to smithi093153278-403 2023-08-27T20:53:27.162 INFO:tasks.rados.rados.0.smithi093.stdout:1900: finishing write tid 4 to smithi093153278-403 2023-08-27T20:53:27.162 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 403 v 250 (ObjNum 1285 snap 0 seq_num 1285) dirty exists 2023-08-27T20:53:27.162 INFO:tasks.rados.rados.0.smithi093.stdout:1900: left oid 403 (ObjNum 1285 snap 0 seq_num 1285) 2023-08-27T20:53:27.162 INFO:tasks.rados.rados.0.smithi093.stdout:1895: done (9 left) 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1896: done (8 left) 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1897: done (7 left) 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1898: done (6 left) 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1899: done (5 left) 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1900: done (4 left) 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1904: done (3 left) 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1905: write oid 87 current snap is 0 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1901: finishing write tid 1 to smithi093153278-424 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1901: finishing write tid 2 to smithi093153278-424 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1901: finishing write tid 3 to smithi093153278-424 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1901: finishing write tid 4 to smithi093153278-424 2023-08-27T20:53:27.163 INFO:tasks.rados.rados.0.smithi093.stdout:1905: seq_num 1288 ranges {447416=584369,1537278=479774,2537050=529882,3524784=122632} 2023-08-27T20:53:27.167 INFO:tasks.rados.rados.0.smithi093.stdout:1905: writing smithi093153278-87 from 447416 to 1031785 tid 1 2023-08-27T20:53:27.172 INFO:tasks.rados.rados.0.smithi093.stdout:1905: writing smithi093153278-87 from 1537278 to 2017052 tid 2 2023-08-27T20:53:27.178 INFO:tasks.rados.rados.0.smithi093.stdout:1905: writing smithi093153278-87 from 2537050 to 3066932 tid 3 2023-08-27T20:53:27.181 INFO:tasks.rados.rados.0.smithi093.stdout:1905: writing smithi093153278-87 from 3524784 to 3647416 tid 4 2023-08-27T20:53:27.181 INFO:tasks.rados.rados.0.smithi093.stdout:1906: delete oid 428 current snap is 0 2023-08-27T20:53:27.182 INFO:tasks.rados.rados.0.smithi093.stdout:1901: finishing write tid 5 to smithi093153278-424 2023-08-27T20:53:27.182 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 424 v 308 (ObjNum 1286 snap 0 seq_num 1286) dirty exists 2023-08-27T20:53:27.182 INFO:tasks.rados.rados.0.smithi093.stdout:1901: left oid 424 (ObjNum 1286 snap 0 seq_num 1286) 2023-08-27T20:53:27.182 INFO:tasks.rados.rados.0.smithi093.stdout:1902: expect (ObjNum 867 snap 0 seq_num 867) 2023-08-27T20:53:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1915000606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1915000606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:27.320 INFO:tasks.rados.rados.0.smithi093.stdout:1903: expect (ObjNum 115 snap 0 seq_num 115) 2023-08-27T20:53:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1915000606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:27.442 INFO:tasks.rados.rados.0.smithi093.stdout:1905: finishing write tid 1 to smithi093153278-87 2023-08-27T20:53:27.442 INFO:tasks.rados.rados.0.smithi093.stdout:1905: finishing write tid 2 to smithi093153278-87 2023-08-27T20:53:27.442 INFO:tasks.rados.rados.0.smithi093.stdout:1905: finishing write tid 3 to smithi093153278-87 2023-08-27T20:53:27.442 INFO:tasks.rados.rados.0.smithi093.stdout:1901: done (4 left) 2023-08-27T20:53:27.442 INFO:tasks.rados.rados.0.smithi093.stdout:1902: done (3 left) 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1903: done (2 left) 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1906: done (1 left) 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1907: read oid 347 snap -1 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1907: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1908: read oid 363 snap -1 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1908: expect (ObjNum 1083 snap 0 seq_num 1083) 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1909: write oid 340 current snap is 0 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1905: finishing write tid 4 to smithi093153278-87 2023-08-27T20:53:27.443 INFO:tasks.rados.rados.0.smithi093.stdout:1905: finishing write tid 5 to smithi093153278-87 2023-08-27T20:53:27.444 INFO:tasks.rados.rados.0.smithi093.stdout:1909: seq_num 1290 ranges {481030=734867,1937889=483443,2875543=552426} 2023-08-27T20:53:27.448 INFO:tasks.rados.rados.0.smithi093.stdout:1909: writing smithi093153278-340 from 481030 to 1215897 tid 1 2023-08-27T20:53:27.455 INFO:tasks.rados.rados.0.smithi093.stdout:1909: writing smithi093153278-340 from 1937889 to 2421332 tid 2 2023-08-27T20:53:27.461 INFO:tasks.rados.rados.0.smithi093.stdout:1909: writing smithi093153278-340 from 2875543 to 3427969 tid 3 2023-08-27T20:53:27.461 INFO:tasks.rados.rados.0.smithi093.stdout:1910: read oid 211 snap -1 2023-08-27T20:53:27.461 INFO:tasks.rados.rados.0.smithi093.stdout:1910: expect (ObjNum 210 snap 0 seq_num 210) 2023-08-27T20:53:27.461 INFO:tasks.rados.rados.0.smithi093.stdout:1911: write oid 320 current snap is 0 2023-08-27T20:53:27.461 INFO:tasks.rados.rados.0.smithi093.stdout:1905: finishing write tid 6 to smithi093153278-87 2023-08-27T20:53:27.462 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 247 (ObjNum 1288 snap 0 seq_num 1288) dirty exists 2023-08-27T20:53:27.462 INFO:tasks.rados.rados.0.smithi093.stdout:1905: left oid 87 (ObjNum 1288 snap 0 seq_num 1288) 2023-08-27T20:53:27.462 INFO:tasks.rados.rados.0.smithi093.stdout:1911: seq_num 1291 ranges {497837=410116,1596371=743454,2902966=763698} 2023-08-27T20:53:27.466 INFO:tasks.rados.rados.0.smithi093.stdout:1911: writing smithi093153278-320 from 497837 to 907953 tid 1 2023-08-27T20:53:27.474 INFO:tasks.rados.rados.0.smithi093.stdout:1911: writing smithi093153278-320 from 1596371 to 2339825 tid 2 2023-08-27T20:53:27.481 INFO:tasks.rados.rados.0.smithi093.stdout:1911: writing smithi093153278-320 from 2902966 to 3666664 tid 3 2023-08-27T20:53:27.482 INFO:tasks.rados.rados.0.smithi093.stdout:1905: done (5 left) 2023-08-27T20:53:27.482 INFO:tasks.rados.rados.0.smithi093.stdout:1912: read oid 376 snap -1 2023-08-27T20:53:27.482 INFO:tasks.rados.rados.0.smithi093.stdout:1912: expect (ObjNum 1039 snap 0 seq_num 1039) 2023-08-27T20:53:27.482 INFO:tasks.rados.rados.0.smithi093.stdout:1913: read oid 47 snap -1 2023-08-27T20:53:27.482 INFO:tasks.rados.rados.0.smithi093.stdout:1913: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:53:27.482 INFO:tasks.rados.rados.0.smithi093.stdout:1914: delete oid 419 current snap is 0 2023-08-27T20:53:27.482 INFO:tasks.rados.rados.0.smithi093.stdout:1907: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:53:27.601 INFO:tasks.rados.rados.0.smithi093.stdout:1908: expect (ObjNum 1083 snap 0 seq_num 1083) 2023-08-27T20:53:27.715 INFO:tasks.rados.rados.0.smithi093.stdout:1909: finishing write tid 1 to smithi093153278-340 2023-08-27T20:53:27.715 INFO:tasks.rados.rados.0.smithi093.stdout:1909: finishing write tid 2 to smithi093153278-340 2023-08-27T20:53:27.715 INFO:tasks.rados.rados.0.smithi093.stdout:1907: done (7 left) 2023-08-27T20:53:27.715 INFO:tasks.rados.rados.0.smithi093.stdout:1908: done (6 left) 2023-08-27T20:53:27.715 INFO:tasks.rados.rados.0.smithi093.stdout:1914: done (5 left) 2023-08-27T20:53:27.716 INFO:tasks.rados.rados.0.smithi093.stdout:1915: write oid 317 current snap is 0 2023-08-27T20:53:27.716 INFO:tasks.rados.rados.0.smithi093.stdout:1909: finishing write tid 3 to smithi093153278-340 2023-08-27T20:53:27.716 INFO:tasks.rados.rados.0.smithi093.stdout:1909: finishing write tid 4 to smithi093153278-340 2023-08-27T20:53:27.716 INFO:tasks.rados.rados.0.smithi093.stdout:1909: finishing write tid 5 to smithi093153278-340 2023-08-27T20:53:27.716 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 340 v 251 (ObjNum 1290 snap 0 seq_num 1290) dirty exists 2023-08-27T20:53:27.716 INFO:tasks.rados.rados.0.smithi093.stdout:1909: left oid 340 (ObjNum 1290 snap 0 seq_num 1290) 2023-08-27T20:53:27.716 INFO:tasks.rados.rados.0.smithi093.stdout:1910: expect (ObjNum 210 snap 0 seq_num 210) 2023-08-27T20:53:27.824 INFO:tasks.rados.rados.0.smithi093.stdout:1911: finishing write tid 1 to smithi093153278-320 2023-08-27T20:53:27.824 INFO:tasks.rados.rados.0.smithi093.stdout:1911: finishing write tid 2 to smithi093153278-320 2023-08-27T20:53:27.824 INFO:tasks.rados.rados.0.smithi093.stdout:1915: seq_num 1293 ranges {531451=676967,1713335=747123,2957812=502595} 2023-08-27T20:53:27.830 INFO:tasks.rados.rados.0.smithi093.stdout:1915: writing smithi093153278-317 from 531451 to 1208418 tid 1 2023-08-27T20:53:27.837 INFO:tasks.rados.rados.0.smithi093.stdout:1915: writing smithi093153278-317 from 1713335 to 2460458 tid 2 2023-08-27T20:53:27.842 INFO:tasks.rados.rados.0.smithi093.stdout:1915: writing smithi093153278-317 from 2957812 to 3460407 tid 3 2023-08-27T20:53:27.843 INFO:tasks.rados.rados.0.smithi093.stdout:1909: done (5 left) 2023-08-27T20:53:27.843 INFO:tasks.rados.rados.0.smithi093.stdout:1910: done (4 left) 2023-08-27T20:53:27.843 INFO:tasks.rados.rados.0.smithi093.stdout:1916: delete oid 335 current snap is 0 2023-08-27T20:53:27.843 INFO:tasks.rados.rados.0.smithi093.stdout:1913: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:53:27.928 INFO:tasks.rados.rados.0.smithi093.stdout:1911: finishing write tid 3 to smithi093153278-320 2023-08-27T20:53:27.928 INFO:tasks.rados.rados.0.smithi093.stdout:1911: finishing write tid 4 to smithi093153278-320 2023-08-27T20:53:27.929 INFO:tasks.rados.rados.0.smithi093.stdout:1913: done (4 left) 2023-08-27T20:53:27.929 INFO:tasks.rados.rados.0.smithi093.stdout:1916: done (3 left) 2023-08-27T20:53:27.929 INFO:tasks.rados.rados.0.smithi093.stdout:1917: write oid 134 current snap is 0 2023-08-27T20:53:27.929 INFO:tasks.rados.rados.0.smithi093.stdout:1911: finishing write tid 5 to smithi093153278-320 2023-08-27T20:53:27.929 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 320 v 347 (ObjNum 1291 snap 0 seq_num 1291) dirty exists 2023-08-27T20:53:27.929 INFO:tasks.rados.rados.0.smithi093.stdout:1911: left oid 320 (ObjNum 1291 snap 0 seq_num 1291) 2023-08-27T20:53:27.929 INFO:tasks.rados.rados.0.smithi093.stdout:1912: expect (ObjNum 1039 snap 0 seq_num 1039) 2023-08-27T20:53:28.112 INFO:tasks.rados.rados.0.smithi093.stdout:1915: finishing write tid 1 to smithi093153278-317 2023-08-27T20:53:28.112 INFO:tasks.rados.rados.0.smithi093.stdout:1915: finishing write tid 2 to smithi093153278-317 2023-08-27T20:53:28.112 INFO:tasks.rados.rados.0.smithi093.stdout:1915: finishing write tid 3 to smithi093153278-317 2023-08-27T20:53:28.112 INFO:tasks.rados.rados.0.smithi093.stdout:1915: finishing write tid 4 to smithi093153278-317 2023-08-27T20:53:28.112 INFO:tasks.rados.rados.0.smithi093.stdout:1915: finishing write tid 5 to smithi093153278-317 2023-08-27T20:53:28.112 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 317 v 256 (ObjNum 1293 snap 0 seq_num 1293) dirty exists 2023-08-27T20:53:28.113 INFO:tasks.rados.rados.0.smithi093.stdout:1915: left oid 317 (ObjNum 1293 snap 0 seq_num 1293) 2023-08-27T20:53:28.113 INFO:tasks.rados.rados.0.smithi093.stdout:1917: seq_num 1295 ranges {565065=427465,1430299=634439,2496305=525139,3472104=292961} 2023-08-27T20:53:28.117 INFO:tasks.rados.rados.0.smithi093.stdout:1917: writing smithi093153278-134 from 565065 to 992530 tid 1 2023-08-27T20:53:28.123 INFO:tasks.rados.rados.0.smithi093.stdout:1917: writing smithi093153278-134 from 1430299 to 2064738 tid 2 2023-08-27T20:53:28.129 INFO:tasks.rados.rados.0.smithi093.stdout:1917: writing smithi093153278-134 from 2496305 to 3021444 tid 3 2023-08-27T20:53:28.133 INFO:tasks.rados.rados.0.smithi093.stdout:1917: writing smithi093153278-134 from 3472104 to 3765065 tid 4 2023-08-27T20:53:28.133 INFO:tasks.rados.rados.0.smithi093.stdout:1911: done (3 left) 2023-08-27T20:53:28.133 INFO:tasks.rados.rados.0.smithi093.stdout:1912: done (2 left) 2023-08-27T20:53:28.133 INFO:tasks.rados.rados.0.smithi093.stdout:1915: done (1 left) 2023-08-27T20:53:28.133 INFO:tasks.rados.rados.0.smithi093.stdout:1918: delete oid 90 current snap is 0 2023-08-27T20:53:28.133 INFO:tasks.rados.rados.0.smithi093.stdout:1917: finishing write tid 1 to smithi093153278-134 2023-08-27T20:53:28.134 INFO:tasks.rados.rados.0.smithi093.stdout:1917: finishing write tid 2 to smithi093153278-134 2023-08-27T20:53:28.134 INFO:tasks.rados.rados.0.smithi093.stdout:1917: finishing write tid 3 to smithi093153278-134 2023-08-27T20:53:28.134 INFO:tasks.rados.rados.0.smithi093.stdout:1918: done (1 left) 2023-08-27T20:53:28.134 INFO:tasks.rados.rados.0.smithi093.stdout:1919: read oid 486 snap -1 2023-08-27T20:53:28.135 INFO:tasks.rados.rados.0.smithi093.stdout:1919: expect (ObjNum 634 snap 0 seq_num 634) 2023-08-27T20:53:28.135 INFO:tasks.rados.rados.0.smithi093.stdout:1920: write oid 382 current snap is 0 2023-08-27T20:53:28.135 INFO:tasks.rados.rados.0.smithi093.stdout:1920: seq_num 1297 ranges {598679=577963,1947263=638108,3351151=447528} 2023-08-27T20:53:28.141 INFO:tasks.rados.rados.0.smithi093.stdout:1920: writing smithi093153278-382 from 598679 to 1176642 tid 1 2023-08-27T20:53:28.149 INFO:tasks.rados.rados.0.smithi093.stdout:1920: writing smithi093153278-382 from 1947263 to 2585371 tid 2 2023-08-27T20:53:28.155 INFO:tasks.rados.rados.0.smithi093.stdout:1920: writing smithi093153278-382 from 3351151 to 3798679 tid 3 2023-08-27T20:53:28.156 INFO:tasks.rados.rados.0.smithi093.stdout:1921: write oid 301 current snap is 0 2023-08-27T20:53:28.156 INFO:tasks.rados.rados.0.smithi093.stdout:1917: finishing write tid 4 to smithi093153278-134 2023-08-27T20:53:28.156 INFO:tasks.rados.rados.0.smithi093.stdout:1921: seq_num 1298 ranges {615486=653212,1889392=498119,2978574=475308} 2023-08-27T20:53:28.162 INFO:tasks.rados.rados.0.smithi093.stdout:1921: writing smithi093153278-301 from 615486 to 1268698 tid 1 2023-08-27T20:53:28.168 INFO:tasks.rados.rados.0.smithi093.stdout:1921: writing smithi093153278-301 from 1889392 to 2387511 tid 2 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1921: writing smithi093153278-301 from 2978574 to 3453882 tid 3 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1922: read oid 184 snap -1 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1922: expect (ObjNum 183 snap 0 seq_num 183) 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1923: write oid 288 current snap is 0 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1917: finishing write tid 5 to smithi093153278-134 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1917: finishing write tid 6 to smithi093153278-134 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 134 v 263 (ObjNum 1295 snap 0 seq_num 1295) dirty exists 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1917: left oid 134 (ObjNum 1295 snap 0 seq_num 1295) 2023-08-27T20:53:28.175 INFO:tasks.rados.rados.0.smithi093.stdout:1919: expect (ObjNum 634 snap 0 seq_num 634) 2023-08-27T20:53:28.326 INFO:tasks.rados.rados.0.smithi093.stdout:1920: finishing write tid 1 to smithi093153278-382 2023-08-27T20:53:28.326 INFO:tasks.rados.rados.0.smithi093.stdout:1920: finishing write tid 2 to smithi093153278-382 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1920: finishing write tid 3 to smithi093153278-382 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1920: finishing write tid 4 to smithi093153278-382 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1920: finishing write tid 5 to smithi093153278-382 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 382 v 314 (ObjNum 1297 snap 0 seq_num 1297) dirty exists 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1920: left oid 382 (ObjNum 1297 snap 0 seq_num 1297) 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1921: finishing write tid 1 to smithi093153278-301 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1921: finishing write tid 2 to smithi093153278-301 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1921: finishing write tid 3 to smithi093153278-301 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1921: finishing write tid 4 to smithi093153278-301 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1921: finishing write tid 5 to smithi093153278-301 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 301 v 295 (ObjNum 1298 snap 0 seq_num 1298) dirty exists 2023-08-27T20:53:28.327 INFO:tasks.rados.rados.0.smithi093.stdout:1921: left oid 301 (ObjNum 1298 snap 0 seq_num 1298) 2023-08-27T20:53:28.328 INFO:tasks.rados.rados.0.smithi093.stdout:1922: expect (ObjNum 183 snap 0 seq_num 183) 2023-08-27T20:53:28.421 INFO:tasks.rados.rados.0.smithi093.stdout:1923: seq_num 1299 ranges {632293=728461,1947874=641777,3005997=686580} 2023-08-27T20:53:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:1923: writing smithi093153278-288 from 632293 to 1360754 tid 1 2023-08-27T20:53:28.435 INFO:tasks.rados.rados.0.smithi093.stdout:1923: writing smithi093153278-288 from 1947874 to 2589651 tid 2 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1923: writing smithi093153278-288 from 3005997 to 3692577 tid 3 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1917: done (5 left) 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1919: done (4 left) 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1920: done (3 left) 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1921: done (2 left) 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1922: done (1 left) 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1924: write oid 296 current snap is 0 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1923: finishing write tid 1 to smithi093153278-288 2023-08-27T20:53:28.441 INFO:tasks.rados.rados.0.smithi093.stdout:1923: finishing write tid 2 to smithi093153278-288 2023-08-27T20:53:28.442 INFO:tasks.rados.rados.0.smithi093.stdout:1924: seq_num 1300 ranges {649100=403710,1606356=501788,2633420=497852,3636005=213095} 2023-08-27T20:53:28.446 INFO:tasks.rados.rados.0.smithi093.stdout:1924: writing smithi093153278-296 from 649100 to 1052810 tid 1 2023-08-27T20:53:28.452 INFO:tasks.rados.rados.0.smithi093.stdout:1924: writing smithi093153278-296 from 1606356 to 2108144 tid 2 2023-08-27T20:53:28.457 INFO:tasks.rados.rados.0.smithi093.stdout:1924: writing smithi093153278-296 from 2633420 to 3131272 tid 3 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1924: writing smithi093153278-296 from 3636005 to 3849100 tid 4 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1925: delete oid 357 current snap is 0 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1923: finishing write tid 3 to smithi093153278-288 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1923: finishing write tid 4 to smithi093153278-288 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1923: finishing write tid 5 to smithi093153278-288 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 288 v 234 (ObjNum 1299 snap 0 seq_num 1299) dirty exists 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1923: left oid 288 (ObjNum 1299 snap 0 seq_num 1299) 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1924: finishing write tid 1 to smithi093153278-296 2023-08-27T20:53:28.462 INFO:tasks.rados.rados.0.smithi093.stdout:1924: finishing write tid 2 to smithi093153278-296 2023-08-27T20:53:28.463 INFO:tasks.rados.rados.0.smithi093.stdout:1924: finishing write tid 3 to smithi093153278-296 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1923: done (2 left) 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1925: done (1 left) 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1926: read oid 166 snap -1 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1926: expect (ObjNum 1041 snap 0 seq_num 1041) 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1927: read oid 87 snap -1 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1927: expect (ObjNum 1288 snap 0 seq_num 1288) 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1928: delete oid 405 current snap is 0 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1924: finishing write tid 4 to smithi093153278-296 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1924: finishing write tid 5 to smithi093153278-296 2023-08-27T20:53:28.464 INFO:tasks.rados.rados.0.smithi093.stdout:1924: finishing write tid 6 to smithi093153278-296 2023-08-27T20:53:28.465 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 296 v 236 (ObjNum 1300 snap 0 seq_num 1300) dirty exists 2023-08-27T20:53:28.465 INFO:tasks.rados.rados.0.smithi093.stdout:1924: left oid 296 (ObjNum 1300 snap 0 seq_num 1300) 2023-08-27T20:53:28.465 INFO:tasks.rados.rados.0.smithi093.stdout:1924: done (3 left) 2023-08-27T20:53:28.465 INFO:tasks.rados.rados.0.smithi093.stdout:1928: done (2 left) 2023-08-27T20:53:28.466 INFO:tasks.rados.rados.0.smithi093.stdout:1929: read oid 378 snap -1 2023-08-27T20:53:28.466 INFO:tasks.rados.rados.0.smithi093.stdout:1929: expect (ObjNum 938 snap 0 seq_num 938) 2023-08-27T20:53:28.466 INFO:tasks.rados.rados.0.smithi093.stdout:1930: write oid 263 current snap is 0 2023-08-27T20:53:28.466 INFO:tasks.rados.rados.0.smithi093.stdout:1930: seq_num 1303 ranges {699521=745810,2181802=649115,3515689=383832} 2023-08-27T20:53:28.473 INFO:tasks.rados.rados.0.smithi093.stdout:1930: writing smithi093153278-263 from 699521 to 1445331 tid 1 2023-08-27T20:53:28.481 INFO:tasks.rados.rados.0.smithi093.stdout:1930: writing smithi093153278-263 from 2181802 to 2830917 tid 2 2023-08-27T20:53:28.487 INFO:tasks.rados.rados.0.smithi093.stdout:1930: writing smithi093153278-263 from 3515689 to 3899521 tid 3 2023-08-27T20:53:28.487 INFO:tasks.rados.rados.0.smithi093.stdout:1931: write oid 51 current snap is 0 2023-08-27T20:53:28.487 INFO:tasks.rados.rados.0.smithi093.stdout:1931: seq_num 1304 ranges {716328=421059,1840284=509126,3143112=659293} 2023-08-27T20:53:28.493 INFO:tasks.rados.rados.0.smithi093.stdout:1931: writing smithi093153278-51 from 716328 to 1137387 tid 1 2023-08-27T20:53:28.500 INFO:tasks.rados.rados.0.smithi093.stdout:1931: writing smithi093153278-51 from 1840284 to 2349410 tid 2 2023-08-27T20:53:28.508 INFO:tasks.rados.rados.0.smithi093.stdout:1931: writing smithi093153278-51 from 3143112 to 3802405 tid 3 2023-08-27T20:53:28.508 INFO:tasks.rados.rados.0.smithi093.stdout:1932: read oid 101 snap -1 2023-08-27T20:53:28.509 INFO:tasks.rados.rados.0.smithi093.stdout:1932: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:53:28.509 INFO:tasks.rados.rados.0.smithi093.stdout:1933: write oid 296 current snap is 0 2023-08-27T20:53:28.509 INFO:tasks.rados.rados.0.smithi093.stdout:1926: expect (ObjNum 1041 snap 0 seq_num 1041) 2023-08-27T20:53:28.656 INFO:tasks.rados.rados.0.smithi093.stdout:1929: expect (ObjNum 938 snap 0 seq_num 938) 2023-08-27T20:53:28.894 INFO:tasks.rados.rados.0.smithi093.stdout:1927: expect (ObjNum 1288 snap 0 seq_num 1288) 2023-08-27T20:53:29.080 INFO:tasks.rados.rados.0.smithi093.stdout:1930: finishing write tid 1 to smithi093153278-263 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:1930: finishing write tid 2 to smithi093153278-263 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:1930: finishing write tid 3 to smithi093153278-263 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:1930: finishing write tid 4 to smithi093153278-263 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:1930: finishing write tid 5 to smithi093153278-263 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 263 v 318 (ObjNum 1303 snap 0 seq_num 1303) dirty exists 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:1930: left oid 263 (ObjNum 1303 snap 0 seq_num 1303) 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:1931: finishing write tid 1 to smithi093153278-51 2023-08-27T20:53:29.081 INFO:tasks.rados.rados.0.smithi093.stdout:1933: seq_num 1305 ranges {733135=496308,1898766=652784,3170535=470565} 2023-08-27T20:53:29.087 INFO:tasks.rados.rados.0.smithi093.stdout:1933: writing smithi093153278-296 from 733135 to 1229443 tid 1 2023-08-27T20:53:29.094 INFO:tasks.rados.rados.0.smithi093.stdout:1933: writing smithi093153278-296 from 1898766 to 2551550 tid 2 2023-08-27T20:53:29.101 INFO:tasks.rados.rados.0.smithi093.stdout:1933: writing smithi093153278-296 from 3170535 to 3641100 tid 3 2023-08-27T20:53:29.101 INFO:tasks.rados.rados.0.smithi093.stdout:1926: done (6 left) 2023-08-27T20:53:29.101 INFO:tasks.rados.rados.0.smithi093.stdout:1927: done (5 left) 2023-08-27T20:53:29.101 INFO:tasks.rados.rados.0.smithi093.stdout:1929: done (4 left) 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1930: done (3 left) 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1934: read oid 274 snap -1 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1934: expect (ObjNum 1216 snap 0 seq_num 1216) 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1935: read oid 154 snap -1 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1935: expect (ObjNum 1263 snap 0 seq_num 1263) 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1936: write oid 102 current snap is 0 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1931: finishing write tid 2 to smithi093153278-51 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1931: finishing write tid 3 to smithi093153278-51 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1931: finishing write tid 4 to smithi093153278-51 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:1931: finishing write tid 5 to smithi093153278-51 2023-08-27T20:53:29.102 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 51 v 255 (ObjNum 1304 snap 0 seq_num 1304) dirty exists 2023-08-27T20:53:29.103 INFO:tasks.rados.rados.0.smithi093.stdout:1931: left oid 51 (ObjNum 1304 snap 0 seq_num 1304) 2023-08-27T20:53:29.103 INFO:tasks.rados.rados.0.smithi093.stdout:1936: seq_num 1306 ranges {749942=571557,1840895=796442,3365252=584690} 2023-08-27T20:53:29.108 INFO:tasks.rados.rados.0.smithi093.stdout:1936: writing smithi093153278-102 from 749942 to 1321499 tid 1 2023-08-27T20:53:29.115 INFO:tasks.rados.rados.0.smithi093.stdout:1936: writing smithi093153278-102 from 1840895 to 2637337 tid 2 2023-08-27T20:53:29.123 INFO:tasks.rados.rados.0.smithi093.stdout:1936: writing smithi093153278-102 from 3365252 to 3949942 tid 3 2023-08-27T20:53:29.123 INFO:tasks.rados.rados.0.smithi093.stdout:1931: done (5 left) 2023-08-27T20:53:29.123 INFO:tasks.rados.rados.0.smithi093.stdout:1937: write oid 21 current snap is 0 2023-08-27T20:53:29.123 INFO:tasks.rados.rados.0.smithi093.stdout:1937: seq_num 1307 ranges {766749=646806,1899377=656453,3109028=609462} 2023-08-27T20:53:29.130 INFO:tasks.rados.rados.0.smithi093.stdout:1937: writing smithi093153278-21 from 766749 to 1413555 tid 1 2023-08-27T20:53:29.137 INFO:tasks.rados.rados.0.smithi093.stdout:1937: writing smithi093153278-21 from 1899377 to 2555830 tid 2 2023-08-27T20:53:29.144 INFO:tasks.rados.rados.0.smithi093.stdout:1937: writing smithi093153278-21 from 3109028 to 3718490 tid 3 2023-08-27T20:53:29.144 INFO:tasks.rados.rados.0.smithi093.stdout:1938: read oid 169 snap -1 2023-08-27T20:53:29.144 INFO:tasks.rados.rados.0.smithi093.stdout:1938: expect (ObjNum 1283 snap 0 seq_num 1283) 2023-08-27T20:53:29.144 INFO:tasks.rados.rados.0.smithi093.stdout:1939: read oid 397 snap -1 2023-08-27T20:53:29.144 INFO:tasks.rados.rados.0.smithi093.stdout:1939: expect deleted 2023-08-27T20:53:29.144 INFO:tasks.rados.rados.0.smithi093.stdout:1940: write oid 248 current snap is 0 2023-08-27T20:53:29.144 INFO:tasks.rados.rados.0.smithi093.stdout:1932: expect (ObjNum 100 snap 0 seq_num 100) 2023-08-27T20:53:29.257 INFO:tasks.rados.rados.0.smithi093.stdout:1933: finishing write tid 1 to smithi093153278-296 2023-08-27T20:53:29.257 INFO:tasks.rados.rados.0.smithi093.stdout:1940: seq_num 1308 ranges {783556=438408,1674212=400111,2736451=420734,3882270=101286} 2023-08-27T20:53:29.264 INFO:tasks.rados.rados.0.smithi093.stdout:1940: writing smithi093153278-248 from 783556 to 1221964 tid 1 2023-08-27T20:53:29.268 INFO:tasks.rados.rados.0.smithi093.stdout:1940: writing smithi093153278-248 from 1674212 to 2074323 tid 2 2023-08-27T20:53:29.274 INFO:tasks.rados.rados.0.smithi093.stdout:1940: writing smithi093153278-248 from 2736451 to 3157185 tid 3 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1940: writing smithi093153278-248 from 3882270 to 3983556 tid 4 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1932: done (8 left) 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1941: read oid 333 snap -1 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1941: expect deleted 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1942: read oid 240 snap -1 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1942: expect (ObjNum 557 snap 0 seq_num 557) 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1943: read oid 81 snap -1 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1943: expect (ObjNum 1279 snap 0 seq_num 1279) 2023-08-27T20:53:29.280 INFO:tasks.rados.rados.0.smithi093.stdout:1944: read oid 74 snap -1 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1944: expect (ObjNum 597 snap 0 seq_num 597) 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1945: read oid 155 snap -1 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1945: expect (ObjNum 1151 snap 0 seq_num 1151) 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1946: read oid 120 snap -1 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1946: expect (ObjNum 1235 snap 0 seq_num 1235) 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1947: write oid 281 current snap is 0 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1933: finishing write tid 2 to smithi093153278-296 2023-08-27T20:53:29.281 INFO:tasks.rados.rados.0.smithi093.stdout:1934: expect (ObjNum 1216 snap 0 seq_num 1216) 2023-08-27T20:53:29.419 INFO:tasks.rados.rados.0.smithi093.stdout:1933: finishing write tid 3 to smithi093153278-296 2023-08-27T20:53:29.419 INFO:tasks.rados.rados.0.smithi093.stdout:1935: expect (ObjNum 1263 snap 0 seq_num 1263) 2023-08-27T20:53:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:1933: finishing write tid 4 to smithi093153278-296 2023-08-27T20:53:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:1933: finishing write tid 5 to smithi093153278-296 2023-08-27T20:53:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 296 v 240 (ObjNum 1305 snap 0 seq_num 1305) dirty exists 2023-08-27T20:53:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:1933: left oid 296 (ObjNum 1305 snap 0 seq_num 1305) 2023-08-27T20:53:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:1936: finishing write tid 1 to smithi093153278-102 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1936: finishing write tid 2 to smithi093153278-102 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1936: finishing write tid 3 to smithi093153278-102 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1936: finishing write tid 4 to smithi093153278-102 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1936: finishing write tid 5 to smithi093153278-102 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 102 v 238 (ObjNum 1306 snap 0 seq_num 1306) dirty exists 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1936: left oid 102 (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1937: finishing write tid 1 to smithi093153278-21 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1937: finishing write tid 2 to smithi093153278-21 2023-08-27T20:53:29.603 INFO:tasks.rados.rados.0.smithi093.stdout:1947: seq_num 1309 ranges {400363=513657,1332694=660122} 2023-08-27T20:53:29.607 INFO:tasks.rados.rados.0.smithi093.stdout:1947: writing smithi093153278-281 from 400363 to 914020 tid 1 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1947: writing smithi093153278-281 from 1332694 to 1992816 tid 2 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1933: done (14 left) 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1934: done (13 left) 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1935: done (12 left) 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1936: done (11 left) 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1948: delete oid 435 current snap is 0 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1937: finishing write tid 3 to smithi093153278-21 2023-08-27T20:53:29.613 INFO:tasks.rados.rados.0.smithi093.stdout:1937: finishing write tid 4 to smithi093153278-21 2023-08-27T20:53:29.614 INFO:tasks.rados.rados.0.smithi093.stdout:1937: finishing write tid 5 to smithi093153278-21 2023-08-27T20:53:29.614 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 21 v 288 (ObjNum 1307 snap 0 seq_num 1307) dirty exists 2023-08-27T20:53:29.614 INFO:tasks.rados.rados.0.smithi093.stdout:1937: left oid 21 (ObjNum 1307 snap 0 seq_num 1307) 2023-08-27T20:53:29.614 INFO:tasks.rados.rados.0.smithi093.stdout:1938: expect (ObjNum 1283 snap 0 seq_num 1283) 2023-08-27T20:53:29.874 INFO:tasks.rados.rados.0.smithi093.stdout:1940: finishing write tid 1 to smithi093153278-248 2023-08-27T20:53:29.874 INFO:tasks.rados.rados.0.smithi093.stdout:1940: finishing write tid 2 to smithi093153278-248 2023-08-27T20:53:29.874 INFO:tasks.rados.rados.0.smithi093.stdout:1940: finishing write tid 3 to smithi093153278-248 2023-08-27T20:53:29.874 INFO:tasks.rados.rados.0.smithi093.stdout:1940: finishing write tid 4 to smithi093153278-248 2023-08-27T20:53:29.874 INFO:tasks.rados.rados.0.smithi093.stdout:1940: finishing write tid 5 to smithi093153278-248 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1940: finishing write tid 6 to smithi093153278-248 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 248 v 191 (ObjNum 1308 snap 0 seq_num 1308) dirty exists 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1940: left oid 248 (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1937: done (11 left) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1938: done (10 left) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1939: done (9 left) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1940: done (8 left) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1941: done (7 left) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1948: done (6 left) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1949: read oid 439 snap -1 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1949: expect (ObjNum 1214 snap 0 seq_num 1214) 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1950: delete oid 271 current snap is 0 2023-08-27T20:53:29.875 INFO:tasks.rados.rados.0.smithi093.stdout:1942: expect (ObjNum 557 snap 0 seq_num 557) 2023-08-27T20:53:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4190574803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4190574803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4190574803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:30.025 INFO:tasks.rados.rados.0.smithi093.stdout:1945: expect (ObjNum 1151 snap 0 seq_num 1151) 2023-08-27T20:53:30.199 INFO:tasks.rados.rados.0.smithi093.stdout:1946: expect (ObjNum 1235 snap 0 seq_num 1235) 2023-08-27T20:53:30.315 INFO:tasks.rados.rados.0.smithi093.stdout:1943: expect (ObjNum 1279 snap 0 seq_num 1279) 2023-08-27T20:53:30.539 INFO:tasks.rados.rados.0.smithi093.stdout:1944: expect (ObjNum 597 snap 0 seq_num 597) 2023-08-27T20:53:30.625 INFO:tasks.rados.rados.0.smithi093.stdout:1947: finishing write tid 1 to smithi093153278-281 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1947: finishing write tid 2 to smithi093153278-281 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1942: done (7 left) 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1943: done (6 left) 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1944: done (5 left) 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1945: done (4 left) 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1946: done (3 left) 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1950: done (2 left) 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1951: write oid 342 current snap is 0 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1947: finishing write tid 3 to smithi093153278-281 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:1947: finishing write tid 4 to smithi093153278-281 2023-08-27T20:53:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 281 v 321 (ObjNum 1309 snap 0 seq_num 1309) dirty exists 2023-08-27T20:53:30.627 INFO:tasks.rados.rados.0.smithi093.stdout:1947: left oid 281 (ObjNum 1309 snap 0 seq_num 1309) 2023-08-27T20:53:30.627 INFO:tasks.rados.rados.0.smithi093.stdout:1951: seq_num 1312 ranges {450784=739404,1791787=258997} 2023-08-27T20:53:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:1951: writing smithi093153278-342 from 450784 to 1190188 tid 1 2023-08-27T20:53:30.636 INFO:tasks.rados.rados.0.smithi093.stdout:1951: writing smithi093153278-342 from 1791787 to 2050784 tid 2 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1947: done (2 left) 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1952: read oid 145 snap -1 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1952: expect (ObjNum 144 snap 0 seq_num 144) 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1953: read oid 264 snap -1 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1953: expect deleted 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1954: read oid 215 snap -1 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1954: expect (ObjNum 1220 snap 0 seq_num 1220) 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1955: delete oid 160 current snap is 0 2023-08-27T20:53:30.637 INFO:tasks.rados.rados.0.smithi093.stdout:1949: expect (ObjNum 1214 snap 0 seq_num 1214) 2023-08-27T20:53:30.708 INFO:tasks.rados.rados.0.smithi093.stdout:1951: finishing write tid 1 to smithi093153278-342 2023-08-27T20:53:30.709 INFO:tasks.rados.rados.0.smithi093.stdout:1951: finishing write tid 2 to smithi093153278-342 2023-08-27T20:53:30.709 INFO:tasks.rados.rados.0.smithi093.stdout:1951: finishing write tid 3 to smithi093153278-342 2023-08-27T20:53:30.709 INFO:tasks.rados.rados.0.smithi093.stdout:1951: finishing write tid 4 to smithi093153278-342 2023-08-27T20:53:30.709 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 342 v 298 (ObjNum 1312 snap 0 seq_num 1312) dirty exists 2023-08-27T20:53:30.709 INFO:tasks.rados.rados.0.smithi093.stdout:1951: left oid 342 (ObjNum 1312 snap 0 seq_num 1312) 2023-08-27T20:53:30.709 INFO:tasks.rados.rados.0.smithi093.stdout:1952: expect (ObjNum 144 snap 0 seq_num 144) 2023-08-27T20:53:30.779 INFO:tasks.rados.rados.0.smithi093.stdout:1954: expect (ObjNum 1220 snap 0 seq_num 1220) 2023-08-27T20:53:30.856 INFO:tasks.rados.rados.0.smithi093.stdout:1949: done (5 left) 2023-08-27T20:53:30.856 INFO:tasks.rados.rados.0.smithi093.stdout:1951: done (4 left) 2023-08-27T20:53:30.857 INFO:tasks.rados.rados.0.smithi093.stdout:1952: done (3 left) 2023-08-27T20:53:30.857 INFO:tasks.rados.rados.0.smithi093.stdout:1953: done (2 left) 2023-08-27T20:53:30.857 INFO:tasks.rados.rados.0.smithi093.stdout:1954: done (1 left) 2023-08-27T20:53:30.857 INFO:tasks.rados.rados.0.smithi093.stdout:1955: done (0 left) 2023-08-27T20:53:30.857 INFO:tasks.rados.rados.0.smithi093.stdout:1956: delete oid 84 current snap is 0 2023-08-27T20:53:30.858 INFO:tasks.rados.rados.0.smithi093.stdout:1956: done (0 left) 2023-08-27T20:53:30.858 INFO:tasks.rados.rados.0.smithi093.stdout:1957: delete oid 240 current snap is 0 2023-08-27T20:53:30.859 INFO:tasks.rados.rados.0.smithi093.stdout:1957: done (0 left) 2023-08-27T20:53:30.859 INFO:tasks.rados.rados.0.smithi093.stdout:1958: read oid 194 snap -1 2023-08-27T20:53:30.859 INFO:tasks.rados.rados.0.smithi093.stdout:1958: expect (ObjNum 193 snap 0 seq_num 193) 2023-08-27T20:53:30.859 INFO:tasks.rados.rados.0.smithi093.stdout:1959: write oid 445 current snap is 0 2023-08-27T20:53:30.860 INFO:tasks.rados.rados.0.smithi093.stdout:1959: seq_num 1316 ranges {518012=756753,2025715=92297} 2023-08-27T20:53:30.867 INFO:tasks.rados.rados.0.smithi093.stdout:1959: writing smithi093153278-445 from 518012 to 1274765 tid 1 2023-08-27T20:53:30.872 INFO:tasks.rados.rados.0.smithi093.stdout:1959: writing smithi093153278-445 from 2025715 to 2118012 tid 2 2023-08-27T20:53:30.872 INFO:tasks.rados.rados.0.smithi093.stdout:1960: write oid 332 current snap is 0 2023-08-27T20:53:30.872 INFO:tasks.rados.rados.0.smithi093.stdout:1960: seq_num 1317 ranges {534819=432002,1684197=450622} 2023-08-27T20:53:30.877 INFO:tasks.rados.rados.0.smithi093.stdout:1960: writing smithi093153278-332 from 534819 to 966821 tid 1 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1960: writing smithi093153278-332 from 1684197 to 2134819 tid 2 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1961: read oid 50 snap -1 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1961: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1962: read oid 264 snap -1 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1962: expect deleted 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1963: write oid 333 current snap is 0 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1959: finishing write tid 1 to smithi093153278-445 2023-08-27T20:53:30.884 INFO:tasks.rados.rados.0.smithi093.stdout:1958: expect (ObjNum 193 snap 0 seq_num 193) 2023-08-27T20:53:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:1959: finishing write tid 2 to smithi093153278-445 2023-08-27T20:53:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:1959: finishing write tid 3 to smithi093153278-445 2023-08-27T20:53:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:1959: finishing write tid 4 to smithi093153278-445 2023-08-27T20:53:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 445 v 350 (ObjNum 1316 snap 0 seq_num 1316) dirty exists 2023-08-27T20:53:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:1959: left oid 445 (ObjNum 1316 snap 0 seq_num 1316) 2023-08-27T20:53:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:1960: finishing write tid 1 to smithi093153278-332 2023-08-27T20:53:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:1963: seq_num 1318 ranges {551626=507251,1626326=418456} 2023-08-27T20:53:30.988 INFO:tasks.rados.rados.0.smithi093.stdout:1963: writing smithi093153278-333 from 551626 to 1058877 tid 1 2023-08-27T20:53:30.994 INFO:tasks.rados.rados.0.smithi093.stdout:1963: writing smithi093153278-333 from 1626326 to 2044782 tid 2 2023-08-27T20:53:30.994 INFO:tasks.rados.rados.0.smithi093.stdout:1958: done (5 left) 2023-08-27T20:53:30.994 INFO:tasks.rados.rados.0.smithi093.stdout:1959: done (4 left) 2023-08-27T20:53:30.994 INFO:tasks.rados.rados.0.smithi093.stdout:1964: read oid 380 snap -1 2023-08-27T20:53:30.994 INFO:tasks.rados.rados.0.smithi093.stdout:1964: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:53:30.994 INFO:tasks.rados.rados.0.smithi093.stdout:1965: write oid 424 current snap is 0 2023-08-27T20:53:30.995 INFO:tasks.rados.rados.0.smithi093.stdout:1965: seq_num 1319 ranges {568433=582500,1684808=483625} 2023-08-27T20:53:31.000 INFO:tasks.rados.rados.0.smithi093.stdout:1965: writing smithi093153278-424 from 568433 to 1150933 tid 1 2023-08-27T20:53:31.006 INFO:tasks.rados.rados.0.smithi093.stdout:1965: writing smithi093153278-424 from 1684808 to 2168433 tid 2 2023-08-27T20:53:31.006 INFO:tasks.rados.rados.0.smithi093.stdout:1962: done (5 left) 2023-08-27T20:53:31.007 INFO:tasks.rados.rados.0.smithi093.stdout:1966: write oid 437 current snap is 0 2023-08-27T20:53:31.007 INFO:tasks.rados.rados.0.smithi093.stdout:1960: finishing write tid 2 to smithi093153278-332 2023-08-27T20:53:31.007 INFO:tasks.rados.rados.0.smithi093.stdout:1960: finishing write tid 3 to smithi093153278-332 2023-08-27T20:53:31.007 INFO:tasks.rados.rados.0.smithi093.stdout:1960: finishing write tid 4 to smithi093153278-332 2023-08-27T20:53:31.007 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 332 v 243 (ObjNum 1317 snap 0 seq_num 1317) dirty exists 2023-08-27T20:53:31.007 INFO:tasks.rados.rados.0.smithi093.stdout:1960: left oid 332 (ObjNum 1317 snap 0 seq_num 1317) 2023-08-27T20:53:31.007 INFO:tasks.rados.rados.0.smithi093.stdout:1961: expect (ObjNum 49 snap 0 seq_num 49) 2023-08-27T20:53:31.089 INFO:tasks.rados.rados.0.smithi093.stdout:1963: finishing write tid 1 to smithi093153278-333 2023-08-27T20:53:31.089 INFO:tasks.rados.rados.0.smithi093.stdout:1963: finishing write tid 2 to smithi093153278-333 2023-08-27T20:53:31.089 INFO:tasks.rados.rados.0.smithi093.stdout:1963: finishing write tid 3 to smithi093153278-333 2023-08-27T20:53:31.089 INFO:tasks.rados.rados.0.smithi093.stdout:1963: finishing write tid 4 to smithi093153278-333 2023-08-27T20:53:31.090 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 333 v 256 (ObjNum 1318 snap 0 seq_num 1318) dirty exists 2023-08-27T20:53:31.090 INFO:tasks.rados.rados.0.smithi093.stdout:1963: left oid 333 (ObjNum 1318 snap 0 seq_num 1318) 2023-08-27T20:53:31.090 INFO:tasks.rados.rados.0.smithi093.stdout:1964: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:53:31.252 INFO:tasks.rados.rados.0.smithi093.stdout:1965: finishing write tid 1 to smithi093153278-424 2023-08-27T20:53:31.252 INFO:tasks.rados.rados.0.smithi093.stdout:1965: finishing write tid 2 to smithi093153278-424 2023-08-27T20:53:31.252 INFO:tasks.rados.rados.0.smithi093.stdout:1965: finishing write tid 3 to smithi093153278-424 2023-08-27T20:53:31.252 INFO:tasks.rados.rados.0.smithi093.stdout:1965: finishing write tid 4 to smithi093153278-424 2023-08-27T20:53:31.252 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 424 v 324 (ObjNum 1319 snap 0 seq_num 1319) dirty exists 2023-08-27T20:53:31.252 INFO:tasks.rados.rados.0.smithi093.stdout:1965: left oid 424 (ObjNum 1319 snap 0 seq_num 1319) 2023-08-27T20:53:31.252 INFO:tasks.rados.rados.0.smithi093.stdout:1966: seq_num 1320 ranges {585240=657749,1743290=422125} 2023-08-27T20:53:31.258 INFO:tasks.rados.rados.0.smithi093.stdout:1966: writing smithi093153278-437 from 585240 to 1242989 tid 1 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1966: writing smithi093153278-437 from 1743290 to 2165415 tid 2 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1960: done (5 left) 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1961: done (4 left) 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1963: done (3 left) 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1964: done (2 left) 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1965: done (1 left) 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1967: read oid 142 snap -1 2023-08-27T20:53:31.264 INFO:tasks.rados.rados.0.smithi093.stdout:1967: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:31.265 INFO:tasks.rados.rados.0.smithi093.stdout:1968: write oid 197 current snap is 0 2023-08-27T20:53:31.265 INFO:tasks.rados.rados.0.smithi093.stdout:1966: finishing write tid 1 to smithi093153278-437 2023-08-27T20:53:31.265 INFO:tasks.rados.rados.0.smithi093.stdout:1968: seq_num 1321 ranges {602047=732998,1801772=400275} 2023-08-27T20:53:31.271 INFO:tasks.rados.rados.0.smithi093.stdout:1968: writing smithi093153278-197 from 602047 to 1335045 tid 1 2023-08-27T20:53:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:1968: writing smithi093153278-197 from 1801772 to 2202047 tid 2 2023-08-27T20:53:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:1969: read oid 378 snap -1 2023-08-27T20:53:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:1969: expect (ObjNum 938 snap 0 seq_num 938) 2023-08-27T20:53:31.277 INFO:tasks.rados.rados.0.smithi093.stdout:1970: read oid 106 snap -1 2023-08-27T20:53:31.277 INFO:tasks.rados.rados.0.smithi093.stdout:1970: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:53:31.277 INFO:tasks.rados.rados.0.smithi093.stdout:1967: expect (ObjNum 141 snap 0 seq_num 141) 2023-08-27T20:53:31.354 INFO:tasks.rados.rados.0.smithi093.stdout:1966: finishing write tid 2 to smithi093153278-437 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1966: finishing write tid 3 to smithi093153278-437 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1966: finishing write tid 4 to smithi093153278-437 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 437 v 292 (ObjNum 1320 snap 0 seq_num 1320) dirty exists 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1966: left oid 437 (ObjNum 1320 snap 0 seq_num 1320) 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1968: finishing write tid 1 to smithi093153278-197 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1968: finishing write tid 2 to smithi093153278-197 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1968: finishing write tid 3 to smithi093153278-197 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1968: finishing write tid 4 to smithi093153278-197 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 197 v 259 (ObjNum 1321 snap 0 seq_num 1321) dirty exists 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1968: left oid 197 (ObjNum 1321 snap 0 seq_num 1321) 2023-08-27T20:53:31.355 INFO:tasks.rados.rados.0.smithi093.stdout:1969: expect (ObjNum 938 snap 0 seq_num 938) 2023-08-27T20:53:31.597 INFO:tasks.rados.rados.0.smithi093.stdout:1970: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:53:31.733 INFO:tasks.rados.rados.0.smithi093.stdout:1971: read oid 222 snap -1 2023-08-27T20:53:31.733 INFO:tasks.rados.rados.0.smithi093.stdout:1971: expect (ObjNum 1265 snap 0 seq_num 1265) 2023-08-27T20:53:31.734 INFO:tasks.rados.rados.0.smithi093.stdout:1966: done (5 left) 2023-08-27T20:53:31.734 INFO:tasks.rados.rados.0.smithi093.stdout:1967: done (4 left) 2023-08-27T20:53:31.734 INFO:tasks.rados.rados.0.smithi093.stdout:1968: done (3 left) 2023-08-27T20:53:31.734 INFO:tasks.rados.rados.0.smithi093.stdout:1969: done (2 left) 2023-08-27T20:53:31.734 INFO:tasks.rados.rados.0.smithi093.stdout:1970: done (1 left) 2023-08-27T20:53:31.734 INFO:tasks.rados.rados.0.smithi093.stdout:1972: write oid 37 current snap is 0 2023-08-27T20:53:31.734 INFO:tasks.rados.rados.0.smithi093.stdout:1972: seq_num 1322 ranges {618854=408247,1743901=425794} 2023-08-27T20:53:31.739 INFO:tasks.rados.rados.0.smithi093.stdout:1972: writing smithi093153278-37 from 618854 to 1027101 tid 1 2023-08-27T20:53:31.745 INFO:tasks.rados.rados.0.smithi093.stdout:1972: writing smithi093153278-37 from 1743901 to 2169695 tid 2 2023-08-27T20:53:31.745 INFO:tasks.rados.rados.0.smithi093.stdout:1973: write oid 411 current snap is 0 2023-08-27T20:53:31.745 INFO:tasks.rados.rados.0.smithi093.stdout:1971: expect (ObjNum 1265 snap 0 seq_num 1265) 2023-08-27T20:53:31.886 INFO:tasks.rados.rados.0.smithi093.stdout:1972: finishing write tid 1 to smithi093153278-37 2023-08-27T20:53:31.886 INFO:tasks.rados.rados.0.smithi093.stdout:1972: finishing write tid 2 to smithi093153278-37 2023-08-27T20:53:31.887 INFO:tasks.rados.rados.0.smithi093.stdout:1972: finishing write tid 3 to smithi093153278-37 2023-08-27T20:53:31.887 INFO:tasks.rados.rados.0.smithi093.stdout:1972: finishing write tid 4 to smithi093153278-37 2023-08-27T20:53:31.887 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 37 v 353 (ObjNum 1322 snap 0 seq_num 1322) dirty exists 2023-08-27T20:53:31.887 INFO:tasks.rados.rados.0.smithi093.stdout:1972: left oid 37 (ObjNum 1322 snap 0 seq_num 1322) 2023-08-27T20:53:31.887 INFO:tasks.rados.rados.0.smithi093.stdout:1973: seq_num 1323 ranges {635661=599849,1918736=316925} 2023-08-27T20:53:31.892 INFO:tasks.rados.rados.0.smithi093.stdout:1973: writing smithi093153278-411 from 635661 to 1235510 tid 1 2023-08-27T20:53:31.898 INFO:tasks.rados.rados.0.smithi093.stdout:1973: writing smithi093153278-411 from 1918736 to 2235661 tid 2 2023-08-27T20:53:31.898 INFO:tasks.rados.rados.0.smithi093.stdout:1971: done (2 left) 2023-08-27T20:53:31.898 INFO:tasks.rados.rados.0.smithi093.stdout:1972: done (1 left) 2023-08-27T20:53:31.899 INFO:tasks.rados.rados.0.smithi093.stdout:1974: write oid 212 current snap is 0 2023-08-27T20:53:31.899 INFO:tasks.rados.rados.0.smithi093.stdout:1973: finishing write tid 1 to smithi093153278-411 2023-08-27T20:53:31.899 INFO:tasks.rados.rados.0.smithi093.stdout:1974: seq_num 1324 ranges {652468=675098,1977218=275250} 2023-08-27T20:53:31.905 INFO:tasks.rados.rados.0.smithi093.stdout:1974: writing smithi093153278-212 from 652468 to 1327566 tid 1 2023-08-27T20:53:31.910 INFO:tasks.rados.rados.0.smithi093.stdout:1974: writing smithi093153278-212 from 1977218 to 2252468 tid 2 2023-08-27T20:53:31.911 INFO:tasks.rados.rados.0.smithi093.stdout:1975: write oid 485 current snap is 0 2023-08-27T20:53:31.911 INFO:tasks.rados.rados.0.smithi093.stdout:1973: finishing write tid 2 to smithi093153278-411 2023-08-27T20:53:31.911 INFO:tasks.rados.rados.0.smithi093.stdout:1975: seq_num 1325 ranges {669275=750347,2035700=233575} 2023-08-27T20:53:31.918 INFO:tasks.rados.rados.0.smithi093.stdout:1975: writing smithi093153278-485 from 669275 to 1419622 tid 1 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1975: writing smithi093153278-485 from 2035700 to 2269275 tid 2 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1976: read oid 423 snap -1 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1976: expect (ObjNum 422 snap 0 seq_num 422) 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1977: read oid 2 snap -1 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1977: expect (ObjNum 1149 snap 0 seq_num 1149) 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1978: write oid 162 current snap is 0 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1973: finishing write tid 3 to smithi093153278-411 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:1973: finishing write tid 4 to smithi093153278-411 2023-08-27T20:53:31.924 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 411 v 246 (ObjNum 1323 snap 0 seq_num 1323) dirty exists 2023-08-27T20:53:31.925 INFO:tasks.rados.rados.0.smithi093.stdout:1973: left oid 411 (ObjNum 1323 snap 0 seq_num 1323) 2023-08-27T20:53:31.925 INFO:tasks.rados.rados.0.smithi093.stdout:1974: finishing write tid 1 to smithi093153278-212 2023-08-27T20:53:31.925 INFO:tasks.rados.rados.0.smithi093.stdout:1978: seq_num 1326 ranges {686082=425596,1577829=433132} 2023-08-27T20:53:31.930 INFO:tasks.rados.rados.0.smithi093.stdout:1978: writing smithi093153278-162 from 686082 to 1111678 tid 1 2023-08-27T20:53:31.936 INFO:tasks.rados.rados.0.smithi093.stdout:1978: writing smithi093153278-162 from 1577829 to 2010961 tid 2 2023-08-27T20:53:31.936 INFO:tasks.rados.rados.0.smithi093.stdout:1973: done (5 left) 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1979: read oid 442 snap -1 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1979: expect (ObjNum 840 snap 0 seq_num 840) 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1980: write oid 439 current snap is 0 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1974: finishing write tid 2 to smithi093153278-212 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1974: finishing write tid 3 to smithi093153278-212 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1974: finishing write tid 4 to smithi093153278-212 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 212 v 249 (ObjNum 1324 snap 0 seq_num 1324) dirty exists 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1974: left oid 212 (ObjNum 1324 snap 0 seq_num 1324) 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1975: finishing write tid 1 to smithi093153278-485 2023-08-27T20:53:31.937 INFO:tasks.rados.rados.0.smithi093.stdout:1980: seq_num 1327 ranges {702889=500845,1636311=576790} 2023-08-27T20:53:31.943 INFO:tasks.rados.rados.0.smithi093.stdout:1980: writing smithi093153278-439 from 702889 to 1203734 tid 1 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1980: writing smithi093153278-439 from 1636311 to 2213101 tid 2 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1974: done (6 left) 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1981: write oid 174 current snap is 0 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1975: finishing write tid 2 to smithi093153278-485 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1975: finishing write tid 3 to smithi093153278-485 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1975: finishing write tid 4 to smithi093153278-485 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 485 v 266 (ObjNum 1325 snap 0 seq_num 1325) dirty exists 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1975: left oid 485 (ObjNum 1325 snap 0 seq_num 1325) 2023-08-27T20:53:31.949 INFO:tasks.rados.rados.0.smithi093.stdout:1981: seq_num 1328 ranges {719696=576094,2094793=224903} 2023-08-27T20:53:31.955 INFO:tasks.rados.rados.0.smithi093.stdout:1981: writing smithi093153278-174 from 719696 to 1295790 tid 1 2023-08-27T20:53:31.961 INFO:tasks.rados.rados.0.smithi093.stdout:1981: writing smithi093153278-174 from 2094793 to 2319696 tid 2 2023-08-27T20:53:31.961 INFO:tasks.rados.rados.0.smithi093.stdout:1975: done (6 left) 2023-08-27T20:53:31.961 INFO:tasks.rados.rados.0.smithi093.stdout:1982: write oid 308 current snap is 0 2023-08-27T20:53:31.961 INFO:tasks.rados.rados.0.smithi093.stdout:1982: seq_num 1329 ranges {736503=651343,2153275=183228} 2023-08-27T20:53:31.968 INFO:tasks.rados.rados.0.smithi093.stdout:1982: writing smithi093153278-308 from 736503 to 1387846 tid 1 2023-08-27T20:53:31.974 INFO:tasks.rados.rados.0.smithi093.stdout:1982: writing smithi093153278-308 from 2153275 to 2336503 tid 2 2023-08-27T20:53:31.974 INFO:tasks.rados.rados.0.smithi093.stdout:1983: write oid 417 current snap is 0 2023-08-27T20:53:31.974 INFO:tasks.rados.rados.0.smithi093.stdout:1977: expect (ObjNum 1149 snap 0 seq_num 1149) 2023-08-27T20:53:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1975178072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1975178072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:32.167 INFO:tasks.rados.rados.0.smithi093.stdout:1978: finishing write tid 1 to smithi093153278-162 2023-08-27T20:53:32.167 INFO:tasks.rados.rados.0.smithi093.stdout:1976: expect (ObjNum 422 snap 0 seq_num 422) 2023-08-27T20:53:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1975178072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:32.262 INFO:tasks.rados.rados.0.smithi093.stdout:1979: expect (ObjNum 840 snap 0 seq_num 840) 2023-08-27T20:53:32.343 INFO:tasks.rados.rados.0.smithi093.stdout:1978: finishing write tid 2 to smithi093153278-162 2023-08-27T20:53:32.343 INFO:tasks.rados.rados.0.smithi093.stdout:1978: finishing write tid 3 to smithi093153278-162 2023-08-27T20:53:32.343 INFO:tasks.rados.rados.0.smithi093.stdout:1978: finishing write tid 4 to smithi093153278-162 2023-08-27T20:53:32.343 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 162 v 241 (ObjNum 1326 snap 0 seq_num 1326) dirty exists 2023-08-27T20:53:32.343 INFO:tasks.rados.rados.0.smithi093.stdout:1978: left oid 162 (ObjNum 1326 snap 0 seq_num 1326) 2023-08-27T20:53:32.343 INFO:tasks.rados.rados.0.smithi093.stdout:1980: finishing write tid 1 to smithi093153278-439 2023-08-27T20:53:32.344 INFO:tasks.rados.rados.0.smithi093.stdout:1980: finishing write tid 2 to smithi093153278-439 2023-08-27T20:53:32.344 INFO:tasks.rados.rados.0.smithi093.stdout:1980: finishing write tid 3 to smithi093153278-439 2023-08-27T20:53:32.344 INFO:tasks.rados.rados.0.smithi093.stdout:1983: seq_num 1330 ranges {753310=726592,2095404=257906} 2023-08-27T20:53:32.351 INFO:tasks.rados.rados.0.smithi093.stdout:1983: writing smithi093153278-417 from 753310 to 1479902 tid 1 2023-08-27T20:53:32.356 INFO:tasks.rados.rados.0.smithi093.stdout:1983: writing smithi093153278-417 from 2095404 to 2353310 tid 2 2023-08-27T20:53:32.356 INFO:tasks.rados.rados.0.smithi093.stdout:1976: done (7 left) 2023-08-27T20:53:32.356 INFO:tasks.rados.rados.0.smithi093.stdout:1977: done (6 left) 2023-08-27T20:53:32.356 INFO:tasks.rados.rados.0.smithi093.stdout:1978: done (5 left) 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1979: done (4 left) 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1984: write oid 173 current snap is 0 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1980: finishing write tid 4 to smithi093153278-439 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 439 v 269 (ObjNum 1327 snap 0 seq_num 1327) dirty exists 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1980: left oid 439 (ObjNum 1327 snap 0 seq_num 1327) 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1981: finishing write tid 1 to smithi093153278-174 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1981: finishing write tid 2 to smithi093153278-174 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1981: finishing write tid 3 to smithi093153278-174 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1981: finishing write tid 4 to smithi093153278-174 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 174 v 272 (ObjNum 1328 snap 0 seq_num 1328) dirty exists 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1981: left oid 174 (ObjNum 1328 snap 0 seq_num 1328) 2023-08-27T20:53:32.357 INFO:tasks.rados.rados.0.smithi093.stdout:1982: finishing write tid 1 to smithi093153278-308 2023-08-27T20:53:32.358 INFO:tasks.rados.rados.0.smithi093.stdout:1982: finishing write tid 2 to smithi093153278-308 2023-08-27T20:53:32.358 INFO:tasks.rados.rados.0.smithi093.stdout:1982: finishing write tid 3 to smithi093153278-308 2023-08-27T20:53:32.358 INFO:tasks.rados.rados.0.smithi093.stdout:1982: finishing write tid 4 to smithi093153278-308 2023-08-27T20:53:32.358 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 308 v 259 (ObjNum 1329 snap 0 seq_num 1329) dirty exists 2023-08-27T20:53:32.358 INFO:tasks.rados.rados.0.smithi093.stdout:1982: left oid 308 (ObjNum 1329 snap 0 seq_num 1329) 2023-08-27T20:53:32.358 INFO:tasks.rados.rados.0.smithi093.stdout:1984: seq_num 1331 ranges {770117=518194,1870239=499878} 2023-08-27T20:53:32.362 INFO:tasks.rados.rados.0.smithi093.stdout:1984: writing smithi093153278-173 from 770117 to 1288311 tid 1 2023-08-27T20:53:32.369 INFO:tasks.rados.rados.0.smithi093.stdout:1984: writing smithi093153278-173 from 1870239 to 2370117 tid 2 2023-08-27T20:53:32.369 INFO:tasks.rados.rados.0.smithi093.stdout:1980: done (4 left) 2023-08-27T20:53:32.369 INFO:tasks.rados.rados.0.smithi093.stdout:1981: done (3 left) 2023-08-27T20:53:32.369 INFO:tasks.rados.rados.0.smithi093.stdout:1982: done (2 left) 2023-08-27T20:53:32.369 INFO:tasks.rados.rados.0.smithi093.stdout:1985: write oid 130 current snap is 0 2023-08-27T20:53:32.369 INFO:tasks.rados.rados.0.smithi093.stdout:1983: finishing write tid 1 to smithi093153278-417 2023-08-27T20:53:32.369 INFO:tasks.rados.rados.0.smithi093.stdout:1985: seq_num 1332 ranges {786924=593443,1928721=458203} 2023-08-27T20:53:32.376 INFO:tasks.rados.rados.0.smithi093.stdout:1985: writing smithi093153278-130 from 786924 to 1380367 tid 1 2023-08-27T20:53:32.382 INFO:tasks.rados.rados.0.smithi093.stdout:1985: writing smithi093153278-130 from 1928721 to 2386924 tid 2 2023-08-27T20:53:32.382 INFO:tasks.rados.rados.0.smithi093.stdout:1986: delete oid 431 current snap is 0 2023-08-27T20:53:32.382 INFO:tasks.rados.rados.0.smithi093.stdout:1983: finishing write tid 2 to smithi093153278-417 2023-08-27T20:53:32.382 INFO:tasks.rados.rados.0.smithi093.stdout:1983: finishing write tid 3 to smithi093153278-417 2023-08-27T20:53:32.382 INFO:tasks.rados.rados.0.smithi093.stdout:1983: finishing write tid 4 to smithi093153278-417 2023-08-27T20:53:32.382 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 417 v 301 (ObjNum 1330 snap 0 seq_num 1330) dirty exists 2023-08-27T20:53:32.382 INFO:tasks.rados.rados.0.smithi093.stdout:1983: left oid 417 (ObjNum 1330 snap 0 seq_num 1330) 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1984: finishing write tid 1 to smithi093153278-173 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1984: finishing write tid 2 to smithi093153278-173 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1984: finishing write tid 3 to smithi093153278-173 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1984: finishing write tid 4 to smithi093153278-173 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 173 v 233 (ObjNum 1331 snap 0 seq_num 1331) dirty exists 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1984: left oid 173 (ObjNum 1331 snap 0 seq_num 1331) 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1985: finishing write tid 1 to smithi093153278-130 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1983: done (3 left) 2023-08-27T20:53:32.383 INFO:tasks.rados.rados.0.smithi093.stdout:1984: done (2 left) 2023-08-27T20:53:32.384 INFO:tasks.rados.rados.0.smithi093.stdout:1986: done (1 left) 2023-08-27T20:53:32.384 INFO:tasks.rados.rados.0.smithi093.stdout:1987: read oid 63 snap -1 2023-08-27T20:53:32.384 INFO:tasks.rados.rados.0.smithi093.stdout:1987: expect deleted 2023-08-27T20:53:32.384 INFO:tasks.rados.rados.0.smithi093.stdout:1988: write oid 275 current snap is 0 2023-08-27T20:53:32.384 INFO:tasks.rados.rados.0.smithi093.stdout:1988: seq_num 1334 ranges {420538=743941,1929332=491206} 2023-08-27T20:53:32.389 INFO:tasks.rados.rados.0.smithi093.stdout:1988: writing smithi093153278-275 from 420538 to 1164479 tid 1 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:1988: writing smithi093153278-275 from 1929332 to 2420538 tid 2 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:1989: delete oid 196 current snap is 0 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:1985: finishing write tid 2 to smithi093153278-130 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:1985: finishing write tid 3 to smithi093153278-130 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:1985: finishing write tid 4 to smithi093153278-130 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 130 v 304 (ObjNum 1332 snap 0 seq_num 1332) dirty exists 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:1985: left oid 130 (ObjNum 1332 snap 0 seq_num 1332) 2023-08-27T20:53:32.397 INFO:tasks.rados.rados.0.smithi093.stdout:1988: finishing write tid 1 to smithi093153278-275 2023-08-27T20:53:32.398 INFO:tasks.rados.rados.0.smithi093.stdout:1985: done (3 left) 2023-08-27T20:53:32.398 INFO:tasks.rados.rados.0.smithi093.stdout:1987: done (2 left) 2023-08-27T20:53:32.398 INFO:tasks.rados.rados.0.smithi093.stdout:1989: done (1 left) 2023-08-27T20:53:32.398 INFO:tasks.rados.rados.0.smithi093.stdout:1990: write oid 74 current snap is 0 2023-08-27T20:53:32.399 INFO:tasks.rados.rados.0.smithi093.stdout:1990: seq_num 1336 ranges {454152=494439,1646296=735124} 2023-08-27T20:53:32.403 INFO:tasks.rados.rados.0.smithi093.stdout:1990: writing smithi093153278-74 from 454152 to 948591 tid 1 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1990: writing smithi093153278-74 from 1646296 to 2381420 tid 2 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1991: read oid 87 snap -1 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1991: expect (ObjNum 1288 snap 0 seq_num 1288) 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1992: read oid 204 snap -1 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1992: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1993: write oid 133 current snap is 0 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1988: finishing write tid 2 to smithi093153278-275 2023-08-27T20:53:32.412 INFO:tasks.rados.rados.0.smithi093.stdout:1988: finishing write tid 3 to smithi093153278-275 2023-08-27T20:53:32.413 INFO:tasks.rados.rados.0.smithi093.stdout:1988: finishing write tid 4 to smithi093153278-275 2023-08-27T20:53:32.413 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 275 v 237 (ObjNum 1334 snap 0 seq_num 1334) dirty exists 2023-08-27T20:53:32.413 INFO:tasks.rados.rados.0.smithi093.stdout:1988: left oid 275 (ObjNum 1334 snap 0 seq_num 1334) 2023-08-27T20:53:32.413 INFO:tasks.rados.rados.0.smithi093.stdout:1990: finishing write tid 1 to smithi093153278-74 2023-08-27T20:53:32.413 INFO:tasks.rados.rados.0.smithi093.stdout:1993: seq_num 1337 ranges {470959=569688,1704778=595135} 2023-08-27T20:53:32.417 INFO:tasks.rados.rados.0.smithi093.stdout:1993: writing smithi093153278-133 from 470959 to 1040647 tid 1 2023-08-27T20:53:32.425 INFO:tasks.rados.rados.0.smithi093.stdout:1993: writing smithi093153278-133 from 1704778 to 2299913 tid 2 2023-08-27T20:53:32.425 INFO:tasks.rados.rados.0.smithi093.stdout:1988: done (4 left) 2023-08-27T20:53:32.425 INFO:tasks.rados.rados.0.smithi093.stdout:1994: write oid 463 current snap is 0 2023-08-27T20:53:32.425 INFO:tasks.rados.rados.0.smithi093.stdout:1994: seq_num 1338 ranges {487766=644937,1646907=738793} 2023-08-27T20:53:32.430 INFO:tasks.rados.rados.0.smithi093.stdout:1994: writing smithi093153278-463 from 487766 to 1132703 tid 1 2023-08-27T20:53:32.438 INFO:tasks.rados.rados.0.smithi093.stdout:1994: writing smithi093153278-463 from 1646907 to 2385700 tid 2 2023-08-27T20:53:32.438 INFO:tasks.rados.rados.0.smithi093.stdout:1995: write oid 362 current snap is 0 2023-08-27T20:53:32.438 INFO:tasks.rados.rados.0.smithi093.stdout:1995: seq_num 1339 ranges {504573=436539,1421742=598804,2437505=67068} 2023-08-27T20:53:32.443 INFO:tasks.rados.rados.0.smithi093.stdout:1995: writing smithi093153278-362 from 504573 to 941112 tid 1 2023-08-27T20:53:32.448 INFO:tasks.rados.rados.0.smithi093.stdout:1995: writing smithi093153278-362 from 1421742 to 2020546 tid 2 2023-08-27T20:53:32.451 INFO:tasks.rados.rados.0.smithi093.stdout:1995: writing smithi093153278-362 from 2437505 to 2504573 tid 3 2023-08-27T20:53:32.451 INFO:tasks.rados.rados.0.smithi093.stdout:1996: write oid 482 current snap is 0 2023-08-27T20:53:32.451 INFO:tasks.rados.rados.0.smithi093.stdout:1996: seq_num 1340 ranges {521380=511788,1480224=742462} 2023-08-27T20:53:32.457 INFO:tasks.rados.rados.0.smithi093.stdout:1996: writing smithi093153278-482 from 521380 to 1033168 tid 1 2023-08-27T20:53:32.463 INFO:tasks.rados.rados.0.smithi093.stdout:1996: writing smithi093153278-482 from 1480224 to 2222686 tid 2 2023-08-27T20:53:32.463 INFO:tasks.rados.rados.0.smithi093.stdout:1997: write oid 20 current snap is 0 2023-08-27T20:53:32.463 INFO:tasks.rados.rados.0.smithi093.stdout:1997: seq_num 1341 ranges {538187=587037,1538706=602473} 2023-08-27T20:53:32.469 INFO:tasks.rados.rados.0.smithi093.stdout:1997: writing smithi093153278-20 from 538187 to 1125224 tid 1 2023-08-27T20:53:32.475 INFO:tasks.rados.rados.0.smithi093.stdout:1997: writing smithi093153278-20 from 1538706 to 2141179 tid 2 2023-08-27T20:53:32.478 INFO:tasks.rados.rados.0.smithi093.stdout:1998: write oid 350 current snap is 0 2023-08-27T20:53:32.478 INFO:tasks.rados.rados.0.smithi093.stdout:1990: finishing write tid 2 to smithi093153278-74 2023-08-27T20:53:32.478 INFO:tasks.rados.rados.0.smithi093.stdout:1992: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:53:32.580 INFO:tasks.rados.rados.0.smithi093.stdout:1990: finishing write tid 3 to smithi093153278-74 2023-08-27T20:53:33.156 INFO:tasks.rados.rados.0.smithi093.stdout:1990: finishing write tid 4 to smithi093153278-74 2023-08-27T20:53:33.156 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 74 v 356 (ObjNum 1336 snap 0 seq_num 1336) dirty exists 2023-08-27T20:53:33.157 INFO:tasks.rados.rados.0.smithi093.stdout:1990: left oid 74 (ObjNum 1336 snap 0 seq_num 1336) 2023-08-27T20:53:33.157 INFO:tasks.rados.rados.0.smithi093.stdout:1993: finishing write tid 1 to smithi093153278-133 2023-08-27T20:53:33.157 INFO:tasks.rados.rados.0.smithi093.stdout:1998: seq_num 1342 ranges {554994=662286,1997188=557806} 2023-08-27T20:53:33.164 INFO:tasks.rados.rados.0.smithi093.stdout:1998: writing smithi093153278-350 from 554994 to 1217280 tid 1 2023-08-27T20:53:33.922 INFO:tasks.rados.rados.0.smithi093.stdout:1998: writing smithi093153278-350 from 1997188 to 2554994 tid 2 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1990: done (8 left) 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1992: done (7 left) 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1999: write oid 464 current snap is 0 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1991: expect (ObjNum 1288 snap 0 seq_num 1288) 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1993: finishing write tid 2 to smithi093153278-133 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1993: finishing write tid 3 to smithi093153278-133 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1993: finishing write tid 4 to smithi093153278-133 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 133 v 295 (ObjNum 1337 snap 0 seq_num 1337) dirty exists 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1993: left oid 133 (ObjNum 1337 snap 0 seq_num 1337) 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1994: finishing write tid 1 to smithi093153278-463 2023-08-27T20:53:33.923 INFO:tasks.rados.rados.0.smithi093.stdout:1994: finishing write tid 2 to smithi093153278-463 2023-08-27T20:53:33.924 INFO:tasks.rados.rados.0.smithi093.stdout:1994: finishing write tid 3 to smithi093153278-463 2023-08-27T20:53:33.924 INFO:tasks.rados.rados.0.smithi093.stdout:1994: finishing write tid 4 to smithi093153278-463 2023-08-27T20:53:33.924 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 463 v 298 (ObjNum 1338 snap 0 seq_num 1338) dirty exists 2023-08-27T20:53:33.924 INFO:tasks.rados.rados.0.smithi093.stdout:1994: left oid 463 (ObjNum 1338 snap 0 seq_num 1338) 2023-08-27T20:53:33.924 INFO:tasks.rados.rados.0.smithi093.stdout:1995: finishing write tid 1 to smithi093153278-362 2023-08-27T20:53:33.924 INFO:tasks.rados.rados.0.smithi093.stdout:1999: seq_num 1343 ranges {571801=737535,1939317=489789} 2023-08-27T20:53:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:1999: writing smithi093153278-464 from 571801 to 1309336 tid 1 2023-08-27T20:53:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:1999: writing smithi093153278-464 from 1939317 to 2429106 tid 2 2023-08-27T20:53:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:1991: done (7 left) 2023-08-27T20:53:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:1993: done (6 left) 2023-08-27T20:53:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:1994: done (5 left) 2023-08-27T20:53:33.932 INFO:tasks.rados.rados.0.smithi093.stdout:2000: write oid 478 current snap is 0 2023-08-27T20:53:33.932 INFO:tasks.rados.rados.0.smithi093.stdout:1995: finishing write tid 2 to smithi093153278-362 2023-08-27T20:53:33.932 INFO:tasks.rados.rados.0.smithi093.stdout:1995: finishing write tid 3 to smithi093153278-362 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:1995: finishing write tid 4 to smithi093153278-362 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:1995: finishing write tid 5 to smithi093153278-362 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 362 v 264 (ObjNum 1339 snap 0 seq_num 1339) dirty exists 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:1995: left oid 362 (ObjNum 1339 snap 0 seq_num 1339) 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:1996: finishing write tid 1 to smithi093153278-482 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:2000: seq_num 1344 ranges {588608=412784,1597799=749800} 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:2000: writing smithi093153278-478 from 588608 to 1001392 tid 1 2023-08-27T20:53:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:2000: writing smithi093153278-478 from 1597799 to 2347599 tid 2 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:1995: done (5 left) 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:2001: delete oid 190 current snap is 0 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:1996: finishing write tid 2 to smithi093153278-482 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:1996: finishing write tid 3 to smithi093153278-482 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:1996: finishing write tid 4 to smithi093153278-482 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 482 v 244 (ObjNum 1340 snap 0 seq_num 1340) dirty exists 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:1996: left oid 482 (ObjNum 1340 snap 0 seq_num 1340) 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:1997: finishing write tid 1 to smithi093153278-20 2023-08-27T20:53:33.934 INFO:tasks.rados.rados.0.smithi093.stdout:1997: finishing write tid 2 to smithi093153278-20 2023-08-27T20:53:33.935 INFO:tasks.rados.rados.0.smithi093.stdout:1997: finishing write tid 3 to smithi093153278-20 2023-08-27T20:53:33.935 INFO:tasks.rados.rados.0.smithi093.stdout:1997: finishing write tid 4 to smithi093153278-20 2023-08-27T20:53:33.937 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 285 (ObjNum 1341 snap 0 seq_num 1341) dirty exists 2023-08-27T20:53:33.937 INFO:tasks.rados.rados.0.smithi093.stdout:1997: left oid 20 (ObjNum 1341 snap 0 seq_num 1341) 2023-08-27T20:53:33.937 INFO:tasks.rados.rados.0.smithi093.stdout:1998: finishing write tid 1 to smithi093153278-350 2023-08-27T20:53:33.937 INFO:tasks.rados.rados.0.smithi093.stdout:1998: finishing write tid 2 to smithi093153278-350 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1998: finishing write tid 3 to smithi093153278-350 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1998: finishing write tid 4 to smithi093153278-350 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 350 v 247 (ObjNum 1342 snap 0 seq_num 1342) dirty exists 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1998: left oid 350 (ObjNum 1342 snap 0 seq_num 1342) 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1999: finishing write tid 1 to smithi093153278-464 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1999: finishing write tid 2 to smithi093153278-464 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1999: finishing write tid 3 to smithi093153278-464 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1999: finishing write tid 4 to smithi093153278-464 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 464 v 275 (ObjNum 1343 snap 0 seq_num 1343) dirty exists 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1999: left oid 464 (ObjNum 1343 snap 0 seq_num 1343) 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:2000: finishing write tid 1 to smithi093153278-478 2023-08-27T20:53:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:1996: done (5 left) 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:1997: done (4 left) 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:1998: done (3 left) 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:1999: done (2 left) 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2001: done (1 left) 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2002: read oid 238 snap -1 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2002: expect (ObjNum 1024 snap 0 seq_num 1024) 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2003: write oid 188 current snap is 0 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2003: seq_num 1346 ranges {622222=679635,1831116=753469} 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2003: writing smithi093153278-188 from 622222 to 1301857 tid 1 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2003: writing smithi093153278-188 from 1831116 to 2584585 tid 2 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2004: write oid 179 current snap is 0 2023-08-27T20:53:33.939 INFO:tasks.rados.rados.0.smithi093.stdout:2004: seq_num 1347 ranges {639029=754884,2173245=465784} 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2004: writing smithi093153278-179 from 639029 to 1393913 tid 1 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2004: writing smithi093153278-179 from 2173245 to 2639029 tid 2 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2005: write oid 200 current snap is 0 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2005: seq_num 1348 ranges {655836=430133,1831727=757138} 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2005: writing smithi093153278-200 from 655836 to 1085969 tid 1 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2005: writing smithi093153278-200 from 1831727 to 2588865 tid 2 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2006: delete oid 208 current snap is 0 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2000: finishing write tid 2 to smithi093153278-478 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2000: finishing write tid 3 to smithi093153278-478 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2000: finishing write tid 4 to smithi093153278-478 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 277 (ObjNum 1344 snap 0 seq_num 1344) dirty exists 2023-08-27T20:53:33.940 INFO:tasks.rados.rados.0.smithi093.stdout:2000: left oid 478 (ObjNum 1344 snap 0 seq_num 1344) 2023-08-27T20:53:33.944 INFO:tasks.rados.rados.0.smithi093.stdout:2002: expect (ObjNum 1024 snap 0 seq_num 1024) 2023-08-27T20:53:33.944 INFO:tasks.rados.rados.0.smithi093.stdout:2003: finishing write tid 1 to smithi093153278-188 2023-08-27T20:53:33.944 INFO:tasks.rados.rados.0.smithi093.stdout:2003: finishing write tid 2 to smithi093153278-188 2023-08-27T20:53:33.944 INFO:tasks.rados.rados.0.smithi093.stdout:2000: done (5 left) 2023-08-27T20:53:33.944 INFO:tasks.rados.rados.0.smithi093.stdout:2002: done (4 left) 2023-08-27T20:53:33.944 INFO:tasks.rados.rados.0.smithi093.stdout:2006: done (3 left) 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2007: write oid 475 current snap is 0 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2003: finishing write tid 3 to smithi093153278-188 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2003: finishing write tid 4 to smithi093153278-188 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 188 v 301 (ObjNum 1346 snap 0 seq_num 1346) dirty exists 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2003: left oid 188 (ObjNum 1346 snap 0 seq_num 1346) 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2004: finishing write tid 1 to smithi093153278-179 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2007: seq_num 1350 ranges {689450=580631,1948691=740759} 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2007: writing smithi093153278-475 from 689450 to 1270081 tid 1 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2007: writing smithi093153278-475 from 1948691 to 2689450 tid 2 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2003: done (3 left) 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2008: read oid 108 snap -1 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2008: expect (ObjNum 1073 snap 0 seq_num 1073) 2023-08-27T20:53:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2009: write oid 193 current snap is 0 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2004: finishing write tid 2 to smithi093153278-179 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2004: finishing write tid 3 to smithi093153278-179 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2004: finishing write tid 4 to smithi093153278-179 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 179 v 288 (ObjNum 1347 snap 0 seq_num 1347) dirty exists 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2004: left oid 179 (ObjNum 1347 snap 0 seq_num 1347) 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2005: finishing write tid 1 to smithi093153278-200 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2009: seq_num 1351 ranges {706257=655880,1890820=504465} 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2009: writing smithi093153278-193 from 706257 to 1362137 tid 1 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2009: writing smithi093153278-193 from 1890820 to 2395285 tid 2 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2004: done (4 left) 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2010: delete oid 237 current snap is 0 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2005: finishing write tid 2 to smithi093153278-200 2023-08-27T20:53:33.946 INFO:tasks.rados.rados.0.smithi093.stdout:2005: finishing write tid 3 to smithi093153278-200 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2005: finishing write tid 4 to smithi093153278-200 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 200 v 328 (ObjNum 1348 snap 0 seq_num 1348) dirty exists 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2005: left oid 200 (ObjNum 1348 snap 0 seq_num 1348) 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2007: finishing write tid 1 to smithi093153278-475 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2008: expect (ObjNum 1073 snap 0 seq_num 1073) 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2007: finishing write tid 2 to smithi093153278-475 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2007: finishing write tid 3 to smithi093153278-475 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2007: finishing write tid 4 to smithi093153278-475 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 475 v 240 (ObjNum 1350 snap 0 seq_num 1350) dirty exists 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2007: left oid 475 (ObjNum 1350 snap 0 seq_num 1350) 2023-08-27T20:53:33.947 INFO:tasks.rados.rados.0.smithi093.stdout:2009: finishing write tid 1 to smithi093153278-193 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2005: done (4 left) 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2007: done (3 left) 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2008: done (2 left) 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2010: done (1 left) 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2011: write oid 258 current snap is 0 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2009: finishing write tid 2 to smithi093153278-193 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2009: finishing write tid 3 to smithi093153278-193 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2009: finishing write tid 4 to smithi093153278-193 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 193 v 194 (ObjNum 1351 snap 0 seq_num 1351) dirty exists 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2009: left oid 193 (ObjNum 1351 snap 0 seq_num 1351) 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2011: seq_num 1353 ranges {739871=406378,1607784=508134} 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2011: writing smithi093153278-258 from 739871 to 1146249 tid 1 2023-08-27T20:53:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2011: writing smithi093153278-258 from 1607784 to 2115918 tid 2 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2009: done (1 left) 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2012: write oid 160 current snap is 0 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2012: seq_num 1354 ranges {756678=597980,1782619=768145} 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2012: writing smithi093153278-160 from 756678 to 1354658 tid 1 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2012: writing smithi093153278-160 from 1782619 to 2550764 tid 2 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2013: write oid 408 current snap is 0 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2011: finishing write tid 1 to smithi093153278-258 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2013: seq_num 1355 ranges {773485=673229,2124748=511803} 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2013: writing smithi093153278-408 from 773485 to 1446714 tid 1 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2013: writing smithi093153278-408 from 2124748 to 2636551 tid 2 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2014: write oid 178 current snap is 0 2023-08-27T20:53:33.949 INFO:tasks.rados.rados.0.smithi093.stdout:2014: seq_num 1356 ranges {790292=748478,2183230=607062} 2023-08-27T20:53:33.950 INFO:tasks.rados.rados.0.smithi093.stdout:2014: writing smithi093153278-178 from 790292 to 1538770 tid 1 2023-08-27T20:53:33.955 INFO:tasks.rados.rados.0.smithi093.stdout:2014: writing smithi093153278-178 from 2183230 to 2790292 tid 2 2023-08-27T20:53:34.016 INFO:tasks.rados.rados.0.smithi093.stdout:2015: read oid 175 snap -1 2023-08-27T20:53:34.016 INFO:tasks.rados.rados.0.smithi093.stdout:2015: expect (ObjNum 1144 snap 0 seq_num 1144) 2023-08-27T20:53:34.016 INFO:tasks.rados.rados.0.smithi093.stdout:2016: read oid 183 snap -1 2023-08-27T20:53:34.017 INFO:tasks.rados.rados.0.smithi093.stdout:2016: expect (ObjNum 1163 snap 0 seq_num 1163) 2023-08-27T20:53:34.017 INFO:tasks.rados.rados.0.smithi093.stdout:2017: read oid 438 snap -1 2023-08-27T20:53:34.017 INFO:tasks.rados.rados.0.smithi093.stdout:2017: expect (ObjNum 1125 snap 0 seq_num 1125) 2023-08-27T20:53:34.017 INFO:tasks.rados.rados.0.smithi093.stdout:2018: read oid 214 snap -1 2023-08-27T20:53:34.017 INFO:tasks.rados.rados.0.smithi093.stdout:2018: expect (ObjNum 213 snap 0 seq_num 213) 2023-08-27T20:53:34.017 INFO:tasks.rados.rados.0.smithi093.stdout:2019: write oid 131 current snap is 0 2023-08-27T20:53:34.017 INFO:tasks.rados.rados.0.smithi093.stdout:2011: finishing write tid 2 to smithi093153278-258 2023-08-27T20:53:34.019 INFO:tasks.rados.rados.0.smithi093.stdout:2011: finishing write tid 3 to smithi093153278-258 2023-08-27T20:53:34.019 INFO:tasks.rados.rados.0.smithi093.stdout:2011: finishing write tid 4 to smithi093153278-258 2023-08-27T20:53:34.019 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 258 v 262 (ObjNum 1353 snap 0 seq_num 1353) dirty exists 2023-08-27T20:53:34.019 INFO:tasks.rados.rados.0.smithi093.stdout:2011: left oid 258 (ObjNum 1353 snap 0 seq_num 1353) 2023-08-27T20:53:34.019 INFO:tasks.rados.rados.0.smithi093.stdout:2012: finishing write tid 1 to smithi093153278-160 2023-08-27T20:53:34.019 INFO:tasks.rados.rados.0.smithi093.stdout:2019: seq_num 1357 ranges {407099=423727,1441712=515472,2582060=225039} 2023-08-27T20:53:34.019 INFO:tasks.rados.rados.0.smithi093.stdout:2019: writing smithi093153278-131 from 407099 to 830826 tid 1 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:2019: writing smithi093153278-131 from 1441712 to 1957184 tid 2 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:2019: writing smithi093153278-131 from 2582060 to 2807099 tid 3 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:2011: done (8 left) 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:2020: write oid 224 current snap is 0 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:2012: finishing write tid 2 to smithi093153278-160 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:2012: finishing write tid 3 to smithi093153278-160 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:2012: finishing write tid 4 to smithi093153278-160 2023-08-27T20:53:34.020 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 160 v 304 (ObjNum 1354 snap 0 seq_num 1354) dirty exists 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2012: left oid 160 (ObjNum 1354 snap 0 seq_num 1354) 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2013: finishing write tid 1 to smithi093153278-408 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2013: finishing write tid 2 to smithi093153278-408 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2013: finishing write tid 3 to smithi093153278-408 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2013: finishing write tid 4 to smithi093153278-408 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 408 v 331 (ObjNum 1355 snap 0 seq_num 1355) dirty exists 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2013: left oid 408 (ObjNum 1355 snap 0 seq_num 1355) 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2014: finishing write tid 1 to smithi093153278-178 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2020: seq_num 1358 ranges {423906=498976,1500194=659130,2609483=214423} 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2020: writing smithi093153278-224 from 423906 to 922882 tid 1 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2020: writing smithi093153278-224 from 1500194 to 2159324 tid 2 2023-08-27T20:53:34.021 INFO:tasks.rados.rados.0.smithi093.stdout:2020: writing smithi093153278-224 from 2609483 to 2823906 tid 3 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2012: done (8 left) 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2013: done (7 left) 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2021: read oid 106 snap -1 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2021: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2022: write oid 344 current snap is 0 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2022: seq_num 1359 ranges {440713=574225,1442323=519141,2636906=203807} 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2022: writing smithi093153278-344 from 440713 to 1014938 tid 1 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2022: writing smithi093153278-344 from 1442323 to 1961464 tid 2 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2022: writing smithi093153278-344 from 2636906 to 2840713 tid 3 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2023: delete oid 116 current snap is 0 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2015: expect (ObjNum 1144 snap 0 seq_num 1144) 2023-08-27T20:53:34.022 INFO:tasks.rados.rados.0.smithi093.stdout:2019: finishing write tid 1 to smithi093153278-131 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2019: finishing write tid 2 to smithi093153278-131 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2017: expect (ObjNum 1125 snap 0 seq_num 1125) 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2014: finishing write tid 2 to smithi093153278-178 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2014: finishing write tid 3 to smithi093153278-178 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2014: finishing write tid 4 to smithi093153278-178 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 178 v 197 (ObjNum 1356 snap 0 seq_num 1356) dirty exists 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2014: left oid 178 (ObjNum 1356 snap 0 seq_num 1356) 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2019: finishing write tid 3 to smithi093153278-131 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2019: finishing write tid 4 to smithi093153278-131 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2014: done (9 left) 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2015: done (8 left) 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2017: done (7 left) 2023-08-27T20:53:34.023 INFO:tasks.rados.rados.0.smithi093.stdout:2023: done (6 left) 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2024: delete oid 440 current snap is 0 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2019: finishing write tid 5 to smithi093153278-131 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 131 v 253 (ObjNum 1357 snap 0 seq_num 1357) dirty exists 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2019: left oid 131 (ObjNum 1357 snap 0 seq_num 1357) 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2016: expect (ObjNum 1163 snap 0 seq_num 1163) 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2018: expect (ObjNum 213 snap 0 seq_num 213) 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2020: finishing write tid 1 to smithi093153278-224 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2020: finishing write tid 2 to smithi093153278-224 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2020: finishing write tid 3 to smithi093153278-224 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2020: finishing write tid 4 to smithi093153278-224 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:2020: finishing write tid 5 to smithi093153278-224 2023-08-27T20:53:34.024 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 224 v 251 (ObjNum 1358 snap 0 seq_num 1358) dirty exists 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2020: left oid 224 (ObjNum 1358 snap 0 seq_num 1358) 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2016: done (6 left) 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2018: done (5 left) 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2019: done (4 left) 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2020: done (3 left) 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2024: done (2 left) 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2025: write oid 310 current snap is 0 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2021: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2022: finishing write tid 1 to smithi093153278-344 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2022: finishing write tid 2 to smithi093153278-344 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2022: finishing write tid 3 to smithi093153278-344 2023-08-27T20:53:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2022: finishing write tid 4 to smithi093153278-344 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2022: finishing write tid 5 to smithi093153278-344 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 344 v 282 (ObjNum 1359 snap 0 seq_num 1359) dirty exists 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2022: left oid 344 (ObjNum 1359 snap 0 seq_num 1359) 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2025: seq_num 1362 ranges {491134=516325,1734122=666468} 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2025: writing smithi093153278-310 from 491134 to 1007459 tid 1 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2025: writing smithi093153278-310 from 1734122 to 2400590 tid 2 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2021: done (2 left) 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2022: done (1 left) 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2026: read oid 422 snap -1 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2026: expect (ObjNum 1153 snap 0 seq_num 1153) 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2027: read oid 147 snap -1 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2027: expect deleted 2023-08-27T20:53:34.026 INFO:tasks.rados.rados.0.smithi093.stdout:2028: write oid 111 current snap is 0 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2025: finishing write tid 1 to smithi093153278-310 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2028: seq_num 1363 ranges {507941=591574,1676251=526479,2746598=161343} 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2028: writing smithi093153278-111 from 507941 to 1099515 tid 1 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2028: writing smithi093153278-111 from 1676251 to 2202730 tid 2 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2028: writing smithi093153278-111 from 2746598 to 2907941 tid 3 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2029: read oid 355 snap -1 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2029: expect (ObjNum 1172 snap 0 seq_num 1172) 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2030: read oid 364 snap -1 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2030: expect (ObjNum 1027 snap 0 seq_num 1027) 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2031: write oid 432 current snap is 0 2023-08-27T20:53:34.027 INFO:tasks.rados.rados.0.smithi093.stdout:2031: seq_num 1364 ranges {524748=666823,1734733=670137} 2023-08-27T20:53:34.083 INFO:tasks.rados.rados.0.smithi093.stdout:2031: writing smithi093153278-432 from 524748 to 1191571 tid 1 2023-08-27T20:53:34.083 INFO:tasks.rados.rados.0.smithi093.stdout:2031: writing smithi093153278-432 from 1734733 to 2404870 tid 2 2023-08-27T20:53:34.083 INFO:tasks.rados.rados.0.smithi093.stdout:2032: write oid 225 current snap is 0 2023-08-27T20:53:34.083 INFO:tasks.rados.rados.0.smithi093.stdout:2032: seq_num 1365 ranges {541555=742072,1793215=530148,2801444=140111} 2023-08-27T20:53:34.083 INFO:tasks.rados.rados.0.smithi093.stdout:2032: writing smithi093153278-225 from 541555 to 1283627 tid 1 2023-08-27T20:53:34.083 INFO:tasks.rados.rados.0.smithi093.stdout:2032: writing smithi093153278-225 from 1793215 to 2323363 tid 2 2023-08-27T20:53:34.083 INFO:tasks.rados.rados.0.smithi093.stdout:2032: writing smithi093153278-225 from 2801444 to 2941555 tid 3 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2033: write oid 75 current snap is 0 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2025: finishing write tid 2 to smithi093153278-310 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2025: finishing write tid 3 to smithi093153278-310 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2033: seq_num 1366 ranges {558362=417321,1451697=673806,2712514=245848} 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2033: writing smithi093153278-75 from 558362 to 975683 tid 1 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2033: writing smithi093153278-75 from 1451697 to 2125503 tid 2 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2033: writing smithi093153278-75 from 2712514 to 2958362 tid 3 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2027: done (8 left) 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2034: read oid 249 snap -1 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2034: expect (ObjNum 820 snap 0 seq_num 820) 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2035: write oid 483 current snap is 0 2023-08-27T20:53:34.084 INFO:tasks.rados.rados.0.smithi093.stdout:2025: finishing write tid 4 to smithi093153278-310 2023-08-27T20:53:34.085 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 310 v 244 (ObjNum 1362 snap 0 seq_num 1362) dirty exists 2023-08-27T20:53:34.085 INFO:tasks.rados.rados.0.smithi093.stdout:2025: left oid 310 (ObjNum 1362 snap 0 seq_num 1362) 2023-08-27T20:53:34.085 INFO:tasks.rados.rados.0.smithi093.stdout:2026: expect (ObjNum 1153 snap 0 seq_num 1153) 2023-08-27T20:53:34.160 INFO:tasks.rados.rados.0.smithi093.stdout:2028: finishing write tid 1 to smithi093153278-111 2023-08-27T20:53:34.343 INFO:tasks.rados.rados.0.smithi093.stdout:2028: finishing write tid 2 to smithi093153278-111 2023-08-27T20:53:34.343 INFO:tasks.rados.rados.0.smithi093.stdout:2035: seq_num 1367 ranges {575169=492570,1793826=417464,2623584=351585} 2023-08-27T20:53:34.344 INFO:tasks.rados.rados.0.smithi093.stdout:2035: writing smithi093153278-483 from 575169 to 1067739 tid 1 2023-08-27T20:53:34.344 INFO:tasks.rados.rados.0.smithi093.stdout:2035: writing smithi093153278-483 from 1793826 to 2211290 tid 2 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2035: writing smithi093153278-483 from 2623584 to 2975169 tid 3 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2025: done (9 left) 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2026: done (8 left) 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2036: write oid 468 current snap is 0 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2036: seq_num 1368 ranges {591976=567819,1852308=677475} 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2036: writing smithi093153278-468 from 591976 to 1159795 tid 1 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2036: writing smithi093153278-468 from 1852308 to 2529783 tid 2 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2037: read oid 338 snap -1 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2037: expect (ObjNum 337 snap 0 seq_num 337) 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2038: delete oid 283 current snap is 0 2023-08-27T20:53:34.345 INFO:tasks.rados.rados.0.smithi093.stdout:2028: finishing write tid 3 to smithi093153278-111 2023-08-27T20:53:34.346 INFO:tasks.rados.rados.0.smithi093.stdout:2028: finishing write tid 4 to smithi093153278-111 2023-08-27T20:53:34.346 INFO:tasks.rados.rados.0.smithi093.stdout:2028: finishing write tid 5 to smithi093153278-111 2023-08-27T20:53:34.346 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 111 v 308 (ObjNum 1363 snap 0 seq_num 1363) dirty exists 2023-08-27T20:53:34.346 INFO:tasks.rados.rados.0.smithi093.stdout:2028: left oid 111 (ObjNum 1363 snap 0 seq_num 1363) 2023-08-27T20:53:34.346 INFO:tasks.rados.rados.0.smithi093.stdout:2030: expect (ObjNum 1027 snap 0 seq_num 1027) 2023-08-27T20:53:34.346 INFO:tasks.rados.rados.0.smithi093.stdout:2029: expect (ObjNum 1172 snap 0 seq_num 1172) 2023-08-27T20:53:34.538 INFO:tasks.rados.rados.0.smithi093.stdout:2031: finishing write tid 1 to smithi093153278-432 2023-08-27T20:53:34.538 INFO:tasks.rados.rados.0.smithi093.stdout:2031: finishing write tid 2 to smithi093153278-432 2023-08-27T20:53:34.538 INFO:tasks.rados.rados.0.smithi093.stdout:2031: finishing write tid 3 to smithi093153278-432 2023-08-27T20:53:34.538 INFO:tasks.rados.rados.0.smithi093.stdout:2031: finishing write tid 4 to smithi093153278-432 2023-08-27T20:53:34.538 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 432 v 241 (ObjNum 1364 snap 0 seq_num 1364) dirty exists 2023-08-27T20:53:34.538 INFO:tasks.rados.rados.0.smithi093.stdout:2031: left oid 432 (ObjNum 1364 snap 0 seq_num 1364) 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2032: finishing write tid 1 to smithi093153278-225 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2028: done (10 left) 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2029: done (9 left) 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2030: done (8 left) 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2031: done (7 left) 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2038: done (6 left) 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2039: delete oid 127 current snap is 0 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2032: finishing write tid 2 to smithi093153278-225 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2032: finishing write tid 3 to smithi093153278-225 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2032: finishing write tid 4 to smithi093153278-225 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:2032: finishing write tid 5 to smithi093153278-225 2023-08-27T20:53:34.539 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 225 v 248 (ObjNum 1365 snap 0 seq_num 1365) dirty exists 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2032: left oid 225 (ObjNum 1365 snap 0 seq_num 1365) 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2033: finishing write tid 1 to smithi093153278-75 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2033: finishing write tid 2 to smithi093153278-75 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2033: finishing write tid 3 to smithi093153278-75 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2033: finishing write tid 4 to smithi093153278-75 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2033: finishing write tid 5 to smithi093153278-75 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 75 v 361 (ObjNum 1366 snap 0 seq_num 1366) dirty exists 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2033: left oid 75 (ObjNum 1366 snap 0 seq_num 1366) 2023-08-27T20:53:34.540 INFO:tasks.rados.rados.0.smithi093.stdout:2034: expect (ObjNum 820 snap 0 seq_num 820) 2023-08-27T20:53:34.747 INFO:tasks.rados.rados.0.smithi093.stdout:2035: finishing write tid 1 to smithi093153278-483 2023-08-27T20:53:34.747 INFO:tasks.rados.rados.0.smithi093.stdout:2035: finishing write tid 2 to smithi093153278-483 2023-08-27T20:53:34.747 INFO:tasks.rados.rados.0.smithi093.stdout:2035: finishing write tid 3 to smithi093153278-483 2023-08-27T20:53:34.747 INFO:tasks.rados.rados.0.smithi093.stdout:2035: finishing write tid 4 to smithi093153278-483 2023-08-27T20:53:34.748 INFO:tasks.rados.rados.0.smithi093.stdout:2035: finishing write tid 5 to smithi093153278-483 2023-08-27T20:53:34.748 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 483 v 255 (ObjNum 1367 snap 0 seq_num 1367) dirty exists 2023-08-27T20:53:34.748 INFO:tasks.rados.rados.0.smithi093.stdout:2035: left oid 483 (ObjNum 1367 snap 0 seq_num 1367) 2023-08-27T20:53:34.748 INFO:tasks.rados.rados.0.smithi093.stdout:2036: finishing write tid 1 to smithi093153278-468 2023-08-27T20:53:34.748 INFO:tasks.rados.rados.0.smithi093.stdout:2037: expect (ObjNum 337 snap 0 seq_num 337) 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2036: finishing write tid 2 to smithi093153278-468 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2036: finishing write tid 3 to smithi093153278-468 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2036: finishing write tid 4 to smithi093153278-468 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 468 v 251 (ObjNum 1368 snap 0 seq_num 1368) dirty exists 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2036: left oid 468 (ObjNum 1368 snap 0 seq_num 1368) 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2032: done (6 left) 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2033: done (5 left) 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2034: done (4 left) 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2035: done (3 left) 2023-08-27T20:53:34.860 INFO:tasks.rados.rados.0.smithi093.stdout:2036: done (2 left) 2023-08-27T20:53:34.861 INFO:tasks.rados.rados.0.smithi093.stdout:2037: done (1 left) 2023-08-27T20:53:34.861 INFO:tasks.rados.rados.0.smithi093.stdout:2039: done (0 left) 2023-08-27T20:53:34.861 INFO:tasks.rados.rados.0.smithi093.stdout:2040: write oid 430 current snap is 0 2023-08-27T20:53:34.861 INFO:tasks.rados.rados.0.smithi093.stdout:2040: seq_num 1371 ranges {642397=509919,1627754=424802,2733276=309121} 2023-08-27T20:53:34.865 INFO:tasks.rados.rados.0.smithi093.stdout:2040: writing smithi093153278-430 from 642397 to 1152316 tid 1 2023-08-27T20:53:34.870 INFO:tasks.rados.rados.0.smithi093.stdout:2040: writing smithi093153278-430 from 1627754 to 2052556 tid 2 2023-08-27T20:53:34.876 INFO:tasks.rados.rados.0.smithi093.stdout:2040: writing smithi093153278-430 from 2733276 to 3042397 tid 3 2023-08-27T20:53:34.876 INFO:tasks.rados.rados.0.smithi093.stdout:2041: write oid 436 current snap is 0 2023-08-27T20:53:34.876 INFO:tasks.rados.rados.0.smithi093.stdout:2040: finishing write tid 1 to smithi093153278-430 2023-08-27T20:53:34.876 INFO:tasks.rados.rados.0.smithi093.stdout:2040: finishing write tid 2 to smithi093153278-430 2023-08-27T20:53:34.877 INFO:tasks.rados.rados.0.smithi093.stdout:2041: seq_num 1372 ranges {659204=585168,1686236=684813} 2023-08-27T20:53:34.883 INFO:tasks.rados.rados.0.smithi093.stdout:2041: writing smithi093153278-436 from 659204 to 1244372 tid 1 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2041: writing smithi093153278-436 from 1686236 to 2371049 tid 2 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2042: read oid 146 snap -1 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2042: expect deleted 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2043: read oid 459 snap -1 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2043: expect (ObjNum 831 snap 0 seq_num 831) 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2044: write oid 308 current snap is 0 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2040: finishing write tid 3 to smithi093153278-430 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2040: finishing write tid 4 to smithi093153278-430 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:2040: finishing write tid 5 to smithi093153278-430 2023-08-27T20:53:34.890 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 430 v 365 (ObjNum 1371 snap 0 seq_num 1371) dirty exists 2023-08-27T20:53:34.891 INFO:tasks.rados.rados.0.smithi093.stdout:2040: left oid 430 (ObjNum 1371 snap 0 seq_num 1371) 2023-08-27T20:53:34.891 INFO:tasks.rados.rados.0.smithi093.stdout:2041: finishing write tid 1 to smithi093153278-436 2023-08-27T20:53:34.891 INFO:tasks.rados.rados.0.smithi093.stdout:2044: seq_num 1373 ranges {676011=660417,1744718=428471,2788122=287889} 2023-08-27T20:53:34.897 INFO:tasks.rados.rados.0.smithi093.stdout:2044: writing smithi093153278-308 from 676011 to 1336428 tid 1 2023-08-27T20:53:34.902 INFO:tasks.rados.rados.0.smithi093.stdout:2044: writing smithi093153278-308 from 1744718 to 2173189 tid 2 2023-08-27T20:53:34.908 INFO:tasks.rados.rados.0.smithi093.stdout:2044: writing smithi093153278-308 from 2788122 to 3076011 tid 3 2023-08-27T20:53:34.908 INFO:tasks.rados.rados.0.smithi093.stdout:2040: done (4 left) 2023-08-27T20:53:34.908 INFO:tasks.rados.rados.0.smithi093.stdout:2045: write oid 314 current snap is 0 2023-08-27T20:53:34.908 INFO:tasks.rados.rados.0.smithi093.stdout:2045: seq_num 1374 ranges {692818=735666,2203200=688482} 2023-08-27T20:53:34.915 INFO:tasks.rados.rados.0.smithi093.stdout:2045: writing smithi093153278-314 from 692818 to 1428484 tid 1 2023-08-27T20:53:34.924 INFO:tasks.rados.rados.0.smithi093.stdout:2045: writing smithi093153278-314 from 2203200 to 2891682 tid 2 2023-08-27T20:53:34.924 INFO:tasks.rados.rados.0.smithi093.stdout:2046: read oid 350 snap -1 2023-08-27T20:53:34.924 INFO:tasks.rados.rados.0.smithi093.stdout:2046: expect (ObjNum 1342 snap 0 seq_num 1342) 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2047: delete oid 467 current snap is 0 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2041: finishing write tid 2 to smithi093153278-436 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2041: finishing write tid 3 to smithi093153278-436 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2041: finishing write tid 4 to smithi093153278-436 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 436 v 200 (ObjNum 1372 snap 0 seq_num 1372) dirty exists 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2041: left oid 436 (ObjNum 1372 snap 0 seq_num 1372) 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2044: finishing write tid 1 to smithi093153278-308 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2044: finishing write tid 2 to smithi093153278-308 2023-08-27T20:53:34.925 INFO:tasks.rados.rados.0.smithi093.stdout:2043: expect (ObjNum 831 snap 0 seq_num 831) 2023-08-27T20:53:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2464836133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2464836133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2464836133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:35.117 INFO:tasks.rados.rados.0.smithi093.stdout:2044: finishing write tid 3 to smithi093153278-308 2023-08-27T20:53:35.117 INFO:tasks.rados.rados.0.smithi093.stdout:2044: finishing write tid 4 to smithi093153278-308 2023-08-27T20:53:35.117 INFO:tasks.rados.rados.0.smithi093.stdout:2044: finishing write tid 5 to smithi093153278-308 2023-08-27T20:53:35.117 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 308 v 268 (ObjNum 1373 snap 0 seq_num 1373) dirty exists 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2044: left oid 308 (ObjNum 1373 snap 0 seq_num 1373) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2045: finishing write tid 1 to smithi093153278-314 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2041: done (6 left) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2042: done (5 left) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2043: done (4 left) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2044: done (3 left) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2047: done (2 left) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2048: read oid 406 snap -1 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2048: expect (ObjNum 1207 snap 0 seq_num 1207) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2049: read oid 201 snap -1 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2049: expect (ObjNum 609 snap 0 seq_num 609) 2023-08-27T20:53:35.118 INFO:tasks.rados.rados.0.smithi093.stdout:2050: read oid 138 snap -1 2023-08-27T20:53:35.119 INFO:tasks.rados.rados.0.smithi093.stdout:2050: expect (ObjNum 845 snap 0 seq_num 845) 2023-08-27T20:53:35.119 INFO:tasks.rados.rados.0.smithi093.stdout:2051: write oid 424 current snap is 0 2023-08-27T20:53:35.119 INFO:tasks.rados.rados.0.smithi093.stdout:2045: finishing write tid 2 to smithi093153278-314 2023-08-27T20:53:35.119 INFO:tasks.rados.rados.0.smithi093.stdout:2051: seq_num 1376 ranges {726432=486164,1803811=692151} 2023-08-27T20:53:35.124 INFO:tasks.rados.rados.0.smithi093.stdout:2051: writing smithi093153278-424 from 726432 to 1212596 tid 1 2023-08-27T20:53:35.132 INFO:tasks.rados.rados.0.smithi093.stdout:2051: writing smithi093153278-424 from 1803811 to 2495962 tid 2 2023-08-27T20:53:35.132 INFO:tasks.rados.rados.0.smithi093.stdout:2052: write oid 480 current snap is 0 2023-08-27T20:53:35.132 INFO:tasks.rados.rados.0.smithi093.stdout:2045: finishing write tid 3 to smithi093153278-314 2023-08-27T20:53:35.132 INFO:tasks.rados.rados.0.smithi093.stdout:2045: finishing write tid 4 to smithi093153278-314 2023-08-27T20:53:35.132 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 314 v 334 (ObjNum 1374 snap 0 seq_num 1374) dirty exists 2023-08-27T20:53:35.132 INFO:tasks.rados.rados.0.smithi093.stdout:2045: left oid 314 (ObjNum 1374 snap 0 seq_num 1374) 2023-08-27T20:53:35.132 INFO:tasks.rados.rados.0.smithi093.stdout:2046: expect (ObjNum 1342 snap 0 seq_num 1342) 2023-08-27T20:53:35.254 INFO:tasks.rados.rados.0.smithi093.stdout:2048: expect (ObjNum 1207 snap 0 seq_num 1207) 2023-08-27T20:53:35.326 INFO:tasks.rados.rados.0.smithi093.stdout:2050: expect (ObjNum 845 snap 0 seq_num 845) 2023-08-27T20:53:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:2049: expect (ObjNum 609 snap 0 seq_num 609) 2023-08-27T20:53:35.519 INFO:tasks.rados.rados.0.smithi093.stdout:2051: finishing write tid 1 to smithi093153278-424 2023-08-27T20:53:35.519 INFO:tasks.rados.rados.0.smithi093.stdout:2051: finishing write tid 2 to smithi093153278-424 2023-08-27T20:53:35.519 INFO:tasks.rados.rados.0.smithi093.stdout:2051: finishing write tid 3 to smithi093153278-424 2023-08-27T20:53:35.519 INFO:tasks.rados.rados.0.smithi093.stdout:2051: finishing write tid 4 to smithi093153278-424 2023-08-27T20:53:35.519 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 424 v 337 (ObjNum 1376 snap 0 seq_num 1376) dirty exists 2023-08-27T20:53:35.519 INFO:tasks.rados.rados.0.smithi093.stdout:2051: left oid 424 (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:53:35.519 INFO:tasks.rados.rados.0.smithi093.stdout:2052: seq_num 1377 ranges {743239=677766,1978646=435809,2897814=245425} 2023-08-27T20:53:35.526 INFO:tasks.rados.rados.0.smithi093.stdout:2052: writing smithi093153278-480 from 743239 to 1421005 tid 1 2023-08-27T20:53:35.531 INFO:tasks.rados.rados.0.smithi093.stdout:2052: writing smithi093153278-480 from 1978646 to 2414455 tid 2 2023-08-27T20:53:35.536 INFO:tasks.rados.rados.0.smithi093.stdout:2052: writing smithi093153278-480 from 2897814 to 3143239 tid 3 2023-08-27T20:53:35.536 INFO:tasks.rados.rados.0.smithi093.stdout:2045: done (6 left) 2023-08-27T20:53:35.536 INFO:tasks.rados.rados.0.smithi093.stdout:2046: done (5 left) 2023-08-27T20:53:35.536 INFO:tasks.rados.rados.0.smithi093.stdout:2048: done (4 left) 2023-08-27T20:53:35.536 INFO:tasks.rados.rados.0.smithi093.stdout:2049: done (3 left) 2023-08-27T20:53:35.536 INFO:tasks.rados.rados.0.smithi093.stdout:2050: done (2 left) 2023-08-27T20:53:35.536 INFO:tasks.rados.rados.0.smithi093.stdout:2051: done (1 left) 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2053: read oid 365 snap -1 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2053: expect (ObjNum 364 snap 0 seq_num 364) 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2054: read oid 20 snap -1 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2054: expect (ObjNum 1341 snap 0 seq_num 1341) 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2055: write oid 212 current snap is 0 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2052: finishing write tid 1 to smithi093153278-480 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2052: finishing write tid 2 to smithi093153278-480 2023-08-27T20:53:35.537 INFO:tasks.rados.rados.0.smithi093.stdout:2055: seq_num 1378 ranges {760046=753015,2037128=695820} 2023-08-27T20:53:35.544 INFO:tasks.rados.rados.0.smithi093.stdout:2055: writing smithi093153278-212 from 760046 to 1513061 tid 1 2023-08-27T20:53:35.551 INFO:tasks.rados.rados.0.smithi093.stdout:2055: writing smithi093153278-212 from 2037128 to 2732948 tid 2 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2056: read oid 113 snap -1 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2056: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2057: write oid 328 current snap is 0 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2052: finishing write tid 3 to smithi093153278-480 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2052: finishing write tid 4 to smithi093153278-480 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2052: finishing write tid 5 to smithi093153278-480 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 480 v 341 (ObjNum 1377 snap 0 seq_num 1377) dirty exists 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2052: left oid 480 (ObjNum 1377 snap 0 seq_num 1377) 2023-08-27T20:53:35.552 INFO:tasks.rados.rados.0.smithi093.stdout:2053: expect (ObjNum 364 snap 0 seq_num 364) 2023-08-27T20:53:35.618 INFO:tasks.rados.rados.0.smithi093.stdout:2054: expect (ObjNum 1341 snap 0 seq_num 1341) 2023-08-27T20:53:35.731 INFO:tasks.rados.rados.0.smithi093.stdout:2055: finishing write tid 1 to smithi093153278-212 2023-08-27T20:53:35.732 INFO:tasks.rados.rados.0.smithi093.stdout:2055: finishing write tid 2 to smithi093153278-212 2023-08-27T20:53:35.732 INFO:tasks.rados.rados.0.smithi093.stdout:2055: finishing write tid 3 to smithi093153278-212 2023-08-27T20:53:35.732 INFO:tasks.rados.rados.0.smithi093.stdout:2055: finishing write tid 4 to smithi093153278-212 2023-08-27T20:53:35.732 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 212 v 256 (ObjNum 1378 snap 0 seq_num 1378) dirty exists 2023-08-27T20:53:35.732 INFO:tasks.rados.rados.0.smithi093.stdout:2055: left oid 212 (ObjNum 1378 snap 0 seq_num 1378) 2023-08-27T20:53:35.732 INFO:tasks.rados.rados.0.smithi093.stdout:2056: expect (ObjNum 112 snap 0 seq_num 112) 2023-08-27T20:53:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:53:35] "GET /metrics HTTP/1.1" 200 - "" "Prometheus/2.43.0" 2023-08-27T20:53:35.853 INFO:tasks.rados.rados.0.smithi093.stdout:2057: seq_num 1379 ranges {776853=428264,1979257=439478,2836307=340546} 2023-08-27T20:53:35.860 INFO:tasks.rados.rados.0.smithi093.stdout:2057: writing smithi093153278-328 from 776853 to 1205117 tid 1 2023-08-27T20:53:35.867 INFO:tasks.rados.rados.0.smithi093.stdout:2057: writing smithi093153278-328 from 1979257 to 2418735 tid 2 2023-08-27T20:53:35.871 INFO:tasks.rados.rados.0.smithi093.stdout:2057: writing smithi093153278-328 from 2836307 to 3176853 tid 3 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2052: done (5 left) 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2053: done (4 left) 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2054: done (3 left) 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2055: done (2 left) 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2056: done (1 left) 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2058: read oid 36 snap -1 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2058: expect (ObjNum 1229 snap 0 seq_num 1229) 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2059: write oid 278 current snap is 0 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2057: finishing write tid 1 to smithi093153278-328 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2057: finishing write tid 2 to smithi093153278-328 2023-08-27T20:53:35.872 INFO:tasks.rados.rados.0.smithi093.stdout:2059: seq_num 1380 ranges {793660=503513,2037739=583136} 2023-08-27T20:53:35.878 INFO:tasks.rados.rados.0.smithi093.stdout:2059: writing smithi093153278-278 from 793660 to 1297173 tid 1 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:2059: writing smithi093153278-278 from 2037739 to 2620875 tid 2 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:2060: delete oid 208 current snap is 0 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:2057: finishing write tid 3 to smithi093153278-328 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:2057: finishing write tid 4 to smithi093153278-328 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:2057: finishing write tid 5 to smithi093153278-328 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 287 (ObjNum 1379 snap 0 seq_num 1379) dirty exists 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:2057: left oid 328 (ObjNum 1379 snap 0 seq_num 1379) 2023-08-27T20:53:35.886 INFO:tasks.rados.rados.0.smithi093.stdout:2058: expect (ObjNum 1229 snap 0 seq_num 1229) 2023-08-27T20:53:36.038 INFO:tasks.rados.rados.0.smithi093.stdout:2059: finishing write tid 1 to smithi093153278-278 2023-08-27T20:53:36.039 INFO:tasks.rados.rados.0.smithi093.stdout:2059: finishing write tid 2 to smithi093153278-278 2023-08-27T20:53:36.039 INFO:tasks.rados.rados.0.smithi093.stdout:2059: finishing write tid 3 to smithi093153278-278 2023-08-27T20:53:36.039 INFO:tasks.rados.rados.0.smithi093.stdout:2059: finishing write tid 4 to smithi093153278-278 2023-08-27T20:53:36.039 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 278 v 368 (ObjNum 1380 snap 0 seq_num 1380) dirty exists 2023-08-27T20:53:36.039 INFO:tasks.rados.rados.0.smithi093.stdout:2059: left oid 278 (ObjNum 1380 snap 0 seq_num 1380) 2023-08-27T20:53:36.039 INFO:tasks.rados.rados.0.smithi093.stdout:2057: done (3 left) 2023-08-27T20:53:36.039 INFO:tasks.rados.rados.0.smithi093.stdout:2058: done (2 left) 2023-08-27T20:53:36.040 INFO:tasks.rados.rados.0.smithi093.stdout:2059: done (1 left) 2023-08-27T20:53:36.040 INFO:tasks.rados.rados.0.smithi093.stdout:2060: done (0 left) 2023-08-27T20:53:36.040 INFO:tasks.rados.rados.0.smithi093.stdout:2061: read oid 405 snap -1 2023-08-27T20:53:36.040 INFO:tasks.rados.rados.0.smithi093.stdout:2061: expect deleted 2023-08-27T20:53:36.040 INFO:tasks.rados.rados.0.smithi093.stdout:2062: read oid 467 snap -1 2023-08-27T20:53:36.040 INFO:tasks.rados.rados.0.smithi093.stdout:2062: expect deleted 2023-08-27T20:53:36.041 INFO:tasks.rados.rados.0.smithi093.stdout:2063: delete oid 273 current snap is 0 2023-08-27T20:53:36.041 INFO:tasks.rados.rados.0.smithi093.stdout:2061: done (2 left) 2023-08-27T20:53:36.041 INFO:tasks.rados.rados.0.smithi093.stdout:2062: done (1 left) 2023-08-27T20:53:36.041 INFO:tasks.rados.rados.0.smithi093.stdout:2063: done (0 left) 2023-08-27T20:53:36.042 INFO:tasks.rados.rados.0.smithi093.stdout:2064: read oid 9 snap -1 2023-08-27T20:53:36.042 INFO:tasks.rados.rados.0.smithi093.stdout:2064: expect (ObjNum 1200 snap 0 seq_num 1200) 2023-08-27T20:53:36.042 INFO:tasks.rados.rados.0.smithi093.stdout:2065: write oid 15 current snap is 0 2023-08-27T20:53:36.042 INFO:tasks.rados.rados.0.smithi093.stdout:2065: seq_num 1383 ranges {444081=729260,1813185=446816,2945999=298082} 2023-08-27T20:53:36.046 INFO:tasks.rados.rados.0.smithi093.stdout:2065: writing smithi093153278-15 from 444081 to 1173341 tid 1 2023-08-27T20:53:36.052 INFO:tasks.rados.rados.0.smithi093.stdout:2065: writing smithi093153278-15 from 1813185 to 2260001 tid 2 2023-08-27T20:53:36.058 INFO:tasks.rados.rados.0.smithi093.stdout:2065: writing smithi093153278-15 from 2945999 to 3244081 tid 3 2023-08-27T20:53:36.058 INFO:tasks.rados.rados.0.smithi093.stdout:2066: read oid 499 snap -1 2023-08-27T20:53:36.058 INFO:tasks.rados.rados.0.smithi093.stdout:2066: expect (ObjNum 953 snap 0 seq_num 953) 2023-08-27T20:53:36.058 INFO:tasks.rados.rados.0.smithi093.stdout:2067: write oid 31 current snap is 0 2023-08-27T20:53:36.058 INFO:tasks.rados.rados.0.smithi093.stdout:2067: seq_num 1384 ranges {460888=520862,1471667=590474,2573422=455407} 2023-08-27T20:53:36.063 INFO:tasks.rados.rados.0.smithi093.stdout:2067: writing smithi093153278-31 from 460888 to 981750 tid 1 2023-08-27T20:53:36.070 INFO:tasks.rados.rados.0.smithi093.stdout:2067: writing smithi093153278-31 from 1471667 to 2062141 tid 2 2023-08-27T20:53:36.075 INFO:tasks.rados.rados.0.smithi093.stdout:2067: writing smithi093153278-31 from 2573422 to 3028829 tid 3 2023-08-27T20:53:36.075 INFO:tasks.rados.rados.0.smithi093.stdout:2068: write oid 417 current snap is 0 2023-08-27T20:53:36.075 INFO:tasks.rados.rados.0.smithi093.stdout:2068: seq_num 1385 ranges {477695=596111,1530149=450485,2600845=666679} 2023-08-27T20:53:36.081 INFO:tasks.rados.rados.0.smithi093.stdout:2068: writing smithi093153278-417 from 477695 to 1073806 tid 1 2023-08-27T20:53:36.086 INFO:tasks.rados.rados.0.smithi093.stdout:2068: writing smithi093153278-417 from 1530149 to 1980634 tid 2 2023-08-27T20:53:36.093 INFO:tasks.rados.rados.0.smithi093.stdout:2068: writing smithi093153278-417 from 2600845 to 3267524 tid 3 2023-08-27T20:53:36.093 INFO:tasks.rados.rados.0.smithi093.stdout:2069: read oid 474 snap -1 2023-08-27T20:53:36.093 INFO:tasks.rados.rados.0.smithi093.stdout:2069: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:53:36.093 INFO:tasks.rados.rados.0.smithi093.stdout:2070: delete oid 10 current snap is 0 2023-08-27T20:53:36.093 INFO:tasks.rados.rados.0.smithi093.stdout:2064: expect (ObjNum 1200 snap 0 seq_num 1200) 2023-08-27T20:53:36.185 INFO:tasks.rados.rados.0.smithi093.stdout:2065: finishing write tid 1 to smithi093153278-15 2023-08-27T20:53:36.185 INFO:tasks.rados.rados.0.smithi093.stdout:2065: finishing write tid 2 to smithi093153278-15 2023-08-27T20:53:36.185 INFO:tasks.rados.rados.0.smithi093.stdout:2064: done (6 left) 2023-08-27T20:53:36.185 INFO:tasks.rados.rados.0.smithi093.stdout:2070: done (5 left) 2023-08-27T20:53:36.185 INFO:tasks.rados.rados.0.smithi093.stdout:2071: read oid 34 snap -1 2023-08-27T20:53:36.185 INFO:tasks.rados.rados.0.smithi093.stdout:2071: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2072: read oid 356 snap -1 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2072: expect (ObjNum 355 snap 0 seq_num 355) 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2073: write oid 421 current snap is 0 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2065: finishing write tid 3 to smithi093153278-15 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2065: finishing write tid 4 to smithi093153278-15 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2065: finishing write tid 5 to smithi093153278-15 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 15 v 308 (ObjNum 1383 snap 0 seq_num 1383) dirty exists 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2065: left oid 15 (ObjNum 1383 snap 0 seq_num 1383) 2023-08-27T20:53:36.186 INFO:tasks.rados.rados.0.smithi093.stdout:2073: seq_num 1387 ranges {511309=746609,2047113=454154,3055691=255618} 2023-08-27T20:53:36.191 INFO:tasks.rados.rados.0.smithi093.stdout:2073: writing smithi093153278-421 from 511309 to 1257918 tid 1 2023-08-27T20:53:36.198 INFO:tasks.rados.rados.0.smithi093.stdout:2073: writing smithi093153278-421 from 2047113 to 2501267 tid 2 2023-08-27T20:53:36.203 INFO:tasks.rados.rados.0.smithi093.stdout:2073: writing smithi093153278-421 from 3055691 to 3311309 tid 3 2023-08-27T20:53:36.204 INFO:tasks.rados.rados.0.smithi093.stdout:2065: done (7 left) 2023-08-27T20:53:36.204 INFO:tasks.rados.rados.0.smithi093.stdout:2074: read oid 227 snap -1 2023-08-27T20:53:36.204 INFO:tasks.rados.rados.0.smithi093.stdout:2074: expect (ObjNum 226 snap 0 seq_num 226) 2023-08-27T20:53:36.204 INFO:tasks.rados.rados.0.smithi093.stdout:2075: write oid 272 current snap is 0 2023-08-27T20:53:36.204 INFO:tasks.rados.rados.0.smithi093.stdout:2066: expect (ObjNum 953 snap 0 seq_num 953) 2023-08-27T20:53:36.284 INFO:tasks.rados.rados.0.smithi093.stdout:2067: finishing write tid 1 to smithi093153278-31 2023-08-27T20:53:36.284 INFO:tasks.rados.rados.0.smithi093.stdout:2075: seq_num 1388 ranges {528116=421858,1589242=597812,2966761=361355} 2023-08-27T20:53:36.289 INFO:tasks.rados.rados.0.smithi093.stdout:2075: writing smithi093153278-272 from 528116 to 949974 tid 1 2023-08-27T20:53:36.296 INFO:tasks.rados.rados.0.smithi093.stdout:2075: writing smithi093153278-272 from 1589242 to 2187054 tid 2 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2075: writing smithi093153278-272 from 2966761 to 3328116 tid 3 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2066: done (8 left) 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2076: read oid 204 snap -1 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2076: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2077: read oid 301 snap -1 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2077: expect (ObjNum 1298 snap 0 seq_num 1298) 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2078: read oid 32 snap -1 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2078: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2079: write oid 61 current snap is 0 2023-08-27T20:53:36.303 INFO:tasks.rados.rados.0.smithi093.stdout:2067: finishing write tid 2 to smithi093153278-31 2023-08-27T20:53:36.304 INFO:tasks.rados.rados.0.smithi093.stdout:2067: finishing write tid 3 to smithi093153278-31 2023-08-27T20:53:36.304 INFO:tasks.rados.rados.0.smithi093.stdout:2079: seq_num 1389 ranges {544923=497107,1647724=457823,2710537=428120} 2023-08-27T20:53:36.308 INFO:tasks.rados.rados.0.smithi093.stdout:2079: writing smithi093153278-61 from 544923 to 1042030 tid 1 2023-08-27T20:53:36.314 INFO:tasks.rados.rados.0.smithi093.stdout:2079: writing smithi093153278-61 from 1647724 to 2105547 tid 2 2023-08-27T20:53:36.320 INFO:tasks.rados.rados.0.smithi093.stdout:2079: writing smithi093153278-61 from 2710537 to 3138657 tid 3 2023-08-27T20:53:36.320 INFO:tasks.rados.rados.0.smithi093.stdout:2080: write oid 89 current snap is 0 2023-08-27T20:53:36.320 INFO:tasks.rados.rados.0.smithi093.stdout:2067: finishing write tid 4 to smithi093153278-31 2023-08-27T20:53:36.321 INFO:tasks.rados.rados.0.smithi093.stdout:2067: finishing write tid 5 to smithi093153278-31 2023-08-27T20:53:36.321 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 31 v 245 (ObjNum 1384 snap 0 seq_num 1384) dirty exists 2023-08-27T20:53:36.321 INFO:tasks.rados.rados.0.smithi093.stdout:2067: left oid 31 (ObjNum 1384 snap 0 seq_num 1384) 2023-08-27T20:53:36.321 INFO:tasks.rados.rados.0.smithi093.stdout:2068: finishing write tid 1 to smithi093153278-417 2023-08-27T20:53:36.321 INFO:tasks.rados.rados.0.smithi093.stdout:2068: finishing write tid 2 to smithi093153278-417 2023-08-27T20:53:36.321 INFO:tasks.rados.rados.0.smithi093.stdout:2080: seq_num 1390 ranges {561730=572356,1706206=601481,3021607=340123} 2023-08-27T20:53:36.326 INFO:tasks.rados.rados.0.smithi093.stdout:2080: writing smithi093153278-89 from 561730 to 1134086 tid 1 2023-08-27T20:53:36.333 INFO:tasks.rados.rados.0.smithi093.stdout:2080: writing smithi093153278-89 from 1706206 to 2307687 tid 2 2023-08-27T20:53:36.339 INFO:tasks.rados.rados.0.smithi093.stdout:2080: writing smithi093153278-89 from 3021607 to 3361730 tid 3 2023-08-27T20:53:36.339 INFO:tasks.rados.rados.0.smithi093.stdout:2067: done (12 left) 2023-08-27T20:53:36.339 INFO:tasks.rados.rados.0.smithi093.stdout:2081: read oid 358 snap -1 2023-08-27T20:53:36.339 INFO:tasks.rados.rados.0.smithi093.stdout:2081: expect (ObjNum 806 snap 0 seq_num 806) 2023-08-27T20:53:36.339 INFO:tasks.rados.rados.0.smithi093.stdout:2082: write oid 329 current snap is 0 2023-08-27T20:53:36.339 INFO:tasks.rados.rados.0.smithi093.stdout:2068: finishing write tid 3 to smithi093153278-417 2023-08-27T20:53:36.340 INFO:tasks.rados.rados.0.smithi093.stdout:2068: finishing write tid 4 to smithi093153278-417 2023-08-27T20:53:36.340 INFO:tasks.rados.rados.0.smithi093.stdout:2068: finishing write tid 5 to smithi093153278-417 2023-08-27T20:53:36.340 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 417 v 313 (ObjNum 1385 snap 0 seq_num 1385) dirty exists 2023-08-27T20:53:36.340 INFO:tasks.rados.rados.0.smithi093.stdout:2068: left oid 417 (ObjNum 1385 snap 0 seq_num 1385) 2023-08-27T20:53:36.340 INFO:tasks.rados.rados.0.smithi093.stdout:2082: seq_num 1391 ranges {578537=647605,1764688=745139,3049030=329507} 2023-08-27T20:53:36.346 INFO:tasks.rados.rados.0.smithi093.stdout:2082: writing smithi093153278-329 from 578537 to 1226142 tid 1 2023-08-27T20:53:36.353 INFO:tasks.rados.rados.0.smithi093.stdout:2082: writing smithi093153278-329 from 1764688 to 2509827 tid 2 2023-08-27T20:53:36.358 INFO:tasks.rados.rados.0.smithi093.stdout:2082: writing smithi093153278-329 from 3049030 to 3378537 tid 3 2023-08-27T20:53:36.358 INFO:tasks.rados.rados.0.smithi093.stdout:2068: done (13 left) 2023-08-27T20:53:36.358 INFO:tasks.rados.rados.0.smithi093.stdout:2083: read oid 317 snap -1 2023-08-27T20:53:36.358 INFO:tasks.rados.rados.0.smithi093.stdout:2083: expect (ObjNum 1293 snap 0 seq_num 1293) 2023-08-27T20:53:36.358 INFO:tasks.rados.rados.0.smithi093.stdout:2084: write oid 434 current snap is 0 2023-08-27T20:53:36.358 INFO:tasks.rados.rados.0.smithi093.stdout:2084: seq_num 1392 ranges {595344=439207,1823170=605150,3076453=318891} 2023-08-27T20:53:36.363 INFO:tasks.rados.rados.0.smithi093.stdout:2084: writing smithi093153278-434 from 595344 to 1034551 tid 1 2023-08-27T20:53:36.371 INFO:tasks.rados.rados.0.smithi093.stdout:2084: writing smithi093153278-434 from 1823170 to 2428320 tid 2 2023-08-27T20:53:36.376 INFO:tasks.rados.rados.0.smithi093.stdout:2084: writing smithi093153278-434 from 3076453 to 3395344 tid 3 2023-08-27T20:53:36.376 INFO:tasks.rados.rados.0.smithi093.stdout:2085: write oid 120 current snap is 0 2023-08-27T20:53:36.377 INFO:tasks.rados.rados.0.smithi093.stdout:2069: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:53:36.473 INFO:tasks.rados.rados.0.smithi093.stdout:2071: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:36.556 INFO:tasks.rados.rados.0.smithi093.stdout:2072: expect (ObjNum 355 snap 0 seq_num 355) 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:2073: finishing write tid 1 to smithi093153278-421 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:2073: finishing write tid 2 to smithi093153278-421 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:2073: finishing write tid 3 to smithi093153278-421 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:2073: finishing write tid 4 to smithi093153278-421 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:2073: finishing write tid 5 to smithi093153278-421 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 421 v 312 (ObjNum 1387 snap 0 seq_num 1387) dirty exists 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:2073: left oid 421 (ObjNum 1387 snap 0 seq_num 1387) 2023-08-27T20:53:36.677 INFO:tasks.rados.rados.0.smithi093.stdout:2085: seq_num 1393 ranges {612151=514456,1881652=748808,3103876=308275} 2023-08-27T20:53:36.682 INFO:tasks.rados.rados.0.smithi093.stdout:2085: writing smithi093153278-120 from 612151 to 1126607 tid 1 2023-08-27T20:53:36.690 INFO:tasks.rados.rados.0.smithi093.stdout:2085: writing smithi093153278-120 from 1881652 to 2630460 tid 2 2023-08-27T20:53:36.695 INFO:tasks.rados.rados.0.smithi093.stdout:2085: writing smithi093153278-120 from 3103876 to 3412151 tid 3 2023-08-27T20:53:36.695 INFO:tasks.rados.rados.0.smithi093.stdout:2069: done (15 left) 2023-08-27T20:53:36.695 INFO:tasks.rados.rados.0.smithi093.stdout:2071: done (14 left) 2023-08-27T20:53:36.695 INFO:tasks.rados.rados.0.smithi093.stdout:2072: done (13 left) 2023-08-27T20:53:36.695 INFO:tasks.rados.rados.0.smithi093.stdout:2073: done (12 left) 2023-08-27T20:53:36.695 INFO:tasks.rados.rados.0.smithi093.stdout:2086: write oid 14 current snap is 0 2023-08-27T20:53:36.695 INFO:tasks.rados.rados.0.smithi093.stdout:2086: seq_num 1394 ranges {628958=589705,1940134=608819,3131299=297659} 2023-08-27T20:53:36.701 INFO:tasks.rados.rados.0.smithi093.stdout:2086: writing smithi093153278-14 from 628958 to 1218663 tid 1 2023-08-27T20:53:36.708 INFO:tasks.rados.rados.0.smithi093.stdout:2086: writing smithi093153278-14 from 1940134 to 2548953 tid 2 2023-08-27T20:53:36.713 INFO:tasks.rados.rados.0.smithi093.stdout:2086: writing smithi093153278-14 from 3131299 to 3428958 tid 3 2023-08-27T20:53:36.714 INFO:tasks.rados.rados.0.smithi093.stdout:2087: write oid 454 current snap is 0 2023-08-27T20:53:36.714 INFO:tasks.rados.rados.0.smithi093.stdout:2074: expect (ObjNum 226 snap 0 seq_num 226) 2023-08-27T20:53:36.828 INFO:tasks.rados.rados.0.smithi093.stdout:2075: finishing write tid 1 to smithi093153278-272 2023-08-27T20:53:36.828 INFO:tasks.rados.rados.0.smithi093.stdout:2075: finishing write tid 2 to smithi093153278-272 2023-08-27T20:53:36.828 INFO:tasks.rados.rados.0.smithi093.stdout:2076: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:53:36.932 INFO:tasks.rados.rados.0.smithi093.stdout:2078: expect (ObjNum 535 snap 0 seq_num 535) 2023-08-27T20:53:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1603542069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1603542069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1603542069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:37.106 INFO:tasks.rados.rados.0.smithi093.stdout:2075: finishing write tid 3 to smithi093153278-272 2023-08-27T20:53:37.106 INFO:tasks.rados.rados.0.smithi093.stdout:2075: finishing write tid 4 to smithi093153278-272 2023-08-27T20:53:37.106 INFO:tasks.rados.rados.0.smithi093.stdout:2075: finishing write tid 5 to smithi093153278-272 2023-08-27T20:53:37.106 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 272 v 317 (ObjNum 1388 snap 0 seq_num 1388) dirty exists 2023-08-27T20:53:37.106 INFO:tasks.rados.rados.0.smithi093.stdout:2075: left oid 272 (ObjNum 1388 snap 0 seq_num 1388) 2023-08-27T20:53:37.106 INFO:tasks.rados.rados.0.smithi093.stdout:2079: finishing write tid 1 to smithi093153278-61 2023-08-27T20:53:37.106 INFO:tasks.rados.rados.0.smithi093.stdout:2087: seq_num 1395 ranges {645765=664954,1998616=752477,3158722=287043} 2023-08-27T20:53:37.113 INFO:tasks.rados.rados.0.smithi093.stdout:2087: writing smithi093153278-454 from 645765 to 1310719 tid 1 2023-08-27T20:53:37.120 INFO:tasks.rados.rados.0.smithi093.stdout:2087: writing smithi093153278-454 from 1998616 to 2751093 tid 2 2023-08-27T20:53:37.124 INFO:tasks.rados.rados.0.smithi093.stdout:2087: writing smithi093153278-454 from 3158722 to 3445765 tid 3 2023-08-27T20:53:37.125 INFO:tasks.rados.rados.0.smithi093.stdout:2074: done (13 left) 2023-08-27T20:53:37.125 INFO:tasks.rados.rados.0.smithi093.stdout:2075: done (12 left) 2023-08-27T20:53:37.125 INFO:tasks.rados.rados.0.smithi093.stdout:2076: done (11 left) 2023-08-27T20:53:37.125 INFO:tasks.rados.rados.0.smithi093.stdout:2078: done (10 left) 2023-08-27T20:53:37.125 INFO:tasks.rados.rados.0.smithi093.stdout:2088: write oid 371 current snap is 0 2023-08-27T20:53:37.125 INFO:tasks.rados.rados.0.smithi093.stdout:2077: expect (ObjNum 1298 snap 0 seq_num 1298) 2023-08-27T20:53:37.351 INFO:tasks.rados.rados.0.smithi093.stdout:2079: finishing write tid 2 to smithi093153278-61 2023-08-27T20:53:37.352 INFO:tasks.rados.rados.0.smithi093.stdout:2079: finishing write tid 3 to smithi093153278-61 2023-08-27T20:53:37.352 INFO:tasks.rados.rados.0.smithi093.stdout:2079: finishing write tid 4 to smithi093153278-61 2023-08-27T20:53:37.352 INFO:tasks.rados.rados.0.smithi093.stdout:2079: finishing write tid 5 to smithi093153278-61 2023-08-27T20:53:37.352 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 61 v 291 (ObjNum 1389 snap 0 seq_num 1389) dirty exists 2023-08-27T20:53:37.352 INFO:tasks.rados.rados.0.smithi093.stdout:2079: left oid 61 (ObjNum 1389 snap 0 seq_num 1389) 2023-08-27T20:53:37.352 INFO:tasks.rados.rados.0.smithi093.stdout:2080: finishing write tid 1 to smithi093153278-89 2023-08-27T20:53:37.352 INFO:tasks.rados.rados.0.smithi093.stdout:2088: seq_num 1396 ranges {662572=740203,1940745=612488,3069792=392780} 2023-08-27T20:53:37.359 INFO:tasks.rados.rados.0.smithi093.stdout:2088: writing smithi093153278-371 from 662572 to 1402775 tid 1 2023-08-27T20:53:37.365 INFO:tasks.rados.rados.0.smithi093.stdout:2088: writing smithi093153278-371 from 1940745 to 2553233 tid 2 2023-08-27T20:53:37.370 INFO:tasks.rados.rados.0.smithi093.stdout:2088: writing smithi093153278-371 from 3069792 to 3462572 tid 3 2023-08-27T20:53:37.370 INFO:tasks.rados.rados.0.smithi093.stdout:2077: done (10 left) 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2079: done (9 left) 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2089: write oid 124 current snap is 0 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2080: finishing write tid 2 to smithi093153278-89 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2080: finishing write tid 3 to smithi093153278-89 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2080: finishing write tid 4 to smithi093153278-89 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2080: finishing write tid 5 to smithi093153278-89 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 89 v 266 (ObjNum 1390 snap 0 seq_num 1390) dirty exists 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2080: left oid 89 (ObjNum 1390 snap 0 seq_num 1390) 2023-08-27T20:53:37.371 INFO:tasks.rados.rados.0.smithi093.stdout:2089: seq_num 1397 ranges {679379=415452,1599227=756146,3097215=382164} 2023-08-27T20:53:37.376 INFO:tasks.rados.rados.0.smithi093.stdout:2089: writing smithi093153278-124 from 679379 to 1094831 tid 1 2023-08-27T20:53:37.383 INFO:tasks.rados.rados.0.smithi093.stdout:2089: writing smithi093153278-124 from 1599227 to 2355373 tid 2 2023-08-27T20:53:37.389 INFO:tasks.rados.rados.0.smithi093.stdout:2089: writing smithi093153278-124 from 3097215 to 3479379 tid 3 2023-08-27T20:53:37.390 INFO:tasks.rados.rados.0.smithi093.stdout:2080: done (9 left) 2023-08-27T20:53:37.390 INFO:tasks.rados.rados.0.smithi093.stdout:2090: read oid 258 snap -1 2023-08-27T20:53:37.390 INFO:tasks.rados.rados.0.smithi093.stdout:2090: expect (ObjNum 1353 snap 0 seq_num 1353) 2023-08-27T20:53:37.390 INFO:tasks.rados.rados.0.smithi093.stdout:2091: read oid 7 snap -1 2023-08-27T20:53:37.390 INFO:tasks.rados.rados.0.smithi093.stdout:2091: expect (ObjNum 976 snap 0 seq_num 976) 2023-08-27T20:53:37.390 INFO:tasks.rados.rados.0.smithi093.stdout:2092: write oid 328 current snap is 0 2023-08-27T20:53:37.390 INFO:tasks.rados.rados.0.smithi093.stdout:2081: expect (ObjNum 806 snap 0 seq_num 806) 2023-08-27T20:53:37.528 INFO:tasks.rados.rados.0.smithi093.stdout:2082: finishing write tid 1 to smithi093153278-329 2023-08-27T20:53:37.528 INFO:tasks.rados.rados.0.smithi093.stdout:2082: finishing write tid 2 to smithi093153278-329 2023-08-27T20:53:37.528 INFO:tasks.rados.rados.0.smithi093.stdout:2092: seq_num 1398 ranges {696186=490701,1657709=616157,2724638=562274} 2023-08-27T20:53:37.534 INFO:tasks.rados.rados.0.smithi093.stdout:2092: writing smithi093153278-328 from 696186 to 1186887 tid 1 2023-08-27T20:53:37.540 INFO:tasks.rados.rados.0.smithi093.stdout:2092: writing smithi093153278-328 from 1657709 to 2273866 tid 2 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2092: writing smithi093153278-328 from 2724638 to 3286912 tid 3 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2081: done (11 left) 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2093: delete oid 178 current snap is 0 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2082: finishing write tid 3 to smithi093153278-329 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2082: finishing write tid 4 to smithi093153278-329 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2082: finishing write tid 5 to smithi093153278-329 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 329 v 272 (ObjNum 1391 snap 0 seq_num 1391) dirty exists 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2082: left oid 329 (ObjNum 1391 snap 0 seq_num 1391) 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2084: finishing write tid 1 to smithi093153278-434 2023-08-27T20:53:37.546 INFO:tasks.rados.rados.0.smithi093.stdout:2083: expect (ObjNum 1293 snap 0 seq_num 1293) 2023-08-27T20:53:37.749 INFO:tasks.rados.rados.0.smithi093.stdout:2084: finishing write tid 2 to smithi093153278-434 2023-08-27T20:53:37.749 INFO:tasks.rados.rados.0.smithi093.stdout:2084: finishing write tid 3 to smithi093153278-434 2023-08-27T20:53:37.749 INFO:tasks.rados.rados.0.smithi093.stdout:2084: finishing write tid 4 to smithi093153278-434 2023-08-27T20:53:37.749 INFO:tasks.rados.rados.0.smithi093.stdout:2084: finishing write tid 5 to smithi093153278-434 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 259 (ObjNum 1392 snap 0 seq_num 1392) dirty exists 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2084: left oid 434 (ObjNum 1392 snap 0 seq_num 1392) 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2082: done (11 left) 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2083: done (10 left) 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2084: done (9 left) 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2093: done (8 left) 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2094: read oid 486 snap -1 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2094: expect (ObjNum 634 snap 0 seq_num 634) 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2095: delete oid 109 current snap is 0 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2085: finishing write tid 1 to smithi093153278-120 2023-08-27T20:53:37.750 INFO:tasks.rados.rados.0.smithi093.stdout:2085: finishing write tid 2 to smithi093153278-120 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2085: finishing write tid 3 to smithi093153278-120 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2085: finishing write tid 4 to smithi093153278-120 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2085: finishing write tid 5 to smithi093153278-120 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 120 v 345 (ObjNum 1393 snap 0 seq_num 1393) dirty exists 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2085: left oid 120 (ObjNum 1393 snap 0 seq_num 1393) 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2086: finishing write tid 1 to smithi093153278-14 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2086: finishing write tid 2 to smithi093153278-14 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2086: finishing write tid 3 to smithi093153278-14 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2086: finishing write tid 4 to smithi093153278-14 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:2086: finishing write tid 5 to smithi093153278-14 2023-08-27T20:53:37.751 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 14 v 255 (ObjNum 1394 snap 0 seq_num 1394) dirty exists 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2086: left oid 14 (ObjNum 1394 snap 0 seq_num 1394) 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2087: finishing write tid 1 to smithi093153278-454 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2087: finishing write tid 2 to smithi093153278-454 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2087: finishing write tid 3 to smithi093153278-454 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2087: finishing write tid 4 to smithi093153278-454 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2087: finishing write tid 5 to smithi093153278-454 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 454 v 321 (ObjNum 1395 snap 0 seq_num 1395) dirty exists 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2087: left oid 454 (ObjNum 1395 snap 0 seq_num 1395) 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2088: finishing write tid 1 to smithi093153278-371 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2088: finishing write tid 2 to smithi093153278-371 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2088: finishing write tid 3 to smithi093153278-371 2023-08-27T20:53:37.752 INFO:tasks.rados.rados.0.smithi093.stdout:2088: finishing write tid 4 to smithi093153278-371 2023-08-27T20:53:37.753 INFO:tasks.rados.rados.0.smithi093.stdout:2088: finishing write tid 5 to smithi093153278-371 2023-08-27T20:53:37.753 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 371 v 263 (ObjNum 1396 snap 0 seq_num 1396) dirty exists 2023-08-27T20:53:37.753 INFO:tasks.rados.rados.0.smithi093.stdout:2088: left oid 371 (ObjNum 1396 snap 0 seq_num 1396) 2023-08-27T20:53:37.753 INFO:tasks.rados.rados.0.smithi093.stdout:2089: finishing write tid 1 to smithi093153278-124 2023-08-27T20:53:37.753 INFO:tasks.rados.rados.0.smithi093.stdout:2089: finishing write tid 2 to smithi093153278-124 2023-08-27T20:53:37.753 INFO:tasks.rados.rados.0.smithi093.stdout:2090: expect (ObjNum 1353 snap 0 seq_num 1353) 2023-08-27T20:53:37.838 INFO:tasks.rados.rados.0.smithi093.stdout:2089: finishing write tid 3 to smithi093153278-124 2023-08-27T20:53:37.838 INFO:tasks.rados.rados.0.smithi093.stdout:2089: finishing write tid 4 to smithi093153278-124 2023-08-27T20:53:37.838 INFO:tasks.rados.rados.0.smithi093.stdout:2089: finishing write tid 5 to smithi093153278-124 2023-08-27T20:53:37.838 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 124 v 279 (ObjNum 1397 snap 0 seq_num 1397) dirty exists 2023-08-27T20:53:37.838 INFO:tasks.rados.rados.0.smithi093.stdout:2089: left oid 124 (ObjNum 1397 snap 0 seq_num 1397) 2023-08-27T20:53:37.838 INFO:tasks.rados.rados.0.smithi093.stdout:2091: expect (ObjNum 976 snap 0 seq_num 976) 2023-08-27T20:53:37.969 INFO:tasks.rados.rados.0.smithi093.stdout:2092: finishing write tid 1 to smithi093153278-328 2023-08-27T20:53:37.969 INFO:tasks.rados.rados.0.smithi093.stdout:2092: finishing write tid 2 to smithi093153278-328 2023-08-27T20:53:37.969 INFO:tasks.rados.rados.0.smithi093.stdout:2092: finishing write tid 3 to smithi093153278-328 2023-08-27T20:53:37.969 INFO:tasks.rados.rados.0.smithi093.stdout:2092: finishing write tid 4 to smithi093153278-328 2023-08-27T20:53:37.969 INFO:tasks.rados.rados.0.smithi093.stdout:2092: finishing write tid 5 to smithi093153278-328 2023-08-27T20:53:37.969 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 295 (ObjNum 1398 snap 0 seq_num 1398) dirty exists 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2092: left oid 328 (ObjNum 1398 snap 0 seq_num 1398) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2085: done (9 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2086: done (8 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2087: done (7 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2088: done (6 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2089: done (5 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2090: done (4 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2091: done (3 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2092: done (2 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2095: done (1 left) 2023-08-27T20:53:37.970 INFO:tasks.rados.rados.0.smithi093.stdout:2096: read oid 289 snap -1 2023-08-27T20:53:37.971 INFO:tasks.rados.rados.0.smithi093.stdout:2096: expect (ObjNum 879 snap 0 seq_num 879) 2023-08-27T20:53:37.971 INFO:tasks.rados.rados.0.smithi093.stdout:2097: write oid 208 current snap is 0 2023-08-27T20:53:37.971 INFO:tasks.rados.rados.0.smithi093.stdout:2094: expect (ObjNum 634 snap 0 seq_num 634) 2023-08-27T20:53:38.119 INFO:tasks.rados.rados.0.smithi093.stdout:2096: expect (ObjNum 879 snap 0 seq_num 879) 2023-08-27T20:53:38.226 INFO:tasks.rados.rados.0.smithi093.stdout:2097: seq_num 1401 ranges {746607=432801,1833155=763484,3206907=339700} 2023-08-27T20:53:38.232 INFO:tasks.rados.rados.0.smithi093.stdout:2097: writing smithi093153278-208 from 746607 to 1179408 tid 1 2023-08-27T20:53:38.240 INFO:tasks.rados.rados.0.smithi093.stdout:2097: writing smithi093153278-208 from 1833155 to 2596639 tid 2 2023-08-27T20:53:38.245 INFO:tasks.rados.rados.0.smithi093.stdout:2097: writing smithi093153278-208 from 3206907 to 3546607 tid 3 2023-08-27T20:53:38.245 INFO:tasks.rados.rados.0.smithi093.stdout:2094: done (2 left) 2023-08-27T20:53:38.245 INFO:tasks.rados.rados.0.smithi093.stdout:2096: done (1 left) 2023-08-27T20:53:38.245 INFO:tasks.rados.rados.0.smithi093.stdout:2098: delete oid 126 current snap is 0 2023-08-27T20:53:38.246 INFO:tasks.rados.rados.0.smithi093.stdout:2097: finishing write tid 1 to smithi093153278-208 2023-08-27T20:53:38.246 INFO:tasks.rados.rados.0.smithi093.stdout:2097: finishing write tid 2 to smithi093153278-208 2023-08-27T20:53:38.247 INFO:tasks.rados.rados.0.smithi093.stdout:2098: done (1 left) 2023-08-27T20:53:38.247 INFO:tasks.rados.rados.0.smithi093.stdout:2099: write oid 476 current snap is 0 2023-08-27T20:53:38.247 INFO:tasks.rados.rados.0.smithi093.stdout:2099: seq_num 1403 ranges {780221=583299,1950119=767153,3261753=318468} 2023-08-27T20:53:38.254 INFO:tasks.rados.rados.0.smithi093.stdout:2099: writing smithi093153278-476 from 780221 to 1363520 tid 1 2023-08-27T20:53:38.261 INFO:tasks.rados.rados.0.smithi093.stdout:2099: writing smithi093153278-476 from 1950119 to 2717272 tid 2 2023-08-27T20:53:38.267 INFO:tasks.rados.rados.0.smithi093.stdout:2099: writing smithi093153278-476 from 3261753 to 3580221 tid 3 2023-08-27T20:53:38.267 INFO:tasks.rados.rados.0.smithi093.stdout:2100: write oid 317 current snap is 0 2023-08-27T20:53:38.267 INFO:tasks.rados.rados.0.smithi093.stdout:2097: finishing write tid 3 to smithi093153278-208 2023-08-27T20:53:38.267 INFO:tasks.rados.rados.0.smithi093.stdout:2097: finishing write tid 4 to smithi093153278-208 2023-08-27T20:53:38.267 INFO:tasks.rados.rados.0.smithi093.stdout:2097: finishing write tid 5 to smithi093153278-208 2023-08-27T20:53:38.267 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 208 v 374 (ObjNum 1401 snap 0 seq_num 1401) dirty exists 2023-08-27T20:53:38.267 INFO:tasks.rados.rados.0.smithi093.stdout:2097: left oid 208 (ObjNum 1401 snap 0 seq_num 1401) 2023-08-27T20:53:38.268 INFO:tasks.rados.rados.0.smithi093.stdout:2099: finishing write tid 1 to smithi093153278-476 2023-08-27T20:53:38.268 INFO:tasks.rados.rados.0.smithi093.stdout:2100: seq_num 1404 ranges {797028=658548,1892248=510811,3172823=424205} 2023-08-27T20:53:38.274 INFO:tasks.rados.rados.0.smithi093.stdout:2100: writing smithi093153278-317 from 797028 to 1455576 tid 1 2023-08-27T20:53:38.279 INFO:tasks.rados.rados.0.smithi093.stdout:2100: writing smithi093153278-317 from 1892248 to 2403059 tid 2 2023-08-27T20:53:38.286 INFO:tasks.rados.rados.0.smithi093.stdout:2100: writing smithi093153278-317 from 3172823 to 3597028 tid 3 2023-08-27T20:53:38.286 INFO:tasks.rados.rados.0.smithi093.stdout:2097: done (2 left) 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2101: read oid 143 snap -1 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2101: expect (ObjNum 1253 snap 0 seq_num 1253) 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2102: read oid 419 snap -1 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2102: expect deleted 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2103: read oid 305 snap -1 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2103: expect (ObjNum 1275 snap 0 seq_num 1275) 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2104: read oid 203 snap -1 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2104: expect (ObjNum 1102 snap 0 seq_num 1102) 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2105: read oid 243 snap -1 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2105: expect (ObjNum 1105 snap 0 seq_num 1105) 2023-08-27T20:53:38.287 INFO:tasks.rados.rados.0.smithi093.stdout:2106: delete oid 346 current snap is 0 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:2099: finishing write tid 2 to smithi093153278-476 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:2099: finishing write tid 3 to smithi093153278-476 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:2099: finishing write tid 4 to smithi093153278-476 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:2099: finishing write tid 5 to smithi093153278-476 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 476 v 316 (ObjNum 1403 snap 0 seq_num 1403) dirty exists 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:2099: left oid 476 (ObjNum 1403 snap 0 seq_num 1403) 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:2100: finishing write tid 1 to smithi093153278-317 2023-08-27T20:53:38.288 INFO:tasks.rados.rados.0.smithi093.stdout:2100: finishing write tid 2 to smithi093153278-317 2023-08-27T20:53:38.289 INFO:tasks.rados.rados.0.smithi093.stdout:2099: done (7 left) 2023-08-27T20:53:38.289 INFO:tasks.rados.rados.0.smithi093.stdout:2102: done (6 left) 2023-08-27T20:53:38.289 INFO:tasks.rados.rados.0.smithi093.stdout:2106: done (5 left) 2023-08-27T20:53:38.289 INFO:tasks.rados.rados.0.smithi093.stdout:2107: write oid 50 current snap is 0 2023-08-27T20:53:38.289 INFO:tasks.rados.rados.0.smithi093.stdout:2107: seq_num 1406 ranges {430642=409046,1609212=514480,2827669=768803} 2023-08-27T20:53:38.293 INFO:tasks.rados.rados.0.smithi093.stdout:2107: writing smithi093153278-50 from 430642 to 839688 tid 1 2023-08-27T20:53:38.300 INFO:tasks.rados.rados.0.smithi093.stdout:2107: writing smithi093153278-50 from 1609212 to 2123692 tid 2 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2107: writing smithi093153278-50 from 2827669 to 3596472 tid 3 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2108: read oid 447 snap -1 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2108: expect (ObjNum 763 snap 0 seq_num 763) 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2109: read oid 172 snap -1 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2109: expect deleted 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2110: write oid 414 current snap is 0 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2100: finishing write tid 3 to smithi093153278-317 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2100: finishing write tid 4 to smithi093153278-317 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:2100: finishing write tid 5 to smithi093153278-317 2023-08-27T20:53:38.309 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 317 v 277 (ObjNum 1404 snap 0 seq_num 1404) dirty exists 2023-08-27T20:53:38.310 INFO:tasks.rados.rados.0.smithi093.stdout:2100: left oid 317 (ObjNum 1404 snap 0 seq_num 1404) 2023-08-27T20:53:38.310 INFO:tasks.rados.rados.0.smithi093.stdout:2110: seq_num 1407 ranges {447449=600648,1784047=774491,2971445=676004} 2023-08-27T20:53:38.314 INFO:tasks.rados.rados.0.smithi093.stdout:2110: writing smithi093153278-414 from 447449 to 1048097 tid 1 2023-08-27T20:53:38.322 INFO:tasks.rados.rados.0.smithi093.stdout:2110: writing smithi093153278-414 from 1784047 to 2558538 tid 2 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2110: writing smithi093153278-414 from 2971445 to 3647449 tid 3 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2100: done (8 left) 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2111: read oid 409 snap -1 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2111: expect (ObjNum 1101 snap 0 seq_num 1101) 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2112: read oid 297 snap -1 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2112: expect (ObjNum 693 snap 0 seq_num 693) 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2113: write oid 136 current snap is 0 2023-08-27T20:53:38.329 INFO:tasks.rados.rados.0.smithi093.stdout:2105: expect (ObjNum 1105 snap 0 seq_num 1105) 2023-08-27T20:53:38.441 INFO:tasks.rados.rados.0.smithi093.stdout:2101: expect (ObjNum 1253 snap 0 seq_num 1253) 2023-08-27T20:53:38.655 INFO:tasks.rados.rados.0.smithi093.stdout:2103: expect (ObjNum 1275 snap 0 seq_num 1275) 2023-08-27T20:53:38.781 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: Manager daemon x is unresponsive, replacing it with standby daemon y 2023-08-27T20:53:38.781 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: osdmap e73: 8 total, 8 up, 8 in 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: mgrmap e23: y(active, starting, since 0.00557249s) 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:53:38.782 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:53:38.783 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[111335]: Manager daemon y is now available 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: Manager daemon x is unresponsive, replacing it with standby daemon y 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: osdmap e73: 8 total, 8 up, 8 in 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: mgrmap e23: y(active, starting, since 0.00557249s) 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:53:38.784 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:53:38.785 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-mon[118902]: Manager daemon y is now available 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:38] ENGINE Bus STOPPING 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:38] ENGINE HTTP Server cherrypy._cpwsgi_server.CPWSGIServer(('::', 9283)) shut down 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:38] ENGINE Bus STOPPED 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:38] ENGINE Bus STARTING 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.647+0000 7f6c8f8b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.647+0000 7f6c8f8b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.786 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.647+0000 7f6c8f8b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.647+0000 7f6c8f8b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.647+0000 7f6c8f8b8700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.653+0000 7f6c8c0b1700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.653+0000 7f6c8c0b1700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.653+0000 7f6c8c0b1700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.653+0000 7f6c8c0b1700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:53:38.653+0000 7f6c8c0b1700 -1 client.0 error registering admin socket command: (17) File exists 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:38] ENGINE Serving on http://:::9283 2023-08-27T20:53:38.787 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:38 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: [27/Aug/2023:20:53:38] ENGINE Bus STARTED 2023-08-27T20:53:38.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: Manager daemon x is unresponsive, replacing it with standby daemon y 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: osdmap e73: 8 total, 8 up, 8 in 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: mgrmap e23: y(active, starting, since 0.00557249s) 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi093.szcdyz"}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata", "who": "foo.smithi136.hhkxbr"}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mgr metadata", "who": "y", "id": "y"}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 0}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 1}]: dispatch 2023-08-27T20:53:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 2}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 3}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 4}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 5}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 6}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata", "id": 7}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mds metadata"}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd metadata"}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata"}]: dispatch 2023-08-27T20:53:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:38 smithi136 ceph-mon[113531]: Manager daemon y is now available 2023-08-27T20:53:38.945 INFO:tasks.rados.rados.0.smithi093.stdout:2107: finishing write tid 1 to smithi093153278-50 2023-08-27T20:53:38.946 INFO:tasks.rados.rados.0.smithi093.stdout:2104: expect (ObjNum 1102 snap 0 seq_num 1102) 2023-08-27T20:53:39.061 INFO:tasks.rados.rados.0.smithi093.stdout:2107: finishing write tid 2 to smithi093153278-50 2023-08-27T20:53:39.061 INFO:tasks.rados.rados.0.smithi093.stdout:2113: seq_num 1408 ranges {464256=675897,1726176=518149,2882515=507700} 2023-08-27T20:53:39.067 INFO:tasks.rados.rados.0.smithi093.stdout:2113: writing smithi093153278-136 from 464256 to 1140153 tid 1 2023-08-27T20:53:39.073 INFO:tasks.rados.rados.0.smithi093.stdout:2113: writing smithi093153278-136 from 1726176 to 2244325 tid 2 2023-08-27T20:53:39.080 INFO:tasks.rados.rados.0.smithi093.stdout:2113: writing smithi093153278-136 from 2882515 to 3390215 tid 3 2023-08-27T20:53:39.080 INFO:tasks.rados.rados.0.smithi093.stdout:2101: done (10 left) 2023-08-27T20:53:39.080 INFO:tasks.rados.rados.0.smithi093.stdout:2103: done (9 left) 2023-08-27T20:53:39.081 INFO:tasks.rados.rados.0.smithi093.stdout:2104: done (8 left) 2023-08-27T20:53:39.081 INFO:tasks.rados.rados.0.smithi093.stdout:2105: done (7 left) 2023-08-27T20:53:39.081 INFO:tasks.rados.rados.0.smithi093.stdout:2114: write oid 486 current snap is 0 2023-08-27T20:53:39.081 INFO:tasks.rados.rados.0.smithi093.stdout:2114: seq_num 1409 ranges {481063=751146,1784658=778160,3309938=371125} 2023-08-27T20:53:39.087 INFO:tasks.rados.rados.0.smithi093.stdout:2114: writing smithi093153278-486 from 481063 to 1232209 tid 1 2023-08-27T20:53:39.094 INFO:tasks.rados.rados.0.smithi093.stdout:2114: writing smithi093153278-486 from 1784658 to 2562818 tid 2 2023-08-27T20:53:39.101 INFO:tasks.rados.rados.0.smithi093.stdout:2114: writing smithi093153278-486 from 3309938 to 3681063 tid 3 2023-08-27T20:53:39.101 INFO:tasks.rados.rados.0.smithi093.stdout:2115: write oid 294 current snap is 0 2023-08-27T20:53:39.101 INFO:tasks.rados.rados.0.smithi093.stdout:2107: finishing write tid 3 to smithi093153278-50 2023-08-27T20:53:39.101 INFO:tasks.rados.rados.0.smithi093.stdout:2107: finishing write tid 4 to smithi093153278-50 2023-08-27T20:53:39.101 INFO:tasks.rados.rados.0.smithi093.stdout:2107: finishing write tid 5 to smithi093153278-50 2023-08-27T20:53:39.101 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 50 v 281 (ObjNum 1406 snap 0 seq_num 1406) dirty exists 2023-08-27T20:53:39.101 INFO:tasks.rados.rados.0.smithi093.stdout:2107: left oid 50 (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:53:39.102 INFO:tasks.rados.rados.0.smithi093.stdout:2108: expect (ObjNum 763 snap 0 seq_num 763) 2023-08-27T20:53:39.246 INFO:tasks.rados.rados.0.smithi093.stdout:2110: finishing write tid 1 to smithi093153278-414 2023-08-27T20:53:39.247 INFO:tasks.rados.rados.0.smithi093.stdout:2110: finishing write tid 2 to smithi093153278-414 2023-08-27T20:53:39.247 INFO:tasks.rados.rados.0.smithi093.stdout:2110: finishing write tid 3 to smithi093153278-414 2023-08-27T20:53:39.247 INFO:tasks.rados.rados.0.smithi093.stdout:2110: finishing write tid 4 to smithi093153278-414 2023-08-27T20:53:39.247 INFO:tasks.rados.rados.0.smithi093.stdout:2112: expect (ObjNum 693 snap 0 seq_num 693) 2023-08-27T20:53:39.408 INFO:tasks.rados.rados.0.smithi093.stdout:2110: finishing write tid 5 to smithi093153278-414 2023-08-27T20:53:39.409 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 414 v 320 (ObjNum 1407 snap 0 seq_num 1407) dirty exists 2023-08-27T20:53:39.409 INFO:tasks.rados.rados.0.smithi093.stdout:2110: left oid 414 (ObjNum 1407 snap 0 seq_num 1407) 2023-08-27T20:53:39.409 INFO:tasks.rados.rados.0.smithi093.stdout:2115: seq_num 1410 ranges {497870=426395,1443140=521818,2537361=530244,3583943=113927} 2023-08-27T20:53:39.413 INFO:tasks.rados.rados.0.smithi093.stdout:2115: writing smithi093153278-294 from 497870 to 924265 tid 1 2023-08-27T20:53:39.419 INFO:tasks.rados.rados.0.smithi093.stdout:2115: writing smithi093153278-294 from 1443140 to 1964958 tid 2 2023-08-27T20:53:39.425 INFO:tasks.rados.rados.0.smithi093.stdout:2115: writing smithi093153278-294 from 2537361 to 3067605 tid 3 2023-08-27T20:53:39.430 INFO:tasks.rados.rados.0.smithi093.stdout:2115: writing smithi093153278-294 from 3583943 to 3697870 tid 4 2023-08-27T20:53:39.430 INFO:tasks.rados.rados.0.smithi093.stdout:2107: done (8 left) 2023-08-27T20:53:39.430 INFO:tasks.rados.rados.0.smithi093.stdout:2108: done (7 left) 2023-08-27T20:53:39.430 INFO:tasks.rados.rados.0.smithi093.stdout:2109: done (6 left) 2023-08-27T20:53:39.431 INFO:tasks.rados.rados.0.smithi093.stdout:2110: done (5 left) 2023-08-27T20:53:39.431 INFO:tasks.rados.rados.0.smithi093.stdout:2112: done (4 left) 2023-08-27T20:53:39.431 INFO:tasks.rados.rados.0.smithi093.stdout:2116: read oid 31 snap -1 2023-08-27T20:53:39.431 INFO:tasks.rados.rados.0.smithi093.stdout:2116: expect (ObjNum 1384 snap 0 seq_num 1384) 2023-08-27T20:53:39.431 INFO:tasks.rados.rados.0.smithi093.stdout:2117: write oid 14 current snap is 0 2023-08-27T20:53:39.432 INFO:tasks.rados.rados.0.smithi093.stdout:2117: seq_num 1411 ranges {514677=501644,1501622=781829,2964784=741516} 2023-08-27T20:53:39.435 INFO:tasks.rados.rados.0.smithi093.stdout:2117: writing smithi093153278-14 from 514677 to 1016321 tid 1 2023-08-27T20:53:39.442 INFO:tasks.rados.rados.0.smithi093.stdout:2117: writing smithi093153278-14 from 1501622 to 2283451 tid 2 2023-08-27T20:53:39.450 INFO:tasks.rados.rados.0.smithi093.stdout:2117: writing smithi093153278-14 from 2964784 to 3706300 tid 3 2023-08-27T20:53:39.450 INFO:tasks.rados.rados.0.smithi093.stdout:2118: write oid 462 current snap is 0 2023-08-27T20:53:39.450 INFO:tasks.rados.rados.0.smithi093.stdout:2111: expect (ObjNum 1101 snap 0 seq_num 1101) 2023-08-27T20:53:39.595 INFO:tasks.rados.rados.0.smithi093.stdout:2113: finishing write tid 1 to smithi093153278-136 2023-08-27T20:53:39.595 INFO:tasks.rados.rados.0.smithi093.stdout:2113: finishing write tid 2 to smithi093153278-136 2023-08-27T20:53:39.596 INFO:tasks.rados.rados.0.smithi093.stdout:2113: finishing write tid 3 to smithi093153278-136 2023-08-27T20:53:39.596 INFO:tasks.rados.rados.0.smithi093.stdout:2113: finishing write tid 4 to smithi093153278-136 2023-08-27T20:53:39.596 INFO:tasks.rados.rados.0.smithi093.stdout:2113: finishing write tid 5 to smithi093153278-136 2023-08-27T20:53:39.596 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 136 v 326 (ObjNum 1408 snap 0 seq_num 1408) dirty exists 2023-08-27T20:53:39.596 INFO:tasks.rados.rados.0.smithi093.stdout:2113: left oid 136 (ObjNum 1408 snap 0 seq_num 1408) 2023-08-27T20:53:39.596 INFO:tasks.rados.rados.0.smithi093.stdout:2114: finishing write tid 1 to smithi093153278-486 2023-08-27T20:53:39.596 INFO:tasks.rados.rados.0.smithi093.stdout:2118: seq_num 1412 ranges {531484=576893,1843751=525487,2875854=669141} 2023-08-27T20:53:39.601 INFO:tasks.rados.rados.0.smithi093.stdout:2118: writing smithi093153278-462 from 531484 to 1108377 tid 1 2023-08-27T20:53:39.608 INFO:tasks.rados.rados.0.smithi093.stdout:2118: writing smithi093153278-462 from 1843751 to 2369238 tid 2 2023-08-27T20:53:39.616 INFO:tasks.rados.rados.0.smithi093.stdout:2118: writing smithi093153278-462 from 2875854 to 3544995 tid 3 2023-08-27T20:53:39.616 INFO:tasks.rados.rados.0.smithi093.stdout:2111: done (6 left) 2023-08-27T20:53:39.616 INFO:tasks.rados.rados.0.smithi093.stdout:2113: done (5 left) 2023-08-27T20:53:39.616 INFO:tasks.rados.rados.0.smithi093.stdout:2119: read oid 483 snap -1 2023-08-27T20:53:39.616 INFO:tasks.rados.rados.0.smithi093.stdout:2119: expect (ObjNum 1367 snap 0 seq_num 1367) 2023-08-27T20:53:39.616 INFO:tasks.rados.rados.0.smithi093.stdout:2120: read oid 80 snap -1 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2120: expect (ObjNum 744 snap 0 seq_num 744) 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2121: write oid 208 current snap is 0 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2114: finishing write tid 2 to smithi093153278-486 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2114: finishing write tid 3 to smithi093153278-486 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2114: finishing write tid 4 to smithi093153278-486 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2114: finishing write tid 5 to smithi093153278-486 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 299 (ObjNum 1409 snap 0 seq_num 1409) dirty exists 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2114: left oid 486 (ObjNum 1409 snap 0 seq_num 1409) 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2115: finishing write tid 1 to smithi093153278-294 2023-08-27T20:53:39.617 INFO:tasks.rados.rados.0.smithi093.stdout:2121: seq_num 1413 ranges {548291=652142,1902233=785498,3303277=445014} 2023-08-27T20:53:39.623 INFO:tasks.rados.rados.0.smithi093.stdout:2121: writing smithi093153278-208 from 548291 to 1200433 tid 1 2023-08-27T20:53:39.631 INFO:tasks.rados.rados.0.smithi093.stdout:2121: writing smithi093153278-208 from 1902233 to 2687731 tid 2 2023-08-27T20:53:39.638 INFO:tasks.rados.rados.0.smithi093.stdout:2121: writing smithi093153278-208 from 3303277 to 3748291 tid 3 2023-08-27T20:53:39.638 INFO:tasks.rados.rados.0.smithi093.stdout:2114: done (7 left) 2023-08-27T20:53:39.638 INFO:tasks.rados.rados.0.smithi093.stdout:2122: write oid 40 current snap is 0 2023-08-27T20:53:39.638 INFO:tasks.rados.rados.0.smithi093.stdout:2115: finishing write tid 2 to smithi093153278-294 2023-08-27T20:53:39.638 INFO:tasks.rados.rados.0.smithi093.stdout:2115: finishing write tid 3 to smithi093153278-294 2023-08-27T20:53:39.638 INFO:tasks.rados.rados.0.smithi093.stdout:2115: finishing write tid 4 to smithi093153278-294 2023-08-27T20:53:39.639 INFO:tasks.rados.rados.0.smithi093.stdout:2115: finishing write tid 5 to smithi093153278-294 2023-08-27T20:53:39.639 INFO:tasks.rados.rados.0.smithi093.stdout:2115: finishing write tid 6 to smithi093153278-294 2023-08-27T20:53:39.639 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 294 v 294 (ObjNum 1410 snap 0 seq_num 1410) dirty exists 2023-08-27T20:53:39.639 INFO:tasks.rados.rados.0.smithi093.stdout:2115: left oid 294 (ObjNum 1410 snap 0 seq_num 1410) 2023-08-27T20:53:39.639 INFO:tasks.rados.rados.0.smithi093.stdout:2116: expect (ObjNum 1384 snap 0 seq_num 1384) 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: [27/Aug/2023:20:53:38] ENGINE Bus STARTING 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: [27/Aug/2023:20:53:38] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:53:39.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: [27/Aug/2023:20:53:38] ENGINE Bus STARTED 2023-08-27T20:53:39.783 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:39.783 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/66937337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:39.783 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:39 smithi136 ceph-mon[113531]: mgrmap e24: y(active, since 1.01469s) 2023-08-27T20:53:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:2117: finishing write tid 1 to smithi093153278-14 2023-08-27T20:53:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:2117: finishing write tid 2 to smithi093153278-14 2023-08-27T20:53:39.787 INFO:tasks.rados.rados.0.smithi093.stdout:2122: seq_num 1414 ranges {565098=727391,1960715=529156,2930700=691685} 2023-08-27T20:53:39.793 INFO:tasks.rados.rados.0.smithi093.stdout:2122: writing smithi093153278-40 from 565098 to 1292489 tid 1 2023-08-27T20:53:39.800 INFO:tasks.rados.rados.0.smithi093.stdout:2122: writing smithi093153278-40 from 1960715 to 2489871 tid 2 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2122: writing smithi093153278-40 from 2930700 to 3622385 tid 3 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2115: done (7 left) 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2116: done (6 left) 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2123: read oid 317 snap -1 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2123: expect (ObjNum 1404 snap 0 seq_num 1404) 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2124: write oid 376 current snap is 0 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2117: finishing write tid 3 to smithi093153278-14 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2117: finishing write tid 4 to smithi093153278-14 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:2117: finishing write tid 5 to smithi093153278-14 2023-08-27T20:53:39.808 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 14 v 259 (ObjNum 1411 snap 0 seq_num 1411) dirty exists 2023-08-27T20:53:39.809 INFO:tasks.rados.rados.0.smithi093.stdout:2117: left oid 14 (ObjNum 1411 snap 0 seq_num 1411) 2023-08-27T20:53:39.809 INFO:tasks.rados.rados.0.smithi093.stdout:2118: finishing write tid 1 to smithi093153278-462 2023-08-27T20:53:39.809 INFO:tasks.rados.rados.0.smithi093.stdout:2124: seq_num 1415 ranges {581905=518993,1735550=672814,2958123=502957} 2023-08-27T20:53:39.815 INFO:tasks.rados.rados.0.smithi093.stdout:2124: writing smithi093153278-376 from 581905 to 1100898 tid 1 2023-08-27T20:53:39.822 INFO:tasks.rados.rados.0.smithi093.stdout:2124: writing smithi093153278-376 from 1735550 to 2408364 tid 2 2023-08-27T20:53:39.828 INFO:tasks.rados.rados.0.smithi093.stdout:2124: writing smithi093153278-376 from 2958123 to 3461080 tid 3 2023-08-27T20:53:39.828 INFO:tasks.rados.rados.0.smithi093.stdout:2117: done (7 left) 2023-08-27T20:53:39.829 INFO:tasks.rados.rados.0.smithi093.stdout:2125: write oid 363 current snap is 0 2023-08-27T20:53:39.829 INFO:tasks.rados.rados.0.smithi093.stdout:2118: finishing write tid 2 to smithi093153278-462 2023-08-27T20:53:39.829 INFO:tasks.rados.rados.0.smithi093.stdout:2118: finishing write tid 3 to smithi093153278-462 2023-08-27T20:53:39.829 INFO:tasks.rados.rados.0.smithi093.stdout:2118: finishing write tid 4 to smithi093153278-462 2023-08-27T20:53:39.829 INFO:tasks.rados.rados.0.smithi093.stdout:2125: seq_num 1416 ranges {598712=594242,1677679=532825,2985546=430582} 2023-08-27T20:53:39.835 INFO:tasks.rados.rados.0.smithi093.stdout:2125: writing smithi093153278-363 from 598712 to 1192954 tid 1 2023-08-27T20:53:39.841 INFO:tasks.rados.rados.0.smithi093.stdout:2125: writing smithi093153278-363 from 1677679 to 2210504 tid 2 2023-08-27T20:53:39.848 INFO:tasks.rados.rados.0.smithi093.stdout:2125: writing smithi093153278-363 from 2985546 to 3416128 tid 3 2023-08-27T20:53:39.848 INFO:tasks.rados.rados.0.smithi093.stdout:2126: write oid 198 current snap is 0 2023-08-27T20:53:39.848 INFO:tasks.rados.rados.0.smithi093.stdout:2118: finishing write tid 5 to smithi093153278-462 2023-08-27T20:53:39.848 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 462 v 298 (ObjNum 1412 snap 0 seq_num 1412) dirty exists 2023-08-27T20:53:39.848 INFO:tasks.rados.rados.0.smithi093.stdout:2118: left oid 462 (ObjNum 1412 snap 0 seq_num 1412) 2023-08-27T20:53:39.848 INFO:tasks.rados.rados.0.smithi093.stdout:2120: expect (ObjNum 744 snap 0 seq_num 744) 2023-08-27T20:53:39.968 INFO:tasks.rados.rados.0.smithi093.stdout:2121: finishing write tid 1 to smithi093153278-208 2023-08-27T20:53:39.968 INFO:tasks.rados.rados.0.smithi093.stdout:2119: expect (ObjNum 1367 snap 0 seq_num 1367) 2023-08-27T20:53:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:53:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:53:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:53:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: [27/Aug/2023:20:53:38] ENGINE Bus STARTING 2023-08-27T20:53:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: [27/Aug/2023:20:53:38] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: [27/Aug/2023:20:53:38] ENGINE Bus STARTED 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/66937337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[118902]: mgrmap e24: y(active, since 1.01469s) 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/mirror_snapshot_schedule"}]: dispatch 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix":"config rm","who":"mgr","name":"mgr/rbd_support/y/trash_purge_schedule"}]: dispatch 2023-08-27T20:53:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: [27/Aug/2023:20:53:38] ENGINE Bus STARTING 2023-08-27T20:53:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: [27/Aug/2023:20:53:38] ENGINE Serving on https://172.21.15.93:7150 2023-08-27T20:53:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: [27/Aug/2023:20:53:38] ENGINE Bus STARTED 2023-08-27T20:53:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/66937337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:39 smithi093 ceph-mon[111335]: mgrmap e24: y(active, since 1.01469s) 2023-08-27T20:53:40.127 INFO:tasks.rados.rados.0.smithi093.stdout:2121: finishing write tid 2 to smithi093153278-208 2023-08-27T20:53:40.127 INFO:tasks.rados.rados.0.smithi093.stdout:2121: finishing write tid 3 to smithi093153278-208 2023-08-27T20:53:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:2121: finishing write tid 4 to smithi093153278-208 2023-08-27T20:53:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:2121: finishing write tid 5 to smithi093153278-208 2023-08-27T20:53:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 208 v 378 (ObjNum 1413 snap 0 seq_num 1413) dirty exists 2023-08-27T20:53:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:2121: left oid 208 (ObjNum 1413 snap 0 seq_num 1413) 2023-08-27T20:53:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:2122: finishing write tid 1 to smithi093153278-40 2023-08-27T20:53:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:2122: finishing write tid 2 to smithi093153278-40 2023-08-27T20:53:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:2126: seq_num 1417 ranges {615519=669491,1736161=676483,2896616=641854} 2023-08-27T20:53:40.134 INFO:tasks.rados.rados.0.smithi093.stdout:2126: writing smithi093153278-198 from 615519 to 1285010 tid 1 2023-08-27T20:53:40.140 INFO:tasks.rados.rados.0.smithi093.stdout:2126: writing smithi093153278-198 from 1736161 to 2412644 tid 2 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2126: writing smithi093153278-198 from 2896616 to 3538470 tid 3 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2118: done (8 left) 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2119: done (7 left) 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2120: done (6 left) 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2121: done (5 left) 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2127: read oid 418 snap -1 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2127: expect (ObjNum 849 snap 0 seq_num 849) 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2128: write oid 455 current snap is 0 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2122: finishing write tid 3 to smithi093153278-40 2023-08-27T20:53:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:2122: finishing write tid 4 to smithi093153278-40 2023-08-27T20:53:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:2128: seq_num 1418 ranges {632326=744740,1794643=536494,3040392=453126} 2023-08-27T20:53:40.155 INFO:tasks.rados.rados.0.smithi093.stdout:2128: writing smithi093153278-455 from 632326 to 1377066 tid 1 2023-08-27T20:53:40.161 INFO:tasks.rados.rados.0.smithi093.stdout:2128: writing smithi093153278-455 from 1794643 to 2331137 tid 2 2023-08-27T20:53:40.167 INFO:tasks.rados.rados.0.smithi093.stdout:2128: writing smithi093153278-455 from 3040392 to 3493518 tid 3 2023-08-27T20:53:40.167 INFO:tasks.rados.rados.0.smithi093.stdout:2129: write oid 466 current snap is 0 2023-08-27T20:53:40.168 INFO:tasks.rados.rados.0.smithi093.stdout:2122: finishing write tid 5 to smithi093153278-40 2023-08-27T20:53:40.168 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 40 v 350 (ObjNum 1414 snap 0 seq_num 1414) dirty exists 2023-08-27T20:53:40.168 INFO:tasks.rados.rados.0.smithi093.stdout:2122: left oid 40 (ObjNum 1414 snap 0 seq_num 1414) 2023-08-27T20:53:40.168 INFO:tasks.rados.rados.0.smithi093.stdout:2124: finishing write tid 1 to smithi093153278-376 2023-08-27T20:53:40.168 INFO:tasks.rados.rados.0.smithi093.stdout:2123: expect (ObjNum 1404 snap 0 seq_num 1404) 2023-08-27T20:53:40.411 INFO:tasks.rados.rados.0.smithi093.stdout:2124: finishing write tid 2 to smithi093153278-376 2023-08-27T20:53:40.411 INFO:tasks.rados.rados.0.smithi093.stdout:2124: finishing write tid 3 to smithi093153278-376 2023-08-27T20:53:40.411 INFO:tasks.rados.rados.0.smithi093.stdout:2124: finishing write tid 4 to smithi093153278-376 2023-08-27T20:53:40.412 INFO:tasks.rados.rados.0.smithi093.stdout:2124: finishing write tid 5 to smithi093153278-376 2023-08-27T20:53:40.412 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 376 v 382 (ObjNum 1415 snap 0 seq_num 1415) dirty exists 2023-08-27T20:53:40.412 INFO:tasks.rados.rados.0.smithi093.stdout:2124: left oid 376 (ObjNum 1415 snap 0 seq_num 1415) 2023-08-27T20:53:40.412 INFO:tasks.rados.rados.0.smithi093.stdout:2125: finishing write tid 1 to smithi093153278-363 2023-08-27T20:53:40.412 INFO:tasks.rados.rados.0.smithi093.stdout:2129: seq_num 1419 ranges {649133=419989,1853125=680152,2951462=664398} 2023-08-27T20:53:40.416 INFO:tasks.rados.rados.0.smithi093.stdout:2129: writing smithi093153278-466 from 649133 to 1069122 tid 1 2023-08-27T20:53:40.424 INFO:tasks.rados.rados.0.smithi093.stdout:2129: writing smithi093153278-466 from 1853125 to 2533277 tid 2 2023-08-27T20:53:40.431 INFO:tasks.rados.rados.0.smithi093.stdout:2129: writing smithi093153278-466 from 2951462 to 3615860 tid 3 2023-08-27T20:53:40.431 INFO:tasks.rados.rados.0.smithi093.stdout:2122: done (7 left) 2023-08-27T20:53:40.431 INFO:tasks.rados.rados.0.smithi093.stdout:2123: done (6 left) 2023-08-27T20:53:40.431 INFO:tasks.rados.rados.0.smithi093.stdout:2124: done (5 left) 2023-08-27T20:53:40.431 INFO:tasks.rados.rados.0.smithi093.stdout:2130: read oid 3 snap -1 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2130: expect (ObjNum 696 snap 0 seq_num 696) 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2131: write oid 84 current snap is 0 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2125: finishing write tid 2 to smithi093153278-363 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2125: finishing write tid 3 to smithi093153278-363 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2125: finishing write tid 4 to smithi093153278-363 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2125: finishing write tid 5 to smithi093153278-363 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 386 (ObjNum 1416 snap 0 seq_num 1416) dirty exists 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2125: left oid 363 (ObjNum 1416 snap 0 seq_num 1416) 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2126: finishing write tid 1 to smithi093153278-198 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2126: finishing write tid 2 to smithi093153278-198 2023-08-27T20:53:40.432 INFO:tasks.rados.rados.0.smithi093.stdout:2131: seq_num 1420 ranges {665940=495238,1795254=540163,2978885=475670} 2023-08-27T20:53:40.437 INFO:tasks.rados.rados.0.smithi093.stdout:2131: writing smithi093153278-84 from 665940 to 1161178 tid 1 2023-08-27T20:53:40.444 INFO:tasks.rados.rados.0.smithi093.stdout:2131: writing smithi093153278-84 from 1795254 to 2335417 tid 2 2023-08-27T20:53:40.451 INFO:tasks.rados.rados.0.smithi093.stdout:2131: writing smithi093153278-84 from 2978885 to 3454555 tid 3 2023-08-27T20:53:40.451 INFO:tasks.rados.rados.0.smithi093.stdout:2125: done (6 left) 2023-08-27T20:53:40.451 INFO:tasks.rados.rados.0.smithi093.stdout:2132: read oid 474 snap -1 2023-08-27T20:53:40.451 INFO:tasks.rados.rados.0.smithi093.stdout:2132: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:53:40.451 INFO:tasks.rados.rados.0.smithi093.stdout:2133: delete oid 390 current snap is 0 2023-08-27T20:53:40.452 INFO:tasks.rados.rados.0.smithi093.stdout:2126: finishing write tid 3 to smithi093153278-198 2023-08-27T20:53:40.452 INFO:tasks.rados.rados.0.smithi093.stdout:2126: finishing write tid 4 to smithi093153278-198 2023-08-27T20:53:40.452 INFO:tasks.rados.rados.0.smithi093.stdout:2126: finishing write tid 5 to smithi093153278-198 2023-08-27T20:53:40.452 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 198 v 390 (ObjNum 1417 snap 0 seq_num 1417) dirty exists 2023-08-27T20:53:40.452 INFO:tasks.rados.rados.0.smithi093.stdout:2126: left oid 198 (ObjNum 1417 snap 0 seq_num 1417) 2023-08-27T20:53:40.452 INFO:tasks.rados.rados.0.smithi093.stdout:2127: expect (ObjNum 849 snap 0 seq_num 849) 2023-08-27T20:53:40.529 INFO:tasks.rados.rados.0.smithi093.stdout:2128: finishing write tid 1 to smithi093153278-455 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2128: finishing write tid 2 to smithi093153278-455 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2128: finishing write tid 3 to smithi093153278-455 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2128: finishing write tid 4 to smithi093153278-455 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2128: finishing write tid 5 to smithi093153278-455 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 455 v 354 (ObjNum 1418 snap 0 seq_num 1418) dirty exists 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2128: left oid 455 (ObjNum 1418 snap 0 seq_num 1418) 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2129: finishing write tid 1 to smithi093153278-466 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2126: done (7 left) 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2127: done (6 left) 2023-08-27T20:53:40.530 INFO:tasks.rados.rados.0.smithi093.stdout:2128: done (5 left) 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2133: done (4 left) 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2134: read oid 409 snap -1 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2134: expect (ObjNum 1101 snap 0 seq_num 1101) 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2135: read oid 118 snap -1 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2135: expect (ObjNum 748 snap 0 seq_num 748) 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2136: write oid 267 current snap is 0 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2129: finishing write tid 2 to smithi093153278-466 2023-08-27T20:53:40.531 INFO:tasks.rados.rados.0.smithi093.stdout:2136: seq_num 1422 ranges {699554=762089,2028571=543832,3150084=614567} 2023-08-27T20:53:40.537 INFO:tasks.rados.rados.0.smithi093.stdout:2136: writing smithi093153278-267 from 699554 to 1461643 tid 1 2023-08-27T20:53:40.543 INFO:tasks.rados.rados.0.smithi093.stdout:2136: writing smithi093153278-267 from 2028571 to 2572403 tid 2 2023-08-27T20:53:40.551 INFO:tasks.rados.rados.0.smithi093.stdout:2136: writing smithi093153278-267 from 3150084 to 3764651 tid 3 2023-08-27T20:53:40.551 INFO:tasks.rados.rados.0.smithi093.stdout:2137: read oid 51 snap -1 2023-08-27T20:53:40.551 INFO:tasks.rados.rados.0.smithi093.stdout:2137: expect (ObjNum 1304 snap 0 seq_num 1304) 2023-08-27T20:53:40.551 INFO:tasks.rados.rados.0.smithi093.stdout:2138: read oid 331 snap -1 2023-08-27T20:53:40.551 INFO:tasks.rados.rados.0.smithi093.stdout:2138: expect (ObjNum 1013 snap 0 seq_num 1013) 2023-08-27T20:53:40.552 INFO:tasks.rados.rados.0.smithi093.stdout:2139: write oid 253 current snap is 0 2023-08-27T20:53:40.552 INFO:tasks.rados.rados.0.smithi093.stdout:2129: finishing write tid 3 to smithi093153278-466 2023-08-27T20:53:40.552 INFO:tasks.rados.rados.0.smithi093.stdout:2129: finishing write tid 4 to smithi093153278-466 2023-08-27T20:53:40.552 INFO:tasks.rados.rados.0.smithi093.stdout:2139: seq_num 1423 ranges {716361=437338,1687053=687490,2777507=425839,3877756=38605} 2023-08-27T20:53:40.558 INFO:tasks.rados.rados.0.smithi093.stdout:2139: writing smithi093153278-253 from 716361 to 1153699 tid 1 2023-08-27T20:53:40.565 INFO:tasks.rados.rados.0.smithi093.stdout:2139: writing smithi093153278-253 from 1687053 to 2374543 tid 2 2023-08-27T20:53:40.570 INFO:tasks.rados.rados.0.smithi093.stdout:2139: writing smithi093153278-253 from 2777507 to 3203346 tid 3 2023-08-27T20:53:40.574 INFO:tasks.rados.rados.0.smithi093.stdout:2139: writing smithi093153278-253 from 3877756 to 3916361 tid 4 2023-08-27T20:53:40.575 INFO:tasks.rados.rados.0.smithi093.stdout:2140: read oid 242 snap -1 2023-08-27T20:53:40.575 INFO:tasks.rados.rados.0.smithi093.stdout:2140: expect (ObjNum 1085 snap 0 seq_num 1085) 2023-08-27T20:53:40.575 INFO:tasks.rados.rados.0.smithi093.stdout:2141: write oid 275 current snap is 0 2023-08-27T20:53:40.575 INFO:tasks.rados.rados.0.smithi093.stdout:2129: finishing write tid 5 to smithi093153278-466 2023-08-27T20:53:40.575 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 466 v 358 (ObjNum 1419 snap 0 seq_num 1419) dirty exists 2023-08-27T20:53:40.575 INFO:tasks.rados.rados.0.smithi093.stdout:2129: left oid 466 (ObjNum 1419 snap 0 seq_num 1419) 2023-08-27T20:53:40.575 INFO:tasks.rados.rados.0.smithi093.stdout:2130: expect (ObjNum 696 snap 0 seq_num 696) 2023-08-27T20:53:40.745 INFO:tasks.rados.rados.0.smithi093.stdout:2131: finishing write tid 1 to smithi093153278-84 2023-08-27T20:53:40.745 INFO:tasks.rados.rados.0.smithi093.stdout:2131: finishing write tid 2 to smithi093153278-84 2023-08-27T20:53:40.745 INFO:tasks.rados.rados.0.smithi093.stdout:2141: seq_num 1424 ranges {733168=512587,2029182=547501,3088577=637111} 2023-08-27T20:53:40.752 INFO:tasks.rados.rados.0.smithi093.stdout:2141: writing smithi093153278-275 from 733168 to 1245755 tid 1 2023-08-27T20:53:40.759 INFO:tasks.rados.rados.0.smithi093.stdout:2141: writing smithi093153278-275 from 2029182 to 2576683 tid 2 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:2141: writing smithi093153278-275 from 3088577 to 3725688 tid 3 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:2129: done (11 left) 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:2130: done (10 left) 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:2142: delete oid 43 current snap is 0 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:2131: finishing write tid 3 to smithi093153278-84 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:2131: finishing write tid 4 to smithi093153278-84 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:2131: finishing write tid 5 to smithi093153278-84 2023-08-27T20:53:40.766 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 84 v 303 (ObjNum 1420 snap 0 seq_num 1420) dirty exists 2023-08-27T20:53:40.767 INFO:tasks.rados.rados.0.smithi093.stdout:2131: left oid 84 (ObjNum 1420 snap 0 seq_num 1420) 2023-08-27T20:53:40.767 INFO:tasks.rados.rados.0.smithi093.stdout:2132: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:53:40.865 INFO:tasks.rados.rados.0.smithi093.stdout:2131: done (10 left) 2023-08-27T20:53:40.865 INFO:tasks.rados.rados.0.smithi093.stdout:2132: done (9 left) 2023-08-27T20:53:40.865 INFO:tasks.rados.rados.0.smithi093.stdout:2142: done (8 left) 2023-08-27T20:53:40.865 INFO:tasks.rados.rados.0.smithi093.stdout:2143: write oid 309 current snap is 0 2023-08-27T20:53:40.865 INFO:tasks.rados.rados.0.smithi093.stdout:2135: expect (ObjNum 748 snap 0 seq_num 748) 2023-08-27T20:53:40.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:40 smithi136 ceph-mon[113531]: pgmap v3: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:40 smithi093 ceph-mon[111335]: pgmap v3: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:40 smithi093 ceph-mon[118902]: pgmap v3: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:41.005 INFO:tasks.rados.rados.0.smithi093.stdout:2134: expect (ObjNum 1101 snap 0 seq_num 1101) 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:2136: finishing write tid 1 to smithi093153278-267 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:2136: finishing write tid 2 to smithi093153278-267 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:2136: finishing write tid 3 to smithi093153278-267 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:2136: finishing write tid 4 to smithi093153278-267 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:2136: finishing write tid 5 to smithi093153278-267 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 267 v 263 (ObjNum 1422 snap 0 seq_num 1422) dirty exists 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:2136: left oid 267 (ObjNum 1422 snap 0 seq_num 1422) 2023-08-27T20:53:41.145 INFO:tasks.rados.rados.0.smithi093.stdout:2143: seq_num 1426 ranges {766782=663085,2146146=551170,3143423=776008} 2023-08-27T20:53:41.153 INFO:tasks.rados.rados.0.smithi093.stdout:2143: writing smithi093153278-309 from 766782 to 1429867 tid 1 2023-08-27T20:53:41.160 INFO:tasks.rados.rados.0.smithi093.stdout:2143: writing smithi093153278-309 from 2146146 to 2697316 tid 2 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2143: writing smithi093153278-309 from 3143423 to 3919431 tid 3 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2134: done (8 left) 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2135: done (7 left) 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2136: done (6 left) 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2144: read oid 163 snap -1 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2144: expect (ObjNum 1028 snap 0 seq_num 1028) 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2145: write oid 404 current snap is 0 2023-08-27T20:53:41.168 INFO:tasks.rados.rados.0.smithi093.stdout:2138: expect (ObjNum 1013 snap 0 seq_num 1013) 2023-08-27T20:53:41.355 INFO:tasks.rados.rados.0.smithi093.stdout:2139: finishing write tid 1 to smithi093153278-253 2023-08-27T20:53:41.355 INFO:tasks.rados.rados.0.smithi093.stdout:2137: expect (ObjNum 1304 snap 0 seq_num 1304) 2023-08-27T20:53:41.605 INFO:tasks.rados.rados.0.smithi093.stdout:2139: finishing write tid 2 to smithi093153278-253 2023-08-27T20:53:41.606 INFO:tasks.rados.rados.0.smithi093.stdout:2139: finishing write tid 3 to smithi093153278-253 2023-08-27T20:53:41.606 INFO:tasks.rados.rados.0.smithi093.stdout:2139: finishing write tid 4 to smithi093153278-253 2023-08-27T20:53:41.606 INFO:tasks.rados.rados.0.smithi093.stdout:2139: finishing write tid 5 to smithi093153278-253 2023-08-27T20:53:41.606 INFO:tasks.rados.rados.0.smithi093.stdout:2139: finishing write tid 6 to smithi093153278-253 2023-08-27T20:53:41.606 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 303 (ObjNum 1423 snap 0 seq_num 1423) dirty exists 2023-08-27T20:53:41.606 INFO:tasks.rados.rados.0.smithi093.stdout:2139: left oid 253 (ObjNum 1423 snap 0 seq_num 1423) 2023-08-27T20:53:41.606 INFO:tasks.rados.rados.0.smithi093.stdout:2140: expect (ObjNum 1085 snap 0 seq_num 1085) 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:2141: finishing write tid 1 to smithi093153278-275 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:2141: finishing write tid 2 to smithi093153278-275 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:2141: finishing write tid 3 to smithi093153278-275 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:2141: finishing write tid 4 to smithi093153278-275 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:2141: finishing write tid 5 to smithi093153278-275 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 275 v 267 (ObjNum 1424 snap 0 seq_num 1424) dirty exists 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:2141: left oid 275 (ObjNum 1424 snap 0 seq_num 1424) 2023-08-27T20:53:41.718 INFO:tasks.rados.rados.0.smithi093.stdout:2145: seq_num 1427 ranges {783589=738334,2204628=694828,3570846=412743} 2023-08-27T20:53:41.726 INFO:tasks.rados.rados.0.smithi093.stdout:2145: writing smithi093153278-404 from 783589 to 1521923 tid 1 2023-08-27T20:53:41.733 INFO:tasks.rados.rados.0.smithi093.stdout:2145: writing smithi093153278-404 from 2204628 to 2899456 tid 2 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2145: writing smithi093153278-404 from 3570846 to 3983589 tid 3 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2137: done (7 left) 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2138: done (6 left) 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2139: done (5 left) 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2140: done (4 left) 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2141: done (3 left) 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2146: write oid 334 current snap is 0 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2143: finishing write tid 1 to smithi093153278-309 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2143: finishing write tid 2 to smithi093153278-309 2023-08-27T20:53:41.740 INFO:tasks.rados.rados.0.smithi093.stdout:2146: seq_num 1428 ranges {400396=413583,1463110=438486} 2023-08-27T20:53:41.744 INFO:tasks.rados.rados.0.smithi093.stdout:2146: writing smithi093153278-334 from 400396 to 813979 tid 1 2023-08-27T20:53:41.750 INFO:tasks.rados.rados.0.smithi093.stdout:2146: writing smithi093153278-334 from 1463110 to 1901596 tid 2 2023-08-27T20:53:41.750 INFO:tasks.rados.rados.0.smithi093.stdout:2147: write oid 108 current snap is 0 2023-08-27T20:53:41.750 INFO:tasks.rados.rados.0.smithi093.stdout:2143: finishing write tid 3 to smithi093153278-309 2023-08-27T20:53:41.750 INFO:tasks.rados.rados.0.smithi093.stdout:2143: finishing write tid 4 to smithi093153278-309 2023-08-27T20:53:41.750 INFO:tasks.rados.rados.0.smithi093.stdout:2143: finishing write tid 5 to smithi093153278-309 2023-08-27T20:53:41.750 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 309 v 259 (ObjNum 1426 snap 0 seq_num 1426) dirty exists 2023-08-27T20:53:41.751 INFO:tasks.rados.rados.0.smithi093.stdout:2143: left oid 309 (ObjNum 1426 snap 0 seq_num 1426) 2023-08-27T20:53:41.751 INFO:tasks.rados.rados.0.smithi093.stdout:2144: expect (ObjNum 1028 snap 0 seq_num 1028) 2023-08-27T20:53:41.904 INFO:tasks.rados.rados.0.smithi093.stdout:2145: finishing write tid 1 to smithi093153278-404 2023-08-27T20:53:41.904 INFO:tasks.rados.rados.0.smithi093.stdout:2145: finishing write tid 2 to smithi093153278-404 2023-08-27T20:53:41.904 INFO:tasks.rados.rados.0.smithi093.stdout:2145: finishing write tid 3 to smithi093153278-404 2023-08-27T20:53:41.904 INFO:tasks.rados.rados.0.smithi093.stdout:2145: finishing write tid 4 to smithi093153278-404 2023-08-27T20:53:41.904 INFO:tasks.rados.rados.0.smithi093.stdout:2145: finishing write tid 5 to smithi093153278-404 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 404 v 302 (ObjNum 1427 snap 0 seq_num 1427) dirty exists 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:2145: left oid 404 (ObjNum 1427 snap 0 seq_num 1427) 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:2146: finishing write tid 1 to smithi093153278-334 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:2146: finishing write tid 2 to smithi093153278-334 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:2146: finishing write tid 3 to smithi093153278-334 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:2146: finishing write tid 4 to smithi093153278-334 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 334 v 259 (ObjNum 1428 snap 0 seq_num 1428) dirty exists 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:2146: left oid 334 (ObjNum 1428 snap 0 seq_num 1428) 2023-08-27T20:53:41.905 INFO:tasks.rados.rados.0.smithi093.stdout:2147: seq_num 1429 ranges {417203=488832,1405239=611964} 2023-08-27T20:53:41.908 INFO:tasks.rados.rados.0.smithi093.stdout:2147: writing smithi093153278-108 from 417203 to 906035 tid 1 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2147: writing smithi093153278-108 from 1405239 to 2017203 tid 2 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2143: done (4 left) 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2144: done (3 left) 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2145: done (2 left) 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2146: done (1 left) 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2148: read oid 128 snap -1 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2148: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:53:41.915 INFO:tasks.rados.rados.0.smithi093.stdout:2149: delete oid 487 current snap is 0 2023-08-27T20:53:41.916 INFO:tasks.rados.rados.0.smithi093.stdout:2147: finishing write tid 1 to smithi093153278-108 2023-08-27T20:53:41.917 INFO:tasks.rados.rados.0.smithi093.stdout:2149: done (2 left) 2023-08-27T20:53:41.917 INFO:tasks.rados.rados.0.smithi093.stdout:2150: read oid 308 snap -1 2023-08-27T20:53:41.917 INFO:tasks.rados.rados.0.smithi093.stdout:2150: expect (ObjNum 1373 snap 0 seq_num 1373) 2023-08-27T20:53:41.917 INFO:tasks.rados.rados.0.smithi093.stdout:2151: read oid 211 snap -1 2023-08-27T20:53:41.917 INFO:tasks.rados.rados.0.smithi093.stdout:2151: expect (ObjNum 210 snap 0 seq_num 210) 2023-08-27T20:53:41.917 INFO:tasks.rados.rados.0.smithi093.stdout:2152: write oid 170 current snap is 0 2023-08-27T20:53:41.917 INFO:tasks.rados.rados.0.smithi093.stdout:2152: seq_num 1431 ranges {450817=755683,1638556=412261} 2023-08-27T20:53:41.924 INFO:tasks.rados.rados.0.smithi093.stdout:2152: writing smithi093153278-170 from 450817 to 1206500 tid 1 2023-08-27T20:53:41.929 INFO:tasks.rados.rados.0.smithi093.stdout:2152: writing smithi093153278-170 from 1638556 to 2050817 tid 2 2023-08-27T20:53:41.929 INFO:tasks.rados.rados.0.smithi093.stdout:2153: read oid 383 snap -1 2023-08-27T20:53:41.929 INFO:tasks.rados.rados.0.smithi093.stdout:2153: expect (ObjNum 937 snap 0 seq_num 937) 2023-08-27T20:53:41.930 INFO:tasks.rados.rados.0.smithi093.stdout:2154: read oid 247 snap -1 2023-08-27T20:53:41.930 INFO:tasks.rados.rados.0.smithi093.stdout:2154: expect (ObjNum 1076 snap 0 seq_num 1076) 2023-08-27T20:53:41.930 INFO:tasks.rados.rados.0.smithi093.stdout:2155: write oid 250 current snap is 0 2023-08-27T20:53:41.930 INFO:tasks.rados.rados.0.smithi093.stdout:2148: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: pgmap v4: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: mgrmap e25: y(active, since 2s) 2023-08-27T20:53:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/463448658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: pgmap v4: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: mgrmap e25: y(active, since 2s) 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/463448658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: pgmap v4: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: mgrmap e25: y(active, since 2s) 2023-08-27T20:53:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/463448658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:41.994 INFO:tasks.rados.rados.0.smithi093.stdout:2147: finishing write tid 2 to smithi093153278-108 2023-08-27T20:53:41.994 INFO:tasks.rados.rados.0.smithi093.stdout:2147: finishing write tid 3 to smithi093153278-108 2023-08-27T20:53:41.994 INFO:tasks.rados.rados.0.smithi093.stdout:2147: finishing write tid 4 to smithi093153278-108 2023-08-27T20:53:41.994 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 108 v 361 (ObjNum 1429 snap 0 seq_num 1429) dirty exists 2023-08-27T20:53:41.994 INFO:tasks.rados.rados.0.smithi093.stdout:2147: left oid 108 (ObjNum 1429 snap 0 seq_num 1429) 2023-08-27T20:53:41.994 INFO:tasks.rados.rados.0.smithi093.stdout:2155: seq_num 1432 ranges {467624=430932,1697038=370586} 2023-08-27T20:53:41.998 INFO:tasks.rados.rados.0.smithi093.stdout:2155: writing smithi093153278-250 from 467624 to 898556 tid 1 2023-08-27T20:53:42.005 INFO:tasks.rados.rados.0.smithi093.stdout:2155: writing smithi093153278-250 from 1697038 to 2067624 tid 2 2023-08-27T20:53:42.005 INFO:tasks.rados.rados.0.smithi093.stdout:2147: done (7 left) 2023-08-27T20:53:42.005 INFO:tasks.rados.rados.0.smithi093.stdout:2148: done (6 left) 2023-08-27T20:53:42.006 INFO:tasks.rados.rados.0.smithi093.stdout:2156: read oid 446 snap -1 2023-08-27T20:53:42.006 INFO:tasks.rados.rados.0.smithi093.stdout:2156: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:53:42.006 INFO:tasks.rados.rados.0.smithi093.stdout:2157: write oid 81 current snap is 0 2023-08-27T20:53:42.006 INFO:tasks.rados.rados.0.smithi093.stdout:2151: expect (ObjNum 210 snap 0 seq_num 210) 2023-08-27T20:53:42.119 INFO:tasks.rados.rados.0.smithi093.stdout:2150: expect (ObjNum 1373 snap 0 seq_num 1373) 2023-08-27T20:53:42.276 INFO:tasks.rados.rados.0.smithi093.stdout:2152: finishing write tid 1 to smithi093153278-170 2023-08-27T20:53:42.276 INFO:tasks.rados.rados.0.smithi093.stdout:2157: seq_num 1433 ranges {484431=506181,1639167=445264} 2023-08-27T20:53:42.281 INFO:tasks.rados.rados.0.smithi093.stdout:2157: writing smithi093153278-81 from 484431 to 990612 tid 1 2023-08-27T20:53:42.287 INFO:tasks.rados.rados.0.smithi093.stdout:2157: writing smithi093153278-81 from 1639167 to 2084431 tid 2 2023-08-27T20:53:42.287 INFO:tasks.rados.rados.0.smithi093.stdout:2150: done (7 left) 2023-08-27T20:53:42.288 INFO:tasks.rados.rados.0.smithi093.stdout:2151: done (6 left) 2023-08-27T20:53:42.288 INFO:tasks.rados.rados.0.smithi093.stdout:2158: read oid 65 snap -1 2023-08-27T20:53:42.288 INFO:tasks.rados.rados.0.smithi093.stdout:2158: expect (ObjNum 868 snap 0 seq_num 868) 2023-08-27T20:53:42.288 INFO:tasks.rados.rados.0.smithi093.stdout:2159: write oid 195 current snap is 0 2023-08-27T20:53:42.288 INFO:tasks.rados.rados.0.smithi093.stdout:2153: expect (ObjNum 937 snap 0 seq_num 937) 2023-08-27T20:53:42.447 INFO:tasks.rados.rados.0.smithi093.stdout:2152: finishing write tid 2 to smithi093153278-170 2023-08-27T20:53:42.448 INFO:tasks.rados.rados.0.smithi093.stdout:2154: expect (ObjNum 1076 snap 0 seq_num 1076) 2023-08-27T20:53:42.526 INFO:tasks.rados.rados.0.smithi093.stdout:2152: finishing write tid 3 to smithi093153278-170 2023-08-27T20:53:42.526 INFO:tasks.rados.rados.0.smithi093.stdout:2152: finishing write tid 4 to smithi093153278-170 2023-08-27T20:53:42.526 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 170 v 269 (ObjNum 1431 snap 0 seq_num 1431) dirty exists 2023-08-27T20:53:42.527 INFO:tasks.rados.rados.0.smithi093.stdout:2152: left oid 170 (ObjNum 1431 snap 0 seq_num 1431) 2023-08-27T20:53:42.527 INFO:tasks.rados.rados.0.smithi093.stdout:2155: finishing write tid 1 to smithi093153278-250 2023-08-27T20:53:42.527 INFO:tasks.rados.rados.0.smithi093.stdout:2155: finishing write tid 2 to smithi093153278-250 2023-08-27T20:53:42.527 INFO:tasks.rados.rados.0.smithi093.stdout:2159: seq_num 1434 ranges {501238=581430,1697649=403589} 2023-08-27T20:53:42.532 INFO:tasks.rados.rados.0.smithi093.stdout:2159: writing smithi093153278-195 from 501238 to 1082668 tid 1 2023-08-27T20:53:42.538 INFO:tasks.rados.rados.0.smithi093.stdout:2159: writing smithi093153278-195 from 1697649 to 2101238 tid 2 2023-08-27T20:53:42.538 INFO:tasks.rados.rados.0.smithi093.stdout:2152: done (7 left) 2023-08-27T20:53:42.538 INFO:tasks.rados.rados.0.smithi093.stdout:2153: done (6 left) 2023-08-27T20:53:42.538 INFO:tasks.rados.rados.0.smithi093.stdout:2154: done (5 left) 2023-08-27T20:53:42.538 INFO:tasks.rados.rados.0.smithi093.stdout:2160: read oid 496 snap -1 2023-08-27T20:53:42.538 INFO:tasks.rados.rados.0.smithi093.stdout:2160: expect (ObjNum 1194 snap 0 seq_num 1194) 2023-08-27T20:53:42.538 INFO:tasks.rados.rados.0.smithi093.stdout:2161: write oid 96 current snap is 0 2023-08-27T20:53:42.539 INFO:tasks.rados.rados.0.smithi093.stdout:2155: finishing write tid 3 to smithi093153278-250 2023-08-27T20:53:42.539 INFO:tasks.rados.rados.0.smithi093.stdout:2155: finishing write tid 4 to smithi093153278-250 2023-08-27T20:53:42.539 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 250 v 393 (ObjNum 1432 snap 0 seq_num 1432) dirty exists 2023-08-27T20:53:42.539 INFO:tasks.rados.rados.0.smithi093.stdout:2155: left oid 250 (ObjNum 1432 snap 0 seq_num 1432) 2023-08-27T20:53:42.539 INFO:tasks.rados.rados.0.smithi093.stdout:2161: seq_num 1435 ranges {518045=656679,1756131=361914} 2023-08-27T20:53:42.544 INFO:tasks.rados.rados.0.smithi093.stdout:2161: writing smithi093153278-96 from 518045 to 1174724 tid 1 2023-08-27T20:53:42.550 INFO:tasks.rados.rados.0.smithi093.stdout:2161: writing smithi093153278-96 from 1756131 to 2118045 tid 2 2023-08-27T20:53:42.550 INFO:tasks.rados.rados.0.smithi093.stdout:2155: done (6 left) 2023-08-27T20:53:42.550 INFO:tasks.rados.rados.0.smithi093.stdout:2162: write oid 83 current snap is 0 2023-08-27T20:53:42.550 INFO:tasks.rados.rados.0.smithi093.stdout:2156: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:53:42.659 INFO:tasks.rados.rados.0.smithi093.stdout:2157: finishing write tid 1 to smithi093153278-81 2023-08-27T20:53:42.659 INFO:tasks.rados.rados.0.smithi093.stdout:2157: finishing write tid 2 to smithi093153278-81 2023-08-27T20:53:42.659 INFO:tasks.rados.rados.0.smithi093.stdout:2162: seq_num 1436 ranges {534852=731928,1814613=320239} 2023-08-27T20:53:42.666 INFO:tasks.rados.rados.0.smithi093.stdout:2162: writing smithi093153278-83 from 534852 to 1266780 tid 1 2023-08-27T20:53:42.671 INFO:tasks.rados.rados.0.smithi093.stdout:2162: writing smithi093153278-83 from 1814613 to 2134852 tid 2 2023-08-27T20:53:42.671 INFO:tasks.rados.rados.0.smithi093.stdout:2156: done (6 left) 2023-08-27T20:53:42.671 INFO:tasks.rados.rados.0.smithi093.stdout:2163: write oid 202 current snap is 0 2023-08-27T20:53:42.671 INFO:tasks.rados.rados.0.smithi093.stdout:2157: finishing write tid 3 to smithi093153278-81 2023-08-27T20:53:42.671 INFO:tasks.rados.rados.0.smithi093.stdout:2163: seq_num 1437 ranges {551659=523530,1873095=278564} 2023-08-27T20:53:42.676 INFO:tasks.rados.rados.0.smithi093.stdout:2163: writing smithi093153278-202 from 551659 to 1075189 tid 1 2023-08-27T20:53:42.683 INFO:tasks.rados.rados.0.smithi093.stdout:2163: writing smithi093153278-202 from 1873095 to 2151659 tid 2 2023-08-27T20:53:42.683 INFO:tasks.rados.rados.0.smithi093.stdout:2164: write oid 247 current snap is 0 2023-08-27T20:53:42.683 INFO:tasks.rados.rados.0.smithi093.stdout:2157: finishing write tid 4 to smithi093153278-81 2023-08-27T20:53:42.683 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 396 (ObjNum 1433 snap 0 seq_num 1433) dirty exists 2023-08-27T20:53:42.683 INFO:tasks.rados.rados.0.smithi093.stdout:2157: left oid 81 (ObjNum 1433 snap 0 seq_num 1433) 2023-08-27T20:53:42.683 INFO:tasks.rados.rados.0.smithi093.stdout:2158: expect (ObjNum 868 snap 0 seq_num 868) 2023-08-27T20:53:42.798 INFO:tasks.rados.rados.0.smithi093.stdout:2159: finishing write tid 1 to smithi093153278-195 2023-08-27T20:53:42.798 INFO:tasks.rados.rados.0.smithi093.stdout:2159: finishing write tid 2 to smithi093153278-195 2023-08-27T20:53:42.798 INFO:tasks.rados.rados.0.smithi093.stdout:2159: finishing write tid 3 to smithi093153278-195 2023-08-27T20:53:42.798 INFO:tasks.rados.rados.0.smithi093.stdout:2164: seq_num 1438 ranges {568466=598779,1931577=236889} 2023-08-27T20:53:42.804 INFO:tasks.rados.rados.0.smithi093.stdout:2164: writing smithi093153278-247 from 568466 to 1167245 tid 1 2023-08-27T20:53:42.810 INFO:tasks.rados.rados.0.smithi093.stdout:2164: writing smithi093153278-247 from 1931577 to 2168466 tid 2 2023-08-27T20:53:42.810 INFO:tasks.rados.rados.0.smithi093.stdout:2157: done (7 left) 2023-08-27T20:53:42.810 INFO:tasks.rados.rados.0.smithi093.stdout:2158: done (6 left) 2023-08-27T20:53:42.810 INFO:tasks.rados.rados.0.smithi093.stdout:2165: delete oid 176 current snap is 0 2023-08-27T20:53:42.810 INFO:tasks.rados.rados.0.smithi093.stdout:2159: finishing write tid 4 to smithi093153278-195 2023-08-27T20:53:42.810 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 262 (ObjNum 1434 snap 0 seq_num 1434) dirty exists 2023-08-27T20:53:42.811 INFO:tasks.rados.rados.0.smithi093.stdout:2159: left oid 195 (ObjNum 1434 snap 0 seq_num 1434) 2023-08-27T20:53:42.811 INFO:tasks.rados.rados.0.smithi093.stdout:2160: expect (ObjNum 1194 snap 0 seq_num 1194) 2023-08-27T20:53:42.898 INFO:tasks.rados.rados.0.smithi093.stdout:2161: finishing write tid 1 to smithi093153278-96 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2161: finishing write tid 2 to smithi093153278-96 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2161: finishing write tid 3 to smithi093153278-96 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2161: finishing write tid 4 to smithi093153278-96 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 96 v 329 (ObjNum 1435 snap 0 seq_num 1435) dirty exists 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2161: left oid 96 (ObjNum 1435 snap 0 seq_num 1435) 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2159: done (6 left) 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2160: done (5 left) 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2161: done (4 left) 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2165: done (3 left) 2023-08-27T20:53:42.899 INFO:tasks.rados.rados.0.smithi093.stdout:2166: read oid 416 snap -1 2023-08-27T20:53:42.900 INFO:tasks.rados.rados.0.smithi093.stdout:2166: expect (ObjNum 1191 snap 0 seq_num 1191) 2023-08-27T20:53:42.900 INFO:tasks.rados.rados.0.smithi093.stdout:2167: write oid 168 current snap is 0 2023-08-27T20:53:42.900 INFO:tasks.rados.rados.0.smithi093.stdout:2162: finishing write tid 1 to smithi093153278-83 2023-08-27T20:53:42.900 INFO:tasks.rados.rados.0.smithi093.stdout:2162: finishing write tid 2 to smithi093153278-83 2023-08-27T20:53:42.900 INFO:tasks.rados.rados.0.smithi093.stdout:2167: seq_num 1440 ranges {602080=749277,2048541=153539} 2023-08-27T20:53:42.906 INFO:tasks.rados.rados.0.smithi093.stdout:2167: writing smithi093153278-168 from 602080 to 1351357 tid 1 2023-08-27T20:53:42.911 INFO:tasks.rados.rados.0.smithi093.stdout:2167: writing smithi093153278-168 from 2048541 to 2202080 tid 2 2023-08-27T20:53:42.911 INFO:tasks.rados.rados.0.smithi093.stdout:2168: write oid 104 current snap is 0 2023-08-27T20:53:42.911 INFO:tasks.rados.rados.0.smithi093.stdout:2162: finishing write tid 3 to smithi093153278-83 2023-08-27T20:53:42.911 INFO:tasks.rados.rados.0.smithi093.stdout:2168: seq_num 1441 ranges {618887=424526,1590670=604158} 2023-08-27T20:53:42.916 INFO:tasks.rados.rados.0.smithi093.stdout:2168: writing smithi093153278-104 from 618887 to 1043413 tid 1 2023-08-27T20:53:42.923 INFO:tasks.rados.rados.0.smithi093.stdout:2168: writing smithi093153278-104 from 1590670 to 2194828 tid 2 2023-08-27T20:53:42.923 INFO:tasks.rados.rados.0.smithi093.stdout:2169: write oid 20 current snap is 0 2023-08-27T20:53:42.923 INFO:tasks.rados.rados.0.smithi093.stdout:2162: finishing write tid 4 to smithi093153278-83 2023-08-27T20:53:42.923 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 83 v 262 (ObjNum 1436 snap 0 seq_num 1436) dirty exists 2023-08-27T20:53:42.923 INFO:tasks.rados.rados.0.smithi093.stdout:2162: left oid 83 (ObjNum 1436 snap 0 seq_num 1436) 2023-08-27T20:53:42.923 INFO:tasks.rados.rados.0.smithi093.stdout:2163: finishing write tid 1 to smithi093153278-202 2023-08-27T20:53:42.923 INFO:tasks.rados.rados.0.smithi093.stdout:2169: seq_num 1442 ranges {635694=499775,1649152=464169} 2023-08-27T20:53:42.929 INFO:tasks.rados.rados.0.smithi093.stdout:2169: writing smithi093153278-20 from 635694 to 1135469 tid 1 2023-08-27T20:53:42.935 INFO:tasks.rados.rados.0.smithi093.stdout:2169: writing smithi093153278-20 from 1649152 to 2113321 tid 2 2023-08-27T20:53:42.935 INFO:tasks.rados.rados.0.smithi093.stdout:2162: done (6 left) 2023-08-27T20:53:42.935 INFO:tasks.rados.rados.0.smithi093.stdout:2170: read oid 34 snap -1 2023-08-27T20:53:42.935 INFO:tasks.rados.rados.0.smithi093.stdout:2170: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:42.935 INFO:tasks.rados.rados.0.smithi093.stdout:2171: read oid 363 snap -1 2023-08-27T20:53:42.936 INFO:tasks.rados.rados.0.smithi093.stdout:2171: expect (ObjNum 1416 snap 0 seq_num 1416) 2023-08-27T20:53:42.936 INFO:tasks.rados.rados.0.smithi093.stdout:2172: write oid 450 current snap is 0 2023-08-27T20:53:42.936 INFO:tasks.rados.rados.0.smithi093.stdout:2163: finishing write tid 2 to smithi093153278-202 2023-08-27T20:53:42.936 INFO:tasks.rados.rados.0.smithi093.stdout:2172: seq_num 1443 ranges {652501=575024,1707634=544867} 2023-08-27T20:53:42.941 INFO:tasks.rados.rados.0.smithi093.stdout:2172: writing smithi093153278-450 from 652501 to 1227525 tid 1 2023-08-27T20:53:42.947 INFO:tasks.rados.rados.0.smithi093.stdout:2172: writing smithi093153278-450 from 1707634 to 2252501 tid 2 2023-08-27T20:53:42.948 INFO:tasks.rados.rados.0.smithi093.stdout:2173: write oid 440 current snap is 0 2023-08-27T20:53:42.948 INFO:tasks.rados.rados.0.smithi093.stdout:2163: finishing write tid 3 to smithi093153278-202 2023-08-27T20:53:42.948 INFO:tasks.rados.rados.0.smithi093.stdout:2163: finishing write tid 4 to smithi093153278-202 2023-08-27T20:53:42.948 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 202 v 324 (ObjNum 1437 snap 0 seq_num 1437) dirty exists 2023-08-27T20:53:42.948 INFO:tasks.rados.rados.0.smithi093.stdout:2163: left oid 202 (ObjNum 1437 snap 0 seq_num 1437) 2023-08-27T20:53:42.948 INFO:tasks.rados.rados.0.smithi093.stdout:2164: finishing write tid 1 to smithi093153278-247 2023-08-27T20:53:42.948 INFO:tasks.rados.rados.0.smithi093.stdout:2173: seq_num 1444 ranges {669308=650273,1766116=467838} 2023-08-27T20:53:42.954 INFO:tasks.rados.rados.0.smithi093.stdout:2173: writing smithi093153278-440 from 669308 to 1319581 tid 1 2023-08-27T20:53:42.960 INFO:tasks.rados.rados.0.smithi093.stdout:2173: writing smithi093153278-440 from 1766116 to 2233954 tid 2 2023-08-27T20:53:42.960 INFO:tasks.rados.rados.0.smithi093.stdout:2163: done (9 left) 2023-08-27T20:53:42.960 INFO:tasks.rados.rados.0.smithi093.stdout:2174: write oid 251 current snap is 0 2023-08-27T20:53:42.960 INFO:tasks.rados.rados.0.smithi093.stdout:2164: finishing write tid 2 to smithi093153278-247 2023-08-27T20:53:42.960 INFO:tasks.rados.rados.0.smithi093.stdout:2174: seq_num 1445 ranges {686115=441875,1824598=461517} 2023-08-27T20:53:42.966 INFO:tasks.rados.rados.0.smithi093.stdout:2174: writing smithi093153278-251 from 686115 to 1127990 tid 1 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2174: writing smithi093153278-251 from 1824598 to 2286115 tid 2 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2175: read oid 7 snap -1 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2175: expect (ObjNum 976 snap 0 seq_num 976) 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2176: read oid 284 snap -1 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2176: expect (ObjNum 1139 snap 0 seq_num 1139) 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2177: read oid 50 snap -1 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2177: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2178: delete oid 406 current snap is 0 2023-08-27T20:53:42.973 INFO:tasks.rados.rados.0.smithi093.stdout:2164: finishing write tid 3 to smithi093153278-247 2023-08-27T20:53:42.974 INFO:tasks.rados.rados.0.smithi093.stdout:2164: finishing write tid 4 to smithi093153278-247 2023-08-27T20:53:42.974 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 247 v 266 (ObjNum 1438 snap 0 seq_num 1438) dirty exists 2023-08-27T20:53:42.974 INFO:tasks.rados.rados.0.smithi093.stdout:2164: left oid 247 (ObjNum 1438 snap 0 seq_num 1438) 2023-08-27T20:53:42.974 INFO:tasks.rados.rados.0.smithi093.stdout:2166: expect (ObjNum 1191 snap 0 seq_num 1191) 2023-08-27T20:53:43.051 INFO:tasks.rados.rados.0.smithi093.stdout:2167: finishing write tid 1 to smithi093153278-168 2023-08-27T20:53:43.051 INFO:tasks.rados.rados.0.smithi093.stdout:2167: finishing write tid 2 to smithi093153278-168 2023-08-27T20:53:43.051 INFO:tasks.rados.rados.0.smithi093.stdout:2167: finishing write tid 3 to smithi093153278-168 2023-08-27T20:53:43.051 INFO:tasks.rados.rados.0.smithi093.stdout:2167: finishing write tid 4 to smithi093153278-168 2023-08-27T20:53:43.051 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 168 v 305 (ObjNum 1440 snap 0 seq_num 1440) dirty exists 2023-08-27T20:53:43.051 INFO:tasks.rados.rados.0.smithi093.stdout:2167: left oid 168 (ObjNum 1440 snap 0 seq_num 1440) 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2168: finishing write tid 1 to smithi093153278-104 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2164: done (13 left) 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2166: done (12 left) 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2167: done (11 left) 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2178: done (10 left) 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2179: write oid 469 current snap is 0 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2168: finishing write tid 2 to smithi093153278-104 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2168: finishing write tid 3 to smithi093153278-104 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2168: finishing write tid 4 to smithi093153278-104 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 104 v 265 (ObjNum 1441 snap 0 seq_num 1441) dirty exists 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2168: left oid 104 (ObjNum 1441 snap 0 seq_num 1441) 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2169: finishing write tid 1 to smithi093153278-20 2023-08-27T20:53:43.052 INFO:tasks.rados.rados.0.smithi093.stdout:2179: seq_num 1447 ranges {719729=592373,1941562=378167} 2023-08-27T20:53:43.057 INFO:tasks.rados.rados.0.smithi093.stdout:2179: writing smithi093153278-469 from 719729 to 1312102 tid 1 2023-08-27T20:53:43.063 INFO:tasks.rados.rados.0.smithi093.stdout:2179: writing smithi093153278-469 from 1941562 to 2319729 tid 2 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2168: done (10 left) 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2180: read oid 295 snap -1 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2180: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2181: read oid 169 snap -1 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2181: expect (ObjNum 1283 snap 0 seq_num 1283) 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2182: delete oid 374 current snap is 0 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2169: finishing write tid 2 to smithi093153278-20 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2169: finishing write tid 3 to smithi093153278-20 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2169: finishing write tid 4 to smithi093153278-20 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 308 (ObjNum 1442 snap 0 seq_num 1442) dirty exists 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2169: left oid 20 (ObjNum 1442 snap 0 seq_num 1442) 2023-08-27T20:53:43.064 INFO:tasks.rados.rados.0.smithi093.stdout:2170: expect (ObjNum 33 snap 0 seq_num 33) 2023-08-27T20:53:43.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:43 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2486M 2023-08-27T20:53:43.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:43.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:43.148 INFO:tasks.rados.rados.0.smithi093.stdout:2171: expect (ObjNum 1416 snap 0 seq_num 1416) 2023-08-27T20:53:43.346 INFO:tasks.rados.rados.0.smithi093.stdout:2172: finishing write tid 1 to smithi093153278-450 2023-08-27T20:53:43.346 INFO:tasks.rados.rados.0.smithi093.stdout:2172: finishing write tid 2 to smithi093153278-450 2023-08-27T20:53:43.346 INFO:tasks.rados.rados.0.smithi093.stdout:2172: finishing write tid 3 to smithi093153278-450 2023-08-27T20:53:43.346 INFO:tasks.rados.rados.0.smithi093.stdout:2172: finishing write tid 4 to smithi093153278-450 2023-08-27T20:53:43.346 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 450 v 249 (ObjNum 1443 snap 0 seq_num 1443) dirty exists 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2172: left oid 450 (ObjNum 1443 snap 0 seq_num 1443) 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2173: finishing write tid 1 to smithi093153278-440 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2169: done (12 left) 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2170: done (11 left) 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2171: done (10 left) 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2172: done (9 left) 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2182: done (8 left) 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2183: write oid 339 current snap is 0 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2173: finishing write tid 2 to smithi093153278-440 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2173: finishing write tid 3 to smithi093153278-440 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:2173: finishing write tid 4 to smithi093153278-440 2023-08-27T20:53:43.347 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 440 v 311 (ObjNum 1444 snap 0 seq_num 1444) dirty exists 2023-08-27T20:53:43.348 INFO:tasks.rados.rados.0.smithi093.stdout:2173: left oid 440 (ObjNum 1444 snap 0 seq_num 1444) 2023-08-27T20:53:43.348 INFO:tasks.rados.rados.0.smithi093.stdout:2174: finishing write tid 1 to smithi093153278-251 2023-08-27T20:53:43.348 INFO:tasks.rados.rados.0.smithi093.stdout:2183: seq_num 1449 ranges {753343=742871,1942173=411170} 2023-08-27T20:53:43.353 INFO:tasks.rados.rados.0.smithi093.stdout:2183: writing smithi093153278-339 from 753343 to 1496214 tid 1 2023-08-27T20:53:43.358 INFO:tasks.rados.rados.0.smithi093.stdout:2183: writing smithi093153278-339 from 1942173 to 2353343 tid 2 2023-08-27T20:53:43.359 INFO:tasks.rados.rados.0.smithi093.stdout:2173: done (8 left) 2023-08-27T20:53:43.359 INFO:tasks.rados.rados.0.smithi093.stdout:2184: write oid 104 current snap is 0 2023-08-27T20:53:43.359 INFO:tasks.rados.rados.0.smithi093.stdout:2184: seq_num 1450 ranges {770150=418120,1600655=478845} 2023-08-27T20:53:43.365 INFO:tasks.rados.rados.0.smithi093.stdout:2184: writing smithi093153278-104 from 770150 to 1188270 tid 1 2023-08-27T20:53:43.370 INFO:tasks.rados.rados.0.smithi093.stdout:2184: writing smithi093153278-104 from 1600655 to 2079500 tid 2 2023-08-27T20:53:43.370 INFO:tasks.rados.rados.0.smithi093.stdout:2185: write oid 472 current snap is 0 2023-08-27T20:53:43.370 INFO:tasks.rados.rados.0.smithi093.stdout:2185: seq_num 1451 ranges {786957=493369,2059137=327820} 2023-08-27T20:53:43.376 INFO:tasks.rados.rados.0.smithi093.stdout:2185: writing smithi093153278-472 from 786957 to 1280326 tid 1 2023-08-27T20:53:43.383 INFO:tasks.rados.rados.0.smithi093.stdout:2185: writing smithi093153278-472 from 2059137 to 2386957 tid 2 2023-08-27T20:53:43.383 INFO:tasks.rados.rados.0.smithi093.stdout:2186: write oid 414 current snap is 0 2023-08-27T20:53:43.383 INFO:tasks.rados.rados.0.smithi093.stdout:2174: finishing write tid 2 to smithi093153278-251 2023-08-27T20:53:43.383 INFO:tasks.rados.rados.0.smithi093.stdout:2174: finishing write tid 3 to smithi093153278-251 2023-08-27T20:53:43.383 INFO:tasks.rados.rados.0.smithi093.stdout:2174: finishing write tid 4 to smithi093153278-251 2023-08-27T20:53:43.383 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 251 v 332 (ObjNum 1445 snap 0 seq_num 1445) dirty exists 2023-08-27T20:53:43.383 INFO:tasks.rados.rados.0.smithi093.stdout:2174: left oid 251 (ObjNum 1445 snap 0 seq_num 1445) 2023-08-27T20:53:43.384 INFO:tasks.rados.rados.0.smithi093.stdout:2175: expect (ObjNum 976 snap 0 seq_num 976) 2023-08-27T20:53:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:43 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2486M 2023-08-27T20:53:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:43 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2486M 2023-08-27T20:53:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:43.517 INFO:tasks.rados.rados.0.smithi093.stdout:2176: expect (ObjNum 1139 snap 0 seq_num 1139) 2023-08-27T20:53:43.619 INFO:tasks.rados.rados.0.smithi093.stdout:2177: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:53:43.783 INFO:tasks.rados.rados.0.smithi093.stdout:2179: finishing write tid 1 to smithi093153278-469 2023-08-27T20:53:43.783 INFO:tasks.rados.rados.0.smithi093.stdout:2181: expect (ObjNum 1283 snap 0 seq_num 1283) 2023-08-27T20:53:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:53:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:53:43.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:53:44.051 INFO:tasks.rados.rados.0.smithi093.stdout:2179: finishing write tid 2 to smithi093153278-469 2023-08-27T20:53:44.051 INFO:tasks.rados.rados.0.smithi093.stdout:2186: seq_num 1452 ranges {403764=568618,1717619=686145} 2023-08-27T20:53:44.056 INFO:tasks.rados.rados.0.smithi093.stdout:2186: writing smithi093153278-414 from 403764 to 972382 tid 1 2023-08-27T20:53:44.065 INFO:tasks.rados.rados.0.smithi093.stdout:2186: writing smithi093153278-414 from 1717619 to 2403764 tid 2 2023-08-27T20:53:44.065 INFO:tasks.rados.rados.0.smithi093.stdout:2174: done (10 left) 2023-08-27T20:53:44.065 INFO:tasks.rados.rados.0.smithi093.stdout:2175: done (9 left) 2023-08-27T20:53:44.065 INFO:tasks.rados.rados.0.smithi093.stdout:2176: done (8 left) 2023-08-27T20:53:44.065 INFO:tasks.rados.rados.0.smithi093.stdout:2177: done (7 left) 2023-08-27T20:53:44.065 INFO:tasks.rados.rados.0.smithi093.stdout:2181: done (6 left) 2023-08-27T20:53:44.066 INFO:tasks.rados.rados.0.smithi093.stdout:2187: write oid 448 current snap is 0 2023-08-27T20:53:44.066 INFO:tasks.rados.rados.0.smithi093.stdout:2179: finishing write tid 3 to smithi093153278-469 2023-08-27T20:53:44.066 INFO:tasks.rados.rados.0.smithi093.stdout:2179: finishing write tid 4 to smithi093153278-469 2023-08-27T20:53:44.066 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 469 v 364 (ObjNum 1447 snap 0 seq_num 1447) dirty exists 2023-08-27T20:53:44.066 INFO:tasks.rados.rados.0.smithi093.stdout:2179: left oid 469 (ObjNum 1447 snap 0 seq_num 1447) 2023-08-27T20:53:44.066 INFO:tasks.rados.rados.0.smithi093.stdout:2180: expect (ObjNum 294 snap 0 seq_num 294) 2023-08-27T20:53:44.167 INFO:tasks.rados.rados.0.smithi093.stdout:2183: finishing write tid 1 to smithi093153278-339 2023-08-27T20:53:44.167 INFO:tasks.rados.rados.0.smithi093.stdout:2183: finishing write tid 2 to smithi093153278-339 2023-08-27T20:53:44.167 INFO:tasks.rados.rados.0.smithi093.stdout:2183: finishing write tid 3 to smithi093153278-339 2023-08-27T20:53:44.167 INFO:tasks.rados.rados.0.smithi093.stdout:2183: finishing write tid 4 to smithi093153278-339 2023-08-27T20:53:44.167 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 339 v 273 (ObjNum 1449 snap 0 seq_num 1449) dirty exists 2023-08-27T20:53:44.167 INFO:tasks.rados.rados.0.smithi093.stdout:2183: left oid 339 (ObjNum 1449 snap 0 seq_num 1449) 2023-08-27T20:53:44.167 INFO:tasks.rados.rados.0.smithi093.stdout:2184: finishing write tid 1 to smithi093153278-104 2023-08-27T20:53:44.168 INFO:tasks.rados.rados.0.smithi093.stdout:2184: finishing write tid 2 to smithi093153278-104 2023-08-27T20:53:44.168 INFO:tasks.rados.rados.0.smithi093.stdout:2184: finishing write tid 3 to smithi093153278-104 2023-08-27T20:53:44.168 INFO:tasks.rados.rados.0.smithi093.stdout:2187: seq_num 1453 ranges {420571=760220,1776101=626172} 2023-08-27T20:53:44.173 INFO:tasks.rados.rados.0.smithi093.stdout:2187: writing smithi093153278-448 from 420571 to 1180791 tid 1 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2187: writing smithi093153278-448 from 1776101 to 2402273 tid 2 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2179: done (6 left) 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2180: done (5 left) 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2183: done (4 left) 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2188: read oid 209 snap -1 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2188: expect (ObjNum 208 snap 0 seq_num 208) 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2189: read oid 102 snap -1 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2189: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2190: write oid 42 current snap is 0 2023-08-27T20:53:44.182 INFO:tasks.rados.rados.0.smithi093.stdout:2184: finishing write tid 4 to smithi093153278-104 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 104 v 269 (ObjNum 1450 snap 0 seq_num 1450) dirty exists 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2184: left oid 104 (ObjNum 1450 snap 0 seq_num 1450) 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2185: finishing write tid 1 to smithi093153278-472 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2185: finishing write tid 2 to smithi093153278-472 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2185: finishing write tid 3 to smithi093153278-472 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2185: finishing write tid 4 to smithi093153278-472 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 472 v 307 (ObjNum 1451 snap 0 seq_num 1451) dirty exists 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2185: left oid 472 (ObjNum 1451 snap 0 seq_num 1451) 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2186: finishing write tid 1 to smithi093153278-414 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2186: finishing write tid 2 to smithi093153278-414 2023-08-27T20:53:44.183 INFO:tasks.rados.rados.0.smithi093.stdout:2186: finishing write tid 3 to smithi093153278-414 2023-08-27T20:53:44.184 INFO:tasks.rados.rados.0.smithi093.stdout:2186: finishing write tid 4 to smithi093153278-414 2023-08-27T20:53:44.184 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 414 v 327 (ObjNum 1452 snap 0 seq_num 1452) dirty exists 2023-08-27T20:53:44.184 INFO:tasks.rados.rados.0.smithi093.stdout:2186: left oid 414 (ObjNum 1452 snap 0 seq_num 1452) 2023-08-27T20:53:44.184 INFO:tasks.rados.rados.0.smithi093.stdout:2190: seq_num 1454 ranges {437378=435469,1434583=769830} 2023-08-27T20:53:44.186 INFO:tasks.rados.rados.0.smithi093.stdout:2190: writing smithi093153278-42 from 437378 to 872847 tid 1 2023-08-27T20:53:44.194 INFO:tasks.rados.rados.0.smithi093.stdout:2190: writing smithi093153278-42 from 1434583 to 2204413 tid 2 2023-08-27T20:53:44.194 INFO:tasks.rados.rados.0.smithi093.stdout:2184: done (6 left) 2023-08-27T20:53:44.194 INFO:tasks.rados.rados.0.smithi093.stdout:2185: done (5 left) 2023-08-27T20:53:44.194 INFO:tasks.rados.rados.0.smithi093.stdout:2186: done (4 left) 2023-08-27T20:53:44.194 INFO:tasks.rados.rados.0.smithi093.stdout:2191: write oid 189 current snap is 0 2023-08-27T20:53:44.194 INFO:tasks.rados.rados.0.smithi093.stdout:2187: finishing write tid 1 to smithi093153278-448 2023-08-27T20:53:44.194 INFO:tasks.rados.rados.0.smithi093.stdout:2191: seq_num 1455 ranges {454185=510718,1493065=629841} 2023-08-27T20:53:44.199 INFO:tasks.rados.rados.0.smithi093.stdout:2191: writing smithi093153278-189 from 454185 to 964903 tid 1 2023-08-27T20:53:44.206 INFO:tasks.rados.rados.0.smithi093.stdout:2191: writing smithi093153278-189 from 1493065 to 2122906 tid 2 2023-08-27T20:53:44.206 INFO:tasks.rados.rados.0.smithi093.stdout:2192: write oid 334 current snap is 0 2023-08-27T20:53:44.206 INFO:tasks.rados.rados.0.smithi093.stdout:2187: finishing write tid 2 to smithi093153278-448 2023-08-27T20:53:44.206 INFO:tasks.rados.rados.0.smithi093.stdout:2192: seq_num 1456 ranges {470992=585967,1551547=773499} 2023-08-27T20:53:44.211 INFO:tasks.rados.rados.0.smithi093.stdout:2192: writing smithi093153278-334 from 470992 to 1056959 tid 1 2023-08-27T20:53:44.219 INFO:tasks.rados.rados.0.smithi093.stdout:2192: writing smithi093153278-334 from 1551547 to 2325046 tid 2 2023-08-27T20:53:44.219 INFO:tasks.rados.rados.0.smithi093.stdout:2193: write oid 348 current snap is 0 2023-08-27T20:53:44.219 INFO:tasks.rados.rados.0.smithi093.stdout:2187: finishing write tid 3 to smithi093153278-448 2023-08-27T20:53:44.219 INFO:tasks.rados.rados.0.smithi093.stdout:2187: finishing write tid 4 to smithi093153278-448 2023-08-27T20:53:44.220 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 448 v 270 (ObjNum 1453 snap 0 seq_num 1453) dirty exists 2023-08-27T20:53:44.220 INFO:tasks.rados.rados.0.smithi093.stdout:2187: left oid 448 (ObjNum 1453 snap 0 seq_num 1453) 2023-08-27T20:53:44.220 INFO:tasks.rados.rados.0.smithi093.stdout:2188: expect (ObjNum 208 snap 0 seq_num 208) 2023-08-27T20:53:44.336 INFO:tasks.rados.rados.0.smithi093.stdout:2189: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:53:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: pgmap v5: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/490796413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:53:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: pgmap v5: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/490796413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: pgmap v5: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/490796413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.4", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.5", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.6", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.7", "name": "osd_memory_target"}]: dispatch 2023-08-27T20:53:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:53:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:53:44.560 INFO:tasks.rados.rados.0.smithi093.stdout:2190: finishing write tid 1 to smithi093153278-42 2023-08-27T20:53:44.560 INFO:tasks.rados.rados.0.smithi093.stdout:2190: finishing write tid 2 to smithi093153278-42 2023-08-27T20:53:44.560 INFO:tasks.rados.rados.0.smithi093.stdout:2190: finishing write tid 3 to smithi093153278-42 2023-08-27T20:53:44.560 INFO:tasks.rados.rados.0.smithi093.stdout:2190: finishing write tid 4 to smithi093153278-42 2023-08-27T20:53:44.560 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 42 v 282 (ObjNum 1454 snap 0 seq_num 1454) dirty exists 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2190: left oid 42 (ObjNum 1454 snap 0 seq_num 1454) 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2191: finishing write tid 1 to smithi093153278-189 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2191: finishing write tid 2 to smithi093153278-189 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2191: finishing write tid 3 to smithi093153278-189 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2191: finishing write tid 4 to smithi093153278-189 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 189 v 276 (ObjNum 1455 snap 0 seq_num 1455) dirty exists 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2191: left oid 189 (ObjNum 1455 snap 0 seq_num 1455) 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2192: finishing write tid 1 to smithi093153278-334 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2192: finishing write tid 2 to smithi093153278-334 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2192: finishing write tid 3 to smithi093153278-334 2023-08-27T20:53:44.561 INFO:tasks.rados.rados.0.smithi093.stdout:2193: seq_num 1457 ranges {487799=661216,1893676=594123} 2023-08-27T20:53:44.566 INFO:tasks.rados.rados.0.smithi093.stdout:2193: writing smithi093153278-348 from 487799 to 1149015 tid 1 2023-08-27T20:53:44.574 INFO:tasks.rados.rados.0.smithi093.stdout:2193: writing smithi093153278-348 from 1893676 to 2487799 tid 2 2023-08-27T20:53:44.574 INFO:tasks.rados.rados.0.smithi093.stdout:2187: done (6 left) 2023-08-27T20:53:44.574 INFO:tasks.rados.rados.0.smithi093.stdout:2188: done (5 left) 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:2189: done (4 left) 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:2190: done (3 left) 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:2191: done (2 left) 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:2194: write oid 266 current snap is 0 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:2192: finishing write tid 4 to smithi093153278-334 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 334 v 272 (ObjNum 1456 snap 0 seq_num 1456) dirty exists 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:2192: left oid 334 (ObjNum 1456 snap 0 seq_num 1456) 2023-08-27T20:53:44.575 INFO:tasks.rados.rados.0.smithi093.stdout:2194: seq_num 1458 ranges {504606=736465,1952158=552448} 2023-08-27T20:53:44.581 INFO:tasks.rados.rados.0.smithi093.stdout:2194: writing smithi093153278-266 from 504606 to 1241071 tid 1 2023-08-27T20:53:44.588 INFO:tasks.rados.rados.0.smithi093.stdout:2194: writing smithi093153278-266 from 1952158 to 2504606 tid 2 2023-08-27T20:53:44.588 INFO:tasks.rados.rados.0.smithi093.stdout:2192: done (2 left) 2023-08-27T20:53:44.588 INFO:tasks.rados.rados.0.smithi093.stdout:2195: delete oid 400 current snap is 0 2023-08-27T20:53:44.588 INFO:tasks.rados.rados.0.smithi093.stdout:2193: finishing write tid 1 to smithi093153278-348 2023-08-27T20:53:44.589 INFO:tasks.rados.rados.0.smithi093.stdout:2193: finishing write tid 2 to smithi093153278-348 2023-08-27T20:53:44.589 INFO:tasks.rados.rados.0.smithi093.stdout:2193: finishing write tid 3 to smithi093153278-348 2023-08-27T20:53:44.589 INFO:tasks.rados.rados.0.smithi093.stdout:2193: finishing write tid 4 to smithi093153278-348 2023-08-27T20:53:44.589 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 348 v 400 (ObjNum 1457 snap 0 seq_num 1457) dirty exists 2023-08-27T20:53:44.589 INFO:tasks.rados.rados.0.smithi093.stdout:2193: left oid 348 (ObjNum 1457 snap 0 seq_num 1457) 2023-08-27T20:53:44.589 INFO:tasks.rados.rados.0.smithi093.stdout:2194: finishing write tid 1 to smithi093153278-266 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2193: done (2 left) 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2195: done (1 left) 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2196: read oid 283 snap -1 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2196: expect deleted 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2197: read oid 191 snap -1 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2197: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2198: read oid 62 snap -1 2023-08-27T20:53:44.590 INFO:tasks.rados.rados.0.smithi093.stdout:2198: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:53:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:2199: write oid 365 current snap is 0 2023-08-27T20:53:44.591 INFO:tasks.rados.rados.0.smithi093.stdout:2199: seq_num 1460 ranges {538220=603316,1785475=752745} 2023-08-27T20:53:44.596 INFO:tasks.rados.rados.0.smithi093.stdout:2199: writing smithi093153278-365 from 538220 to 1141536 tid 1 2023-08-27T20:53:44.604 INFO:tasks.rados.rados.0.smithi093.stdout:2199: writing smithi093153278-365 from 1785475 to 2538220 tid 2 2023-08-27T20:53:44.604 INFO:tasks.rados.rados.0.smithi093.stdout:2200: read oid 14 snap -1 2023-08-27T20:53:44.604 INFO:tasks.rados.rados.0.smithi093.stdout:2200: expect (ObjNum 1411 snap 0 seq_num 1411) 2023-08-27T20:53:44.604 INFO:tasks.rados.rados.0.smithi093.stdout:2201: read oid 393 snap -1 2023-08-27T20:53:44.605 INFO:tasks.rados.rados.0.smithi093.stdout:2201: expect (ObjNum 1186 snap 0 seq_num 1186) 2023-08-27T20:53:44.605 INFO:tasks.rados.rados.0.smithi093.stdout:2202: write oid 218 current snap is 0 2023-08-27T20:53:44.605 INFO:tasks.rados.rados.0.smithi093.stdout:2202: seq_num 1461 ranges {555027=678565,1727604=640848} 2023-08-27T20:53:44.611 INFO:tasks.rados.rados.0.smithi093.stdout:2202: writing smithi093153278-218 from 555027 to 1233592 tid 1 2023-08-27T20:53:44.618 INFO:tasks.rados.rados.0.smithi093.stdout:2202: writing smithi093153278-218 from 1727604 to 2368452 tid 2 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2196: done (7 left) 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2203: read oid 163 snap -1 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2203: expect (ObjNum 1028 snap 0 seq_num 1028) 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2204: delete oid 75 current snap is 0 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2194: finishing write tid 2 to smithi093153278-266 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2194: finishing write tid 3 to smithi093153278-266 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2194: finishing write tid 4 to smithi093153278-266 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 266 v 275 (ObjNum 1458 snap 0 seq_num 1458) dirty exists 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2194: left oid 266 (ObjNum 1458 snap 0 seq_num 1458) 2023-08-27T20:53:44.619 INFO:tasks.rados.rados.0.smithi093.stdout:2197: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:53:44.742 INFO:tasks.rados.rados.0.smithi093.stdout:2198: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:53:44.956 INFO:tasks.rados.rados.0.smithi093.stdout:2199: finishing write tid 1 to smithi093153278-365 2023-08-27T20:53:44.956 INFO:tasks.rados.rados.0.smithi093.stdout:2199: finishing write tid 2 to smithi093153278-365 2023-08-27T20:53:44.956 INFO:tasks.rados.rados.0.smithi093.stdout:2194: done (8 left) 2023-08-27T20:53:44.956 INFO:tasks.rados.rados.0.smithi093.stdout:2197: done (7 left) 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:2198: done (6 left) 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:2204: done (5 left) 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:2205: delete oid 176 current snap is 0 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:2199: finishing write tid 3 to smithi093153278-365 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:2199: finishing write tid 4 to smithi093153278-365 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 365 v 330 (ObjNum 1460 snap 0 seq_num 1460) dirty exists 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:2199: left oid 365 (ObjNum 1460 snap 0 seq_num 1460) 2023-08-27T20:53:44.957 INFO:tasks.rados.rados.0.smithi093.stdout:2200: expect (ObjNum 1411 snap 0 seq_num 1411) 2023-08-27T20:53:45.185 INFO:tasks.rados.rados.0.smithi093.stdout:2202: finishing write tid 1 to smithi093153278-218 2023-08-27T20:53:45.185 INFO:tasks.rados.rados.0.smithi093.stdout:2201: expect (ObjNum 1186 snap 0 seq_num 1186) 2023-08-27T20:53:45.404 INFO:tasks.rados.rados.0.smithi093.stdout:2203: expect (ObjNum 1028 snap 0 seq_num 1028) 2023-08-27T20:53:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:45 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi136 to 2486M 2023-08-27T20:53:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:45 smithi136 ceph-mon[113531]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:53:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:45 smithi136 ceph-mon[113531]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:53:45.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:53:45] "GET /metrics HTTP/1.1" 200 33297 "" "Prometheus/2.43.0" 2023-08-27T20:53:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:45 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi136 to 2486M 2023-08-27T20:53:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:45 smithi093 ceph-mon[111335]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:53:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:45 smithi093 ceph-mon[111335]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:53:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:45 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi136 to 2486M 2023-08-27T20:53:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:45 smithi093 ceph-mon[118902]: Updating smithi093:/etc/ceph/ceph.conf 2023-08-27T20:53:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:45 smithi093 ceph-mon[118902]: Updating smithi136:/etc/ceph/ceph.conf 2023-08-27T20:53:45.543 INFO:tasks.rados.rados.0.smithi093.stdout:2202: finishing write tid 2 to smithi093153278-218 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2202: finishing write tid 3 to smithi093153278-218 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2202: finishing write tid 4 to smithi093153278-218 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 218 v 278 (ObjNum 1461 snap 0 seq_num 1461) dirty exists 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2202: left oid 218 (ObjNum 1461 snap 0 seq_num 1461) 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2199: done (5 left) 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2200: done (4 left) 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2201: done (3 left) 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2202: done (2 left) 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2203: done (1 left) 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2205: done (0 left) 2023-08-27T20:53:45.544 INFO:tasks.rados.rados.0.smithi093.stdout:2206: write oid 117 current snap is 0 2023-08-27T20:53:45.545 INFO:tasks.rados.rados.0.smithi093.stdout:2206: seq_num 1464 ranges {605448=504312,1903050=702398} 2023-08-27T20:53:45.549 INFO:tasks.rados.rados.0.smithi093.stdout:2206: writing smithi093153278-117 from 605448 to 1109760 tid 1 2023-08-27T20:53:45.557 INFO:tasks.rados.rados.0.smithi093.stdout:2206: writing smithi093153278-117 from 1903050 to 2605448 tid 2 2023-08-27T20:53:45.558 INFO:tasks.rados.rados.0.smithi093.stdout:2207: read oid 173 snap -1 2023-08-27T20:53:45.558 INFO:tasks.rados.rados.0.smithi093.stdout:2207: expect (ObjNum 1331 snap 0 seq_num 1331) 2023-08-27T20:53:45.558 INFO:tasks.rados.rados.0.smithi093.stdout:2208: write oid 316 current snap is 0 2023-08-27T20:53:45.558 INFO:tasks.rados.rados.0.smithi093.stdout:2206: finishing write tid 1 to smithi093153278-117 2023-08-27T20:53:45.558 INFO:tasks.rados.rados.0.smithi093.stdout:2208: seq_num 1465 ranges {622255=579561,1845179=531833} 2023-08-27T20:53:45.564 INFO:tasks.rados.rados.0.smithi093.stdout:2208: writing smithi093153278-316 from 622255 to 1201816 tid 1 2023-08-27T20:53:45.571 INFO:tasks.rados.rados.0.smithi093.stdout:2208: writing smithi093153278-316 from 1845179 to 2377012 tid 2 2023-08-27T20:53:45.571 INFO:tasks.rados.rados.0.smithi093.stdout:2209: read oid 289 snap -1 2023-08-27T20:53:45.571 INFO:tasks.rados.rados.0.smithi093.stdout:2209: expect (ObjNum 879 snap 0 seq_num 879) 2023-08-27T20:53:45.571 INFO:tasks.rados.rados.0.smithi093.stdout:2210: delete oid 484 current snap is 0 2023-08-27T20:53:45.571 INFO:tasks.rados.rados.0.smithi093.stdout:2206: finishing write tid 2 to smithi093153278-117 2023-08-27T20:53:45.571 INFO:tasks.rados.rados.0.smithi093.stdout:2206: finishing write tid 3 to smithi093153278-117 2023-08-27T20:53:45.571 INFO:tasks.rados.rados.0.smithi093.stdout:2206: finishing write tid 4 to smithi093153278-117 2023-08-27T20:53:45.572 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 117 v 306 (ObjNum 1464 snap 0 seq_num 1464) dirty exists 2023-08-27T20:53:45.572 INFO:tasks.rados.rados.0.smithi093.stdout:2206: left oid 117 (ObjNum 1464 snap 0 seq_num 1464) 2023-08-27T20:53:45.572 INFO:tasks.rados.rados.0.smithi093.stdout:2207: expect (ObjNum 1331 snap 0 seq_num 1331) 2023-08-27T20:53:45.694 INFO:tasks.rados.rados.0.smithi093.stdout:2208: finishing write tid 1 to smithi093153278-316 2023-08-27T20:53:45.694 INFO:tasks.rados.rados.0.smithi093.stdout:2209: expect (ObjNum 879 snap 0 seq_num 879) 2023-08-27T20:53:45.802 INFO:tasks.rados.rados.0.smithi093.stdout:2208: finishing write tid 2 to smithi093153278-316 2023-08-27T20:53:45.802 INFO:tasks.rados.rados.0.smithi093.stdout:2208: finishing write tid 3 to smithi093153278-316 2023-08-27T20:53:45.802 INFO:tasks.rados.rados.0.smithi093.stdout:2206: done (4 left) 2023-08-27T20:53:45.802 INFO:tasks.rados.rados.0.smithi093.stdout:2207: done (3 left) 2023-08-27T20:53:45.802 INFO:tasks.rados.rados.0.smithi093.stdout:2209: done (2 left) 2023-08-27T20:53:45.802 INFO:tasks.rados.rados.0.smithi093.stdout:2210: done (1 left) 2023-08-27T20:53:45.802 INFO:tasks.rados.rados.0.smithi093.stdout:2211: delete oid 229 current snap is 0 2023-08-27T20:53:45.803 INFO:tasks.rados.rados.0.smithi093.stdout:2208: finishing write tid 4 to smithi093153278-316 2023-08-27T20:53:45.803 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 316 v 335 (ObjNum 1465 snap 0 seq_num 1465) dirty exists 2023-08-27T20:53:45.803 INFO:tasks.rados.rados.0.smithi093.stdout:2208: left oid 316 (ObjNum 1465 snap 0 seq_num 1465) 2023-08-27T20:53:45.804 INFO:tasks.rados.rados.0.smithi093.stdout:2208: done (1 left) 2023-08-27T20:53:45.804 INFO:tasks.rados.rados.0.smithi093.stdout:2211: done (0 left) 2023-08-27T20:53:45.804 INFO:tasks.rados.rados.0.smithi093.stdout:2212: write oid 449 current snap is 0 2023-08-27T20:53:45.804 INFO:tasks.rados.rados.0.smithi093.stdout:2212: seq_num 1468 ranges {672676=521661,1736978=795513} 2023-08-27T20:53:45.810 INFO:tasks.rados.rados.0.smithi093.stdout:2212: writing smithi093153278-449 from 672676 to 1194337 tid 1 2023-08-27T20:53:45.817 INFO:tasks.rados.rados.0.smithi093.stdout:2212: writing smithi093153278-449 from 1736978 to 2532491 tid 2 2023-08-27T20:53:45.818 INFO:tasks.rados.rados.0.smithi093.stdout:2213: read oid 384 snap -1 2023-08-27T20:53:45.818 INFO:tasks.rados.rados.0.smithi093.stdout:2213: expect (ObjNum 808 snap 0 seq_num 808) 2023-08-27T20:53:45.818 INFO:tasks.rados.rados.0.smithi093.stdout:2214: read oid 306 snap -1 2023-08-27T20:53:45.818 INFO:tasks.rados.rados.0.smithi093.stdout:2214: expect (ObjNum 305 snap 0 seq_num 305) 2023-08-27T20:53:45.818 INFO:tasks.rados.rados.0.smithi093.stdout:2215: write oid 245 current snap is 0 2023-08-27T20:53:45.818 INFO:tasks.rados.rados.0.smithi093.stdout:2212: finishing write tid 1 to smithi093153278-449 2023-08-27T20:53:45.818 INFO:tasks.rados.rados.0.smithi093.stdout:2215: seq_num 1469 ranges {689483=596910,2079107=539171} 2023-08-27T20:53:45.824 INFO:tasks.rados.rados.0.smithi093.stdout:2215: writing smithi093153278-245 from 689483 to 1286393 tid 1 2023-08-27T20:53:45.832 INFO:tasks.rados.rados.0.smithi093.stdout:2215: writing smithi093153278-245 from 2079107 to 2618278 tid 2 2023-08-27T20:53:45.832 INFO:tasks.rados.rados.0.smithi093.stdout:2216: write oid 201 current snap is 0 2023-08-27T20:53:45.832 INFO:tasks.rados.rados.0.smithi093.stdout:2216: seq_num 1470 ranges {706290=672159,2137589=568701} 2023-08-27T20:53:45.839 INFO:tasks.rados.rados.0.smithi093.stdout:2216: writing smithi093153278-201 from 706290 to 1378449 tid 1 2023-08-27T20:53:45.847 INFO:tasks.rados.rados.0.smithi093.stdout:2216: writing smithi093153278-201 from 2137589 to 2706290 tid 2 2023-08-27T20:53:45.847 INFO:tasks.rados.rados.0.smithi093.stdout:2217: delete oid 66 current snap is 0 2023-08-27T20:53:45.847 INFO:tasks.rados.rados.0.smithi093.stdout:2213: expect (ObjNum 808 snap 0 seq_num 808) 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2212: finishing write tid 2 to smithi093153278-449 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2212: finishing write tid 3 to smithi093153278-449 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2212: finishing write tid 4 to smithi093153278-449 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 367 (ObjNum 1468 snap 0 seq_num 1468) dirty exists 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2212: left oid 449 (ObjNum 1468 snap 0 seq_num 1468) 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2212: done (5 left) 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2213: done (4 left) 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2217: done (3 left) 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2218: write oid 338 current snap is 0 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2215: finishing write tid 1 to smithi093153278-245 2023-08-27T20:53:45.997 INFO:tasks.rados.rados.0.smithi093.stdout:2214: expect (ObjNum 305 snap 0 seq_num 305) 2023-08-27T20:53:46.100 INFO:tasks.rados.rados.0.smithi093.stdout:2215: finishing write tid 2 to smithi093153278-245 2023-08-27T20:53:46.100 INFO:tasks.rados.rados.0.smithi093.stdout:2215: finishing write tid 3 to smithi093153278-245 2023-08-27T20:53:46.100 INFO:tasks.rados.rados.0.smithi093.stdout:2215: finishing write tid 4 to smithi093153278-245 2023-08-27T20:53:46.100 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 245 v 406 (ObjNum 1469 snap 0 seq_num 1469) dirty exists 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:2215: left oid 245 (ObjNum 1469 snap 0 seq_num 1469) 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:2216: finishing write tid 1 to smithi093153278-201 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:2216: finishing write tid 2 to smithi093153278-201 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:2216: finishing write tid 3 to smithi093153278-201 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:2216: finishing write tid 4 to smithi093153278-201 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 201 v 285 (ObjNum 1470 snap 0 seq_num 1470) dirty exists 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:2216: left oid 201 (ObjNum 1470 snap 0 seq_num 1470) 2023-08-27T20:53:46.101 INFO:tasks.rados.rados.0.smithi093.stdout:2218: seq_num 1472 ranges {739904=422657,1854553=402851} 2023-08-27T20:53:46.106 INFO:tasks.rados.rados.0.smithi093.stdout:2218: writing smithi093153278-338 from 739904 to 1162561 tid 1 2023-08-27T20:53:46.113 INFO:tasks.rados.rados.0.smithi093.stdout:2218: writing smithi093153278-338 from 1854553 to 2257404 tid 2 2023-08-27T20:53:46.113 INFO:tasks.rados.rados.0.smithi093.stdout:2214: done (3 left) 2023-08-27T20:53:46.113 INFO:tasks.rados.rados.0.smithi093.stdout:2215: done (2 left) 2023-08-27T20:53:46.113 INFO:tasks.rados.rados.0.smithi093.stdout:2216: done (1 left) 2023-08-27T20:53:46.113 INFO:tasks.rados.rados.0.smithi093.stdout:2219: write oid 459 current snap is 0 2023-08-27T20:53:46.113 INFO:tasks.rados.rados.0.smithi093.stdout:2218: finishing write tid 1 to smithi093153278-338 2023-08-27T20:53:46.113 INFO:tasks.rados.rados.0.smithi093.stdout:2219: seq_num 1473 ranges {756711=497906,1913035=546509} 2023-08-27T20:53:46.119 INFO:tasks.rados.rados.0.smithi093.stdout:2219: writing smithi093153278-459 from 756711 to 1254617 tid 1 2023-08-27T20:53:46.126 INFO:tasks.rados.rados.0.smithi093.stdout:2219: writing smithi093153278-459 from 1913035 to 2459544 tid 2 2023-08-27T20:53:46.126 INFO:tasks.rados.rados.0.smithi093.stdout:2220: delete oid 356 current snap is 0 2023-08-27T20:53:46.126 INFO:tasks.rados.rados.0.smithi093.stdout:2218: finishing write tid 2 to smithi093153278-338 2023-08-27T20:53:46.126 INFO:tasks.rados.rados.0.smithi093.stdout:2218: finishing write tid 3 to smithi093153278-338 2023-08-27T20:53:46.127 INFO:tasks.rados.rados.0.smithi093.stdout:2218: finishing write tid 4 to smithi093153278-338 2023-08-27T20:53:46.127 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 338 v 315 (ObjNum 1472 snap 0 seq_num 1472) dirty exists 2023-08-27T20:53:46.127 INFO:tasks.rados.rados.0.smithi093.stdout:2218: left oid 338 (ObjNum 1472 snap 0 seq_num 1472) 2023-08-27T20:53:46.127 INFO:tasks.rados.rados.0.smithi093.stdout:2219: finishing write tid 1 to smithi093153278-459 2023-08-27T20:53:46.128 INFO:tasks.rados.rados.0.smithi093.stdout:2218: done (2 left) 2023-08-27T20:53:46.129 INFO:tasks.rados.rados.0.smithi093.stdout:2220: done (1 left) 2023-08-27T20:53:46.129 INFO:tasks.rados.rados.0.smithi093.stdout:2221: read oid 344 snap -1 2023-08-27T20:53:46.129 INFO:tasks.rados.rados.0.smithi093.stdout:2221: expect (ObjNum 1359 snap 0 seq_num 1359) 2023-08-27T20:53:46.129 INFO:tasks.rados.rados.0.smithi093.stdout:2222: read oid 480 snap -1 2023-08-27T20:53:46.129 INFO:tasks.rados.rados.0.smithi093.stdout:2222: expect (ObjNum 1377 snap 0 seq_num 1377) 2023-08-27T20:53:46.129 INFO:tasks.rados.rados.0.smithi093.stdout:2223: write oid 237 current snap is 0 2023-08-27T20:53:46.129 INFO:tasks.rados.rados.0.smithi093.stdout:2223: seq_num 1475 ranges {790325=764757,2029999=550178} 2023-08-27T20:53:46.137 INFO:tasks.rados.rados.0.smithi093.stdout:2223: writing smithi093153278-237 from 790325 to 1555082 tid 1 2023-08-27T20:53:46.143 INFO:tasks.rados.rados.0.smithi093.stdout:2223: writing smithi093153278-237 from 2029999 to 2580177 tid 2 2023-08-27T20:53:46.144 INFO:tasks.rados.rados.0.smithi093.stdout:2224: write oid 33 current snap is 0 2023-08-27T20:53:46.144 INFO:tasks.rados.rados.0.smithi093.stdout:2219: finishing write tid 2 to smithi093153278-459 2023-08-27T20:53:46.144 INFO:tasks.rados.rados.0.smithi093.stdout:2224: seq_num 1476 ranges {407132=440006,1288481=693836,2616455=190677} 2023-08-27T20:53:46.148 INFO:tasks.rados.rados.0.smithi093.stdout:2224: writing smithi093153278-33 from 407132 to 847138 tid 1 2023-08-27T20:53:46.154 INFO:tasks.rados.rados.0.smithi093.stdout:2224: writing smithi093153278-33 from 1288481 to 1982317 tid 2 2023-08-27T20:53:46.159 INFO:tasks.rados.rados.0.smithi093.stdout:2224: writing smithi093153278-33 from 2616455 to 2807132 tid 3 2023-08-27T20:53:46.159 INFO:tasks.rados.rados.0.smithi093.stdout:2225: delete oid 301 current snap is 0 2023-08-27T20:53:46.159 INFO:tasks.rados.rados.0.smithi093.stdout:2219: finishing write tid 3 to smithi093153278-459 2023-08-27T20:53:46.160 INFO:tasks.rados.rados.0.smithi093.stdout:2219: finishing write tid 4 to smithi093153278-459 2023-08-27T20:53:46.160 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 459 v 279 (ObjNum 1473 snap 0 seq_num 1473) dirty exists 2023-08-27T20:53:46.160 INFO:tasks.rados.rados.0.smithi093.stdout:2219: left oid 459 (ObjNum 1473 snap 0 seq_num 1473) 2023-08-27T20:53:46.160 INFO:tasks.rados.rados.0.smithi093.stdout:2222: expect (ObjNum 1377 snap 0 seq_num 1377) 2023-08-27T20:53:46.291 INFO:tasks.rados.rados.0.smithi093.stdout:2221: expect (ObjNum 1359 snap 0 seq_num 1359) 2023-08-27T20:53:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: pgmap v6: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2223: finishing write tid 1 to smithi093153278-237 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2223: finishing write tid 2 to smithi093153278-237 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2223: finishing write tid 3 to smithi093153278-237 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2223: finishing write tid 4 to smithi093153278-237 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 237 v 309 (ObjNum 1475 snap 0 seq_num 1475) dirty exists 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2223: left oid 237 (ObjNum 1475 snap 0 seq_num 1475) 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2224: finishing write tid 1 to smithi093153278-33 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2219: done (5 left) 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2221: done (4 left) 2023-08-27T20:53:46.444 INFO:tasks.rados.rados.0.smithi093.stdout:2222: done (3 left) 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2223: done (2 left) 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2225: done (1 left) 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2226: read oid 318 snap -1 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2226: expect deleted 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2227: write oid 151 current snap is 0 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2224: finishing write tid 2 to smithi093153278-33 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2224: finishing write tid 3 to smithi093153278-33 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2224: finishing write tid 4 to smithi093153278-33 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2224: finishing write tid 5 to smithi093153278-33 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 33 v 313 (ObjNum 1476 snap 0 seq_num 1476) dirty exists 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2224: left oid 33 (ObjNum 1476 snap 0 seq_num 1476) 2023-08-27T20:53:46.445 INFO:tasks.rados.rados.0.smithi093.stdout:2227: seq_num 1478 ranges {440746=590504,1689092=697505} 2023-08-27T20:53:46.448 INFO:tasks.rados.rados.0.smithi093.stdout:2227: writing smithi093153278-151 from 440746 to 1031250 tid 1 2023-08-27T20:53:46.456 INFO:tasks.rados.rados.0.smithi093.stdout:2227: writing smithi093153278-151 from 1689092 to 2386597 tid 2 2023-08-27T20:53:46.456 INFO:tasks.rados.rados.0.smithi093.stdout:2224: done (2 left) 2023-08-27T20:53:46.457 INFO:tasks.rados.rados.0.smithi093.stdout:2228: write oid 422 current snap is 0 2023-08-27T20:53:46.457 INFO:tasks.rados.rados.0.smithi093.stdout:2228: seq_num 1479 ranges {457553=665753,1747574=557516} 2023-08-27T20:53:46.462 INFO:tasks.rados.rados.0.smithi093.stdout:2228: writing smithi093153278-422 from 457553 to 1123306 tid 1 2023-08-27T20:53:46.469 INFO:tasks.rados.rados.0.smithi093.stdout:2228: writing smithi093153278-422 from 1747574 to 2305090 tid 2 2023-08-27T20:53:46.469 INFO:tasks.rados.rados.0.smithi093.stdout:2229: write oid 156 current snap is 0 2023-08-27T20:53:46.470 INFO:tasks.rados.rados.0.smithi093.stdout:2229: seq_num 1480 ranges {474360=741002,1806056=701174} 2023-08-27T20:53:46.476 INFO:tasks.rados.rados.0.smithi093.stdout:2229: writing smithi093153278-156 from 474360 to 1215362 tid 1 2023-08-27T20:53:46.594 INFO:tasks.rados.rados.0.smithi093.stdout:2229: writing smithi093153278-156 from 1806056 to 2507230 tid 2 2023-08-27T20:53:46.595 INFO:tasks.rados.rados.0.smithi093.stdout:2230: read oid 302 snap -1 2023-08-27T20:53:46.595 INFO:tasks.rados.rados.0.smithi093.stdout:2230: expect (ObjNum 1195 snap 0 seq_num 1195) 2023-08-27T20:53:46.595 INFO:tasks.rados.rados.0.smithi093.stdout:2231: delete oid 325 current snap is 0 2023-08-27T20:53:46.595 INFO:tasks.rados.rados.0.smithi093.stdout:2227: finishing write tid 1 to smithi093153278-151 2023-08-27T20:53:46.595 INFO:tasks.rados.rados.0.smithi093.stdout:2227: finishing write tid 2 to smithi093153278-151 2023-08-27T20:53:46.596 INFO:tasks.rados.rados.0.smithi093.stdout:2227: finishing write tid 3 to smithi093153278-151 2023-08-27T20:53:46.596 INFO:tasks.rados.rados.0.smithi093.stdout:2227: finishing write tid 4 to smithi093153278-151 2023-08-27T20:53:46.596 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 151 v 285 (ObjNum 1478 snap 0 seq_num 1478) dirty exists 2023-08-27T20:53:46.596 INFO:tasks.rados.rados.0.smithi093.stdout:2227: left oid 151 (ObjNum 1478 snap 0 seq_num 1478) 2023-08-27T20:53:46.596 INFO:tasks.rados.rados.0.smithi093.stdout:2228: finishing write tid 1 to smithi093153278-422 2023-08-27T20:53:46.596 INFO:tasks.rados.rados.0.smithi093.stdout:2228: finishing write tid 2 to smithi093153278-422 2023-08-27T20:53:46.597 INFO:tasks.rados.rados.0.smithi093.stdout:2228: finishing write tid 3 to smithi093153278-422 2023-08-27T20:53:46.597 INFO:tasks.rados.rados.0.smithi093.stdout:2228: finishing write tid 4 to smithi093153278-422 2023-08-27T20:53:46.597 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 422 v 281 (ObjNum 1479 snap 0 seq_num 1479) dirty exists 2023-08-27T20:53:46.597 INFO:tasks.rados.rados.0.smithi093.stdout:2228: left oid 422 (ObjNum 1479 snap 0 seq_num 1479) 2023-08-27T20:53:46.597 INFO:tasks.rados.rados.0.smithi093.stdout:2229: finishing write tid 1 to smithi093153278-156 2023-08-27T20:53:46.597 INFO:tasks.rados.rados.0.smithi093.stdout:2226: done (5 left) 2023-08-27T20:53:46.598 INFO:tasks.rados.rados.0.smithi093.stdout:2227: done (4 left) 2023-08-27T20:53:46.598 INFO:tasks.rados.rados.0.smithi093.stdout:2228: done (3 left) 2023-08-27T20:53:46.598 INFO:tasks.rados.rados.0.smithi093.stdout:2231: done (2 left) 2023-08-27T20:53:46.598 INFO:tasks.rados.rados.0.smithi093.stdout:2232: write oid 220 current snap is 0 2023-08-27T20:53:46.598 INFO:tasks.rados.rados.0.smithi093.stdout:2232: seq_num 1482 ranges {507974=491500,1406667=704843,2548287=359687} 2023-08-27T20:53:46.599 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:53:46.599 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:53:46.599 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: pgmap v6: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:46.599 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:53:46.599 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:53:46.600 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:53:46.600 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:53:46.600 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.600 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.600 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.600 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.601 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.601 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:46.601 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:53:46.601 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.conf 2023-08-27T20:53:46.601 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: pgmap v6: 89 pgs: 89 active+clean; 1.2 GiB data, 2.2 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:53:46.602 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: Updating smithi093:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:53:46.602 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: Updating smithi136:/etc/ceph/ceph.client.admin.keyring 2023-08-27T20:53:46.602 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: Updating smithi093:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:53:46.602 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: Updating smithi136:/var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/config/ceph.client.admin.keyring 2023-08-27T20:53:46.602 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.603 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.603 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.603 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.603 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:53:46.603 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:53:46.604 INFO:tasks.rados.rados.0.smithi093.stdout:2232: writing smithi093153278-220 from 507974 to 999474 tid 1 2023-08-27T20:53:46.604 INFO:tasks.rados.rados.0.smithi093.stdout:2232: writing smithi093153278-220 from 1406667 to 2111510 tid 2 2023-08-27T20:53:46.604 INFO:tasks.rados.rados.0.smithi093.stdout:2232: writing smithi093153278-220 from 2548287 to 2907974 tid 3 2023-08-27T20:53:46.604 INFO:tasks.rados.rados.0.smithi093.stdout:2233: read oid 319 snap -1 2023-08-27T20:53:46.604 INFO:tasks.rados.rados.0.smithi093.stdout:2233: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2234: write oid 387 current snap is 0 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2234: seq_num 1483 ranges {524781=683102,1981502=564854} 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2234: writing smithi093153278-387 from 524781 to 1207883 tid 1 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2234: writing smithi093153278-387 from 1981502 to 2546356 tid 2 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2235: write oid 315 current snap is 0 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2230: expect (ObjNum 1195 snap 0 seq_num 1195) 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2229: finishing write tid 2 to smithi093153278-156 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2229: finishing write tid 3 to smithi093153278-156 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2229: finishing write tid 4 to smithi093153278-156 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 156 v 288 (ObjNum 1480 snap 0 seq_num 1480) dirty exists 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2229: left oid 156 (ObjNum 1480 snap 0 seq_num 1480) 2023-08-27T20:53:46.605 INFO:tasks.rados.rados.0.smithi093.stdout:2232: finishing write tid 1 to smithi093153278-220 2023-08-27T20:53:46.606 INFO:tasks.rados.rados.0.smithi093.stdout:2235: seq_num 1484 ranges {541588=758351,2039984=708512} 2023-08-27T20:53:46.606 INFO:tasks.rados.rados.0.smithi093.stdout:2235: writing smithi093153278-315 from 541588 to 1299939 tid 1 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2235: writing smithi093153278-315 from 2039984 to 2748496 tid 2 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2229: done (5 left) 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2230: done (4 left) 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2236: write oid 322 current snap is 0 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2232: finishing write tid 2 to smithi093153278-220 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2232: finishing write tid 3 to smithi093153278-220 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2232: finishing write tid 4 to smithi093153278-220 2023-08-27T20:53:46.613 INFO:tasks.rados.rados.0.smithi093.stdout:2236: seq_num 1485 ranges {558395=433600,1698466=568523,2863262=95133} 2023-08-27T20:53:46.618 INFO:tasks.rados.rados.0.smithi093.stdout:2236: writing smithi093153278-322 from 558395 to 991995 tid 1 2023-08-27T20:53:46.625 INFO:tasks.rados.rados.0.smithi093.stdout:2236: writing smithi093153278-322 from 1698466 to 2266989 tid 2 2023-08-27T20:53:46.629 INFO:tasks.rados.rados.0.smithi093.stdout:2236: writing smithi093153278-322 from 2863262 to 2958395 tid 3 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2237: read oid 280 snap -1 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2237: expect (ObjNum 822 snap 0 seq_num 822) 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2238: read oid 377 snap -1 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2238: expect (ObjNum 1132 snap 0 seq_num 1132) 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2239: read oid 385 snap -1 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2239: expect (ObjNum 1204 snap 0 seq_num 1204) 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2240: write oid 321 current snap is 0 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2232: finishing write tid 5 to smithi093153278-220 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 220 v 372 (ObjNum 1482 snap 0 seq_num 1482) dirty exists 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2232: left oid 220 (ObjNum 1482 snap 0 seq_num 1482) 2023-08-27T20:53:46.630 INFO:tasks.rados.rados.0.smithi093.stdout:2233: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:53:46.784 INFO:tasks.rados.rados.0.smithi093.stdout:2234: finishing write tid 1 to smithi093153278-387 2023-08-27T20:53:46.784 INFO:tasks.rados.rados.0.smithi093.stdout:2234: finishing write tid 2 to smithi093153278-387 2023-08-27T20:53:46.784 INFO:tasks.rados.rados.0.smithi093.stdout:2234: finishing write tid 3 to smithi093153278-387 2023-08-27T20:53:46.784 INFO:tasks.rados.rados.0.smithi093.stdout:2234: finishing write tid 4 to smithi093153278-387 2023-08-27T20:53:46.784 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 387 v 288 (ObjNum 1483 snap 0 seq_num 1483) dirty exists 2023-08-27T20:53:46.784 INFO:tasks.rados.rados.0.smithi093.stdout:2234: left oid 387 (ObjNum 1483 snap 0 seq_num 1483) 2023-08-27T20:53:46.785 INFO:tasks.rados.rados.0.smithi093.stdout:2235: finishing write tid 1 to smithi093153278-315 2023-08-27T20:53:46.785 INFO:tasks.rados.rados.0.smithi093.stdout:2240: seq_num 1486 ranges {575202=508849,1640595=712181} 2023-08-27T20:53:46.789 INFO:tasks.rados.rados.0.smithi093.stdout:2240: writing smithi093153278-321 from 575202 to 1084051 tid 1 2023-08-27T20:53:46.797 INFO:tasks.rados.rados.0.smithi093.stdout:2240: writing smithi093153278-321 from 1640595 to 2352776 tid 2 2023-08-27T20:53:46.797 INFO:tasks.rados.rados.0.smithi093.stdout:2235: finishing write tid 2 to smithi093153278-315 2023-08-27T20:53:46.797 INFO:tasks.rados.rados.0.smithi093.stdout:2235: finishing write tid 3 to smithi093153278-315 2023-08-27T20:53:46.797 INFO:tasks.rados.rados.0.smithi093.stdout:2235: finishing write tid 4 to smithi093153278-315 2023-08-27T20:53:46.797 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 316 (ObjNum 1484 snap 0 seq_num 1484) dirty exists 2023-08-27T20:53:46.797 INFO:tasks.rados.rados.0.smithi093.stdout:2235: left oid 315 (ObjNum 1484 snap 0 seq_num 1484) 2023-08-27T20:53:46.797 INFO:tasks.rados.rados.0.smithi093.stdout:2236: finishing write tid 1 to smithi093153278-322 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2236: finishing write tid 2 to smithi093153278-322 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2232: done (8 left) 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2233: done (7 left) 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2234: done (6 left) 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2235: done (5 left) 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2241: delete oid 142 current snap is 0 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2236: finishing write tid 3 to smithi093153278-322 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2236: finishing write tid 4 to smithi093153278-322 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2236: finishing write tid 5 to smithi093153278-322 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 322 v 311 (ObjNum 1485 snap 0 seq_num 1485) dirty exists 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2236: left oid 322 (ObjNum 1485 snap 0 seq_num 1485) 2023-08-27T20:53:46.798 INFO:tasks.rados.rados.0.smithi093.stdout:2239: expect (ObjNum 1204 snap 0 seq_num 1204) 2023-08-27T20:53:46.893 INFO:tasks.rados.rados.0.smithi093.stdout:2238: expect (ObjNum 1132 snap 0 seq_num 1132) 2023-08-27T20:53:47.058 INFO:tasks.rados.rados.0.smithi093.stdout:2237: expect (ObjNum 822 snap 0 seq_num 822) 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2240: finishing write tid 1 to smithi093153278-321 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2240: finishing write tid 2 to smithi093153278-321 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2240: finishing write tid 3 to smithi093153278-321 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2240: finishing write tid 4 to smithi093153278-321 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 321 v 333 (ObjNum 1486 snap 0 seq_num 1486) dirty exists 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2240: left oid 321 (ObjNum 1486 snap 0 seq_num 1486) 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2236: done (5 left) 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2237: done (4 left) 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2238: done (3 left) 2023-08-27T20:53:47.228 INFO:tasks.rados.rados.0.smithi093.stdout:2239: done (2 left) 2023-08-27T20:53:47.229 INFO:tasks.rados.rados.0.smithi093.stdout:2240: done (1 left) 2023-08-27T20:53:47.229 INFO:tasks.rados.rados.0.smithi093.stdout:2241: done (0 left) 2023-08-27T20:53:47.229 INFO:tasks.rados.rados.0.smithi093.stdout:2242: delete oid 281 current snap is 0 2023-08-27T20:53:47.229 INFO:tasks.rados.rados.0.smithi093.stdout:2242: done (0 left) 2023-08-27T20:53:47.229 INFO:tasks.rados.rados.0.smithi093.stdout:2243: delete oid 148 current snap is 0 2023-08-27T20:53:47.231 INFO:tasks.rados.rados.0.smithi093.stdout:2243: done (0 left) 2023-08-27T20:53:47.231 INFO:tasks.rados.rados.0.smithi093.stdout:2244: write oid 168 current snap is 0 2023-08-27T20:53:47.231 INFO:tasks.rados.rados.0.smithi093.stdout:2244: seq_num 1490 ranges {642430=409845,1758170=719519} 2023-08-27T20:53:47.236 INFO:tasks.rados.rados.0.smithi093.stdout:2244: writing smithi093153278-168 from 642430 to 1052275 tid 1 2023-08-27T20:53:47.244 INFO:tasks.rados.rados.0.smithi093.stdout:2244: writing smithi093153278-168 from 1758170 to 2477689 tid 2 2023-08-27T20:53:47.244 INFO:tasks.rados.rados.0.smithi093.stdout:2245: write oid 16 current snap is 0 2023-08-27T20:53:47.244 INFO:tasks.rados.rados.0.smithi093.stdout:2244: finishing write tid 1 to smithi093153278-168 2023-08-27T20:53:47.244 INFO:tasks.rados.rados.0.smithi093.stdout:2245: seq_num 1491 ranges {659237=601447,1933005=463177} 2023-08-27T20:53:47.251 INFO:tasks.rados.rados.0.smithi093.stdout:2245: writing smithi093153278-16 from 659237 to 1260684 tid 1 2023-08-27T20:53:47.257 INFO:tasks.rados.rados.0.smithi093.stdout:2245: writing smithi093153278-16 from 1933005 to 2396182 tid 2 2023-08-27T20:53:47.257 INFO:tasks.rados.rados.0.smithi093.stdout:2246: write oid 418 current snap is 0 2023-08-27T20:53:47.258 INFO:tasks.rados.rados.0.smithi093.stdout:2244: finishing write tid 2 to smithi093153278-168 2023-08-27T20:53:47.258 INFO:tasks.rados.rados.0.smithi093.stdout:2244: finishing write tid 3 to smithi093153278-168 2023-08-27T20:53:47.258 INFO:tasks.rados.rados.0.smithi093.stdout:2244: finishing write tid 4 to smithi093153278-168 2023-08-27T20:53:47.258 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 168 v 318 (ObjNum 1490 snap 0 seq_num 1490) dirty exists 2023-08-27T20:53:47.258 INFO:tasks.rados.rados.0.smithi093.stdout:2244: left oid 168 (ObjNum 1490 snap 0 seq_num 1490) 2023-08-27T20:53:47.258 INFO:tasks.rados.rados.0.smithi093.stdout:2245: finishing write tid 1 to smithi093153278-16 2023-08-27T20:53:47.258 INFO:tasks.rados.rados.0.smithi093.stdout:2246: seq_num 1492 ranges {676044=676696,1991487=723188} 2023-08-27T20:53:47.264 INFO:tasks.rados.rados.0.smithi093.stdout:2246: writing smithi093153278-418 from 676044 to 1352740 tid 1 2023-08-27T20:53:47.272 INFO:tasks.rados.rados.0.smithi093.stdout:2246: writing smithi093153278-418 from 1991487 to 2714675 tid 2 2023-08-27T20:53:47.272 INFO:tasks.rados.rados.0.smithi093.stdout:2244: done (2 left) 2023-08-27T20:53:47.272 INFO:tasks.rados.rados.0.smithi093.stdout:2247: write oid 163 current snap is 0 2023-08-27T20:53:47.272 INFO:tasks.rados.rados.0.smithi093.stdout:2245: finishing write tid 2 to smithi093153278-16 2023-08-27T20:53:47.272 INFO:tasks.rados.rados.0.smithi093.stdout:2247: seq_num 1493 ranges {692851=751945,2049969=466846} 2023-08-27T20:53:47.280 INFO:tasks.rados.rados.0.smithi093.stdout:2247: writing smithi093153278-163 from 692851 to 1444796 tid 1 2023-08-27T20:53:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:2247: writing smithi093153278-163 from 2049969 to 2516815 tid 2 2023-08-27T20:53:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:2248: write oid 392 current snap is 0 2023-08-27T20:53:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:2245: finishing write tid 3 to smithi093153278-16 2023-08-27T20:53:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:2245: finishing write tid 4 to smithi093153278-16 2023-08-27T20:53:47.286 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 16 v 291 (ObjNum 1491 snap 0 seq_num 1491) dirty exists 2023-08-27T20:53:47.287 INFO:tasks.rados.rados.0.smithi093.stdout:2245: left oid 16 (ObjNum 1491 snap 0 seq_num 1491) 2023-08-27T20:53:47.287 INFO:tasks.rados.rados.0.smithi093.stdout:2248: seq_num 1494 ranges {709658=427194,1592098=726857,2761010=348648} 2023-08-27T20:53:47.292 INFO:tasks.rados.rados.0.smithi093.stdout:2248: writing smithi093153278-392 from 709658 to 1136852 tid 1 2023-08-27T20:53:47.299 INFO:tasks.rados.rados.0.smithi093.stdout:2248: writing smithi093153278-392 from 1592098 to 2318955 tid 2 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2248: writing smithi093153278-392 from 2761010 to 3109658 tid 3 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2245: done (3 left) 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2249: delete oid 144 current snap is 0 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2246: finishing write tid 1 to smithi093153278-418 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2246: finishing write tid 2 to smithi093153278-418 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2246: finishing write tid 3 to smithi093153278-418 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2246: finishing write tid 4 to smithi093153278-418 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 418 v 319 (ObjNum 1492 snap 0 seq_num 1492) dirty exists 2023-08-27T20:53:47.304 INFO:tasks.rados.rados.0.smithi093.stdout:2246: left oid 418 (ObjNum 1492 snap 0 seq_num 1492) 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:2247: finishing write tid 1 to smithi093153278-163 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:2247: finishing write tid 2 to smithi093153278-163 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:2247: finishing write tid 3 to smithi093153278-163 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:2247: finishing write tid 4 to smithi093153278-163 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 163 v 274 (ObjNum 1493 snap 0 seq_num 1493) dirty exists 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:2247: left oid 163 (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:2248: finishing write tid 1 to smithi093153278-392 2023-08-27T20:53:47.305 INFO:tasks.rados.rados.0.smithi093.stdout:2248: finishing write tid 2 to smithi093153278-392 2023-08-27T20:53:47.306 INFO:tasks.rados.rados.0.smithi093.stdout:2246: done (3 left) 2023-08-27T20:53:47.306 INFO:tasks.rados.rados.0.smithi093.stdout:2247: done (2 left) 2023-08-27T20:53:47.306 INFO:tasks.rados.rados.0.smithi093.stdout:2249: done (1 left) 2023-08-27T20:53:47.306 INFO:tasks.rados.rados.0.smithi093.stdout:2250: write oid 148 current snap is 0 2023-08-27T20:53:47.306 INFO:tasks.rados.rados.0.smithi093.stdout:2250: seq_num 1496 ranges {743272=577692,2109062=730526} 2023-08-27T20:53:47.312 INFO:tasks.rados.rados.0.smithi093.stdout:2250: writing smithi093153278-148 from 743272 to 1320964 tid 1 2023-08-27T20:53:47.320 INFO:tasks.rados.rados.0.smithi093.stdout:2250: writing smithi093153278-148 from 2109062 to 2839588 tid 2 2023-08-27T20:53:47.320 INFO:tasks.rados.rados.0.smithi093.stdout:2251: delete oid 160 current snap is 0 2023-08-27T20:53:47.321 INFO:tasks.rados.rados.0.smithi093.stdout:2248: finishing write tid 3 to smithi093153278-392 2023-08-27T20:53:47.321 INFO:tasks.rados.rados.0.smithi093.stdout:2248: finishing write tid 4 to smithi093153278-392 2023-08-27T20:53:47.321 INFO:tasks.rados.rados.0.smithi093.stdout:2248: finishing write tid 5 to smithi093153278-392 2023-08-27T20:53:47.321 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 392 v 315 (ObjNum 1494 snap 0 seq_num 1494) dirty exists 2023-08-27T20:53:47.321 INFO:tasks.rados.rados.0.smithi093.stdout:2248: left oid 392 (ObjNum 1494 snap 0 seq_num 1494) 2023-08-27T20:53:47.321 INFO:tasks.rados.rados.0.smithi093.stdout:2250: finishing write tid 1 to smithi093153278-148 2023-08-27T20:53:47.321 INFO:tasks.rados.rados.0.smithi093.stdout:2248: done (2 left) 2023-08-27T20:53:47.322 INFO:tasks.rados.rados.0.smithi093.stdout:2251: done (1 left) 2023-08-27T20:53:47.322 INFO:tasks.rados.rados.0.smithi093.stdout:2252: write oid 113 current snap is 0 2023-08-27T20:53:47.322 INFO:tasks.rados.rados.0.smithi093.stdout:2252: seq_num 1498 ranges {776886=444543,1826026=734195,2987055=189831} 2023-08-27T20:53:47.328 INFO:tasks.rados.rados.0.smithi093.stdout:2252: writing smithi093153278-113 from 776886 to 1221429 tid 1 2023-08-27T20:53:47.335 INFO:tasks.rados.rados.0.smithi093.stdout:2252: writing smithi093153278-113 from 1826026 to 2560221 tid 2 2023-08-27T20:53:47.339 INFO:tasks.rados.rados.0.smithi093.stdout:2252: writing smithi093153278-113 from 2987055 to 3176886 tid 3 2023-08-27T20:53:47.339 INFO:tasks.rados.rados.0.smithi093.stdout:2253: delete oid 430 current snap is 0 2023-08-27T20:53:47.340 INFO:tasks.rados.rados.0.smithi093.stdout:2250: finishing write tid 2 to smithi093153278-148 2023-08-27T20:53:47.340 INFO:tasks.rados.rados.0.smithi093.stdout:2250: finishing write tid 3 to smithi093153278-148 2023-08-27T20:53:47.340 INFO:tasks.rados.rados.0.smithi093.stdout:2250: finishing write tid 4 to smithi093153278-148 2023-08-27T20:53:47.340 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 148 v 270 (ObjNum 1496 snap 0 seq_num 1496) dirty exists 2023-08-27T20:53:47.340 INFO:tasks.rados.rados.0.smithi093.stdout:2250: left oid 148 (ObjNum 1496 snap 0 seq_num 1496) 2023-08-27T20:53:47.340 INFO:tasks.rados.rados.0.smithi093.stdout:2252: finishing write tid 1 to smithi093153278-113 2023-08-27T20:53:47.340 INFO:tasks.rados.rados.0.smithi093.stdout:2252: finishing write tid 2 to smithi093153278-113 2023-08-27T20:53:47.341 INFO:tasks.rados.rados.0.smithi093.stdout:2250: done (2 left) 2023-08-27T20:53:47.341 INFO:tasks.rados.rados.0.smithi093.stdout:2253: done (1 left) 2023-08-27T20:53:47.341 INFO:tasks.rados.rados.0.smithi093.stdout:2254: read oid 389 snap -1 2023-08-27T20:53:47.341 INFO:tasks.rados.rados.0.smithi093.stdout:2254: expect deleted 2023-08-27T20:53:47.341 INFO:tasks.rados.rados.0.smithi093.stdout:2255: write oid 88 current snap is 0 2023-08-27T20:53:47.341 INFO:tasks.rados.rados.0.smithi093.stdout:2255: seq_num 1500 ranges {410500=595041,1542990=621511,2925548=284952} 2023-08-27T20:53:47.346 INFO:tasks.rados.rados.0.smithi093.stdout:2255: writing smithi093153278-88 from 410500 to 1005541 tid 1 2023-08-27T20:53:47.352 INFO:tasks.rados.rados.0.smithi093.stdout:2255: writing smithi093153278-88 from 1542990 to 2164501 tid 2 2023-08-27T20:53:47.358 INFO:tasks.rados.rados.0.smithi093.stdout:2255: writing smithi093153278-88 from 2925548 to 3210500 tid 3 2023-08-27T20:53:47.359 INFO:tasks.rados.rados.0.smithi093.stdout:2256: read oid 208 snap -1 2023-08-27T20:53:47.359 INFO:tasks.rados.rados.0.smithi093.stdout:2256: expect (ObjNum 1413 snap 0 seq_num 1413) 2023-08-27T20:53:47.359 INFO:tasks.rados.rados.0.smithi093.stdout:2257: read oid 179 snap -1 2023-08-27T20:53:47.359 INFO:tasks.rados.rados.0.smithi093.stdout:2257: expect (ObjNum 1347 snap 0 seq_num 1347) 2023-08-27T20:53:47.359 INFO:tasks.rados.rados.0.smithi093.stdout:2258: write oid 11 current snap is 0 2023-08-27T20:53:47.359 INFO:tasks.rados.rados.0.smithi093.stdout:2258: seq_num 1501 ranges {427307=670290,1601472=481522,2552971=483056} 2023-08-27T20:53:47.365 INFO:tasks.rados.rados.0.smithi093.stdout:2258: writing smithi093153278-11 from 427307 to 1097597 tid 1 2023-08-27T20:53:47.370 INFO:tasks.rados.rados.0.smithi093.stdout:2258: writing smithi093153278-11 from 1601472 to 2082994 tid 2 2023-08-27T20:53:47.376 INFO:tasks.rados.rados.0.smithi093.stdout:2258: writing smithi093153278-11 from 2552971 to 3036027 tid 3 2023-08-27T20:53:47.376 INFO:tasks.rados.rados.0.smithi093.stdout:2259: write oid 382 current snap is 0 2023-08-27T20:53:47.376 INFO:tasks.rados.rados.0.smithi093.stdout:2252: finishing write tid 3 to smithi093153278-113 2023-08-27T20:53:47.376 INFO:tasks.rados.rados.0.smithi093.stdout:2252: finishing write tid 4 to smithi093153278-113 2023-08-27T20:53:47.376 INFO:tasks.rados.rados.0.smithi093.stdout:2252: finishing write tid 5 to smithi093153278-113 2023-08-27T20:53:47.376 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 113 v 253 (ObjNum 1498 snap 0 seq_num 1498) dirty exists 2023-08-27T20:53:47.376 INFO:tasks.rados.rados.0.smithi093.stdout:2252: left oid 113 (ObjNum 1498 snap 0 seq_num 1498) 2023-08-27T20:53:47.377 INFO:tasks.rados.rados.0.smithi093.stdout:2255: finishing write tid 1 to smithi093153278-88 2023-08-27T20:53:47.377 INFO:tasks.rados.rados.0.smithi093.stdout:2259: seq_num 1502 ranges {444114=745539,1943601=625180} 2023-08-27T20:53:47.382 INFO:tasks.rados.rados.0.smithi093.stdout:2259: writing smithi093153278-382 from 444114 to 1189653 tid 1 2023-08-27T20:53:47.390 INFO:tasks.rados.rados.0.smithi093.stdout:2259: writing smithi093153278-382 from 1943601 to 2568781 tid 2 2023-08-27T20:53:47.390 INFO:tasks.rados.rados.0.smithi093.stdout:2252: done (6 left) 2023-08-27T20:53:47.391 INFO:tasks.rados.rados.0.smithi093.stdout:2254: done (5 left) 2023-08-27T20:53:47.391 INFO:tasks.rados.rados.0.smithi093.stdout:2260: write oid 162 current snap is 0 2023-08-27T20:53:47.391 INFO:tasks.rados.rados.0.smithi093.stdout:2255: finishing write tid 2 to smithi093153278-88 2023-08-27T20:53:47.391 INFO:tasks.rados.rados.0.smithi093.stdout:2260: seq_num 1503 ranges {460921=420788,1602083=485191,2491464=621953} 2023-08-27T20:53:47.395 INFO:tasks.rados.rados.0.smithi093.stdout:2260: writing smithi093153278-162 from 460921 to 881709 tid 1 2023-08-27T20:53:47.402 INFO:tasks.rados.rados.0.smithi093.stdout:2260: writing smithi093153278-162 from 1602083 to 2087274 tid 2 2023-08-27T20:53:47.407 INFO:tasks.rados.rados.0.smithi093.stdout:2260: writing smithi093153278-162 from 2491464 to 3113417 tid 3 2023-08-27T20:53:47.408 INFO:tasks.rados.rados.0.smithi093.stdout:2261: write oid 159 current snap is 0 2023-08-27T20:53:47.408 INFO:tasks.rados.rados.0.smithi093.stdout:2255: finishing write tid 3 to smithi093153278-88 2023-08-27T20:53:47.408 INFO:tasks.rados.rados.0.smithi093.stdout:2255: finishing write tid 4 to smithi093153278-88 2023-08-27T20:53:47.408 INFO:tasks.rados.rados.0.smithi093.stdout:2255: finishing write tid 5 to smithi093153278-88 2023-08-27T20:53:47.408 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 338 (ObjNum 1500 snap 0 seq_num 1500) dirty exists 2023-08-27T20:53:47.408 INFO:tasks.rados.rados.0.smithi093.stdout:2255: left oid 88 (ObjNum 1500 snap 0 seq_num 1500) 2023-08-27T20:53:47.408 INFO:tasks.rados.rados.0.smithi093.stdout:2261: seq_num 1504 ranges {477728=496037,1660565=628849,2918887=358841} 2023-08-27T20:53:47.412 INFO:tasks.rados.rados.0.smithi093.stdout:2261: writing smithi093153278-159 from 477728 to 973765 tid 1 2023-08-27T20:53:47.420 INFO:tasks.rados.rados.0.smithi093.stdout:2261: writing smithi093153278-159 from 1660565 to 2289414 tid 2 2023-08-27T20:53:47.425 INFO:tasks.rados.rados.0.smithi093.stdout:2261: writing smithi093153278-159 from 2918887 to 3277728 tid 3 2023-08-27T20:53:47.426 INFO:tasks.rados.rados.0.smithi093.stdout:2255: done (6 left) 2023-08-27T20:53:47.426 INFO:tasks.rados.rados.0.smithi093.stdout:2262: write oid 83 current snap is 0 2023-08-27T20:53:47.426 INFO:tasks.rados.rados.0.smithi093.stdout:2257: expect (ObjNum 1347 snap 0 seq_num 1347) 2023-08-27T20:53:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:47 smithi136 ceph-mon[113531]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:53:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:47 smithi136 ceph-mon[113531]: Failed to find standby mgr for failover. Retrying in 2 seconds 2023-08-27T20:53:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2635018763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:47 smithi093 ceph-mon[111335]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:53:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:47 smithi093 ceph-mon[111335]: Failed to find standby mgr for failover. Retrying in 2 seconds 2023-08-27T20:53:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2635018763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:47 smithi093 ceph-mon[118902]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:53:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:47 smithi093 ceph-mon[118902]: Failed to find standby mgr for failover. Retrying in 2 seconds 2023-08-27T20:53:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2635018763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:47.558 INFO:tasks.rados.rados.0.smithi093.stdout:2256: expect (ObjNum 1413 snap 0 seq_num 1413) 2023-08-27T20:53:47.720 INFO:tasks.rados.rados.0.smithi093.stdout:2258: finishing write tid 1 to smithi093153278-11 2023-08-27T20:53:47.720 INFO:tasks.rados.rados.0.smithi093.stdout:2258: finishing write tid 2 to smithi093153278-11 2023-08-27T20:53:47.720 INFO:tasks.rados.rados.0.smithi093.stdout:2258: finishing write tid 3 to smithi093153278-11 2023-08-27T20:53:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:2258: finishing write tid 4 to smithi093153278-11 2023-08-27T20:53:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:2258: finishing write tid 5 to smithi093153278-11 2023-08-27T20:53:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 11 v 257 (ObjNum 1501 snap 0 seq_num 1501) dirty exists 2023-08-27T20:53:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:2258: left oid 11 (ObjNum 1501 snap 0 seq_num 1501) 2023-08-27T20:53:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:2259: finishing write tid 1 to smithi093153278-382 2023-08-27T20:53:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:2262: seq_num 1505 ranges {494535=571286,1719047=488860,2946310=348225} 2023-08-27T20:53:47.725 INFO:tasks.rados.rados.0.smithi093.stdout:2262: writing smithi093153278-83 from 494535 to 1065821 tid 1 2023-08-27T20:53:47.731 INFO:tasks.rados.rados.0.smithi093.stdout:2262: writing smithi093153278-83 from 1719047 to 2207907 tid 2 2023-08-27T20:53:47.738 INFO:tasks.rados.rados.0.smithi093.stdout:2262: writing smithi093153278-83 from 2946310 to 3294535 tid 3 2023-08-27T20:53:47.738 INFO:tasks.rados.rados.0.smithi093.stdout:2256: done (6 left) 2023-08-27T20:53:47.738 INFO:tasks.rados.rados.0.smithi093.stdout:2257: done (5 left) 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2258: done (4 left) 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2263: write oid 63 current snap is 0 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2259: finishing write tid 2 to smithi093153278-382 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2259: finishing write tid 3 to smithi093153278-382 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2259: finishing write tid 4 to smithi093153278-382 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 382 v 376 (ObjNum 1502 snap 0 seq_num 1502) dirty exists 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2259: left oid 382 (ObjNum 1502 snap 0 seq_num 1502) 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2260: finishing write tid 1 to smithi093153278-162 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2260: finishing write tid 2 to smithi093153278-162 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2260: finishing write tid 3 to smithi093153278-162 2023-08-27T20:53:47.739 INFO:tasks.rados.rados.0.smithi093.stdout:2260: finishing write tid 4 to smithi093153278-162 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2260: finishing write tid 5 to smithi093153278-162 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 162 v 267 (ObjNum 1503 snap 0 seq_num 1503) dirty exists 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2260: left oid 162 (ObjNum 1503 snap 0 seq_num 1503) 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2261: finishing write tid 1 to smithi093153278-159 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2261: finishing write tid 2 to smithi093153278-159 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2261: finishing write tid 3 to smithi093153278-159 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2261: finishing write tid 4 to smithi093153278-159 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2261: finishing write tid 5 to smithi093153278-159 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 159 v 261 (ObjNum 1504 snap 0 seq_num 1504) dirty exists 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2261: left oid 159 (ObjNum 1504 snap 0 seq_num 1504) 2023-08-27T20:53:47.740 INFO:tasks.rados.rados.0.smithi093.stdout:2263: seq_num 1506 ranges {511342=762888,1777529=632518,2973733=337609} 2023-08-27T20:53:47.745 INFO:tasks.rados.rados.0.smithi093.stdout:2263: writing smithi093153278-63 from 511342 to 1274230 tid 1 2023-08-27T20:53:47.751 INFO:tasks.rados.rados.0.smithi093.stdout:2263: writing smithi093153278-63 from 1777529 to 2410047 tid 2 2023-08-27T20:53:47.756 INFO:tasks.rados.rados.0.smithi093.stdout:2263: writing smithi093153278-63 from 2973733 to 3311342 tid 3 2023-08-27T20:53:47.756 INFO:tasks.rados.rados.0.smithi093.stdout:2259: done (4 left) 2023-08-27T20:53:47.756 INFO:tasks.rados.rados.0.smithi093.stdout:2260: done (3 left) 2023-08-27T20:53:47.756 INFO:tasks.rados.rados.0.smithi093.stdout:2261: done (2 left) 2023-08-27T20:53:47.757 INFO:tasks.rados.rados.0.smithi093.stdout:2264: write oid 443 current snap is 0 2023-08-27T20:53:47.757 INFO:tasks.rados.rados.0.smithi093.stdout:2262: finishing write tid 1 to smithi093153278-83 2023-08-27T20:53:47.757 INFO:tasks.rados.rados.0.smithi093.stdout:2262: finishing write tid 2 to smithi093153278-83 2023-08-27T20:53:47.757 INFO:tasks.rados.rados.0.smithi093.stdout:2264: seq_num 1507 ranges {528149=438137,1436011=492529,2601156=667041} 2023-08-27T20:53:47.761 INFO:tasks.rados.rados.0.smithi093.stdout:2264: writing smithi093153278-443 from 528149 to 966286 tid 1 2023-08-27T20:53:47.767 INFO:tasks.rados.rados.0.smithi093.stdout:2264: writing smithi093153278-443 from 1436011 to 1928540 tid 2 2023-08-27T20:53:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:2264: writing smithi093153278-443 from 2601156 to 3268197 tid 3 2023-08-27T20:53:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:2265: read oid 415 snap -1 2023-08-27T20:53:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:2265: expect (ObjNum 918 snap 0 seq_num 918) 2023-08-27T20:53:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:2266: read oid 478 snap -1 2023-08-27T20:53:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:2266: expect (ObjNum 1344 snap 0 seq_num 1344) 2023-08-27T20:53:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:2267: read oid 203 snap -1 2023-08-27T20:53:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:2267: expect (ObjNum 1102 snap 0 seq_num 1102) 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2268: read oid 208 snap -1 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2268: expect (ObjNum 1413 snap 0 seq_num 1413) 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2269: write oid 328 current snap is 0 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2262: finishing write tid 3 to smithi093153278-83 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2262: finishing write tid 4 to smithi093153278-83 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2262: finishing write tid 5 to smithi093153278-83 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 83 v 285 (ObjNum 1505 snap 0 seq_num 1505) dirty exists 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2262: left oid 83 (ObjNum 1505 snap 0 seq_num 1505) 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2263: finishing write tid 1 to smithi093153278-63 2023-08-27T20:53:47.776 INFO:tasks.rados.rados.0.smithi093.stdout:2269: seq_num 1508 ranges {544956=513386,1494493=636187,2628579=594666} 2023-08-27T20:53:47.781 INFO:tasks.rados.rados.0.smithi093.stdout:2269: writing smithi093153278-328 from 544956 to 1058342 tid 1 2023-08-27T20:53:47.787 INFO:tasks.rados.rados.0.smithi093.stdout:2269: writing smithi093153278-328 from 1494493 to 2130680 tid 2 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2269: writing smithi093153278-328 from 2628579 to 3223245 tid 3 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2262: done (7 left) 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2270: read oid 31 snap -1 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2270: expect (ObjNum 1384 snap 0 seq_num 1384) 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2271: delete oid 307 current snap is 0 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2263: finishing write tid 2 to smithi093153278-63 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2263: finishing write tid 3 to smithi093153278-63 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2263: finishing write tid 4 to smithi093153278-63 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:2263: finishing write tid 5 to smithi093153278-63 2023-08-27T20:53:47.794 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 63 v 274 (ObjNum 1506 snap 0 seq_num 1506) dirty exists 2023-08-27T20:53:47.795 INFO:tasks.rados.rados.0.smithi093.stdout:2263: left oid 63 (ObjNum 1506 snap 0 seq_num 1506) 2023-08-27T20:53:47.795 INFO:tasks.rados.rados.0.smithi093.stdout:2264: finishing write tid 1 to smithi093153278-443 2023-08-27T20:53:47.795 INFO:tasks.rados.rados.0.smithi093.stdout:2264: finishing write tid 2 to smithi093153278-443 2023-08-27T20:53:47.795 INFO:tasks.rados.rados.0.smithi093.stdout:2267: expect (ObjNum 1102 snap 0 seq_num 1102) 2023-08-27T20:53:47.904 INFO:tasks.rados.rados.0.smithi093.stdout:2268: expect (ObjNum 1413 snap 0 seq_num 1413) 2023-08-27T20:53:48.065 INFO:tasks.rados.rados.0.smithi093.stdout:2264: finishing write tid 3 to smithi093153278-443 2023-08-27T20:53:48.065 INFO:tasks.rados.rados.0.smithi093.stdout:2264: finishing write tid 4 to smithi093153278-443 2023-08-27T20:53:48.065 INFO:tasks.rados.rados.0.smithi093.stdout:2264: finishing write tid 5 to smithi093153278-443 2023-08-27T20:53:48.065 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 341 (ObjNum 1507 snap 0 seq_num 1507) dirty exists 2023-08-27T20:53:48.065 INFO:tasks.rados.rados.0.smithi093.stdout:2264: left oid 443 (ObjNum 1507 snap 0 seq_num 1507) 2023-08-27T20:53:48.065 INFO:tasks.rados.rados.0.smithi093.stdout:2265: expect (ObjNum 918 snap 0 seq_num 918) 2023-08-27T20:53:48.218 INFO:tasks.rados.rados.0.smithi093.stdout:2266: expect (ObjNum 1344 snap 0 seq_num 1344) 2023-08-27T20:53:48.384 INFO:tasks.rados.rados.0.smithi093.stdout:2269: finishing write tid 1 to smithi093153278-328 2023-08-27T20:53:48.384 INFO:tasks.rados.rados.0.smithi093.stdout:2269: finishing write tid 2 to smithi093153278-328 2023-08-27T20:53:48.384 INFO:tasks.rados.rados.0.smithi093.stdout:2263: done (8 left) 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2264: done (7 left) 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2265: done (6 left) 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2266: done (5 left) 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2267: done (4 left) 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2268: done (3 left) 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2271: done (2 left) 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2272: delete oid 6 current snap is 0 2023-08-27T20:53:48.385 INFO:tasks.rados.rados.0.smithi093.stdout:2270: expect (ObjNum 1384 snap 0 seq_num 1384) 2023-08-27T20:53:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:48 smithi136 ceph-mon[113531]: pgmap v7: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2023-08-27T20:53:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:48 smithi093 ceph-mon[111335]: pgmap v7: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2023-08-27T20:53:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:48 smithi093 ceph-mon[118902]: pgmap v7: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 47 MiB/s rd, 6.9 MiB/s wr, 67 op/s 2023-08-27T20:53:48.527 INFO:tasks.rados.rados.0.smithi093.stdout:2269: finishing write tid 3 to smithi093153278-328 2023-08-27T20:53:48.527 INFO:tasks.rados.rados.0.smithi093.stdout:2269: finishing write tid 4 to smithi093153278-328 2023-08-27T20:53:48.527 INFO:tasks.rados.rados.0.smithi093.stdout:2270: done (2 left) 2023-08-27T20:53:48.527 INFO:tasks.rados.rados.0.smithi093.stdout:2272: done (1 left) 2023-08-27T20:53:48.527 INFO:tasks.rados.rados.0.smithi093.stdout:2273: write oid 288 current snap is 0 2023-08-27T20:53:48.528 INFO:tasks.rados.rados.0.smithi093.stdout:2269: finishing write tid 5 to smithi093153278-328 2023-08-27T20:53:48.528 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 323 (ObjNum 1508 snap 0 seq_num 1508) dirty exists 2023-08-27T20:53:48.528 INFO:tasks.rados.rados.0.smithi093.stdout:2269: left oid 328 (ObjNum 1508 snap 0 seq_num 1508) 2023-08-27T20:53:48.528 INFO:tasks.rados.rados.0.smithi093.stdout:2273: seq_num 1511 ranges {595377=739133,1953586=499867,3110848=284529} 2023-08-27T20:53:48.534 INFO:tasks.rados.rados.0.smithi093.stdout:2273: writing smithi093153278-288 from 595377 to 1334510 tid 1 2023-08-27T20:53:48.540 INFO:tasks.rados.rados.0.smithi093.stdout:2273: writing smithi093153278-288 from 1953586 to 2453453 tid 2 2023-08-27T20:53:48.545 INFO:tasks.rados.rados.0.smithi093.stdout:2273: writing smithi093153278-288 from 3110848 to 3395377 tid 3 2023-08-27T20:53:48.546 INFO:tasks.rados.rados.0.smithi093.stdout:2269: done (1 left) 2023-08-27T20:53:48.546 INFO:tasks.rados.rados.0.smithi093.stdout:2274: write oid 208 current snap is 0 2023-08-27T20:53:48.546 INFO:tasks.rados.rados.0.smithi093.stdout:2274: seq_num 1512 ranges {612184=414382,1612068=643525,3021918=390266} 2023-08-27T20:53:48.551 INFO:tasks.rados.rados.0.smithi093.stdout:2274: writing smithi093153278-208 from 612184 to 1026566 tid 1 2023-08-27T20:53:48.558 INFO:tasks.rados.rados.0.smithi093.stdout:2274: writing smithi093153278-208 from 1612068 to 2255593 tid 2 2023-08-27T20:53:48.564 INFO:tasks.rados.rados.0.smithi093.stdout:2274: writing smithi093153278-208 from 3021918 to 3412184 tid 3 2023-08-27T20:53:48.565 INFO:tasks.rados.rados.0.smithi093.stdout:2275: write oid 94 current snap is 0 2023-08-27T20:53:48.565 INFO:tasks.rados.rados.0.smithi093.stdout:2273: finishing write tid 1 to smithi093153278-288 2023-08-27T20:53:48.565 INFO:tasks.rados.rados.0.smithi093.stdout:2273: finishing write tid 2 to smithi093153278-288 2023-08-27T20:53:48.565 INFO:tasks.rados.rados.0.smithi093.stdout:2273: finishing write tid 3 to smithi093153278-288 2023-08-27T20:53:48.565 INFO:tasks.rados.rados.0.smithi093.stdout:2275: seq_num 1513 ranges {628991=605984,1786903=787183,3165694=263297} 2023-08-27T20:53:48.571 INFO:tasks.rados.rados.0.smithi093.stdout:2275: writing smithi093153278-94 from 628991 to 1234975 tid 1 2023-08-27T20:53:48.578 INFO:tasks.rados.rados.0.smithi093.stdout:2275: writing smithi093153278-94 from 1786903 to 2574086 tid 2 2023-08-27T20:53:48.583 INFO:tasks.rados.rados.0.smithi093.stdout:2275: writing smithi093153278-94 from 3165694 to 3428991 tid 3 2023-08-27T20:53:48.583 INFO:tasks.rados.rados.0.smithi093.stdout:2276: write oid 66 current snap is 0 2023-08-27T20:53:48.583 INFO:tasks.rados.rados.0.smithi093.stdout:2273: finishing write tid 4 to smithi093153278-288 2023-08-27T20:53:48.584 INFO:tasks.rados.rados.0.smithi093.stdout:2276: seq_num 1514 ranges {645798=681233,1729032=647194,3076764=369034} 2023-08-27T20:53:48.590 INFO:tasks.rados.rados.0.smithi093.stdout:2276: writing smithi093153278-66 from 645798 to 1327031 tid 1 2023-08-27T20:53:48.596 INFO:tasks.rados.rados.0.smithi093.stdout:2276: writing smithi093153278-66 from 1729032 to 2376226 tid 2 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2276: writing smithi093153278-66 from 3076764 to 3445798 tid 3 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2277: write oid 383 current snap is 0 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2273: finishing write tid 5 to smithi093153278-288 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 288 v 271 (ObjNum 1511 snap 0 seq_num 1511) dirty exists 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2273: left oid 288 (ObjNum 1511 snap 0 seq_num 1511) 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2274: finishing write tid 1 to smithi093153278-208 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2274: finishing write tid 2 to smithi093153278-208 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2274: finishing write tid 3 to smithi093153278-208 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2274: finishing write tid 4 to smithi093153278-208 2023-08-27T20:53:48.603 INFO:tasks.rados.rados.0.smithi093.stdout:2277: seq_num 1515 ranges {662605=756482,2187514=790852} 2023-08-27T20:53:48.610 INFO:tasks.rados.rados.0.smithi093.stdout:2277: writing smithi093153278-383 from 662605 to 1419087 tid 1 2023-08-27T20:53:48.619 INFO:tasks.rados.rados.0.smithi093.stdout:2277: writing smithi093153278-383 from 2187514 to 2978366 tid 2 2023-08-27T20:53:48.619 INFO:tasks.rados.rados.0.smithi093.stdout:2273: done (4 left) 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2278: delete oid 481 current snap is 0 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2274: finishing write tid 5 to smithi093153278-208 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 208 v 411 (ObjNum 1512 snap 0 seq_num 1512) dirty exists 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2274: left oid 208 (ObjNum 1512 snap 0 seq_num 1512) 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2275: finishing write tid 1 to smithi093153278-94 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2275: finishing write tid 2 to smithi093153278-94 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2275: finishing write tid 3 to smithi093153278-94 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2275: finishing write tid 4 to smithi093153278-94 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2275: finishing write tid 5 to smithi093153278-94 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 94 v 207 (ObjNum 1513 snap 0 seq_num 1513) dirty exists 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2275: left oid 94 (ObjNum 1513 snap 0 seq_num 1513) 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2276: finishing write tid 1 to smithi093153278-66 2023-08-27T20:53:48.620 INFO:tasks.rados.rados.0.smithi093.stdout:2276: finishing write tid 2 to smithi093153278-66 2023-08-27T20:53:48.621 INFO:tasks.rados.rados.0.smithi093.stdout:2276: finishing write tid 3 to smithi093153278-66 2023-08-27T20:53:48.621 INFO:tasks.rados.rados.0.smithi093.stdout:2276: finishing write tid 4 to smithi093153278-66 2023-08-27T20:53:48.621 INFO:tasks.rados.rados.0.smithi093.stdout:2276: finishing write tid 5 to smithi093153278-66 2023-08-27T20:53:48.621 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 66 v 380 (ObjNum 1514 snap 0 seq_num 1514) dirty exists 2023-08-27T20:53:48.621 INFO:tasks.rados.rados.0.smithi093.stdout:2276: left oid 66 (ObjNum 1514 snap 0 seq_num 1514) 2023-08-27T20:53:48.621 INFO:tasks.rados.rados.0.smithi093.stdout:2277: finishing write tid 1 to smithi093153278-383 2023-08-27T20:53:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:2274: done (4 left) 2023-08-27T20:53:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:2275: done (3 left) 2023-08-27T20:53:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:2276: done (2 left) 2023-08-27T20:53:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:2278: done (1 left) 2023-08-27T20:53:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:2279: write oid 214 current snap is 0 2023-08-27T20:53:48.622 INFO:tasks.rados.rados.0.smithi093.stdout:2279: seq_num 1517 ranges {696219=506980,1904478=794521,3159033=337186} 2023-08-27T20:53:48.628 INFO:tasks.rados.rados.0.smithi093.stdout:2279: writing smithi093153278-214 from 696219 to 1203199 tid 1 2023-08-27T20:53:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:2279: writing smithi093153278-214 from 1904478 to 2698999 tid 2 2023-08-27T20:53:48.641 INFO:tasks.rados.rados.0.smithi093.stdout:2279: writing smithi093153278-214 from 3159033 to 3496219 tid 3 2023-08-27T20:53:48.641 INFO:tasks.rados.rados.0.smithi093.stdout:2280: read oid 51 snap -1 2023-08-27T20:53:48.641 INFO:tasks.rados.rados.0.smithi093.stdout:2280: expect (ObjNum 1304 snap 0 seq_num 1304) 2023-08-27T20:53:48.641 INFO:tasks.rados.rados.0.smithi093.stdout:2281: write oid 180 current snap is 0 2023-08-27T20:53:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:2277: finishing write tid 2 to smithi093153278-383 2023-08-27T20:53:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:2277: finishing write tid 3 to smithi093153278-383 2023-08-27T20:53:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:2277: finishing write tid 4 to smithi093153278-383 2023-08-27T20:53:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 383 v 383 (ObjNum 1515 snap 0 seq_num 1515) dirty exists 2023-08-27T20:53:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:2277: left oid 383 (ObjNum 1515 snap 0 seq_num 1515) 2023-08-27T20:53:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:2279: finishing write tid 1 to smithi093153278-214 2023-08-27T20:53:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:2281: seq_num 1518 ranges {713026=582229,1962960=654532,3186456=326570} 2023-08-27T20:53:48.648 INFO:tasks.rados.rados.0.smithi093.stdout:2281: writing smithi093153278-180 from 713026 to 1295255 tid 1 2023-08-27T20:53:48.655 INFO:tasks.rados.rados.0.smithi093.stdout:2281: writing smithi093153278-180 from 1962960 to 2617492 tid 2 2023-08-27T20:53:48.661 INFO:tasks.rados.rados.0.smithi093.stdout:2281: writing smithi093153278-180 from 3186456 to 3513026 tid 3 2023-08-27T20:53:48.661 INFO:tasks.rados.rados.0.smithi093.stdout:2277: done (3 left) 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2282: read oid 50 snap -1 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2282: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2283: read oid 418 snap -1 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2283: expect (ObjNum 1492 snap 0 seq_num 1492) 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2284: read oid 102 snap -1 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2284: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2285: write oid 442 current snap is 0 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2279: finishing write tid 2 to smithi093153278-214 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2279: finishing write tid 3 to smithi093153278-214 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2279: finishing write tid 4 to smithi093153278-214 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:2279: finishing write tid 5 to smithi093153278-214 2023-08-27T20:53:48.662 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 214 v 211 (ObjNum 1517 snap 0 seq_num 1517) dirty exists 2023-08-27T20:53:48.663 INFO:tasks.rados.rados.0.smithi093.stdout:2279: left oid 214 (ObjNum 1517 snap 0 seq_num 1517) 2023-08-27T20:53:48.663 INFO:tasks.rados.rados.0.smithi093.stdout:2280: expect (ObjNum 1304 snap 0 seq_num 1304) 2023-08-27T20:53:48.895 INFO:tasks.rados.rados.0.smithi093.stdout:2281: finishing write tid 1 to smithi093153278-180 2023-08-27T20:53:48.896 INFO:tasks.rados.rados.0.smithi093.stdout:2281: finishing write tid 2 to smithi093153278-180 2023-08-27T20:53:48.896 INFO:tasks.rados.rados.0.smithi093.stdout:2285: seq_num 1519 ranges {729833=657478,1905089=798190,3497526=32307} 2023-08-27T20:53:48.902 INFO:tasks.rados.rados.0.smithi093.stdout:2285: writing smithi093153278-442 from 729833 to 1387311 tid 1 2023-08-27T20:53:48.909 INFO:tasks.rados.rados.0.smithi093.stdout:2285: writing smithi093153278-442 from 1905089 to 2703279 tid 2 2023-08-27T20:53:48.915 INFO:tasks.rados.rados.0.smithi093.stdout:2285: writing smithi093153278-442 from 3497526 to 3529833 tid 3 2023-08-27T20:53:48.915 INFO:tasks.rados.rados.0.smithi093.stdout:2279: done (6 left) 2023-08-27T20:53:48.915 INFO:tasks.rados.rados.0.smithi093.stdout:2280: done (5 left) 2023-08-27T20:53:48.915 INFO:tasks.rados.rados.0.smithi093.stdout:2286: delete oid 490 current snap is 0 2023-08-27T20:53:48.915 INFO:tasks.rados.rados.0.smithi093.stdout:2281: finishing write tid 3 to smithi093153278-180 2023-08-27T20:53:48.915 INFO:tasks.rados.rados.0.smithi093.stdout:2281: finishing write tid 4 to smithi093153278-180 2023-08-27T20:53:48.916 INFO:tasks.rados.rados.0.smithi093.stdout:2281: finishing write tid 5 to smithi093153278-180 2023-08-27T20:53:48.916 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 180 v 283 (ObjNum 1518 snap 0 seq_num 1518) dirty exists 2023-08-27T20:53:48.916 INFO:tasks.rados.rados.0.smithi093.stdout:2281: left oid 180 (ObjNum 1518 snap 0 seq_num 1518) 2023-08-27T20:53:48.916 INFO:tasks.rados.rados.0.smithi093.stdout:2282: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:53:49.071 INFO:tasks.rados.rados.0.smithi093.stdout:2283: expect (ObjNum 1492 snap 0 seq_num 1492) 2023-08-27T20:53:49.204 INFO:tasks.rados.rados.0.smithi093.stdout:2284: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2285: finishing write tid 1 to smithi093153278-442 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2285: finishing write tid 2 to smithi093153278-442 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2285: finishing write tid 3 to smithi093153278-442 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2285: finishing write tid 4 to smithi093153278-442 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2285: finishing write tid 5 to smithi093153278-442 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 442 v 345 (ObjNum 1519 snap 0 seq_num 1519) dirty exists 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2285: left oid 442 (ObjNum 1519 snap 0 seq_num 1519) 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2281: done (5 left) 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2282: done (4 left) 2023-08-27T20:53:49.411 INFO:tasks.rados.rados.0.smithi093.stdout:2283: done (3 left) 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2284: done (2 left) 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2285: done (1 left) 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2286: done (0 left) 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2287: read oid 439 snap -1 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2287: expect (ObjNum 1327 snap 0 seq_num 1327) 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2288: read oid 453 snap -1 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2288: expect (ObjNum 1187 snap 0 seq_num 1187) 2023-08-27T20:53:49.412 INFO:tasks.rados.rados.0.smithi093.stdout:2289: delete oid 229 current snap is 0 2023-08-27T20:53:49.413 INFO:tasks.rados.rados.0.smithi093.stdout:2289: done (2 left) 2023-08-27T20:53:49.413 INFO:tasks.rados.rados.0.smithi093.stdout:2290: delete oid 430 current snap is 0 2023-08-27T20:53:49.414 INFO:tasks.rados.rados.0.smithi093.stdout:2290: done (2 left) 2023-08-27T20:53:49.414 INFO:tasks.rados.rados.0.smithi093.stdout:2291: read oid 63 snap -1 2023-08-27T20:53:49.414 INFO:tasks.rados.rados.0.smithi093.stdout:2291: expect (ObjNum 1506 snap 0 seq_num 1506) 2023-08-27T20:53:49.414 INFO:tasks.rados.rados.0.smithi093.stdout:2292: write oid 363 current snap is 0 2023-08-27T20:53:49.414 INFO:tasks.rados.rados.0.smithi093.stdout:2292: seq_num 1523 ranges {797061=674827,2139017=405528,3207218=389843} 2023-08-27T20:53:49.421 INFO:tasks.rados.rados.0.smithi093.stdout:2292: writing smithi093153278-363 from 797061 to 1471888 tid 1 2023-08-27T20:53:49.427 INFO:tasks.rados.rados.0.smithi093.stdout:2292: writing smithi093153278-363 from 2139017 to 2544545 tid 2 2023-08-27T20:53:49.433 INFO:tasks.rados.rados.0.smithi093.stdout:2292: writing smithi093153278-363 from 3207218 to 3597061 tid 3 2023-08-27T20:53:49.433 INFO:tasks.rados.rados.0.smithi093.stdout:2293: read oid 383 snap -1 2023-08-27T20:53:49.433 INFO:tasks.rados.rados.0.smithi093.stdout:2293: expect (ObjNum 1515 snap 0 seq_num 1515) 2023-08-27T20:53:49.434 INFO:tasks.rados.rados.0.smithi093.stdout:2294: write oid 200 current snap is 0 2023-08-27T20:53:49.434 INFO:tasks.rados.rados.0.smithi093.stdout:2287: expect (ObjNum 1327 snap 0 seq_num 1327) 2023-08-27T20:53:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:49 smithi136 ceph-mon[113531]: Failed to find standby mgr for failover. Retrying in 8 seconds 2023-08-27T20:53:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3129412470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:49 smithi093 ceph-mon[111335]: Failed to find standby mgr for failover. Retrying in 8 seconds 2023-08-27T20:53:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3129412470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:49 smithi093 ceph-mon[118902]: Failed to find standby mgr for failover. Retrying in 8 seconds 2023-08-27T20:53:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3129412470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:49.537 INFO:tasks.rados.rados.0.smithi093.stdout:2292: finishing write tid 1 to smithi093153278-363 2023-08-27T20:53:49.537 INFO:tasks.rados.rados.0.smithi093.stdout:2292: finishing write tid 2 to smithi093153278-363 2023-08-27T20:53:49.538 INFO:tasks.rados.rados.0.smithi093.stdout:2294: seq_num 1524 ranges {413868=750076,1797499=549186,3118288=495580} 2023-08-27T20:53:49.544 INFO:tasks.rados.rados.0.smithi093.stdout:2294: writing smithi093153278-200 from 413868 to 1163944 tid 1 2023-08-27T20:53:49.550 INFO:tasks.rados.rados.0.smithi093.stdout:2294: writing smithi093153278-200 from 1797499 to 2346685 tid 2 2023-08-27T20:53:49.557 INFO:tasks.rados.rados.0.smithi093.stdout:2294: writing smithi093153278-200 from 3118288 to 3613868 tid 3 2023-08-27T20:53:49.558 INFO:tasks.rados.rados.0.smithi093.stdout:2287: done (5 left) 2023-08-27T20:53:49.558 INFO:tasks.rados.rados.0.smithi093.stdout:2295: write oid 232 current snap is 0 2023-08-27T20:53:49.558 INFO:tasks.rados.rados.0.smithi093.stdout:2293: expect (ObjNum 1515 snap 0 seq_num 1515) 2023-08-27T20:53:49.737 INFO:tasks.rados.rados.0.smithi093.stdout:2292: finishing write tid 3 to smithi093153278-363 2023-08-27T20:53:49.738 INFO:tasks.rados.rados.0.smithi093.stdout:2292: finishing write tid 4 to smithi093153278-363 2023-08-27T20:53:49.738 INFO:tasks.rados.rados.0.smithi093.stdout:2288: expect (ObjNum 1187 snap 0 seq_num 1187) 2023-08-27T20:53:49.908 INFO:tasks.rados.rados.0.smithi093.stdout:2292: finishing write tid 5 to smithi093153278-363 2023-08-27T20:53:49.909 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 416 (ObjNum 1523 snap 0 seq_num 1523) dirty exists 2023-08-27T20:53:49.909 INFO:tasks.rados.rados.0.smithi093.stdout:2292: left oid 363 (ObjNum 1523 snap 0 seq_num 1523) 2023-08-27T20:53:49.909 INFO:tasks.rados.rados.0.smithi093.stdout:2291: expect (ObjNum 1506 snap 0 seq_num 1506) 2023-08-27T20:53:50.007 INFO:tasks.rados.rados.0.smithi093.stdout:2294: finishing write tid 1 to smithi093153278-200 2023-08-27T20:53:50.007 INFO:tasks.rados.rados.0.smithi093.stdout:2294: finishing write tid 2 to smithi093153278-200 2023-08-27T20:53:50.007 INFO:tasks.rados.rados.0.smithi093.stdout:2294: finishing write tid 3 to smithi093153278-200 2023-08-27T20:53:50.007 INFO:tasks.rados.rados.0.smithi093.stdout:2294: finishing write tid 4 to smithi093153278-200 2023-08-27T20:53:50.007 INFO:tasks.rados.rados.0.smithi093.stdout:2294: finishing write tid 5 to smithi093153278-200 2023-08-27T20:53:50.008 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 200 v 387 (ObjNum 1524 snap 0 seq_num 1524) dirty exists 2023-08-27T20:53:50.008 INFO:tasks.rados.rados.0.smithi093.stdout:2294: left oid 200 (ObjNum 1524 snap 0 seq_num 1524) 2023-08-27T20:53:50.008 INFO:tasks.rados.rados.0.smithi093.stdout:2295: seq_num 1525 ranges {430675=425325,1455981=409197,2462064=535349,3463076=167599} 2023-08-27T20:53:50.011 INFO:tasks.rados.rados.0.smithi093.stdout:2295: writing smithi093153278-232 from 430675 to 856000 tid 1 2023-08-27T20:53:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:2295: writing smithi093153278-232 from 1455981 to 1865178 tid 2 2023-08-27T20:53:50.023 INFO:tasks.rados.rados.0.smithi093.stdout:2295: writing smithi093153278-232 from 2462064 to 2997413 tid 3 2023-08-27T20:53:50.027 INFO:tasks.rados.rados.0.smithi093.stdout:2295: writing smithi093153278-232 from 3463076 to 3630675 tid 4 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2288: done (5 left) 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2291: done (4 left) 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2292: done (3 left) 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2293: done (2 left) 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2294: done (1 left) 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2296: read oid 305 snap -1 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2296: expect (ObjNum 1275 snap 0 seq_num 1275) 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2297: write oid 135 current snap is 0 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2295: finishing write tid 1 to smithi093153278-232 2023-08-27T20:53:50.028 INFO:tasks.rados.rados.0.smithi093.stdout:2297: seq_num 1526 ranges {447482=500574,1514463=552855,2489487=462974,3645668=1814} 2023-08-27T20:53:50.032 INFO:tasks.rados.rados.0.smithi093.stdout:2297: writing smithi093153278-135 from 447482 to 948056 tid 1 2023-08-27T20:53:50.038 INFO:tasks.rados.rados.0.smithi093.stdout:2297: writing smithi093153278-135 from 1514463 to 2067318 tid 2 2023-08-27T20:53:50.043 INFO:tasks.rados.rados.0.smithi093.stdout:2297: writing smithi093153278-135 from 2489487 to 2952461 tid 3 2023-08-27T20:53:50.047 INFO:tasks.rados.rados.0.smithi093.stdout:2297: writing smithi093153278-135 from 3645668 to 3647482 tid 4 2023-08-27T20:53:50.047 INFO:tasks.rados.rados.0.smithi093.stdout:2298: write oid 309 current snap is 0 2023-08-27T20:53:50.048 INFO:tasks.rados.rados.0.smithi093.stdout:2295: finishing write tid 2 to smithi093153278-232 2023-08-27T20:53:50.048 INFO:tasks.rados.rados.0.smithi093.stdout:2295: finishing write tid 3 to smithi093153278-232 2023-08-27T20:53:50.048 INFO:tasks.rados.rados.0.smithi093.stdout:2298: seq_num 1527 ranges {464289=575823,1456592=412866,2400557=674246,3595554=68735} 2023-08-27T20:53:50.053 INFO:tasks.rados.rados.0.smithi093.stdout:2298: writing smithi093153278-309 from 464289 to 1040112 tid 1 2023-08-27T20:53:50.057 INFO:tasks.rados.rados.0.smithi093.stdout:2298: writing smithi093153278-309 from 1456592 to 1869458 tid 2 2023-08-27T20:53:50.064 INFO:tasks.rados.rados.0.smithi093.stdout:2298: writing smithi093153278-309 from 2400557 to 3074803 tid 3 2023-08-27T20:53:50.067 INFO:tasks.rados.rados.0.smithi093.stdout:2298: writing smithi093153278-309 from 3595554 to 3664289 tid 4 2023-08-27T20:53:50.067 INFO:tasks.rados.rados.0.smithi093.stdout:2299: read oid 444 snap -1 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:2299: expect (ObjNum 1221 snap 0 seq_num 1221) 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:2300: delete oid 158 current snap is 0 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:2295: finishing write tid 4 to smithi093153278-232 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:2295: finishing write tid 5 to smithi093153278-232 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:2295: finishing write tid 6 to smithi093153278-232 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 232 v 320 (ObjNum 1525 snap 0 seq_num 1525) dirty exists 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:2295: left oid 232 (ObjNum 1525 snap 0 seq_num 1525) 2023-08-27T20:53:50.068 INFO:tasks.rados.rados.0.smithi093.stdout:2296: expect (ObjNum 1275 snap 0 seq_num 1275) 2023-08-27T20:53:50.301 INFO:tasks.rados.rados.0.smithi093.stdout:2297: finishing write tid 1 to smithi093153278-135 2023-08-27T20:53:50.301 INFO:tasks.rados.rados.0.smithi093.stdout:2297: finishing write tid 2 to smithi093153278-135 2023-08-27T20:53:50.301 INFO:tasks.rados.rados.0.smithi093.stdout:2295: done (5 left) 2023-08-27T20:53:50.301 INFO:tasks.rados.rados.0.smithi093.stdout:2296: done (4 left) 2023-08-27T20:53:50.301 INFO:tasks.rados.rados.0.smithi093.stdout:2300: done (3 left) 2023-08-27T20:53:50.301 INFO:tasks.rados.rados.0.smithi093.stdout:2301: write oid 345 current snap is 0 2023-08-27T20:53:50.301 INFO:tasks.rados.rados.0.smithi093.stdout:2297: finishing write tid 3 to smithi093153278-135 2023-08-27T20:53:50.302 INFO:tasks.rados.rados.0.smithi093.stdout:2297: finishing write tid 4 to smithi093153278-135 2023-08-27T20:53:50.302 INFO:tasks.rados.rados.0.smithi093.stdout:2297: finishing write tid 5 to smithi093153278-135 2023-08-27T20:53:50.302 INFO:tasks.rados.rados.0.smithi093.stdout:2297: finishing write tid 6 to smithi093153278-135 2023-08-27T20:53:50.302 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 135 v 279 (ObjNum 1526 snap 0 seq_num 1526) dirty exists 2023-08-27T20:53:50.302 INFO:tasks.rados.rados.0.smithi093.stdout:2297: left oid 135 (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:53:50.302 INFO:tasks.rados.rados.0.smithi093.stdout:2298: finishing write tid 1 to smithi093153278-309 2023-08-27T20:53:50.302 INFO:tasks.rados.rados.0.smithi093.stdout:2301: seq_num 1529 ranges {497903=442674,1689909=416535,2571756=696790} 2023-08-27T20:53:50.305 INFO:tasks.rados.rados.0.smithi093.stdout:2301: writing smithi093153278-345 from 497903 to 940577 tid 1 2023-08-27T20:53:50.312 INFO:tasks.rados.rados.0.smithi093.stdout:2301: writing smithi093153278-345 from 1689909 to 2106444 tid 2 2023-08-27T20:53:50.319 INFO:tasks.rados.rados.0.smithi093.stdout:2301: writing smithi093153278-345 from 2571756 to 3268546 tid 3 2023-08-27T20:53:50.319 INFO:tasks.rados.rados.0.smithi093.stdout:2297: done (3 left) 2023-08-27T20:53:50.319 INFO:tasks.rados.rados.0.smithi093.stdout:2302: read oid 24 snap -1 2023-08-27T20:53:50.320 INFO:tasks.rados.rados.0.smithi093.stdout:2302: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:53:50.320 INFO:tasks.rados.rados.0.smithi093.stdout:2303: read oid 248 snap -1 2023-08-27T20:53:50.320 INFO:tasks.rados.rados.0.smithi093.stdout:2303: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:50.320 INFO:tasks.rados.rados.0.smithi093.stdout:2304: write oid 395 current snap is 0 2023-08-27T20:53:50.320 INFO:tasks.rados.rados.0.smithi093.stdout:2298: finishing write tid 2 to smithi093153278-309 2023-08-27T20:53:50.320 INFO:tasks.rados.rados.0.smithi093.stdout:2298: finishing write tid 3 to smithi093153278-309 2023-08-27T20:53:50.320 INFO:tasks.rados.rados.0.smithi093.stdout:2304: seq_num 1530 ranges {514710=517923,1748391=560193,2999179=624415} 2023-08-27T20:53:50.324 INFO:tasks.rados.rados.0.smithi093.stdout:2304: writing smithi093153278-395 from 514710 to 1032633 tid 1 2023-08-27T20:53:50.332 INFO:tasks.rados.rados.0.smithi093.stdout:2304: writing smithi093153278-395 from 1748391 to 2308584 tid 2 2023-08-27T20:53:50.340 INFO:tasks.rados.rados.0.smithi093.stdout:2304: writing smithi093153278-395 from 2999179 to 3623594 tid 3 2023-08-27T20:53:50.340 INFO:tasks.rados.rados.0.smithi093.stdout:2305: read oid 408 snap -1 2023-08-27T20:53:50.340 INFO:tasks.rados.rados.0.smithi093.stdout:2305: expect (ObjNum 1355 snap 0 seq_num 1355) 2023-08-27T20:53:50.340 INFO:tasks.rados.rados.0.smithi093.stdout:2298: finishing write tid 4 to smithi093153278-309 2023-08-27T20:53:50.340 INFO:tasks.rados.rados.0.smithi093.stdout:2298: finishing write tid 5 to smithi093153278-309 2023-08-27T20:53:50.341 INFO:tasks.rados.rados.0.smithi093.stdout:2306: read oid 290 snap -1 2023-08-27T20:53:50.341 INFO:tasks.rados.rados.0.smithi093.stdout:2306: expect (ObjNum 711 snap 0 seq_num 711) 2023-08-27T20:53:50.341 INFO:tasks.rados.rados.0.smithi093.stdout:2307: read oid 178 snap -1 2023-08-27T20:53:50.341 INFO:tasks.rados.rados.0.smithi093.stdout:2307: expect deleted 2023-08-27T20:53:50.341 INFO:tasks.rados.rados.0.smithi093.stdout:2308: delete oid 57 current snap is 0 2023-08-27T20:53:50.341 INFO:tasks.rados.rados.0.smithi093.stdout:2298: finishing write tid 6 to smithi093153278-309 2023-08-27T20:53:50.341 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 309 v 276 (ObjNum 1527 snap 0 seq_num 1527) dirty exists 2023-08-27T20:53:50.342 INFO:tasks.rados.rados.0.smithi093.stdout:2298: left oid 309 (ObjNum 1527 snap 0 seq_num 1527) 2023-08-27T20:53:50.342 INFO:tasks.rados.rados.0.smithi093.stdout:2299: expect (ObjNum 1221 snap 0 seq_num 1221) 2023-08-27T20:53:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:50 smithi136 ceph-mon[113531]: pgmap v8: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 5.4 MiB/s wr, 52 op/s 2023-08-27T20:53:50.466 INFO:tasks.rados.rados.0.smithi093.stdout:2301: finishing write tid 1 to smithi093153278-345 2023-08-27T20:53:50.466 INFO:tasks.rados.rados.0.smithi093.stdout:2301: finishing write tid 2 to smithi093153278-345 2023-08-27T20:53:50.466 INFO:tasks.rados.rados.0.smithi093.stdout:2298: done (9 left) 2023-08-27T20:53:50.466 INFO:tasks.rados.rados.0.smithi093.stdout:2299: done (8 left) 2023-08-27T20:53:50.466 INFO:tasks.rados.rados.0.smithi093.stdout:2308: done (7 left) 2023-08-27T20:53:50.467 INFO:tasks.rados.rados.0.smithi093.stdout:2309: write oid 349 current snap is 0 2023-08-27T20:53:50.467 INFO:tasks.rados.rados.0.smithi093.stdout:2304: finishing write tid 1 to smithi093153278-395 2023-08-27T20:53:50.467 INFO:tasks.rados.rados.0.smithi093.stdout:2301: finishing write tid 3 to smithi093153278-345 2023-08-27T20:53:50.467 INFO:tasks.rados.rados.0.smithi093.stdout:2301: finishing write tid 4 to smithi093153278-345 2023-08-27T20:53:50.467 INFO:tasks.rados.rados.0.smithi093.stdout:2301: finishing write tid 5 to smithi093153278-345 2023-08-27T20:53:50.467 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 345 v 215 (ObjNum 1529 snap 0 seq_num 1529) dirty exists 2023-08-27T20:53:50.467 INFO:tasks.rados.rados.0.smithi093.stdout:2301: left oid 345 (ObjNum 1529 snap 0 seq_num 1529) 2023-08-27T20:53:50.468 INFO:tasks.rados.rados.0.smithi093.stdout:2302: expect (ObjNum 23 snap 0 seq_num 23) 2023-08-27T20:53:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:50 smithi093 ceph-mon[111335]: pgmap v8: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 5.4 MiB/s wr, 52 op/s 2023-08-27T20:53:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:50 smithi093 ceph-mon[118902]: pgmap v8: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 37 MiB/s rd, 5.4 MiB/s wr, 52 op/s 2023-08-27T20:53:50.542 INFO:tasks.rados.rados.0.smithi093.stdout:2304: finishing write tid 2 to smithi093153278-395 2023-08-27T20:53:50.543 INFO:tasks.rados.rados.0.smithi093.stdout:2303: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:50.758 INFO:tasks.rados.rados.0.smithi093.stdout:2304: finishing write tid 3 to smithi093153278-395 2023-08-27T20:53:50.759 INFO:tasks.rados.rados.0.smithi093.stdout:2304: finishing write tid 4 to smithi093153278-395 2023-08-27T20:53:50.759 INFO:tasks.rados.rados.0.smithi093.stdout:2304: finishing write tid 5 to smithi093153278-395 2023-08-27T20:53:50.759 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 420 (ObjNum 1530 snap 0 seq_num 1530) dirty exists 2023-08-27T20:53:50.759 INFO:tasks.rados.rados.0.smithi093.stdout:2304: left oid 395 (ObjNum 1530 snap 0 seq_num 1530) 2023-08-27T20:53:50.759 INFO:tasks.rados.rados.0.smithi093.stdout:2309: seq_num 1532 ranges {548324=668421,1749002=563862,2937672=646959} 2023-08-27T20:53:50.765 INFO:tasks.rados.rados.0.smithi093.stdout:2309: writing smithi093153278-349 from 548324 to 1216745 tid 1 2023-08-27T20:53:50.771 INFO:tasks.rados.rados.0.smithi093.stdout:2309: writing smithi093153278-349 from 1749002 to 2312864 tid 2 2023-08-27T20:53:50.779 INFO:tasks.rados.rados.0.smithi093.stdout:2309: writing smithi093153278-349 from 2937672 to 3584631 tid 3 2023-08-27T20:53:50.779 INFO:tasks.rados.rados.0.smithi093.stdout:2301: done (7 left) 2023-08-27T20:53:50.779 INFO:tasks.rados.rados.0.smithi093.stdout:2302: done (6 left) 2023-08-27T20:53:50.779 INFO:tasks.rados.rados.0.smithi093.stdout:2303: done (5 left) 2023-08-27T20:53:50.779 INFO:tasks.rados.rados.0.smithi093.stdout:2304: done (4 left) 2023-08-27T20:53:50.779 INFO:tasks.rados.rados.0.smithi093.stdout:2307: done (3 left) 2023-08-27T20:53:50.780 INFO:tasks.rados.rados.0.smithi093.stdout:2310: write oid 389 current snap is 0 2023-08-27T20:53:50.780 INFO:tasks.rados.rados.0.smithi093.stdout:2305: expect (ObjNum 1355 snap 0 seq_num 1355) 2023-08-27T20:53:50.870 INFO:tasks.rados.rados.0.smithi093.stdout:2306: expect (ObjNum 711 snap 0 seq_num 711) 2023-08-27T20:53:50.997 INFO:tasks.rados.rados.0.smithi093.stdout:2309: finishing write tid 1 to smithi093153278-349 2023-08-27T20:53:50.997 INFO:tasks.rados.rados.0.smithi093.stdout:2309: finishing write tid 2 to smithi093153278-349 2023-08-27T20:53:50.997 INFO:tasks.rados.rados.0.smithi093.stdout:2309: finishing write tid 3 to smithi093153278-349 2023-08-27T20:53:50.997 INFO:tasks.rados.rados.0.smithi093.stdout:2309: finishing write tid 4 to smithi093153278-349 2023-08-27T20:53:50.997 INFO:tasks.rados.rados.0.smithi093.stdout:2309: finishing write tid 5 to smithi093153278-349 2023-08-27T20:53:50.997 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 349 v 391 (ObjNum 1532 snap 0 seq_num 1532) dirty exists 2023-08-27T20:53:50.998 INFO:tasks.rados.rados.0.smithi093.stdout:2309: left oid 349 (ObjNum 1532 snap 0 seq_num 1532) 2023-08-27T20:53:50.998 INFO:tasks.rados.rados.0.smithi093.stdout:2310: seq_num 1533 ranges {565131=743670,1807484=423873,2965095=458231} 2023-08-27T20:53:51.004 INFO:tasks.rados.rados.0.smithi093.stdout:2310: writing smithi093153278-389 from 565131 to 1308801 tid 1 2023-08-27T20:53:51.009 INFO:tasks.rados.rados.0.smithi093.stdout:2310: writing smithi093153278-389 from 1807484 to 2231357 tid 2 2023-08-27T20:53:51.016 INFO:tasks.rados.rados.0.smithi093.stdout:2310: writing smithi093153278-389 from 2965095 to 3423326 tid 3 2023-08-27T20:53:51.016 INFO:tasks.rados.rados.0.smithi093.stdout:2305: done (3 left) 2023-08-27T20:53:51.016 INFO:tasks.rados.rados.0.smithi093.stdout:2306: done (2 left) 2023-08-27T20:53:51.017 INFO:tasks.rados.rados.0.smithi093.stdout:2309: done (1 left) 2023-08-27T20:53:51.017 INFO:tasks.rados.rados.0.smithi093.stdout:2311: read oid 418 snap -1 2023-08-27T20:53:51.017 INFO:tasks.rados.rados.0.smithi093.stdout:2311: expect (ObjNum 1492 snap 0 seq_num 1492) 2023-08-27T20:53:51.017 INFO:tasks.rados.rados.0.smithi093.stdout:2312: write oid 193 current snap is 0 2023-08-27T20:53:51.017 INFO:tasks.rados.rados.0.smithi093.stdout:2310: finishing write tid 1 to smithi093153278-389 2023-08-27T20:53:51.017 INFO:tasks.rados.rados.0.smithi093.stdout:2310: finishing write tid 2 to smithi093153278-389 2023-08-27T20:53:51.017 INFO:tasks.rados.rados.0.smithi093.stdout:2312: seq_num 1534 ranges {581938=418919,1465966=567531,2592518=669503} 2023-08-27T20:53:51.022 INFO:tasks.rados.rados.0.smithi093.stdout:2312: writing smithi093153278-193 from 581938 to 1000857 tid 1 2023-08-27T20:53:51.028 INFO:tasks.rados.rados.0.smithi093.stdout:2312: writing smithi093153278-193 from 1465966 to 2033497 tid 2 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2312: writing smithi093153278-193 from 2592518 to 3262021 tid 3 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2313: read oid 443 snap -1 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2313: expect (ObjNum 1507 snap 0 seq_num 1507) 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2314: read oid 47 snap -1 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2314: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2315: write oid 441 current snap is 0 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2310: finishing write tid 3 to smithi093153278-389 2023-08-27T20:53:51.035 INFO:tasks.rados.rados.0.smithi093.stdout:2315: seq_num 1535 ranges {598745=494168,1808095=427542,2903588=597128} 2023-08-27T20:53:51.041 INFO:tasks.rados.rados.0.smithi093.stdout:2315: writing smithi093153278-441 from 598745 to 1092913 tid 1 2023-08-27T20:53:51.048 INFO:tasks.rados.rados.0.smithi093.stdout:2315: writing smithi093153278-441 from 1808095 to 2235637 tid 2 2023-08-27T20:53:51.055 INFO:tasks.rados.rados.0.smithi093.stdout:2315: writing smithi093153278-441 from 2903588 to 3500716 tid 3 2023-08-27T20:53:51.055 INFO:tasks.rados.rados.0.smithi093.stdout:2316: delete oid 413 current snap is 0 2023-08-27T20:53:51.055 INFO:tasks.rados.rados.0.smithi093.stdout:2310: finishing write tid 4 to smithi093153278-389 2023-08-27T20:53:51.055 INFO:tasks.rados.rados.0.smithi093.stdout:2310: finishing write tid 5 to smithi093153278-389 2023-08-27T20:53:51.055 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 389 v 342 (ObjNum 1533 snap 0 seq_num 1533) dirty exists 2023-08-27T20:53:51.055 INFO:tasks.rados.rados.0.smithi093.stdout:2310: left oid 389 (ObjNum 1533 snap 0 seq_num 1533) 2023-08-27T20:53:51.055 INFO:tasks.rados.rados.0.smithi093.stdout:2311: expect (ObjNum 1492 snap 0 seq_num 1492) 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2312: finishing write tid 1 to smithi093153278-193 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2312: finishing write tid 2 to smithi093153278-193 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2310: done (6 left) 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2311: done (5 left) 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2316: done (4 left) 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2317: write oid 314 current snap is 0 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2312: finishing write tid 3 to smithi093153278-193 2023-08-27T20:53:51.191 INFO:tasks.rados.rados.0.smithi093.stdout:2313: expect (ObjNum 1507 snap 0 seq_num 1507) 2023-08-27T20:53:51.370 INFO:tasks.rados.rados.0.smithi093.stdout:2312: finishing write tid 4 to smithi093153278-193 2023-08-27T20:53:51.370 INFO:tasks.rados.rados.0.smithi093.stdout:2312: finishing write tid 5 to smithi093153278-193 2023-08-27T20:53:51.370 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 193 v 219 (ObjNum 1534 snap 0 seq_num 1534) dirty exists 2023-08-27T20:53:51.370 INFO:tasks.rados.rados.0.smithi093.stdout:2312: left oid 193 (ObjNum 1534 snap 0 seq_num 1534) 2023-08-27T20:53:51.370 INFO:tasks.rados.rados.0.smithi093.stdout:2314: expect (ObjNum 46 snap 0 seq_num 46) 2023-08-27T20:53:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/849961319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:51.455 INFO:tasks.rados.rados.0.smithi093.stdout:2315: finishing write tid 1 to smithi093153278-441 2023-08-27T20:53:51.456 INFO:tasks.rados.rados.0.smithi093.stdout:2315: finishing write tid 2 to smithi093153278-441 2023-08-27T20:53:51.456 INFO:tasks.rados.rados.0.smithi093.stdout:2315: finishing write tid 3 to smithi093153278-441 2023-08-27T20:53:51.456 INFO:tasks.rados.rados.0.smithi093.stdout:2315: finishing write tid 4 to smithi093153278-441 2023-08-27T20:53:51.456 INFO:tasks.rados.rados.0.smithi093.stdout:2315: finishing write tid 5 to smithi093153278-441 2023-08-27T20:53:51.456 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 289 (ObjNum 1535 snap 0 seq_num 1535) dirty exists 2023-08-27T20:53:51.456 INFO:tasks.rados.rados.0.smithi093.stdout:2315: left oid 441 (ObjNum 1535 snap 0 seq_num 1535) 2023-08-27T20:53:51.456 INFO:tasks.rados.rados.0.smithi093.stdout:2317: seq_num 1537 ranges {632359=761019,2041412=714858,3358434=473925} 2023-08-27T20:53:51.463 INFO:tasks.rados.rados.0.smithi093.stdout:2317: writing smithi093153278-314 from 632359 to 1393378 tid 1 2023-08-27T20:53:51.470 INFO:tasks.rados.rados.0.smithi093.stdout:2317: writing smithi093153278-314 from 2041412 to 2756270 tid 2 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2317: writing smithi093153278-314 from 3358434 to 3832359 tid 3 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2312: done (4 left) 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2313: done (3 left) 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2314: done (2 left) 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2315: done (1 left) 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2318: write oid 266 current snap is 0 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2317: finishing write tid 1 to smithi093153278-314 2023-08-27T20:53:51.477 INFO:tasks.rados.rados.0.smithi093.stdout:2318: seq_num 1538 ranges {649166=436268,1699894=574869,2702210=430944} 2023-08-27T20:53:51.482 INFO:tasks.rados.rados.0.smithi093.stdout:2318: writing smithi093153278-266 from 649166 to 1085434 tid 1 2023-08-27T20:53:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/849961319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/849961319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:51.489 INFO:tasks.rados.rados.0.smithi093.stdout:2318: writing smithi093153278-266 from 1699894 to 2274763 tid 2 2023-08-27T20:53:51.494 INFO:tasks.rados.rados.0.smithi093.stdout:2318: writing smithi093153278-266 from 2702210 to 3133154 tid 3 2023-08-27T20:53:51.494 INFO:tasks.rados.rados.0.smithi093.stdout:2319: write oid 57 current snap is 0 2023-08-27T20:53:51.494 INFO:tasks.rados.rados.0.smithi093.stdout:2317: finishing write tid 2 to smithi093153278-314 2023-08-27T20:53:51.494 INFO:tasks.rados.rados.0.smithi093.stdout:2319: seq_num 1539 ranges {665973=511517,1642023=718527,2896927=642216} 2023-08-27T20:53:51.502 INFO:tasks.rados.rados.0.smithi093.stdout:2319: writing smithi093153278-57 from 665973 to 1177490 tid 1 2023-08-27T20:53:51.506 INFO:tasks.rados.rados.0.smithi093.stdout:2319: writing smithi093153278-57 from 1642023 to 2360550 tid 2 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:2319: writing smithi093153278-57 from 2896927 to 3539143 tid 3 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:2320: read oid 37 snap -1 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:2320: expect (ObjNum 1322 snap 0 seq_num 1322) 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:2321: write oid 397 current snap is 0 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:2317: finishing write tid 3 to smithi093153278-314 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:2317: finishing write tid 4 to smithi093153278-314 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:2317: finishing write tid 5 to smithi093153278-314 2023-08-27T20:53:51.515 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 314 v 395 (ObjNum 1537 snap 0 seq_num 1537) dirty exists 2023-08-27T20:53:51.516 INFO:tasks.rados.rados.0.smithi093.stdout:2317: left oid 314 (ObjNum 1537 snap 0 seq_num 1537) 2023-08-27T20:53:51.516 INFO:tasks.rados.rados.0.smithi093.stdout:2318: finishing write tid 1 to smithi093153278-266 2023-08-27T20:53:51.516 INFO:tasks.rados.rados.0.smithi093.stdout:2318: finishing write tid 2 to smithi093153278-266 2023-08-27T20:53:51.516 INFO:tasks.rados.rados.0.smithi093.stdout:2321: seq_num 1540 ranges {682780=586766,1700505=578538,3040703=569841} 2023-08-27T20:53:51.520 INFO:tasks.rados.rados.0.smithi093.stdout:2321: writing smithi093153278-397 from 682780 to 1269546 tid 1 2023-08-27T20:53:51.526 INFO:tasks.rados.rados.0.smithi093.stdout:2321: writing smithi093153278-397 from 1700505 to 2279043 tid 2 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2321: writing smithi093153278-397 from 3040703 to 3610544 tid 3 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2317: done (4 left) 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2322: read oid 332 snap -1 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2322: expect (ObjNum 1317 snap 0 seq_num 1317) 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2323: write oid 82 current snap is 0 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2318: finishing write tid 3 to smithi093153278-266 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2318: finishing write tid 4 to smithi093153278-266 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:2318: finishing write tid 5 to smithi093153278-266 2023-08-27T20:53:51.534 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 266 v 289 (ObjNum 1538 snap 0 seq_num 1538) dirty exists 2023-08-27T20:53:51.535 INFO:tasks.rados.rados.0.smithi093.stdout:2318: left oid 266 (ObjNum 1538 snap 0 seq_num 1538) 2023-08-27T20:53:51.535 INFO:tasks.rados.rados.0.smithi093.stdout:2319: finishing write tid 1 to smithi093153278-57 2023-08-27T20:53:51.535 INFO:tasks.rados.rados.0.smithi093.stdout:2319: finishing write tid 2 to smithi093153278-57 2023-08-27T20:53:51.535 INFO:tasks.rados.rados.0.smithi093.stdout:2323: seq_num 1541 ranges {699587=662015,2158987=722196,3351773=547814} 2023-08-27T20:53:51.541 INFO:tasks.rados.rados.0.smithi093.stdout:2323: writing smithi093153278-82 from 699587 to 1361602 tid 1 2023-08-27T20:53:51.549 INFO:tasks.rados.rados.0.smithi093.stdout:2323: writing smithi093153278-82 from 2158987 to 2881183 tid 2 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2323: writing smithi093153278-82 from 3351773 to 3899587 tid 3 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2318: done (5 left) 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2324: read oid 248 snap -1 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2324: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2325: delete oid 216 current snap is 0 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2319: finishing write tid 3 to smithi093153278-57 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2319: finishing write tid 4 to smithi093153278-57 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2319: finishing write tid 5 to smithi093153278-57 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 57 v 293 (ObjNum 1539 snap 0 seq_num 1539) dirty exists 2023-08-27T20:53:51.556 INFO:tasks.rados.rados.0.smithi093.stdout:2319: left oid 57 (ObjNum 1539 snap 0 seq_num 1539) 2023-08-27T20:53:51.557 INFO:tasks.rados.rados.0.smithi093.stdout:2320: expect (ObjNum 1322 snap 0 seq_num 1322) 2023-08-27T20:53:51.678 INFO:tasks.rados.rados.0.smithi093.stdout:2321: finishing write tid 1 to smithi093153278-397 2023-08-27T20:53:51.678 INFO:tasks.rados.rados.0.smithi093.stdout:2321: finishing write tid 2 to smithi093153278-397 2023-08-27T20:53:51.679 INFO:tasks.rados.rados.0.smithi093.stdout:2319: done (6 left) 2023-08-27T20:53:51.679 INFO:tasks.rados.rados.0.smithi093.stdout:2320: done (5 left) 2023-08-27T20:53:51.679 INFO:tasks.rados.rados.0.smithi093.stdout:2325: done (4 left) 2023-08-27T20:53:51.679 INFO:tasks.rados.rados.0.smithi093.stdout:2326: write oid 310 current snap is 0 2023-08-27T20:53:51.679 INFO:tasks.rados.rados.0.smithi093.stdout:2322: expect (ObjNum 1317 snap 0 seq_num 1317) 2023-08-27T20:53:51.803 INFO:tasks.rados.rados.0.smithi093.stdout:2321: finishing write tid 3 to smithi093153278-397 2023-08-27T20:53:51.803 INFO:tasks.rados.rados.0.smithi093.stdout:2321: finishing write tid 4 to smithi093153278-397 2023-08-27T20:53:51.803 INFO:tasks.rados.rados.0.smithi093.stdout:2326: seq_num 1543 ranges {733201=412513,1759598=725865,3006619=403657} 2023-08-27T20:53:51.808 INFO:tasks.rados.rados.0.smithi093.stdout:2326: writing smithi093153278-310 from 733201 to 1145714 tid 1 2023-08-27T20:53:51.816 INFO:tasks.rados.rados.0.smithi093.stdout:2326: writing smithi093153278-310 from 1759598 to 2485463 tid 2 2023-08-27T20:53:51.822 INFO:tasks.rados.rados.0.smithi093.stdout:2326: writing smithi093153278-310 from 3006619 to 3410276 tid 3 2023-08-27T20:53:51.822 INFO:tasks.rados.rados.0.smithi093.stdout:2322: done (4 left) 2023-08-27T20:53:51.822 INFO:tasks.rados.rados.0.smithi093.stdout:2327: read oid 474 snap -1 2023-08-27T20:53:51.822 INFO:tasks.rados.rados.0.smithi093.stdout:2327: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:53:51.822 INFO:tasks.rados.rados.0.smithi093.stdout:2328: write oid 10 current snap is 0 2023-08-27T20:53:51.822 INFO:tasks.rados.rados.0.smithi093.stdout:2321: finishing write tid 5 to smithi093153278-397 2023-08-27T20:53:51.823 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 397 v 346 (ObjNum 1540 snap 0 seq_num 1540) dirty exists 2023-08-27T20:53:51.823 INFO:tasks.rados.rados.0.smithi093.stdout:2321: left oid 397 (ObjNum 1540 snap 0 seq_num 1540) 2023-08-27T20:53:51.823 INFO:tasks.rados.rados.0.smithi093.stdout:2323: finishing write tid 1 to smithi093153278-82 2023-08-27T20:53:51.823 INFO:tasks.rados.rados.0.smithi093.stdout:2323: finishing write tid 2 to smithi093153278-82 2023-08-27T20:53:51.823 INFO:tasks.rados.rados.0.smithi093.stdout:2328: seq_num 1544 ranges {750008=604115,1934433=585876,3150395=731282} 2023-08-27T20:53:51.828 INFO:tasks.rados.rados.0.smithi093.stdout:2328: writing smithi093153278-10 from 750008 to 1354123 tid 1 2023-08-27T20:53:51.835 INFO:tasks.rados.rados.0.smithi093.stdout:2328: writing smithi093153278-10 from 1934433 to 2520309 tid 2 2023-08-27T20:53:51.843 INFO:tasks.rados.rados.0.smithi093.stdout:2328: writing smithi093153278-10 from 3150395 to 3881677 tid 3 2023-08-27T20:53:51.843 INFO:tasks.rados.rados.0.smithi093.stdout:2321: done (5 left) 2023-08-27T20:53:51.843 INFO:tasks.rados.rados.0.smithi093.stdout:2329: write oid 18 current snap is 0 2023-08-27T20:53:51.843 INFO:tasks.rados.rados.0.smithi093.stdout:2323: finishing write tid 3 to smithi093153278-82 2023-08-27T20:53:51.843 INFO:tasks.rados.rados.0.smithi093.stdout:2323: finishing write tid 4 to smithi093153278-82 2023-08-27T20:53:51.843 INFO:tasks.rados.rados.0.smithi093.stdout:2329: seq_num 1545 ranges {766815=679364,1992915=729534,3177818=542554} 2023-08-27T20:53:51.850 INFO:tasks.rados.rados.0.smithi093.stdout:2329: writing smithi093153278-18 from 766815 to 1446179 tid 1 2023-08-27T20:53:51.857 INFO:tasks.rados.rados.0.smithi093.stdout:2329: writing smithi093153278-18 from 1992915 to 2722449 tid 2 2023-08-27T20:53:51.863 INFO:tasks.rados.rados.0.smithi093.stdout:2329: writing smithi093153278-18 from 3177818 to 3720372 tid 3 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2330: read oid 44 snap -1 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2330: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2331: read oid 329 snap -1 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2331: expect (ObjNum 1391 snap 0 seq_num 1391) 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2332: write oid 251 current snap is 0 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2323: finishing write tid 5 to smithi093153278-82 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 82 v 324 (ObjNum 1541 snap 0 seq_num 1541) dirty exists 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2323: left oid 82 (ObjNum 1541 snap 0 seq_num 1541) 2023-08-27T20:53:51.864 INFO:tasks.rados.rados.0.smithi093.stdout:2324: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:52.076 INFO:tasks.rados.rados.0.smithi093.stdout:2326: finishing write tid 1 to smithi093153278-310 2023-08-27T20:53:52.076 INFO:tasks.rados.rados.0.smithi093.stdout:2326: finishing write tid 2 to smithi093153278-310 2023-08-27T20:53:52.076 INFO:tasks.rados.rados.0.smithi093.stdout:2326: finishing write tid 3 to smithi093153278-310 2023-08-27T20:53:52.076 INFO:tasks.rados.rados.0.smithi093.stdout:2326: finishing write tid 4 to smithi093153278-310 2023-08-27T20:53:52.076 INFO:tasks.rados.rados.0.smithi093.stdout:2326: finishing write tid 5 to smithi093153278-310 2023-08-27T20:53:52.076 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 310 v 280 (ObjNum 1543 snap 0 seq_num 1543) dirty exists 2023-08-27T20:53:52.077 INFO:tasks.rados.rados.0.smithi093.stdout:2326: left oid 310 (ObjNum 1543 snap 0 seq_num 1543) 2023-08-27T20:53:52.077 INFO:tasks.rados.rados.0.smithi093.stdout:2327: expect (ObjNum 769 snap 0 seq_num 769) 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:2328: finishing write tid 1 to smithi093153278-10 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:2328: finishing write tid 2 to smithi093153278-10 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:2328: finishing write tid 3 to smithi093153278-10 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:2328: finishing write tid 4 to smithi093153278-10 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:2328: finishing write tid 5 to smithi093153278-10 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 10 v 265 (ObjNum 1544 snap 0 seq_num 1544) dirty exists 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:2328: left oid 10 (ObjNum 1544 snap 0 seq_num 1544) 2023-08-27T20:53:52.169 INFO:tasks.rados.rados.0.smithi093.stdout:2332: seq_num 1546 ranges {783622=754613,2051397=589545,3205241=753826} 2023-08-27T20:53:52.177 INFO:tasks.rados.rados.0.smithi093.stdout:2332: writing smithi093153278-251 from 783622 to 1538235 tid 1 2023-08-27T20:53:52.183 INFO:tasks.rados.rados.0.smithi093.stdout:2332: writing smithi093153278-251 from 2051397 to 2640942 tid 2 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2332: writing smithi093153278-251 from 3205241 to 3959067 tid 3 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2323: done (8 left) 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2324: done (7 left) 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2326: done (6 left) 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2327: done (5 left) 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2328: done (4 left) 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2333: read oid 50 snap -1 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2333: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:53:52.191 INFO:tasks.rados.rados.0.smithi093.stdout:2334: read oid 263 snap -1 2023-08-27T20:53:52.192 INFO:tasks.rados.rados.0.smithi093.stdout:2334: expect (ObjNum 1303 snap 0 seq_num 1303) 2023-08-27T20:53:52.192 INFO:tasks.rados.rados.0.smithi093.stdout:2335: write oid 113 current snap is 0 2023-08-27T20:53:52.192 INFO:tasks.rados.rados.0.smithi093.stdout:2329: finishing write tid 1 to smithi093153278-18 2023-08-27T20:53:52.192 INFO:tasks.rados.rados.0.smithi093.stdout:2329: finishing write tid 2 to smithi093153278-18 2023-08-27T20:53:52.192 INFO:tasks.rados.rados.0.smithi093.stdout:2335: seq_num 1547 ranges {400429=429862,1593526=406903} 2023-08-27T20:53:52.195 INFO:tasks.rados.rados.0.smithi093.stdout:2335: writing smithi093153278-113 from 400429 to 830291 tid 1 2023-08-27T20:53:52.202 INFO:tasks.rados.rados.0.smithi093.stdout:2335: writing smithi093153278-113 from 1593526 to 2000429 tid 2 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2336: read oid 472 snap -1 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2336: expect (ObjNum 1451 snap 0 seq_num 1451) 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2337: delete oid 262 current snap is 0 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2329: finishing write tid 3 to smithi093153278-18 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2329: finishing write tid 4 to smithi093153278-18 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2329: finishing write tid 5 to smithi093153278-18 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 18 v 350 (ObjNum 1545 snap 0 seq_num 1545) dirty exists 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2329: left oid 18 (ObjNum 1545 snap 0 seq_num 1545) 2023-08-27T20:53:52.203 INFO:tasks.rados.rados.0.smithi093.stdout:2330: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:53:52.311 INFO:tasks.rados.rados.0.smithi093.stdout:2331: expect (ObjNum 1391 snap 0 seq_num 1391) 2023-08-27T20:53:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:52 smithi136 ceph-mon[113531]: pgmap v9: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.9 MiB/s wr, 91 op/s 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2332: finishing write tid 1 to smithi093153278-251 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2332: finishing write tid 2 to smithi093153278-251 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2332: finishing write tid 3 to smithi093153278-251 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2329: done (8 left) 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2330: done (7 left) 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2331: done (6 left) 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2337: done (5 left) 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2338: read oid 418 snap -1 2023-08-27T20:53:52.451 INFO:tasks.rados.rados.0.smithi093.stdout:2338: expect (ObjNum 1492 snap 0 seq_num 1492) 2023-08-27T20:53:52.452 INFO:tasks.rados.rados.0.smithi093.stdout:2339: write oid 282 current snap is 0 2023-08-27T20:53:52.452 INFO:tasks.rados.rados.0.smithi093.stdout:2339: seq_num 1549 ranges {434043=580360,1710490=323553} 2023-08-27T20:53:52.456 INFO:tasks.rados.rados.0.smithi093.stdout:2339: writing smithi093153278-282 from 434043 to 1014403 tid 1 2023-08-27T20:53:52.462 INFO:tasks.rados.rados.0.smithi093.stdout:2339: writing smithi093153278-282 from 1710490 to 2034043 tid 2 2023-08-27T20:53:52.462 INFO:tasks.rados.rados.0.smithi093.stdout:2340: read oid 40 snap -1 2023-08-27T20:53:52.462 INFO:tasks.rados.rados.0.smithi093.stdout:2340: expect (ObjNum 1414 snap 0 seq_num 1414) 2023-08-27T20:53:52.463 INFO:tasks.rados.rados.0.smithi093.stdout:2341: write oid 103 current snap is 0 2023-08-27T20:53:52.463 INFO:tasks.rados.rados.0.smithi093.stdout:2332: finishing write tid 4 to smithi093153278-251 2023-08-27T20:53:52.463 INFO:tasks.rados.rados.0.smithi093.stdout:2332: finishing write tid 5 to smithi093153278-251 2023-08-27T20:53:52.463 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 251 v 349 (ObjNum 1546 snap 0 seq_num 1546) dirty exists 2023-08-27T20:53:52.463 INFO:tasks.rados.rados.0.smithi093.stdout:2332: left oid 251 (ObjNum 1546 snap 0 seq_num 1546) 2023-08-27T20:53:52.463 INFO:tasks.rados.rados.0.smithi093.stdout:2341: seq_num 1550 ranges {450850=655609,1768972=281878} 2023-08-27T20:53:52.468 INFO:tasks.rados.rados.0.smithi093.stdout:2341: writing smithi093153278-103 from 450850 to 1106459 tid 1 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2341: writing smithi093153278-103 from 1768972 to 2050850 tid 2 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2332: done (8 left) 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2342: read oid 215 snap -1 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2342: expect (ObjNum 1220 snap 0 seq_num 1220) 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2343: read oid 406 snap -1 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2343: expect deleted 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2344: write oid 105 current snap is 0 2023-08-27T20:53:52.474 INFO:tasks.rados.rados.0.smithi093.stdout:2334: expect (ObjNum 1303 snap 0 seq_num 1303) 2023-08-27T20:53:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:52 smithi093 ceph-mon[111335]: pgmap v9: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.9 MiB/s wr, 91 op/s 2023-08-27T20:53:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:52 smithi093 ceph-mon[118902]: pgmap v9: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 9.9 MiB/s wr, 91 op/s 2023-08-27T20:53:52.657 INFO:tasks.rados.rados.0.smithi093.stdout:2335: finishing write tid 1 to smithi093153278-113 2023-08-27T20:53:52.657 INFO:tasks.rados.rados.0.smithi093.stdout:2333: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:53:52.814 INFO:tasks.rados.rados.0.smithi093.stdout:2335: finishing write tid 2 to smithi093153278-113 2023-08-27T20:53:52.814 INFO:tasks.rados.rados.0.smithi093.stdout:2335: finishing write tid 3 to smithi093153278-113 2023-08-27T20:53:52.814 INFO:tasks.rados.rados.0.smithi093.stdout:2335: finishing write tid 4 to smithi093153278-113 2023-08-27T20:53:52.814 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 113 v 268 (ObjNum 1547 snap 0 seq_num 1547) dirty exists 2023-08-27T20:53:52.814 INFO:tasks.rados.rados.0.smithi093.stdout:2335: left oid 113 (ObjNum 1547 snap 0 seq_num 1547) 2023-08-27T20:53:52.814 INFO:tasks.rados.rados.0.smithi093.stdout:2336: expect (ObjNum 1451 snap 0 seq_num 1451) 2023-08-27T20:53:52.890 INFO:tasks.rados.rados.0.smithi093.stdout:2338: expect (ObjNum 1492 snap 0 seq_num 1492) 2023-08-27T20:53:53.021 INFO:tasks.rados.rados.0.smithi093.stdout:2339: finishing write tid 1 to smithi093153278-282 2023-08-27T20:53:53.021 INFO:tasks.rados.rados.0.smithi093.stdout:2339: finishing write tid 2 to smithi093153278-282 2023-08-27T20:53:53.021 INFO:tasks.rados.rados.0.smithi093.stdout:2339: finishing write tid 3 to smithi093153278-282 2023-08-27T20:53:53.021 INFO:tasks.rados.rados.0.smithi093.stdout:2339: finishing write tid 4 to smithi093153278-282 2023-08-27T20:53:53.022 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 282 v 222 (ObjNum 1549 snap 0 seq_num 1549) dirty exists 2023-08-27T20:53:53.022 INFO:tasks.rados.rados.0.smithi093.stdout:2339: left oid 282 (ObjNum 1549 snap 0 seq_num 1549) 2023-08-27T20:53:53.022 INFO:tasks.rados.rados.0.smithi093.stdout:2340: expect (ObjNum 1414 snap 0 seq_num 1414) 2023-08-27T20:53:53.292 INFO:tasks.rados.rados.0.smithi093.stdout:2341: finishing write tid 1 to smithi093153278-103 2023-08-27T20:53:53.292 INFO:tasks.rados.rados.0.smithi093.stdout:2344: seq_num 1551 ranges {467657=447211,1427454=640203} 2023-08-27T20:53:53.297 INFO:tasks.rados.rados.0.smithi093.stdout:2344: writing smithi093153278-105 from 467657 to 914868 tid 1 2023-08-27T20:53:53.304 INFO:tasks.rados.rados.0.smithi093.stdout:2344: writing smithi093153278-105 from 1427454 to 2067657 tid 2 2023-08-27T20:53:53.304 INFO:tasks.rados.rados.0.smithi093.stdout:2333: done (10 left) 2023-08-27T20:53:53.304 INFO:tasks.rados.rados.0.smithi093.stdout:2334: done (9 left) 2023-08-27T20:53:53.304 INFO:tasks.rados.rados.0.smithi093.stdout:2335: done (8 left) 2023-08-27T20:53:53.304 INFO:tasks.rados.rados.0.smithi093.stdout:2336: done (7 left) 2023-08-27T20:53:53.305 INFO:tasks.rados.rados.0.smithi093.stdout:2338: done (6 left) 2023-08-27T20:53:53.305 INFO:tasks.rados.rados.0.smithi093.stdout:2339: done (5 left) 2023-08-27T20:53:53.305 INFO:tasks.rados.rados.0.smithi093.stdout:2340: done (4 left) 2023-08-27T20:53:53.305 INFO:tasks.rados.rados.0.smithi093.stdout:2345: read oid 410 snap -1 2023-08-27T20:53:53.305 INFO:tasks.rados.rados.0.smithi093.stdout:2345: expect deleted 2023-08-27T20:53:53.305 INFO:tasks.rados.rados.0.smithi093.stdout:2346: write oid 6 current snap is 0 2023-08-27T20:53:53.305 INFO:tasks.rados.rados.0.smithi093.stdout:2346: seq_num 1552 ranges {484464=522460,1485936=484199} 2023-08-27T20:53:53.309 INFO:tasks.rados.rados.0.smithi093.stdout:2346: writing smithi093153278-6 from 484464 to 1006924 tid 1 2023-08-27T20:53:53.315 INFO:tasks.rados.rados.0.smithi093.stdout:2346: writing smithi093153278-6 from 1485936 to 1970135 tid 2 2023-08-27T20:53:53.315 INFO:tasks.rados.rados.0.smithi093.stdout:2347: read oid 199 snap -1 2023-08-27T20:53:53.315 INFO:tasks.rados.rados.0.smithi093.stdout:2347: expect (ObjNum 198 snap 0 seq_num 198) 2023-08-27T20:53:53.315 INFO:tasks.rados.rados.0.smithi093.stdout:2348: write oid 217 current snap is 0 2023-08-27T20:53:53.315 INFO:tasks.rados.rados.0.smithi093.stdout:2341: finishing write tid 2 to smithi093153278-103 2023-08-27T20:53:53.315 INFO:tasks.rados.rados.0.smithi093.stdout:2341: finishing write tid 3 to smithi093153278-103 2023-08-27T20:53:53.315 INFO:tasks.rados.rados.0.smithi093.stdout:2348: seq_num 1553 ranges {501271=597709,1544418=556853} 2023-08-27T20:53:53.321 INFO:tasks.rados.rados.0.smithi093.stdout:2348: writing smithi093153278-217 from 501271 to 1098980 tid 1 2023-08-27T20:53:53.327 INFO:tasks.rados.rados.0.smithi093.stdout:2348: writing smithi093153278-217 from 1544418 to 2101271 tid 2 2023-08-27T20:53:53.327 INFO:tasks.rados.rados.0.smithi093.stdout:2343: done (7 left) 2023-08-27T20:53:53.327 INFO:tasks.rados.rados.0.smithi093.stdout:2349: write oid 205 current snap is 0 2023-08-27T20:53:53.327 INFO:tasks.rados.rados.0.smithi093.stdout:2341: finishing write tid 4 to smithi093153278-103 2023-08-27T20:53:53.327 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 103 v 271 (ObjNum 1550 snap 0 seq_num 1550) dirty exists 2023-08-27T20:53:53.328 INFO:tasks.rados.rados.0.smithi093.stdout:2341: left oid 103 (ObjNum 1550 snap 0 seq_num 1550) 2023-08-27T20:53:53.328 INFO:tasks.rados.rados.0.smithi093.stdout:2342: expect (ObjNum 1220 snap 0 seq_num 1220) 2023-08-27T20:53:53.408 INFO:tasks.rados.rados.0.smithi093.stdout:2344: finishing write tid 1 to smithi093153278-105 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2344: finishing write tid 2 to smithi093153278-105 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2344: finishing write tid 3 to smithi093153278-105 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2344: finishing write tid 4 to smithi093153278-105 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 105 v 283 (ObjNum 1551 snap 0 seq_num 1551) dirty exists 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2344: left oid 105 (ObjNum 1551 snap 0 seq_num 1551) 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2346: finishing write tid 1 to smithi093153278-6 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2346: finishing write tid 2 to smithi093153278-6 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2346: finishing write tid 3 to smithi093153278-6 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:2346: finishing write tid 4 to smithi093153278-6 2023-08-27T20:53:53.409 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 6 v 295 (ObjNum 1552 snap 0 seq_num 1552) dirty exists 2023-08-27T20:53:53.410 INFO:tasks.rados.rados.0.smithi093.stdout:2346: left oid 6 (ObjNum 1552 snap 0 seq_num 1552) 2023-08-27T20:53:53.410 INFO:tasks.rados.rados.0.smithi093.stdout:2347: expect (ObjNum 198 snap 0 seq_num 198) 2023-08-27T20:53:53.513 INFO:tasks.rados.rados.0.smithi093.stdout:2348: finishing write tid 1 to smithi093153278-217 2023-08-27T20:53:53.514 INFO:tasks.rados.rados.0.smithi093.stdout:2348: finishing write tid 2 to smithi093153278-217 2023-08-27T20:53:53.514 INFO:tasks.rados.rados.0.smithi093.stdout:2349: seq_num 1554 ranges {518078=672958,1602900=487868} 2023-08-27T20:53:53.520 INFO:tasks.rados.rados.0.smithi093.stdout:2349: writing smithi093153278-205 from 518078 to 1191036 tid 1 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2349: writing smithi093153278-205 from 1602900 to 2090768 tid 2 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2341: done (7 left) 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2342: done (6 left) 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2344: done (5 left) 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2345: done (4 left) 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2346: done (3 left) 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2347: done (2 left) 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2350: read oid 120 snap -1 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2350: expect (ObjNum 1393 snap 0 seq_num 1393) 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2351: read oid 471 snap -1 2023-08-27T20:53:53.526 INFO:tasks.rados.rados.0.smithi093.stdout:2351: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:53:53.527 INFO:tasks.rados.rados.0.smithi093.stdout:2352: write oid 78 current snap is 0 2023-08-27T20:53:53.527 INFO:tasks.rados.rados.0.smithi093.stdout:2348: finishing write tid 3 to smithi093153278-217 2023-08-27T20:53:53.527 INFO:tasks.rados.rados.0.smithi093.stdout:2348: finishing write tid 4 to smithi093153278-217 2023-08-27T20:53:53.527 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 217 v 354 (ObjNum 1553 snap 0 seq_num 1553) dirty exists 2023-08-27T20:53:53.527 INFO:tasks.rados.rados.0.smithi093.stdout:2348: left oid 217 (ObjNum 1553 snap 0 seq_num 1553) 2023-08-27T20:53:53.527 INFO:tasks.rados.rados.0.smithi093.stdout:2349: finishing write tid 1 to smithi093153278-205 2023-08-27T20:53:53.527 INFO:tasks.rados.rados.0.smithi093.stdout:2352: seq_num 1555 ranges {534885=748207,1945029=189856} 2023-08-27T20:53:53.532 INFO:tasks.rados.rados.0.smithi093.stdout:2352: writing smithi093153278-78 from 534885 to 1283092 tid 1 2023-08-27T20:53:53.537 INFO:tasks.rados.rados.0.smithi093.stdout:2352: writing smithi093153278-78 from 1945029 to 2134885 tid 2 2023-08-27T20:53:53.537 INFO:tasks.rados.rados.0.smithi093.stdout:2348: done (4 left) 2023-08-27T20:53:53.537 INFO:tasks.rados.rados.0.smithi093.stdout:2353: write oid 275 current snap is 0 2023-08-27T20:53:53.538 INFO:tasks.rados.rados.0.smithi093.stdout:2353: seq_num 1556 ranges {551692=423456,1603511=491537} 2023-08-27T20:53:53.542 INFO:tasks.rados.rados.0.smithi093.stdout:2353: writing smithi093153278-275 from 551692 to 975148 tid 1 2023-08-27T20:53:53.550 INFO:tasks.rados.rados.0.smithi093.stdout:2353: writing smithi093153278-275 from 1603511 to 2095048 tid 2 2023-08-27T20:53:53.550 INFO:tasks.rados.rados.0.smithi093.stdout:2354: read oid 139 snap -1 2023-08-27T20:53:53.550 INFO:tasks.rados.rados.0.smithi093.stdout:2354: expect (ObjNum 1218 snap 0 seq_num 1218) 2023-08-27T20:53:53.550 INFO:tasks.rados.rados.0.smithi093.stdout:2355: write oid 242 current snap is 0 2023-08-27T20:53:53.551 INFO:tasks.rados.rados.0.smithi093.stdout:2349: finishing write tid 2 to smithi093153278-205 2023-08-27T20:53:53.551 INFO:tasks.rados.rados.0.smithi093.stdout:2349: finishing write tid 3 to smithi093153278-205 2023-08-27T20:53:53.551 INFO:tasks.rados.rados.0.smithi093.stdout:2349: finishing write tid 4 to smithi093153278-205 2023-08-27T20:53:53.551 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 205 v 225 (ObjNum 1554 snap 0 seq_num 1554) dirty exists 2023-08-27T20:53:53.551 INFO:tasks.rados.rados.0.smithi093.stdout:2349: left oid 205 (ObjNum 1554 snap 0 seq_num 1554) 2023-08-27T20:53:53.551 INFO:tasks.rados.rados.0.smithi093.stdout:2350: expect (ObjNum 1393 snap 0 seq_num 1393) 2023-08-27T20:53:53.732 INFO:tasks.rados.rados.0.smithi093.stdout:2352: finishing write tid 1 to smithi093153278-78 2023-08-27T20:53:53.732 INFO:tasks.rados.rados.0.smithi093.stdout:2352: finishing write tid 2 to smithi093153278-78 2023-08-27T20:53:53.732 INFO:tasks.rados.rados.0.smithi093.stdout:2352: finishing write tid 3 to smithi093153278-78 2023-08-27T20:53:53.732 INFO:tasks.rados.rados.0.smithi093.stdout:2352: finishing write tid 4 to smithi093153278-78 2023-08-27T20:53:53.732 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 282 (ObjNum 1555 snap 0 seq_num 1555) dirty exists 2023-08-27T20:53:53.732 INFO:tasks.rados.rados.0.smithi093.stdout:2352: left oid 78 (ObjNum 1555 snap 0 seq_num 1555) 2023-08-27T20:53:53.733 INFO:tasks.rados.rados.0.smithi093.stdout:2351: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:53:53.853 INFO:tasks.rados.rados.0.smithi093.stdout:2353: finishing write tid 1 to smithi093153278-275 2023-08-27T20:53:53.853 INFO:tasks.rados.rados.0.smithi093.stdout:2353: finishing write tid 2 to smithi093153278-275 2023-08-27T20:53:53.853 INFO:tasks.rados.rados.0.smithi093.stdout:2353: finishing write tid 3 to smithi093153278-275 2023-08-27T20:53:53.853 INFO:tasks.rados.rados.0.smithi093.stdout:2353: finishing write tid 4 to smithi093153278-275 2023-08-27T20:53:53.853 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 275 v 286 (ObjNum 1556 snap 0 seq_num 1556) dirty exists 2023-08-27T20:53:53.853 INFO:tasks.rados.rados.0.smithi093.stdout:2353: left oid 275 (ObjNum 1556 snap 0 seq_num 1556) 2023-08-27T20:53:53.853 INFO:tasks.rados.rados.0.smithi093.stdout:2354: expect (ObjNum 1218 snap 0 seq_num 1218) 2023-08-27T20:53:53.960 INFO:tasks.rados.rados.0.smithi093.stdout:2355: seq_num 1557 ranges {568499=498705,1661993=506506} 2023-08-27T20:53:53.966 INFO:tasks.rados.rados.0.smithi093.stdout:2355: writing smithi093153278-242 from 568499 to 1067204 tid 1 2023-08-27T20:53:53.972 INFO:tasks.rados.rados.0.smithi093.stdout:2355: writing smithi093153278-242 from 1661993 to 2168499 tid 2 2023-08-27T20:53:53.972 INFO:tasks.rados.rados.0.smithi093.stdout:2349: done (6 left) 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2350: done (5 left) 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2351: done (4 left) 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2352: done (3 left) 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2353: done (2 left) 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2354: done (1 left) 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2356: read oid 416 snap -1 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2356: expect (ObjNum 1191 snap 0 seq_num 1191) 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2357: write oid 315 current snap is 0 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2355: finishing write tid 1 to smithi093153278-242 2023-08-27T20:53:53.973 INFO:tasks.rados.rados.0.smithi093.stdout:2357: seq_num 1558 ranges {585306=573954,1720475=464831} 2023-08-27T20:53:53.978 INFO:tasks.rados.rados.0.smithi093.stdout:2357: writing smithi093153278-315 from 585306 to 1159260 tid 1 2023-08-27T20:53:53.984 INFO:tasks.rados.rados.0.smithi093.stdout:2357: writing smithi093153278-315 from 1720475 to 2185306 tid 2 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2358: read oid 466 snap -1 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2358: expect (ObjNum 1419 snap 0 seq_num 1419) 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2355: finishing write tid 2 to smithi093153278-242 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2359: read oid 105 snap -1 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2359: expect (ObjNum 1551 snap 0 seq_num 1551) 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2360: read oid 453 snap -1 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2360: expect (ObjNum 1187 snap 0 seq_num 1187) 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2361: read oid 46 snap -1 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2361: expect deleted 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2362: read oid 248 snap -1 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2362: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:53.985 INFO:tasks.rados.rados.0.smithi093.stdout:2363: write oid 69 current snap is 0 2023-08-27T20:53:53.986 INFO:tasks.rados.rados.0.smithi093.stdout:2363: seq_num 1559 ranges {602113=765556,1778957=423156} 2023-08-27T20:53:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:53:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:53:53.514Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:53:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:53:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:53:53.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:53:53.992 INFO:tasks.rados.rados.0.smithi093.stdout:2363: writing smithi093153278-69 from 602113 to 1367669 tid 1 2023-08-27T20:53:53.997 INFO:tasks.rados.rados.0.smithi093.stdout:2363: writing smithi093153278-69 from 1778957 to 2202113 tid 2 2023-08-27T20:53:53.997 INFO:tasks.rados.rados.0.smithi093.stdout:2364: write oid 489 current snap is 0 2023-08-27T20:53:53.997 INFO:tasks.rados.rados.0.smithi093.stdout:2355: finishing write tid 3 to smithi093153278-242 2023-08-27T20:53:53.997 INFO:tasks.rados.rados.0.smithi093.stdout:2355: finishing write tid 4 to smithi093153278-242 2023-08-27T20:53:53.997 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 242 v 296 (ObjNum 1557 snap 0 seq_num 1557) dirty exists 2023-08-27T20:53:53.997 INFO:tasks.rados.rados.0.smithi093.stdout:2355: left oid 242 (ObjNum 1557 snap 0 seq_num 1557) 2023-08-27T20:53:53.997 INFO:tasks.rados.rados.0.smithi093.stdout:2356: expect (ObjNum 1191 snap 0 seq_num 1191) 2023-08-27T20:53:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:2357: finishing write tid 1 to smithi093153278-315 2023-08-27T20:53:54.075 INFO:tasks.rados.rados.0.smithi093.stdout:2359: expect (ObjNum 1551 snap 0 seq_num 1551) 2023-08-27T20:53:54.151 INFO:tasks.rados.rados.0.smithi093.stdout:2357: finishing write tid 2 to smithi093153278-315 2023-08-27T20:53:54.151 INFO:tasks.rados.rados.0.smithi093.stdout:2357: finishing write tid 3 to smithi093153278-315 2023-08-27T20:53:54.152 INFO:tasks.rados.rados.0.smithi093.stdout:2364: seq_num 1560 ranges {618920=440805,1837439=381481} 2023-08-27T20:53:54.157 INFO:tasks.rados.rados.0.smithi093.stdout:2364: writing smithi093153278-489 from 618920 to 1059725 tid 1 2023-08-27T20:53:54.164 INFO:tasks.rados.rados.0.smithi093.stdout:2364: writing smithi093153278-489 from 1837439 to 2218920 tid 2 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2355: done (9 left) 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2356: done (8 left) 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2359: done (7 left) 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2361: done (6 left) 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2365: read oid 252 snap -1 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2365: expect (ObjNum 587 snap 0 seq_num 587) 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2366: read oid 201 snap -1 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2366: expect (ObjNum 1470 snap 0 seq_num 1470) 2023-08-27T20:53:54.165 INFO:tasks.rados.rados.0.smithi093.stdout:2367: read oid 218 snap -1 2023-08-27T20:53:54.166 INFO:tasks.rados.rados.0.smithi093.stdout:2367: expect (ObjNum 1461 snap 0 seq_num 1461) 2023-08-27T20:53:54.166 INFO:tasks.rados.rados.0.smithi093.stdout:2357: finishing write tid 4 to smithi093153278-315 2023-08-27T20:53:54.166 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 326 (ObjNum 1558 snap 0 seq_num 1558) dirty exists 2023-08-27T20:53:54.166 INFO:tasks.rados.rados.0.smithi093.stdout:2357: left oid 315 (ObjNum 1558 snap 0 seq_num 1558) 2023-08-27T20:53:54.166 INFO:tasks.rados.rados.0.smithi093.stdout:2358: expect (ObjNum 1419 snap 0 seq_num 1419) 2023-08-27T20:53:54.384 INFO:tasks.rados.rados.0.smithi093.stdout:2362: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:54 smithi136 ceph-mon[113531]: pgmap v10: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:53:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/425005231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:53:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:54 smithi093 ceph-mon[111335]: pgmap v10: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:53:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/425005231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:53:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:54 smithi093 ceph-mon[118902]: pgmap v10: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:53:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/425005231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:53:54.600 INFO:tasks.rados.rados.0.smithi093.stdout:2360: expect (ObjNum 1187 snap 0 seq_num 1187) 2023-08-27T20:53:54.771 INFO:tasks.rados.rados.0.smithi093.stdout:2363: finishing write tid 1 to smithi093153278-69 2023-08-27T20:53:54.772 INFO:tasks.rados.rados.0.smithi093.stdout:2363: finishing write tid 2 to smithi093153278-69 2023-08-27T20:53:54.772 INFO:tasks.rados.rados.0.smithi093.stdout:2363: finishing write tid 3 to smithi093153278-69 2023-08-27T20:53:54.772 INFO:tasks.rados.rados.0.smithi093.stdout:2363: finishing write tid 4 to smithi093153278-69 2023-08-27T20:53:54.772 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 69 v 352 (ObjNum 1559 snap 0 seq_num 1559) dirty exists 2023-08-27T20:53:54.772 INFO:tasks.rados.rados.0.smithi093.stdout:2363: left oid 69 (ObjNum 1559 snap 0 seq_num 1559) 2023-08-27T20:53:54.772 INFO:tasks.rados.rados.0.smithi093.stdout:2364: finishing write tid 1 to smithi093153278-489 2023-08-27T20:53:54.773 INFO:tasks.rados.rados.0.smithi093.stdout:2367: expect (ObjNum 1461 snap 0 seq_num 1461) 2023-08-27T20:53:54.905 INFO:tasks.rados.rados.0.smithi093.stdout:2364: finishing write tid 2 to smithi093153278-489 2023-08-27T20:53:54.905 INFO:tasks.rados.rados.0.smithi093.stdout:2364: finishing write tid 3 to smithi093153278-489 2023-08-27T20:53:54.905 INFO:tasks.rados.rados.0.smithi093.stdout:2364: finishing write tid 4 to smithi093153278-489 2023-08-27T20:53:54.905 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 489 v 289 (ObjNum 1560 snap 0 seq_num 1560) dirty exists 2023-08-27T20:53:54.905 INFO:tasks.rados.rados.0.smithi093.stdout:2364: left oid 489 (ObjNum 1560 snap 0 seq_num 1560) 2023-08-27T20:53:54.905 INFO:tasks.rados.rados.0.smithi093.stdout:2366: expect (ObjNum 1470 snap 0 seq_num 1470) 2023-08-27T20:53:55.045 INFO:tasks.rados.rados.0.smithi093.stdout:2365: expect (ObjNum 587 snap 0 seq_num 587) 2023-08-27T20:53:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:2368: read oid 128 snap -1 2023-08-27T20:53:55.224 INFO:tasks.rados.rados.0.smithi093.stdout:2368: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:53:55.225 INFO:tasks.rados.rados.0.smithi093.stdout:2357: done (9 left) 2023-08-27T20:53:55.225 INFO:tasks.rados.rados.0.smithi093.stdout:2358: done (8 left) 2023-08-27T20:53:55.225 INFO:tasks.rados.rados.0.smithi093.stdout:2360: done (7 left) 2023-08-27T20:53:55.225 INFO:tasks.rados.rados.0.smithi093.stdout:2362: done (6 left) 2023-08-27T20:53:55.225 INFO:tasks.rados.rados.0.smithi093.stdout:2363: done (5 left) 2023-08-27T20:53:55.225 INFO:tasks.rados.rados.0.smithi093.stdout:2364: done (4 left) 2023-08-27T20:53:55.226 INFO:tasks.rados.rados.0.smithi093.stdout:2365: done (3 left) 2023-08-27T20:53:55.226 INFO:tasks.rados.rados.0.smithi093.stdout:2366: done (2 left) 2023-08-27T20:53:55.226 INFO:tasks.rados.rados.0.smithi093.stdout:2367: done (1 left) 2023-08-27T20:53:55.226 INFO:tasks.rados.rados.0.smithi093.stdout:2369: read oid 86 snap -1 2023-08-27T20:53:55.226 INFO:tasks.rados.rados.0.smithi093.stdout:2369: expect (ObjNum 1126 snap 0 seq_num 1126) 2023-08-27T20:53:55.227 INFO:tasks.rados.rados.0.smithi093.stdout:2370: write oid 168 current snap is 0 2023-08-27T20:53:55.227 INFO:tasks.rados.rados.0.smithi093.stdout:2370: seq_num 1561 ranges {635727=516054,1895921=339806} 2023-08-27T20:53:55.230 INFO:tasks.rados.rados.0.smithi093.stdout:2370: writing smithi093153278-168 from 635727 to 1151781 tid 1 2023-08-27T20:53:55.236 INFO:tasks.rados.rados.0.smithi093.stdout:2370: writing smithi093153278-168 from 1895921 to 2235727 tid 2 2023-08-27T20:53:55.236 INFO:tasks.rados.rados.0.smithi093.stdout:2371: read oid 75 snap -1 2023-08-27T20:53:55.236 INFO:tasks.rados.rados.0.smithi093.stdout:2371: expect deleted 2023-08-27T20:53:55.237 INFO:tasks.rados.rados.0.smithi093.stdout:2372: read oid 252 snap -1 2023-08-27T20:53:55.237 INFO:tasks.rados.rados.0.smithi093.stdout:2372: expect (ObjNum 587 snap 0 seq_num 587) 2023-08-27T20:53:55.237 INFO:tasks.rados.rados.0.smithi093.stdout:2373: write oid 236 current snap is 0 2023-08-27T20:53:55.237 INFO:tasks.rados.rados.0.smithi093.stdout:2373: seq_num 1562 ranges {652534=591303,1954403=298131} 2023-08-27T20:53:55.243 INFO:tasks.rados.rados.0.smithi093.stdout:2373: writing smithi093153278-236 from 652534 to 1243837 tid 1 2023-08-27T20:53:55.249 INFO:tasks.rados.rados.0.smithi093.stdout:2373: writing smithi093153278-236 from 1954403 to 2252534 tid 2 2023-08-27T20:53:55.249 INFO:tasks.rados.rados.0.smithi093.stdout:2374: read oid 403 snap -1 2023-08-27T20:53:55.250 INFO:tasks.rados.rados.0.smithi093.stdout:2374: expect (ObjNum 1285 snap 0 seq_num 1285) 2023-08-27T20:53:55.250 INFO:tasks.rados.rados.0.smithi093.stdout:2375: read oid 408 snap -1 2023-08-27T20:53:55.250 INFO:tasks.rados.rados.0.smithi093.stdout:2375: expect (ObjNum 1355 snap 0 seq_num 1355) 2023-08-27T20:53:55.250 INFO:tasks.rados.rados.0.smithi093.stdout:2376: write oid 439 current snap is 0 2023-08-27T20:53:55.250 INFO:tasks.rados.rados.0.smithi093.stdout:2368: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:53:55.315 INFO:tasks.rados.rados.0.smithi093.stdout:2369: expect (ObjNum 1126 snap 0 seq_num 1126) 2023-08-27T20:53:55.479 INFO:tasks.rados.rados.0.smithi093.stdout:2370: finishing write tid 1 to smithi093153278-168 2023-08-27T20:53:55.479 INFO:tasks.rados.rados.0.smithi093.stdout:2376: seq_num 1563 ranges {669341=666552,2012885=256456} 2023-08-27T20:53:55.486 INFO:tasks.rados.rados.0.smithi093.stdout:2376: writing smithi093153278-439 from 669341 to 1335893 tid 1 2023-08-27T20:53:55.493 INFO:tasks.rados.rados.0.smithi093.stdout:2376: writing smithi093153278-439 from 2012885 to 2269341 tid 2 2023-08-27T20:53:55.493 INFO:tasks.rados.rados.0.smithi093.stdout:2368: done (8 left) 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2369: done (7 left) 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2377: read oid 323 snap -1 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2377: expect (ObjNum 1008 snap 0 seq_num 1008) 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2378: write oid 34 current snap is 0 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2370: finishing write tid 2 to smithi093153278-168 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2370: finishing write tid 3 to smithi093153278-168 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2370: finishing write tid 4 to smithi093153278-168 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 168 v 327 (ObjNum 1561 snap 0 seq_num 1561) dirty exists 2023-08-27T20:53:55.494 INFO:tasks.rados.rados.0.smithi093.stdout:2370: left oid 168 (ObjNum 1561 snap 0 seq_num 1561) 2023-08-27T20:53:55.495 INFO:tasks.rados.rados.0.smithi093.stdout:2372: expect (ObjNum 587 snap 0 seq_num 587) 2023-08-27T20:53:55.679 INFO:tasks.rados.rados.0.smithi093.stdout:2373: finishing write tid 1 to smithi093153278-236 2023-08-27T20:53:55.680 INFO:tasks.rados.rados.0.smithi093.stdout:2373: finishing write tid 2 to smithi093153278-236 2023-08-27T20:53:55.680 INFO:tasks.rados.rados.0.smithi093.stdout:2373: finishing write tid 3 to smithi093153278-236 2023-08-27T20:53:55.680 INFO:tasks.rados.rados.0.smithi093.stdout:2373: finishing write tid 4 to smithi093153278-236 2023-08-27T20:53:55.680 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 236 v 293 (ObjNum 1562 snap 0 seq_num 1562) dirty exists 2023-08-27T20:53:55.680 INFO:tasks.rados.rados.0.smithi093.stdout:2373: left oid 236 (ObjNum 1562 snap 0 seq_num 1562) 2023-08-27T20:53:55.681 INFO:tasks.rados.rados.0.smithi093.stdout:2374: expect (ObjNum 1285 snap 0 seq_num 1285) 2023-08-27T20:53:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:53:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:53:55] "GET /metrics HTTP/1.1" 200 242428 "" "Prometheus/2.43.0" 2023-08-27T20:53:55.890 INFO:tasks.rados.rados.0.smithi093.stdout:2375: expect (ObjNum 1355 snap 0 seq_num 1355) 2023-08-27T20:53:55.976 INFO:tasks.rados.rados.0.smithi093.stdout:2376: finishing write tid 1 to smithi093153278-439 2023-08-27T20:53:55.976 INFO:tasks.rados.rados.0.smithi093.stdout:2376: finishing write tid 2 to smithi093153278-439 2023-08-27T20:53:55.976 INFO:tasks.rados.rados.0.smithi093.stdout:2378: seq_num 1564 ranges {686148=741801,1955014=331134} 2023-08-27T20:53:55.983 INFO:tasks.rados.rados.0.smithi093.stdout:2378: writing smithi093153278-34 from 686148 to 1427949 tid 1 2023-08-27T20:53:55.989 INFO:tasks.rados.rados.0.smithi093.stdout:2378: writing smithi093153278-34 from 1955014 to 2286148 tid 2 2023-08-27T20:53:55.989 INFO:tasks.rados.rados.0.smithi093.stdout:2370: done (8 left) 2023-08-27T20:53:55.989 INFO:tasks.rados.rados.0.smithi093.stdout:2371: done (7 left) 2023-08-27T20:53:55.989 INFO:tasks.rados.rados.0.smithi093.stdout:2372: done (6 left) 2023-08-27T20:53:55.989 INFO:tasks.rados.rados.0.smithi093.stdout:2373: done (5 left) 2023-08-27T20:53:55.989 INFO:tasks.rados.rados.0.smithi093.stdout:2374: done (4 left) 2023-08-27T20:53:55.989 INFO:tasks.rados.rados.0.smithi093.stdout:2375: done (3 left) 2023-08-27T20:53:55.990 INFO:tasks.rados.rados.0.smithi093.stdout:2379: write oid 215 current snap is 0 2023-08-27T20:53:55.990 INFO:tasks.rados.rados.0.smithi093.stdout:2376: finishing write tid 3 to smithi093153278-439 2023-08-27T20:53:55.990 INFO:tasks.rados.rados.0.smithi093.stdout:2376: finishing write tid 4 to smithi093153278-439 2023-08-27T20:53:55.990 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 439 v 296 (ObjNum 1563 snap 0 seq_num 1563) dirty exists 2023-08-27T20:53:55.990 INFO:tasks.rados.rados.0.smithi093.stdout:2376: left oid 439 (ObjNum 1563 snap 0 seq_num 1563) 2023-08-27T20:53:55.990 INFO:tasks.rados.rados.0.smithi093.stdout:2377: expect (ObjNum 1008 snap 0 seq_num 1008) 2023-08-27T20:53:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:2378: finishing write tid 1 to smithi093153278-34 2023-08-27T20:53:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:2378: finishing write tid 2 to smithi093153278-34 2023-08-27T20:53:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:2378: finishing write tid 3 to smithi093153278-34 2023-08-27T20:53:56.149 INFO:tasks.rados.rados.0.smithi093.stdout:2378: finishing write tid 4 to smithi093153278-34 2023-08-27T20:53:56.150 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 34 v 299 (ObjNum 1564 snap 0 seq_num 1564) dirty exists 2023-08-27T20:53:56.150 INFO:tasks.rados.rados.0.smithi093.stdout:2378: left oid 34 (ObjNum 1564 snap 0 seq_num 1564) 2023-08-27T20:53:56.150 INFO:tasks.rados.rados.0.smithi093.stdout:2379: seq_num 1565 ranges {702955=417050,1613496=649871} 2023-08-27T20:53:56.155 INFO:tasks.rados.rados.0.smithi093.stdout:2379: writing smithi093153278-215 from 702955 to 1120005 tid 1 2023-08-27T20:53:56.162 INFO:tasks.rados.rados.0.smithi093.stdout:2379: writing smithi093153278-215 from 1613496 to 2263367 tid 2 2023-08-27T20:53:56.162 INFO:tasks.rados.rados.0.smithi093.stdout:2376: done (3 left) 2023-08-27T20:53:56.162 INFO:tasks.rados.rados.0.smithi093.stdout:2377: done (2 left) 2023-08-27T20:53:56.162 INFO:tasks.rados.rados.0.smithi093.stdout:2378: done (1 left) 2023-08-27T20:53:56.162 INFO:tasks.rados.rados.0.smithi093.stdout:2380: write oid 14 current snap is 0 2023-08-27T20:53:56.162 INFO:tasks.rados.rados.0.smithi093.stdout:2379: finishing write tid 1 to smithi093153278-215 2023-08-27T20:53:56.162 INFO:tasks.rados.rados.0.smithi093.stdout:2380: seq_num 1566 ranges {719762=492299,1671978=509882} 2023-08-27T20:53:56.168 INFO:tasks.rados.rados.0.smithi093.stdout:2380: writing smithi093153278-14 from 719762 to 1212061 tid 1 2023-08-27T20:53:56.174 INFO:tasks.rados.rados.0.smithi093.stdout:2380: writing smithi093153278-14 from 1671978 to 2181860 tid 2 2023-08-27T20:53:56.174 INFO:tasks.rados.rados.0.smithi093.stdout:2381: read oid 132 snap -1 2023-08-27T20:53:56.174 INFO:tasks.rados.rados.0.smithi093.stdout:2381: expect (ObjNum 1052 snap 0 seq_num 1052) 2023-08-27T20:53:56.174 INFO:tasks.rados.rados.0.smithi093.stdout:2382: delete oid 369 current snap is 0 2023-08-27T20:53:56.174 INFO:tasks.rados.rados.0.smithi093.stdout:2379: finishing write tid 2 to smithi093153278-215 2023-08-27T20:53:56.175 INFO:tasks.rados.rados.0.smithi093.stdout:2379: finishing write tid 3 to smithi093153278-215 2023-08-27T20:53:56.175 INFO:tasks.rados.rados.0.smithi093.stdout:2379: finishing write tid 4 to smithi093153278-215 2023-08-27T20:53:56.175 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 215 v 302 (ObjNum 1565 snap 0 seq_num 1565) dirty exists 2023-08-27T20:53:56.175 INFO:tasks.rados.rados.0.smithi093.stdout:2379: left oid 215 (ObjNum 1565 snap 0 seq_num 1565) 2023-08-27T20:53:56.175 INFO:tasks.rados.rados.0.smithi093.stdout:2380: finishing write tid 1 to smithi093153278-14 2023-08-27T20:53:56.176 INFO:tasks.rados.rados.0.smithi093.stdout:2379: done (3 left) 2023-08-27T20:53:56.176 INFO:tasks.rados.rados.0.smithi093.stdout:2382: done (2 left) 2023-08-27T20:53:56.176 INFO:tasks.rados.rados.0.smithi093.stdout:2383: read oid 371 snap -1 2023-08-27T20:53:56.176 INFO:tasks.rados.rados.0.smithi093.stdout:2383: expect (ObjNum 1396 snap 0 seq_num 1396) 2023-08-27T20:53:56.176 INFO:tasks.rados.rados.0.smithi093.stdout:2384: read oid 389 snap -1 2023-08-27T20:53:56.176 INFO:tasks.rados.rados.0.smithi093.stdout:2384: expect (ObjNum 1533 snap 0 seq_num 1533) 2023-08-27T20:53:56.176 INFO:tasks.rados.rados.0.smithi093.stdout:2385: delete oid 115 current snap is 0 2023-08-27T20:53:56.177 INFO:tasks.rados.rados.0.smithi093.stdout:2385: done (4 left) 2023-08-27T20:53:56.177 INFO:tasks.rados.rados.0.smithi093.stdout:2386: write oid 183 current snap is 0 2023-08-27T20:53:56.177 INFO:tasks.rados.rados.0.smithi093.stdout:2386: seq_num 1569 ranges {770183=434399,1847424=522759} 2023-08-27T20:53:56.185 INFO:tasks.rados.rados.0.smithi093.stdout:2386: writing smithi093153278-183 from 770183 to 1204582 tid 1 2023-08-27T20:53:56.192 INFO:tasks.rados.rados.0.smithi093.stdout:2386: writing smithi093153278-183 from 1847424 to 2370183 tid 2 2023-08-27T20:53:56.192 INFO:tasks.rados.rados.0.smithi093.stdout:2387: write oid 483 current snap is 0 2023-08-27T20:53:56.192 INFO:tasks.rados.rados.0.smithi093.stdout:2387: seq_num 1570 ranges {786990=509648,1905906=481084} 2023-08-27T20:53:56.199 INFO:tasks.rados.rados.0.smithi093.stdout:2387: writing smithi093153278-483 from 786990 to 1296638 tid 1 2023-08-27T20:53:56.205 INFO:tasks.rados.rados.0.smithi093.stdout:2387: writing smithi093153278-483 from 1905906 to 2386990 tid 2 2023-08-27T20:53:56.205 INFO:tasks.rados.rados.0.smithi093.stdout:2388: write oid 233 current snap is 0 2023-08-27T20:53:56.205 INFO:tasks.rados.rados.0.smithi093.stdout:2388: seq_num 1571 ranges {403797=584897,1564388=660878} 2023-08-27T20:53:56.210 INFO:tasks.rados.rados.0.smithi093.stdout:2388: writing smithi093153278-233 from 403797 to 988694 tid 1 2023-08-27T20:53:56.217 INFO:tasks.rados.rados.0.smithi093.stdout:2388: writing smithi093153278-233 from 1564388 to 2225266 tid 2 2023-08-27T20:53:56.217 INFO:tasks.rados.rados.0.smithi093.stdout:2389: read oid 246 snap -1 2023-08-27T20:53:56.217 INFO:tasks.rados.rados.0.smithi093.stdout:2389: expect deleted 2023-08-27T20:53:56.218 INFO:tasks.rados.rados.0.smithi093.stdout:2390: read oid 497 snap -1 2023-08-27T20:53:56.218 INFO:tasks.rados.rados.0.smithi093.stdout:2390: expect (ObjNum 1222 snap 0 seq_num 1222) 2023-08-27T20:53:56.218 INFO:tasks.rados.rados.0.smithi093.stdout:2391: write oid 20 current snap is 0 2023-08-27T20:53:56.218 INFO:tasks.rados.rados.0.smithi093.stdout:2391: seq_num 1572 ranges {420604=660146,1506517=404536} 2023-08-27T20:53:56.224 INFO:tasks.rados.rados.0.smithi093.stdout:2391: writing smithi093153278-20 from 420604 to 1080750 tid 1 2023-08-27T20:53:56.229 INFO:tasks.rados.rados.0.smithi093.stdout:2391: writing smithi093153278-20 from 1506517 to 1911053 tid 2 2023-08-27T20:53:56.229 INFO:tasks.rados.rados.0.smithi093.stdout:2392: read oid 191 snap -1 2023-08-27T20:53:56.229 INFO:tasks.rados.rados.0.smithi093.stdout:2392: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:53:56.229 INFO:tasks.rados.rados.0.smithi093.stdout:2393: write oid 357 current snap is 0 2023-08-27T20:53:56.229 INFO:tasks.rados.rados.0.smithi093.stdout:2380: finishing write tid 2 to smithi093153278-14 2023-08-27T20:53:56.230 INFO:tasks.rados.rados.0.smithi093.stdout:2380: finishing write tid 3 to smithi093153278-14 2023-08-27T20:53:56.230 INFO:tasks.rados.rados.0.smithi093.stdout:2393: seq_num 1573 ranges {437411=735395,1964999=472412} 2023-08-27T20:53:56.235 INFO:tasks.rados.rados.0.smithi093.stdout:2393: writing smithi093153278-357 from 437411 to 1172806 tid 1 2023-08-27T20:53:56.243 INFO:tasks.rados.rados.0.smithi093.stdout:2393: writing smithi093153278-357 from 1964999 to 2437411 tid 2 2023-08-27T20:53:56.243 INFO:tasks.rados.rados.0.smithi093.stdout:2394: write oid 429 current snap is 0 2023-08-27T20:53:56.243 INFO:tasks.rados.rados.0.smithi093.stdout:2380: finishing write tid 4 to smithi093153278-14 2023-08-27T20:53:56.243 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 14 v 292 (ObjNum 1566 snap 0 seq_num 1566) dirty exists 2023-08-27T20:53:56.243 INFO:tasks.rados.rados.0.smithi093.stdout:2380: left oid 14 (ObjNum 1566 snap 0 seq_num 1566) 2023-08-27T20:53:56.243 INFO:tasks.rados.rados.0.smithi093.stdout:2384: expect (ObjNum 1533 snap 0 seq_num 1533) 2023-08-27T20:53:56.360 INFO:tasks.rados.rados.0.smithi093.stdout:2381: expect (ObjNum 1052 snap 0 seq_num 1052) 2023-08-27T20:53:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:56 smithi136 ceph-mon[113531]: pgmap v11: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:53:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2039068971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:56.476 INFO:tasks.rados.rados.0.smithi093.stdout:2383: expect (ObjNum 1396 snap 0 seq_num 1396) 2023-08-27T20:53:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:56 smithi093 ceph-mon[111335]: pgmap v11: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:53:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2039068971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:56 smithi093 ceph-mon[118902]: pgmap v11: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:53:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2039068971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:56.615 INFO:tasks.rados.rados.0.smithi093.stdout:2386: finishing write tid 1 to smithi093153278-183 2023-08-27T20:53:56.615 INFO:tasks.rados.rados.0.smithi093.stdout:2386: finishing write tid 2 to smithi093153278-183 2023-08-27T20:53:56.615 INFO:tasks.rados.rados.0.smithi093.stdout:2386: finishing write tid 3 to smithi093153278-183 2023-08-27T20:53:56.616 INFO:tasks.rados.rados.0.smithi093.stdout:2386: finishing write tid 4 to smithi093153278-183 2023-08-27T20:53:56.616 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 183 v 298 (ObjNum 1569 snap 0 seq_num 1569) dirty exists 2023-08-27T20:53:56.616 INFO:tasks.rados.rados.0.smithi093.stdout:2386: left oid 183 (ObjNum 1569 snap 0 seq_num 1569) 2023-08-27T20:53:56.616 INFO:tasks.rados.rados.0.smithi093.stdout:2387: finishing write tid 1 to smithi093153278-483 2023-08-27T20:53:56.616 INFO:tasks.rados.rados.0.smithi093.stdout:2394: seq_num 1574 ranges {454218=526997,1739834=408205} 2023-08-27T20:53:56.620 INFO:tasks.rados.rados.0.smithi093.stdout:2394: writing smithi093153278-429 from 454218 to 981215 tid 1 2023-08-27T20:53:56.627 INFO:tasks.rados.rados.0.smithi093.stdout:2394: writing smithi093153278-429 from 1739834 to 2148039 tid 2 2023-08-27T20:53:56.627 INFO:tasks.rados.rados.0.smithi093.stdout:2380: done (12 left) 2023-08-27T20:53:56.627 INFO:tasks.rados.rados.0.smithi093.stdout:2381: done (11 left) 2023-08-27T20:53:56.627 INFO:tasks.rados.rados.0.smithi093.stdout:2383: done (10 left) 2023-08-27T20:53:56.628 INFO:tasks.rados.rados.0.smithi093.stdout:2384: done (9 left) 2023-08-27T20:53:56.628 INFO:tasks.rados.rados.0.smithi093.stdout:2386: done (8 left) 2023-08-27T20:53:56.628 INFO:tasks.rados.rados.0.smithi093.stdout:2395: write oid 41 current snap is 0 2023-08-27T20:53:56.628 INFO:tasks.rados.rados.0.smithi093.stdout:2387: finishing write tid 2 to smithi093153278-483 2023-08-27T20:53:56.628 INFO:tasks.rados.rados.0.smithi093.stdout:2387: finishing write tid 3 to smithi093153278-483 2023-08-27T20:53:56.628 INFO:tasks.rados.rados.0.smithi093.stdout:2395: seq_num 1575 ranges {471025=602246,1798316=668216} 2023-08-27T20:53:56.632 INFO:tasks.rados.rados.0.smithi093.stdout:2395: writing smithi093153278-41 from 471025 to 1073271 tid 1 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2395: writing smithi093153278-41 from 1798316 to 2466532 tid 2 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2396: read oid 234 snap -1 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2396: expect (ObjNum 1202 snap 0 seq_num 1202) 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2397: read oid 307 snap -1 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2397: expect deleted 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2398: read oid 228 snap -1 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2398: expect (ObjNum 973 snap 0 seq_num 973) 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2399: read oid 181 snap -1 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2399: expect (ObjNum 180 snap 0 seq_num 180) 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2400: write oid 15 current snap is 0 2023-08-27T20:53:56.641 INFO:tasks.rados.rados.0.smithi093.stdout:2387: finishing write tid 4 to smithi093153278-483 2023-08-27T20:53:56.642 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 483 v 279 (ObjNum 1570 snap 0 seq_num 1570) dirty exists 2023-08-27T20:53:56.642 INFO:tasks.rados.rados.0.smithi093.stdout:2387: left oid 483 (ObjNum 1570 snap 0 seq_num 1570) 2023-08-27T20:53:56.642 INFO:tasks.rados.rados.0.smithi093.stdout:2388: finishing write tid 1 to smithi093153278-233 2023-08-27T20:53:56.642 INFO:tasks.rados.rados.0.smithi093.stdout:2400: seq_num 1576 ranges {487832=677495,1740445=411874} 2023-08-27T20:53:56.647 INFO:tasks.rados.rados.0.smithi093.stdout:2400: writing smithi093153278-15 from 487832 to 1165327 tid 1 2023-08-27T20:53:56.654 INFO:tasks.rados.rados.0.smithi093.stdout:2400: writing smithi093153278-15 from 1740445 to 2152319 tid 2 2023-08-27T20:53:56.654 INFO:tasks.rados.rados.0.smithi093.stdout:2387: done (13 left) 2023-08-27T20:53:56.654 INFO:tasks.rados.rados.0.smithi093.stdout:2401: write oid 14 current snap is 0 2023-08-27T20:53:56.654 INFO:tasks.rados.rados.0.smithi093.stdout:2401: seq_num 1577 ranges {504639=752744,1798927=671885} 2023-08-27T20:53:56.661 INFO:tasks.rados.rados.0.smithi093.stdout:2401: writing smithi093153278-14 from 504639 to 1257383 tid 1 2023-08-27T20:53:56.668 INFO:tasks.rados.rados.0.smithi093.stdout:2401: writing smithi093153278-14 from 1798927 to 2470812 tid 2 2023-08-27T20:53:56.668 INFO:tasks.rados.rados.0.smithi093.stdout:2402: read oid 431 snap -1 2023-08-27T20:53:56.668 INFO:tasks.rados.rados.0.smithi093.stdout:2402: expect deleted 2023-08-27T20:53:56.668 INFO:tasks.rados.rados.0.smithi093.stdout:2403: write oid 125 current snap is 0 2023-08-27T20:53:56.668 INFO:tasks.rados.rados.0.smithi093.stdout:2388: finishing write tid 2 to smithi093153278-233 2023-08-27T20:53:56.668 INFO:tasks.rados.rados.0.smithi093.stdout:2388: finishing write tid 3 to smithi093153278-233 2023-08-27T20:53:56.668 INFO:tasks.rados.rados.0.smithi093.stdout:2388: finishing write tid 4 to smithi093153278-233 2023-08-27T20:53:56.669 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 233 v 329 (ObjNum 1571 snap 0 seq_num 1571) dirty exists 2023-08-27T20:53:56.669 INFO:tasks.rados.rados.0.smithi093.stdout:2388: left oid 233 (ObjNum 1571 snap 0 seq_num 1571) 2023-08-27T20:53:56.669 INFO:tasks.rados.rados.0.smithi093.stdout:2390: expect (ObjNum 1222 snap 0 seq_num 1222) 2023-08-27T20:53:56.781 INFO:tasks.rados.rados.0.smithi093.stdout:2391: finishing write tid 1 to smithi093153278-20 2023-08-27T20:53:56.781 INFO:tasks.rados.rados.0.smithi093.stdout:2392: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:53:56.895 INFO:tasks.rados.rados.0.smithi093.stdout:2391: finishing write tid 2 to smithi093153278-20 2023-08-27T20:53:56.895 INFO:tasks.rados.rados.0.smithi093.stdout:2391: finishing write tid 3 to smithi093153278-20 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2391: finishing write tid 4 to smithi093153278-20 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 331 (ObjNum 1572 snap 0 seq_num 1572) dirty exists 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2391: left oid 20 (ObjNum 1572 snap 0 seq_num 1572) 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2393: finishing write tid 1 to smithi093153278-357 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2393: finishing write tid 2 to smithi093153278-357 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2393: finishing write tid 3 to smithi093153278-357 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2393: finishing write tid 4 to smithi093153278-357 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 357 v 275 (ObjNum 1573 snap 0 seq_num 1573) dirty exists 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2393: left oid 357 (ObjNum 1573 snap 0 seq_num 1573) 2023-08-27T20:53:56.896 INFO:tasks.rados.rados.0.smithi093.stdout:2403: seq_num 1578 ranges {521446=427993,1457409=415543,2301012=220434} 2023-08-27T20:53:56.900 INFO:tasks.rados.rados.0.smithi093.stdout:2403: writing smithi093153278-125 from 521446 to 949439 tid 1 2023-08-27T20:53:56.905 INFO:tasks.rados.rados.0.smithi093.stdout:2403: writing smithi093153278-125 from 1457409 to 1872952 tid 2 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2403: writing smithi093153278-125 from 2301012 to 2521446 tid 3 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2388: done (15 left) 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2389: done (14 left) 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2390: done (13 left) 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2391: done (12 left) 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2392: done (11 left) 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2393: done (10 left) 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2404: read oid 216 snap -1 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2404: expect deleted 2023-08-27T20:53:56.910 INFO:tasks.rados.rados.0.smithi093.stdout:2405: write oid 250 current snap is 0 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:2394: finishing write tid 1 to smithi093153278-429 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:2394: finishing write tid 2 to smithi093153278-429 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:2394: finishing write tid 3 to smithi093153278-429 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:2394: finishing write tid 4 to smithi093153278-429 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 429 v 355 (ObjNum 1574 snap 0 seq_num 1574) dirty exists 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:2394: left oid 429 (ObjNum 1574 snap 0 seq_num 1574) 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:2395: finishing write tid 1 to smithi093153278-41 2023-08-27T20:53:56.911 INFO:tasks.rados.rados.0.smithi093.stdout:2405: seq_num 1579 ranges {538253=503242,1515891=675554} 2023-08-27T20:53:56.914 INFO:tasks.rados.rados.0.smithi093.stdout:2405: writing smithi093153278-250 from 538253 to 1041495 tid 1 2023-08-27T20:53:56.921 INFO:tasks.rados.rados.0.smithi093.stdout:2405: writing smithi093153278-250 from 1515891 to 2191445 tid 2 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2394: done (11 left) 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2406: read oid 195 snap -1 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2406: expect (ObjNum 1434 snap 0 seq_num 1434) 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2407: read oid 424 snap -1 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2407: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2408: write oid 134 current snap is 0 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2395: finishing write tid 2 to smithi093153278-41 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2395: finishing write tid 3 to smithi093153278-41 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2395: finishing write tid 4 to smithi093153278-41 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 41 v 423 (ObjNum 1575 snap 0 seq_num 1575) dirty exists 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2395: left oid 41 (ObjNum 1575 snap 0 seq_num 1575) 2023-08-27T20:53:56.922 INFO:tasks.rados.rados.0.smithi093.stdout:2396: expect (ObjNum 1202 snap 0 seq_num 1202) 2023-08-27T20:53:57.026 INFO:tasks.rados.rados.0.smithi093.stdout:2400: finishing write tid 1 to smithi093153278-15 2023-08-27T20:53:57.026 INFO:tasks.rados.rados.0.smithi093.stdout:2398: expect (ObjNum 973 snap 0 seq_num 973) 2023-08-27T20:53:57.122 INFO:tasks.rados.rados.0.smithi093.stdout:2399: expect (ObjNum 180 snap 0 seq_num 180) 2023-08-27T20:53:57.217 INFO:tasks.rados.rados.0.smithi093.stdout:2400: finishing write tid 2 to smithi093153278-15 2023-08-27T20:53:57.217 INFO:tasks.rados.rados.0.smithi093.stdout:2400: finishing write tid 3 to smithi093153278-15 2023-08-27T20:53:57.217 INFO:tasks.rados.rados.0.smithi093.stdout:2400: finishing write tid 4 to smithi093153278-15 2023-08-27T20:53:57.217 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 15 v 357 (ObjNum 1576 snap 0 seq_num 1576) dirty exists 2023-08-27T20:53:57.218 INFO:tasks.rados.rados.0.smithi093.stdout:2400: left oid 15 (ObjNum 1576 snap 0 seq_num 1576) 2023-08-27T20:53:57.218 INFO:tasks.rados.rados.0.smithi093.stdout:2401: finishing write tid 1 to smithi093153278-14 2023-08-27T20:53:57.218 INFO:tasks.rados.rados.0.smithi093.stdout:2408: seq_num 1580 ranges {555060=578491,1858020=419212} 2023-08-27T20:53:57.222 INFO:tasks.rados.rados.0.smithi093.stdout:2408: writing smithi093153278-134 from 555060 to 1133551 tid 1 2023-08-27T20:53:57.230 INFO:tasks.rados.rados.0.smithi093.stdout:2408: writing smithi093153278-134 from 1858020 to 2277232 tid 2 2023-08-27T20:53:57.230 INFO:tasks.rados.rados.0.smithi093.stdout:2395: done (13 left) 2023-08-27T20:53:57.230 INFO:tasks.rados.rados.0.smithi093.stdout:2396: done (12 left) 2023-08-27T20:53:57.231 INFO:tasks.rados.rados.0.smithi093.stdout:2397: done (11 left) 2023-08-27T20:53:57.231 INFO:tasks.rados.rados.0.smithi093.stdout:2398: done (10 left) 2023-08-27T20:53:57.231 INFO:tasks.rados.rados.0.smithi093.stdout:2399: done (9 left) 2023-08-27T20:53:57.231 INFO:tasks.rados.rados.0.smithi093.stdout:2400: done (8 left) 2023-08-27T20:53:57.231 INFO:tasks.rados.rados.0.smithi093.stdout:2409: delete oid 24 current snap is 0 2023-08-27T20:53:57.232 INFO:tasks.rados.rados.0.smithi093.stdout:2401: finishing write tid 2 to smithi093153278-14 2023-08-27T20:53:57.232 INFO:tasks.rados.rados.0.smithi093.stdout:2401: finishing write tid 3 to smithi093153278-14 2023-08-27T20:53:57.232 INFO:tasks.rados.rados.0.smithi093.stdout:2401: finishing write tid 4 to smithi093153278-14 2023-08-27T20:53:57.232 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 14 v 295 (ObjNum 1577 snap 0 seq_num 1577) dirty exists 2023-08-27T20:53:57.232 INFO:tasks.rados.rados.0.smithi093.stdout:2401: left oid 14 (ObjNum 1577 snap 0 seq_num 1577) 2023-08-27T20:53:57.232 INFO:tasks.rados.rados.0.smithi093.stdout:2403: finishing write tid 1 to smithi093153278-125 2023-08-27T20:53:57.233 INFO:tasks.rados.rados.0.smithi093.stdout:2403: finishing write tid 2 to smithi093153278-125 2023-08-27T20:53:57.233 INFO:tasks.rados.rados.0.smithi093.stdout:2403: finishing write tid 3 to smithi093153278-125 2023-08-27T20:53:57.233 INFO:tasks.rados.rados.0.smithi093.stdout:2403: finishing write tid 4 to smithi093153278-125 2023-08-27T20:53:57.233 INFO:tasks.rados.rados.0.smithi093.stdout:2403: finishing write tid 5 to smithi093153278-125 2023-08-27T20:53:57.233 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 125 v 333 (ObjNum 1578 snap 0 seq_num 1578) dirty exists 2023-08-27T20:53:57.234 INFO:tasks.rados.rados.0.smithi093.stdout:2403: left oid 125 (ObjNum 1578 snap 0 seq_num 1578) 2023-08-27T20:53:57.234 INFO:tasks.rados.rados.0.smithi093.stdout:2405: finishing write tid 1 to smithi093153278-250 2023-08-27T20:53:57.234 INFO:tasks.rados.rados.0.smithi093.stdout:2405: finishing write tid 2 to smithi093153278-250 2023-08-27T20:53:57.234 INFO:tasks.rados.rados.0.smithi093.stdout:2405: finishing write tid 3 to smithi093153278-250 2023-08-27T20:53:57.234 INFO:tasks.rados.rados.0.smithi093.stdout:2405: finishing write tid 4 to smithi093153278-250 2023-08-27T20:53:57.234 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 250 v 426 (ObjNum 1579 snap 0 seq_num 1579) dirty exists 2023-08-27T20:53:57.235 INFO:tasks.rados.rados.0.smithi093.stdout:2405: left oid 250 (ObjNum 1579 snap 0 seq_num 1579) 2023-08-27T20:53:57.235 INFO:tasks.rados.rados.0.smithi093.stdout:2407: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:53:57.403 INFO:tasks.rados.rados.0.smithi093.stdout:2406: expect (ObjNum 1434 snap 0 seq_num 1434) 2023-08-27T20:53:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:57 smithi136 ceph-mon[113531]: Failed to find standby mgr for failover. Retrying in 15 seconds 2023-08-27T20:53:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:57 smithi093 ceph-mon[111335]: Failed to find standby mgr for failover. Retrying in 15 seconds 2023-08-27T20:53:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:57 smithi093 ceph-mon[118902]: Failed to find standby mgr for failover. Retrying in 15 seconds 2023-08-27T20:53:57.581 INFO:tasks.rados.rados.0.smithi093.stdout:2408: finishing write tid 1 to smithi093153278-134 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2408: finishing write tid 2 to smithi093153278-134 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2408: finishing write tid 3 to smithi093153278-134 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2408: finishing write tid 4 to smithi093153278-134 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 134 v 305 (ObjNum 1580 snap 0 seq_num 1580) dirty exists 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2408: left oid 134 (ObjNum 1580 snap 0 seq_num 1580) 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2401: done (8 left) 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2402: done (7 left) 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2403: done (6 left) 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2404: done (5 left) 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2405: done (4 left) 2023-08-27T20:53:57.582 INFO:tasks.rados.rados.0.smithi093.stdout:2406: done (3 left) 2023-08-27T20:53:57.583 INFO:tasks.rados.rados.0.smithi093.stdout:2407: done (2 left) 2023-08-27T20:53:57.583 INFO:tasks.rados.rados.0.smithi093.stdout:2408: done (1 left) 2023-08-27T20:53:57.583 INFO:tasks.rados.rados.0.smithi093.stdout:2409: done (0 left) 2023-08-27T20:53:57.583 INFO:tasks.rados.rados.0.smithi093.stdout:2410: write oid 118 current snap is 0 2023-08-27T20:53:57.583 INFO:tasks.rados.rados.0.smithi093.stdout:2410: seq_num 1582 ranges {588674=445342,1691337=422881} 2023-08-27T20:53:57.586 INFO:tasks.rados.rados.0.smithi093.stdout:2410: writing smithi093153278-118 from 588674 to 1034016 tid 1 2023-08-27T20:53:57.593 INFO:tasks.rados.rados.0.smithi093.stdout:2410: writing smithi093153278-118 from 1691337 to 2114218 tid 2 2023-08-27T20:53:57.593 INFO:tasks.rados.rados.0.smithi093.stdout:2411: write oid 300 current snap is 0 2023-08-27T20:53:57.593 INFO:tasks.rados.rados.0.smithi093.stdout:2410: finishing write tid 1 to smithi093153278-118 2023-08-27T20:53:57.593 INFO:tasks.rados.rados.0.smithi093.stdout:2411: seq_num 1583 ranges {605481=520591,1749819=682892} 2023-08-27T20:53:57.598 INFO:tasks.rados.rados.0.smithi093.stdout:2411: writing smithi093153278-300 from 605481 to 1126072 tid 1 2023-08-27T20:53:57.606 INFO:tasks.rados.rados.0.smithi093.stdout:2411: writing smithi093153278-300 from 1749819 to 2432711 tid 2 2023-08-27T20:53:57.606 INFO:tasks.rados.rados.0.smithi093.stdout:2412: read oid 464 snap -1 2023-08-27T20:53:57.606 INFO:tasks.rados.rados.0.smithi093.stdout:2412: expect (ObjNum 1343 snap 0 seq_num 1343) 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2413: read oid 367 snap -1 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2413: expect (ObjNum 719 snap 0 seq_num 719) 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2414: read oid 217 snap -1 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2414: expect (ObjNum 1553 snap 0 seq_num 1553) 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2415: delete oid 66 current snap is 0 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2410: finishing write tid 2 to smithi093153278-118 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2410: finishing write tid 3 to smithi093153278-118 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2410: finishing write tid 4 to smithi093153278-118 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 118 v 298 (ObjNum 1582 snap 0 seq_num 1582) dirty exists 2023-08-27T20:53:57.607 INFO:tasks.rados.rados.0.smithi093.stdout:2410: left oid 118 (ObjNum 1582 snap 0 seq_num 1582) 2023-08-27T20:53:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:2411: finishing write tid 1 to smithi093153278-300 2023-08-27T20:53:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:2410: done (5 left) 2023-08-27T20:53:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:2415: done (4 left) 2023-08-27T20:53:57.608 INFO:tasks.rados.rados.0.smithi093.stdout:2416: delete oid 92 current snap is 0 2023-08-27T20:53:57.609 INFO:tasks.rados.rados.0.smithi093.stdout:2414: expect (ObjNum 1553 snap 0 seq_num 1553) 2023-08-27T20:53:57.672 INFO:tasks.rados.rados.0.smithi093.stdout:2414: done (4 left) 2023-08-27T20:53:57.673 INFO:tasks.rados.rados.0.smithi093.stdout:2416: done (3 left) 2023-08-27T20:53:57.673 INFO:tasks.rados.rados.0.smithi093.stdout:2417: read oid 369 snap -1 2023-08-27T20:53:57.673 INFO:tasks.rados.rados.0.smithi093.stdout:2417: expect deleted 2023-08-27T20:53:57.673 INFO:tasks.rados.rados.0.smithi093.stdout:2418: write oid 262 current snap is 0 2023-08-27T20:53:57.673 INFO:tasks.rados.rados.0.smithi093.stdout:2411: finishing write tid 2 to smithi093153278-300 2023-08-27T20:53:57.673 INFO:tasks.rados.rados.0.smithi093.stdout:2411: finishing write tid 3 to smithi093153278-300 2023-08-27T20:53:57.673 INFO:tasks.rados.rados.0.smithi093.stdout:2413: expect (ObjNum 719 snap 0 seq_num 719) 2023-08-27T20:53:57.753 INFO:tasks.rados.rados.0.smithi093.stdout:2412: expect (ObjNum 1343 snap 0 seq_num 1343) 2023-08-27T20:53:57.898 INFO:tasks.rados.rados.0.smithi093.stdout:2411: finishing write tid 4 to smithi093153278-300 2023-08-27T20:53:57.898 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 300 v 283 (ObjNum 1583 snap 0 seq_num 1583) dirty exists 2023-08-27T20:53:57.898 INFO:tasks.rados.rados.0.smithi093.stdout:2411: left oid 300 (ObjNum 1583 snap 0 seq_num 1583) 2023-08-27T20:53:57.898 INFO:tasks.rados.rados.0.smithi093.stdout:2418: seq_num 1586 ranges {655902=746338,1808912=430219} 2023-08-27T20:53:57.905 INFO:tasks.rados.rados.0.smithi093.stdout:2418: writing smithi093153278-262 from 655902 to 1402240 tid 1 2023-08-27T20:53:57.910 INFO:tasks.rados.rados.0.smithi093.stdout:2418: writing smithi093153278-262 from 1808912 to 2239131 tid 2 2023-08-27T20:53:57.911 INFO:tasks.rados.rados.0.smithi093.stdout:2411: done (4 left) 2023-08-27T20:53:57.911 INFO:tasks.rados.rados.0.smithi093.stdout:2412: done (3 left) 2023-08-27T20:53:57.911 INFO:tasks.rados.rados.0.smithi093.stdout:2413: done (2 left) 2023-08-27T20:53:57.911 INFO:tasks.rados.rados.0.smithi093.stdout:2419: write oid 4 current snap is 0 2023-08-27T20:53:57.911 INFO:tasks.rados.rados.0.smithi093.stdout:2418: finishing write tid 1 to smithi093153278-262 2023-08-27T20:53:57.911 INFO:tasks.rados.rados.0.smithi093.stdout:2419: seq_num 1587 ranges {672709=421587,1867394=573877} 2023-08-27T20:53:57.916 INFO:tasks.rados.rados.0.smithi093.stdout:2419: writing smithi093153278-4 from 672709 to 1094296 tid 1 2023-08-27T20:53:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:2419: writing smithi093153278-4 from 1867394 to 2441271 tid 2 2023-08-27T20:53:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:2417: done (2 left) 2023-08-27T20:53:57.924 INFO:tasks.rados.rados.0.smithi093.stdout:2420: read oid 25 snap -1 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2420: expect (ObjNum 912 snap 0 seq_num 912) 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2421: write oid 290 current snap is 0 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2418: finishing write tid 2 to smithi093153278-262 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2418: finishing write tid 3 to smithi093153278-262 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2418: finishing write tid 4 to smithi093153278-262 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 262 v 360 (ObjNum 1586 snap 0 seq_num 1586) dirty exists 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2418: left oid 262 (ObjNum 1586 snap 0 seq_num 1586) 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2419: finishing write tid 1 to smithi093153278-4 2023-08-27T20:53:57.925 INFO:tasks.rados.rados.0.smithi093.stdout:2421: seq_num 1588 ranges {689516=496836,1809523=433888} 2023-08-27T20:53:57.931 INFO:tasks.rados.rados.0.smithi093.stdout:2421: writing smithi093153278-290 from 689516 to 1186352 tid 1 2023-08-27T20:53:57.936 INFO:tasks.rados.rados.0.smithi093.stdout:2421: writing smithi093153278-290 from 1809523 to 2243411 tid 2 2023-08-27T20:53:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:2418: done (3 left) 2023-08-27T20:53:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:2422: write oid 11 current snap is 0 2023-08-27T20:53:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:2419: finishing write tid 2 to smithi093153278-4 2023-08-27T20:53:57.937 INFO:tasks.rados.rados.0.smithi093.stdout:2422: seq_num 1589 ranges {706323=688438,1984358=577546} 2023-08-27T20:53:57.944 INFO:tasks.rados.rados.0.smithi093.stdout:2422: writing smithi093153278-11 from 706323 to 1394761 tid 1 2023-08-27T20:53:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:2422: writing smithi093153278-11 from 1984358 to 2561904 tid 2 2023-08-27T20:53:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:2423: write oid 125 current snap is 0 2023-08-27T20:53:57.951 INFO:tasks.rados.rados.0.smithi093.stdout:2423: seq_num 1590 ranges {723130=763687,2042840=437557} 2023-08-27T20:53:57.959 INFO:tasks.rados.rados.0.smithi093.stdout:2423: writing smithi093153278-125 from 723130 to 1486817 tid 1 2023-08-27T20:53:57.964 INFO:tasks.rados.rados.0.smithi093.stdout:2423: writing smithi093153278-125 from 2042840 to 2480397 tid 2 2023-08-27T20:53:57.965 INFO:tasks.rados.rados.0.smithi093.stdout:2424: write oid 64 current snap is 0 2023-08-27T20:53:57.965 INFO:tasks.rados.rados.0.smithi093.stdout:2419: finishing write tid 3 to smithi093153278-4 2023-08-27T20:53:57.965 INFO:tasks.rados.rados.0.smithi093.stdout:2419: finishing write tid 4 to smithi093153278-4 2023-08-27T20:53:57.965 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 4 v 363 (ObjNum 1587 snap 0 seq_num 1587) dirty exists 2023-08-27T20:53:57.965 INFO:tasks.rados.rados.0.smithi093.stdout:2419: left oid 4 (ObjNum 1587 snap 0 seq_num 1587) 2023-08-27T20:53:57.965 INFO:tasks.rados.rados.0.smithi093.stdout:2420: expect (ObjNum 912 snap 0 seq_num 912) 2023-08-27T20:53:58.122 INFO:tasks.rados.rados.0.smithi093.stdout:2421: finishing write tid 1 to smithi093153278-290 2023-08-27T20:53:58.122 INFO:tasks.rados.rados.0.smithi093.stdout:2424: seq_num 1591 ranges {739937=438936,1701322=581215} 2023-08-27T20:53:58.128 INFO:tasks.rados.rados.0.smithi093.stdout:2424: writing smithi093153278-64 from 739937 to 1178873 tid 1 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2424: writing smithi093153278-64 from 1701322 to 2282537 tid 2 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2419: done (5 left) 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2420: done (4 left) 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2425: read oid 417 snap -1 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2425: expect (ObjNum 1385 snap 0 seq_num 1385) 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2426: read oid 72 snap -1 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2426: expect (ObjNum 944 snap 0 seq_num 944) 2023-08-27T20:53:58.135 INFO:tasks.rados.rados.0.smithi093.stdout:2427: write oid 89 current snap is 0 2023-08-27T20:53:58.136 INFO:tasks.rados.rados.0.smithi093.stdout:2427: seq_num 1592 ranges {756744=514185,2043451=441226} 2023-08-27T20:53:58.142 INFO:tasks.rados.rados.0.smithi093.stdout:2427: writing smithi093153278-89 from 756744 to 1270929 tid 1 2023-08-27T20:53:58.149 INFO:tasks.rados.rados.0.smithi093.stdout:2427: writing smithi093153278-89 from 2043451 to 2484677 tid 2 2023-08-27T20:53:58.149 INFO:tasks.rados.rados.0.smithi093.stdout:2428: read oid 425 snap -1 2023-08-27T20:53:58.149 INFO:tasks.rados.rados.0.smithi093.stdout:2428: expect (ObjNum 1282 snap 0 seq_num 1282) 2023-08-27T20:53:58.149 INFO:tasks.rados.rados.0.smithi093.stdout:2429: read oid 430 snap -1 2023-08-27T20:53:58.149 INFO:tasks.rados.rados.0.smithi093.stdout:2429: expect deleted 2023-08-27T20:53:58.150 INFO:tasks.rados.rados.0.smithi093.stdout:2430: write oid 88 current snap is 0 2023-08-27T20:53:58.150 INFO:tasks.rados.rados.0.smithi093.stdout:2430: seq_num 1593 ranges {773551=589434,2101933=584884} 2023-08-27T20:53:58.157 INFO:tasks.rados.rados.0.smithi093.stdout:2430: writing smithi093153278-88 from 773551 to 1362985 tid 1 2023-08-27T20:53:58.164 INFO:tasks.rados.rados.0.smithi093.stdout:2430: writing smithi093153278-88 from 2101933 to 2686817 tid 2 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2431: write oid 487 current snap is 0 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2421: finishing write tid 2 to smithi093153278-290 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2421: finishing write tid 3 to smithi093153278-290 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2421: finishing write tid 4 to smithi093153278-290 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 290 v 285 (ObjNum 1588 snap 0 seq_num 1588) dirty exists 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2421: left oid 290 (ObjNum 1588 snap 0 seq_num 1588) 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2422: finishing write tid 1 to smithi093153278-11 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2422: finishing write tid 2 to smithi093153278-11 2023-08-27T20:53:58.165 INFO:tasks.rados.rados.0.smithi093.stdout:2431: seq_num 1594 ranges {790358=664683,2160415=444895} 2023-08-27T20:53:58.172 INFO:tasks.rados.rados.0.smithi093.stdout:2431: writing smithi093153278-487 from 790358 to 1455041 tid 1 2023-08-27T20:53:58.179 INFO:tasks.rados.rados.0.smithi093.stdout:2431: writing smithi093153278-487 from 2160415 to 2605310 tid 2 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2421: done (10 left) 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2432: read oid 441 snap -1 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2432: expect (ObjNum 1535 snap 0 seq_num 1535) 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2433: read oid 142 snap -1 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2433: expect deleted 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2434: write oid 136 current snap is 0 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2422: finishing write tid 3 to smithi093153278-11 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2422: finishing write tid 4 to smithi093153278-11 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 11 v 278 (ObjNum 1589 snap 0 seq_num 1589) dirty exists 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2422: left oid 11 (ObjNum 1589 snap 0 seq_num 1589) 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2423: finishing write tid 1 to smithi093153278-125 2023-08-27T20:53:58.180 INFO:tasks.rados.rados.0.smithi093.stdout:2423: finishing write tid 2 to smithi093153278-125 2023-08-27T20:53:58.181 INFO:tasks.rados.rados.0.smithi093.stdout:2434: seq_num 1595 ranges {407165=739932,1818897=588553} 2023-08-27T20:53:58.185 INFO:tasks.rados.rados.0.smithi093.stdout:2434: writing smithi093153278-136 from 407165 to 1147097 tid 1 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:2434: writing smithi093153278-136 from 1818897 to 2407450 tid 2 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:2422: done (12 left) 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:2435: write oid 195 current snap is 0 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:2423: finishing write tid 3 to smithi093153278-125 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:2423: finishing write tid 4 to smithi093153278-125 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 125 v 336 (ObjNum 1590 snap 0 seq_num 1590) dirty exists 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:2423: left oid 125 (ObjNum 1590 snap 0 seq_num 1590) 2023-08-27T20:53:58.193 INFO:tasks.rados.rados.0.smithi093.stdout:2435: seq_num 1596 ranges {423972=415181,1361026=448564,2561920=262052} 2023-08-27T20:53:58.197 INFO:tasks.rados.rados.0.smithi093.stdout:2435: writing smithi093153278-195 from 423972 to 839153 tid 1 2023-08-27T20:53:58.203 INFO:tasks.rados.rados.0.smithi093.stdout:2435: writing smithi093153278-195 from 1361026 to 1809590 tid 2 2023-08-27T20:53:58.208 INFO:tasks.rados.rados.0.smithi093.stdout:2435: writing smithi093153278-195 from 2561920 to 2823972 tid 3 2023-08-27T20:53:58.208 INFO:tasks.rados.rados.0.smithi093.stdout:2423: done (12 left) 2023-08-27T20:53:58.209 INFO:tasks.rados.rados.0.smithi093.stdout:2436: write oid 396 current snap is 0 2023-08-27T20:53:58.209 INFO:tasks.rados.rados.0.smithi093.stdout:2424: finishing write tid 1 to smithi093153278-64 2023-08-27T20:53:58.209 INFO:tasks.rados.rados.0.smithi093.stdout:2436: seq_num 1597 ranges {440779=606783,1535861=592222,2589343=251436} 2023-08-27T20:53:58.214 INFO:tasks.rados.rados.0.smithi093.stdout:2436: writing smithi093153278-396 from 440779 to 1047562 tid 1 2023-08-27T20:53:58.220 INFO:tasks.rados.rados.0.smithi093.stdout:2436: writing smithi093153278-396 from 1535861 to 2128083 tid 2 2023-08-27T20:53:58.224 INFO:tasks.rados.rados.0.smithi093.stdout:2436: writing smithi093153278-396 from 2589343 to 2840779 tid 3 2023-08-27T20:53:58.224 INFO:tasks.rados.rados.0.smithi093.stdout:2437: write oid 225 current snap is 0 2023-08-27T20:53:58.224 INFO:tasks.rados.rados.0.smithi093.stdout:2437: seq_num 1598 ranges {457586=682032,1594343=735880} 2023-08-27T20:53:58.230 INFO:tasks.rados.rados.0.smithi093.stdout:2437: writing smithi093153278-225 from 457586 to 1139618 tid 1 2023-08-27T20:53:58.237 INFO:tasks.rados.rados.0.smithi093.stdout:2437: writing smithi093153278-225 from 1594343 to 2330223 tid 2 2023-08-27T20:53:58.237 INFO:tasks.rados.rados.0.smithi093.stdout:2438: write oid 167 current snap is 0 2023-08-27T20:53:58.238 INFO:tasks.rados.rados.0.smithi093.stdout:2424: finishing write tid 2 to smithi093153278-64 2023-08-27T20:53:58.238 INFO:tasks.rados.rados.0.smithi093.stdout:2424: finishing write tid 3 to smithi093153278-64 2023-08-27T20:53:58.238 INFO:tasks.rados.rados.0.smithi093.stdout:2424: finishing write tid 4 to smithi093153278-64 2023-08-27T20:53:58.238 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 64 v 228 (ObjNum 1591 snap 0 seq_num 1591) dirty exists 2023-08-27T20:53:58.238 INFO:tasks.rados.rados.0.smithi093.stdout:2424: left oid 64 (ObjNum 1591 snap 0 seq_num 1591) 2023-08-27T20:53:58.238 INFO:tasks.rados.rados.0.smithi093.stdout:2425: expect (ObjNum 1385 snap 0 seq_num 1385) 2023-08-27T20:53:58.411 INFO:tasks.rados.rados.0.smithi093.stdout:2427: finishing write tid 1 to smithi093153278-89 2023-08-27T20:53:58.411 INFO:tasks.rados.rados.0.smithi093.stdout:2426: expect (ObjNum 944 snap 0 seq_num 944) 2023-08-27T20:53:58.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:58 smithi136 ceph-mon[113531]: pgmap v12: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s rd, 13 MiB/s wr, 122 op/s 2023-08-27T20:53:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:58 smithi093 ceph-mon[111335]: pgmap v12: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s rd, 13 MiB/s wr, 122 op/s 2023-08-27T20:53:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:58 smithi093 ceph-mon[118902]: pgmap v12: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s rd, 13 MiB/s wr, 122 op/s 2023-08-27T20:53:58.616 INFO:tasks.rados.rados.0.smithi093.stdout:2427: finishing write tid 2 to smithi093153278-89 2023-08-27T20:53:58.617 INFO:tasks.rados.rados.0.smithi093.stdout:2427: finishing write tid 3 to smithi093153278-89 2023-08-27T20:53:58.617 INFO:tasks.rados.rados.0.smithi093.stdout:2427: finishing write tid 4 to smithi093153278-89 2023-08-27T20:53:58.617 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 89 v 299 (ObjNum 1592 snap 0 seq_num 1592) dirty exists 2023-08-27T20:53:58.617 INFO:tasks.rados.rados.0.smithi093.stdout:2427: left oid 89 (ObjNum 1592 snap 0 seq_num 1592) 2023-08-27T20:53:58.617 INFO:tasks.rados.rados.0.smithi093.stdout:2428: expect (ObjNum 1282 snap 0 seq_num 1282) 2023-08-27T20:53:58.802 INFO:tasks.rados.rados.0.smithi093.stdout:2430: finishing write tid 1 to smithi093153278-88 2023-08-27T20:53:58.802 INFO:tasks.rados.rados.0.smithi093.stdout:2430: finishing write tid 2 to smithi093153278-88 2023-08-27T20:53:58.802 INFO:tasks.rados.rados.0.smithi093.stdout:2430: finishing write tid 3 to smithi093153278-88 2023-08-27T20:53:58.802 INFO:tasks.rados.rados.0.smithi093.stdout:2430: finishing write tid 4 to smithi093153278-88 2023-08-27T20:53:58.802 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 366 (ObjNum 1593 snap 0 seq_num 1593) dirty exists 2023-08-27T20:53:58.802 INFO:tasks.rados.rados.0.smithi093.stdout:2430: left oid 88 (ObjNum 1593 snap 0 seq_num 1593) 2023-08-27T20:53:58.803 INFO:tasks.rados.rados.0.smithi093.stdout:2431: finishing write tid 1 to smithi093153278-487 2023-08-27T20:53:58.803 INFO:tasks.rados.rados.0.smithi093.stdout:2431: finishing write tid 2 to smithi093153278-487 2023-08-27T20:53:58.803 INFO:tasks.rados.rados.0.smithi093.stdout:2431: finishing write tid 3 to smithi093153278-487 2023-08-27T20:53:58.803 INFO:tasks.rados.rados.0.smithi093.stdout:2438: seq_num 1599 ranges {474393=757281,1652825=595891} 2023-08-27T20:53:58.808 INFO:tasks.rados.rados.0.smithi093.stdout:2438: writing smithi093153278-167 from 474393 to 1231674 tid 1 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2438: writing smithi093153278-167 from 1652825 to 2248716 tid 2 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2424: done (14 left) 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2425: done (13 left) 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2426: done (12 left) 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2427: done (11 left) 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2428: done (10 left) 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2429: done (9 left) 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2430: done (8 left) 2023-08-27T20:53:58.815 INFO:tasks.rados.rados.0.smithi093.stdout:2433: done (7 left) 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:2439: read oid 449 snap -1 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:2439: expect (ObjNum 1468 snap 0 seq_num 1468) 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:2440: write oid 238 current snap is 0 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:2431: finishing write tid 4 to smithi093153278-487 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 487 v 324 (ObjNum 1594 snap 0 seq_num 1594) dirty exists 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:2431: left oid 487 (ObjNum 1594 snap 0 seq_num 1594) 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:2434: finishing write tid 1 to smithi093153278-136 2023-08-27T20:53:58.816 INFO:tasks.rados.rados.0.smithi093.stdout:2432: expect (ObjNum 1535 snap 0 seq_num 1535) 2023-08-27T20:53:59.032 INFO:tasks.rados.rados.0.smithi093.stdout:2434: finishing write tid 2 to smithi093153278-136 2023-08-27T20:53:59.033 INFO:tasks.rados.rados.0.smithi093.stdout:2434: finishing write tid 3 to smithi093153278-136 2023-08-27T20:53:59.033 INFO:tasks.rados.rados.0.smithi093.stdout:2434: finishing write tid 4 to smithi093153278-136 2023-08-27T20:53:59.033 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 136 v 358 (ObjNum 1595 snap 0 seq_num 1595) dirty exists 2023-08-27T20:53:59.033 INFO:tasks.rados.rados.0.smithi093.stdout:2434: left oid 136 (ObjNum 1595 snap 0 seq_num 1595) 2023-08-27T20:53:59.033 INFO:tasks.rados.rados.0.smithi093.stdout:2435: finishing write tid 1 to smithi093153278-195 2023-08-27T20:53:59.033 INFO:tasks.rados.rados.0.smithi093.stdout:2440: seq_num 1600 ranges {491200=432530,1594954=739549} 2023-08-27T20:53:59.037 INFO:tasks.rados.rados.0.smithi093.stdout:2440: writing smithi093153278-238 from 491200 to 923730 tid 1 2023-08-27T20:53:59.046 INFO:tasks.rados.rados.0.smithi093.stdout:2440: writing smithi093153278-238 from 1594954 to 2334503 tid 2 2023-08-27T20:53:59.046 INFO:tasks.rados.rados.0.smithi093.stdout:2431: done (8 left) 2023-08-27T20:53:59.046 INFO:tasks.rados.rados.0.smithi093.stdout:2432: done (7 left) 2023-08-27T20:53:59.046 INFO:tasks.rados.rados.0.smithi093.stdout:2434: done (6 left) 2023-08-27T20:53:59.046 INFO:tasks.rados.rados.0.smithi093.stdout:2441: write oid 113 current snap is 0 2023-08-27T20:53:59.046 INFO:tasks.rados.rados.0.smithi093.stdout:2435: finishing write tid 2 to smithi093153278-195 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2435: finishing write tid 3 to smithi093153278-195 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2435: finishing write tid 4 to smithi093153278-195 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2435: finishing write tid 5 to smithi093153278-195 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 288 (ObjNum 1596 snap 0 seq_num 1596) dirty exists 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2435: left oid 195 (ObjNum 1596 snap 0 seq_num 1596) 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2436: finishing write tid 1 to smithi093153278-396 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2436: finishing write tid 2 to smithi093153278-396 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2436: finishing write tid 3 to smithi093153278-396 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2436: finishing write tid 4 to smithi093153278-396 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2436: finishing write tid 5 to smithi093153278-396 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 396 v 370 (ObjNum 1597 snap 0 seq_num 1597) dirty exists 2023-08-27T20:53:59.047 INFO:tasks.rados.rados.0.smithi093.stdout:2436: left oid 396 (ObjNum 1597 snap 0 seq_num 1597) 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:2437: finishing write tid 1 to smithi093153278-225 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:2437: finishing write tid 2 to smithi093153278-225 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:2437: finishing write tid 3 to smithi093153278-225 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:2437: finishing write tid 4 to smithi093153278-225 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 225 v 301 (ObjNum 1598 snap 0 seq_num 1598) dirty exists 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:2437: left oid 225 (ObjNum 1598 snap 0 seq_num 1598) 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:2438: finishing write tid 1 to smithi093153278-167 2023-08-27T20:53:59.048 INFO:tasks.rados.rados.0.smithi093.stdout:2441: seq_num 1601 ranges {508007=507779,1653436=599560} 2023-08-27T20:53:59.051 INFO:tasks.rados.rados.0.smithi093.stdout:2441: writing smithi093153278-113 from 508007 to 1015786 tid 1 2023-08-27T20:53:59.058 INFO:tasks.rados.rados.0.smithi093.stdout:2441: writing smithi093153278-113 from 1653436 to 2252996 tid 2 2023-08-27T20:53:59.058 INFO:tasks.rados.rados.0.smithi093.stdout:2435: done (6 left) 2023-08-27T20:53:59.058 INFO:tasks.rados.rados.0.smithi093.stdout:2436: done (5 left) 2023-08-27T20:53:59.058 INFO:tasks.rados.rados.0.smithi093.stdout:2437: done (4 left) 2023-08-27T20:53:59.058 INFO:tasks.rados.rados.0.smithi093.stdout:2442: write oid 160 current snap is 0 2023-08-27T20:53:59.058 INFO:tasks.rados.rados.0.smithi093.stdout:2442: seq_num 1602 ranges {524814=583028,1711918=743218} 2023-08-27T20:53:59.064 INFO:tasks.rados.rados.0.smithi093.stdout:2442: writing smithi093153278-160 from 524814 to 1107842 tid 1 2023-08-27T20:53:59.072 INFO:tasks.rados.rados.0.smithi093.stdout:2442: writing smithi093153278-160 from 1711918 to 2455136 tid 2 2023-08-27T20:53:59.072 INFO:tasks.rados.rados.0.smithi093.stdout:2443: read oid 248 snap -1 2023-08-27T20:53:59.072 INFO:tasks.rados.rados.0.smithi093.stdout:2443: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:59.072 INFO:tasks.rados.rados.0.smithi093.stdout:2444: delete oid 94 current snap is 0 2023-08-27T20:53:59.072 INFO:tasks.rados.rados.0.smithi093.stdout:2438: finishing write tid 2 to smithi093153278-167 2023-08-27T20:53:59.072 INFO:tasks.rados.rados.0.smithi093.stdout:2438: finishing write tid 3 to smithi093153278-167 2023-08-27T20:53:59.072 INFO:tasks.rados.rados.0.smithi093.stdout:2438: finishing write tid 4 to smithi093153278-167 2023-08-27T20:53:59.073 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 167 v 399 (ObjNum 1599 snap 0 seq_num 1599) dirty exists 2023-08-27T20:53:59.073 INFO:tasks.rados.rados.0.smithi093.stdout:2438: left oid 167 (ObjNum 1599 snap 0 seq_num 1599) 2023-08-27T20:53:59.073 INFO:tasks.rados.rados.0.smithi093.stdout:2439: expect (ObjNum 1468 snap 0 seq_num 1468) 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2440: finishing write tid 1 to smithi093153278-238 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2440: finishing write tid 2 to smithi093153278-238 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2440: finishing write tid 3 to smithi093153278-238 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2440: finishing write tid 4 to smithi093153278-238 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 238 v 429 (ObjNum 1600 snap 0 seq_num 1600) dirty exists 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2440: left oid 238 (ObjNum 1600 snap 0 seq_num 1600) 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2441: finishing write tid 1 to smithi093153278-113 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2438: done (6 left) 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2439: done (5 left) 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2440: done (4 left) 2023-08-27T20:53:59.221 INFO:tasks.rados.rados.0.smithi093.stdout:2444: done (3 left) 2023-08-27T20:53:59.222 INFO:tasks.rados.rados.0.smithi093.stdout:2445: read oid 119 snap -1 2023-08-27T20:53:59.222 INFO:tasks.rados.rados.0.smithi093.stdout:2445: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:53:59.222 INFO:tasks.rados.rados.0.smithi093.stdout:2446: write oid 356 current snap is 0 2023-08-27T20:53:59.222 INFO:tasks.rados.rados.0.smithi093.stdout:2441: finishing write tid 2 to smithi093153278-113 2023-08-27T20:53:59.222 INFO:tasks.rados.rados.0.smithi093.stdout:2441: finishing write tid 3 to smithi093153278-113 2023-08-27T20:53:59.222 INFO:tasks.rados.rados.0.smithi093.stdout:2446: seq_num 1604 ranges {558428=733526,1712529=746887,2948598=9830} 2023-08-27T20:53:59.227 INFO:tasks.rados.rados.0.smithi093.stdout:2446: writing smithi093153278-356 from 558428 to 1291954 tid 1 2023-08-27T20:53:59.233 INFO:tasks.rados.rados.0.smithi093.stdout:2446: writing smithi093153278-356 from 1712529 to 2459416 tid 2 2023-08-27T20:53:59.237 INFO:tasks.rados.rados.0.smithi093.stdout:2446: writing smithi093153278-356 from 2948598 to 2958428 tid 3 2023-08-27T20:53:59.237 INFO:tasks.rados.rados.0.smithi093.stdout:2447: write oid 243 current snap is 0 2023-08-27T20:53:59.237 INFO:tasks.rados.rados.0.smithi093.stdout:2441: finishing write tid 4 to smithi093153278-113 2023-08-27T20:53:59.237 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 113 v 281 (ObjNum 1601 snap 0 seq_num 1601) dirty exists 2023-08-27T20:53:59.237 INFO:tasks.rados.rados.0.smithi093.stdout:2441: left oid 113 (ObjNum 1601 snap 0 seq_num 1601) 2023-08-27T20:53:59.237 INFO:tasks.rados.rados.0.smithi093.stdout:2442: finishing write tid 1 to smithi093153278-160 2023-08-27T20:53:59.237 INFO:tasks.rados.rados.0.smithi093.stdout:2442: finishing write tid 2 to smithi093153278-160 2023-08-27T20:53:59.238 INFO:tasks.rados.rados.0.smithi093.stdout:2442: finishing write tid 3 to smithi093153278-160 2023-08-27T20:53:59.238 INFO:tasks.rados.rados.0.smithi093.stdout:2442: finishing write tid 4 to smithi093153278-160 2023-08-27T20:53:59.238 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 160 v 373 (ObjNum 1602 snap 0 seq_num 1602) dirty exists 2023-08-27T20:53:59.238 INFO:tasks.rados.rados.0.smithi093.stdout:2442: left oid 160 (ObjNum 1602 snap 0 seq_num 1602) 2023-08-27T20:53:59.238 INFO:tasks.rados.rados.0.smithi093.stdout:2447: seq_num 1605 ranges {575235=525128,1887364=606898} 2023-08-27T20:53:59.242 INFO:tasks.rados.rados.0.smithi093.stdout:2447: writing smithi093153278-243 from 575235 to 1100363 tid 1 2023-08-27T20:53:59.250 INFO:tasks.rados.rados.0.smithi093.stdout:2447: writing smithi093153278-243 from 1887364 to 2494262 tid 2 2023-08-27T20:53:59.250 INFO:tasks.rados.rados.0.smithi093.stdout:2441: done (5 left) 2023-08-27T20:53:59.250 INFO:tasks.rados.rados.0.smithi093.stdout:2442: done (4 left) 2023-08-27T20:53:59.250 INFO:tasks.rados.rados.0.smithi093.stdout:2448: write oid 93 current snap is 0 2023-08-27T20:53:59.250 INFO:tasks.rados.rados.0.smithi093.stdout:2443: expect (ObjNum 1308 snap 0 seq_num 1308) 2023-08-27T20:53:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/926455878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:53:59 smithi136 ceph-mon[113531]: pgmap v13: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 8.7 MiB/s wr, 83 op/s 2023-08-27T20:53:59.463 INFO:tasks.rados.rados.0.smithi093.stdout:2445: expect (ObjNum 118 snap 0 seq_num 118) 2023-08-27T20:53:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/926455878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:53:59 smithi093 ceph-mon[111335]: pgmap v13: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 8.7 MiB/s wr, 83 op/s 2023-08-27T20:53:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/926455878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:53:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:53:59 smithi093 ceph-mon[118902]: pgmap v13: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 8.7 MiB/s wr, 83 op/s 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2446: finishing write tid 1 to smithi093153278-356 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2446: finishing write tid 2 to smithi093153278-356 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2446: finishing write tid 3 to smithi093153278-356 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2446: finishing write tid 4 to smithi093153278-356 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2446: finishing write tid 5 to smithi093153278-356 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 356 v 292 (ObjNum 1604 snap 0 seq_num 1604) dirty exists 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2446: left oid 356 (ObjNum 1604 snap 0 seq_num 1604) 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2447: finishing write tid 1 to smithi093153278-243 2023-08-27T20:53:59.590 INFO:tasks.rados.rados.0.smithi093.stdout:2448: seq_num 1606 ranges {592042=600377,1945846=750556} 2023-08-27T20:53:59.596 INFO:tasks.rados.rados.0.smithi093.stdout:2448: writing smithi093153278-93 from 592042 to 1192419 tid 1 2023-08-27T20:53:59.604 INFO:tasks.rados.rados.0.smithi093.stdout:2448: writing smithi093153278-93 from 1945846 to 2696402 tid 2 2023-08-27T20:53:59.604 INFO:tasks.rados.rados.0.smithi093.stdout:2443: done (4 left) 2023-08-27T20:53:59.604 INFO:tasks.rados.rados.0.smithi093.stdout:2445: done (3 left) 2023-08-27T20:53:59.604 INFO:tasks.rados.rados.0.smithi093.stdout:2446: done (2 left) 2023-08-27T20:53:59.605 INFO:tasks.rados.rados.0.smithi093.stdout:2449: write oid 495 current snap is 0 2023-08-27T20:53:59.605 INFO:tasks.rados.rados.0.smithi093.stdout:2447: finishing write tid 2 to smithi093153278-243 2023-08-27T20:53:59.605 INFO:tasks.rados.rados.0.smithi093.stdout:2447: finishing write tid 3 to smithi093153278-243 2023-08-27T20:53:59.605 INFO:tasks.rados.rados.0.smithi093.stdout:2447: finishing write tid 4 to smithi093153278-243 2023-08-27T20:53:59.605 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 243 v 376 (ObjNum 1605 snap 0 seq_num 1605) dirty exists 2023-08-27T20:53:59.605 INFO:tasks.rados.rados.0.smithi093.stdout:2447: left oid 243 (ObjNum 1605 snap 0 seq_num 1605) 2023-08-27T20:53:59.605 INFO:tasks.rados.rados.0.smithi093.stdout:2449: seq_num 1607 ranges {608849=675626,2004328=610567} 2023-08-27T20:53:59.611 INFO:tasks.rados.rados.0.smithi093.stdout:2449: writing smithi093153278-495 from 608849 to 1284475 tid 1 2023-08-27T20:53:59.619 INFO:tasks.rados.rados.0.smithi093.stdout:2449: writing smithi093153278-495 from 2004328 to 2614895 tid 2 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2447: done (2 left) 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2450: read oid 436 snap -1 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2450: expect (ObjNum 1372 snap 0 seq_num 1372) 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2451: read oid 467 snap -1 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2451: expect deleted 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2452: read oid 489 snap -1 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2452: expect (ObjNum 1560 snap 0 seq_num 1560) 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2453: write oid 316 current snap is 0 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2448: finishing write tid 1 to smithi093153278-93 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2448: finishing write tid 2 to smithi093153278-93 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2448: finishing write tid 3 to smithi093153278-93 2023-08-27T20:53:59.620 INFO:tasks.rados.rados.0.smithi093.stdout:2448: finishing write tid 4 to smithi093153278-93 2023-08-27T20:53:59.621 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 93 v 292 (ObjNum 1606 snap 0 seq_num 1606) dirty exists 2023-08-27T20:53:59.621 INFO:tasks.rados.rados.0.smithi093.stdout:2448: left oid 93 (ObjNum 1606 snap 0 seq_num 1606) 2023-08-27T20:53:59.621 INFO:tasks.rados.rados.0.smithi093.stdout:2449: finishing write tid 1 to smithi093153278-495 2023-08-27T20:53:59.621 INFO:tasks.rados.rados.0.smithi093.stdout:2453: seq_num 1608 ranges {625656=750875,2062810=754225} 2023-08-27T20:53:59.626 INFO:tasks.rados.rados.0.smithi093.stdout:2453: writing smithi093153278-316 from 625656 to 1376531 tid 1 2023-08-27T20:53:59.634 INFO:tasks.rados.rados.0.smithi093.stdout:2453: writing smithi093153278-316 from 2062810 to 2817035 tid 2 2023-08-27T20:53:59.634 INFO:tasks.rados.rados.0.smithi093.stdout:2448: done (5 left) 2023-08-27T20:53:59.634 INFO:tasks.rados.rados.0.smithi093.stdout:2454: delete oid 187 current snap is 0 2023-08-27T20:53:59.634 INFO:tasks.rados.rados.0.smithi093.stdout:2449: finishing write tid 2 to smithi093153278-495 2023-08-27T20:53:59.634 INFO:tasks.rados.rados.0.smithi093.stdout:2449: finishing write tid 3 to smithi093153278-495 2023-08-27T20:53:59.635 INFO:tasks.rados.rados.0.smithi093.stdout:2452: expect (ObjNum 1560 snap 0 seq_num 1560) 2023-08-27T20:53:59.755 INFO:tasks.rados.rados.0.smithi093.stdout:2449: finishing write tid 4 to smithi093153278-495 2023-08-27T20:53:59.755 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 495 v 379 (ObjNum 1607 snap 0 seq_num 1607) dirty exists 2023-08-27T20:53:59.755 INFO:tasks.rados.rados.0.smithi093.stdout:2449: left oid 495 (ObjNum 1607 snap 0 seq_num 1607) 2023-08-27T20:53:59.755 INFO:tasks.rados.rados.0.smithi093.stdout:2450: expect (ObjNum 1372 snap 0 seq_num 1372) 2023-08-27T20:53:59.933 INFO:tasks.rados.rados.0.smithi093.stdout:2453: finishing write tid 1 to smithi093153278-316 2023-08-27T20:53:59.933 INFO:tasks.rados.rados.0.smithi093.stdout:2453: finishing write tid 2 to smithi093153278-316 2023-08-27T20:53:59.933 INFO:tasks.rados.rados.0.smithi093.stdout:2453: finishing write tid 3 to smithi093153278-316 2023-08-27T20:53:59.933 INFO:tasks.rados.rados.0.smithi093.stdout:2453: finishing write tid 4 to smithi093153278-316 2023-08-27T20:53:59.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 316 v 361 (ObjNum 1608 snap 0 seq_num 1608) dirty exists 2023-08-27T20:53:59.933 INFO:tasks.rados.rados.0.smithi093.stdout:2453: left oid 316 (ObjNum 1608 snap 0 seq_num 1608) 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2449: done (5 left) 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2450: done (4 left) 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2451: done (3 left) 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2452: done (2 left) 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2453: done (1 left) 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2454: done (0 left) 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2455: write oid 332 current snap is 0 2023-08-27T20:53:59.934 INFO:tasks.rados.rados.0.smithi093.stdout:2455: seq_num 1610 ranges {659270=501373,1663421=757894} 2023-08-27T20:53:59.938 INFO:tasks.rados.rados.0.smithi093.stdout:2455: writing smithi093153278-332 from 659270 to 1160643 tid 1 2023-08-27T20:53:59.946 INFO:tasks.rados.rados.0.smithi093.stdout:2455: writing smithi093153278-332 from 1663421 to 2421315 tid 2 2023-08-27T20:53:59.946 INFO:tasks.rados.rados.0.smithi093.stdout:2456: write oid 476 current snap is 0 2023-08-27T20:53:59.946 INFO:tasks.rados.rados.0.smithi093.stdout:2455: finishing write tid 1 to smithi093153278-332 2023-08-27T20:53:59.946 INFO:tasks.rados.rados.0.smithi093.stdout:2456: seq_num 1611 ranges {676077=576622,1721903=501552,2740559=335518} 2023-08-27T20:53:59.952 INFO:tasks.rados.rados.0.smithi093.stdout:2456: writing smithi093153278-476 from 676077 to 1252699 tid 1 2023-08-27T20:53:59.958 INFO:tasks.rados.rados.0.smithi093.stdout:2456: writing smithi093153278-476 from 1721903 to 2223455 tid 2 2023-08-27T20:53:59.963 INFO:tasks.rados.rados.0.smithi093.stdout:2456: writing smithi093153278-476 from 2740559 to 3076077 tid 3 2023-08-27T20:53:59.963 INFO:tasks.rados.rados.0.smithi093.stdout:2457: write oid 261 current snap is 0 2023-08-27T20:53:59.963 INFO:tasks.rados.rados.0.smithi093.stdout:2455: finishing write tid 2 to smithi093153278-332 2023-08-27T20:53:59.963 INFO:tasks.rados.rados.0.smithi093.stdout:2457: seq_num 1612 ranges {692884=768224,1896738=761563} 2023-08-27T20:53:59.970 INFO:tasks.rados.rados.0.smithi093.stdout:2457: writing smithi093153278-261 from 692884 to 1461108 tid 1 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2457: writing smithi093153278-261 from 1896738 to 2658301 tid 2 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2458: write oid 325 current snap is 0 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2455: finishing write tid 3 to smithi093153278-332 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2455: finishing write tid 4 to smithi093153278-332 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 332 v 295 (ObjNum 1610 snap 0 seq_num 1610) dirty exists 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2455: left oid 332 (ObjNum 1610 snap 0 seq_num 1610) 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2456: finishing write tid 1 to smithi093153278-476 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2456: finishing write tid 2 to smithi093153278-476 2023-08-27T20:53:59.978 INFO:tasks.rados.rados.0.smithi093.stdout:2458: seq_num 1613 ranges {709691=443473,1838867=505221,2795405=314286} 2023-08-27T20:53:59.983 INFO:tasks.rados.rados.0.smithi093.stdout:2458: writing smithi093153278-325 from 709691 to 1153164 tid 1 2023-08-27T20:53:59.990 INFO:tasks.rados.rados.0.smithi093.stdout:2458: writing smithi093153278-325 from 1838867 to 2344088 tid 2 2023-08-27T20:53:59.995 INFO:tasks.rados.rados.0.smithi093.stdout:2458: writing smithi093153278-325 from 2795405 to 3109691 tid 3 2023-08-27T20:53:59.995 INFO:tasks.rados.rados.0.smithi093.stdout:2455: done (3 left) 2023-08-27T20:53:59.995 INFO:tasks.rados.rados.0.smithi093.stdout:2459: read oid 122 snap -1 2023-08-27T20:53:59.995 INFO:tasks.rados.rados.0.smithi093.stdout:2459: expect (ObjNum 1098 snap 0 seq_num 1098) 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2460: read oid 127 snap -1 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2460: expect deleted 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2461: read oid 455 snap -1 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2461: expect (ObjNum 1418 snap 0 seq_num 1418) 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2462: read oid 329 snap -1 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2462: expect (ObjNum 1391 snap 0 seq_num 1391) 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2463: read oid 355 snap -1 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2463: expect (ObjNum 1172 snap 0 seq_num 1172) 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2456: finishing write tid 3 to smithi093153278-476 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2456: finishing write tid 4 to smithi093153278-476 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2464: read oid 223 snap -1 2023-08-27T20:53:59.996 INFO:tasks.rados.rados.0.smithi093.stdout:2464: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:53:59.997 INFO:tasks.rados.rados.0.smithi093.stdout:2465: write oid 364 current snap is 0 2023-08-27T20:53:59.997 INFO:tasks.rados.rados.0.smithi093.stdout:2465: seq_num 1614 ranges {726498=518722,1897349=765232} 2023-08-27T20:54:00.001 INFO:tasks.rados.rados.0.smithi093.stdout:2465: writing smithi093153278-364 from 726498 to 1245220 tid 1 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:2465: writing smithi093153278-364 from 1897349 to 2662581 tid 2 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:2466: read oid 256 snap -1 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:2466: expect (ObjNum 745 snap 0 seq_num 745) 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:2467: write oid 217 current snap is 0 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:2456: finishing write tid 5 to smithi093153278-476 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 476 v 383 (ObjNum 1611 snap 0 seq_num 1611) dirty exists 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:2456: left oid 476 (ObjNum 1611 snap 0 seq_num 1611) 2023-08-27T20:54:00.010 INFO:tasks.rados.rados.0.smithi093.stdout:2457: finishing write tid 1 to smithi093153278-261 2023-08-27T20:54:00.011 INFO:tasks.rados.rados.0.smithi093.stdout:2467: seq_num 1615 ranges {743305=593971,1955831=508890} 2023-08-27T20:54:00.017 INFO:tasks.rados.rados.0.smithi093.stdout:2467: writing smithi093153278-217 from 743305 to 1337276 tid 1 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2467: writing smithi093153278-217 from 1955831 to 2464721 tid 2 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2456: done (11 left) 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2468: read oid 163 snap -1 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2468: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2469: write oid 485 current snap is 0 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2457: finishing write tid 2 to smithi093153278-261 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2457: finishing write tid 3 to smithi093153278-261 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2457: finishing write tid 4 to smithi093153278-261 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 261 v 327 (ObjNum 1612 snap 0 seq_num 1612) dirty exists 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2457: left oid 261 (ObjNum 1612 snap 0 seq_num 1612) 2023-08-27T20:54:00.024 INFO:tasks.rados.rados.0.smithi093.stdout:2458: finishing write tid 1 to smithi093153278-325 2023-08-27T20:54:00.025 INFO:tasks.rados.rados.0.smithi093.stdout:2458: finishing write tid 2 to smithi093153278-325 2023-08-27T20:54:00.025 INFO:tasks.rados.rados.0.smithi093.stdout:2469: seq_num 1616 ranges {760112=669220,2014313=768901} 2023-08-27T20:54:00.031 INFO:tasks.rados.rados.0.smithi093.stdout:2469: writing smithi093153278-485 from 760112 to 1429332 tid 1 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2469: writing smithi093153278-485 from 2014313 to 2783214 tid 2 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2457: done (12 left) 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2470: read oid 492 snap -1 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2470: expect (ObjNum 904 snap 0 seq_num 904) 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2471: read oid 49 snap -1 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2471: expect (ObjNum 1189 snap 0 seq_num 1189) 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2472: read oid 483 snap -1 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2472: expect (ObjNum 1570 snap 0 seq_num 1570) 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2473: read oid 96 snap -1 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2473: expect (ObjNum 1435 snap 0 seq_num 1435) 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:54:00.040 INFO:tasks.rados.rados.0.smithi093.stdout:2458: finishing write tid 3 to smithi093153278-325 2023-08-27T20:54:00.041 INFO:tasks.rados.rados.0.smithi093.stdout:2458: finishing write tid 4 to smithi093153278-325 2023-08-27T20:54:00.041 INFO:tasks.rados.rados.0.smithi093.stdout:2458: finishing write tid 5 to smithi093153278-325 2023-08-27T20:54:00.041 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 325 v 365 (ObjNum 1613 snap 0 seq_num 1613) dirty exists 2023-08-27T20:54:00.041 INFO:tasks.rados.rados.0.smithi093.stdout:2458: left oid 325 (ObjNum 1613 snap 0 seq_num 1613) 2023-08-27T20:54:00.041 INFO:tasks.rados.rados.0.smithi093.stdout:2461: expect (ObjNum 1418 snap 0 seq_num 1418) 2023-08-27T20:54:00.219 INFO:tasks.rados.rados.0.smithi093.stdout:2459: expect (ObjNum 1098 snap 0 seq_num 1098) 2023-08-27T20:54:00.354 INFO:tasks.rados.rados.0.smithi093.stdout:2465: finishing write tid 1 to smithi093153278-364 2023-08-27T20:54:00.354 INFO:tasks.rados.rados.0.smithi093.stdout:2463: expect (ObjNum 1172 snap 0 seq_num 1172) 2023-08-27T20:54:00.577 INFO:tasks.rados.rados.0.smithi093.stdout:2465: finishing write tid 2 to smithi093153278-364 2023-08-27T20:54:00.577 INFO:tasks.rados.rados.0.smithi093.stdout:2465: finishing write tid 3 to smithi093153278-364 2023-08-27T20:54:00.577 INFO:tasks.rados.rados.0.smithi093.stdout:2465: finishing write tid 4 to smithi093153278-364 2023-08-27T20:54:00.577 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 364 v 284 (ObjNum 1614 snap 0 seq_num 1614) dirty exists 2023-08-27T20:54:00.577 INFO:tasks.rados.rados.0.smithi093.stdout:2465: left oid 364 (ObjNum 1614 snap 0 seq_num 1614) 2023-08-27T20:54:00.577 INFO:tasks.rados.rados.0.smithi093.stdout:2466: expect (ObjNum 745 snap 0 seq_num 745) 2023-08-27T20:54:00.689 INFO:tasks.rados.rados.0.smithi093.stdout:2464: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:54:00.778 INFO:tasks.rados.rados.0.smithi093.stdout:2462: expect (ObjNum 1391 snap 0 seq_num 1391) 2023-08-27T20:54:00.918 INFO:tasks.rados.rados.0.smithi093.stdout:2467: finishing write tid 1 to smithi093153278-217 2023-08-27T20:54:00.918 INFO:tasks.rados.rados.0.smithi093.stdout:2467: finishing write tid 2 to smithi093153278-217 2023-08-27T20:54:00.918 INFO:tasks.rados.rados.0.smithi093.stdout:2467: finishing write tid 3 to smithi093153278-217 2023-08-27T20:54:00.919 INFO:tasks.rados.rados.0.smithi093.stdout:2467: finishing write tid 4 to smithi093153278-217 2023-08-27T20:54:00.919 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 217 v 386 (ObjNum 1615 snap 0 seq_num 1615) dirty exists 2023-08-27T20:54:00.919 INFO:tasks.rados.rados.0.smithi093.stdout:2467: left oid 217 (ObjNum 1615 snap 0 seq_num 1615) 2023-08-27T20:54:00.919 INFO:tasks.rados.rados.0.smithi093.stdout:2468: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:54:01.089 INFO:tasks.rados.rados.0.smithi093.stdout:2469: finishing write tid 1 to smithi093153278-485 2023-08-27T20:54:01.089 INFO:tasks.rados.rados.0.smithi093.stdout:2473: expect (ObjNum 1435 snap 0 seq_num 1435) 2023-08-27T20:54:01.192 INFO:tasks.rados.rados.0.smithi093.stdout:2469: finishing write tid 2 to smithi093153278-485 2023-08-27T20:54:01.193 INFO:tasks.rados.rados.0.smithi093.stdout:2469: finishing write tid 3 to smithi093153278-485 2023-08-27T20:54:01.193 INFO:tasks.rados.rados.0.smithi093.stdout:2472: expect (ObjNum 1570 snap 0 seq_num 1570) 2023-08-27T20:54:01.313 INFO:tasks.rados.rados.0.smithi093.stdout:2469: finishing write tid 4 to smithi093153278-485 2023-08-27T20:54:01.314 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 485 v 308 (ObjNum 1616 snap 0 seq_num 1616) dirty exists 2023-08-27T20:54:01.314 INFO:tasks.rados.rados.0.smithi093.stdout:2469: left oid 485 (ObjNum 1616 snap 0 seq_num 1616) 2023-08-27T20:54:01.314 INFO:tasks.rados.rados.0.smithi093.stdout:2470: expect (ObjNum 904 snap 0 seq_num 904) 2023-08-27T20:54:01.512 INFO:tasks.rados.rados.0.smithi093.stdout:2471: expect (ObjNum 1189 snap 0 seq_num 1189) 2023-08-27T20:54:01.815 INFO:tasks.rados.rados.0.smithi093.stdout:2458: done (15 left) 2023-08-27T20:54:01.815 INFO:tasks.rados.rados.0.smithi093.stdout:2459: done (14 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2460: done (13 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2461: done (12 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2462: done (11 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2463: done (10 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2464: done (9 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2465: done (8 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2466: done (7 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2467: done (6 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2468: done (5 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2469: done (4 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2470: done (3 left) 2023-08-27T20:54:01.816 INFO:tasks.rados.rados.0.smithi093.stdout:2471: done (2 left) 2023-08-27T20:54:01.817 INFO:tasks.rados.rados.0.smithi093.stdout:2472: done (1 left) 2023-08-27T20:54:01.817 INFO:tasks.rados.rados.0.smithi093.stdout:2473: done (0 left) 2023-08-27T20:54:01.817 INFO:tasks.rados.rados.0.smithi093.stdout:2474: read oid 256 snap -1 2023-08-27T20:54:01.817 INFO:tasks.rados.rados.0.smithi093.stdout:2474: expect (ObjNum 745 snap 0 seq_num 745) 2023-08-27T20:54:01.817 INFO:tasks.rados.rados.0.smithi093.stdout:2475: write oid 296 current snap is 0 2023-08-27T20:54:01.817 INFO:tasks.rados.rados.0.smithi093.stdout:2475: seq_num 1617 ranges {776919=744469,1956442=512559} 2023-08-27T20:54:01.823 INFO:tasks.rados.rados.0.smithi093.stdout:2475: writing smithi093153278-296 from 776919 to 1521388 tid 1 2023-08-27T20:54:01.828 INFO:tasks.rados.rados.0.smithi093.stdout:2475: writing smithi093153278-296 from 1956442 to 2469001 tid 2 2023-08-27T20:54:01.828 INFO:tasks.rados.rados.0.smithi093.stdout:2476: write oid 79 current snap is 0 2023-08-27T20:54:01.828 INFO:tasks.rados.rados.0.smithi093.stdout:2476: seq_num 1618 ranges {793726=419718,1614924=772570,2932520=261206} 2023-08-27T20:54:01.835 INFO:tasks.rados.rados.0.smithi093.stdout:2476: writing smithi093153278-79 from 793726 to 1213444 tid 1 2023-08-27T20:54:01.841 INFO:tasks.rados.rados.0.smithi093.stdout:2476: writing smithi093153278-79 from 1614924 to 2387494 tid 2 2023-08-27T20:54:01.846 INFO:tasks.rados.rados.0.smithi093.stdout:2476: writing smithi093153278-79 from 2932520 to 3193726 tid 3 2023-08-27T20:54:01.846 INFO:tasks.rados.rados.0.smithi093.stdout:2477: delete oid 92 current snap is 0 2023-08-27T20:54:01.846 INFO:tasks.rados.rados.0.smithi093.stdout:2474: expect (ObjNum 745 snap 0 seq_num 745) 2023-08-27T20:54:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:01 smithi136 ceph-mon[113531]: pgmap v14: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 12 MiB/s wr, 119 op/s 2023-08-27T20:54:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/785449264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2475: finishing write tid 1 to smithi093153278-296 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2475: finishing write tid 2 to smithi093153278-296 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2475: finishing write tid 3 to smithi093153278-296 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2475: finishing write tid 4 to smithi093153278-296 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 296 v 298 (ObjNum 1617 snap 0 seq_num 1617) dirty exists 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2475: left oid 296 (ObjNum 1617 snap 0 seq_num 1617) 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2476: finishing write tid 1 to smithi093153278-79 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2476: finishing write tid 2 to smithi093153278-79 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2476: finishing write tid 3 to smithi093153278-79 2023-08-27T20:54:01.956 INFO:tasks.rados.rados.0.smithi093.stdout:2476: finishing write tid 4 to smithi093153278-79 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2476: finishing write tid 5 to smithi093153278-79 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 79 v 433 (ObjNum 1618 snap 0 seq_num 1618) dirty exists 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2476: left oid 79 (ObjNum 1618 snap 0 seq_num 1618) 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2474: done (3 left) 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2475: done (2 left) 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2476: done (1 left) 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2477: done (0 left) 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2478: read oid 279 snap -1 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2478: expect (ObjNum 841 snap 0 seq_num 841) 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2479: read oid 482 snap -1 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2479: expect (ObjNum 1340 snap 0 seq_num 1340) 2023-08-27T20:54:01.957 INFO:tasks.rados.rados.0.smithi093.stdout:2480: read oid 83 snap -1 2023-08-27T20:54:01.958 INFO:tasks.rados.rados.0.smithi093.stdout:2480: expect (ObjNum 1505 snap 0 seq_num 1505) 2023-08-27T20:54:01.958 INFO:tasks.rados.rados.0.smithi093.stdout:2481: read oid 419 snap -1 2023-08-27T20:54:01.958 INFO:tasks.rados.rados.0.smithi093.stdout:2481: expect deleted 2023-08-27T20:54:01.958 INFO:tasks.rados.rados.0.smithi093.stdout:2482: delete oid 155 current snap is 0 2023-08-27T20:54:01.958 INFO:tasks.rados.rados.0.smithi093.stdout:2481: done (4 left) 2023-08-27T20:54:01.958 INFO:tasks.rados.rados.0.smithi093.stdout:2482: done (3 left) 2023-08-27T20:54:01.959 INFO:tasks.rados.rados.0.smithi093.stdout:2483: write oid 333 current snap is 0 2023-08-27T20:54:01.959 INFO:tasks.rados.rados.0.smithi093.stdout:2483: seq_num 1621 ranges {444147=761818,1790370=519897,2898436=345711} 2023-08-27T20:54:01.964 INFO:tasks.rados.rados.0.smithi093.stdout:2483: writing smithi093153278-333 from 444147 to 1205965 tid 1 2023-08-27T20:54:01.970 INFO:tasks.rados.rados.0.smithi093.stdout:2483: writing smithi093153278-333 from 1790370 to 2310267 tid 2 2023-08-27T20:54:01.975 INFO:tasks.rados.rados.0.smithi093.stdout:2483: writing smithi093153278-333 from 2898436 to 3244147 tid 3 2023-08-27T20:54:01.975 INFO:tasks.rados.rados.0.smithi093.stdout:2484: write oid 187 current snap is 0 2023-08-27T20:54:01.975 INFO:tasks.rados.rados.0.smithi093.stdout:2484: seq_num 1622 ranges {460954=437067,1448852=663555,2525859=735095} 2023-08-27T20:54:01.980 INFO:tasks.rados.rados.0.smithi093.stdout:2484: writing smithi093153278-187 from 460954 to 898021 tid 1 2023-08-27T20:54:01.986 INFO:tasks.rados.rados.0.smithi093.stdout:2484: writing smithi093153278-187 from 1448852 to 2112407 tid 2 2023-08-27T20:54:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:01 smithi093 ceph-mon[118902]: pgmap v14: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 12 MiB/s wr, 119 op/s 2023-08-27T20:54:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/785449264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:01 smithi093 ceph-mon[111335]: pgmap v14: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 12 MiB/s wr, 119 op/s 2023-08-27T20:54:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/785449264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:01.993 INFO:tasks.rados.rados.0.smithi093.stdout:2484: writing smithi093153278-187 from 2525859 to 3260954 tid 3 2023-08-27T20:54:01.993 INFO:tasks.rados.rados.0.smithi093.stdout:2485: read oid 134 snap -1 2023-08-27T20:54:01.993 INFO:tasks.rados.rados.0.smithi093.stdout:2485: expect (ObjNum 1580 snap 0 seq_num 1580) 2023-08-27T20:54:01.993 INFO:tasks.rados.rados.0.smithi093.stdout:2486: write oid 314 current snap is 0 2023-08-27T20:54:01.994 INFO:tasks.rados.rados.0.smithi093.stdout:2478: expect (ObjNum 841 snap 0 seq_num 841) 2023-08-27T20:54:02.078 INFO:tasks.rados.rados.0.smithi093.stdout:2480: expect (ObjNum 1505 snap 0 seq_num 1505) 2023-08-27T20:54:02.261 INFO:tasks.rados.rados.0.smithi093.stdout:2479: expect (ObjNum 1340 snap 0 seq_num 1340) 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:2483: finishing write tid 1 to smithi093153278-333 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:2483: finishing write tid 2 to smithi093153278-333 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:2483: finishing write tid 3 to smithi093153278-333 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:2483: finishing write tid 4 to smithi093153278-333 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:2483: finishing write tid 5 to smithi093153278-333 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 333 v 303 (ObjNum 1621 snap 0 seq_num 1621) dirty exists 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:2483: left oid 333 (ObjNum 1621 snap 0 seq_num 1621) 2023-08-27T20:54:02.377 INFO:tasks.rados.rados.0.smithi093.stdout:2486: seq_num 1623 ranges {477761=512316,1507334=523566,2553282=599771} 2023-08-27T20:54:02.382 INFO:tasks.rados.rados.0.smithi093.stdout:2486: writing smithi093153278-314 from 477761 to 990077 tid 1 2023-08-27T20:54:02.387 INFO:tasks.rados.rados.0.smithi093.stdout:2486: writing smithi093153278-314 from 1507334 to 2030900 tid 2 2023-08-27T20:54:02.394 INFO:tasks.rados.rados.0.smithi093.stdout:2486: writing smithi093153278-314 from 2553282 to 3153053 tid 3 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2478: done (6 left) 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2479: done (5 left) 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2480: done (4 left) 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2483: done (3 left) 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2487: read oid 45 snap -1 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2487: expect (ObjNum 1205 snap 0 seq_num 1205) 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2488: write oid 391 current snap is 0 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2484: finishing write tid 1 to smithi093153278-187 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2484: finishing write tid 2 to smithi093153278-187 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2484: finishing write tid 3 to smithi093153278-187 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2484: finishing write tid 4 to smithi093153278-187 2023-08-27T20:54:02.395 INFO:tasks.rados.rados.0.smithi093.stdout:2484: finishing write tid 5 to smithi093153278-187 2023-08-27T20:54:02.396 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 187 v 306 (ObjNum 1622 snap 0 seq_num 1622) dirty exists 2023-08-27T20:54:02.396 INFO:tasks.rados.rados.0.smithi093.stdout:2484: left oid 187 (ObjNum 1622 snap 0 seq_num 1622) 2023-08-27T20:54:02.396 INFO:tasks.rados.rados.0.smithi093.stdout:2485: expect (ObjNum 1580 snap 0 seq_num 1580) 2023-08-27T20:54:02.564 INFO:tasks.rados.rados.0.smithi093.stdout:2488: seq_num 1624 ranges {494568=587565,1565816=667224,2980705=313863} 2023-08-27T20:54:02.570 INFO:tasks.rados.rados.0.smithi093.stdout:2488: writing smithi093153278-391 from 494568 to 1082133 tid 1 2023-08-27T20:54:02.576 INFO:tasks.rados.rados.0.smithi093.stdout:2488: writing smithi093153278-391 from 1565816 to 2233040 tid 2 2023-08-27T20:54:02.582 INFO:tasks.rados.rados.0.smithi093.stdout:2488: writing smithi093153278-391 from 2980705 to 3294568 tid 3 2023-08-27T20:54:02.582 INFO:tasks.rados.rados.0.smithi093.stdout:2484: done (4 left) 2023-08-27T20:54:02.583 INFO:tasks.rados.rados.0.smithi093.stdout:2485: done (3 left) 2023-08-27T20:54:02.583 INFO:tasks.rados.rados.0.smithi093.stdout:2489: write oid 141 current snap is 0 2023-08-27T20:54:02.583 INFO:tasks.rados.rados.0.smithi093.stdout:2486: finishing write tid 1 to smithi093153278-314 2023-08-27T20:54:02.583 INFO:tasks.rados.rados.0.smithi093.stdout:2486: finishing write tid 2 to smithi093153278-314 2023-08-27T20:54:02.583 INFO:tasks.rados.rados.0.smithi093.stdout:2489: seq_num 1625 ranges {511375=662814,1907945=527235,2891775=419600} 2023-08-27T20:54:02.588 INFO:tasks.rados.rados.0.smithi093.stdout:2489: writing smithi093153278-141 from 511375 to 1174189 tid 1 2023-08-27T20:54:02.595 INFO:tasks.rados.rados.0.smithi093.stdout:2489: writing smithi093153278-141 from 1907945 to 2435180 tid 2 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2489: writing smithi093153278-141 from 2891775 to 3311375 tid 3 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2490: read oid 463 snap -1 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2490: expect (ObjNum 1338 snap 0 seq_num 1338) 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2491: delete oid 63 current snap is 0 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2486: finishing write tid 3 to smithi093153278-314 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2486: finishing write tid 4 to smithi093153278-314 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2486: finishing write tid 5 to smithi093153278-314 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 314 v 403 (ObjNum 1623 snap 0 seq_num 1623) dirty exists 2023-08-27T20:54:02.601 INFO:tasks.rados.rados.0.smithi093.stdout:2486: left oid 314 (ObjNum 1623 snap 0 seq_num 1623) 2023-08-27T20:54:02.602 INFO:tasks.rados.rados.0.smithi093.stdout:2487: expect (ObjNum 1205 snap 0 seq_num 1205) 2023-08-27T20:54:02.693 INFO:tasks.rados.rados.0.smithi093.stdout:2488: finishing write tid 1 to smithi093153278-391 2023-08-27T20:54:02.693 INFO:tasks.rados.rados.0.smithi093.stdout:2488: finishing write tid 2 to smithi093153278-391 2023-08-27T20:54:02.693 INFO:tasks.rados.rados.0.smithi093.stdout:2488: finishing write tid 3 to smithi093153278-391 2023-08-27T20:54:02.693 INFO:tasks.rados.rados.0.smithi093.stdout:2486: done (5 left) 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2487: done (4 left) 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2491: done (3 left) 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2492: write oid 411 current snap is 0 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2488: finishing write tid 4 to smithi093153278-391 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2488: finishing write tid 5 to smithi093153278-391 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 391 v 369 (ObjNum 1624 snap 0 seq_num 1624) dirty exists 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2488: left oid 391 (ObjNum 1624 snap 0 seq_num 1624) 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2489: finishing write tid 1 to smithi093153278-141 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2489: finishing write tid 2 to smithi093153278-141 2023-08-27T20:54:02.694 INFO:tasks.rados.rados.0.smithi093.stdout:2492: seq_num 1627 ranges {544989=529665,1741262=530904,3062974=282015} 2023-08-27T20:54:02.699 INFO:tasks.rados.rados.0.smithi093.stdout:2492: writing smithi093153278-411 from 544989 to 1074654 tid 1 2023-08-27T20:54:02.705 INFO:tasks.rados.rados.0.smithi093.stdout:2492: writing smithi093153278-411 from 1741262 to 2272166 tid 2 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2492: writing smithi093153278-411 from 3062974 to 3344989 tid 3 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2488: done (3 left) 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2493: read oid 216 snap -1 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2493: expect deleted 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2494: read oid 172 snap -1 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2494: expect deleted 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2495: write oid 75 current snap is 0 2023-08-27T20:54:02.712 INFO:tasks.rados.rados.0.smithi093.stdout:2495: seq_num 1628 ranges {561796=604914,1799744=674562,3090397=271399} 2023-08-27T20:54:02.718 INFO:tasks.rados.rados.0.smithi093.stdout:2495: writing smithi093153278-75 from 561796 to 1166710 tid 1 2023-08-27T20:54:02.725 INFO:tasks.rados.rados.0.smithi093.stdout:2495: writing smithi093153278-75 from 1799744 to 2474306 tid 2 2023-08-27T20:54:02.730 INFO:tasks.rados.rados.0.smithi093.stdout:2495: writing smithi093153278-75 from 3090397 to 3361796 tid 3 2023-08-27T20:54:02.730 INFO:tasks.rados.rados.0.smithi093.stdout:2496: write oid 477 current snap is 0 2023-08-27T20:54:02.730 INFO:tasks.rados.rados.0.smithi093.stdout:2496: seq_num 1629 ranges {578603=680163,1741873=534573,3001467=377136} 2023-08-27T20:54:02.737 INFO:tasks.rados.rados.0.smithi093.stdout:2496: writing smithi093153278-477 from 578603 to 1258766 tid 1 2023-08-27T20:54:02.743 INFO:tasks.rados.rados.0.smithi093.stdout:2496: writing smithi093153278-477 from 1741873 to 2276446 tid 2 2023-08-27T20:54:02.749 INFO:tasks.rados.rados.0.smithi093.stdout:2496: writing smithi093153278-477 from 3001467 to 3378603 tid 3 2023-08-27T20:54:02.749 INFO:tasks.rados.rados.0.smithi093.stdout:2497: read oid 176 snap -1 2023-08-27T20:54:02.749 INFO:tasks.rados.rados.0.smithi093.stdout:2497: expect deleted 2023-08-27T20:54:02.749 INFO:tasks.rados.rados.0.smithi093.stdout:2498: write oid 48 current snap is 0 2023-08-27T20:54:02.749 INFO:tasks.rados.rados.0.smithi093.stdout:2498: seq_num 1630 ranges {595410=755412,1800355=678231,3028890=366520} 2023-08-27T20:54:02.757 INFO:tasks.rados.rados.0.smithi093.stdout:2498: writing smithi093153278-48 from 595410 to 1350822 tid 1 2023-08-27T20:54:02.763 INFO:tasks.rados.rados.0.smithi093.stdout:2498: writing smithi093153278-48 from 1800355 to 2478586 tid 2 2023-08-27T20:54:02.768 INFO:tasks.rados.rados.0.smithi093.stdout:2498: writing smithi093153278-48 from 3028890 to 3395410 tid 3 2023-08-27T20:54:02.768 INFO:tasks.rados.rados.0.smithi093.stdout:2499: write oid 198 current snap is 0 2023-08-27T20:54:02.769 INFO:tasks.rados.rados.0.smithi093.stdout:2489: finishing write tid 3 to smithi093153278-141 2023-08-27T20:54:02.769 INFO:tasks.rados.rados.0.smithi093.stdout:2489: finishing write tid 4 to smithi093153278-141 2023-08-27T20:54:02.769 INFO:tasks.rados.rados.0.smithi093.stdout:2489: finishing write tid 5 to smithi093153278-141 2023-08-27T20:54:02.769 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 141 v 310 (ObjNum 1625 snap 0 seq_num 1625) dirty exists 2023-08-27T20:54:02.769 INFO:tasks.rados.rados.0.smithi093.stdout:2489: left oid 141 (ObjNum 1625 snap 0 seq_num 1625) 2023-08-27T20:54:02.769 INFO:tasks.rados.rados.0.smithi093.stdout:2490: expect (ObjNum 1338 snap 0 seq_num 1338) 2023-08-27T20:54:02.845 INFO:tasks.rados.rados.0.smithi093.stdout:2492: finishing write tid 1 to smithi093153278-411 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2492: finishing write tid 2 to smithi093153278-411 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2492: finishing write tid 3 to smithi093153278-411 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2492: finishing write tid 4 to smithi093153278-411 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2492: finishing write tid 5 to smithi093153278-411 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 411 v 302 (ObjNum 1627 snap 0 seq_num 1627) dirty exists 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2492: left oid 411 (ObjNum 1627 snap 0 seq_num 1627) 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2495: finishing write tid 1 to smithi093153278-75 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2495: finishing write tid 2 to smithi093153278-75 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2495: finishing write tid 3 to smithi093153278-75 2023-08-27T20:54:02.846 INFO:tasks.rados.rados.0.smithi093.stdout:2499: seq_num 1631 ranges {612217=430661,1458837=538242,2656313=522653} 2023-08-27T20:54:02.851 INFO:tasks.rados.rados.0.smithi093.stdout:2499: writing smithi093153278-198 from 612217 to 1042878 tid 1 2023-08-27T20:54:02.856 INFO:tasks.rados.rados.0.smithi093.stdout:2499: writing smithi093153278-198 from 1458837 to 1997079 tid 2 2023-08-27T20:54:02.863 INFO:tasks.rados.rados.0.smithi093.stdout:2499: writing smithi093153278-198 from 2656313 to 3178966 tid 3 2023-08-27T20:54:02.863 INFO:tasks.rados.rados.0.smithi093.stdout:2489: done (9 left) 2023-08-27T20:54:02.863 INFO:tasks.rados.rados.0.smithi093.stdout:2490: done (8 left) 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2492: done (7 left) 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2493: done (6 left) 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2494: done (5 left) 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2500: read oid 108 snap -1 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2500: expect (ObjNum 1429 snap 0 seq_num 1429) 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2501: write oid 415 current snap is 0 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2495: finishing write tid 4 to smithi093153278-75 2023-08-27T20:54:02.864 INFO:tasks.rados.rados.0.smithi093.stdout:2501: seq_num 1632 ranges {629024=505910,1917319=681900,3083736=345288} 2023-08-27T20:54:02.869 INFO:tasks.rados.rados.0.smithi093.stdout:2501: writing smithi093153278-415 from 629024 to 1134934 tid 1 2023-08-27T20:54:02.877 INFO:tasks.rados.rados.0.smithi093.stdout:2501: writing smithi093153278-415 from 1917319 to 2599219 tid 2 2023-08-27T20:54:02.882 INFO:tasks.rados.rados.0.smithi093.stdout:2501: writing smithi093153278-415 from 3083736 to 3429024 tid 3 2023-08-27T20:54:02.882 INFO:tasks.rados.rados.0.smithi093.stdout:2502: delete oid 480 current snap is 0 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2495: finishing write tid 5 to smithi093153278-75 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 75 v 437 (ObjNum 1628 snap 0 seq_num 1628) dirty exists 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2495: left oid 75 (ObjNum 1628 snap 0 seq_num 1628) 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2496: finishing write tid 1 to smithi093153278-477 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2496: finishing write tid 2 to smithi093153278-477 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2496: finishing write tid 3 to smithi093153278-477 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2496: finishing write tid 4 to smithi093153278-477 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2496: finishing write tid 5 to smithi093153278-477 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 477 v 407 (ObjNum 1629 snap 0 seq_num 1629) dirty exists 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2496: left oid 477 (ObjNum 1629 snap 0 seq_num 1629) 2023-08-27T20:54:02.883 INFO:tasks.rados.rados.0.smithi093.stdout:2498: finishing write tid 1 to smithi093153278-48 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2498: finishing write tid 2 to smithi093153278-48 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2498: finishing write tid 3 to smithi093153278-48 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2498: finishing write tid 4 to smithi093153278-48 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2498: finishing write tid 5 to smithi093153278-48 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 48 v 314 (ObjNum 1630 snap 0 seq_num 1630) dirty exists 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2498: left oid 48 (ObjNum 1630 snap 0 seq_num 1630) 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2499: finishing write tid 1 to smithi093153278-198 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2499: finishing write tid 2 to smithi093153278-198 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2499: finishing write tid 3 to smithi093153278-198 2023-08-27T20:54:02.884 INFO:tasks.rados.rados.0.smithi093.stdout:2500: expect (ObjNum 1429 snap 0 seq_num 1429) 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2499: finishing write tid 4 to smithi093153278-198 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2499: finishing write tid 5 to smithi093153278-198 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 198 v 441 (ObjNum 1631 snap 0 seq_num 1631) dirty exists 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2499: left oid 198 (ObjNum 1631 snap 0 seq_num 1631) 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2501: finishing write tid 1 to smithi093153278-415 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2501: finishing write tid 2 to smithi093153278-415 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2495: done (7 left) 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2496: done (6 left) 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2497: done (5 left) 2023-08-27T20:54:02.982 INFO:tasks.rados.rados.0.smithi093.stdout:2498: done (4 left) 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2499: done (3 left) 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2500: done (2 left) 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2502: done (1 left) 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2503: read oid 398 snap -1 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2503: expect (ObjNum 397 snap 0 seq_num 397) 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2504: write oid 441 current snap is 0 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2501: finishing write tid 3 to smithi093153278-415 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2501: finishing write tid 4 to smithi093153278-415 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2501: finishing write tid 5 to smithi093153278-415 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 415 v 373 (ObjNum 1632 snap 0 seq_num 1632) dirty exists 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2501: left oid 415 (ObjNum 1632 snap 0 seq_num 1632) 2023-08-27T20:54:02.983 INFO:tasks.rados.rados.0.smithi093.stdout:2504: seq_num 1634 ranges {662638=656408,1917930=685569,3022229=440409} 2023-08-27T20:54:02.988 INFO:tasks.rados.rados.0.smithi093.stdout:2504: writing smithi093153278-441 from 662638 to 1319046 tid 1 2023-08-27T20:54:02.995 INFO:tasks.rados.rados.0.smithi093.stdout:2504: writing smithi093153278-441 from 1917930 to 2603499 tid 2 2023-08-27T20:54:03.001 INFO:tasks.rados.rados.0.smithi093.stdout:2504: writing smithi093153278-441 from 3022229 to 3462638 tid 3 2023-08-27T20:54:03.001 INFO:tasks.rados.rados.0.smithi093.stdout:2501: done (2 left) 2023-08-27T20:54:03.001 INFO:tasks.rados.rados.0.smithi093.stdout:2505: read oid 491 snap -1 2023-08-27T20:54:03.001 INFO:tasks.rados.rados.0.smithi093.stdout:2505: expect (ObjNum 1110 snap 0 seq_num 1110) 2023-08-27T20:54:03.001 INFO:tasks.rados.rados.0.smithi093.stdout:2506: read oid 392 snap -1 2023-08-27T20:54:03.002 INFO:tasks.rados.rados.0.smithi093.stdout:2506: expect (ObjNum 1494 snap 0 seq_num 1494) 2023-08-27T20:54:03.002 INFO:tasks.rados.rados.0.smithi093.stdout:2507: read oid 230 snap -1 2023-08-27T20:54:03.002 INFO:tasks.rados.rados.0.smithi093.stdout:2507: expect (ObjNum 1177 snap 0 seq_num 1177) 2023-08-27T20:54:03.002 INFO:tasks.rados.rados.0.smithi093.stdout:2508: write oid 116 current snap is 0 2023-08-27T20:54:03.002 INFO:tasks.rados.rados.0.smithi093.stdout:2508: seq_num 1635 ranges {679445=448010,1692765=429227,2766005=567741} 2023-08-27T20:54:03.007 INFO:tasks.rados.rados.0.smithi093.stdout:2508: writing smithi093153278-116 from 679445 to 1127455 tid 1 2023-08-27T20:54:03.013 INFO:tasks.rados.rados.0.smithi093.stdout:2508: writing smithi093153278-116 from 1692765 to 2121992 tid 2 2023-08-27T20:54:03.020 INFO:tasks.rados.rados.0.smithi093.stdout:2508: writing smithi093153278-116 from 2766005 to 3333746 tid 3 2023-08-27T20:54:03.020 INFO:tasks.rados.rados.0.smithi093.stdout:2509: read oid 400 snap -1 2023-08-27T20:54:03.020 INFO:tasks.rados.rados.0.smithi093.stdout:2509: expect deleted 2023-08-27T20:54:03.020 INFO:tasks.rados.rados.0.smithi093.stdout:2510: read oid 125 snap -1 2023-08-27T20:54:03.020 INFO:tasks.rados.rados.0.smithi093.stdout:2510: expect (ObjNum 1590 snap 0 seq_num 1590) 2023-08-27T20:54:03.020 INFO:tasks.rados.rados.0.smithi093.stdout:2511: write oid 412 current snap is 0 2023-08-27T20:54:03.020 INFO:tasks.rados.rados.0.smithi093.stdout:2503: expect (ObjNum 397 snap 0 seq_num 397) 2023-08-27T20:54:03.104 INFO:tasks.rados.rados.0.smithi093.stdout:2504: finishing write tid 1 to smithi093153278-441 2023-08-27T20:54:03.104 INFO:tasks.rados.rados.0.smithi093.stdout:2504: finishing write tid 2 to smithi093153278-441 2023-08-27T20:54:03.104 INFO:tasks.rados.rados.0.smithi093.stdout:2505: expect (ObjNum 1110 snap 0 seq_num 1110) 2023-08-27T20:54:03.262 INFO:tasks.rados.rados.0.smithi093.stdout:2507: expect (ObjNum 1177 snap 0 seq_num 1177) 2023-08-27T20:54:03.485 INFO:tasks.rados.rados.0.smithi093.stdout:2504: finishing write tid 3 to smithi093153278-441 2023-08-27T20:54:03.485 INFO:tasks.rados.rados.0.smithi093.stdout:2504: finishing write tid 4 to smithi093153278-441 2023-08-27T20:54:03.485 INFO:tasks.rados.rados.0.smithi093.stdout:2504: finishing write tid 5 to smithi093153278-441 2023-08-27T20:54:03.485 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 307 (ObjNum 1634 snap 0 seq_num 1634) dirty exists 2023-08-27T20:54:03.485 INFO:tasks.rados.rados.0.smithi093.stdout:2504: left oid 441 (ObjNum 1634 snap 0 seq_num 1634) 2023-08-27T20:54:03.485 INFO:tasks.rados.rados.0.smithi093.stdout:2506: expect (ObjNum 1494 snap 0 seq_num 1494) 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:2508: finishing write tid 1 to smithi093153278-116 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:2508: finishing write tid 2 to smithi093153278-116 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:2508: finishing write tid 3 to smithi093153278-116 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:2508: finishing write tid 4 to smithi093153278-116 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:2508: finishing write tid 5 to smithi093153278-116 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 116 v 318 (ObjNum 1635 snap 0 seq_num 1635) dirty exists 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:2508: left oid 116 (ObjNum 1635 snap 0 seq_num 1635) 2023-08-27T20:54:03.582 INFO:tasks.rados.rados.0.smithi093.stdout:2510: expect (ObjNum 1590 snap 0 seq_num 1590) 2023-08-27T20:54:03.727 INFO:tasks.rados.rados.0.smithi093.stdout:2511: seq_num 1636 ranges {696252=523259,1751247=689238,3193428=302824} 2023-08-27T20:54:03.734 INFO:tasks.rados.rados.0.smithi093.stdout:2511: writing smithi093153278-412 from 696252 to 1219511 tid 1 2023-08-27T20:54:03.740 INFO:tasks.rados.rados.0.smithi093.stdout:2511: writing smithi093153278-412 from 1751247 to 2440485 tid 2 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2511: writing smithi093153278-412 from 3193428 to 3496252 tid 3 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2503: done (8 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2504: done (7 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2505: done (6 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2506: done (5 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2507: done (4 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2508: done (3 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2509: done (2 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2510: done (1 left) 2023-08-27T20:54:03.747 INFO:tasks.rados.rados.0.smithi093.stdout:2512: read oid 395 snap -1 2023-08-27T20:54:03.748 INFO:tasks.rados.rados.0.smithi093.stdout:2512: expect (ObjNum 1530 snap 0 seq_num 1530) 2023-08-27T20:54:03.748 INFO:tasks.rados.rados.0.smithi093.stdout:2513: write oid 321 current snap is 0 2023-08-27T20:54:03.748 INFO:tasks.rados.rados.0.smithi093.stdout:2511: finishing write tid 1 to smithi093153278-412 2023-08-27T20:54:03.748 INFO:tasks.rados.rados.0.smithi093.stdout:2511: finishing write tid 2 to smithi093153278-412 2023-08-27T20:54:03.748 INFO:tasks.rados.rados.0.smithi093.stdout:2513: seq_num 1637 ranges {713059=598508,2093376=432896,3104498=408561} 2023-08-27T20:54:03.754 INFO:tasks.rados.rados.0.smithi093.stdout:2513: writing smithi093153278-321 from 713059 to 1311567 tid 1 2023-08-27T20:54:03.760 INFO:tasks.rados.rados.0.smithi093.stdout:2513: writing smithi093153278-321 from 2093376 to 2526272 tid 2 2023-08-27T20:54:03.766 INFO:tasks.rados.rados.0.smithi093.stdout:2513: writing smithi093153278-321 from 3104498 to 3513059 tid 3 2023-08-27T20:54:03.766 INFO:tasks.rados.rados.0.smithi093.stdout:2514: write oid 147 current snap is 0 2023-08-27T20:54:03.766 INFO:tasks.rados.rados.0.smithi093.stdout:2514: seq_num 1638 ranges {729866=673757,2151858=692907} 2023-08-27T20:54:03.773 INFO:tasks.rados.rados.0.smithi093.stdout:2514: writing smithi093153278-147 from 729866 to 1403623 tid 1 2023-08-27T20:54:03.781 INFO:tasks.rados.rados.0.smithi093.stdout:2514: writing smithi093153278-147 from 2151858 to 2844765 tid 2 2023-08-27T20:54:03.781 INFO:tasks.rados.rados.0.smithi093.stdout:2515: write oid 409 current snap is 0 2023-08-27T20:54:03.782 INFO:tasks.rados.rados.0.smithi093.stdout:2511: finishing write tid 3 to smithi093153278-412 2023-08-27T20:54:03.782 INFO:tasks.rados.rados.0.smithi093.stdout:2515: seq_num 1639 ranges {746673=749006,2210340=436565,3159344=387329} 2023-08-27T20:54:03.789 INFO:tasks.rados.rados.0.smithi093.stdout:2515: writing smithi093153278-409 from 746673 to 1495679 tid 1 2023-08-27T20:54:03.795 INFO:tasks.rados.rados.0.smithi093.stdout:2515: writing smithi093153278-409 from 2210340 to 2646905 tid 2 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2515: writing smithi093153278-409 from 3159344 to 3546673 tid 3 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2516: read oid 242 snap -1 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2516: expect (ObjNum 1557 snap 0 seq_num 1557) 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2517: write oid 129 current snap is 0 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2511: finishing write tid 4 to smithi093153278-412 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2511: finishing write tid 5 to smithi093153278-412 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 412 v 302 (ObjNum 1636 snap 0 seq_num 1636) dirty exists 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2511: left oid 412 (ObjNum 1636 snap 0 seq_num 1636) 2023-08-27T20:54:03.801 INFO:tasks.rados.rados.0.smithi093.stdout:2512: expect (ObjNum 1530 snap 0 seq_num 1530) 2023-08-27T20:54:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:03 smithi136 ceph-mon[113531]: pgmap v15: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.2 MiB/s wr, 74 op/s 2023-08-27T20:54:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1126996398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:03 smithi093 ceph-mon[111335]: pgmap v15: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.2 MiB/s wr, 74 op/s 2023-08-27T20:54:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1126996398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:03 smithi093 ceph-mon[118902]: pgmap v15: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.2 MiB/s wr, 74 op/s 2023-08-27T20:54:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1126996398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:03.514Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:03.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:04.013 INFO:tasks.rados.rados.0.smithi093.stdout:2513: finishing write tid 1 to smithi093153278-321 2023-08-27T20:54:04.013 INFO:tasks.rados.rados.0.smithi093.stdout:2513: finishing write tid 2 to smithi093153278-321 2023-08-27T20:54:04.013 INFO:tasks.rados.rados.0.smithi093.stdout:2513: finishing write tid 3 to smithi093153278-321 2023-08-27T20:54:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:2513: finishing write tid 4 to smithi093153278-321 2023-08-27T20:54:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:2513: finishing write tid 5 to smithi093153278-321 2023-08-27T20:54:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 321 v 390 (ObjNum 1637 snap 0 seq_num 1637) dirty exists 2023-08-27T20:54:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:2513: left oid 321 (ObjNum 1637 snap 0 seq_num 1637) 2023-08-27T20:54:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:2514: finishing write tid 1 to smithi093153278-147 2023-08-27T20:54:04.014 INFO:tasks.rados.rados.0.smithi093.stdout:2517: seq_num 1640 ranges {763480=424255,1868822=696576,3186767=376713} 2023-08-27T20:54:04.019 INFO:tasks.rados.rados.0.smithi093.stdout:2517: writing smithi093153278-129 from 763480 to 1187735 tid 1 2023-08-27T20:54:04.027 INFO:tasks.rados.rados.0.smithi093.stdout:2517: writing smithi093153278-129 from 1868822 to 2565398 tid 2 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2517: writing smithi093153278-129 from 3186767 to 3563480 tid 3 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2511: done (6 left) 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2512: done (5 left) 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2513: done (4 left) 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2518: read oid 500 snap -1 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2518: expect (ObjNum 875 snap 0 seq_num 875) 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2519: write oid 486 current snap is 0 2023-08-27T20:54:04.033 INFO:tasks.rados.rados.0.smithi093.stdout:2514: finishing write tid 2 to smithi093153278-147 2023-08-27T20:54:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:2514: finishing write tid 3 to smithi093153278-147 2023-08-27T20:54:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:2514: finishing write tid 4 to smithi093153278-147 2023-08-27T20:54:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 147 v 289 (ObjNum 1638 snap 0 seq_num 1638) dirty exists 2023-08-27T20:54:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:2514: left oid 147 (ObjNum 1638 snap 0 seq_num 1638) 2023-08-27T20:54:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:2515: finishing write tid 1 to smithi093153278-409 2023-08-27T20:54:04.034 INFO:tasks.rados.rados.0.smithi093.stdout:2519: seq_num 1641 ranges {780287=499504,1810951=440234,2697837=468079} 2023-08-27T20:54:04.039 INFO:tasks.rados.rados.0.smithi093.stdout:2519: writing smithi093153278-486 from 780287 to 1279791 tid 1 2023-08-27T20:54:04.045 INFO:tasks.rados.rados.0.smithi093.stdout:2519: writing smithi093153278-486 from 1810951 to 2251185 tid 2 2023-08-27T20:54:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:2519: writing smithi093153278-486 from 2697837 to 3165916 tid 3 2023-08-27T20:54:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:2514: done (5 left) 2023-08-27T20:54:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:2520: read oid 238 snap -1 2023-08-27T20:54:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:2520: expect (ObjNum 1600 snap 0 seq_num 1600) 2023-08-27T20:54:04.050 INFO:tasks.rados.rados.0.smithi093.stdout:2521: write oid 61 current snap is 0 2023-08-27T20:54:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:2515: finishing write tid 2 to smithi093153278-409 2023-08-27T20:54:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:2515: finishing write tid 3 to smithi093153278-409 2023-08-27T20:54:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:2515: finishing write tid 4 to smithi093153278-409 2023-08-27T20:54:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:2515: finishing write tid 5 to smithi093153278-409 2023-08-27T20:54:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 409 v 293 (ObjNum 1639 snap 0 seq_num 1639) dirty exists 2023-08-27T20:54:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:2515: left oid 409 (ObjNum 1639 snap 0 seq_num 1639) 2023-08-27T20:54:04.051 INFO:tasks.rados.rados.0.smithi093.stdout:2516: expect (ObjNum 1557 snap 0 seq_num 1557) 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:2517: finishing write tid 1 to smithi093153278-129 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:2517: finishing write tid 2 to smithi093153278-129 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:2517: finishing write tid 3 to smithi093153278-129 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:2517: finishing write tid 4 to smithi093153278-129 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:2517: finishing write tid 5 to smithi093153278-129 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 129 v 297 (ObjNum 1640 snap 0 seq_num 1640) dirty exists 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:2517: left oid 129 (ObjNum 1640 snap 0 seq_num 1640) 2023-08-27T20:54:04.178 INFO:tasks.rados.rados.0.smithi093.stdout:2518: expect (ObjNum 875 snap 0 seq_num 875) 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:2519: finishing write tid 1 to smithi093153278-486 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:2519: finishing write tid 2 to smithi093153278-486 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:2519: finishing write tid 3 to smithi093153278-486 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:2519: finishing write tid 4 to smithi093153278-486 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:2519: finishing write tid 5 to smithi093153278-486 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 340 (ObjNum 1641 snap 0 seq_num 1641) dirty exists 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:2519: left oid 486 (ObjNum 1641 snap 0 seq_num 1641) 2023-08-27T20:54:04.335 INFO:tasks.rados.rados.0.smithi093.stdout:2520: expect (ObjNum 1600 snap 0 seq_num 1600) 2023-08-27T20:54:04.492 INFO:tasks.rados.rados.0.smithi093.stdout:2521: seq_num 1642 ranges {797094=574753,1869433=700245,3125260=471834} 2023-08-27T20:54:04.499 INFO:tasks.rados.rados.0.smithi093.stdout:2521: writing smithi093153278-61 from 797094 to 1371847 tid 1 2023-08-27T20:54:04.505 INFO:tasks.rados.rados.0.smithi093.stdout:2521: writing smithi093153278-61 from 1869433 to 2569678 tid 2 2023-08-27T20:54:04.512 INFO:tasks.rados.rados.0.smithi093.stdout:2521: writing smithi093153278-61 from 3125260 to 3597094 tid 3 2023-08-27T20:54:04.512 INFO:tasks.rados.rados.0.smithi093.stdout:2515: done (6 left) 2023-08-27T20:54:04.512 INFO:tasks.rados.rados.0.smithi093.stdout:2516: done (5 left) 2023-08-27T20:54:04.512 INFO:tasks.rados.rados.0.smithi093.stdout:2517: done (4 left) 2023-08-27T20:54:04.512 INFO:tasks.rados.rados.0.smithi093.stdout:2518: done (3 left) 2023-08-27T20:54:04.512 INFO:tasks.rados.rados.0.smithi093.stdout:2519: done (2 left) 2023-08-27T20:54:04.512 INFO:tasks.rados.rados.0.smithi093.stdout:2520: done (1 left) 2023-08-27T20:54:04.513 INFO:tasks.rados.rados.0.smithi093.stdout:2522: read oid 325 snap -1 2023-08-27T20:54:04.513 INFO:tasks.rados.rados.0.smithi093.stdout:2522: expect (ObjNum 1613 snap 0 seq_num 1613) 2023-08-27T20:54:04.513 INFO:tasks.rados.rados.0.smithi093.stdout:2523: read oid 459 snap -1 2023-08-27T20:54:04.513 INFO:tasks.rados.rados.0.smithi093.stdout:2523: expect (ObjNum 1473 snap 0 seq_num 1473) 2023-08-27T20:54:04.513 INFO:tasks.rados.rados.0.smithi093.stdout:2524: delete oid 235 current snap is 0 2023-08-27T20:54:04.513 INFO:tasks.rados.rados.0.smithi093.stdout:2521: finishing write tid 1 to smithi093153278-61 2023-08-27T20:54:04.513 INFO:tasks.rados.rados.0.smithi093.stdout:2521: finishing write tid 2 to smithi093153278-61 2023-08-27T20:54:04.514 INFO:tasks.rados.rados.0.smithi093.stdout:2524: done (3 left) 2023-08-27T20:54:04.514 INFO:tasks.rados.rados.0.smithi093.stdout:2525: read oid 470 snap -1 2023-08-27T20:54:04.514 INFO:tasks.rados.rados.0.smithi093.stdout:2525: expect (ObjNum 1011 snap 0 seq_num 1011) 2023-08-27T20:54:04.514 INFO:tasks.rados.rados.0.smithi093.stdout:2526: read oid 414 snap -1 2023-08-27T20:54:04.515 INFO:tasks.rados.rados.0.smithi093.stdout:2526: expect (ObjNum 1452 snap 0 seq_num 1452) 2023-08-27T20:54:04.515 INFO:tasks.rados.rados.0.smithi093.stdout:2527: read oid 298 snap -1 2023-08-27T20:54:04.515 INFO:tasks.rados.rados.0.smithi093.stdout:2527: expect (ObjNum 297 snap 0 seq_num 297) 2023-08-27T20:54:04.515 INFO:tasks.rados.rados.0.smithi093.stdout:2528: write oid 476 current snap is 0 2023-08-27T20:54:04.515 INFO:tasks.rados.rados.0.smithi093.stdout:2528: seq_num 1644 ranges {430708=441604,1302750=703914,2496459=701895} 2023-08-27T20:54:04.519 INFO:tasks.rados.rados.0.smithi093.stdout:2528: writing smithi093153278-476 from 430708 to 872312 tid 1 2023-08-27T20:54:04.525 INFO:tasks.rados.rados.0.smithi093.stdout:2528: writing smithi093153278-476 from 1302750 to 2006664 tid 2 2023-08-27T20:54:04.532 INFO:tasks.rados.rados.0.smithi093.stdout:2528: writing smithi093153278-476 from 2496459 to 3198354 tid 3 2023-08-27T20:54:04.532 INFO:tasks.rados.rados.0.smithi093.stdout:2529: read oid 406 snap -1 2023-08-27T20:54:04.532 INFO:tasks.rados.rados.0.smithi093.stdout:2529: expect deleted 2023-08-27T20:54:04.532 INFO:tasks.rados.rados.0.smithi093.stdout:2530: write oid 211 current snap is 0 2023-08-27T20:54:04.532 INFO:tasks.rados.rados.0.smithi093.stdout:2521: finishing write tid 3 to smithi093153278-61 2023-08-27T20:54:04.533 INFO:tasks.rados.rados.0.smithi093.stdout:2530: seq_num 1645 ranges {447515=516853,1644879=447572,2807529=629520} 2023-08-27T20:54:04.537 INFO:tasks.rados.rados.0.smithi093.stdout:2530: writing smithi093153278-211 from 447515 to 964368 tid 1 2023-08-27T20:54:04.544 INFO:tasks.rados.rados.0.smithi093.stdout:2530: writing smithi093153278-211 from 1644879 to 2092451 tid 2 2023-08-27T20:54:04.551 INFO:tasks.rados.rados.0.smithi093.stdout:2530: writing smithi093153278-211 from 2807529 to 3437049 tid 3 2023-08-27T20:54:04.551 INFO:tasks.rados.rados.0.smithi093.stdout:2531: write oid 438 current snap is 0 2023-08-27T20:54:04.552 INFO:tasks.rados.rados.0.smithi093.stdout:2521: finishing write tid 4 to smithi093153278-61 2023-08-27T20:54:04.552 INFO:tasks.rados.rados.0.smithi093.stdout:2521: finishing write tid 5 to smithi093153278-61 2023-08-27T20:54:04.552 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 61 v 344 (ObjNum 1642 snap 0 seq_num 1642) dirty exists 2023-08-27T20:54:04.552 INFO:tasks.rados.rados.0.smithi093.stdout:2521: left oid 61 (ObjNum 1642 snap 0 seq_num 1642) 2023-08-27T20:54:04.552 INFO:tasks.rados.rados.0.smithi093.stdout:2522: expect (ObjNum 1613 snap 0 seq_num 1613) 2023-08-27T20:54:04.651 INFO:tasks.rados.rados.0.smithi093.stdout:2523: expect (ObjNum 1473 snap 0 seq_num 1473) 2023-08-27T20:54:04.783 INFO:tasks.rados.rados.0.smithi093.stdout:2526: expect (ObjNum 1452 snap 0 seq_num 1452) 2023-08-27T20:54:04.887 INFO:tasks.rados.rados.0.smithi093.stdout:2525: expect (ObjNum 1011 snap 0 seq_num 1011) 2023-08-27T20:54:05.053 INFO:tasks.rados.rados.0.smithi093.stdout:2527: expect (ObjNum 297 snap 0 seq_num 297) 2023-08-27T20:54:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:2528: finishing write tid 1 to smithi093153278-476 2023-08-27T20:54:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:2528: finishing write tid 2 to smithi093153278-476 2023-08-27T20:54:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:2531: seq_num 1646 ranges {464322=592102,1703361=591230,2718599=440792} 2023-08-27T20:54:05.162 INFO:tasks.rados.rados.0.smithi093.stdout:2531: writing smithi093153278-438 from 464322 to 1056424 tid 1 2023-08-27T20:54:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:2531: writing smithi093153278-438 from 1703361 to 2294591 tid 2 2023-08-27T20:54:05.174 INFO:tasks.rados.rados.0.smithi093.stdout:2531: writing smithi093153278-438 from 2718599 to 3159391 tid 3 2023-08-27T20:54:05.174 INFO:tasks.rados.rados.0.smithi093.stdout:2521: done (9 left) 2023-08-27T20:54:05.174 INFO:tasks.rados.rados.0.smithi093.stdout:2522: done (8 left) 2023-08-27T20:54:05.174 INFO:tasks.rados.rados.0.smithi093.stdout:2523: done (7 left) 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2525: done (6 left) 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2526: done (5 left) 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2527: done (4 left) 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2529: done (3 left) 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2532: write oid 210 current snap is 0 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2528: finishing write tid 3 to smithi093153278-476 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2528: finishing write tid 4 to smithi093153278-476 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:2528: finishing write tid 5 to smithi093153278-476 2023-08-27T20:54:05.175 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 476 v 394 (ObjNum 1644 snap 0 seq_num 1644) dirty exists 2023-08-27T20:54:05.176 INFO:tasks.rados.rados.0.smithi093.stdout:2528: left oid 476 (ObjNum 1644 snap 0 seq_num 1644) 2023-08-27T20:54:05.176 INFO:tasks.rados.rados.0.smithi093.stdout:2530: finishing write tid 1 to smithi093153278-211 2023-08-27T20:54:05.176 INFO:tasks.rados.rados.0.smithi093.stdout:2530: finishing write tid 2 to smithi093153278-211 2023-08-27T20:54:05.176 INFO:tasks.rados.rados.0.smithi093.stdout:2532: seq_num 1647 ranges {481129=667351,1761843=451241,2862375=652064} 2023-08-27T20:54:05.180 INFO:tasks.rados.rados.0.smithi093.stdout:2532: writing smithi093153278-210 from 481129 to 1148480 tid 1 2023-08-27T20:54:05.186 INFO:tasks.rados.rados.0.smithi093.stdout:2532: writing smithi093153278-210 from 1761843 to 2213084 tid 2 2023-08-27T20:54:05.193 INFO:tasks.rados.rados.0.smithi093.stdout:2532: writing smithi093153278-210 from 2862375 to 3514439 tid 3 2023-08-27T20:54:05.193 INFO:tasks.rados.rados.0.smithi093.stdout:2528: done (3 left) 2023-08-27T20:54:05.193 INFO:tasks.rados.rados.0.smithi093.stdout:2533: write oid 137 current snap is 0 2023-08-27T20:54:05.193 INFO:tasks.rados.rados.0.smithi093.stdout:2530: finishing write tid 3 to smithi093153278-211 2023-08-27T20:54:05.194 INFO:tasks.rados.rados.0.smithi093.stdout:2530: finishing write tid 4 to smithi093153278-211 2023-08-27T20:54:05.194 INFO:tasks.rados.rados.0.smithi093.stdout:2533: seq_num 1648 ranges {497936=742600,1820325=594899,2889798=463336} 2023-08-27T20:54:05.200 INFO:tasks.rados.rados.0.smithi093.stdout:2533: writing smithi093153278-137 from 497936 to 1240536 tid 1 2023-08-27T20:54:05.206 INFO:tasks.rados.rados.0.smithi093.stdout:2533: writing smithi093153278-137 from 1820325 to 2415224 tid 2 2023-08-27T20:54:05.212 INFO:tasks.rados.rados.0.smithi093.stdout:2533: writing smithi093153278-137 from 2889798 to 3353134 tid 3 2023-08-27T20:54:05.212 INFO:tasks.rados.rados.0.smithi093.stdout:2534: write oid 182 current snap is 0 2023-08-27T20:54:05.212 INFO:tasks.rados.rados.0.smithi093.stdout:2530: finishing write tid 5 to smithi093153278-211 2023-08-27T20:54:05.212 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 211 v 445 (ObjNum 1645 snap 0 seq_num 1645) dirty exists 2023-08-27T20:54:05.213 INFO:tasks.rados.rados.0.smithi093.stdout:2530: left oid 211 (ObjNum 1645 snap 0 seq_num 1645) 2023-08-27T20:54:05.213 INFO:tasks.rados.rados.0.smithi093.stdout:2531: finishing write tid 1 to smithi093153278-438 2023-08-27T20:54:05.213 INFO:tasks.rados.rados.0.smithi093.stdout:2531: finishing write tid 2 to smithi093153278-438 2023-08-27T20:54:05.213 INFO:tasks.rados.rados.0.smithi093.stdout:2531: finishing write tid 3 to smithi093153278-438 2023-08-27T20:54:05.213 INFO:tasks.rados.rados.0.smithi093.stdout:2531: finishing write tid 4 to smithi093153278-438 2023-08-27T20:54:05.213 INFO:tasks.rados.rados.0.smithi093.stdout:2534: seq_num 1649 ranges {514743=417849,1362454=454910,2400868=790961} 2023-08-27T20:54:05.217 INFO:tasks.rados.rados.0.smithi093.stdout:2534: writing smithi093153278-182 from 514743 to 932592 tid 1 2023-08-27T20:54:05.222 INFO:tasks.rados.rados.0.smithi093.stdout:2534: writing smithi093153278-182 from 1362454 to 1817364 tid 2 2023-08-27T20:54:05.231 INFO:tasks.rados.rados.0.smithi093.stdout:2534: writing smithi093153278-182 from 2400868 to 3191829 tid 3 2023-08-27T20:54:05.231 INFO:tasks.rados.rados.0.smithi093.stdout:2530: done (4 left) 2023-08-27T20:54:05.231 INFO:tasks.rados.rados.0.smithi093.stdout:2535: read oid 86 snap -1 2023-08-27T20:54:05.231 INFO:tasks.rados.rados.0.smithi093.stdout:2535: expect (ObjNum 1126 snap 0 seq_num 1126) 2023-08-27T20:54:05.231 INFO:tasks.rados.rados.0.smithi093.stdout:2536: read oid 406 snap -1 2023-08-27T20:54:05.231 INFO:tasks.rados.rados.0.smithi093.stdout:2536: expect deleted 2023-08-27T20:54:05.231 INFO:tasks.rados.rados.0.smithi093.stdout:2537: delete oid 97 current snap is 0 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2531: finishing write tid 5 to smithi093153278-438 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 438 v 311 (ObjNum 1646 snap 0 seq_num 1646) dirty exists 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2531: left oid 438 (ObjNum 1646 snap 0 seq_num 1646) 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2532: finishing write tid 1 to smithi093153278-210 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2532: finishing write tid 2 to smithi093153278-210 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2532: finishing write tid 3 to smithi093153278-210 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2532: finishing write tid 4 to smithi093153278-210 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2532: finishing write tid 5 to smithi093153278-210 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 210 v 449 (ObjNum 1647 snap 0 seq_num 1647) dirty exists 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2532: left oid 210 (ObjNum 1647 snap 0 seq_num 1647) 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2533: finishing write tid 1 to smithi093153278-137 2023-08-27T20:54:05.232 INFO:tasks.rados.rados.0.smithi093.stdout:2533: finishing write tid 2 to smithi093153278-137 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:2533: finishing write tid 3 to smithi093153278-137 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:2533: finishing write tid 4 to smithi093153278-137 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:2533: finishing write tid 5 to smithi093153278-137 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 137 v 398 (ObjNum 1648 snap 0 seq_num 1648) dirty exists 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:2533: left oid 137 (ObjNum 1648 snap 0 seq_num 1648) 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:2534: finishing write tid 1 to smithi093153278-182 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:2534: finishing write tid 2 to smithi093153278-182 2023-08-27T20:54:05.233 INFO:tasks.rados.rados.0.smithi093.stdout:2531: done (6 left) 2023-08-27T20:54:05.234 INFO:tasks.rados.rados.0.smithi093.stdout:2532: done (5 left) 2023-08-27T20:54:05.234 INFO:tasks.rados.rados.0.smithi093.stdout:2533: done (4 left) 2023-08-27T20:54:05.234 INFO:tasks.rados.rados.0.smithi093.stdout:2536: done (3 left) 2023-08-27T20:54:05.234 INFO:tasks.rados.rados.0.smithi093.stdout:2537: done (2 left) 2023-08-27T20:54:05.234 INFO:tasks.rados.rados.0.smithi093.stdout:2538: write oid 298 current snap is 0 2023-08-27T20:54:05.234 INFO:tasks.rados.rados.0.smithi093.stdout:2538: seq_num 1651 ranges {548357=684700,1995771=458579,2972067=413505} 2023-08-27T20:54:05.239 INFO:tasks.rados.rados.0.smithi093.stdout:2538: writing smithi093153278-298 from 548357 to 1233057 tid 1 2023-08-27T20:54:05.246 INFO:tasks.rados.rados.0.smithi093.stdout:2538: writing smithi093153278-298 from 1995771 to 2454350 tid 2 2023-08-27T20:54:05.251 INFO:tasks.rados.rados.0.smithi093.stdout:2538: writing smithi093153278-298 from 2972067 to 3385572 tid 3 2023-08-27T20:54:05.251 INFO:tasks.rados.rados.0.smithi093.stdout:2539: delete oid 188 current snap is 0 2023-08-27T20:54:05.251 INFO:tasks.rados.rados.0.smithi093.stdout:2534: finishing write tid 3 to smithi093153278-182 2023-08-27T20:54:05.252 INFO:tasks.rados.rados.0.smithi093.stdout:2534: finishing write tid 4 to smithi093153278-182 2023-08-27T20:54:05.252 INFO:tasks.rados.rados.0.smithi093.stdout:2534: finishing write tid 5 to smithi093153278-182 2023-08-27T20:54:05.252 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 182 v 332 (ObjNum 1649 snap 0 seq_num 1649) dirty exists 2023-08-27T20:54:05.252 INFO:tasks.rados.rados.0.smithi093.stdout:2534: left oid 182 (ObjNum 1649 snap 0 seq_num 1649) 2023-08-27T20:54:05.252 INFO:tasks.rados.rados.0.smithi093.stdout:2535: expect (ObjNum 1126 snap 0 seq_num 1126) 2023-08-27T20:54:05.421 INFO:tasks.rados.rados.0.smithi093.stdout:2538: finishing write tid 1 to smithi093153278-298 2023-08-27T20:54:05.421 INFO:tasks.rados.rados.0.smithi093.stdout:2538: finishing write tid 2 to smithi093153278-298 2023-08-27T20:54:05.421 INFO:tasks.rados.rados.0.smithi093.stdout:2538: finishing write tid 3 to smithi093153278-298 2023-08-27T20:54:05.421 INFO:tasks.rados.rados.0.smithi093.stdout:2538: finishing write tid 4 to smithi093153278-298 2023-08-27T20:54:05.421 INFO:tasks.rados.rados.0.smithi093.stdout:2534: done (3 left) 2023-08-27T20:54:05.421 INFO:tasks.rados.rados.0.smithi093.stdout:2535: done (2 left) 2023-08-27T20:54:05.421 INFO:tasks.rados.rados.0.smithi093.stdout:2539: done (1 left) 2023-08-27T20:54:05.422 INFO:tasks.rados.rados.0.smithi093.stdout:2540: write oid 180 current snap is 0 2023-08-27T20:54:05.422 INFO:tasks.rados.rados.0.smithi093.stdout:2538: finishing write tid 5 to smithi093153278-298 2023-08-27T20:54:05.422 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 298 v 313 (ObjNum 1651 snap 0 seq_num 1651) dirty exists 2023-08-27T20:54:05.422 INFO:tasks.rados.rados.0.smithi093.stdout:2538: left oid 298 (ObjNum 1651 snap 0 seq_num 1651) 2023-08-27T20:54:05.422 INFO:tasks.rados.rados.0.smithi093.stdout:2540: seq_num 1653 ranges {581971=435198,1712735=462248,2626913=436049,3752375=29596} 2023-08-27T20:54:05.426 INFO:tasks.rados.rados.0.smithi093.stdout:2540: writing smithi093153278-180 from 581971 to 1017169 tid 1 2023-08-27T20:54:05.432 INFO:tasks.rados.rados.0.smithi093.stdout:2540: writing smithi093153278-180 from 1712735 to 2174983 tid 2 2023-08-27T20:54:05.437 INFO:tasks.rados.rados.0.smithi093.stdout:2540: writing smithi093153278-180 from 2626913 to 3062962 tid 3 2023-08-27T20:54:05.442 INFO:tasks.rados.rados.0.smithi093.stdout:2540: writing smithi093153278-180 from 3752375 to 3781971 tid 4 2023-08-27T20:54:05.442 INFO:tasks.rados.rados.0.smithi093.stdout:2538: done (1 left) 2023-08-27T20:54:05.442 INFO:tasks.rados.rados.0.smithi093.stdout:2541: write oid 296 current snap is 0 2023-08-27T20:54:05.442 INFO:tasks.rados.rados.0.smithi093.stdout:2540: finishing write tid 1 to smithi093153278-180 2023-08-27T20:54:05.442 INFO:tasks.rados.rados.0.smithi093.stdout:2541: seq_num 1654 ranges {598778=510447,1654864=605906,2937983=763674} 2023-08-27T20:54:05.448 INFO:tasks.rados.rados.0.smithi093.stdout:2541: writing smithi093153278-296 from 598778 to 1109225 tid 1 2023-08-27T20:54:05.454 INFO:tasks.rados.rados.0.smithi093.stdout:2541: writing smithi093153278-296 from 1654864 to 2260770 tid 2 2023-08-27T20:54:05.462 INFO:tasks.rados.rados.0.smithi093.stdout:2541: writing smithi093153278-296 from 2937983 to 3701657 tid 3 2023-08-27T20:54:05.462 INFO:tasks.rados.rados.0.smithi093.stdout:2542: write oid 369 current snap is 0 2023-08-27T20:54:05.462 INFO:tasks.rados.rados.0.smithi093.stdout:2540: finishing write tid 2 to smithi093153278-180 2023-08-27T20:54:05.462 INFO:tasks.rados.rados.0.smithi093.stdout:2542: seq_num 1655 ranges {615585=585696,1713346=465917,2965406=574946} 2023-08-27T20:54:05.468 INFO:tasks.rados.rados.0.smithi093.stdout:2542: writing smithi093153278-369 from 615585 to 1201281 tid 1 2023-08-27T20:54:05.474 INFO:tasks.rados.rados.0.smithi093.stdout:2542: writing smithi093153278-369 from 1713346 to 2179263 tid 2 2023-08-27T20:54:05.482 INFO:tasks.rados.rados.0.smithi093.stdout:2542: writing smithi093153278-369 from 2965406 to 3540352 tid 3 2023-08-27T20:54:05.482 INFO:tasks.rados.rados.0.smithi093.stdout:2543: write oid 474 current snap is 0 2023-08-27T20:54:05.482 INFO:tasks.rados.rados.0.smithi093.stdout:2540: finishing write tid 3 to smithi093153278-180 2023-08-27T20:54:05.483 INFO:tasks.rados.rados.0.smithi093.stdout:2540: finishing write tid 4 to smithi093153278-180 2023-08-27T20:54:05.483 INFO:tasks.rados.rados.0.smithi093.stdout:2540: finishing write tid 5 to smithi093153278-180 2023-08-27T20:54:05.483 INFO:tasks.rados.rados.0.smithi093.stdout:2543: seq_num 1656 ranges {632392=660945,1771828=609575,2992829=786218} 2023-08-27T20:54:05.489 INFO:tasks.rados.rados.0.smithi093.stdout:2543: writing smithi093153278-474 from 632392 to 1293337 tid 1 2023-08-27T20:54:05.494 INFO:tasks.rados.rados.0.smithi093.stdout:2543: writing smithi093153278-474 from 1771828 to 2381403 tid 2 2023-08-27T20:54:05.502 INFO:tasks.rados.rados.0.smithi093.stdout:2543: writing smithi093153278-474 from 2992829 to 3779047 tid 3 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2544: write oid 253 current snap is 0 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2540: finishing write tid 6 to smithi093153278-180 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 180 v 316 (ObjNum 1653 snap 0 seq_num 1653) dirty exists 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2540: left oid 180 (ObjNum 1653 snap 0 seq_num 1653) 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2541: finishing write tid 1 to smithi093153278-296 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2541: finishing write tid 2 to smithi093153278-296 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2541: finishing write tid 3 to smithi093153278-296 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2541: finishing write tid 4 to smithi093153278-296 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2541: finishing write tid 5 to smithi093153278-296 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 296 v 306 (ObjNum 1654 snap 0 seq_num 1654) dirty exists 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2541: left oid 296 (ObjNum 1654 snap 0 seq_num 1654) 2023-08-27T20:54:05.503 INFO:tasks.rados.rados.0.smithi093.stdout:2542: finishing write tid 1 to smithi093153278-369 2023-08-27T20:54:05.504 INFO:tasks.rados.rados.0.smithi093.stdout:2542: finishing write tid 2 to smithi093153278-369 2023-08-27T20:54:05.504 INFO:tasks.rados.rados.0.smithi093.stdout:2544: seq_num 1657 ranges {649199=736194,1830310=753233,3303899=545300} 2023-08-27T20:54:05.509 INFO:tasks.rados.rados.0.smithi093.stdout:2544: writing smithi093153278-253 from 649199 to 1385393 tid 1 2023-08-27T20:54:05.516 INFO:tasks.rados.rados.0.smithi093.stdout:2544: writing smithi093153278-253 from 1830310 to 2583543 tid 2 2023-08-27T20:54:05.523 INFO:tasks.rados.rados.0.smithi093.stdout:2544: writing smithi093153278-253 from 3303899 to 3849199 tid 3 2023-08-27T20:54:05.523 INFO:tasks.rados.rados.0.smithi093.stdout:2540: done (4 left) 2023-08-27T20:54:05.523 INFO:tasks.rados.rados.0.smithi093.stdout:2541: done (3 left) 2023-08-27T20:54:05.524 INFO:tasks.rados.rados.0.smithi093.stdout:2545: write oid 255 current snap is 0 2023-08-27T20:54:05.524 INFO:tasks.rados.rados.0.smithi093.stdout:2542: finishing write tid 3 to smithi093153278-369 2023-08-27T20:54:05.524 INFO:tasks.rados.rados.0.smithi093.stdout:2542: finishing write tid 4 to smithi093153278-369 2023-08-27T20:54:05.524 INFO:tasks.rados.rados.0.smithi093.stdout:2545: seq_num 1658 ranges {666006=527796,1888792=613244,3047675=525115} 2023-08-27T20:54:05.530 INFO:tasks.rados.rados.0.smithi093.stdout:2545: writing smithi093153278-255 from 666006 to 1193802 tid 1 2023-08-27T20:54:05.537 INFO:tasks.rados.rados.0.smithi093.stdout:2545: writing smithi093153278-255 from 1888792 to 2502036 tid 2 2023-08-27T20:54:05.543 INFO:tasks.rados.rados.0.smithi093.stdout:2545: writing smithi093153278-255 from 3047675 to 3572790 tid 3 2023-08-27T20:54:05.543 INFO:tasks.rados.rados.0.smithi093.stdout:2546: read oid 180 snap -1 2023-08-27T20:54:05.544 INFO:tasks.rados.rados.0.smithi093.stdout:2546: expect (ObjNum 1653 snap 0 seq_num 1653) 2023-08-27T20:54:05.544 INFO:tasks.rados.rados.0.smithi093.stdout:2547: write oid 383 current snap is 0 2023-08-27T20:54:05.544 INFO:tasks.rados.rados.0.smithi093.stdout:2542: finishing write tid 5 to smithi093153278-369 2023-08-27T20:54:05.544 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 369 v 335 (ObjNum 1655 snap 0 seq_num 1655) dirty exists 2023-08-27T20:54:05.544 INFO:tasks.rados.rados.0.smithi093.stdout:2542: left oid 369 (ObjNum 1655 snap 0 seq_num 1655) 2023-08-27T20:54:05.544 INFO:tasks.rados.rados.0.smithi093.stdout:2543: finishing write tid 1 to smithi093153278-474 2023-08-27T20:54:05.544 INFO:tasks.rados.rados.0.smithi093.stdout:2543: finishing write tid 2 to smithi093153278-474 2023-08-27T20:54:05.545 INFO:tasks.rados.rados.0.smithi093.stdout:2547: seq_num 1659 ranges {682813=603045,1947274=756902,3358745=524068} 2023-08-27T20:54:05.550 INFO:tasks.rados.rados.0.smithi093.stdout:2547: writing smithi093153278-383 from 682813 to 1285858 tid 1 2023-08-27T20:54:05.558 INFO:tasks.rados.rados.0.smithi093.stdout:2547: writing smithi093153278-383 from 1947274 to 2704176 tid 2 2023-08-27T20:54:05.565 INFO:tasks.rados.rados.0.smithi093.stdout:2547: writing smithi093153278-383 from 3358745 to 3882813 tid 3 2023-08-27T20:54:05.565 INFO:tasks.rados.rados.0.smithi093.stdout:2542: done (5 left) 2023-08-27T20:54:05.565 INFO:tasks.rados.rados.0.smithi093.stdout:2548: delete oid 497 current snap is 0 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2543: finishing write tid 3 to smithi093153278-474 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2543: finishing write tid 4 to smithi093153278-474 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2543: finishing write tid 5 to smithi093153278-474 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 474 v 306 (ObjNum 1656 snap 0 seq_num 1656) dirty exists 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2543: left oid 474 (ObjNum 1656 snap 0 seq_num 1656) 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2544: finishing write tid 1 to smithi093153278-253 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2544: finishing write tid 2 to smithi093153278-253 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2544: finishing write tid 3 to smithi093153278-253 2023-08-27T20:54:05.566 INFO:tasks.rados.rados.0.smithi093.stdout:2544: finishing write tid 4 to smithi093153278-253 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2544: finishing write tid 5 to smithi093153278-253 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 336 (ObjNum 1657 snap 0 seq_num 1657) dirty exists 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2544: left oid 253 (ObjNum 1657 snap 0 seq_num 1657) 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2545: finishing write tid 1 to smithi093153278-255 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2545: finishing write tid 2 to smithi093153278-255 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2545: finishing write tid 3 to smithi093153278-255 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2545: finishing write tid 4 to smithi093153278-255 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2545: finishing write tid 5 to smithi093153278-255 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 255 v 453 (ObjNum 1658 snap 0 seq_num 1658) dirty exists 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2545: left oid 255 (ObjNum 1658 snap 0 seq_num 1658) 2023-08-27T20:54:05.567 INFO:tasks.rados.rados.0.smithi093.stdout:2547: finishing write tid 1 to smithi093153278-383 2023-08-27T20:54:05.568 INFO:tasks.rados.rados.0.smithi093.stdout:2546: expect (ObjNum 1653 snap 0 seq_num 1653) 2023-08-27T20:54:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:54:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:54:05] "GET /metrics HTTP/1.1" 200 242428 "" "Prometheus/2.43.0" 2023-08-27T20:54:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:05 smithi093 ceph-mon[111335]: pgmap v16: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.2 MiB/s wr, 74 op/s 2023-08-27T20:54:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3175871152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:05 smithi093 ceph-mon[118902]: pgmap v16: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.2 MiB/s wr, 74 op/s 2023-08-27T20:54:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3175871152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:2547: finishing write tid 2 to smithi093153278-383 2023-08-27T20:54:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:2543: done (5 left) 2023-08-27T20:54:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:2544: done (4 left) 2023-08-27T20:54:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:2545: done (3 left) 2023-08-27T20:54:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:2546: done (2 left) 2023-08-27T20:54:05.862 INFO:tasks.rados.rados.0.smithi093.stdout:2548: done (1 left) 2023-08-27T20:54:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:2549: write oid 406 current snap is 0 2023-08-27T20:54:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:2547: finishing write tid 3 to smithi093153278-383 2023-08-27T20:54:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:2547: finishing write tid 4 to smithi093153278-383 2023-08-27T20:54:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:2547: finishing write tid 5 to smithi093153278-383 2023-08-27T20:54:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 383 v 412 (ObjNum 1659 snap 0 seq_num 1659) dirty exists 2023-08-27T20:54:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:2547: left oid 383 (ObjNum 1659 snap 0 seq_num 1659) 2023-08-27T20:54:05.863 INFO:tasks.rados.rados.0.smithi093.stdout:2549: seq_num 1661 ranges {716427=753543,2064238=760571,3413591=502836} 2023-08-27T20:54:05.869 INFO:tasks.rados.rados.0.smithi093.stdout:2549: writing smithi093153278-406 from 716427 to 1469970 tid 1 2023-08-27T20:54:05.877 INFO:tasks.rados.rados.0.smithi093.stdout:2549: writing smithi093153278-406 from 2064238 to 2824809 tid 2 2023-08-27T20:54:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:2549: writing smithi093153278-406 from 3413591 to 3916427 tid 3 2023-08-27T20:54:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:2547: done (1 left) 2023-08-27T20:54:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:2550: write oid 107 current snap is 0 2023-08-27T20:54:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:2549: finishing write tid 1 to smithi093153278-406 2023-08-27T20:54:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:2549: finishing write tid 2 to smithi093153278-406 2023-08-27T20:54:05.883 INFO:tasks.rados.rados.0.smithi093.stdout:2550: seq_num 1662 ranges {733234=428792,1606367=620582,2641014=570203,3664879=268355} 2023-08-27T20:54:05.889 INFO:tasks.rados.rados.0.smithi093.stdout:2550: writing smithi093153278-107 from 733234 to 1162026 tid 1 2023-08-27T20:54:05.895 INFO:tasks.rados.rados.0.smithi093.stdout:2550: writing smithi093153278-107 from 1606367 to 2226949 tid 2 2023-08-27T20:54:05.900 INFO:tasks.rados.rados.0.smithi093.stdout:2550: writing smithi093153278-107 from 2641014 to 3211217 tid 3 2023-08-27T20:54:05.905 INFO:tasks.rados.rados.0.smithi093.stdout:2550: writing smithi093153278-107 from 3664879 to 3933234 tid 4 2023-08-27T20:54:05.905 INFO:tasks.rados.rados.0.smithi093.stdout:2551: write oid 45 current snap is 0 2023-08-27T20:54:05.905 INFO:tasks.rados.rados.0.smithi093.stdout:2549: finishing write tid 3 to smithi093153278-406 2023-08-27T20:54:05.905 INFO:tasks.rados.rados.0.smithi093.stdout:2549: finishing write tid 4 to smithi093153278-406 2023-08-27T20:54:05.905 INFO:tasks.rados.rados.0.smithi093.stdout:2549: finishing write tid 5 to smithi093153278-406 2023-08-27T20:54:05.905 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 406 v 310 (ObjNum 1661 snap 0 seq_num 1661) dirty exists 2023-08-27T20:54:05.905 INFO:tasks.rados.rados.0.smithi093.stdout:2549: left oid 406 (ObjNum 1661 snap 0 seq_num 1661) 2023-08-27T20:54:05.906 INFO:tasks.rados.rados.0.smithi093.stdout:2550: finishing write tid 1 to smithi093153278-107 2023-08-27T20:54:05.906 INFO:tasks.rados.rados.0.smithi093.stdout:2550: finishing write tid 2 to smithi093153278-107 2023-08-27T20:54:05.906 INFO:tasks.rados.rados.0.smithi093.stdout:2550: finishing write tid 3 to smithi093153278-107 2023-08-27T20:54:05.906 INFO:tasks.rados.rados.0.smithi093.stdout:2551: seq_num 1663 ranges {750041=504041,1664849=764240,2952084=497828} 2023-08-27T20:54:05.911 INFO:tasks.rados.rados.0.smithi093.stdout:2551: writing smithi093153278-45 from 750041 to 1254082 tid 1 2023-08-27T20:54:05.917 INFO:tasks.rados.rados.0.smithi093.stdout:2551: writing smithi093153278-45 from 1664849 to 2429089 tid 2 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2551: writing smithi093153278-45 from 2952084 to 3449912 tid 3 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2549: done (2 left) 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2552: delete oid 302 current snap is 0 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2550: finishing write tid 4 to smithi093153278-107 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2550: finishing write tid 5 to smithi093153278-107 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2550: finishing write tid 6 to smithi093153278-107 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 107 v 321 (ObjNum 1662 snap 0 seq_num 1662) dirty exists 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2550: left oid 107 (ObjNum 1662 snap 0 seq_num 1662) 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2551: finishing write tid 1 to smithi093153278-45 2023-08-27T20:54:05.924 INFO:tasks.rados.rados.0.smithi093.stdout:2551: finishing write tid 2 to smithi093153278-45 2023-08-27T20:54:05.925 INFO:tasks.rados.rados.0.smithi093.stdout:2550: done (2 left) 2023-08-27T20:54:05.925 INFO:tasks.rados.rados.0.smithi093.stdout:2552: done (1 left) 2023-08-27T20:54:05.925 INFO:tasks.rados.rados.0.smithi093.stdout:2553: write oid 359 current snap is 0 2023-08-27T20:54:05.925 INFO:tasks.rados.rados.0.smithi093.stdout:2553: seq_num 1665 ranges {783655=770892,2298166=767909,3639636=344019} 2023-08-27T20:54:05.933 INFO:tasks.rados.rados.0.smithi093.stdout:2553: writing smithi093153278-359 from 783655 to 1554547 tid 1 2023-08-27T20:54:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:05 smithi136 ceph-mon[113531]: pgmap v16: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.2 MiB/s wr, 74 op/s 2023-08-27T20:54:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3175871152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:05.941 INFO:tasks.rados.rados.0.smithi093.stdout:2553: writing smithi093153278-359 from 2298166 to 3066075 tid 2 2023-08-27T20:54:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:2553: writing smithi093153278-359 from 3639636 to 3983655 tid 3 2023-08-27T20:54:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:2554: read oid 54 snap -1 2023-08-27T20:54:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:2554: expect (ObjNum 53 snap 0 seq_num 53) 2023-08-27T20:54:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:2555: write oid 268 current snap is 0 2023-08-27T20:54:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:2551: finishing write tid 3 to smithi093153278-45 2023-08-27T20:54:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:2555: seq_num 1666 ranges {400462=446141,1440295=560167} 2023-08-27T20:54:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:2555: writing smithi093153278-268 from 400462 to 846603 tid 1 2023-08-27T20:54:05.958 INFO:tasks.rados.rados.0.smithi093.stdout:2555: writing smithi093153278-268 from 1440295 to 2000462 tid 2 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2556: read oid 177 snap -1 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2556: expect (ObjNum 176 snap 0 seq_num 176) 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2557: write oid 394 current snap is 0 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2551: finishing write tid 4 to smithi093153278-45 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2551: finishing write tid 5 to smithi093153278-45 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 45 v 297 (ObjNum 1663 snap 0 seq_num 1663) dirty exists 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2551: left oid 45 (ObjNum 1663 snap 0 seq_num 1663) 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2553: finishing write tid 1 to smithi093153278-359 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2553: finishing write tid 2 to smithi093153278-359 2023-08-27T20:54:05.959 INFO:tasks.rados.rados.0.smithi093.stdout:2557: seq_num 1667 ranges {417269=521390,1498777=518492} 2023-08-27T20:54:05.963 INFO:tasks.rados.rados.0.smithi093.stdout:2557: writing smithi093153278-394 from 417269 to 938659 tid 1 2023-08-27T20:54:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:2557: writing smithi093153278-394 from 1498777 to 2017269 tid 2 2023-08-27T20:54:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:2551: done (5 left) 2023-08-27T20:54:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:2558: read oid 367 snap -1 2023-08-27T20:54:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:2558: expect (ObjNum 719 snap 0 seq_num 719) 2023-08-27T20:54:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:2559: write oid 56 current snap is 0 2023-08-27T20:54:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:2553: finishing write tid 3 to smithi093153278-359 2023-08-27T20:54:05.971 INFO:tasks.rados.rados.0.smithi093.stdout:2553: finishing write tid 4 to smithi093153278-359 2023-08-27T20:54:05.971 INFO:tasks.rados.rados.0.smithi093.stdout:2553: finishing write tid 5 to smithi093153278-359 2023-08-27T20:54:05.971 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 359 v 457 (ObjNum 1665 snap 0 seq_num 1665) dirty exists 2023-08-27T20:54:05.971 INFO:tasks.rados.rados.0.smithi093.stdout:2553: left oid 359 (ObjNum 1665 snap 0 seq_num 1665) 2023-08-27T20:54:05.971 INFO:tasks.rados.rados.0.smithi093.stdout:2559: seq_num 1668 ranges {434076=596639,1557259=476817} 2023-08-27T20:54:05.975 INFO:tasks.rados.rados.0.smithi093.stdout:2559: writing smithi093153278-56 from 434076 to 1030715 tid 1 2023-08-27T20:54:05.981 INFO:tasks.rados.rados.0.smithi093.stdout:2559: writing smithi093153278-56 from 1557259 to 2034076 tid 2 2023-08-27T20:54:05.981 INFO:tasks.rados.rados.0.smithi093.stdout:2553: done (6 left) 2023-08-27T20:54:05.981 INFO:tasks.rados.rados.0.smithi093.stdout:2560: read oid 379 snap -1 2023-08-27T20:54:05.982 INFO:tasks.rados.rados.0.smithi093.stdout:2560: expect (ObjNum 378 snap 0 seq_num 378) 2023-08-27T20:54:05.982 INFO:tasks.rados.rados.0.smithi093.stdout:2561: write oid 45 current snap is 0 2023-08-27T20:54:05.982 INFO:tasks.rados.rados.0.smithi093.stdout:2554: expect (ObjNum 53 snap 0 seq_num 53) 2023-08-27T20:54:06.073 INFO:tasks.rados.rados.0.smithi093.stdout:2555: finishing write tid 1 to smithi093153278-268 2023-08-27T20:54:06.073 INFO:tasks.rados.rados.0.smithi093.stdout:2555: finishing write tid 2 to smithi093153278-268 2023-08-27T20:54:06.074 INFO:tasks.rados.rados.0.smithi093.stdout:2555: finishing write tid 3 to smithi093153278-268 2023-08-27T20:54:06.074 INFO:tasks.rados.rados.0.smithi093.stdout:2555: finishing write tid 4 to smithi093153278-268 2023-08-27T20:54:06.074 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 268 v 313 (ObjNum 1666 snap 0 seq_num 1666) dirty exists 2023-08-27T20:54:06.074 INFO:tasks.rados.rados.0.smithi093.stdout:2555: left oid 268 (ObjNum 1666 snap 0 seq_num 1666) 2023-08-27T20:54:06.074 INFO:tasks.rados.rados.0.smithi093.stdout:2556: expect (ObjNum 176 snap 0 seq_num 176) 2023-08-27T20:54:06.168 INFO:tasks.rados.rados.0.smithi093.stdout:2557: finishing write tid 1 to smithi093153278-394 2023-08-27T20:54:06.168 INFO:tasks.rados.rados.0.smithi093.stdout:2557: finishing write tid 2 to smithi093153278-394 2023-08-27T20:54:06.168 INFO:tasks.rados.rados.0.smithi093.stdout:2557: finishing write tid 3 to smithi093153278-394 2023-08-27T20:54:06.168 INFO:tasks.rados.rados.0.smithi093.stdout:2557: finishing write tid 4 to smithi093153278-394 2023-08-27T20:54:06.168 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 394 v 347 (ObjNum 1667 snap 0 seq_num 1667) dirty exists 2023-08-27T20:54:06.168 INFO:tasks.rados.rados.0.smithi093.stdout:2557: left oid 394 (ObjNum 1667 snap 0 seq_num 1667) 2023-08-27T20:54:06.168 INFO:tasks.rados.rados.0.smithi093.stdout:2561: seq_num 1669 ranges {450883=671888,1615741=435142} 2023-08-27T20:54:06.174 INFO:tasks.rados.rados.0.smithi093.stdout:2561: writing smithi093153278-45 from 450883 to 1122771 tid 1 2023-08-27T20:54:06.180 INFO:tasks.rados.rados.0.smithi093.stdout:2561: writing smithi093153278-45 from 1615741 to 2050883 tid 2 2023-08-27T20:54:06.180 INFO:tasks.rados.rados.0.smithi093.stdout:2554: done (7 left) 2023-08-27T20:54:06.180 INFO:tasks.rados.rados.0.smithi093.stdout:2555: done (6 left) 2023-08-27T20:54:06.180 INFO:tasks.rados.rados.0.smithi093.stdout:2556: done (5 left) 2023-08-27T20:54:06.180 INFO:tasks.rados.rados.0.smithi093.stdout:2557: done (4 left) 2023-08-27T20:54:06.180 INFO:tasks.rados.rados.0.smithi093.stdout:2562: write oid 455 current snap is 0 2023-08-27T20:54:06.180 INFO:tasks.rados.rados.0.smithi093.stdout:2558: expect (ObjNum 719 snap 0 seq_num 719) 2023-08-27T20:54:06.262 INFO:tasks.rados.rados.0.smithi093.stdout:2559: finishing write tid 1 to smithi093153278-56 2023-08-27T20:54:06.262 INFO:tasks.rados.rados.0.smithi093.stdout:2560: expect (ObjNum 378 snap 0 seq_num 378) 2023-08-27T20:54:06.335 INFO:tasks.rados.rados.0.smithi093.stdout:2559: finishing write tid 2 to smithi093153278-56 2023-08-27T20:54:06.335 INFO:tasks.rados.rados.0.smithi093.stdout:2559: finishing write tid 3 to smithi093153278-56 2023-08-27T20:54:06.336 INFO:tasks.rados.rados.0.smithi093.stdout:2559: finishing write tid 4 to smithi093153278-56 2023-08-27T20:54:06.336 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 56 v 300 (ObjNum 1668 snap 0 seq_num 1668) dirty exists 2023-08-27T20:54:06.336 INFO:tasks.rados.rados.0.smithi093.stdout:2559: left oid 56 (ObjNum 1668 snap 0 seq_num 1668) 2023-08-27T20:54:06.336 INFO:tasks.rados.rados.0.smithi093.stdout:2562: seq_num 1670 ranges {467690=747137,1957870=109820} 2023-08-27T20:54:06.342 INFO:tasks.rados.rados.0.smithi093.stdout:2562: writing smithi093153278-455 from 467690 to 1214827 tid 1 2023-08-27T20:54:06.347 INFO:tasks.rados.rados.0.smithi093.stdout:2562: writing smithi093153278-455 from 1957870 to 2067690 tid 2 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2558: done (4 left) 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2559: done (3 left) 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2560: done (2 left) 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2563: read oid 358 snap -1 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2563: expect (ObjNum 806 snap 0 seq_num 806) 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2564: write oid 433 current snap is 0 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2561: finishing write tid 1 to smithi093153278-45 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2561: finishing write tid 2 to smithi093153278-45 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2561: finishing write tid 3 to smithi093153278-45 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2561: finishing write tid 4 to smithi093153278-45 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 45 v 300 (ObjNum 1669 snap 0 seq_num 1669) dirty exists 2023-08-27T20:54:06.348 INFO:tasks.rados.rados.0.smithi093.stdout:2561: left oid 45 (ObjNum 1669 snap 0 seq_num 1669) 2023-08-27T20:54:06.349 INFO:tasks.rados.rados.0.smithi093.stdout:2564: seq_num 1671 ranges {484497=422386,1616352=468145} 2023-08-27T20:54:06.352 INFO:tasks.rados.rados.0.smithi093.stdout:2564: writing smithi093153278-433 from 484497 to 906883 tid 1 2023-08-27T20:54:06.359 INFO:tasks.rados.rados.0.smithi093.stdout:2564: writing smithi093153278-433 from 1616352 to 2084497 tid 2 2023-08-27T20:54:06.359 INFO:tasks.rados.rados.0.smithi093.stdout:2561: done (3 left) 2023-08-27T20:54:06.359 INFO:tasks.rados.rados.0.smithi093.stdout:2565: write oid 144 current snap is 0 2023-08-27T20:54:06.359 INFO:tasks.rados.rados.0.smithi093.stdout:2562: finishing write tid 1 to smithi093153278-455 2023-08-27T20:54:06.359 INFO:tasks.rados.rados.0.smithi093.stdout:2565: seq_num 1672 ranges {501304=497635,1674834=426470} 2023-08-27T20:54:06.364 INFO:tasks.rados.rados.0.smithi093.stdout:2565: writing smithi093153278-144 from 501304 to 998939 tid 1 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2565: writing smithi093153278-144 from 1674834 to 2101304 tid 2 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2566: read oid 48 snap -1 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2566: expect (ObjNum 1630 snap 0 seq_num 1630) 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2567: write oid 78 current snap is 0 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2562: finishing write tid 2 to smithi093153278-455 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2562: finishing write tid 3 to smithi093153278-455 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2562: finishing write tid 4 to smithi093153278-455 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 455 v 415 (ObjNum 1670 snap 0 seq_num 1670) dirty exists 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2562: left oid 455 (ObjNum 1670 snap 0 seq_num 1670) 2023-08-27T20:54:06.371 INFO:tasks.rados.rados.0.smithi093.stdout:2567: seq_num 1673 ranges {518111=689237,1849669=268442} 2023-08-27T20:54:06.377 INFO:tasks.rados.rados.0.smithi093.stdout:2567: writing smithi093153278-78 from 518111 to 1207348 tid 1 2023-08-27T20:54:06.382 INFO:tasks.rados.rados.0.smithi093.stdout:2567: writing smithi093153278-78 from 1849669 to 2118111 tid 2 2023-08-27T20:54:06.382 INFO:tasks.rados.rados.0.smithi093.stdout:2562: done (5 left) 2023-08-27T20:54:06.382 INFO:tasks.rados.rados.0.smithi093.stdout:2568: write oid 218 current snap is 0 2023-08-27T20:54:06.383 INFO:tasks.rados.rados.0.smithi093.stdout:2563: expect (ObjNum 806 snap 0 seq_num 806) 2023-08-27T20:54:06.520 INFO:tasks.rados.rados.0.smithi093.stdout:2564: finishing write tid 1 to smithi093153278-433 2023-08-27T20:54:06.520 INFO:tasks.rados.rados.0.smithi093.stdout:2564: finishing write tid 2 to smithi093153278-433 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2564: finishing write tid 3 to smithi093153278-433 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2564: finishing write tid 4 to smithi093153278-433 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 433 v 376 (ObjNum 1671 snap 0 seq_num 1671) dirty exists 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2564: left oid 433 (ObjNum 1671 snap 0 seq_num 1671) 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2565: finishing write tid 1 to smithi093153278-144 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2565: finishing write tid 2 to smithi093153278-144 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2565: finishing write tid 3 to smithi093153278-144 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2565: finishing write tid 4 to smithi093153278-144 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 144 v 338 (ObjNum 1672 snap 0 seq_num 1672) dirty exists 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2565: left oid 144 (ObjNum 1672 snap 0 seq_num 1672) 2023-08-27T20:54:06.521 INFO:tasks.rados.rados.0.smithi093.stdout:2568: seq_num 1674 ranges {534918=764486,1791798=343120} 2023-08-27T20:54:06.527 INFO:tasks.rados.rados.0.smithi093.stdout:2568: writing smithi093153278-218 from 534918 to 1299404 tid 1 2023-08-27T20:54:06.532 INFO:tasks.rados.rados.0.smithi093.stdout:2568: writing smithi093153278-218 from 1791798 to 2134918 tid 2 2023-08-27T20:54:06.533 INFO:tasks.rados.rados.0.smithi093.stdout:2563: done (5 left) 2023-08-27T20:54:06.533 INFO:tasks.rados.rados.0.smithi093.stdout:2564: done (4 left) 2023-08-27T20:54:06.533 INFO:tasks.rados.rados.0.smithi093.stdout:2565: done (3 left) 2023-08-27T20:54:06.533 INFO:tasks.rados.rados.0.smithi093.stdout:2569: delete oid 43 current snap is 0 2023-08-27T20:54:06.533 INFO:tasks.rados.rados.0.smithi093.stdout:2566: expect (ObjNum 1630 snap 0 seq_num 1630) 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2567: finishing write tid 1 to smithi093153278-78 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2567: finishing write tid 2 to smithi093153278-78 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2567: finishing write tid 3 to smithi093153278-78 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2567: finishing write tid 4 to smithi093153278-78 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 303 (ObjNum 1673 snap 0 seq_num 1673) dirty exists 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2567: left oid 78 (ObjNum 1673 snap 0 seq_num 1673) 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2566: done (3 left) 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2567: done (2 left) 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2569: done (1 left) 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2570: write oid 419 current snap is 0 2023-08-27T20:54:06.692 INFO:tasks.rados.rados.0.smithi093.stdout:2568: finishing write tid 1 to smithi093153278-218 2023-08-27T20:54:06.693 INFO:tasks.rados.rados.0.smithi093.stdout:2568: finishing write tid 2 to smithi093153278-218 2023-08-27T20:54:06.693 INFO:tasks.rados.rados.0.smithi093.stdout:2568: finishing write tid 3 to smithi093153278-218 2023-08-27T20:54:06.693 INFO:tasks.rados.rados.0.smithi093.stdout:2568: finishing write tid 4 to smithi093153278-218 2023-08-27T20:54:06.693 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 218 v 316 (ObjNum 1674 snap 0 seq_num 1674) dirty exists 2023-08-27T20:54:06.693 INFO:tasks.rados.rados.0.smithi093.stdout:2568: left oid 218 (ObjNum 1674 snap 0 seq_num 1674) 2023-08-27T20:54:06.693 INFO:tasks.rados.rados.0.smithi093.stdout:2570: seq_num 1676 ranges {568532=514984,1508762=529912} 2023-08-27T20:54:06.697 INFO:tasks.rados.rados.0.smithi093.stdout:2570: writing smithi093153278-419 from 568532 to 1083516 tid 1 2023-08-27T20:54:06.702 INFO:tasks.rados.rados.0.smithi093.stdout:2570: writing smithi093153278-419 from 1508762 to 2038674 tid 2 2023-08-27T20:54:06.702 INFO:tasks.rados.rados.0.smithi093.stdout:2568: done (1 left) 2023-08-27T20:54:06.703 INFO:tasks.rados.rados.0.smithi093.stdout:2571: read oid 94 snap -1 2023-08-27T20:54:06.703 INFO:tasks.rados.rados.0.smithi093.stdout:2571: expect deleted 2023-08-27T20:54:06.703 INFO:tasks.rados.rados.0.smithi093.stdout:2572: delete oid 130 current snap is 0 2023-08-27T20:54:06.703 INFO:tasks.rados.rados.0.smithi093.stdout:2570: finishing write tid 1 to smithi093153278-419 2023-08-27T20:54:06.704 INFO:tasks.rados.rados.0.smithi093.stdout:2571: done (2 left) 2023-08-27T20:54:06.704 INFO:tasks.rados.rados.0.smithi093.stdout:2572: done (1 left) 2023-08-27T20:54:06.704 INFO:tasks.rados.rados.0.smithi093.stdout:2573: write oid 318 current snap is 0 2023-08-27T20:54:06.704 INFO:tasks.rados.rados.0.smithi093.stdout:2573: seq_num 1678 ranges {602146=665482,1909373=292773} 2023-08-27T20:54:06.711 INFO:tasks.rados.rados.0.smithi093.stdout:2573: writing smithi093153278-318 from 602146 to 1267628 tid 1 2023-08-27T20:54:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:2573: writing smithi093153278-318 from 1909373 to 2202146 tid 2 2023-08-27T20:54:06.716 INFO:tasks.rados.rados.0.smithi093.stdout:2574: read oid 272 snap -1 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:2574: expect (ObjNum 1388 snap 0 seq_num 1388) 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:2575: delete oid 468 current snap is 0 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:2570: finishing write tid 2 to smithi093153278-419 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:2570: finishing write tid 3 to smithi093153278-419 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:2570: finishing write tid 4 to smithi093153278-419 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 419 v 418 (ObjNum 1676 snap 0 seq_num 1676) dirty exists 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:2570: left oid 419 (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:54:06.717 INFO:tasks.rados.rados.0.smithi093.stdout:2573: finishing write tid 1 to smithi093153278-318 2023-08-27T20:54:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:2573: finishing write tid 2 to smithi093153278-318 2023-08-27T20:54:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:2573: finishing write tid 3 to smithi093153278-318 2023-08-27T20:54:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:2573: finishing write tid 4 to smithi093153278-318 2023-08-27T20:54:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 318 v 321 (ObjNum 1678 snap 0 seq_num 1678) dirty exists 2023-08-27T20:54:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:2573: left oid 318 (ObjNum 1678 snap 0 seq_num 1678) 2023-08-27T20:54:06.719 INFO:tasks.rados.rados.0.smithi093.stdout:2570: done (3 left) 2023-08-27T20:54:06.720 INFO:tasks.rados.rados.0.smithi093.stdout:2573: done (2 left) 2023-08-27T20:54:06.720 INFO:tasks.rados.rados.0.smithi093.stdout:2575: done (1 left) 2023-08-27T20:54:06.720 INFO:tasks.rados.rados.0.smithi093.stdout:2576: read oid 9 snap -1 2023-08-27T20:54:06.720 INFO:tasks.rados.rados.0.smithi093.stdout:2576: expect (ObjNum 1200 snap 0 seq_num 1200) 2023-08-27T20:54:06.720 INFO:tasks.rados.rados.0.smithi093.stdout:2577: write oid 295 current snap is 0 2023-08-27T20:54:06.720 INFO:tasks.rados.rados.0.smithi093.stdout:2577: seq_num 1680 ranges {635760=415980,1626337=537250} 2023-08-27T20:54:06.724 INFO:tasks.rados.rados.0.smithi093.stdout:2577: writing smithi093153278-295 from 635760 to 1051740 tid 1 2023-08-27T20:54:06.731 INFO:tasks.rados.rados.0.smithi093.stdout:2577: writing smithi093153278-295 from 1626337 to 2163587 tid 2 2023-08-27T20:54:06.731 INFO:tasks.rados.rados.0.smithi093.stdout:2578: write oid 219 current snap is 0 2023-08-27T20:54:06.731 INFO:tasks.rados.rados.0.smithi093.stdout:2578: seq_num 1681 ranges {652567=607582,1801172=451395} 2023-08-27T20:54:06.738 INFO:tasks.rados.rados.0.smithi093.stdout:2578: writing smithi093153278-219 from 652567 to 1260149 tid 1 2023-08-27T20:54:06.744 INFO:tasks.rados.rados.0.smithi093.stdout:2578: writing smithi093153278-219 from 1801172 to 2252567 tid 2 2023-08-27T20:54:06.744 INFO:tasks.rados.rados.0.smithi093.stdout:2579: read oid 163 snap -1 2023-08-27T20:54:06.744 INFO:tasks.rados.rados.0.smithi093.stdout:2579: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:54:06.744 INFO:tasks.rados.rados.0.smithi093.stdout:2580: delete oid 70 current snap is 0 2023-08-27T20:54:06.744 INFO:tasks.rados.rados.0.smithi093.stdout:2574: expect (ObjNum 1388 snap 0 seq_num 1388) 2023-08-27T20:54:06.887 INFO:tasks.rados.rados.0.smithi093.stdout:2576: expect (ObjNum 1200 snap 0 seq_num 1200) 2023-08-27T20:54:06.972 INFO:tasks.rados.rados.0.smithi093.stdout:2577: finishing write tid 1 to smithi093153278-295 2023-08-27T20:54:07.018 INFO:tasks.rados.rados.0.smithi093.stdout:2577: finishing write tid 2 to smithi093153278-295 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:2577: finishing write tid 3 to smithi093153278-295 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:2577: finishing write tid 4 to smithi093153278-295 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 295 v 306 (ObjNum 1680 snap 0 seq_num 1680) dirty exists 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:2577: left oid 295 (ObjNum 1680 snap 0 seq_num 1680) 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:2578: finishing write tid 1 to smithi093153278-219 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:2578: finishing write tid 2 to smithi093153278-219 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:2578: finishing write tid 3 to smithi093153278-219 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:2578: finishing write tid 4 to smithi093153278-219 2023-08-27T20:54:07.020 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 219 v 325 (ObjNum 1681 snap 0 seq_num 1681) dirty exists 2023-08-27T20:54:07.021 INFO:tasks.rados.rados.0.smithi093.stdout:2578: left oid 219 (ObjNum 1681 snap 0 seq_num 1681) 2023-08-27T20:54:07.021 INFO:tasks.rados.rados.0.smithi093.stdout:2579: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:54:07.145 INFO:tasks.rados.rados.0.smithi093.stdout:2574: done (5 left) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2576: done (4 left) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2577: done (3 left) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2578: done (2 left) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2579: done (1 left) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2580: done (0 left) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2581: read oid 364 snap -1 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2581: expect (ObjNum 1614 snap 0 seq_num 1614) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2582: read oid 383 snap -1 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2582: expect (ObjNum 1659 snap 0 seq_num 1659) 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2583: read oid 424 snap -1 2023-08-27T20:54:07.146 INFO:tasks.rados.rados.0.smithi093.stdout:2583: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:54:07.147 INFO:tasks.rados.rados.0.smithi093.stdout:2584: delete oid 381 current snap is 0 2023-08-27T20:54:07.147 INFO:tasks.rados.rados.0.smithi093.stdout:2584: done (3 left) 2023-08-27T20:54:07.148 INFO:tasks.rados.rados.0.smithi093.stdout:2585: read oid 443 snap -1 2023-08-27T20:54:07.148 INFO:tasks.rados.rados.0.smithi093.stdout:2585: expect (ObjNum 1507 snap 0 seq_num 1507) 2023-08-27T20:54:07.148 INFO:tasks.rados.rados.0.smithi093.stdout:2586: write oid 447 current snap is 0 2023-08-27T20:54:07.148 INFO:tasks.rados.rados.0.smithi093.stdout:2586: seq_num 1684 ranges {702988=433329,1860265=442723} 2023-08-27T20:54:07.154 INFO:tasks.rados.rados.0.smithi093.stdout:2586: writing smithi093153278-447 from 702988 to 1136317 tid 1 2023-08-27T20:54:07.161 INFO:tasks.rados.rados.0.smithi093.stdout:2586: writing smithi093153278-447 from 1860265 to 2302988 tid 2 2023-08-27T20:54:07.161 INFO:tasks.rados.rados.0.smithi093.stdout:2587: read oid 307 snap -1 2023-08-27T20:54:07.161 INFO:tasks.rados.rados.0.smithi093.stdout:2587: expect deleted 2023-08-27T20:54:07.162 INFO:tasks.rados.rados.0.smithi093.stdout:2588: write oid 235 current snap is 0 2023-08-27T20:54:07.162 INFO:tasks.rados.rados.0.smithi093.stdout:2588: seq_num 1685 ranges {719795=508578,1918747=401048} 2023-08-27T20:54:07.168 INFO:tasks.rados.rados.0.smithi093.stdout:2588: writing smithi093153278-235 from 719795 to 1228373 tid 1 2023-08-27T20:54:07.174 INFO:tasks.rados.rados.0.smithi093.stdout:2588: writing smithi093153278-235 from 1918747 to 2319795 tid 2 2023-08-27T20:54:07.174 INFO:tasks.rados.rados.0.smithi093.stdout:2589: write oid 51 current snap is 0 2023-08-27T20:54:07.175 INFO:tasks.rados.rados.0.smithi093.stdout:2581: expect (ObjNum 1614 snap 0 seq_num 1614) 2023-08-27T20:54:07.338 INFO:tasks.rados.rados.0.smithi093.stdout:2585: expect (ObjNum 1507 snap 0 seq_num 1507) 2023-08-27T20:54:07.522 INFO:tasks.rados.rados.0.smithi093.stdout:2582: expect (ObjNum 1659 snap 0 seq_num 1659) 2023-08-27T20:54:07.734 INFO:tasks.rados.rados.0.smithi093.stdout:2583: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:54:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:2586: finishing write tid 1 to smithi093153278-447 2023-08-27T20:54:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:2586: finishing write tid 2 to smithi093153278-447 2023-08-27T20:54:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:2586: finishing write tid 3 to smithi093153278-447 2023-08-27T20:54:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:2586: finishing write tid 4 to smithi093153278-447 2023-08-27T20:54:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 447 v 421 (ObjNum 1684 snap 0 seq_num 1684) dirty exists 2023-08-27T20:54:07.914 INFO:tasks.rados.rados.0.smithi093.stdout:2586: left oid 447 (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:54:07.915 INFO:tasks.rados.rados.0.smithi093.stdout:2588: finishing write tid 1 to smithi093153278-235 2023-08-27T20:54:07.915 INFO:tasks.rados.rados.0.smithi093.stdout:2589: seq_num 1686 ranges {736602=583827,1860876=475726} 2023-08-27T20:54:07.920 INFO:tasks.rados.rados.0.smithi093.stdout:2589: writing smithi093153278-51 from 736602 to 1320429 tid 1 2023-08-27T20:54:07.927 INFO:tasks.rados.rados.0.smithi093.stdout:2589: writing smithi093153278-51 from 1860876 to 2336602 tid 2 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2581: done (7 left) 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2582: done (6 left) 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2583: done (5 left) 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2585: done (4 left) 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2586: done (3 left) 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2587: done (2 left) 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2590: read oid 195 snap -1 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2590: expect (ObjNum 1596 snap 0 seq_num 1596) 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2591: write oid 152 current snap is 0 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2588: finishing write tid 2 to smithi093153278-235 2023-08-27T20:54:07.928 INFO:tasks.rados.rados.0.smithi093.stdout:2588: finishing write tid 3 to smithi093153278-235 2023-08-27T20:54:07.929 INFO:tasks.rados.rados.0.smithi093.stdout:2588: finishing write tid 4 to smithi093153278-235 2023-08-27T20:54:07.929 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 235 v 317 (ObjNum 1685 snap 0 seq_num 1685) dirty exists 2023-08-27T20:54:07.929 INFO:tasks.rados.rados.0.smithi093.stdout:2588: left oid 235 (ObjNum 1685 snap 0 seq_num 1685) 2023-08-27T20:54:07.929 INFO:tasks.rados.rados.0.smithi093.stdout:2591: seq_num 1687 ranges {753409=659076,1919358=434051} 2023-08-27T20:54:07.935 INFO:tasks.rados.rados.0.smithi093.stdout:2591: writing smithi093153278-152 from 753409 to 1412485 tid 1 2023-08-27T20:54:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:07 smithi136 ceph-mon[113531]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 84 MiB/s rd, 11 MiB/s wr, 113 op/s 2023-08-27T20:54:07.940 INFO:tasks.rados.rados.0.smithi093.stdout:2591: writing smithi093153278-152 from 1919358 to 2353409 tid 2 2023-08-27T20:54:07.940 INFO:tasks.rados.rados.0.smithi093.stdout:2588: done (3 left) 2023-08-27T20:54:07.941 INFO:tasks.rados.rados.0.smithi093.stdout:2592: write oid 368 current snap is 0 2023-08-27T20:54:07.941 INFO:tasks.rados.rados.0.smithi093.stdout:2589: finishing write tid 1 to smithi093153278-51 2023-08-27T20:54:07.941 INFO:tasks.rados.rados.0.smithi093.stdout:2592: seq_num 1688 ranges {770216=450678,1694193=551926} 2023-08-27T20:54:07.947 INFO:tasks.rados.rados.0.smithi093.stdout:2592: writing smithi093153278-368 from 770216 to 1220894 tid 1 2023-08-27T20:54:07.953 INFO:tasks.rados.rados.0.smithi093.stdout:2592: writing smithi093153278-368 from 1694193 to 2246119 tid 2 2023-08-27T20:54:07.953 INFO:tasks.rados.rados.0.smithi093.stdout:2593: write oid 207 current snap is 0 2023-08-27T20:54:07.953 INFO:tasks.rados.rados.0.smithi093.stdout:2593: seq_num 1689 ranges {787023=525927,1752675=634348} 2023-08-27T20:54:07.960 INFO:tasks.rados.rados.0.smithi093.stdout:2593: writing smithi093153278-207 from 787023 to 1312950 tid 1 2023-08-27T20:54:07.966 INFO:tasks.rados.rados.0.smithi093.stdout:2593: writing smithi093153278-207 from 1752675 to 2387023 tid 2 2023-08-27T20:54:07.966 INFO:tasks.rados.rados.0.smithi093.stdout:2594: write oid 226 current snap is 0 2023-08-27T20:54:07.967 INFO:tasks.rados.rados.0.smithi093.stdout:2589: finishing write tid 2 to smithi093153278-51 2023-08-27T20:54:07.967 INFO:tasks.rados.rados.0.smithi093.stdout:2589: finishing write tid 3 to smithi093153278-51 2023-08-27T20:54:07.967 INFO:tasks.rados.rados.0.smithi093.stdout:2589: finishing write tid 4 to smithi093153278-51 2023-08-27T20:54:07.967 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 51 v 309 (ObjNum 1686 snap 0 seq_num 1686) dirty exists 2023-08-27T20:54:07.967 INFO:tasks.rados.rados.0.smithi093.stdout:2589: left oid 51 (ObjNum 1686 snap 0 seq_num 1686) 2023-08-27T20:54:07.967 INFO:tasks.rados.rados.0.smithi093.stdout:2594: seq_num 1690 ranges {403830=601176,1694804=555595} 2023-08-27T20:54:07.972 INFO:tasks.rados.rados.0.smithi093.stdout:2594: writing smithi093153278-226 from 403830 to 1005006 tid 1 2023-08-27T20:54:07.979 INFO:tasks.rados.rados.0.smithi093.stdout:2594: writing smithi093153278-226 from 1694804 to 2250399 tid 2 2023-08-27T20:54:07.979 INFO:tasks.rados.rados.0.smithi093.stdout:2589: done (5 left) 2023-08-27T20:54:07.979 INFO:tasks.rados.rados.0.smithi093.stdout:2595: read oid 171 snap -1 2023-08-27T20:54:07.979 INFO:tasks.rados.rados.0.smithi093.stdout:2595: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:54:07.979 INFO:tasks.rados.rados.0.smithi093.stdout:2596: read oid 170 snap -1 2023-08-27T20:54:07.979 INFO:tasks.rados.rados.0.smithi093.stdout:2596: expect (ObjNum 1431 snap 0 seq_num 1431) 2023-08-27T20:54:07.980 INFO:tasks.rados.rados.0.smithi093.stdout:2597: write oid 257 current snap is 0 2023-08-27T20:54:07.980 INFO:tasks.rados.rados.0.smithi093.stdout:2597: seq_num 1691 ranges {420637=676425,1753286=667351} 2023-08-27T20:54:07.985 INFO:tasks.rados.rados.0.smithi093.stdout:2597: writing smithi093153278-257 from 420637 to 1097062 tid 1 2023-08-27T20:54:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:07 smithi093 ceph-mon[111335]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 84 MiB/s rd, 11 MiB/s wr, 113 op/s 2023-08-27T20:54:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:07 smithi093 ceph-mon[118902]: pgmap v17: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 84 MiB/s rd, 11 MiB/s wr, 113 op/s 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2597: writing smithi093153278-257 from 1753286 to 2420637 tid 2 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2598: read oid 224 snap -1 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2598: expect (ObjNum 1358 snap 0 seq_num 1358) 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2599: read oid 423 snap -1 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2599: expect (ObjNum 422 snap 0 seq_num 422) 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2600: read oid 358 snap -1 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2600: expect (ObjNum 806 snap 0 seq_num 806) 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2601: read oid 16 snap -1 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2601: expect (ObjNum 1491 snap 0 seq_num 1491) 2023-08-27T20:54:07.994 INFO:tasks.rados.rados.0.smithi093.stdout:2602: read oid 63 snap -1 2023-08-27T20:54:07.995 INFO:tasks.rados.rados.0.smithi093.stdout:2602: expect deleted 2023-08-27T20:54:07.995 INFO:tasks.rados.rados.0.smithi093.stdout:2603: read oid 405 snap -1 2023-08-27T20:54:07.995 INFO:tasks.rados.rados.0.smithi093.stdout:2603: expect deleted 2023-08-27T20:54:07.995 INFO:tasks.rados.rados.0.smithi093.stdout:2604: read oid 91 snap -1 2023-08-27T20:54:07.995 INFO:tasks.rados.rados.0.smithi093.stdout:2604: expect (ObjNum 826 snap 0 seq_num 826) 2023-08-27T20:54:07.995 INFO:tasks.rados.rados.0.smithi093.stdout:2590: expect (ObjNum 1596 snap 0 seq_num 1596) 2023-08-27T20:54:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:2591: finishing write tid 1 to smithi093153278-152 2023-08-27T20:54:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:2591: finishing write tid 2 to smithi093153278-152 2023-08-27T20:54:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:2591: finishing write tid 3 to smithi093153278-152 2023-08-27T20:54:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:2591: finishing write tid 4 to smithi093153278-152 2023-08-27T20:54:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 152 v 350 (ObjNum 1687 snap 0 seq_num 1687) dirty exists 2023-08-27T20:54:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:2591: left oid 152 (ObjNum 1687 snap 0 seq_num 1687) 2023-08-27T20:54:08.148 INFO:tasks.rados.rados.0.smithi093.stdout:2592: finishing write tid 1 to smithi093153278-368 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2592: finishing write tid 2 to smithi093153278-368 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2592: finishing write tid 3 to smithi093153278-368 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2590: done (14 left) 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2591: done (13 left) 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2605: read oid 258 snap -1 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2605: expect (ObjNum 1353 snap 0 seq_num 1353) 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2606: write oid 92 current snap is 0 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2592: finishing write tid 4 to smithi093153278-368 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 368 v 324 (ObjNum 1688 snap 0 seq_num 1688) dirty exists 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2592: left oid 368 (ObjNum 1688 snap 0 seq_num 1688) 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2593: finishing write tid 1 to smithi093153278-207 2023-08-27T20:54:08.149 INFO:tasks.rados.rados.0.smithi093.stdout:2606: seq_num 1692 ranges {437444=751674,1811768=559264} 2023-08-27T20:54:08.155 INFO:tasks.rados.rados.0.smithi093.stdout:2606: writing smithi093153278-92 from 437444 to 1189118 tid 1 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2606: writing smithi093153278-92 from 1811768 to 2371032 tid 2 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2592: done (14 left) 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2607: write oid 82 current snap is 0 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2593: finishing write tid 2 to smithi093153278-207 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2593: finishing write tid 3 to smithi093153278-207 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2593: finishing write tid 4 to smithi093153278-207 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 207 v 340 (ObjNum 1689 snap 0 seq_num 1689) dirty exists 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2593: left oid 207 (ObjNum 1689 snap 0 seq_num 1689) 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2594: finishing write tid 1 to smithi093153278-226 2023-08-27T20:54:08.162 INFO:tasks.rados.rados.0.smithi093.stdout:2594: finishing write tid 2 to smithi093153278-226 2023-08-27T20:54:08.163 INFO:tasks.rados.rados.0.smithi093.stdout:2594: finishing write tid 3 to smithi093153278-226 2023-08-27T20:54:08.163 INFO:tasks.rados.rados.0.smithi093.stdout:2594: finishing write tid 4 to smithi093153278-226 2023-08-27T20:54:08.163 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 226 v 353 (ObjNum 1690 snap 0 seq_num 1690) dirty exists 2023-08-27T20:54:08.163 INFO:tasks.rados.rados.0.smithi093.stdout:2594: left oid 226 (ObjNum 1690 snap 0 seq_num 1690) 2023-08-27T20:54:08.163 INFO:tasks.rados.rados.0.smithi093.stdout:2607: seq_num 1693 ranges {454251=426923,1470250=702922} 2023-08-27T20:54:08.166 INFO:tasks.rados.rados.0.smithi093.stdout:2607: writing smithi093153278-82 from 454251 to 881174 tid 1 2023-08-27T20:54:08.173 INFO:tasks.rados.rados.0.smithi093.stdout:2607: writing smithi093153278-82 from 1470250 to 2173172 tid 2 2023-08-27T20:54:08.173 INFO:tasks.rados.rados.0.smithi093.stdout:2593: done (14 left) 2023-08-27T20:54:08.173 INFO:tasks.rados.rados.0.smithi093.stdout:2594: done (13 left) 2023-08-27T20:54:08.174 INFO:tasks.rados.rados.0.smithi093.stdout:2608: write oid 197 current snap is 0 2023-08-27T20:54:08.174 INFO:tasks.rados.rados.0.smithi093.stdout:2608: seq_num 1694 ranges {471058=502172,1412379=446580} 2023-08-27T20:54:08.178 INFO:tasks.rados.rados.0.smithi093.stdout:2608: writing smithi093153278-197 from 471058 to 973230 tid 1 2023-08-27T20:54:08.184 INFO:tasks.rados.rados.0.smithi093.stdout:2608: writing smithi093153278-197 from 1412379 to 1858959 tid 2 2023-08-27T20:54:08.184 INFO:tasks.rados.rados.0.smithi093.stdout:2609: write oid 484 current snap is 0 2023-08-27T20:54:08.184 INFO:tasks.rados.rados.0.smithi093.stdout:2597: finishing write tid 1 to smithi093153278-257 2023-08-27T20:54:08.184 INFO:tasks.rados.rados.0.smithi093.stdout:2596: expect (ObjNum 1431 snap 0 seq_num 1431) 2023-08-27T20:54:08.269 INFO:tasks.rados.rados.0.smithi093.stdout:2595: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:54:08.358 INFO:tasks.rados.rados.0.smithi093.stdout:2597: finishing write tid 2 to smithi093153278-257 2023-08-27T20:54:08.358 INFO:tasks.rados.rados.0.smithi093.stdout:2597: finishing write tid 3 to smithi093153278-257 2023-08-27T20:54:08.358 INFO:tasks.rados.rados.0.smithi093.stdout:2597: finishing write tid 4 to smithi093153278-257 2023-08-27T20:54:08.358 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 257 v 460 (ObjNum 1691 snap 0 seq_num 1691) dirty exists 2023-08-27T20:54:08.358 INFO:tasks.rados.rados.0.smithi093.stdout:2597: left oid 257 (ObjNum 1691 snap 0 seq_num 1691) 2023-08-27T20:54:08.359 INFO:tasks.rados.rados.0.smithi093.stdout:2600: expect (ObjNum 806 snap 0 seq_num 806) 2023-08-27T20:54:08.496 INFO:tasks.rados.rados.0.smithi093.stdout:2599: expect (ObjNum 422 snap 0 seq_num 422) 2023-08-27T20:54:08.593 INFO:tasks.rados.rados.0.smithi093.stdout:2598: expect (ObjNum 1358 snap 0 seq_num 1358) 2023-08-27T20:54:08.746 INFO:tasks.rados.rados.0.smithi093.stdout:2601: expect (ObjNum 1491 snap 0 seq_num 1491) 2023-08-27T20:54:08.886 INFO:tasks.rados.rados.0.smithi093.stdout:2605: expect (ObjNum 1353 snap 0 seq_num 1353) 2023-08-27T20:54:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4171045204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:08.974 INFO:tasks.rados.rados.0.smithi093.stdout:2604: expect (ObjNum 826 snap 0 seq_num 826) 2023-08-27T20:54:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4171045204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4171045204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:09.156 INFO:tasks.rados.rados.0.smithi093.stdout:2606: finishing write tid 1 to smithi093153278-92 2023-08-27T20:54:09.156 INFO:tasks.rados.rados.0.smithi093.stdout:2606: finishing write tid 2 to smithi093153278-92 2023-08-27T20:54:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:2606: finishing write tid 3 to smithi093153278-92 2023-08-27T20:54:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:2606: finishing write tid 4 to smithi093153278-92 2023-08-27T20:54:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 92 v 303 (ObjNum 1692 snap 0 seq_num 1692) dirty exists 2023-08-27T20:54:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:2606: left oid 92 (ObjNum 1692 snap 0 seq_num 1692) 2023-08-27T20:54:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:2607: finishing write tid 1 to smithi093153278-82 2023-08-27T20:54:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:2609: seq_num 1695 ranges {487865=577421,1470861=706591} 2023-08-27T20:54:09.162 INFO:tasks.rados.rados.0.smithi093.stdout:2609: writing smithi093153278-484 from 487865 to 1065286 tid 1 2023-08-27T20:54:09.169 INFO:tasks.rados.rados.0.smithi093.stdout:2609: writing smithi093153278-484 from 1470861 to 2177452 tid 2 2023-08-27T20:54:09.169 INFO:tasks.rados.rados.0.smithi093.stdout:2595: done (14 left) 2023-08-27T20:54:09.169 INFO:tasks.rados.rados.0.smithi093.stdout:2596: done (13 left) 2023-08-27T20:54:09.169 INFO:tasks.rados.rados.0.smithi093.stdout:2597: done (12 left) 2023-08-27T20:54:09.169 INFO:tasks.rados.rados.0.smithi093.stdout:2598: done (11 left) 2023-08-27T20:54:09.169 INFO:tasks.rados.rados.0.smithi093.stdout:2599: done (10 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2600: done (9 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2601: done (8 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2602: done (7 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2603: done (6 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2604: done (5 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2605: done (4 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2606: done (3 left) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2610: read oid 316 snap -1 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2610: expect (ObjNum 1608 snap 0 seq_num 1608) 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2611: write oid 181 current snap is 0 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2607: finishing write tid 2 to smithi093153278-82 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2607: finishing write tid 3 to smithi093153278-82 2023-08-27T20:54:09.170 INFO:tasks.rados.rados.0.smithi093.stdout:2607: finishing write tid 4 to smithi093153278-82 2023-08-27T20:54:09.171 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 82 v 342 (ObjNum 1693 snap 0 seq_num 1693) dirty exists 2023-08-27T20:54:09.171 INFO:tasks.rados.rados.0.smithi093.stdout:2607: left oid 82 (ObjNum 1693 snap 0 seq_num 1693) 2023-08-27T20:54:09.171 INFO:tasks.rados.rados.0.smithi093.stdout:2608: finishing write tid 1 to smithi093153278-197 2023-08-27T20:54:09.171 INFO:tasks.rados.rados.0.smithi093.stdout:2608: finishing write tid 2 to smithi093153278-197 2023-08-27T20:54:09.171 INFO:tasks.rados.rados.0.smithi093.stdout:2611: seq_num 1696 ranges {504672=769023,2045696=450249} 2023-08-27T20:54:09.175 INFO:tasks.rados.rados.0.smithi093.stdout:2611: writing smithi093153278-181 from 504672 to 1273695 tid 1 2023-08-27T20:54:09.183 INFO:tasks.rados.rados.0.smithi093.stdout:2611: writing smithi093153278-181 from 2045696 to 2495945 tid 2 2023-08-27T20:54:09.183 INFO:tasks.rados.rados.0.smithi093.stdout:2607: done (4 left) 2023-08-27T20:54:09.183 INFO:tasks.rados.rados.0.smithi093.stdout:2612: read oid 83 snap -1 2023-08-27T20:54:09.183 INFO:tasks.rados.rados.0.smithi093.stdout:2612: expect (ObjNum 1505 snap 0 seq_num 1505) 2023-08-27T20:54:09.183 INFO:tasks.rados.rados.0.smithi093.stdout:2613: read oid 331 snap -1 2023-08-27T20:54:09.183 INFO:tasks.rados.rados.0.smithi093.stdout:2613: expect (ObjNum 1013 snap 0 seq_num 1013) 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2614: read oid 303 snap -1 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2614: expect deleted 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2615: read oid 128 snap -1 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2615: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2616: read oid 205 snap -1 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2616: expect (ObjNum 1554 snap 0 seq_num 1554) 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2617: write oid 436 current snap is 0 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2608: finishing write tid 3 to smithi093153278-197 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2608: finishing write tid 4 to smithi093153278-197 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 197 v 327 (ObjNum 1694 snap 0 seq_num 1694) dirty exists 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2608: left oid 197 (ObjNum 1694 snap 0 seq_num 1694) 2023-08-27T20:54:09.184 INFO:tasks.rados.rados.0.smithi093.stdout:2609: finishing write tid 1 to smithi093153278-484 2023-08-27T20:54:09.185 INFO:tasks.rados.rados.0.smithi093.stdout:2609: finishing write tid 2 to smithi093153278-484 2023-08-27T20:54:09.185 INFO:tasks.rados.rados.0.smithi093.stdout:2609: finishing write tid 3 to smithi093153278-484 2023-08-27T20:54:09.185 INFO:tasks.rados.rados.0.smithi093.stdout:2609: finishing write tid 4 to smithi093153278-484 2023-08-27T20:54:09.185 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 484 v 463 (ObjNum 1695 snap 0 seq_num 1695) dirty exists 2023-08-27T20:54:09.185 INFO:tasks.rados.rados.0.smithi093.stdout:2609: left oid 484 (ObjNum 1695 snap 0 seq_num 1695) 2023-08-27T20:54:09.185 INFO:tasks.rados.rados.0.smithi093.stdout:2610: expect (ObjNum 1608 snap 0 seq_num 1608) 2023-08-27T20:54:09.367 INFO:tasks.rados.rados.0.smithi093.stdout:2611: finishing write tid 1 to smithi093153278-181 2023-08-27T20:54:09.367 INFO:tasks.rados.rados.0.smithi093.stdout:2612: expect (ObjNum 1505 snap 0 seq_num 1505) 2023-08-27T20:54:09.551 INFO:tasks.rados.rados.0.smithi093.stdout:2616: expect (ObjNum 1554 snap 0 seq_num 1554) 2023-08-27T20:54:09.689 INFO:tasks.rados.rados.0.smithi093.stdout:2611: finishing write tid 2 to smithi093153278-181 2023-08-27T20:54:09.689 INFO:tasks.rados.rados.0.smithi093.stdout:2611: finishing write tid 3 to smithi093153278-181 2023-08-27T20:54:09.689 INFO:tasks.rados.rados.0.smithi093.stdout:2611: finishing write tid 4 to smithi093153278-181 2023-08-27T20:54:09.689 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 181 v 310 (ObjNum 1696 snap 0 seq_num 1696) dirty exists 2023-08-27T20:54:09.689 INFO:tasks.rados.rados.0.smithi093.stdout:2611: left oid 181 (ObjNum 1696 snap 0 seq_num 1696) 2023-08-27T20:54:09.689 INFO:tasks.rados.rados.0.smithi093.stdout:2613: expect (ObjNum 1013 snap 0 seq_num 1013) 2023-08-27T20:54:09.864 INFO:tasks.rados.rados.0.smithi093.stdout:2615: expect (ObjNum 127 snap 0 seq_num 127) 2023-08-27T20:54:09.935 INFO:tasks.rados.rados.0.smithi093.stdout:2617: seq_num 1697 ranges {521479=444272,1704178=710260} 2023-08-27T20:54:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:09 smithi136 ceph-mon[113531]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 MiB/s rd, 7.2 MiB/s wr, 75 op/s 2023-08-27T20:54:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:09.940 INFO:tasks.rados.rados.0.smithi093.stdout:2617: writing smithi093153278-436 from 521479 to 965751 tid 1 2023-08-27T20:54:09.948 INFO:tasks.rados.rados.0.smithi093.stdout:2617: writing smithi093153278-436 from 1704178 to 2414438 tid 2 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2608: done (9 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2609: done (8 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2610: done (7 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2611: done (6 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2612: done (5 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2613: done (4 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2614: done (3 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2615: done (2 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2616: done (1 left) 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2618: write oid 412 current snap is 0 2023-08-27T20:54:09.949 INFO:tasks.rados.rados.0.smithi093.stdout:2617: finishing write tid 1 to smithi093153278-436 2023-08-27T20:54:09.950 INFO:tasks.rados.rados.0.smithi093.stdout:2618: seq_num 1698 ranges {538286=519521,1762660=453918} 2023-08-27T20:54:09.954 INFO:tasks.rados.rados.0.smithi093.stdout:2618: writing smithi093153278-412 from 538286 to 1057807 tid 1 2023-08-27T20:54:09.961 INFO:tasks.rados.rados.0.smithi093.stdout:2618: writing smithi093153278-412 from 1762660 to 2216578 tid 2 2023-08-27T20:54:09.961 INFO:tasks.rados.rados.0.smithi093.stdout:2619: read oid 92 snap -1 2023-08-27T20:54:09.961 INFO:tasks.rados.rados.0.smithi093.stdout:2619: expect (ObjNum 1692 snap 0 seq_num 1692) 2023-08-27T20:54:09.961 INFO:tasks.rados.rados.0.smithi093.stdout:2620: read oid 416 snap -1 2023-08-27T20:54:09.961 INFO:tasks.rados.rados.0.smithi093.stdout:2620: expect (ObjNum 1191 snap 0 seq_num 1191) 2023-08-27T20:54:09.961 INFO:tasks.rados.rados.0.smithi093.stdout:2621: write oid 444 current snap is 0 2023-08-27T20:54:09.962 INFO:tasks.rados.rados.0.smithi093.stdout:2617: finishing write tid 2 to smithi093153278-436 2023-08-27T20:54:09.962 INFO:tasks.rados.rados.0.smithi093.stdout:2617: finishing write tid 3 to smithi093153278-436 2023-08-27T20:54:09.962 INFO:tasks.rados.rados.0.smithi093.stdout:2617: finishing write tid 4 to smithi093153278-436 2023-08-27T20:54:09.962 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 436 v 233 (ObjNum 1697 snap 0 seq_num 1697) dirty exists 2023-08-27T20:54:09.962 INFO:tasks.rados.rados.0.smithi093.stdout:2617: left oid 436 (ObjNum 1697 snap 0 seq_num 1697) 2023-08-27T20:54:09.962 INFO:tasks.rados.rados.0.smithi093.stdout:2618: finishing write tid 1 to smithi093153278-412 2023-08-27T20:54:09.962 INFO:tasks.rados.rados.0.smithi093.stdout:2621: seq_num 1699 ranges {555093=594770,1704789=713929} 2023-08-27T20:54:09.967 INFO:tasks.rados.rados.0.smithi093.stdout:2621: writing smithi093153278-444 from 555093 to 1149863 tid 1 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:2621: writing smithi093153278-444 from 1704789 to 2418718 tid 2 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:2617: done (4 left) 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:2622: write oid 325 current snap is 0 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:2618: finishing write tid 2 to smithi093153278-412 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:2618: finishing write tid 3 to smithi093153278-412 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:2618: finishing write tid 4 to smithi093153278-412 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 412 v 313 (ObjNum 1698 snap 0 seq_num 1698) dirty exists 2023-08-27T20:54:09.975 INFO:tasks.rados.rados.0.smithi093.stdout:2618: left oid 412 (ObjNum 1698 snap 0 seq_num 1698) 2023-08-27T20:54:09.976 INFO:tasks.rados.rados.0.smithi093.stdout:2619: expect (ObjNum 1692 snap 0 seq_num 1692) 2023-08-27T20:54:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:09 smithi093 ceph-mon[111335]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 MiB/s rd, 7.2 MiB/s wr, 75 op/s 2023-08-27T20:54:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:09 smithi093 ceph-mon[118902]: pgmap v18: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 54 MiB/s rd, 7.2 MiB/s wr, 75 op/s 2023-08-27T20:54:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:10.060 INFO:tasks.rados.rados.0.smithi093.stdout:2621: finishing write tid 1 to smithi093153278-444 2023-08-27T20:54:10.060 INFO:tasks.rados.rados.0.smithi093.stdout:2622: seq_num 1700 ranges {571900=670019,1763271=457587} 2023-08-27T20:54:10.067 INFO:tasks.rados.rados.0.smithi093.stdout:2622: writing smithi093153278-325 from 571900 to 1241919 tid 1 2023-08-27T20:54:10.073 INFO:tasks.rados.rados.0.smithi093.stdout:2622: writing smithi093153278-325 from 1763271 to 2220858 tid 2 2023-08-27T20:54:10.073 INFO:tasks.rados.rados.0.smithi093.stdout:2618: done (4 left) 2023-08-27T20:54:10.073 INFO:tasks.rados.rados.0.smithi093.stdout:2619: done (3 left) 2023-08-27T20:54:10.073 INFO:tasks.rados.rados.0.smithi093.stdout:2623: write oid 274 current snap is 0 2023-08-27T20:54:10.073 INFO:tasks.rados.rados.0.smithi093.stdout:2620: expect (ObjNum 1191 snap 0 seq_num 1191) 2023-08-27T20:54:10.150 INFO:tasks.rados.rados.0.smithi093.stdout:2621: finishing write tid 2 to smithi093153278-444 2023-08-27T20:54:10.150 INFO:tasks.rados.rados.0.smithi093.stdout:2621: finishing write tid 3 to smithi093153278-444 2023-08-27T20:54:10.150 INFO:tasks.rados.rados.0.smithi093.stdout:2621: finishing write tid 4 to smithi093153278-444 2023-08-27T20:54:10.151 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 444 v 343 (ObjNum 1699 snap 0 seq_num 1699) dirty exists 2023-08-27T20:54:10.151 INFO:tasks.rados.rados.0.smithi093.stdout:2621: left oid 444 (ObjNum 1699 snap 0 seq_num 1699) 2023-08-27T20:54:10.151 INFO:tasks.rados.rados.0.smithi093.stdout:2623: seq_num 1701 ranges {588707=745268,1821753=717598} 2023-08-27T20:54:10.157 INFO:tasks.rados.rados.0.smithi093.stdout:2623: writing smithi093153278-274 from 588707 to 1333975 tid 1 2023-08-27T20:54:10.164 INFO:tasks.rados.rados.0.smithi093.stdout:2623: writing smithi093153278-274 from 1821753 to 2539351 tid 2 2023-08-27T20:54:10.165 INFO:tasks.rados.rados.0.smithi093.stdout:2620: done (3 left) 2023-08-27T20:54:10.165 INFO:tasks.rados.rados.0.smithi093.stdout:2621: done (2 left) 2023-08-27T20:54:10.165 INFO:tasks.rados.rados.0.smithi093.stdout:2624: write oid 311 current snap is 0 2023-08-27T20:54:10.165 INFO:tasks.rados.rados.0.smithi093.stdout:2622: finishing write tid 1 to smithi093153278-325 2023-08-27T20:54:10.165 INFO:tasks.rados.rados.0.smithi093.stdout:2624: seq_num 1702 ranges {605514=420517,1480235=461256,2356169=249345} 2023-08-27T20:54:10.170 INFO:tasks.rados.rados.0.smithi093.stdout:2624: writing smithi093153278-311 from 605514 to 1026031 tid 1 2023-08-27T20:54:10.175 INFO:tasks.rados.rados.0.smithi093.stdout:2624: writing smithi093153278-311 from 1480235 to 1941491 tid 2 2023-08-27T20:54:10.179 INFO:tasks.rados.rados.0.smithi093.stdout:2624: writing smithi093153278-311 from 2356169 to 2605514 tid 3 2023-08-27T20:54:10.179 INFO:tasks.rados.rados.0.smithi093.stdout:2625: write oid 496 current snap is 0 2023-08-27T20:54:10.179 INFO:tasks.rados.rados.0.smithi093.stdout:2622: finishing write tid 2 to smithi093153278-325 2023-08-27T20:54:10.179 INFO:tasks.rados.rados.0.smithi093.stdout:2622: finishing write tid 3 to smithi093153278-325 2023-08-27T20:54:10.180 INFO:tasks.rados.rados.0.smithi093.stdout:2622: finishing write tid 4 to smithi093153278-325 2023-08-27T20:54:10.180 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 325 v 380 (ObjNum 1700 snap 0 seq_num 1700) dirty exists 2023-08-27T20:54:10.180 INFO:tasks.rados.rados.0.smithi093.stdout:2622: left oid 325 (ObjNum 1700 snap 0 seq_num 1700) 2023-08-27T20:54:10.180 INFO:tasks.rados.rados.0.smithi093.stdout:2623: finishing write tid 1 to smithi093153278-274 2023-08-27T20:54:10.180 INFO:tasks.rados.rados.0.smithi093.stdout:2623: finishing write tid 2 to smithi093153278-274 2023-08-27T20:54:10.180 INFO:tasks.rados.rados.0.smithi093.stdout:2625: seq_num 1703 ranges {622321=612119,1938717=683604} 2023-08-27T20:54:10.186 INFO:tasks.rados.rados.0.smithi093.stdout:2625: writing smithi093153278-496 from 622321 to 1234440 tid 1 2023-08-27T20:54:10.194 INFO:tasks.rados.rados.0.smithi093.stdout:2625: writing smithi093153278-496 from 1938717 to 2622321 tid 2 2023-08-27T20:54:10.194 INFO:tasks.rados.rados.0.smithi093.stdout:2622: done (3 left) 2023-08-27T20:54:10.194 INFO:tasks.rados.rados.0.smithi093.stdout:2626: write oid 66 current snap is 0 2023-08-27T20:54:10.194 INFO:tasks.rados.rados.0.smithi093.stdout:2623: finishing write tid 3 to smithi093153278-274 2023-08-27T20:54:10.194 INFO:tasks.rados.rados.0.smithi093.stdout:2623: finishing write tid 4 to smithi093153278-274 2023-08-27T20:54:10.194 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 274 v 346 (ObjNum 1701 snap 0 seq_num 1701) dirty exists 2023-08-27T20:54:10.194 INFO:tasks.rados.rados.0.smithi093.stdout:2623: left oid 274 (ObjNum 1701 snap 0 seq_num 1701) 2023-08-27T20:54:10.195 INFO:tasks.rados.rados.0.smithi093.stdout:2624: finishing write tid 1 to smithi093153278-311 2023-08-27T20:54:10.195 INFO:tasks.rados.rados.0.smithi093.stdout:2624: finishing write tid 2 to smithi093153278-311 2023-08-27T20:54:10.195 INFO:tasks.rados.rados.0.smithi093.stdout:2626: seq_num 1704 ranges {639128=687368,1997199=464925} 2023-08-27T20:54:10.201 INFO:tasks.rados.rados.0.smithi093.stdout:2626: writing smithi093153278-66 from 639128 to 1326496 tid 1 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2626: writing smithi093153278-66 from 1997199 to 2462124 tid 2 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2623: done (3 left) 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2627: read oid 80 snap -1 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2627: expect (ObjNum 744 snap 0 seq_num 744) 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2628: read oid 74 snap -1 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2628: expect (ObjNum 1336 snap 0 seq_num 1336) 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2629: read oid 406 snap -1 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2629: expect (ObjNum 1661 snap 0 seq_num 1661) 2023-08-27T20:54:10.208 INFO:tasks.rados.rados.0.smithi093.stdout:2630: write oid 259 current snap is 0 2023-08-27T20:54:10.209 INFO:tasks.rados.rados.0.smithi093.stdout:2630: seq_num 1705 ranges {655935=762617,2055681=600254} 2023-08-27T20:54:10.215 INFO:tasks.rados.rados.0.smithi093.stdout:2630: writing smithi093153278-259 from 655935 to 1418552 tid 1 2023-08-27T20:54:10.223 INFO:tasks.rados.rados.0.smithi093.stdout:2630: writing smithi093153278-259 from 2055681 to 2655935 tid 2 2023-08-27T20:54:10.223 INFO:tasks.rados.rados.0.smithi093.stdout:2631: read oid 104 snap -1 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2631: expect (ObjNum 1450 snap 0 seq_num 1450) 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2632: read oid 354 snap -1 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2632: expect deleted 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2633: delete oid 225 current snap is 0 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2624: finishing write tid 3 to smithi093153278-311 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2624: finishing write tid 4 to smithi093153278-311 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2624: finishing write tid 5 to smithi093153278-311 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 311 v 320 (ObjNum 1702 snap 0 seq_num 1702) dirty exists 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2624: left oid 311 (ObjNum 1702 snap 0 seq_num 1702) 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2625: finishing write tid 1 to smithi093153278-496 2023-08-27T20:54:10.224 INFO:tasks.rados.rados.0.smithi093.stdout:2625: finishing write tid 2 to smithi093153278-496 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2625: finishing write tid 3 to smithi093153278-496 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2625: finishing write tid 4 to smithi093153278-496 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 496 v 345 (ObjNum 1703 snap 0 seq_num 1703) dirty exists 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2625: left oid 496 (ObjNum 1703 snap 0 seq_num 1703) 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2626: finishing write tid 1 to smithi093153278-66 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2626: finishing write tid 2 to smithi093153278-66 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2626: finishing write tid 3 to smithi093153278-66 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2626: finishing write tid 4 to smithi093153278-66 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 66 v 424 (ObjNum 1704 snap 0 seq_num 1704) dirty exists 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2626: left oid 66 (ObjNum 1704 snap 0 seq_num 1704) 2023-08-27T20:54:10.225 INFO:tasks.rados.rados.0.smithi093.stdout:2627: expect (ObjNum 744 snap 0 seq_num 744) 2023-08-27T20:54:10.337 INFO:tasks.rados.rados.0.smithi093.stdout:2628: expect (ObjNum 1336 snap 0 seq_num 1336) 2023-08-27T20:54:10.475 INFO:tasks.rados.rados.0.smithi093.stdout:2630: finishing write tid 1 to smithi093153278-259 2023-08-27T20:54:10.475 INFO:tasks.rados.rados.0.smithi093.stdout:2629: expect (ObjNum 1661 snap 0 seq_num 1661) 2023-08-27T20:54:10.578 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: 2023-08-27T20:54:10.321+0000 7f6cb8109700 -1 log_channel(cephadm) log [ERR] : Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T20:54:10.603 INFO:tasks.rados.rados.0.smithi093.stdout:2631: expect (ObjNum 1450 snap 0 seq_num 1450) 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2630: finishing write tid 2 to smithi093153278-259 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2630: finishing write tid 3 to smithi093153278-259 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2630: finishing write tid 4 to smithi093153278-259 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 259 v 287 (ObjNum 1705 snap 0 seq_num 1705) dirty exists 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2630: left oid 259 (ObjNum 1705 snap 0 seq_num 1705) 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2624: done (9 left) 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2625: done (8 left) 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2626: done (7 left) 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2627: done (6 left) 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2628: done (5 left) 2023-08-27T20:54:10.710 INFO:tasks.rados.rados.0.smithi093.stdout:2629: done (4 left) 2023-08-27T20:54:10.711 INFO:tasks.rados.rados.0.smithi093.stdout:2630: done (3 left) 2023-08-27T20:54:10.711 INFO:tasks.rados.rados.0.smithi093.stdout:2631: done (2 left) 2023-08-27T20:54:10.711 INFO:tasks.rados.rados.0.smithi093.stdout:2632: done (1 left) 2023-08-27T20:54:10.711 INFO:tasks.rados.rados.0.smithi093.stdout:2633: done (0 left) 2023-08-27T20:54:10.711 INFO:tasks.rados.rados.0.smithi093.stdout:2634: write oid 34 current snap is 0 2023-08-27T20:54:10.711 INFO:tasks.rados.rados.0.smithi093.stdout:2634: seq_num 1707 ranges {689549=513115,1656292=612252} 2023-08-27T20:54:10.715 INFO:tasks.rados.rados.0.smithi093.stdout:2634: writing smithi093153278-34 from 689549 to 1202664 tid 1 2023-08-27T20:54:10.722 INFO:tasks.rados.rados.0.smithi093.stdout:2634: writing smithi093153278-34 from 1656292 to 2268544 tid 2 2023-08-27T20:54:10.722 INFO:tasks.rados.rados.0.smithi093.stdout:2635: read oid 285 snap -1 2023-08-27T20:54:10.722 INFO:tasks.rados.rados.0.smithi093.stdout:2635: expect (ObjNum 1245 snap 0 seq_num 1245) 2023-08-27T20:54:10.722 INFO:tasks.rados.rados.0.smithi093.stdout:2636: write oid 219 current snap is 0 2023-08-27T20:54:10.722 INFO:tasks.rados.rados.0.smithi093.stdout:2634: finishing write tid 1 to smithi093153278-34 2023-08-27T20:54:10.722 INFO:tasks.rados.rados.0.smithi093.stdout:2636: seq_num 1708 ranges {706356=588364,1714774=472263} 2023-08-27T20:54:10.729 INFO:tasks.rados.rados.0.smithi093.stdout:2636: writing smithi093153278-219 from 706356 to 1294720 tid 1 2023-08-27T20:54:10.734 INFO:tasks.rados.rados.0.smithi093.stdout:2636: writing smithi093153278-219 from 1714774 to 2187037 tid 2 2023-08-27T20:54:10.734 INFO:tasks.rados.rados.0.smithi093.stdout:2637: write oid 167 current snap is 0 2023-08-27T20:54:10.734 INFO:tasks.rados.rados.0.smithi093.stdout:2634: finishing write tid 2 to smithi093153278-34 2023-08-27T20:54:10.734 INFO:tasks.rados.rados.0.smithi093.stdout:2634: finishing write tid 3 to smithi093153278-34 2023-08-27T20:54:10.734 INFO:tasks.rados.rados.0.smithi093.stdout:2637: seq_num 1709 ranges {723163=663613,2173256=549907} 2023-08-27T20:54:10.741 INFO:tasks.rados.rados.0.smithi093.stdout:2637: writing smithi093153278-167 from 723163 to 1386776 tid 1 2023-08-27T20:54:10.749 INFO:tasks.rados.rados.0.smithi093.stdout:2637: writing smithi093153278-167 from 2173256 to 2723163 tid 2 2023-08-27T20:54:10.749 INFO:tasks.rados.rados.0.smithi093.stdout:2638: write oid 69 current snap is 0 2023-08-27T20:54:10.749 INFO:tasks.rados.rados.0.smithi093.stdout:2634: finishing write tid 4 to smithi093153278-34 2023-08-27T20:54:10.749 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 34 v 320 (ObjNum 1707 snap 0 seq_num 1707) dirty exists 2023-08-27T20:54:10.749 INFO:tasks.rados.rados.0.smithi093.stdout:2634: left oid 34 (ObjNum 1707 snap 0 seq_num 1707) 2023-08-27T20:54:10.749 INFO:tasks.rados.rados.0.smithi093.stdout:2635: expect (ObjNum 1245 snap 0 seq_num 1245) 2023-08-27T20:54:10.895 INFO:tasks.rados.rados.0.smithi093.stdout:2636: finishing write tid 1 to smithi093153278-219 2023-08-27T20:54:10.895 INFO:tasks.rados.rados.0.smithi093.stdout:2636: finishing write tid 2 to smithi093153278-219 2023-08-27T20:54:10.895 INFO:tasks.rados.rados.0.smithi093.stdout:2638: seq_num 1710 ranges {739970=738862,2231738=475932} 2023-08-27T20:54:10.903 INFO:tasks.rados.rados.0.smithi093.stdout:2638: writing smithi093153278-69 from 739970 to 1478832 tid 1 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2638: writing smithi093153278-69 from 2231738 to 2707670 tid 2 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2634: done (4 left) 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2635: done (3 left) 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2639: delete oid 297 current snap is 0 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2636: finishing write tid 3 to smithi093153278-219 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2636: finishing write tid 4 to smithi093153278-219 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 219 v 329 (ObjNum 1708 snap 0 seq_num 1708) dirty exists 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2636: left oid 219 (ObjNum 1708 snap 0 seq_num 1708) 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2637: finishing write tid 1 to smithi093153278-167 2023-08-27T20:54:10.911 INFO:tasks.rados.rados.0.smithi093.stdout:2637: finishing write tid 2 to smithi093153278-167 2023-08-27T20:54:10.912 INFO:tasks.rados.rados.0.smithi093.stdout:2637: finishing write tid 3 to smithi093153278-167 2023-08-27T20:54:10.912 INFO:tasks.rados.rados.0.smithi093.stdout:2637: finishing write tid 4 to smithi093153278-167 2023-08-27T20:54:10.912 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 167 v 427 (ObjNum 1709 snap 0 seq_num 1709) dirty exists 2023-08-27T20:54:10.912 INFO:tasks.rados.rados.0.smithi093.stdout:2637: left oid 167 (ObjNum 1709 snap 0 seq_num 1709) 2023-08-27T20:54:10.912 INFO:tasks.rados.rados.0.smithi093.stdout:2638: finishing write tid 1 to smithi093153278-69 2023-08-27T20:54:10.914 INFO:tasks.rados.rados.0.smithi093.stdout:2636: done (3 left) 2023-08-27T20:54:10.914 INFO:tasks.rados.rados.0.smithi093.stdout:2637: done (2 left) 2023-08-27T20:54:10.914 INFO:tasks.rados.rados.0.smithi093.stdout:2639: done (1 left) 2023-08-27T20:54:10.914 INFO:tasks.rados.rados.0.smithi093.stdout:2640: write oid 220 current snap is 0 2023-08-27T20:54:10.914 INFO:tasks.rados.rados.0.smithi093.stdout:2640: seq_num 1712 ranges {773584=605713,1948702=479601} 2023-08-27T20:54:10.919 INFO:tasks.rados.rados.0.smithi093.stdout:2640: writing smithi093153278-220 from 773584 to 1379297 tid 1 2023-08-27T20:54:10.925 INFO:tasks.rados.rados.0.smithi093.stdout:2640: writing smithi093153278-220 from 1948702 to 2428303 tid 2 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2641: read oid 108 snap -1 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2641: expect (ObjNum 1429 snap 0 seq_num 1429) 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2642: read oid 33 snap -1 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2642: expect (ObjNum 1476 snap 0 seq_num 1476) 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2643: read oid 231 snap -1 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2643: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2638: finishing write tid 2 to smithi093153278-69 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2638: finishing write tid 3 to smithi093153278-69 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:2638: finishing write tid 4 to smithi093153278-69 2023-08-27T20:54:10.926 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 69 v 383 (ObjNum 1710 snap 0 seq_num 1710) dirty exists 2023-08-27T20:54:10.927 INFO:tasks.rados.rados.0.smithi093.stdout:2638: left oid 69 (ObjNum 1710 snap 0 seq_num 1710) 2023-08-27T20:54:10.927 INFO:tasks.rados.rados.0.smithi093.stdout:2640: finishing write tid 1 to smithi093153278-220 2023-08-27T20:54:10.927 INFO:tasks.rados.rados.0.smithi093.stdout:2644: read oid 109 snap -1 2023-08-27T20:54:10.927 INFO:tasks.rados.rados.0.smithi093.stdout:2644: expect deleted 2023-08-27T20:54:10.927 INFO:tasks.rados.rados.0.smithi093.stdout:2638: done (5 left) 2023-08-27T20:54:10.927 INFO:tasks.rados.rados.0.smithi093.stdout:2645: write oid 346 current snap is 0 2023-08-27T20:54:10.927 INFO:tasks.rados.rados.0.smithi093.stdout:2645: seq_num 1713 ranges {790391=680962,2007184=623259} 2023-08-27T20:54:10.933 INFO:tasks.rados.rados.0.smithi093.stdout:2645: writing smithi093153278-346 from 790391 to 1471353 tid 1 2023-08-27T20:54:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/948267940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:54:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.940 INFO:tasks.rados.rados.0.smithi093.stdout:2645: writing smithi093153278-346 from 2007184 to 2630443 tid 2 2023-08-27T20:54:10.940 INFO:tasks.rados.rados.0.smithi093.stdout:2646: write oid 48 current snap is 0 2023-08-27T20:54:10.940 INFO:tasks.rados.rados.0.smithi093.stdout:2643: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/948267940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:54:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/948267940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "quorum_status"}]: dispatch 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:54:10.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:54:10.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:54:10.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:11.061 INFO:tasks.rados.rados.0.smithi093.stdout:2641: expect (ObjNum 1429 snap 0 seq_num 1429) 2023-08-27T20:54:11.155 INFO:tasks.rados.rados.0.smithi093.stdout:2640: finishing write tid 2 to smithi093153278-220 2023-08-27T20:54:11.155 INFO:tasks.rados.rados.0.smithi093.stdout:2640: finishing write tid 3 to smithi093153278-220 2023-08-27T20:54:11.155 INFO:tasks.rados.rados.0.smithi093.stdout:2646: seq_num 1714 ranges {407198=756211,1665666=483270,2685245=121953} 2023-08-27T20:54:11.161 INFO:tasks.rados.rados.0.smithi093.stdout:2646: writing smithi093153278-48 from 407198 to 1163409 tid 1 2023-08-27T20:54:11.166 INFO:tasks.rados.rados.0.smithi093.stdout:2646: writing smithi093153278-48 from 1665666 to 2148936 tid 2 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2646: writing smithi093153278-48 from 2685245 to 2807198 tid 3 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2641: done (6 left) 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2643: done (5 left) 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2644: done (4 left) 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2647: read oid 290 snap -1 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2647: expect (ObjNum 1588 snap 0 seq_num 1588) 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2648: write oid 37 current snap is 0 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2640: finishing write tid 4 to smithi093153278-220 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 220 v 430 (ObjNum 1712 snap 0 seq_num 1712) dirty exists 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2640: left oid 220 (ObjNum 1712 snap 0 seq_num 1712) 2023-08-27T20:54:11.171 INFO:tasks.rados.rados.0.smithi093.stdout:2642: expect (ObjNum 1476 snap 0 seq_num 1476) 2023-08-27T20:54:11.257 INFO:tasks.rados.rados.0.smithi093.stdout:2645: finishing write tid 1 to smithi093153278-346 2023-08-27T20:54:11.257 INFO:tasks.rados.rados.0.smithi093.stdout:2648: seq_num 1715 ranges {424005=431460,1607795=626928} 2023-08-27T20:54:11.261 INFO:tasks.rados.rados.0.smithi093.stdout:2648: writing smithi093153278-37 from 424005 to 855465 tid 1 2023-08-27T20:54:11.269 INFO:tasks.rados.rados.0.smithi093.stdout:2648: writing smithi093153278-37 from 1607795 to 2234723 tid 2 2023-08-27T20:54:11.269 INFO:tasks.rados.rados.0.smithi093.stdout:2640: done (5 left) 2023-08-27T20:54:11.269 INFO:tasks.rados.rados.0.smithi093.stdout:2642: done (4 left) 2023-08-27T20:54:11.269 INFO:tasks.rados.rados.0.smithi093.stdout:2649: delete oid 472 current snap is 0 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2645: finishing write tid 2 to smithi093153278-346 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2645: finishing write tid 3 to smithi093153278-346 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2645: finishing write tid 4 to smithi093153278-346 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 346 v 433 (ObjNum 1713 snap 0 seq_num 1713) dirty exists 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2645: left oid 346 (ObjNum 1713 snap 0 seq_num 1713) 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2646: finishing write tid 1 to smithi093153278-48 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2646: finishing write tid 2 to smithi093153278-48 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2646: finishing write tid 3 to smithi093153278-48 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2646: finishing write tid 4 to smithi093153278-48 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2646: finishing write tid 5 to smithi093153278-48 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 48 v 333 (ObjNum 1714 snap 0 seq_num 1714) dirty exists 2023-08-27T20:54:11.270 INFO:tasks.rados.rados.0.smithi093.stdout:2646: left oid 48 (ObjNum 1714 snap 0 seq_num 1714) 2023-08-27T20:54:11.271 INFO:tasks.rados.rados.0.smithi093.stdout:2647: expect (ObjNum 1588 snap 0 seq_num 1588) 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2648: finishing write tid 1 to smithi093153278-37 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2648: finishing write tid 2 to smithi093153278-37 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2648: finishing write tid 3 to smithi093153278-37 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2648: finishing write tid 4 to smithi093153278-37 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 37 v 466 (ObjNum 1715 snap 0 seq_num 1715) dirty exists 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2648: left oid 37 (ObjNum 1715 snap 0 seq_num 1715) 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2645: done (4 left) 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2646: done (3 left) 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2647: done (2 left) 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2648: done (1 left) 2023-08-27T20:54:11.390 INFO:tasks.rados.rados.0.smithi093.stdout:2649: done (0 left) 2023-08-27T20:54:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:2650: read oid 11 snap -1 2023-08-27T20:54:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:2650: expect (ObjNum 1589 snap 0 seq_num 1589) 2023-08-27T20:54:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:2651: delete oid 143 current snap is 0 2023-08-27T20:54:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:2651: done (1 left) 2023-08-27T20:54:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:2652: write oid 137 current snap is 0 2023-08-27T20:54:11.391 INFO:tasks.rados.rados.0.smithi093.stdout:2652: seq_num 1718 ranges {474426=657207,1783241=774255} 2023-08-27T20:54:11.397 INFO:tasks.rados.rados.0.smithi093.stdout:2652: writing smithi093153278-137 from 474426 to 1131633 tid 1 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2652: writing smithi093153278-137 from 1783241 to 2557496 tid 2 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2653: read oid 399 snap -1 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2653: expect (ObjNum 523 snap 0 seq_num 523) 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2654: read oid 419 snap -1 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2654: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2655: read oid 214 snap -1 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2655: expect (ObjNum 1517 snap 0 seq_num 1517) 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2656: write oid 392 current snap is 0 2023-08-27T20:54:11.406 INFO:tasks.rados.rados.0.smithi093.stdout:2650: expect (ObjNum 1589 snap 0 seq_num 1589) 2023-08-27T20:54:11.521 INFO:tasks.rados.rados.0.smithi093.stdout:2652: finishing write tid 1 to smithi093153278-137 2023-08-27T20:54:11.521 INFO:tasks.rados.rados.0.smithi093.stdout:2656: seq_num 1719 ranges {491233=448809,1441723=634266,2589654=301579} 2023-08-27T20:54:11.526 INFO:tasks.rados.rados.0.smithi093.stdout:2656: writing smithi093153278-392 from 491233 to 940042 tid 1 2023-08-27T20:54:11.532 INFO:tasks.rados.rados.0.smithi093.stdout:2656: writing smithi093153278-392 from 1441723 to 2075989 tid 2 2023-08-27T20:54:11.537 INFO:tasks.rados.rados.0.smithi093.stdout:2656: writing smithi093153278-392 from 2589654 to 2891233 tid 3 2023-08-27T20:54:11.537 INFO:tasks.rados.rados.0.smithi093.stdout:2650: done (5 left) 2023-08-27T20:54:11.537 INFO:tasks.rados.rados.0.smithi093.stdout:2657: read oid 160 snap -1 2023-08-27T20:54:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:2657: expect (ObjNum 1602 snap 0 seq_num 1602) 2023-08-27T20:54:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:2658: write oid 305 current snap is 0 2023-08-27T20:54:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:2654: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:54:11.605 INFO:tasks.rados.rados.0.smithi093.stdout:2652: finishing write tid 2 to smithi093153278-137 2023-08-27T20:54:11.605 INFO:tasks.rados.rados.0.smithi093.stdout:2652: finishing write tid 3 to smithi093153278-137 2023-08-27T20:54:11.605 INFO:tasks.rados.rados.0.smithi093.stdout:2652: finishing write tid 4 to smithi093153278-137 2023-08-27T20:54:11.606 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 137 v 403 (ObjNum 1718 snap 0 seq_num 1718) dirty exists 2023-08-27T20:54:11.606 INFO:tasks.rados.rados.0.smithi093.stdout:2652: left oid 137 (ObjNum 1718 snap 0 seq_num 1718) 2023-08-27T20:54:11.606 INFO:tasks.rados.rados.0.smithi093.stdout:2653: expect (ObjNum 523 snap 0 seq_num 523) 2023-08-27T20:54:11.734 INFO:tasks.rados.rados.0.smithi093.stdout:2655: expect (ObjNum 1517 snap 0 seq_num 1517) 2023-08-27T20:54:11.858 INFO:tasks.rados.rados.0.smithi093.stdout:2656: finishing write tid 1 to smithi093153278-392 2023-08-27T20:54:11.858 INFO:tasks.rados.rados.0.smithi093.stdout:2656: finishing write tid 2 to smithi093153278-392 2023-08-27T20:54:11.858 INFO:tasks.rados.rados.0.smithi093.stdout:2657: expect (ObjNum 1602 snap 0 seq_num 1602) 2023-08-27T20:54:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:11 smithi136 ceph-mon[113531]: Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T20:54:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:11 smithi136 ceph-mon[113531]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:54:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:11 smithi136 ceph-mon[113531]: pgmap v19: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 MiB/s rd, 12 MiB/s wr, 120 op/s 2023-08-27T20:54:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:11 smithi136 ceph-mon[113531]: Health check failed: Upgrade: Need standby mgr daemon (UPGRADE_NO_STANDBY_MGR) 2023-08-27T20:54:11.948 INFO:tasks.rados.rados.0.smithi093.stdout:2656: finishing write tid 3 to smithi093153278-392 2023-08-27T20:54:11.948 INFO:tasks.rados.rados.0.smithi093.stdout:2656: finishing write tid 4 to smithi093153278-392 2023-08-27T20:54:11.948 INFO:tasks.rados.rados.0.smithi093.stdout:2656: finishing write tid 5 to smithi093153278-392 2023-08-27T20:54:11.948 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 392 v 352 (ObjNum 1719 snap 0 seq_num 1719) dirty exists 2023-08-27T20:54:11.948 INFO:tasks.rados.rados.0.smithi093.stdout:2656: left oid 392 (ObjNum 1719 snap 0 seq_num 1719) 2023-08-27T20:54:11.948 INFO:tasks.rados.rados.0.smithi093.stdout:2658: seq_num 1720 ranges {508040=524058,1500205=777924} 2023-08-27T20:54:11.953 INFO:tasks.rados.rados.0.smithi093.stdout:2658: writing smithi093153278-305 from 508040 to 1032098 tid 1 2023-08-27T20:54:11.960 INFO:tasks.rados.rados.0.smithi093.stdout:2658: writing smithi093153278-305 from 1500205 to 2278129 tid 2 2023-08-27T20:54:11.960 INFO:tasks.rados.rados.0.smithi093.stdout:2652: done (6 left) 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2653: done (5 left) 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2654: done (4 left) 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2655: done (3 left) 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2656: done (2 left) 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2657: done (1 left) 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2659: read oid 94 snap -1 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2659: expect deleted 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2660: write oid 178 current snap is 0 2023-08-27T20:54:11.961 INFO:tasks.rados.rados.0.smithi093.stdout:2660: seq_num 1721 ranges {524847=599307,1558687=637935,2644500=280347} 2023-08-27T20:54:11.966 INFO:tasks.rados.rados.0.smithi093.stdout:2660: writing smithi093153278-178 from 524847 to 1124154 tid 1 2023-08-27T20:54:11.972 INFO:tasks.rados.rados.0.smithi093.stdout:2660: writing smithi093153278-178 from 1558687 to 2196622 tid 2 2023-08-27T20:54:11.976 INFO:tasks.rados.rados.0.smithi093.stdout:2660: writing smithi093153278-178 from 2644500 to 2924847 tid 3 2023-08-27T20:54:11.977 INFO:tasks.rados.rados.0.smithi093.stdout:2661: read oid 268 snap -1 2023-08-27T20:54:11.977 INFO:tasks.rados.rados.0.smithi093.stdout:2661: expect (ObjNum 1666 snap 0 seq_num 1666) 2023-08-27T20:54:11.977 INFO:tasks.rados.rados.0.smithi093.stdout:2662: write oid 310 current snap is 0 2023-08-27T20:54:11.977 INFO:tasks.rados.rados.0.smithi093.stdout:2658: finishing write tid 1 to smithi093153278-305 2023-08-27T20:54:11.977 INFO:tasks.rados.rados.0.smithi093.stdout:2662: seq_num 1722 ranges {541654=674556,1617169=781593} 2023-08-27T20:54:11.983 INFO:tasks.rados.rados.0.smithi093.stdout:2662: writing smithi093153278-310 from 541654 to 1216210 tid 1 2023-08-27T20:54:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[111335]: Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T20:54:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[111335]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:54:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[111335]: pgmap v19: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 MiB/s rd, 12 MiB/s wr, 120 op/s 2023-08-27T20:54:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[111335]: Health check failed: Upgrade: Need standby mgr daemon (UPGRADE_NO_STANDBY_MGR) 2023-08-27T20:54:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[118902]: Upgrade: Paused due to UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T20:54:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[118902]: Upgrade: Need to upgrade myself (mgr.y) 2023-08-27T20:54:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[118902]: pgmap v19: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 83 MiB/s rd, 12 MiB/s wr, 120 op/s 2023-08-27T20:54:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-mon[118902]: Health check failed: Upgrade: Need standby mgr daemon (UPGRADE_NO_STANDBY_MGR) 2023-08-27T20:54:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:11.625Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:11.990 INFO:tasks.rados.rados.0.smithi093.stdout:2662: writing smithi093153278-310 from 1617169 to 2398762 tid 2 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2663: read oid 10 snap -1 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2663: expect (ObjNum 1544 snap 0 seq_num 1544) 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2664: read oid 479 snap -1 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2664: expect (ObjNum 1266 snap 0 seq_num 1266) 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2665: write oid 301 current snap is 0 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2658: finishing write tid 2 to smithi093153278-305 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2658: finishing write tid 3 to smithi093153278-305 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2658: finishing write tid 4 to smithi093153278-305 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 305 v 290 (ObjNum 1720 snap 0 seq_num 1720) dirty exists 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2658: left oid 305 (ObjNum 1720 snap 0 seq_num 1720) 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2660: finishing write tid 1 to smithi093153278-178 2023-08-27T20:54:11.991 INFO:tasks.rados.rados.0.smithi093.stdout:2660: finishing write tid 2 to smithi093153278-178 2023-08-27T20:54:11.992 INFO:tasks.rados.rados.0.smithi093.stdout:2665: seq_num 1723 ranges {558461=749805,1959298=641604} 2023-08-27T20:54:11.998 INFO:tasks.rados.rados.0.smithi093.stdout:2665: writing smithi093153278-301 from 558461 to 1308266 tid 1 2023-08-27T20:54:12.005 INFO:tasks.rados.rados.0.smithi093.stdout:2665: writing smithi093153278-301 from 1959298 to 2600902 tid 2 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2658: done (7 left) 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2659: done (6 left) 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2666: read oid 19 snap -1 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2666: expect (ObjNum 18 snap 0 seq_num 18) 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2667: write oid 202 current snap is 0 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2660: finishing write tid 3 to smithi093153278-178 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2660: finishing write tid 4 to smithi093153278-178 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2660: finishing write tid 5 to smithi093153278-178 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 178 v 237 (ObjNum 1721 snap 0 seq_num 1721) dirty exists 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2660: left oid 178 (ObjNum 1721 snap 0 seq_num 1721) 2023-08-27T20:54:12.006 INFO:tasks.rados.rados.0.smithi093.stdout:2661: expect (ObjNum 1666 snap 0 seq_num 1666) 2023-08-27T20:54:12.080 INFO:tasks.rados.rados.0.smithi093.stdout:2662: finishing write tid 1 to smithi093153278-310 2023-08-27T20:54:12.080 INFO:tasks.rados.rados.0.smithi093.stdout:2662: finishing write tid 2 to smithi093153278-310 2023-08-27T20:54:12.080 INFO:tasks.rados.rados.0.smithi093.stdout:2664: expect (ObjNum 1266 snap 0 seq_num 1266) 2023-08-27T20:54:12.215 INFO:tasks.rados.rados.0.smithi093.stdout:2662: finishing write tid 3 to smithi093153278-310 2023-08-27T20:54:12.215 INFO:tasks.rados.rados.0.smithi093.stdout:2662: finishing write tid 4 to smithi093153278-310 2023-08-27T20:54:12.215 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 310 v 336 (ObjNum 1722 snap 0 seq_num 1722) dirty exists 2023-08-27T20:54:12.215 INFO:tasks.rados.rados.0.smithi093.stdout:2662: left oid 310 (ObjNum 1722 snap 0 seq_num 1722) 2023-08-27T20:54:12.215 INFO:tasks.rados.rados.0.smithi093.stdout:2663: expect (ObjNum 1544 snap 0 seq_num 1544) 2023-08-27T20:54:12.341 INFO:tasks.rados.rados.0.smithi093.stdout:2665: finishing write tid 1 to smithi093153278-301 2023-08-27T20:54:12.341 INFO:tasks.rados.rados.0.smithi093.stdout:2667: seq_num 1724 ranges {575268=425054,1617780=785262} 2023-08-27T20:54:12.346 INFO:tasks.rados.rados.0.smithi093.stdout:2667: writing smithi093153278-202 from 575268 to 1000322 tid 1 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2667: writing smithi093153278-202 from 1617780 to 2403042 tid 2 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2660: done (7 left) 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2661: done (6 left) 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2662: done (5 left) 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2663: done (4 left) 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2664: done (3 left) 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2668: write oid 252 current snap is 0 2023-08-27T20:54:12.355 INFO:tasks.rados.rados.0.smithi093.stdout:2665: finishing write tid 2 to smithi093153278-301 2023-08-27T20:54:12.356 INFO:tasks.rados.rados.0.smithi093.stdout:2665: finishing write tid 3 to smithi093153278-301 2023-08-27T20:54:12.356 INFO:tasks.rados.rados.0.smithi093.stdout:2668: seq_num 1725 ranges {592075=500303,1676262=645273,2754192=237883} 2023-08-27T20:54:12.360 INFO:tasks.rados.rados.0.smithi093.stdout:2668: writing smithi093153278-252 from 592075 to 1092378 tid 1 2023-08-27T20:54:12.367 INFO:tasks.rados.rados.0.smithi093.stdout:2668: writing smithi093153278-252 from 1676262 to 2321535 tid 2 2023-08-27T20:54:12.371 INFO:tasks.rados.rados.0.smithi093.stdout:2668: writing smithi093153278-252 from 2754192 to 2992075 tid 3 2023-08-27T20:54:12.371 INFO:tasks.rados.rados.0.smithi093.stdout:2669: delete oid 174 current snap is 0 2023-08-27T20:54:12.372 INFO:tasks.rados.rados.0.smithi093.stdout:2665: finishing write tid 4 to smithi093153278-301 2023-08-27T20:54:12.372 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 301 v 386 (ObjNum 1723 snap 0 seq_num 1723) dirty exists 2023-08-27T20:54:12.372 INFO:tasks.rados.rados.0.smithi093.stdout:2665: left oid 301 (ObjNum 1723 snap 0 seq_num 1723) 2023-08-27T20:54:12.372 INFO:tasks.rados.rados.0.smithi093.stdout:2666: expect (ObjNum 18 snap 0 seq_num 18) 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2667: finishing write tid 1 to smithi093153278-202 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2667: finishing write tid 2 to smithi093153278-202 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2667: finishing write tid 3 to smithi093153278-202 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2667: finishing write tid 4 to smithi093153278-202 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 202 v 406 (ObjNum 1724 snap 0 seq_num 1724) dirty exists 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2667: left oid 202 (ObjNum 1724 snap 0 seq_num 1724) 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2668: finishing write tid 1 to smithi093153278-252 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2665: done (4 left) 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2666: done (3 left) 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2667: done (2 left) 2023-08-27T20:54:12.445 INFO:tasks.rados.rados.0.smithi093.stdout:2669: done (1 left) 2023-08-27T20:54:12.446 INFO:tasks.rados.rados.0.smithi093.stdout:2670: read oid 183 snap -1 2023-08-27T20:54:12.446 INFO:tasks.rados.rados.0.smithi093.stdout:2670: expect (ObjNum 1569 snap 0 seq_num 1569) 2023-08-27T20:54:12.446 INFO:tasks.rados.rados.0.smithi093.stdout:2671: write oid 94 current snap is 0 2023-08-27T20:54:12.446 INFO:tasks.rados.rados.0.smithi093.stdout:2668: finishing write tid 2 to smithi093153278-252 2023-08-27T20:54:12.446 INFO:tasks.rados.rados.0.smithi093.stdout:2671: seq_num 1727 ranges {625689=767154,1793226=648942} 2023-08-27T20:54:12.452 INFO:tasks.rados.rados.0.smithi093.stdout:2671: writing smithi093153278-94 from 625689 to 1392843 tid 1 2023-08-27T20:54:12.922 INFO:tasks.rados.rados.0.smithi093.stdout:2671: writing smithi093153278-94 from 1793226 to 2442168 tid 2 2023-08-27T20:54:12.922 INFO:tasks.rados.rados.0.smithi093.stdout:2672: write oid 281 current snap is 0 2023-08-27T20:54:12.922 INFO:tasks.rados.rados.0.smithi093.stdout:2668: finishing write tid 3 to smithi093153278-252 2023-08-27T20:54:12.922 INFO:tasks.rados.rados.0.smithi093.stdout:2668: finishing write tid 4 to smithi093153278-252 2023-08-27T20:54:12.927 INFO:tasks.rados.rados.0.smithi093.stdout:2668: finishing write tid 5 to smithi093153278-252 2023-08-27T20:54:12.927 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 252 v 313 (ObjNum 1725 snap 0 seq_num 1725) dirty exists 2023-08-27T20:54:12.928 INFO:tasks.rados.rados.0.smithi093.stdout:2668: left oid 252 (ObjNum 1725 snap 0 seq_num 1725) 2023-08-27T20:54:12.928 INFO:tasks.rados.rados.0.smithi093.stdout:2672: seq_num 1728 ranges {642496=442403,1851708=792600} 2023-08-27T20:54:12.928 INFO:tasks.rados.rados.0.smithi093.stdout:2672: writing smithi093153278-281 from 642496 to 1084899 tid 1 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2672: writing smithi093153278-281 from 1851708 to 2644308 tid 2 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2668: done (3 left) 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2673: read oid 417 snap -1 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2673: expect (ObjNum 1385 snap 0 seq_num 1385) 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2674: write oid 128 current snap is 0 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2674: seq_num 1729 ranges {659303=517652,1910190=652611} 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2674: writing smithi093153278-128 from 659303 to 1176955 tid 1 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2674: writing smithi093153278-128 from 1910190 to 2562801 tid 2 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2675: read oid 382 snap -1 2023-08-27T20:54:12.929 INFO:tasks.rados.rados.0.smithi093.stdout:2675: expect (ObjNum 1502 snap 0 seq_num 1502) 2023-08-27T20:54:12.931 INFO:tasks.rados.rados.0.smithi093.stdout:2676: read oid 90 snap -1 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2676: expect deleted 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2677: read oid 12 snap -1 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2677: expect (ObjNum 1277 snap 0 seq_num 1277) 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2678: write oid 56 current snap is 0 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2678: seq_num 1730 ranges {676110=592901,1968672=796269} 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2678: writing smithi093153278-56 from 676110 to 1269011 tid 1 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2678: writing smithi093153278-56 from 1968672 to 2764941 tid 2 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2679: read oid 133 snap -1 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2679: expect (ObjNum 1337 snap 0 seq_num 1337) 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2680: write oid 105 current snap is 0 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2670: expect (ObjNum 1569 snap 0 seq_num 1569) 2023-08-27T20:54:12.932 INFO:tasks.rados.rados.0.smithi093.stdout:2671: finishing write tid 1 to smithi093153278-94 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2671: finishing write tid 2 to smithi093153278-94 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2671: finishing write tid 3 to smithi093153278-94 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2671: finishing write tid 4 to smithi093153278-94 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 94 v 240 (ObjNum 1727 snap 0 seq_num 1727) dirty exists 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2671: left oid 94 (ObjNum 1727 snap 0 seq_num 1727) 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2672: finishing write tid 1 to smithi093153278-281 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2672: finishing write tid 2 to smithi093153278-281 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2672: finishing write tid 3 to smithi093153278-281 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2672: finishing write tid 4 to smithi093153278-281 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 281 v 436 (ObjNum 1728 snap 0 seq_num 1728) dirty exists 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2672: left oid 281 (ObjNum 1728 snap 0 seq_num 1728) 2023-08-27T20:54:12.933 INFO:tasks.rados.rados.0.smithi093.stdout:2673: expect (ObjNum 1385 snap 0 seq_num 1385) 2023-08-27T20:54:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:2674: finishing write tid 1 to smithi093153278-128 2023-08-27T20:54:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:2674: finishing write tid 2 to smithi093153278-128 2023-08-27T20:54:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:2674: finishing write tid 3 to smithi093153278-128 2023-08-27T20:54:12.934 INFO:tasks.rados.rados.0.smithi093.stdout:2675: expect (ObjNum 1502 snap 0 seq_num 1502) 2023-08-27T20:54:12.979 INFO:tasks.rados.rados.0.smithi093.stdout:2674: finishing write tid 4 to smithi093153278-128 2023-08-27T20:54:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 128 v 469 (ObjNum 1729 snap 0 seq_num 1729) dirty exists 2023-08-27T20:54:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:2674: left oid 128 (ObjNum 1729 snap 0 seq_num 1729) 2023-08-27T20:54:13.539 INFO:tasks.rados.rados.0.smithi093.stdout:2677: expect (ObjNum 1277 snap 0 seq_num 1277) 2023-08-27T20:54:13.549 INFO:tasks.rados.rados.0.smithi093.stdout:2678: finishing write tid 1 to smithi093153278-56 2023-08-27T20:54:13.572 INFO:tasks.rados.rados.0.smithi093.stdout:2679: expect (ObjNum 1337 snap 0 seq_num 1337) 2023-08-27T20:54:13.572 INFO:tasks.rados.rados.0.smithi093.stdout:2678: finishing write tid 2 to smithi093153278-56 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2678: finishing write tid 3 to smithi093153278-56 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2678: finishing write tid 4 to smithi093153278-56 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 56 v 306 (ObjNum 1730 snap 0 seq_num 1730) dirty exists 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2678: left oid 56 (ObjNum 1730 snap 0 seq_num 1730) 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2680: seq_num 1731 ranges {692917=668150,1910801=539927,3086024=6893} 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2680: writing smithi093153278-105 from 692917 to 1361067 tid 1 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2680: writing smithi093153278-105 from 1910801 to 2450728 tid 2 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2680: writing smithi093153278-105 from 3086024 to 3092917 tid 3 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2670: done (10 left) 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2671: done (9 left) 2023-08-27T20:54:13.573 INFO:tasks.rados.rados.0.smithi093.stdout:2672: done (8 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2673: done (7 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2674: done (6 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2675: done (5 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2676: done (4 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2677: done (3 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2678: done (2 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2679: done (1 left) 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2681: write oid 33 current snap is 0 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2680: finishing write tid 1 to smithi093153278-105 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2680: finishing write tid 2 to smithi093153278-105 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2681: seq_num 1732 ranges {709724=743399,1969283=799938} 2023-08-27T20:54:13.574 INFO:tasks.rados.rados.0.smithi093.stdout:2681: writing smithi093153278-33 from 709724 to 1453123 tid 1 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2681: writing smithi093153278-33 from 1969283 to 2769221 tid 2 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2682: write oid 355 current snap is 0 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2682: seq_num 1733 ranges {726531=418648,1627765=543596,2740870=385661} 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2682: writing smithi093153278-355 from 726531 to 1145179 tid 1 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2682: writing smithi093153278-355 from 1627765 to 2171361 tid 2 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2682: writing smithi093153278-355 from 2740870 to 3126531 tid 3 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2683: write oid 125 current snap is 0 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2683: seq_num 1734 ranges {743338=610250,1802600=403607,2884646=258692} 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2683: writing smithi093153278-125 from 743338 to 1353588 tid 1 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2683: writing smithi093153278-125 from 1802600 to 2206207 tid 2 2023-08-27T20:54:13.575 INFO:tasks.rados.rados.0.smithi093.stdout:2683: writing smithi093153278-125 from 2884646 to 3143338 tid 3 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2684: read oid 299 snap -1 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2684: expect (ObjNum 773 snap 0 seq_num 773) 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2685: write oid 139 current snap is 0 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2680: finishing write tid 3 to smithi093153278-105 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2685: seq_num 1735 ranges {760145=685499,2144729=547265} 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2685: writing smithi093153278-139 from 760145 to 1445644 tid 1 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2685: writing smithi093153278-139 from 2144729 to 2691994 tid 2 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2686: read oid 4 snap -1 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2686: expect (ObjNum 1587 snap 0 seq_num 1587) 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2687: read oid 340 snap -1 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2687: expect (ObjNum 1290 snap 0 seq_num 1290) 2023-08-27T20:54:13.576 INFO:tasks.rados.rados.0.smithi093.stdout:2688: write oid 211 current snap is 0 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2680: finishing write tid 4 to smithi093153278-105 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2680: finishing write tid 5 to smithi093153278-105 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 105 v 340 (ObjNum 1731 snap 0 seq_num 1731) dirty exists 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2680: left oid 105 (ObjNum 1731 snap 0 seq_num 1731) 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2681: finishing write tid 1 to smithi093153278-33 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2681: finishing write tid 2 to smithi093153278-33 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2681: finishing write tid 3 to smithi093153278-33 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2688: seq_num 1736 ranges {776952=760748,2203211=407276} 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2688: writing smithi093153278-211 from 776952 to 1537700 tid 1 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2688: writing smithi093153278-211 from 2203211 to 2610487 tid 2 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2680: done (8 left) 2023-08-27T20:54:13.577 INFO:tasks.rados.rados.0.smithi093.stdout:2689: read oid 342 snap -1 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2689: expect (ObjNum 1312 snap 0 seq_num 1312) 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2690: read oid 498 snap -1 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2690: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2691: write oid 156 current snap is 0 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2681: finishing write tid 4 to smithi093153278-33 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 33 v 356 (ObjNum 1732 snap 0 seq_num 1732) dirty exists 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2681: left oid 33 (ObjNum 1732 snap 0 seq_num 1732) 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2682: finishing write tid 1 to smithi093153278-355 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2682: finishing write tid 2 to smithi093153278-355 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2682: finishing write tid 3 to smithi093153278-355 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2682: finishing write tid 4 to smithi093153278-355 2023-08-27T20:54:13.578 INFO:tasks.rados.rados.0.smithi093.stdout:2682: finishing write tid 5 to smithi093153278-355 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 355 v 310 (ObjNum 1733 snap 0 seq_num 1733) dirty exists 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2682: left oid 355 (ObjNum 1733 snap 0 seq_num 1733) 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2683: finishing write tid 1 to smithi093153278-125 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2691: seq_num 1737 ranges {793759=435997,1861693=550934,2850562=343197} 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2691: writing smithi093153278-156 from 793759 to 1229756 tid 1 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2691: writing smithi093153278-156 from 1861693 to 2412627 tid 2 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2691: writing smithi093153278-156 from 2850562 to 3193759 tid 3 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2681: done (10 left) 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2682: done (9 left) 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2692: read oid 27 snap -1 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2692: expect (ObjNum 26 snap 0 seq_num 26) 2023-08-27T20:54:13.579 INFO:tasks.rados.rados.0.smithi093.stdout:2693: read oid 49 snap -1 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2693: expect (ObjNum 1189 snap 0 seq_num 1189) 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2694: read oid 159 snap -1 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2694: expect (ObjNum 1504 snap 0 seq_num 1504) 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2695: write oid 432 current snap is 0 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2683: finishing write tid 2 to smithi093153278-125 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2695: seq_num 1738 ranges {410566=511246,1520175=410945,2477985=604876} 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2695: writing smithi093153278-432 from 410566 to 921812 tid 1 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2695: writing smithi093153278-432 from 1520175 to 1931120 tid 2 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2695: writing smithi093153278-432 from 2477985 to 3082861 tid 3 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2696: write oid 42 current snap is 0 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2683: finishing write tid 3 to smithi093153278-125 2023-08-27T20:54:13.580 INFO:tasks.rados.rados.0.smithi093.stdout:2683: finishing write tid 4 to smithi093153278-125 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2696: seq_num 1739 ranges {427373=586495,1462304=554603,2789055=416148} 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2696: writing smithi093153278-42 from 427373 to 1013868 tid 1 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2696: writing smithi093153278-42 from 1462304 to 2016907 tid 2 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2696: writing smithi093153278-42 from 2789055 to 3205203 tid 3 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2697: read oid 402 snap -1 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2697: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2698: read oid 407 snap -1 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2698: expect (ObjNum 1136 snap 0 seq_num 1136) 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2683: finishing write tid 5 to smithi093153278-125 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 125 v 360 (ObjNum 1734 snap 0 seq_num 1734) dirty exists 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2683: left oid 125 (ObjNum 1734 snap 0 seq_num 1734) 2023-08-27T20:54:13.581 INFO:tasks.rados.rados.0.smithi093.stdout:2684: expect (ObjNum 773 snap 0 seq_num 773) 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2685: finishing write tid 1 to smithi093153278-139 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2685: finishing write tid 2 to smithi093153278-139 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2685: finishing write tid 3 to smithi093153278-139 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2683: done (15 left) 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2684: done (14 left) 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2699: read oid 196 snap -1 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2699: expect deleted 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2700: write oid 74 current snap is 0 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2685: finishing write tid 4 to smithi093153278-139 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 139 v 343 (ObjNum 1735 snap 0 seq_num 1735) dirty exists 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2685: left oid 139 (ObjNum 1735 snap 0 seq_num 1735) 2023-08-27T20:54:13.582 INFO:tasks.rados.rados.0.smithi093.stdout:2686: expect (ObjNum 1587 snap 0 seq_num 1587) 2023-08-27T20:54:13.661 INFO:tasks.rados.rados.0.smithi093.stdout:2687: expect (ObjNum 1290 snap 0 seq_num 1290) 2023-08-27T20:54:13.856 INFO:tasks.rados.rados.0.smithi093.stdout:2688: finishing write tid 1 to smithi093153278-211 2023-08-27T20:54:14.188 INFO:tasks.rados.rados.0.smithi093.stdout:2688: finishing write tid 2 to smithi093153278-211 2023-08-27T20:54:14.188 INFO:tasks.rados.rados.0.smithi093.stdout:2689: expect (ObjNum 1312 snap 0 seq_num 1312) 2023-08-27T20:54:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:13 smithi136 ceph-mon[113531]: pgmap v20: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.6 MiB/s wr, 83 op/s 2023-08-27T20:54:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/247921703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:14.216 INFO:tasks.rados.rados.0.smithi093.stdout:2688: finishing write tid 3 to smithi093153278-211 2023-08-27T20:54:14.216 INFO:tasks.rados.rados.0.smithi093.stdout:2688: finishing write tid 4 to smithi093153278-211 2023-08-27T20:54:14.216 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 211 v 472 (ObjNum 1736 snap 0 seq_num 1736) dirty exists 2023-08-27T20:54:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:2688: left oid 211 (ObjNum 1736 snap 0 seq_num 1736) 2023-08-27T20:54:14.217 INFO:tasks.rados.rados.0.smithi093.stdout:2690: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:14.220 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:13 smithi093 ceph-mon[111335]: pgmap v20: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.6 MiB/s wr, 83 op/s 2023-08-27T20:54:14.220 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/247921703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:14.220 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:13 smithi093 ceph-mon[118902]: pgmap v20: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.6 MiB/s wr, 83 op/s 2023-08-27T20:54:14.220 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/247921703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:14.220 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:13.514Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:14.221 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:13.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:14.223 INFO:tasks.rados.rados.0.smithi093.stdout:2691: finishing write tid 1 to smithi093153278-156 2023-08-27T20:54:14.223 INFO:tasks.rados.rados.0.smithi093.stdout:2691: finishing write tid 2 to smithi093153278-156 2023-08-27T20:54:14.223 INFO:tasks.rados.rados.0.smithi093.stdout:2700: seq_num 1740 ranges {444180=661744,1520786=414614,2532831=627420} 2023-08-27T20:54:14.223 INFO:tasks.rados.rados.0.smithi093.stdout:2700: writing smithi093153278-74 from 444180 to 1105924 tid 1 2023-08-27T20:54:14.224 INFO:tasks.rados.rados.0.smithi093.stdout:2700: writing smithi093153278-74 from 1520786 to 1935400 tid 2 2023-08-27T20:54:14.224 INFO:tasks.rados.rados.0.smithi093.stdout:2700: writing smithi093153278-74 from 2532831 to 3160251 tid 3 2023-08-27T20:54:14.224 INFO:tasks.rados.rados.0.smithi093.stdout:2685: done (15 left) 2023-08-27T20:54:14.224 INFO:tasks.rados.rados.0.smithi093.stdout:2686: done (14 left) 2023-08-27T20:54:14.224 INFO:tasks.rados.rados.0.smithi093.stdout:2687: done (13 left) 2023-08-27T20:54:14.224 INFO:tasks.rados.rados.0.smithi093.stdout:2688: done (12 left) 2023-08-27T20:54:14.225 INFO:tasks.rados.rados.0.smithi093.stdout:2689: done (11 left) 2023-08-27T20:54:14.225 INFO:tasks.rados.rados.0.smithi093.stdout:2690: done (10 left) 2023-08-27T20:54:14.225 INFO:tasks.rados.rados.0.smithi093.stdout:2701: write oid 263 current snap is 0 2023-08-27T20:54:14.225 INFO:tasks.rados.rados.0.smithi093.stdout:2701: seq_num 1741 ranges {460987=453346,1695621=558272,2960254=300733} 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2701: writing smithi093153278-263 from 460987 to 914333 tid 1 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2701: writing smithi093153278-263 from 1695621 to 2253893 tid 2 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2701: writing smithi093153278-263 from 2960254 to 3260987 tid 3 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2702: read oid 489 snap -1 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2702: expect (ObjNum 1560 snap 0 seq_num 1560) 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2703: write oid 36 current snap is 0 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2691: finishing write tid 3 to smithi093153278-156 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2691: finishing write tid 4 to smithi093153278-156 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2703: seq_num 1742 ranges {477794=528595,1754103=701930,2987677=290117} 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2703: writing smithi093153278-36 from 477794 to 1006389 tid 1 2023-08-27T20:54:14.226 INFO:tasks.rados.rados.0.smithi093.stdout:2703: writing smithi093153278-36 from 1754103 to 2456033 tid 2 2023-08-27T20:54:14.227 INFO:tasks.rados.rados.0.smithi093.stdout:2703: writing smithi093153278-36 from 2987677 to 3277794 tid 3 2023-08-27T20:54:14.227 INFO:tasks.rados.rados.0.smithi093.stdout:2704: read oid 296 snap -1 2023-08-27T20:54:14.227 INFO:tasks.rados.rados.0.smithi093.stdout:2704: expect (ObjNum 1654 snap 0 seq_num 1654) 2023-08-27T20:54:14.227 INFO:tasks.rados.rados.0.smithi093.stdout:2705: read oid 303 snap -1 2023-08-27T20:54:14.227 INFO:tasks.rados.rados.0.smithi093.stdout:2705: expect deleted 2023-08-27T20:54:14.344 INFO:tasks.rados.rados.0.smithi093.stdout:2706: write oid 88 current snap is 0 2023-08-27T20:54:14.344 INFO:tasks.rados.rados.0.smithi093.stdout:2691: finishing write tid 5 to smithi093153278-156 2023-08-27T20:54:14.345 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 156 v 317 (ObjNum 1737 snap 0 seq_num 1737) dirty exists 2023-08-27T20:54:14.345 INFO:tasks.rados.rados.0.smithi093.stdout:2691: left oid 156 (ObjNum 1737 snap 0 seq_num 1737) 2023-08-27T20:54:14.345 INFO:tasks.rados.rados.0.smithi093.stdout:2692: expect (ObjNum 26 snap 0 seq_num 26) 2023-08-27T20:54:14.345 INFO:tasks.rados.rados.0.smithi093.stdout:2694: expect (ObjNum 1504 snap 0 seq_num 1504) 2023-08-27T20:54:14.390 INFO:tasks.rados.rados.0.smithi093.stdout:2695: finishing write tid 1 to smithi093153278-432 2023-08-27T20:54:14.390 INFO:tasks.rados.rados.0.smithi093.stdout:2693: expect (ObjNum 1189 snap 0 seq_num 1189) 2023-08-27T20:54:14.695 INFO:tasks.rados.rados.0.smithi093.stdout:2695: finishing write tid 2 to smithi093153278-432 2023-08-27T20:54:15.549 INFO:tasks.rados.rados.0.smithi093.stdout:2695: finishing write tid 3 to smithi093153278-432 2023-08-27T20:54:15.549 INFO:tasks.rados.rados.0.smithi093.stdout:2695: finishing write tid 4 to smithi093153278-432 2023-08-27T20:54:15.549 INFO:tasks.rados.rados.0.smithi093.stdout:2706: seq_num 1743 ranges {494601=603844,1812585=561941,3015100=279501} 2023-08-27T20:54:15.553 INFO:tasks.rados.rados.0.smithi093.stdout:2706: writing smithi093153278-88 from 494601 to 1098445 tid 1 2023-08-27T20:54:15.553 INFO:tasks.rados.rados.0.smithi093.stdout:2706: writing smithi093153278-88 from 1812585 to 2374526 tid 2 2023-08-27T20:54:15.554 INFO:tasks.rados.rados.0.smithi093.stdout:2706: writing smithi093153278-88 from 3015100 to 3294601 tid 3 2023-08-27T20:54:15.554 INFO:tasks.rados.rados.0.smithi093.stdout:2691: done (15 left) 2023-08-27T20:54:15.554 INFO:tasks.rados.rados.0.smithi093.stdout:2692: done (14 left) 2023-08-27T20:54:16.187 INFO:tasks.rados.rados.0.smithi093.stdout:2693: done (13 left) 2023-08-27T20:54:16.187 INFO:tasks.rados.rados.0.smithi093.stdout:2694: done (12 left) 2023-08-27T20:54:16.187 INFO:tasks.rados.rados.0.smithi093.stdout:2707: write oid 22 current snap is 0 2023-08-27T20:54:16.552 INFO:tasks.rados.rados.0.smithi093.stdout:2695: finishing write tid 5 to smithi093153278-432 2023-08-27T20:54:16.552 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 432 v 294 (ObjNum 1738 snap 0 seq_num 1738) dirty exists 2023-08-27T20:54:16.553 INFO:tasks.rados.rados.0.smithi093.stdout:2695: left oid 432 (ObjNum 1738 snap 0 seq_num 1738) 2023-08-27T20:54:18.598 INFO:tasks.rados.rados.0.smithi093.stdout:2696: finishing write tid 1 to smithi093153278-42 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2696: finishing write tid 2 to smithi093153278-42 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2696: finishing write tid 3 to smithi093153278-42 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2696: finishing write tid 4 to smithi093153278-42 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2696: finishing write tid 5 to smithi093153278-42 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 42 v 325 (ObjNum 1739 snap 0 seq_num 1739) dirty exists 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2696: left oid 42 (ObjNum 1739 snap 0 seq_num 1739) 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2697: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2698: expect (ObjNum 1136 snap 0 seq_num 1136) 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2700: finishing write tid 1 to smithi093153278-74 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2700: finishing write tid 2 to smithi093153278-74 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2700: finishing write tid 3 to smithi093153278-74 2023-08-27T20:54:18.599 INFO:tasks.rados.rados.0.smithi093.stdout:2700: finishing write tid 4 to smithi093153278-74 2023-08-27T20:54:18.767 INFO:tasks.rados.rados.0.smithi093.stdout:2700: finishing write tid 5 to smithi093153278-74 2023-08-27T20:54:18.767 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 74 v 476 (ObjNum 1740 snap 0 seq_num 1740) dirty exists 2023-08-27T20:54:18.767 INFO:tasks.rados.rados.0.smithi093.stdout:2700: left oid 74 (ObjNum 1740 snap 0 seq_num 1740) 2023-08-27T20:54:18.767 INFO:tasks.rados.rados.0.smithi093.stdout:2701: finishing write tid 1 to smithi093153278-263 2023-08-27T20:54:18.767 INFO:tasks.rados.rados.0.smithi093.stdout:2707: seq_num 1744 ranges {511408=679093,1754714=705599,2926170=385238} 2023-08-27T20:54:18.768 INFO:tasks.rados.rados.0.smithi093.stdout:2707: writing smithi093153278-22 from 511408 to 1190501 tid 1 2023-08-27T20:54:18.937 INFO:tasks.rados.rados.0.smithi093.stdout:2707: writing smithi093153278-22 from 1754714 to 2460313 tid 2 2023-08-27T20:54:18.937 INFO:tasks.rados.rados.0.smithi093.stdout:2707: writing smithi093153278-22 from 2926170 to 3311408 tid 3 2023-08-27T20:54:18.937 INFO:tasks.rados.rados.0.smithi093.stdout:2695: done (12 left) 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2696: done (11 left) 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2697: done (10 left) 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2698: done (9 left) 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2699: done (8 left) 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2700: done (7 left) 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2708: read oid 2 snap -1 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2708: expect (ObjNum 1149 snap 0 seq_num 1149) 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2709: write oid 39 current snap is 0 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2701: finishing write tid 2 to smithi093153278-263 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2701: finishing write tid 3 to smithi093153278-263 2023-08-27T20:54:19.093 INFO:tasks.rados.rados.0.smithi093.stdout:2709: seq_num 1745 ranges {528215=754342,1813196=565610,2953593=374622} 2023-08-27T20:54:19.094 INFO:tasks.rados.rados.0.smithi093.stdout:2709: writing smithi093153278-39 from 528215 to 1282557 tid 1 2023-08-27T20:54:19.094 INFO:tasks.rados.rados.0.smithi093.stdout:2709: writing smithi093153278-39 from 1813196 to 2378806 tid 2 2023-08-27T20:54:19.094 INFO:tasks.rados.rados.0.smithi093.stdout:2709: writing smithi093153278-39 from 2953593 to 3328215 tid 3 2023-08-27T20:54:19.094 INFO:tasks.rados.rados.0.smithi093.stdout:2710: delete oid 150 current snap is 0 2023-08-27T20:54:19.094 INFO:tasks.rados.rados.0.smithi093.stdout:2701: finishing write tid 4 to smithi093153278-263 2023-08-27T20:54:19.094 INFO:tasks.rados.rados.0.smithi093.stdout:2701: finishing write tid 5 to smithi093153278-263 2023-08-27T20:54:19.094 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 263 v 440 (ObjNum 1741 snap 0 seq_num 1741) dirty exists 2023-08-27T20:54:19.102 INFO:tasks.rados.rados.0.smithi093.stdout:2701: left oid 263 (ObjNum 1741 snap 0 seq_num 1741) 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2702: expect (ObjNum 1560 snap 0 seq_num 1560) 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2703: finishing write tid 1 to smithi093153278-36 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2703: finishing write tid 2 to smithi093153278-36 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2703: finishing write tid 3 to smithi093153278-36 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2703: finishing write tid 4 to smithi093153278-36 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2703: finishing write tid 5 to smithi093153278-36 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 349 (ObjNum 1742 snap 0 seq_num 1742) dirty exists 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2703: left oid 36 (ObjNum 1742 snap 0 seq_num 1742) 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2704: expect (ObjNum 1654 snap 0 seq_num 1654) 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2706: finishing write tid 1 to smithi093153278-88 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2706: finishing write tid 2 to smithi093153278-88 2023-08-27T20:54:19.103 INFO:tasks.rados.rados.0.smithi093.stdout:2701: done (9 left) 2023-08-27T20:54:19.104 INFO:tasks.rados.rados.0.smithi093.stdout:2702: done (8 left) 2023-08-27T20:54:19.104 INFO:tasks.rados.rados.0.smithi093.stdout:2703: done (7 left) 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:2704: done (6 left) 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:2705: done (5 left) 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:2710: done (4 left) 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:2711: write oid 124 current snap is 0 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:2706: finishing write tid 3 to smithi093153278-88 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:2706: finishing write tid 4 to smithi093153278-88 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:2706: finishing write tid 5 to smithi093153278-88 2023-08-27T20:54:19.255 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 410 (ObjNum 1743 snap 0 seq_num 1743) dirty exists 2023-08-27T20:54:19.256 INFO:tasks.rados.rados.0.smithi093.stdout:2706: left oid 88 (ObjNum 1743 snap 0 seq_num 1743) 2023-08-27T20:54:19.256 INFO:tasks.rados.rados.0.smithi093.stdout:2707: finishing write tid 1 to smithi093153278-22 2023-08-27T20:54:19.256 INFO:tasks.rados.rados.0.smithi093.stdout:2707: finishing write tid 2 to smithi093153278-22 2023-08-27T20:54:19.256 INFO:tasks.rados.rados.0.smithi093.stdout:2711: seq_num 1747 ranges {561829=504840,1530160=569279,2608439=739030} 2023-08-27T20:54:19.256 INFO:tasks.rados.rados.0.smithi093.stdout:2711: writing smithi093153278-124 from 561829 to 1066669 tid 1 2023-08-27T20:54:19.256 INFO:tasks.rados.rados.0.smithi093.stdout:2711: writing smithi093153278-124 from 1530160 to 2099439 tid 2 2023-08-27T20:54:19.257 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:15 smithi136 ceph-mon[113531]: pgmap v21: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.6 MiB/s wr, 83 op/s 2023-08-27T20:54:19.467 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1485931810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:19.467 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:17 smithi136 ceph-mon[113531]: pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 13 MiB/s wr, 128 op/s 2023-08-27T20:54:19.468 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3794003252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:19.468 INFO:tasks.rados.rados.0.smithi093.stdout:2711: writing smithi093153278-124 from 2608439 to 3347469 tid 3 2023-08-27T20:54:19.471 INFO:tasks.rados.rados.0.smithi093.stdout:2706: done (4 left) 2023-08-27T20:54:19.474 INFO:tasks.rados.rados.0.smithi093.stdout:2712: read oid 277 snap -1 2023-08-27T20:54:19.475 INFO:tasks.rados.rados.0.smithi093.stdout:2712: expect (ObjNum 1198 snap 0 seq_num 1198) 2023-08-27T20:54:19.476 INFO:tasks.rados.rados.0.smithi093.stdout:2713: read oid 36 snap -1 2023-08-27T20:54:19.629 INFO:tasks.rados.rados.0.smithi093.stdout:2713: expect (ObjNum 1742 snap 0 seq_num 1742) 2023-08-27T20:54:19.629 INFO:tasks.rados.rados.0.smithi093.stdout:2714: write oid 43 current snap is 0 2023-08-27T20:54:19.629 INFO:tasks.rados.rados.0.smithi093.stdout:2708: expect (ObjNum 1149 snap 0 seq_num 1149) 2023-08-27T20:54:19.630 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:54:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:54:15] "GET /metrics HTTP/1.1" 200 242420 "" "Prometheus/2.43.0" 2023-08-27T20:54:19.631 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:15 smithi093 ceph-mon[111335]: pgmap v21: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.6 MiB/s wr, 83 op/s 2023-08-27T20:54:19.631 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1485931810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:19.632 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:15 smithi093 ceph-mon[118902]: pgmap v21: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.6 MiB/s wr, 83 op/s 2023-08-27T20:54:19.632 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1485931810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:19.655 INFO:tasks.rados.rados.0.smithi093.stdout:2707: finishing write tid 3 to smithi093153278-22 2023-08-27T20:54:19.797 INFO:tasks.rados.rados.0.smithi093.stdout:2707: finishing write tid 4 to smithi093153278-22 2023-08-27T20:54:19.797 INFO:tasks.rados.rados.0.smithi093.stdout:2707: finishing write tid 5 to smithi093153278-22 2023-08-27T20:54:19.797 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 22 v 347 (ObjNum 1744 snap 0 seq_num 1744) dirty exists 2023-08-27T20:54:19.797 INFO:tasks.rados.rados.0.smithi093.stdout:2707: left oid 22 (ObjNum 1744 snap 0 seq_num 1744) 2023-08-27T20:54:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:2709: finishing write tid 1 to smithi093153278-39 2023-08-27T20:54:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:2714: seq_num 1748 ranges {578636=580089,1872289=712937,3319509=59127} 2023-08-27T20:54:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:2714: writing smithi093153278-43 from 578636 to 1158725 tid 1 2023-08-27T20:54:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:2714: writing smithi093153278-43 from 1872289 to 2585226 tid 2 2023-08-27T20:54:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:2714: writing smithi093153278-43 from 3319509 to 3378636 tid 3 2023-08-27T20:54:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:2707: done (6 left) 2023-08-27T20:54:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:2708: done (5 left) 2023-08-27T20:54:20.133 INFO:tasks.rados.rados.0.smithi093.stdout:2715: write oid 241 current snap is 0 2023-08-27T20:54:20.133 INFO:tasks.rados.rados.0.smithi093.stdout:2709: finishing write tid 2 to smithi093153278-39 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2709: finishing write tid 3 to smithi093153278-39 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2709: finishing write tid 4 to smithi093153278-39 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2709: finishing write tid 5 to smithi093153278-39 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 39 v 298 (ObjNum 1745 snap 0 seq_num 1745) dirty exists 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2709: left oid 39 (ObjNum 1745 snap 0 seq_num 1745) 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2711: finishing write tid 1 to smithi093153278-124 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2711: finishing write tid 2 to smithi093153278-124 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2712: expect (ObjNum 1198 snap 0 seq_num 1198) 2023-08-27T20:54:20.134 INFO:tasks.rados.rados.0.smithi093.stdout:2711: finishing write tid 3 to smithi093153278-124 2023-08-27T20:54:20.135 INFO:tasks.rados.rados.0.smithi093.stdout:2711: finishing write tid 4 to smithi093153278-124 2023-08-27T20:54:20.135 INFO:tasks.rados.rados.0.smithi093.stdout:2711: finishing write tid 5 to smithi093153278-124 2023-08-27T20:54:20.135 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 124 v 329 (ObjNum 1747 snap 0 seq_num 1747) dirty exists 2023-08-27T20:54:20.135 INFO:tasks.rados.rados.0.smithi093.stdout:2711: left oid 124 (ObjNum 1747 snap 0 seq_num 1747) 2023-08-27T20:54:20.135 INFO:tasks.rados.rados.0.smithi093.stdout:2713: expect (ObjNum 1742 snap 0 seq_num 1742) 2023-08-27T20:54:20.135 INFO:tasks.rados.rados.0.smithi093.stdout:2714: finishing write tid 1 to smithi093153278-43 2023-08-27T20:54:20.135 INFO:tasks.rados.rados.0.smithi093.stdout:2714: finishing write tid 2 to smithi093153278-43 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2714: finishing write tid 3 to smithi093153278-43 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2714: finishing write tid 4 to smithi093153278-43 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2714: finishing write tid 5 to smithi093153278-43 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 43 v 414 (ObjNum 1748 snap 0 seq_num 1748) dirty exists 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2714: left oid 43 (ObjNum 1748 snap 0 seq_num 1748) 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2715: seq_num 1749 ranges {595443=771691,2047124=572948,3063285=332158} 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2715: writing smithi093153278-241 from 595443 to 1367134 tid 1 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2715: writing smithi093153278-241 from 2047124 to 2620072 tid 2 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2715: writing smithi093153278-241 from 3063285 to 3395443 tid 3 2023-08-27T20:54:20.138 INFO:tasks.rados.rados.0.smithi093.stdout:2709: done (5 left) 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2711: done (4 left) 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2712: done (3 left) 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2713: done (2 left) 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2714: done (1 left) 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2716: write oid 323 current snap is 0 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2715: finishing write tid 1 to smithi093153278-241 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2715: finishing write tid 2 to smithi093153278-241 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2716: seq_num 1750 ranges {612250=446940,1705606=716606,3090708=321542} 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2716: writing smithi093153278-323 from 612250 to 1059190 tid 1 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2716: writing smithi093153278-323 from 1705606 to 2422212 tid 2 2023-08-27T20:54:20.139 INFO:tasks.rados.rados.0.smithi093.stdout:2716: writing smithi093153278-323 from 3090708 to 3412250 tid 3 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2717: write oid 140 current snap is 0 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2715: finishing write tid 3 to smithi093153278-241 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2715: finishing write tid 4 to smithi093153278-241 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2717: seq_num 1751 ranges {629057=522189,1764088=576617,3118131=310926} 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2717: writing smithi093153278-140 from 629057 to 1151246 tid 1 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2717: writing smithi093153278-140 from 1764088 to 2340705 tid 2 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2717: writing smithi093153278-140 from 3118131 to 3429057 tid 3 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2718: write oid 386 current snap is 0 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2715: finishing write tid 5 to smithi093153278-241 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 241 v 317 (ObjNum 1749 snap 0 seq_num 1749) dirty exists 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2715: left oid 241 (ObjNum 1749 snap 0 seq_num 1749) 2023-08-27T20:54:20.140 INFO:tasks.rados.rados.0.smithi093.stdout:2716: finishing write tid 1 to smithi093153278-323 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2716: finishing write tid 2 to smithi093153278-323 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2718: seq_num 1752 ranges {645864=597438,1706217=720275,3029201=416663} 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2718: writing smithi093153278-386 from 645864 to 1243302 tid 1 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2718: writing smithi093153278-386 from 1706217 to 2426492 tid 2 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2718: writing smithi093153278-386 from 3029201 to 3445864 tid 3 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2715: done (3 left) 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2719: read oid 53 snap -1 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2719: expect (ObjNum 860 snap 0 seq_num 860) 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2720: write oid 43 current snap is 0 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2716: finishing write tid 3 to smithi093153278-323 2023-08-27T20:54:20.141 INFO:tasks.rados.rados.0.smithi093.stdout:2716: finishing write tid 4 to smithi093153278-323 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2720: seq_num 1753 ranges {662671=672687,1764699=580286,3056624=406047} 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2720: writing smithi093153278-43 from 662671 to 1335358 tid 1 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2720: writing smithi093153278-43 from 1764699 to 2344985 tid 2 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2720: writing smithi093153278-43 from 3056624 to 3462671 tid 3 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2721: write oid 405 current snap is 0 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2716: finishing write tid 5 to smithi093153278-323 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 323 v 356 (ObjNum 1750 snap 0 seq_num 1750) dirty exists 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2716: left oid 323 (ObjNum 1750 snap 0 seq_num 1750) 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2717: finishing write tid 1 to smithi093153278-140 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2717: finishing write tid 2 to smithi093153278-140 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2721: seq_num 1754 ranges {679478=747936,2223181=723944} 2023-08-27T20:54:20.142 INFO:tasks.rados.rados.0.smithi093.stdout:2721: writing smithi093153278-405 from 679478 to 1427414 tid 1 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2721: writing smithi093153278-405 from 2223181 to 2947125 tid 2 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2716: done (5 left) 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2722: read oid 452 snap -1 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2722: expect (ObjNum 920 snap 0 seq_num 920) 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2723: write oid 37 current snap is 0 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2717: finishing write tid 3 to smithi093153278-140 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2717: finishing write tid 4 to smithi093153278-140 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2717: finishing write tid 5 to smithi093153278-140 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 140 v 325 (ObjNum 1751 snap 0 seq_num 1751) dirty exists 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2717: left oid 140 (ObjNum 1751 snap 0 seq_num 1751) 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2718: finishing write tid 1 to smithi093153278-386 2023-08-27T20:54:20.143 INFO:tasks.rados.rados.0.smithi093.stdout:2718: finishing write tid 2 to smithi093153278-386 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2718: finishing write tid 3 to smithi093153278-386 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2718: finishing write tid 4 to smithi093153278-386 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2718: finishing write tid 5 to smithi093153278-386 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 386 v 314 (ObjNum 1752 snap 0 seq_num 1752) dirty exists 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2718: left oid 386 (ObjNum 1752 snap 0 seq_num 1752) 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2719: expect (ObjNum 860 snap 0 seq_num 860) 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2720: finishing write tid 1 to smithi093153278-43 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2720: finishing write tid 2 to smithi093153278-43 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2720: finishing write tid 3 to smithi093153278-43 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2720: finishing write tid 4 to smithi093153278-43 2023-08-27T20:54:20.144 INFO:tasks.rados.rados.0.smithi093.stdout:2720: finishing write tid 5 to smithi093153278-43 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 43 v 418 (ObjNum 1753 snap 0 seq_num 1753) dirty exists 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:2720: left oid 43 (ObjNum 1753 snap 0 seq_num 1753) 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:2721: finishing write tid 1 to smithi093153278-405 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:2721: finishing write tid 2 to smithi093153278-405 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:2721: finishing write tid 3 to smithi093153278-405 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:2723: seq_num 1755 ranges {696285=423185,1881663=467602,3111470=384815} 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:2723: writing smithi093153278-37 from 696285 to 1119470 tid 1 2023-08-27T20:54:20.145 INFO:tasks.rados.rados.0.smithi093.stdout:2723: writing smithi093153278-37 from 1881663 to 2349265 tid 2 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2723: writing smithi093153278-37 from 3111470 to 3496285 tid 3 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2717: done (6 left) 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2718: done (5 left) 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2719: done (4 left) 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2720: done (3 left) 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2724: delete oid 94 current snap is 0 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2721: finishing write tid 4 to smithi093153278-405 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 405 v 359 (ObjNum 1754 snap 0 seq_num 1754) dirty exists 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2721: left oid 405 (ObjNum 1754 snap 0 seq_num 1754) 2023-08-27T20:54:20.299 INFO:tasks.rados.rados.0.smithi093.stdout:2722: expect (ObjNum 920 snap 0 seq_num 920) 2023-08-27T20:54:20.300 INFO:tasks.rados.rados.0.smithi093.stdout:2723: finishing write tid 1 to smithi093153278-37 2023-08-27T20:54:20.300 INFO:tasks.rados.rados.0.smithi093.stdout:2723: finishing write tid 2 to smithi093153278-37 2023-08-27T20:54:20.300 INFO:tasks.rados.rados.0.smithi093.stdout:2723: finishing write tid 3 to smithi093153278-37 2023-08-27T20:54:20.300 INFO:tasks.rados.rados.0.smithi093.stdout:2721: done (3 left) 2023-08-27T20:54:20.300 INFO:tasks.rados.rados.0.smithi093.stdout:2722: done (2 left) 2023-08-27T20:54:20.300 INFO:tasks.rados.rados.0.smithi093.stdout:2724: done (1 left) 2023-08-27T20:54:20.302 INFO:tasks.rados.rados.0.smithi093.stdout:2725: read oid 297 snap -1 2023-08-27T20:54:20.302 INFO:tasks.rados.rados.0.smithi093.stdout:2725: expect deleted 2023-08-27T20:54:20.305 INFO:tasks.rados.rados.0.smithi093.stdout:2726: read oid 96 snap -1 2023-08-27T20:54:20.305 INFO:tasks.rados.rados.0.smithi093.stdout:2726: expect (ObjNum 1435 snap 0 seq_num 1435) 2023-08-27T20:54:20.305 INFO:tasks.rados.rados.0.smithi093.stdout:2727: read oid 217 snap -1 2023-08-27T20:54:20.305 INFO:tasks.rados.rados.0.smithi093.stdout:2727: expect (ObjNum 1615 snap 0 seq_num 1615) 2023-08-27T20:54:20.305 INFO:tasks.rados.rados.0.smithi093.stdout:2728: write oid 378 current snap is 0 2023-08-27T20:54:20.305 INFO:tasks.rados.rados.0.smithi093.stdout:2723: finishing write tid 4 to smithi093153278-37 2023-08-27T20:54:20.307 INFO:tasks.rados.rados.0.smithi093.stdout:2728: seq_num 1757 ranges {729899=690036,1998627=471271,3166316=363583} 2023-08-27T20:54:20.307 INFO:tasks.rados.rados.0.smithi093.stdout:2728: writing smithi093153278-378 from 729899 to 1419935 tid 1 2023-08-27T20:54:20.307 INFO:tasks.rados.rados.0.smithi093.stdout:2728: writing smithi093153278-378 from 1998627 to 2469898 tid 2 2023-08-27T20:54:20.307 INFO:tasks.rados.rados.0.smithi093.stdout:2728: writing smithi093153278-378 from 3166316 to 3529899 tid 3 2023-08-27T20:54:20.307 INFO:tasks.rados.rados.0.smithi093.stdout:2729: write oid 132 current snap is 0 2023-08-27T20:54:20.307 INFO:tasks.rados.rados.0.smithi093.stdout:2723: finishing write tid 5 to smithi093153278-37 2023-08-27T20:54:20.307 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 37 v 480 (ObjNum 1755 snap 0 seq_num 1755) dirty exists 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2723: left oid 37 (ObjNum 1755 snap 0 seq_num 1755) 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2729: seq_num 1758 ranges {746706=765285,2057109=731282,3193739=352967} 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2729: writing smithi093153278-132 from 746706 to 1511991 tid 1 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2729: writing smithi093153278-132 from 2057109 to 2788391 tid 2 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2729: writing smithi093153278-132 from 3193739 to 3546706 tid 3 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2723: done (5 left) 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2730: read oid 498 snap -1 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2730: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2731: read oid 484 snap -1 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2731: expect (ObjNum 1695 snap 0 seq_num 1695) 2023-08-27T20:54:20.308 INFO:tasks.rados.rados.0.smithi093.stdout:2732: write oid 325 current snap is 0 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2726: expect (ObjNum 1435 snap 0 seq_num 1435) 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2727: expect (ObjNum 1615 snap 0 seq_num 1615) 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2728: finishing write tid 1 to smithi093153278-378 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2728: finishing write tid 2 to smithi093153278-378 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2728: finishing write tid 3 to smithi093153278-378 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2728: finishing write tid 4 to smithi093153278-378 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2728: finishing write tid 5 to smithi093153278-378 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 378 v 363 (ObjNum 1757 snap 0 seq_num 1757) dirty exists 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2728: left oid 378 (ObjNum 1757 snap 0 seq_num 1757) 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2729: finishing write tid 1 to smithi093153278-132 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2732: seq_num 1759 ranges {763513=440534,1715591=474940,2821162=423353} 2023-08-27T20:54:20.309 INFO:tasks.rados.rados.0.smithi093.stdout:2732: writing smithi093153278-325 from 763513 to 1204047 tid 1 2023-08-27T20:54:20.310 INFO:tasks.rados.rados.0.smithi093.stdout:2732: writing smithi093153278-325 from 1715591 to 2190531 tid 2 2023-08-27T20:54:20.310 INFO:tasks.rados.rados.0.smithi093.stdout:2732: writing smithi093153278-325 from 2821162 to 3244515 tid 3 2023-08-27T20:54:20.310 INFO:tasks.rados.rados.0.smithi093.stdout:2725: done (7 left) 2023-08-27T20:54:20.310 INFO:tasks.rados.rados.0.smithi093.stdout:2726: done (6 left) 2023-08-27T20:54:20.310 INFO:tasks.rados.rados.0.smithi093.stdout:2727: done (5 left) 2023-08-27T20:54:20.310 INFO:tasks.rados.rados.0.smithi093.stdout:2728: done (4 left) 2023-08-27T20:54:20.310 INFO:tasks.rados.rados.0.smithi093.stdout:2733: delete oid 126 current snap is 0 2023-08-27T20:54:20.311 INFO:tasks.rados.rados.0.smithi093.stdout:2729: finishing write tid 2 to smithi093153278-132 2023-08-27T20:54:20.311 INFO:tasks.rados.rados.0.smithi093.stdout:2729: finishing write tid 3 to smithi093153278-132 2023-08-27T20:54:20.311 INFO:tasks.rados.rados.0.smithi093.stdout:2731: expect (ObjNum 1695 snap 0 seq_num 1695) 2023-08-27T20:54:20.311 INFO:tasks.rados.rados.0.smithi093.stdout:2729: finishing write tid 4 to smithi093153278-132 2023-08-27T20:54:20.311 INFO:tasks.rados.rados.0.smithi093.stdout:2729: finishing write tid 5 to smithi093153278-132 2023-08-27T20:54:20.311 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 132 v 351 (ObjNum 1758 snap 0 seq_num 1758) dirty exists 2023-08-27T20:54:20.311 INFO:tasks.rados.rados.0.smithi093.stdout:2729: left oid 132 (ObjNum 1758 snap 0 seq_num 1758) 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2730: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2732: finishing write tid 1 to smithi093153278-325 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2732: finishing write tid 2 to smithi093153278-325 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2732: finishing write tid 3 to smithi093153278-325 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2732: finishing write tid 4 to smithi093153278-325 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2729: done (4 left) 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2730: done (3 left) 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2731: done (2 left) 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2733: done (1 left) 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2734: write oid 434 current snap is 0 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:2732: finishing write tid 5 to smithi093153278-325 2023-08-27T20:54:20.312 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 325 v 390 (ObjNum 1759 snap 0 seq_num 1759) dirty exists 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2732: left oid 325 (ObjNum 1759 snap 0 seq_num 1759) 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2734: seq_num 1761 ranges {797127=591032,2116202=478609,3159655=437472} 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2734: writing smithi093153278-434 from 797127 to 1388159 tid 1 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2734: writing smithi093153278-434 from 2116202 to 2594811 tid 2 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2734: writing smithi093153278-434 from 3159655 to 3597127 tid 3 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2732: done (1 left) 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2735: delete oid 168 current snap is 0 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2734: finishing write tid 1 to smithi093153278-434 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2734: finishing write tid 2 to smithi093153278-434 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2735: done (1 left) 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2736: write oid 68 current snap is 0 2023-08-27T20:54:20.313 INFO:tasks.rados.rados.0.smithi093.stdout:2736: seq_num 1763 ranges {430741=741530,1833166=482278,2814501=584794} 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2736: writing smithi093153278-68 from 430741 to 1172271 tid 1 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2736: writing smithi093153278-68 from 1833166 to 2315444 tid 2 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2736: writing smithi093153278-68 from 2814501 to 3399295 tid 3 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2737: write oid 54 current snap is 0 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2734: finishing write tid 3 to smithi093153278-434 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2734: finishing write tid 4 to smithi093153278-434 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2734: finishing write tid 5 to smithi093153278-434 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 321 (ObjNum 1761 snap 0 seq_num 1761) dirty exists 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2734: left oid 434 (ObjNum 1761 snap 0 seq_num 1761) 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2736: finishing write tid 1 to smithi093153278-68 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2737: seq_num 1764 ranges {447548=533132,1491648=742289,2841924=796066} 2023-08-27T20:54:20.314 INFO:tasks.rados.rados.0.smithi093.stdout:2737: writing smithi093153278-54 from 447548 to 980680 tid 1 2023-08-27T20:54:20.315 INFO:tasks.rados.rados.0.smithi093.stdout:2737: writing smithi093153278-54 from 1491648 to 2233937 tid 2 2023-08-27T20:54:20.315 INFO:tasks.rados.rados.0.smithi093.stdout:2737: writing smithi093153278-54 from 2841924 to 3637990 tid 3 2023-08-27T20:54:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:2734: done (2 left) 2023-08-27T20:54:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:2738: read oid 131 snap -1 2023-08-27T20:54:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:2738: expect (ObjNum 1357 snap 0 seq_num 1357) 2023-08-27T20:54:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:2739: write oid 249 current snap is 0 2023-08-27T20:54:20.460 INFO:tasks.rados.rados.0.smithi093.stdout:2736: finishing write tid 2 to smithi093153278-68 2023-08-27T20:54:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:2736: finishing write tid 3 to smithi093153278-68 2023-08-27T20:54:20.464 INFO:tasks.rados.rados.0.smithi093.stdout:2736: finishing write tid 4 to smithi093153278-68 2023-08-27T20:54:20.465 INFO:tasks.rados.rados.0.smithi093.stdout:2736: finishing write tid 5 to smithi093153278-68 2023-08-27T20:54:20.465 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 68 v 367 (ObjNum 1763 snap 0 seq_num 1763) dirty exists 2023-08-27T20:54:20.465 INFO:tasks.rados.rados.0.smithi093.stdout:2736: left oid 68 (ObjNum 1763 snap 0 seq_num 1763) 2023-08-27T20:54:20.465 INFO:tasks.rados.rados.0.smithi093.stdout:2737: finishing write tid 1 to smithi093153278-54 2023-08-27T20:54:20.471 INFO:tasks.rados.rados.0.smithi093.stdout:2737: finishing write tid 2 to smithi093153278-54 2023-08-27T20:54:20.471 INFO:tasks.rados.rados.0.smithi093.stdout:2739: seq_num 1765 ranges {464355=608381,1550130=485947,2469347=607338,3549144=115211} 2023-08-27T20:54:20.471 INFO:tasks.rados.rados.0.smithi093.stdout:2739: writing smithi093153278-249 from 464355 to 1072736 tid 1 2023-08-27T20:54:20.471 INFO:tasks.rados.rados.0.smithi093.stdout:2739: writing smithi093153278-249 from 1550130 to 2036077 tid 2 2023-08-27T20:54:20.471 INFO:tasks.rados.rados.0.smithi093.stdout:2739: writing smithi093153278-249 from 2469347 to 3076685 tid 3 2023-08-27T20:54:20.471 INFO:tasks.rados.rados.0.smithi093.stdout:2739: writing smithi093153278-249 from 3549144 to 3664355 tid 4 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2736: done (3 left) 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2740: delete oid 169 current snap is 0 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2739: finishing write tid 1 to smithi093153278-249 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2738: expect (ObjNum 1357 snap 0 seq_num 1357) 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2737: finishing write tid 3 to smithi093153278-54 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2737: finishing write tid 4 to smithi093153278-54 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2738: done (3 left) 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2740: done (2 left) 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2741: write oid 409 current snap is 0 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2737: finishing write tid 5 to smithi093153278-54 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 54 v 329 (ObjNum 1764 snap 0 seq_num 1764) dirty exists 2023-08-27T20:54:20.472 INFO:tasks.rados.rados.0.smithi093.stdout:2737: left oid 54 (ObjNum 1764 snap 0 seq_num 1764) 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2739: finishing write tid 2 to smithi093153278-249 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2741: seq_num 1767 ranges {497969=758879,1667094=489616,2924193=629882} 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2741: writing smithi093153278-409 from 497969 to 1256848 tid 1 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2741: writing smithi093153278-409 from 1667094 to 2156710 tid 2 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2741: writing smithi093153278-409 from 2924193 to 3554075 tid 3 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2737: done (2 left) 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2742: write oid 60 current snap is 0 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2739: finishing write tid 3 to smithi093153278-249 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2742: seq_num 1768 ranges {514776=434128,1609223=633274,2718910=557507} 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2742: writing smithi093153278-60 from 514776 to 948904 tid 1 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2742: writing smithi093153278-60 from 1609223 to 2242497 tid 2 2023-08-27T20:54:20.473 INFO:tasks.rados.rados.0.smithi093.stdout:2742: writing smithi093153278-60 from 2718910 to 3276417 tid 3 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2743: write oid 201 current snap is 0 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2739: finishing write tid 4 to smithi093153278-249 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2739: finishing write tid 5 to smithi093153278-249 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2743: seq_num 1769 ranges {531583=509377,1667705=493285,2862686=768779} 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2743: writing smithi093153278-201 from 531583 to 1040960 tid 1 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2743: writing smithi093153278-201 from 1667705 to 2160990 tid 2 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2743: writing smithi093153278-201 from 2862686 to 3631465 tid 3 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2744: read oid 25 snap -1 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2744: expect (ObjNum 912 snap 0 seq_num 912) 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2745: read oid 422 snap -1 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2745: expect (ObjNum 1479 snap 0 seq_num 1479) 2023-08-27T20:54:20.474 INFO:tasks.rados.rados.0.smithi093.stdout:2746: write oid 457 current snap is 0 2023-08-27T20:54:20.475 INFO:tasks.rados.rados.0.smithi093.stdout:2739: finishing write tid 6 to smithi093153278-249 2023-08-27T20:54:20.475 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 249 v 365 (ObjNum 1765 snap 0 seq_num 1765) dirty exists 2023-08-27T20:54:20.475 INFO:tasks.rados.rados.0.smithi093.stdout:2739: left oid 249 (ObjNum 1765 snap 0 seq_num 1765) 2023-08-27T20:54:20.475 INFO:tasks.rados.rados.0.smithi093.stdout:2741: finishing write tid 1 to smithi093153278-409 2023-08-27T20:54:20.475 INFO:tasks.rados.rados.0.smithi093.stdout:2741: finishing write tid 2 to smithi093153278-409 2023-08-27T20:54:20.475 INFO:tasks.rados.rados.0.smithi093.stdout:2746: seq_num 1770 ranges {548390=584626,1726187=636943,2773756=580051} 2023-08-27T20:54:20.475 INFO:tasks.rados.rados.0.smithi093.stdout:2746: writing smithi093153278-457 from 548390 to 1133016 tid 1 2023-08-27T20:54:20.614 INFO:tasks.rados.rados.0.smithi093.stdout:2746: writing smithi093153278-457 from 1726187 to 2363130 tid 2 2023-08-27T20:54:20.614 INFO:tasks.rados.rados.0.smithi093.stdout:2746: writing smithi093153278-457 from 2773756 to 3353807 tid 3 2023-08-27T20:54:20.614 INFO:tasks.rados.rados.0.smithi093.stdout:2739: done (6 left) 2023-08-27T20:54:20.614 INFO:tasks.rados.rados.0.smithi093.stdout:2747: write oid 267 current snap is 0 2023-08-27T20:54:20.614 INFO:tasks.rados.rados.0.smithi093.stdout:2741: finishing write tid 3 to smithi093153278-409 2023-08-27T20:54:20.614 INFO:tasks.rados.rados.0.smithi093.stdout:2741: finishing write tid 4 to smithi093153278-409 2023-08-27T20:54:20.614 INFO:tasks.rados.rados.0.smithi093.stdout:2741: finishing write tid 5 to smithi093153278-409 2023-08-27T20:54:20.615 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 409 v 325 (ObjNum 1767 snap 0 seq_num 1767) dirty exists 2023-08-27T20:54:20.615 INFO:tasks.rados.rados.0.smithi093.stdout:2741: left oid 409 (ObjNum 1767 snap 0 seq_num 1767) 2023-08-27T20:54:20.615 INFO:tasks.rados.rados.0.smithi093.stdout:2742: finishing write tid 1 to smithi093153278-60 2023-08-27T20:54:20.615 INFO:tasks.rados.rados.0.smithi093.stdout:2742: finishing write tid 2 to smithi093153278-60 2023-08-27T20:54:20.615 INFO:tasks.rados.rados.0.smithi093.stdout:2747: seq_num 1771 ranges {565197=659875,1784669=496954,2917532=791323} 2023-08-27T20:54:20.615 INFO:tasks.rados.rados.0.smithi093.stdout:2747: writing smithi093153278-267 from 565197 to 1225072 tid 1 2023-08-27T20:54:20.616 INFO:tasks.rados.rados.0.smithi093.stdout:2747: writing smithi093153278-267 from 1784669 to 2281623 tid 2 2023-08-27T20:54:20.616 INFO:tasks.rados.rados.0.smithi093.stdout:2747: writing smithi093153278-267 from 2917532 to 3708855 tid 3 2023-08-27T20:54:20.616 INFO:tasks.rados.rados.0.smithi093.stdout:2741: done (6 left) 2023-08-27T20:54:20.616 INFO:tasks.rados.rados.0.smithi093.stdout:2748: write oid 316 current snap is 0 2023-08-27T20:54:20.616 INFO:tasks.rados.rados.0.smithi093.stdout:2742: finishing write tid 3 to smithi093153278-60 2023-08-27T20:54:20.617 INFO:tasks.rados.rados.0.smithi093.stdout:2742: finishing write tid 4 to smithi093153278-60 2023-08-27T20:54:20.617 INFO:tasks.rados.rados.0.smithi093.stdout:2748: seq_num 1772 ranges {582004=451477,1443151=640612,2544955=602595,3612817=169187} 2023-08-27T20:54:20.617 INFO:tasks.rados.rados.0.smithi093.stdout:2748: writing smithi093153278-316 from 582004 to 1033481 tid 1 2023-08-27T20:54:20.618 INFO:tasks.rados.rados.0.smithi093.stdout:2748: writing smithi093153278-316 from 1443151 to 2083763 tid 2 2023-08-27T20:54:20.618 INFO:tasks.rados.rados.0.smithi093.stdout:2748: writing smithi093153278-316 from 2544955 to 3147550 tid 3 2023-08-27T20:54:20.618 INFO:tasks.rados.rados.0.smithi093.stdout:2748: writing smithi093153278-316 from 3612817 to 3782004 tid 4 2023-08-27T20:54:20.618 INFO:tasks.rados.rados.0.smithi093.stdout:2749: write oid 216 current snap is 0 2023-08-27T20:54:20.618 INFO:tasks.rados.rados.0.smithi093.stdout:2742: finishing write tid 5 to smithi093153278-60 2023-08-27T20:54:20.618 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 60 v 329 (ObjNum 1768 snap 0 seq_num 1768) dirty exists 2023-08-27T20:54:20.618 INFO:tasks.rados.rados.0.smithi093.stdout:2742: left oid 60 (ObjNum 1768 snap 0 seq_num 1768) 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2743: finishing write tid 1 to smithi093153278-201 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2743: finishing write tid 2 to smithi093153278-201 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2743: finishing write tid 3 to smithi093153278-201 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2743: finishing write tid 4 to smithi093153278-201 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2749: seq_num 1773 ranges {598811=526726,1901633=500623,2972378=530220} 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2749: writing smithi093153278-216 from 598811 to 1125537 tid 1 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2749: writing smithi093153278-216 from 1901633 to 2402256 tid 2 2023-08-27T20:54:20.619 INFO:tasks.rados.rados.0.smithi093.stdout:2749: writing smithi093153278-216 from 2972378 to 3502598 tid 3 2023-08-27T20:54:20.620 INFO:tasks.rados.rados.0.smithi093.stdout:2742: done (7 left) 2023-08-27T20:54:20.620 INFO:tasks.rados.rados.0.smithi093.stdout:2750: read oid 297 snap -1 2023-08-27T20:54:20.621 INFO:tasks.rados.rados.0.smithi093.stdout:2750: expect deleted 2023-08-27T20:54:20.621 INFO:tasks.rados.rados.0.smithi093.stdout:2751: read oid 351 snap -1 2023-08-27T20:54:20.621 INFO:tasks.rados.rados.0.smithi093.stdout:2751: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:54:20.623 INFO:tasks.rados.rados.0.smithi093.stdout:2752: read oid 414 snap -1 2023-08-27T20:54:20.623 INFO:tasks.rados.rados.0.smithi093.stdout:2752: expect (ObjNum 1452 snap 0 seq_num 1452) 2023-08-27T20:54:20.623 INFO:tasks.rados.rados.0.smithi093.stdout:2753: delete oid 190 current snap is 0 2023-08-27T20:54:20.623 INFO:tasks.rados.rados.0.smithi093.stdout:2743: finishing write tid 5 to smithi093153278-201 2023-08-27T20:54:20.623 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 201 v 322 (ObjNum 1769 snap 0 seq_num 1769) dirty exists 2023-08-27T20:54:20.623 INFO:tasks.rados.rados.0.smithi093.stdout:2743: left oid 201 (ObjNum 1769 snap 0 seq_num 1769) 2023-08-27T20:54:20.623 INFO:tasks.rados.rados.0.smithi093.stdout:2744: expect (ObjNum 912 snap 0 seq_num 912) 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2745: expect (ObjNum 1479 snap 0 seq_num 1479) 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2746: finishing write tid 1 to smithi093153278-457 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2746: finishing write tid 2 to smithi093153278-457 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2746: finishing write tid 3 to smithi093153278-457 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2746: finishing write tid 4 to smithi093153278-457 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2746: finishing write tid 5 to smithi093153278-457 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 457 v 371 (ObjNum 1770 snap 0 seq_num 1770) dirty exists 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2746: left oid 457 (ObjNum 1770 snap 0 seq_num 1770) 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2747: finishing write tid 1 to smithi093153278-267 2023-08-27T20:54:20.624 INFO:tasks.rados.rados.0.smithi093.stdout:2747: finishing write tid 2 to smithi093153278-267 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2743: done (10 left) 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2744: done (9 left) 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2745: done (8 left) 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2746: done (7 left) 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2753: done (6 left) 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2754: read oid 289 snap -1 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2754: expect (ObjNum 879 snap 0 seq_num 879) 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2755: read oid 476 snap -1 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2755: expect (ObjNum 1644 snap 0 seq_num 1644) 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2756: read oid 429 snap -1 2023-08-27T20:54:20.625 INFO:tasks.rados.rados.0.smithi093.stdout:2756: expect (ObjNum 1574 snap 0 seq_num 1574) 2023-08-27T20:54:20.626 INFO:tasks.rados.rados.0.smithi093.stdout:2757: read oid 231 snap -1 2023-08-27T20:54:20.626 INFO:tasks.rados.rados.0.smithi093.stdout:2757: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:54:20.626 INFO:tasks.rados.rados.0.smithi093.stdout:2758: write oid 153 current snap is 0 2023-08-27T20:54:20.626 INFO:tasks.rados.rados.0.smithi093.stdout:2747: finishing write tid 3 to smithi093153278-267 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2747: finishing write tid 4 to smithi093153278-267 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2747: finishing write tid 5 to smithi093153278-267 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 267 v 355 (ObjNum 1771 snap 0 seq_num 1771) dirty exists 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2747: left oid 267 (ObjNum 1771 snap 0 seq_num 1771) 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2748: finishing write tid 1 to smithi093153278-316 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2748: finishing write tid 2 to smithi093153278-316 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2748: finishing write tid 3 to smithi093153278-316 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2758: seq_num 1775 ranges {632425=677224,2018597=504292,3027224=552764} 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2758: writing smithi093153278-153 from 632425 to 1309649 tid 1 2023-08-27T20:54:20.627 INFO:tasks.rados.rados.0.smithi093.stdout:2758: writing smithi093153278-153 from 2018597 to 2522889 tid 2 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2758: writing smithi093153278-153 from 3027224 to 3579988 tid 3 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2747: done (10 left) 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2759: write oid 43 current snap is 0 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2748: finishing write tid 4 to smithi093153278-316 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2748: finishing write tid 5 to smithi093153278-316 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2748: finishing write tid 6 to smithi093153278-316 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 316 v 395 (ObjNum 1772 snap 0 seq_num 1772) dirty exists 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2748: left oid 316 (ObjNum 1772 snap 0 seq_num 1772) 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2749: finishing write tid 1 to smithi093153278-216 2023-08-27T20:54:20.628 INFO:tasks.rados.rados.0.smithi093.stdout:2749: finishing write tid 2 to smithi093153278-216 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2759: seq_num 1776 ranges {649232=752473,1960726=647950,3338294=510938} 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2759: writing smithi093153278-43 from 649232 to 1401705 tid 1 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2759: writing smithi093153278-43 from 1960726 to 2608676 tid 2 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2759: writing smithi093153278-43 from 3338294 to 3849232 tid 3 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2748: done (10 left) 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2760: write oid 317 current snap is 0 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2749: finishing write tid 3 to smithi093153278-216 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2752: expect (ObjNum 1452 snap 0 seq_num 1452) 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2749: finishing write tid 4 to smithi093153278-216 2023-08-27T20:54:20.629 INFO:tasks.rados.rados.0.smithi093.stdout:2749: finishing write tid 5 to smithi093153278-216 2023-08-27T20:54:20.630 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 216 v 375 (ObjNum 1773 snap 0 seq_num 1773) dirty exists 2023-08-27T20:54:20.630 INFO:tasks.rados.rados.0.smithi093.stdout:2749: left oid 216 (ObjNum 1773 snap 0 seq_num 1773) 2023-08-27T20:54:20.630 INFO:tasks.rados.rados.0.smithi093.stdout:2751: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:54:20.630 INFO:tasks.rados.rados.0.smithi093.stdout:2754: expect (ObjNum 879 snap 0 seq_num 879) 2023-08-27T20:54:20.631 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:17 smithi093 ceph-mon[111335]: pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 13 MiB/s wr, 128 op/s 2023-08-27T20:54:20.631 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3794003252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:20.631 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:17 smithi093 ceph-mon[118902]: pgmap v22: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 13 MiB/s wr, 128 op/s 2023-08-27T20:54:20.631 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3794003252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:20.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:19 smithi136 ceph-mon[113531]: pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 MiB/s rd, 9.1 MiB/s wr, 89 op/s 2023-08-27T20:54:20.643 INFO:tasks.rados.rados.0.smithi093.stdout:2756: expect (ObjNum 1574 snap 0 seq_num 1574) 2023-08-27T20:54:20.777 INFO:tasks.rados.rados.0.smithi093.stdout:2755: expect (ObjNum 1644 snap 0 seq_num 1644) 2023-08-27T20:54:20.777 INFO:tasks.rados.rados.0.smithi093.stdout:2758: finishing write tid 1 to smithi093153278-153 2023-08-27T20:54:20.777 INFO:tasks.rados.rados.0.smithi093.stdout:2757: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:54:20.777 INFO:tasks.rados.rados.0.smithi093.stdout:2758: finishing write tid 2 to smithi093153278-153 2023-08-27T20:54:20.777 INFO:tasks.rados.rados.0.smithi093.stdout:2758: finishing write tid 3 to smithi093153278-153 2023-08-27T20:54:20.778 INFO:tasks.rados.rados.0.smithi093.stdout:2758: finishing write tid 4 to smithi093153278-153 2023-08-27T20:54:20.778 INFO:tasks.rados.rados.0.smithi093.stdout:2758: finishing write tid 5 to smithi093153278-153 2023-08-27T20:54:20.778 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 153 v 369 (ObjNum 1775 snap 0 seq_num 1775) dirty exists 2023-08-27T20:54:20.778 INFO:tasks.rados.rados.0.smithi093.stdout:2758: left oid 153 (ObjNum 1775 snap 0 seq_num 1775) 2023-08-27T20:54:20.778 INFO:tasks.rados.rados.0.smithi093.stdout:2760: seq_num 1777 ranges {666039=427722,1619208=507961,2565717=691661,3660365=205674} 2023-08-27T20:54:20.778 INFO:tasks.rados.rados.0.smithi093.stdout:2760: writing smithi093153278-317 from 666039 to 1093761 tid 1 2023-08-27T20:54:20.779 INFO:tasks.rados.rados.0.smithi093.stdout:2760: writing smithi093153278-317 from 1619208 to 2127169 tid 2 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2760: writing smithi093153278-317 from 2565717 to 3257378 tid 3 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2760: writing smithi093153278-317 from 3660365 to 3866039 tid 4 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2749: done (10 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2750: done (9 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2751: done (8 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2752: done (7 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2754: done (6 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2755: done (5 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2756: done (4 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2757: done (3 left) 2023-08-27T20:54:20.787 INFO:tasks.rados.rados.0.smithi093.stdout:2758: done (2 left) 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2761: write oid 479 current snap is 0 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2759: finishing write tid 1 to smithi093153278-43 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2759: finishing write tid 2 to smithi093153278-43 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2759: finishing write tid 3 to smithi093153278-43 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2759: finishing write tid 4 to smithi093153278-43 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2761: seq_num 1778 ranges {682846=502971,1677690=651619,2993140=502933} 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2761: writing smithi093153278-479 from 682846 to 1185817 tid 1 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2761: writing smithi093153278-479 from 1677690 to 2329309 tid 2 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2761: writing smithi093153278-479 from 2993140 to 3496073 tid 3 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2762: read oid 14 snap -1 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2762: expect (ObjNum 1577 snap 0 seq_num 1577) 2023-08-27T20:54:20.788 INFO:tasks.rados.rados.0.smithi093.stdout:2763: write oid 331 current snap is 0 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2763: seq_num 1779 ranges {699653=694573,1852525=795277,3420563=479090} 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2763: writing smithi093153278-331 from 699653 to 1394226 tid 1 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2763: writing smithi093153278-331 from 1852525 to 2647802 tid 2 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2763: writing smithi093153278-331 from 3420563 to 3899653 tid 3 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2764: write oid 82 current snap is 0 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2759: finishing write tid 5 to smithi093153278-43 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 43 v 422 (ObjNum 1776 snap 0 seq_num 1776) dirty exists 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2759: left oid 43 (ObjNum 1776 snap 0 seq_num 1776) 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2760: finishing write tid 1 to smithi093153278-317 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2760: finishing write tid 2 to smithi093153278-317 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2760: finishing write tid 3 to smithi093153278-317 2023-08-27T20:54:20.789 INFO:tasks.rados.rados.0.smithi093.stdout:2760: finishing write tid 4 to smithi093153278-317 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2760: finishing write tid 5 to smithi093153278-317 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2760: finishing write tid 6 to smithi093153278-317 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 317 v 327 (ObjNum 1777 snap 0 seq_num 1777) dirty exists 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2760: left oid 317 (ObjNum 1777 snap 0 seq_num 1777) 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2761: finishing write tid 1 to smithi093153278-479 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2761: finishing write tid 2 to smithi093153278-479 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2764: seq_num 1780 ranges {716460=769822,2194654=655288,3447986=468474} 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2764: writing smithi093153278-82 from 716460 to 1486282 tid 1 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2764: writing smithi093153278-82 from 2194654 to 2849942 tid 2 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2764: writing smithi093153278-82 from 3447986 to 3916460 tid 3 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2759: done (5 left) 2023-08-27T20:54:20.790 INFO:tasks.rados.rados.0.smithi093.stdout:2760: done (4 left) 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2765: read oid 351 snap -1 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2765: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2766: write oid 188 current snap is 0 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2761: finishing write tid 3 to smithi093153278-479 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2761: finishing write tid 4 to smithi093153278-479 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2761: finishing write tid 5 to smithi093153278-479 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 479 v 333 (ObjNum 1778 snap 0 seq_num 1778) dirty exists 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2761: left oid 479 (ObjNum 1778 snap 0 seq_num 1778) 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2762: expect (ObjNum 1577 snap 0 seq_num 1577) 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2763: finishing write tid 1 to smithi093153278-331 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2763: finishing write tid 2 to smithi093153278-331 2023-08-27T20:54:20.791 INFO:tasks.rados.rados.0.smithi093.stdout:2763: finishing write tid 3 to smithi093153278-331 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2763: finishing write tid 4 to smithi093153278-331 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2763: finishing write tid 5 to smithi093153278-331 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 331 v 484 (ObjNum 1779 snap 0 seq_num 1779) dirty exists 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2763: left oid 331 (ObjNum 1779 snap 0 seq_num 1779) 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2764: finishing write tid 1 to smithi093153278-82 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2766: seq_num 1781 ranges {733267=445071,1853136=798946,3359056=574211} 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2766: writing smithi093153278-188 from 733267 to 1178338 tid 1 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2766: writing smithi093153278-188 from 1853136 to 2652082 tid 2 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2766: writing smithi093153278-188 from 3359056 to 3933267 tid 3 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2761: done (5 left) 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2762: done (4 left) 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2763: done (3 left) 2023-08-27T20:54:20.792 INFO:tasks.rados.rados.0.smithi093.stdout:2767: read oid 22 snap -1 2023-08-27T20:54:20.793 INFO:tasks.rados.rados.0.smithi093.stdout:2767: expect (ObjNum 1744 snap 0 seq_num 1744) 2023-08-27T20:54:20.936 INFO:tasks.rados.rados.0.smithi093.stdout:2768: write oid 327 current snap is 0 2023-08-27T20:54:20.936 INFO:tasks.rados.rados.0.smithi093.stdout:2764: finishing write tid 2 to smithi093153278-82 2023-08-27T20:54:20.936 INFO:tasks.rados.rados.0.smithi093.stdout:2764: finishing write tid 3 to smithi093153278-82 2023-08-27T20:54:20.936 INFO:tasks.rados.rados.0.smithi093.stdout:2768: seq_num 1782 ranges {750074=520320,1911618=658957,3102832=664374} 2023-08-27T20:54:20.936 INFO:tasks.rados.rados.0.smithi093.stdout:2768: writing smithi093153278-327 from 750074 to 1270394 tid 1 2023-08-27T20:54:20.936 INFO:tasks.rados.rados.0.smithi093.stdout:2768: writing smithi093153278-327 from 1911618 to 2570575 tid 2 2023-08-27T20:54:20.936 INFO:tasks.rados.rados.0.smithi093.stdout:2768: writing smithi093153278-327 from 3102832 to 3767206 tid 3 2023-08-27T20:54:20.937 INFO:tasks.rados.rados.0.smithi093.stdout:2769: read oid 133 snap -1 2023-08-27T20:54:20.937 INFO:tasks.rados.rados.0.smithi093.stdout:2769: expect (ObjNum 1337 snap 0 seq_num 1337) 2023-08-27T20:54:20.937 INFO:tasks.rados.rados.0.smithi093.stdout:2770: write oid 151 current snap is 0 2023-08-27T20:54:20.937 INFO:tasks.rados.rados.0.smithi093.stdout:2764: finishing write tid 4 to smithi093153278-82 2023-08-27T20:54:20.939 INFO:tasks.rados.rados.0.smithi093.stdout:2764: finishing write tid 5 to smithi093153278-82 2023-08-27T20:54:20.939 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 82 v 355 (ObjNum 1780 snap 0 seq_num 1780) dirty exists 2023-08-27T20:54:20.939 INFO:tasks.rados.rados.0.smithi093.stdout:2764: left oid 82 (ObjNum 1780 snap 0 seq_num 1780) 2023-08-27T20:54:20.939 INFO:tasks.rados.rados.0.smithi093.stdout:2765: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:54:20.939 INFO:tasks.rados.rados.0.smithi093.stdout:2766: finishing write tid 1 to smithi093153278-188 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2766: finishing write tid 2 to smithi093153278-188 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2766: finishing write tid 3 to smithi093153278-188 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2766: finishing write tid 4 to smithi093153278-188 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2766: finishing write tid 5 to smithi093153278-188 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 188 v 426 (ObjNum 1781 snap 0 seq_num 1781) dirty exists 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2766: left oid 188 (ObjNum 1781 snap 0 seq_num 1781) 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2767: expect (ObjNum 1744 snap 0 seq_num 1744) 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2768: finishing write tid 1 to smithi093153278-327 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2768: finishing write tid 2 to smithi093153278-327 2023-08-27T20:54:20.940 INFO:tasks.rados.rados.0.smithi093.stdout:2768: finishing write tid 3 to smithi093153278-327 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2768: finishing write tid 4 to smithi093153278-327 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2768: finishing write tid 5 to smithi093153278-327 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 327 v 430 (ObjNum 1782 snap 0 seq_num 1782) dirty exists 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2768: left oid 327 (ObjNum 1782 snap 0 seq_num 1782) 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2769: expect (ObjNum 1337 snap 0 seq_num 1337) 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2770: seq_num 1783 ranges {766881=595569,1970100=402615,3013902=475646} 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2770: writing smithi093153278-151 from 766881 to 1362450 tid 1 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2770: writing smithi093153278-151 from 1970100 to 2372715 tid 2 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2770: writing smithi093153278-151 from 3013902 to 3489548 tid 3 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2764: done (6 left) 2023-08-27T20:54:20.941 INFO:tasks.rados.rados.0.smithi093.stdout:2765: done (5 left) 2023-08-27T20:54:20.944 INFO:tasks.rados.rados.0.smithi093.stdout:2766: done (4 left) 2023-08-27T20:54:20.944 INFO:tasks.rados.rados.0.smithi093.stdout:2767: done (3 left) 2023-08-27T20:54:20.944 INFO:tasks.rados.rados.0.smithi093.stdout:2768: done (2 left) 2023-08-27T20:54:20.944 INFO:tasks.rados.rados.0.smithi093.stdout:2769: done (1 left) 2023-08-27T20:54:20.944 INFO:tasks.rados.rados.0.smithi093.stdout:2771: read oid 150 snap -1 2023-08-27T20:54:20.944 INFO:tasks.rados.rados.0.smithi093.stdout:2771: expect deleted 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2772: read oid 433 snap -1 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2772: expect (ObjNum 1671 snap 0 seq_num 1671) 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2773: read oid 350 snap -1 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2773: expect (ObjNum 1342 snap 0 seq_num 1342) 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2774: delete oid 464 current snap is 0 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2770: finishing write tid 1 to smithi093153278-151 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2770: finishing write tid 2 to smithi093153278-151 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2772: expect (ObjNum 1671 snap 0 seq_num 1671) 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2770: finishing write tid 3 to smithi093153278-151 2023-08-27T20:54:20.945 INFO:tasks.rados.rados.0.smithi093.stdout:2770: finishing write tid 4 to smithi093153278-151 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2773: expect (ObjNum 1342 snap 0 seq_num 1342) 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2770: finishing write tid 5 to smithi093153278-151 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 151 v 333 (ObjNum 1783 snap 0 seq_num 1783) dirty exists 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2770: left oid 151 (ObjNum 1783 snap 0 seq_num 1783) 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2770: done (4 left) 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2771: done (3 left) 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2772: done (2 left) 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2773: done (1 left) 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2774: done (0 left) 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2775: write oid 401 current snap is 0 2023-08-27T20:54:20.946 INFO:tasks.rados.rados.0.smithi093.stdout:2775: seq_num 1785 ranges {400495=746067,1570711=406284} 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2775: writing smithi093153278-401 from 400495 to 1146562 tid 1 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2775: writing smithi093153278-401 from 1570711 to 1976995 tid 2 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2776: write oid 132 current snap is 0 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2775: finishing write tid 1 to smithi093153278-401 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2776: seq_num 1786 ranges {417302=421316,1629193=388109} 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2776: writing smithi093153278-132 from 417302 to 838618 tid 1 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2776: writing smithi093153278-132 from 1629193 to 2017302 tid 2 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2777: read oid 258 snap -1 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2777: expect (ObjNum 1353 snap 0 seq_num 1353) 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2778: read oid 309 snap -1 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2778: expect (ObjNum 1527 snap 0 seq_num 1527) 2023-08-27T20:54:20.947 INFO:tasks.rados.rados.0.smithi093.stdout:2779: read oid 41 snap -1 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2779: expect (ObjNum 1575 snap 0 seq_num 1575) 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2780: delete oid 196 current snap is 0 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2775: finishing write tid 2 to smithi093153278-401 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2775: finishing write tid 3 to smithi093153278-401 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2775: finishing write tid 4 to smithi093153278-401 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 401 v 244 (ObjNum 1785 snap 0 seq_num 1785) dirty exists 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2775: left oid 401 (ObjNum 1785 snap 0 seq_num 1785) 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2776: finishing write tid 1 to smithi093153278-132 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2775: done (5 left) 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2780: done (4 left) 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2781: write oid 133 current snap is 0 2023-08-27T20:54:20.948 INFO:tasks.rados.rados.0.smithi093.stdout:2781: seq_num 1788 ranges {450916=688167,1862510=188406} 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2781: writing smithi093153278-133 from 450916 to 1139083 tid 1 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2781: writing smithi093153278-133 from 1862510 to 2050916 tid 2 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2782: write oid 460 current snap is 0 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2782: seq_num 1789 ranges {467723=763416,1804639=263084} 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2782: writing smithi093153278-460 from 467723 to 1231139 tid 1 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2782: writing smithi093153278-460 from 1804639 to 2067723 tid 2 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2783: read oid 207 snap -1 2023-08-27T20:54:20.949 INFO:tasks.rados.rados.0.smithi093.stdout:2783: expect (ObjNum 1689 snap 0 seq_num 1689) 2023-08-27T20:54:20.951 INFO:tasks.rados.rados.0.smithi093.stdout:2784: read oid 253 snap -1 2023-08-27T20:54:20.951 INFO:tasks.rados.rados.0.smithi093.stdout:2784: expect (ObjNum 1657 snap 0 seq_num 1657) 2023-08-27T20:54:20.951 INFO:tasks.rados.rados.0.smithi093.stdout:2785: delete oid 302 current snap is 0 2023-08-27T20:54:20.951 INFO:tasks.rados.rados.0.smithi093.stdout:2779: expect (ObjNum 1575 snap 0 seq_num 1575) 2023-08-27T20:54:20.951 INFO:tasks.rados.rados.0.smithi093.stdout:2776: finishing write tid 2 to smithi093153278-132 2023-08-27T20:54:20.951 INFO:tasks.rados.rados.0.smithi093.stdout:2779: done (8 left) 2023-08-27T20:54:20.953 INFO:tasks.rados.rados.0.smithi093.stdout:2785: done (7 left) 2023-08-27T20:54:20.953 INFO:tasks.rados.rados.0.smithi093.stdout:2786: write oid 13 current snap is 0 2023-08-27T20:54:20.953 INFO:tasks.rados.rados.0.smithi093.stdout:2776: finishing write tid 3 to smithi093153278-132 2023-08-27T20:54:20.953 INFO:tasks.rados.rados.0.smithi093.stdout:2776: finishing write tid 4 to smithi093153278-132 2023-08-27T20:54:20.953 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 132 v 358 (ObjNum 1786 snap 0 seq_num 1786) dirty exists 2023-08-27T20:54:20.953 INFO:tasks.rados.rados.0.smithi093.stdout:2776: left oid 132 (ObjNum 1786 snap 0 seq_num 1786) 2023-08-27T20:54:20.953 INFO:tasks.rados.rados.0.smithi093.stdout:2786: seq_num 1791 ranges {501337=513914,1521603=417291} 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2786: writing smithi093153278-13 from 501337 to 1015251 tid 1 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2786: writing smithi093153278-13 from 1521603 to 1938894 tid 2 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2776: done (7 left) 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2787: write oid 326 current snap is 0 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2781: finishing write tid 1 to smithi093153278-133 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2781: finishing write tid 2 to smithi093153278-133 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2781: finishing write tid 3 to smithi093153278-133 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2787: seq_num 1792 ranges {518144=589163,1580085=538059} 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2787: writing smithi093153278-326 from 518144 to 1107307 tid 1 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2787: writing smithi093153278-326 from 1580085 to 2118144 tid 2 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2788: write oid 78 current snap is 0 2023-08-27T20:54:20.954 INFO:tasks.rados.rados.0.smithi093.stdout:2781: finishing write tid 4 to smithi093153278-133 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 133 v 433 (ObjNum 1788 snap 0 seq_num 1788) dirty exists 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2781: left oid 133 (ObjNum 1788 snap 0 seq_num 1788) 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2777: expect (ObjNum 1353 snap 0 seq_num 1353) 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2778: expect (ObjNum 1527 snap 0 seq_num 1527) 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2782: finishing write tid 1 to smithi093153278-460 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2782: finishing write tid 2 to smithi093153278-460 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2782: finishing write tid 3 to smithi093153278-460 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2782: finishing write tid 4 to smithi093153278-460 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 460 v 301 (ObjNum 1789 snap 0 seq_num 1789) dirty exists 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2782: left oid 460 (ObjNum 1789 snap 0 seq_num 1789) 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2783: expect (ObjNum 1689 snap 0 seq_num 1689) 2023-08-27T20:54:20.955 INFO:tasks.rados.rados.0.smithi093.stdout:2784: expect (ObjNum 1657 snap 0 seq_num 1657) 2023-08-27T20:54:20.956 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:19 smithi093 ceph-mon[111335]: pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 MiB/s rd, 9.1 MiB/s wr, 89 op/s 2023-08-27T20:54:20.956 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:19 smithi093 ceph-mon[118902]: pgmap v23: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 62 MiB/s rd, 9.1 MiB/s wr, 89 op/s 2023-08-27T20:54:20.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2313363761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:21.276 INFO:tasks.rados.rados.0.smithi093.stdout:2786: finishing write tid 1 to smithi093153278-13 2023-08-27T20:54:21.750 INFO:tasks.rados.rados.0.smithi093.stdout:2786: finishing write tid 2 to smithi093153278-13 2023-08-27T20:54:21.750 INFO:tasks.rados.rados.0.smithi093.stdout:2786: finishing write tid 3 to smithi093153278-13 2023-08-27T20:54:21.750 INFO:tasks.rados.rados.0.smithi093.stdout:2786: finishing write tid 4 to smithi093153278-13 2023-08-27T20:54:21.750 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 13 v 317 (ObjNum 1791 snap 0 seq_num 1791) dirty exists 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2786: left oid 13 (ObjNum 1791 snap 0 seq_num 1791) 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2788: seq_num 1793 ranges {534951=664412,1922214=212737} 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2788: writing smithi093153278-78 from 534951 to 1199363 tid 1 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2788: writing smithi093153278-78 from 1922214 to 2134951 tid 2 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2777: done (8 left) 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2778: done (7 left) 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2781: done (6 left) 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2782: done (5 left) 2023-08-27T20:54:21.751 INFO:tasks.rados.rados.0.smithi093.stdout:2783: done (4 left) 2023-08-27T20:54:21.763 INFO:tasks.rados.rados.0.smithi093.stdout:2784: done (3 left) 2023-08-27T20:54:21.763 INFO:tasks.rados.rados.0.smithi093.stdout:2786: done (2 left) 2023-08-27T20:54:21.905 INFO:tasks.rados.rados.0.smithi093.stdout:2789: read oid 498 snap -1 2023-08-27T20:54:21.905 INFO:tasks.rados.rados.0.smithi093.stdout:2789: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2790: read oid 29 snap -1 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2790: expect (ObjNum 1127 snap 0 seq_num 1127) 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2791: write oid 395 current snap is 0 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2787: finishing write tid 1 to smithi093153278-326 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2787: finishing write tid 2 to smithi093153278-326 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2787: finishing write tid 3 to smithi093153278-326 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2787: finishing write tid 4 to smithi093153278-326 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 304 (ObjNum 1792 snap 0 seq_num 1792) dirty exists 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2787: left oid 326 (ObjNum 1792 snap 0 seq_num 1792) 2023-08-27T20:54:21.906 INFO:tasks.rados.rados.0.smithi093.stdout:2791: seq_num 1794 ranges {551758=739661,1980696=171062} 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2791: writing smithi093153278-395 from 551758 to 1291419 tid 1 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2791: writing smithi093153278-395 from 1980696 to 2151758 tid 2 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2787: done (4 left) 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2792: delete oid 65 current snap is 0 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2788: finishing write tid 1 to smithi093153278-78 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2788: finishing write tid 2 to smithi093153278-78 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2788: finishing write tid 3 to smithi093153278-78 2023-08-27T20:54:21.907 INFO:tasks.rados.rados.0.smithi093.stdout:2788: finishing write tid 4 to smithi093153278-78 2023-08-27T20:54:21.908 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 332 (ObjNum 1793 snap 0 seq_num 1793) dirty exists 2023-08-27T20:54:21.915 INFO:tasks.rados.rados.0.smithi093.stdout:2788: left oid 78 (ObjNum 1793 snap 0 seq_num 1793) 2023-08-27T20:54:21.915 INFO:tasks.rados.rados.0.smithi093.stdout:2790: expect (ObjNum 1127 snap 0 seq_num 1127) 2023-08-27T20:54:21.916 INFO:tasks.rados.rados.0.smithi093.stdout:2789: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:21.916 INFO:tasks.rados.rados.0.smithi093.stdout:2791: finishing write tid 1 to smithi093153278-395 2023-08-27T20:54:21.917 INFO:tasks.rados.rados.0.smithi093.stdout:2791: finishing write tid 2 to smithi093153278-395 2023-08-27T20:54:21.918 INFO:tasks.rados.rados.0.smithi093.stdout:2791: finishing write tid 3 to smithi093153278-395 2023-08-27T20:54:21.918 INFO:tasks.rados.rados.0.smithi093.stdout:2791: finishing write tid 4 to smithi093153278-395 2023-08-27T20:54:21.919 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 487 (ObjNum 1794 snap 0 seq_num 1794) dirty exists 2023-08-27T20:54:21.919 INFO:tasks.rados.rados.0.smithi093.stdout:2791: left oid 395 (ObjNum 1794 snap 0 seq_num 1794) 2023-08-27T20:54:21.919 INFO:tasks.rados.rados.0.smithi093.stdout:2788: done (4 left) 2023-08-27T20:54:21.923 INFO:tasks.rados.rados.0.smithi093.stdout:2789: done (3 left) 2023-08-27T20:54:21.923 INFO:tasks.rados.rados.0.smithi093.stdout:2790: done (2 left) 2023-08-27T20:54:21.924 INFO:tasks.rados.rados.0.smithi093.stdout:2791: done (1 left) 2023-08-27T20:54:21.924 INFO:tasks.rados.rados.0.smithi093.stdout:2792: done (0 left) 2023-08-27T20:54:21.924 INFO:tasks.rados.rados.0.smithi093.stdout:2793: write oid 470 current snap is 0 2023-08-27T20:54:22.061 INFO:tasks.rados.rados.0.smithi093.stdout:2793: seq_num 1796 ranges {585372=606512,1814013=371359} 2023-08-27T20:54:22.061 INFO:tasks.rados.rados.0.smithi093.stdout:2793: writing smithi093153278-470 from 585372 to 1191884 tid 1 2023-08-27T20:54:22.061 INFO:tasks.rados.rados.0.smithi093.stdout:2793: writing smithi093153278-470 from 1814013 to 2185372 tid 2 2023-08-27T20:54:22.066 INFO:tasks.rados.rados.0.smithi093.stdout:2794: read oid 312 snap -1 2023-08-27T20:54:22.067 INFO:tasks.rados.rados.0.smithi093.stdout:2794: expect (ObjNum 1242 snap 0 seq_num 1242) 2023-08-27T20:54:22.074 INFO:tasks.rados.rados.0.smithi093.stdout:2795: read oid 61 snap -1 2023-08-27T20:54:22.074 INFO:tasks.rados.rados.0.smithi093.stdout:2795: expect (ObjNum 1642 snap 0 seq_num 1642) 2023-08-27T20:54:22.074 INFO:tasks.rados.rados.0.smithi093.stdout:2796: write oid 414 current snap is 0 2023-08-27T20:54:22.074 INFO:tasks.rados.rados.0.smithi093.stdout:2793: finishing write tid 1 to smithi093153278-470 2023-08-27T20:54:22.074 INFO:tasks.rados.rados.0.smithi093.stdout:2796: seq_num 1797 ranges {602179=681761,1756142=428298} 2023-08-27T20:54:22.074 INFO:tasks.rados.rados.0.smithi093.stdout:2796: writing smithi093153278-414 from 602179 to 1283940 tid 1 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2796: writing smithi093153278-414 from 1756142 to 2184440 tid 2 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2793: finishing write tid 2 to smithi093153278-470 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2793: finishing write tid 3 to smithi093153278-470 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2793: finishing write tid 4 to smithi093153278-470 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 470 v 490 (ObjNum 1796 snap 0 seq_num 1796) dirty exists 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2793: left oid 470 (ObjNum 1796 snap 0 seq_num 1796) 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2796: finishing write tid 1 to smithi093153278-414 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2793: done (3 left) 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2797: write oid 230 current snap is 0 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2797: seq_num 1798 ranges {618986=757010,1814624=404362} 2023-08-27T20:54:22.075 INFO:tasks.rados.rados.0.smithi093.stdout:2797: writing smithi093153278-230 from 618986 to 1375996 tid 1 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2797: writing smithi093153278-230 from 1814624 to 2218986 tid 2 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2798: write oid 404 current snap is 0 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2794: expect (ObjNum 1242 snap 0 seq_num 1242) 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2796: finishing write tid 2 to smithi093153278-414 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2796: finishing write tid 3 to smithi093153278-414 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2796: finishing write tid 4 to smithi093153278-414 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 414 v 436 (ObjNum 1797 snap 0 seq_num 1797) dirty exists 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2796: left oid 414 (ObjNum 1797 snap 0 seq_num 1797) 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2795: expect (ObjNum 1642 snap 0 seq_num 1642) 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2797: finishing write tid 1 to smithi093153278-230 2023-08-27T20:54:22.076 INFO:tasks.rados.rados.0.smithi093.stdout:2797: finishing write tid 2 to smithi093153278-230 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2797: finishing write tid 3 to smithi093153278-230 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2797: finishing write tid 4 to smithi093153278-230 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 358 (ObjNum 1798 snap 0 seq_num 1798) dirty exists 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2797: left oid 230 (ObjNum 1798 snap 0 seq_num 1798) 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2798: seq_num 1799 ranges {635793=432259,1473106=431967} 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2798: writing smithi093153278-404 from 635793 to 1068052 tid 1 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2798: writing smithi093153278-404 from 1473106 to 1905073 tid 2 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2794: done (4 left) 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2795: done (3 left) 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2796: done (2 left) 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2797: done (1 left) 2023-08-27T20:54:22.077 INFO:tasks.rados.rados.0.smithi093.stdout:2798: finishing write tid 1 to smithi093153278-404 2023-08-27T20:54:22.078 INFO:tasks.rados.rados.0.smithi093.stdout:2799: read oid 128 snap -1 2023-08-27T20:54:22.078 INFO:tasks.rados.rados.0.smithi093.stdout:2799: expect (ObjNum 1729 snap 0 seq_num 1729) 2023-08-27T20:54:22.078 INFO:tasks.rados.rados.0.smithi093.stdout:2800: read oid 16 snap -1 2023-08-27T20:54:22.078 INFO:tasks.rados.rados.0.smithi093.stdout:2800: expect (ObjNum 1491 snap 0 seq_num 1491) 2023-08-27T20:54:22.078 INFO:tasks.rados.rados.0.smithi093.stdout:2801: write oid 108 current snap is 0 2023-08-27T20:54:22.078 INFO:tasks.rados.rados.0.smithi093.stdout:2801: seq_num 1800 ranges {652600=507508,1931588=321012} 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2801: writing smithi093153278-108 from 652600 to 1160108 tid 1 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2801: writing smithi093153278-108 from 1931588 to 2252600 tid 2 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2802: write oid 280 current snap is 0 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2802: seq_num 1801 ranges {669407=582757,1873717=395690} 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2802: writing smithi093153278-280 from 669407 to 1252164 tid 1 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2802: writing smithi093153278-280 from 1873717 to 2269407 tid 2 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2803: read oid 290 snap -1 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2803: expect (ObjNum 1588 snap 0 seq_num 1588) 2023-08-27T20:54:22.083 INFO:tasks.rados.rados.0.smithi093.stdout:2804: read oid 489 snap -1 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2804: expect (ObjNum 1560 snap 0 seq_num 1560) 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2805: read oid 346 snap -1 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2805: expect (ObjNum 1713 snap 0 seq_num 1713) 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2806: write oid 207 current snap is 0 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2798: finishing write tid 2 to smithi093153278-404 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2798: finishing write tid 3 to smithi093153278-404 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2798: finishing write tid 4 to smithi093153278-404 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 404 v 361 (ObjNum 1799 snap 0 seq_num 1799) dirty exists 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2798: left oid 404 (ObjNum 1799 snap 0 seq_num 1799) 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2799: expect (ObjNum 1729 snap 0 seq_num 1729) 2023-08-27T20:54:22.084 INFO:tasks.rados.rados.0.smithi093.stdout:2800: expect (ObjNum 1491 snap 0 seq_num 1491) 2023-08-27T20:54:22.085 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2313363761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:22.085 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2313363761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:22.087 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:21 smithi136 ceph-mon[113531]: pgmap v24: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 14 MiB/s wr, 132 op/s 2023-08-27T20:54:22.224 INFO:tasks.rados.rados.0.smithi093.stdout:2801: finishing write tid 1 to smithi093153278-108 2023-08-27T20:54:22.224 INFO:tasks.rados.rados.0.smithi093.stdout:2801: finishing write tid 2 to smithi093153278-108 2023-08-27T20:54:22.224 INFO:tasks.rados.rados.0.smithi093.stdout:2801: finishing write tid 3 to smithi093153278-108 2023-08-27T20:54:22.225 INFO:tasks.rados.rados.0.smithi093.stdout:2801: finishing write tid 4 to smithi093153278-108 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 108 v 445 (ObjNum 1800 snap 0 seq_num 1800) dirty exists 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2801: left oid 108 (ObjNum 1800 snap 0 seq_num 1800) 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2802: finishing write tid 1 to smithi093153278-280 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2806: seq_num 1802 ranges {686214=774359,2048552=237662} 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2806: writing smithi093153278-207 from 686214 to 1460573 tid 1 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2806: writing smithi093153278-207 from 2048552 to 2286214 tid 2 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2798: done (8 left) 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2799: done (7 left) 2023-08-27T20:54:22.227 INFO:tasks.rados.rados.0.smithi093.stdout:2800: done (6 left) 2023-08-27T20:54:22.228 INFO:tasks.rados.rados.0.smithi093.stdout:2801: done (5 left) 2023-08-27T20:54:22.228 INFO:tasks.rados.rados.0.smithi093.stdout:2807: read oid 179 snap -1 2023-08-27T20:54:22.230 INFO:tasks.rados.rados.0.smithi093.stdout:2807: expect (ObjNum 1347 snap 0 seq_num 1347) 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2808: read oid 456 snap -1 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2808: expect (ObjNum 565 snap 0 seq_num 565) 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2809: read oid 208 snap -1 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2809: expect (ObjNum 1512 snap 0 seq_num 1512) 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2810: read oid 192 snap -1 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2810: expect (ObjNum 1130 snap 0 seq_num 1130) 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2811: write oid 41 current snap is 0 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2811: seq_num 1803 ranges {703021=449608,1707034=595987} 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2811: writing smithi093153278-41 from 703021 to 1152629 tid 1 2023-08-27T20:54:22.231 INFO:tasks.rados.rados.0.smithi093.stdout:2811: writing smithi093153278-41 from 1707034 to 2303021 tid 2 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2812: read oid 314 snap -1 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2812: expect (ObjNum 1623 snap 0 seq_num 1623) 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2813: write oid 79 current snap is 0 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2804: expect (ObjNum 1560 snap 0 seq_num 1560) 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2802: finishing write tid 2 to smithi093153278-280 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2802: finishing write tid 3 to smithi093153278-280 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2802: finishing write tid 4 to smithi093153278-280 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 280 v 320 (ObjNum 1801 snap 0 seq_num 1801) dirty exists 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2802: left oid 280 (ObjNum 1801 snap 0 seq_num 1801) 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2813: seq_num 1804 ranges {719828=524857,1765516=554312} 2023-08-27T20:54:22.232 INFO:tasks.rados.rados.0.smithi093.stdout:2813: writing smithi093153278-79 from 719828 to 1244685 tid 1 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2813: writing smithi093153278-79 from 1765516 to 2319828 tid 2 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2802: done (11 left) 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2804: done (10 left) 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2814: read oid 370 snap -1 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2814: expect (ObjNum 369 snap 0 seq_num 369) 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2815: read oid 399 snap -1 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2815: expect (ObjNum 523 snap 0 seq_num 523) 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2816: read oid 328 snap -1 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2816: expect (ObjNum 1508 snap 0 seq_num 1508) 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2817: delete oid 310 current snap is 0 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2805: expect (ObjNum 1713 snap 0 seq_num 1713) 2023-08-27T20:54:22.233 INFO:tasks.rados.rados.0.smithi093.stdout:2803: expect (ObjNum 1588 snap 0 seq_num 1588) 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2806: finishing write tid 1 to smithi093153278-207 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2806: finishing write tid 2 to smithi093153278-207 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2806: finishing write tid 3 to smithi093153278-207 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2806: finishing write tid 4 to smithi093153278-207 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 207 v 379 (ObjNum 1802 snap 0 seq_num 1802) dirty exists 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2806: left oid 207 (ObjNum 1802 snap 0 seq_num 1802) 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2807: expect (ObjNum 1347 snap 0 seq_num 1347) 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2809: expect (ObjNum 1512 snap 0 seq_num 1512) 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2808: expect (ObjNum 565 snap 0 seq_num 565) 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2811: finishing write tid 1 to smithi093153278-41 2023-08-27T20:54:22.234 INFO:tasks.rados.rados.0.smithi093.stdout:2810: expect (ObjNum 1130 snap 0 seq_num 1130) 2023-08-27T20:54:22.235 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:21 smithi093 ceph-mon[111335]: pgmap v24: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 14 MiB/s wr, 132 op/s 2023-08-27T20:54:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:21 smithi093 ceph-mon[118902]: pgmap v24: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 14 MiB/s wr, 132 op/s 2023-08-27T20:54:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:21.625Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:21.626Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:22.241 INFO:tasks.rados.rados.0.smithi093.stdout:2811: finishing write tid 2 to smithi093153278-41 2023-08-27T20:54:22.241 INFO:tasks.rados.rados.0.smithi093.stdout:2811: finishing write tid 3 to smithi093153278-41 2023-08-27T20:54:22.241 INFO:tasks.rados.rados.0.smithi093.stdout:2811: finishing write tid 4 to smithi093153278-41 2023-08-27T20:54:22.241 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 41 v 493 (ObjNum 1803 snap 0 seq_num 1803) dirty exists 2023-08-27T20:54:22.241 INFO:tasks.rados.rados.0.smithi093.stdout:2811: left oid 41 (ObjNum 1803 snap 0 seq_num 1803) 2023-08-27T20:54:22.241 INFO:tasks.rados.rados.0.smithi093.stdout:2803: done (13 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2805: done (12 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2806: done (11 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2807: done (10 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2808: done (9 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2809: done (8 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2810: done (7 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2811: done (6 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2817: done (5 left) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2818: read oid 83 snap -1 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2818: expect (ObjNum 1505 snap 0 seq_num 1505) 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2819: write oid 181 current snap is 0 2023-08-27T20:54:22.242 INFO:tasks.rados.rados.0.smithi093.stdout:2812: expect (ObjNum 1623 snap 0 seq_num 1623) 2023-08-27T20:54:22.274 INFO:tasks.rados.rados.0.smithi093.stdout:2813: finishing write tid 1 to smithi093153278-79 2023-08-27T20:54:22.389 INFO:tasks.rados.rados.0.smithi093.stdout:2819: seq_num 1806 ranges {753442=675355,2166127=187315} 2023-08-27T20:54:22.390 INFO:tasks.rados.rados.0.smithi093.stdout:2819: writing smithi093153278-181 from 753442 to 1428797 tid 1 2023-08-27T20:54:22.390 INFO:tasks.rados.rados.0.smithi093.stdout:2819: writing smithi093153278-181 from 2166127 to 2353442 tid 2 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2812: done (6 left) 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2820: delete oid 192 current snap is 0 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2814: expect (ObjNum 369 snap 0 seq_num 369) 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2813: finishing write tid 2 to smithi093153278-79 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2813: finishing write tid 3 to smithi093153278-79 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2813: finishing write tid 4 to smithi093153278-79 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 79 v 496 (ObjNum 1804 snap 0 seq_num 1804) dirty exists 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2813: left oid 79 (ObjNum 1804 snap 0 seq_num 1804) 2023-08-27T20:54:22.391 INFO:tasks.rados.rados.0.smithi093.stdout:2815: expect (ObjNum 523 snap 0 seq_num 523) 2023-08-27T20:54:22.480 INFO:tasks.rados.rados.0.smithi093.stdout:2816: expect (ObjNum 1508 snap 0 seq_num 1508) 2023-08-27T20:54:22.757 INFO:tasks.rados.rados.0.smithi093.stdout:2818: expect (ObjNum 1505 snap 0 seq_num 1505) 2023-08-27T20:54:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/742306374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:23.043 INFO:tasks.rados.rados.0.smithi093.stdout:2819: finishing write tid 1 to smithi093153278-181 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2819: finishing write tid 2 to smithi093153278-181 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2819: finishing write tid 3 to smithi093153278-181 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2819: finishing write tid 4 to smithi093153278-181 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 181 v 331 (ObjNum 1806 snap 0 seq_num 1806) dirty exists 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2819: left oid 181 (ObjNum 1806 snap 0 seq_num 1806) 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2813: done (6 left) 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2814: done (5 left) 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2815: done (4 left) 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2816: done (3 left) 2023-08-27T20:54:23.044 INFO:tasks.rados.rados.0.smithi093.stdout:2818: done (2 left) 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2819: done (1 left) 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2820: done (0 left) 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2821: write oid 21 current snap is 0 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2821: seq_num 1808 ranges {787056=425853,1883091=503965} 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2821: writing smithi093153278-21 from 787056 to 1212909 tid 1 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2821: writing smithi093153278-21 from 1883091 to 2387056 tid 2 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2822: write oid 237 current snap is 0 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2821: finishing write tid 1 to smithi093153278-21 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2822: seq_num 1809 ranges {403863=501102,1425220=733959} 2023-08-27T20:54:23.045 INFO:tasks.rados.rados.0.smithi093.stdout:2822: writing smithi093153278-237 from 403863 to 904965 tid 1 2023-08-27T20:54:23.046 INFO:tasks.rados.rados.0.smithi093.stdout:2822: writing smithi093153278-237 from 1425220 to 2159179 tid 2 2023-08-27T20:54:23.047 INFO:tasks.rados.rados.0.smithi093.stdout:2823: write oid 450 current snap is 0 2023-08-27T20:54:23.047 INFO:tasks.rados.rados.0.smithi093.stdout:2821: finishing write tid 2 to smithi093153278-21 2023-08-27T20:54:23.047 INFO:tasks.rados.rados.0.smithi093.stdout:2823: seq_num 1810 ranges {420670=692704,1600055=593970} 2023-08-27T20:54:23.047 INFO:tasks.rados.rados.0.smithi093.stdout:2823: writing smithi093153278-450 from 420670 to 1113374 tid 1 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2823: writing smithi093153278-450 from 1600055 to 2194025 tid 2 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2824: read oid 425 snap -1 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2824: expect (ObjNum 1282 snap 0 seq_num 1282) 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2825: read oid 239 snap -1 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2825: expect (ObjNum 238 snap 0 seq_num 238) 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2826: write oid 329 current snap is 0 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2821: finishing write tid 3 to smithi093153278-21 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2821: finishing write tid 4 to smithi093153278-21 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 21 v 439 (ObjNum 1808 snap 0 seq_num 1808) dirty exists 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2821: left oid 21 (ObjNum 1808 snap 0 seq_num 1808) 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2822: finishing write tid 1 to smithi093153278-237 2023-08-27T20:54:23.048 INFO:tasks.rados.rados.0.smithi093.stdout:2826: seq_num 1811 ranges {437477=767953,1658537=737628} 2023-08-27T20:54:23.049 INFO:tasks.rados.rados.0.smithi093.stdout:2826: writing smithi093153278-329 from 437477 to 1205430 tid 1 2023-08-27T20:54:23.049 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/742306374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:23.049 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/742306374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:23.052 INFO:tasks.rados.rados.0.smithi093.stdout:2826: writing smithi093153278-329 from 1658537 to 2396165 tid 2 2023-08-27T20:54:23.069 INFO:tasks.rados.rados.0.smithi093.stdout:2821: done (5 left) 2023-08-27T20:54:23.069 INFO:tasks.rados.rados.0.smithi093.stdout:2827: write oid 46 current snap is 0 2023-08-27T20:54:23.069 INFO:tasks.rados.rados.0.smithi093.stdout:2822: finishing write tid 2 to smithi093153278-237 2023-08-27T20:54:23.069 INFO:tasks.rados.rados.0.smithi093.stdout:2822: finishing write tid 3 to smithi093153278-237 2023-08-27T20:54:23.069 INFO:tasks.rados.rados.0.smithi093.stdout:2827: seq_num 1812 ranges {454284=443202,1317019=597639,2376463=77821} 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2827: writing smithi093153278-46 from 454284 to 897486 tid 1 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2827: writing smithi093153278-46 from 1317019 to 1914658 tid 2 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2827: writing smithi093153278-46 from 2376463 to 2454284 tid 3 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2828: read oid 288 snap -1 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2828: expect (ObjNum 1511 snap 0 seq_num 1511) 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2829: write oid 137 current snap is 0 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2822: finishing write tid 4 to smithi093153278-237 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 237 v 372 (ObjNum 1809 snap 0 seq_num 1809) dirty exists 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2822: left oid 237 (ObjNum 1809 snap 0 seq_num 1809) 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2823: finishing write tid 1 to smithi093153278-450 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2829: seq_num 1813 ranges {471091=518451,1659148=741297} 2023-08-27T20:54:23.070 INFO:tasks.rados.rados.0.smithi093.stdout:2829: writing smithi093153278-137 from 471091 to 989542 tid 1 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2829: writing smithi093153278-137 from 1659148 to 2400445 tid 2 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2822: done (7 left) 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2830: write oid 304 current snap is 0 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2830: seq_num 1814 ranges {487898=593700,1717630=484955} 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2830: writing smithi093153278-304 from 487898 to 1081598 tid 1 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2830: writing smithi093153278-304 from 1717630 to 2202585 tid 2 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2831: read oid 98 snap -1 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2831: expect deleted 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2832: write oid 222 current snap is 0 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2823: finishing write tid 2 to smithi093153278-450 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2823: finishing write tid 3 to smithi093153278-450 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:2823: finishing write tid 4 to smithi093153278-450 2023-08-27T20:54:23.071 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 450 v 307 (ObjNum 1810 snap 0 seq_num 1810) dirty exists 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2823: left oid 450 (ObjNum 1810 snap 0 seq_num 1810) 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2832: seq_num 1815 ranges {504705=668949,1776112=728593} 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2832: writing smithi093153278-222 from 504705 to 1173654 tid 1 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2832: writing smithi093153278-222 from 1776112 to 2504705 tid 2 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2823: done (9 left) 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2833: read oid 167 snap -1 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2833: expect (ObjNum 1709 snap 0 seq_num 1709) 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2834: delete oid 181 current snap is 0 2023-08-27T20:54:23.072 INFO:tasks.rados.rados.0.smithi093.stdout:2825: expect (ObjNum 238 snap 0 seq_num 238) 2023-08-27T20:54:23.109 INFO:tasks.rados.rados.0.smithi093.stdout:2824: expect (ObjNum 1282 snap 0 seq_num 1282) 2023-08-27T20:54:23.296 INFO:tasks.rados.rados.0.smithi093.stdout:2826: finishing write tid 1 to smithi093153278-329 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2826: finishing write tid 2 to smithi093153278-329 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2826: finishing write tid 3 to smithi093153278-329 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2826: finishing write tid 4 to smithi093153278-329 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 329 v 334 (ObjNum 1811 snap 0 seq_num 1811) dirty exists 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2826: left oid 329 (ObjNum 1811 snap 0 seq_num 1811) 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2827: finishing write tid 1 to smithi093153278-46 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2827: finishing write tid 2 to smithi093153278-46 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2827: finishing write tid 3 to smithi093153278-46 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2827: finishing write tid 4 to smithi093153278-46 2023-08-27T20:54:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:2824: done (10 left) 2023-08-27T20:54:23.370 INFO:tasks.rados.rados.0.smithi093.stdout:2825: done (9 left) 2023-08-27T20:54:23.370 INFO:tasks.rados.rados.0.smithi093.stdout:2826: done (8 left) 2023-08-27T20:54:23.370 INFO:tasks.rados.rados.0.smithi093.stdout:2834: done (7 left) 2023-08-27T20:54:23.370 INFO:tasks.rados.rados.0.smithi093.stdout:2835: write oid 312 current snap is 0 2023-08-27T20:54:23.370 INFO:tasks.rados.rados.0.smithi093.stdout:2827: finishing write tid 5 to smithi093153278-46 2023-08-27T20:54:23.370 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 46 v 500 (ObjNum 1812 snap 0 seq_num 1812) dirty exists 2023-08-27T20:54:23.371 INFO:tasks.rados.rados.0.smithi093.stdout:2827: left oid 46 (ObjNum 1812 snap 0 seq_num 1812) 2023-08-27T20:54:23.371 INFO:tasks.rados.rados.0.smithi093.stdout:2835: seq_num 1817 ranges {538319=535800,1493076=748635} 2023-08-27T20:54:23.371 INFO:tasks.rados.rados.0.smithi093.stdout:2835: writing smithi093153278-312 from 538319 to 1074119 tid 1 2023-08-27T20:54:23.374 INFO:tasks.rados.rados.0.smithi093.stdout:2835: writing smithi093153278-312 from 1493076 to 2241711 tid 2 2023-08-27T20:54:23.374 INFO:tasks.rados.rados.0.smithi093.stdout:2827: done (7 left) 2023-08-27T20:54:23.374 INFO:tasks.rados.rados.0.smithi093.stdout:2836: write oid 164 current snap is 0 2023-08-27T20:54:23.374 INFO:tasks.rados.rados.0.smithi093.stdout:2829: finishing write tid 1 to smithi093153278-137 2023-08-27T20:54:23.374 INFO:tasks.rados.rados.0.smithi093.stdout:2828: expect (ObjNum 1511 snap 0 seq_num 1511) 2023-08-27T20:54:23.492 INFO:tasks.rados.rados.0.smithi093.stdout:2829: finishing write tid 2 to smithi093153278-137 2023-08-27T20:54:23.531 INFO:tasks.rados.rados.0.smithi093.stdout:2829: finishing write tid 3 to smithi093153278-137 2023-08-27T20:54:23.532 INFO:tasks.rados.rados.0.smithi093.stdout:2829: finishing write tid 4 to smithi093153278-137 2023-08-27T20:54:23.532 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 137 v 442 (ObjNum 1813 snap 0 seq_num 1813) dirty exists 2023-08-27T20:54:23.532 INFO:tasks.rados.rados.0.smithi093.stdout:2829: left oid 137 (ObjNum 1813 snap 0 seq_num 1813) 2023-08-27T20:54:23.532 INFO:tasks.rados.rados.0.smithi093.stdout:2830: finishing write tid 1 to smithi093153278-304 2023-08-27T20:54:23.532 INFO:tasks.rados.rados.0.smithi093.stdout:2836: seq_num 1818 ranges {555126=611049,1951558=492293} 2023-08-27T20:54:23.533 INFO:tasks.rados.rados.0.smithi093.stdout:2836: writing smithi093153278-164 from 555126 to 1166175 tid 1 2023-08-27T20:54:23.852 INFO:tasks.rados.rados.0.smithi093.stdout:2836: writing smithi093153278-164 from 1951558 to 2443851 tid 2 2023-08-27T20:54:23.852 INFO:tasks.rados.rados.0.smithi093.stdout:2828: done (7 left) 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2829: done (6 left) 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2837: read oid 321 snap -1 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2837: expect (ObjNum 1637 snap 0 seq_num 1637) 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2838: delete oid 217 current snap is 0 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2830: finishing write tid 2 to smithi093153278-304 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2830: finishing write tid 3 to smithi093153278-304 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2830: finishing write tid 4 to smithi093153278-304 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 304 v 445 (ObjNum 1814 snap 0 seq_num 1814) dirty exists 2023-08-27T20:54:23.853 INFO:tasks.rados.rados.0.smithi093.stdout:2830: left oid 304 (ObjNum 1814 snap 0 seq_num 1814) 2023-08-27T20:54:23.855 INFO:tasks.rados.rados.0.smithi093.stdout:2832: finishing write tid 1 to smithi093153278-222 2023-08-27T20:54:23.855 INFO:tasks.rados.rados.0.smithi093.stdout:2832: finishing write tid 2 to smithi093153278-222 2023-08-27T20:54:23.855 INFO:tasks.rados.rados.0.smithi093.stdout:2832: finishing write tid 3 to smithi093153278-222 2023-08-27T20:54:23.855 INFO:tasks.rados.rados.0.smithi093.stdout:2832: finishing write tid 4 to smithi093153278-222 2023-08-27T20:54:23.855 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 222 v 398 (ObjNum 1815 snap 0 seq_num 1815) dirty exists 2023-08-27T20:54:23.858 INFO:tasks.rados.rados.0.smithi093.stdout:2832: left oid 222 (ObjNum 1815 snap 0 seq_num 1815) 2023-08-27T20:54:23.858 INFO:tasks.rados.rados.0.smithi093.stdout:2833: expect (ObjNum 1709 snap 0 seq_num 1709) 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:2835: finishing write tid 1 to smithi093153278-312 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:2835: finishing write tid 2 to smithi093153278-312 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:2835: finishing write tid 3 to smithi093153278-312 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:2835: finishing write tid 4 to smithi093153278-312 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 312 v 338 (ObjNum 1817 snap 0 seq_num 1817) dirty exists 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:2835: left oid 312 (ObjNum 1817 snap 0 seq_num 1817) 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:2836: finishing write tid 1 to smithi093153278-164 2023-08-27T20:54:23.859 INFO:tasks.rados.rados.0.smithi093.stdout:2830: done (7 left) 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2831: done (6 left) 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2832: done (5 left) 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2833: done (4 left) 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2835: done (3 left) 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2838: done (2 left) 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2839: write oid 289 current snap is 0 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2836: finishing write tid 2 to smithi093153278-164 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2836: finishing write tid 3 to smithi093153278-164 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2836: finishing write tid 4 to smithi093153278-164 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 164 v 338 (ObjNum 1818 snap 0 seq_num 1818) dirty exists 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2836: left oid 164 (ObjNum 1818 snap 0 seq_num 1818) 2023-08-27T20:54:23.860 INFO:tasks.rados.rados.0.smithi093.stdout:2837: expect (ObjNum 1637 snap 0 seq_num 1637) 2023-08-27T20:54:23.865 INFO:tasks.rados.rados.0.smithi093.stdout:2839: seq_num 1820 ranges {588740=761547,2068522=495962} 2023-08-27T20:54:23.872 INFO:tasks.rados.rados.0.smithi093.stdout:2839: writing smithi093153278-289 from 588740 to 1350287 tid 1 2023-08-27T20:54:24.015 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:23 smithi136 ceph-mon[113531]: pgmap v25: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 9.4 MiB/s wr, 86 op/s 2023-08-27T20:54:24.015 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:24.015 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:24.017 INFO:tasks.rados.rados.0.smithi093.stdout:2839: writing smithi093153278-289 from 2068522 to 2564484 tid 2 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2836: done (2 left) 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2837: done (1 left) 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2840: delete oid 222 current snap is 0 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2839: finishing write tid 1 to smithi093153278-289 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2840: done (1 left) 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2841: delete oid 396 current snap is 0 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2839: finishing write tid 2 to smithi093153278-289 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2841: done (1 left) 2023-08-27T20:54:24.018 INFO:tasks.rados.rados.0.smithi093.stdout:2842: read oid 205 snap -1 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2842: expect (ObjNum 1554 snap 0 seq_num 1554) 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2843: read oid 36 snap -1 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2843: expect (ObjNum 1742 snap 0 seq_num 1742) 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2844: read oid 343 snap -1 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2844: expect (ObjNum 577 snap 0 seq_num 577) 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2845: write oid 226 current snap is 0 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2839: finishing write tid 3 to smithi093153278-289 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2845: seq_num 1823 ranges {639161=587294,1727615=759642} 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2845: writing smithi093153278-226 from 639161 to 1226455 tid 1 2023-08-27T20:54:24.019 INFO:tasks.rados.rados.0.smithi093.stdout:2845: writing smithi093153278-226 from 1727615 to 2487257 tid 2 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2846: write oid 230 current snap is 0 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2839: finishing write tid 4 to smithi093153278-289 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 289 v 382 (ObjNum 1820 snap 0 seq_num 1820) dirty exists 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2839: left oid 289 (ObjNum 1820 snap 0 seq_num 1820) 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2846: seq_num 1824 ranges {655968=662543,1786097=503300} 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2846: writing smithi093153278-230 from 655968 to 1318511 tid 1 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2846: writing smithi093153278-230 from 1786097 to 2289397 tid 2 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2839: done (5 left) 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2847: write oid 376 current snap is 0 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2847: seq_num 1825 ranges {672775=454145,1844579=763311} 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2847: writing smithi093153278-376 from 672775 to 1126920 tid 1 2023-08-27T20:54:24.020 INFO:tasks.rados.rados.0.smithi093.stdout:2847: writing smithi093153278-376 from 1844579 to 2607890 tid 2 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2848: read oid 378 snap -1 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2848: expect (ObjNum 1757 snap 0 seq_num 1757) 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2849: read oid 206 snap -1 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2849: expect (ObjNum 205 snap 0 seq_num 205) 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2850: read oid 127 snap -1 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2850: expect deleted 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2851: write oid 47 current snap is 0 2023-08-27T20:54:24.021 INFO:tasks.rados.rados.0.smithi093.stdout:2843: expect (ObjNum 1742 snap 0 seq_num 1742) 2023-08-27T20:54:24.021 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:23.515Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:24.022 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:23.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:24.022 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-mon[111335]: pgmap v25: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 9.4 MiB/s wr, 86 op/s 2023-08-27T20:54:24.022 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:24.022 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:24.022 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-mon[118902]: pgmap v25: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 9.4 MiB/s wr, 86 op/s 2023-08-27T20:54:24.022 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:54:24.022 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:24.138 INFO:tasks.rados.rados.0.smithi093.stdout:2842: expect (ObjNum 1554 snap 0 seq_num 1554) 2023-08-27T20:54:24.278 INFO:tasks.rados.rados.0.smithi093.stdout:2845: finishing write tid 1 to smithi093153278-226 2023-08-27T20:54:24.344 INFO:tasks.rados.rados.0.smithi093.stdout:2844: expect (ObjNum 577 snap 0 seq_num 577) 2023-08-27T20:54:24.444 INFO:tasks.rados.rados.0.smithi093.stdout:2845: finishing write tid 2 to smithi093153278-226 2023-08-27T20:54:24.504 INFO:tasks.rados.rados.0.smithi093.stdout:2845: finishing write tid 3 to smithi093153278-226 2023-08-27T20:54:24.504 INFO:tasks.rados.rados.0.smithi093.stdout:2851: seq_num 1826 ranges {689582=529394,1903061=506969} 2023-08-27T20:54:24.510 INFO:tasks.rados.rados.0.smithi093.stdout:2851: writing smithi093153278-47 from 689582 to 1218976 tid 1 2023-08-27T20:54:24.510 INFO:tasks.rados.rados.0.smithi093.stdout:2851: writing smithi093153278-47 from 1903061 to 2410030 tid 2 2023-08-27T20:54:24.510 INFO:tasks.rados.rados.0.smithi093.stdout:2842: done (9 left) 2023-08-27T20:54:24.510 INFO:tasks.rados.rados.0.smithi093.stdout:2843: done (8 left) 2023-08-27T20:54:24.517 INFO:tasks.rados.rados.0.smithi093.stdout:2844: done (7 left) 2023-08-27T20:54:24.517 INFO:tasks.rados.rados.0.smithi093.stdout:2852: read oid 460 snap -1 2023-08-27T20:54:24.517 INFO:tasks.rados.rados.0.smithi093.stdout:2852: expect (ObjNum 1789 snap 0 seq_num 1789) 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2853: read oid 232 snap -1 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2853: expect (ObjNum 1525 snap 0 seq_num 1525) 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2854: read oid 348 snap -1 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2854: expect (ObjNum 1457 snap 0 seq_num 1457) 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2855: write oid 148 current snap is 0 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2855: seq_num 1827 ranges {706389=604643,1961543=650627} 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2855: writing smithi093153278-148 from 706389 to 1311032 tid 1 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2855: writing smithi093153278-148 from 1961543 to 2612170 tid 2 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2856: write oid 137 current snap is 0 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:2845: finishing write tid 4 to smithi093153278-226 2023-08-27T20:54:24.518 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 226 v 375 (ObjNum 1823 snap 0 seq_num 1823) dirty exists 2023-08-27T20:54:24.519 INFO:tasks.rados.rados.0.smithi093.stdout:2845: left oid 226 (ObjNum 1823 snap 0 seq_num 1823) 2023-08-27T20:54:24.519 INFO:tasks.rados.rados.0.smithi093.stdout:2846: finishing write tid 1 to smithi093153278-230 2023-08-27T20:54:24.519 INFO:tasks.rados.rados.0.smithi093.stdout:2856: seq_num 1828 ranges {723196=679892,2020025=510638} 2023-08-27T20:54:24.519 INFO:tasks.rados.rados.0.smithi093.stdout:2856: writing smithi093153278-137 from 723196 to 1403088 tid 1 2023-08-27T20:54:24.519 INFO:tasks.rados.rados.0.smithi093.stdout:2856: writing smithi093153278-137 from 2020025 to 2530663 tid 2 2023-08-27T20:54:24.519 INFO:tasks.rados.rados.0.smithi093.stdout:2845: done (11 left) 2023-08-27T20:54:24.519 INFO:tasks.rados.rados.0.smithi093.stdout:2857: write oid 340 current snap is 0 2023-08-27T20:54:24.520 INFO:tasks.rados.rados.0.smithi093.stdout:2846: finishing write tid 2 to smithi093153278-230 2023-08-27T20:54:24.520 INFO:tasks.rados.rados.0.smithi093.stdout:2846: finishing write tid 3 to smithi093153278-230 2023-08-27T20:54:24.520 INFO:tasks.rados.rados.0.smithi093.stdout:2846: finishing write tid 4 to smithi093153278-230 2023-08-27T20:54:24.520 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 364 (ObjNum 1824 snap 0 seq_num 1824) dirty exists 2023-08-27T20:54:24.520 INFO:tasks.rados.rados.0.smithi093.stdout:2846: left oid 230 (ObjNum 1824 snap 0 seq_num 1824) 2023-08-27T20:54:24.520 INFO:tasks.rados.rados.0.smithi093.stdout:2847: finishing write tid 1 to smithi093153278-376 2023-08-27T20:54:24.520 INFO:tasks.rados.rados.0.smithi093.stdout:2857: seq_num 1829 ranges {740003=755141,2078507=654296} 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2857: writing smithi093153278-340 from 740003 to 1495144 tid 1 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2857: writing smithi093153278-340 from 2078507 to 2732803 tid 2 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2846: done (11 left) 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2858: read oid 99 snap -1 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2858: expect (ObjNum 969 snap 0 seq_num 969) 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2859: read oid 430 snap -1 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2859: expect deleted 2023-08-27T20:54:24.521 INFO:tasks.rados.rados.0.smithi093.stdout:2860: read oid 500 snap -1 2023-08-27T20:54:24.522 INFO:tasks.rados.rados.0.smithi093.stdout:2860: expect (ObjNum 875 snap 0 seq_num 875) 2023-08-27T20:54:24.522 INFO:tasks.rados.rados.0.smithi093.stdout:2861: write oid 167 current snap is 0 2023-08-27T20:54:24.522 INFO:tasks.rados.rados.0.smithi093.stdout:2861: seq_num 1830 ranges {756810=430390,1620636=514307} 2023-08-27T20:54:24.522 INFO:tasks.rados.rados.0.smithi093.stdout:2861: writing smithi093153278-167 from 756810 to 1187200 tid 1 2023-08-27T20:54:24.523 INFO:tasks.rados.rados.0.smithi093.stdout:2861: writing smithi093153278-167 from 1620636 to 2134943 tid 2 2023-08-27T20:54:24.524 INFO:tasks.rados.rados.0.smithi093.stdout:2862: read oid 20 snap -1 2023-08-27T20:54:24.524 INFO:tasks.rados.rados.0.smithi093.stdout:2862: expect (ObjNum 1572 snap 0 seq_num 1572) 2023-08-27T20:54:24.524 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:54:24.524 INFO:tasks.rados.rados.0.smithi093.stdout:2847: finishing write tid 2 to smithi093153278-376 2023-08-27T20:54:24.524 INFO:tasks.rados.rados.0.smithi093.stdout:2847: finishing write tid 3 to smithi093153278-376 2023-08-27T20:54:24.524 INFO:tasks.rados.rados.0.smithi093.stdout:2847: finishing write tid 4 to smithi093153278-376 2023-08-27T20:54:24.524 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 376 v 503 (ObjNum 1825 snap 0 seq_num 1825) dirty exists 2023-08-27T20:54:24.525 INFO:tasks.rados.rados.0.smithi093.stdout:2847: left oid 376 (ObjNum 1825 snap 0 seq_num 1825) 2023-08-27T20:54:24.525 INFO:tasks.rados.rados.0.smithi093.stdout:2848: expect (ObjNum 1757 snap 0 seq_num 1757) 2023-08-27T20:54:24.752 INFO:tasks.rados.rados.0.smithi093.stdout:2849: expect (ObjNum 205 snap 0 seq_num 205) 2023-08-27T20:54:24.860 INFO:tasks.rados.rados.0.smithi093.stdout:2851: finishing write tid 1 to smithi093153278-47 2023-08-27T20:54:24.907 INFO:tasks.rados.rados.0.smithi093.stdout:2852: expect (ObjNum 1789 snap 0 seq_num 1789) 2023-08-27T20:54:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3969963391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:24.985 INFO:tasks.rados.rados.0.smithi093.stdout:2851: finishing write tid 2 to smithi093153278-47 2023-08-27T20:54:24.985 INFO:tasks.rados.rados.0.smithi093.stdout:2851: finishing write tid 3 to smithi093153278-47 2023-08-27T20:54:24.985 INFO:tasks.rados.rados.0.smithi093.stdout:2851: finishing write tid 4 to smithi093153278-47 2023-08-27T20:54:24.985 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 47 v 378 (ObjNum 1826 snap 0 seq_num 1826) dirty exists 2023-08-27T20:54:24.985 INFO:tasks.rados.rados.0.smithi093.stdout:2851: left oid 47 (ObjNum 1826 snap 0 seq_num 1826) 2023-08-27T20:54:24.985 INFO:tasks.rados.rados.0.smithi093.stdout:2853: expect (ObjNum 1525 snap 0 seq_num 1525) 2023-08-27T20:54:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3969963391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3969963391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:25.126 INFO:tasks.rados.rados.0.smithi093.stdout:2854: expect (ObjNum 1457 snap 0 seq_num 1457) 2023-08-27T20:54:25.245 INFO:tasks.rados.rados.0.smithi093.stdout:2855: finishing write tid 1 to smithi093153278-148 2023-08-27T20:54:25.245 INFO:tasks.rados.rados.0.smithi093.stdout:2855: finishing write tid 2 to smithi093153278-148 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2855: finishing write tid 3 to smithi093153278-148 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2855: finishing write tid 4 to smithi093153278-148 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 148 v 336 (ObjNum 1827 snap 0 seq_num 1827) dirty exists 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2855: left oid 148 (ObjNum 1827 snap 0 seq_num 1827) 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2856: finishing write tid 1 to smithi093153278-137 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2847: done (15 left) 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2848: done (14 left) 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2849: done (13 left) 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2850: done (12 left) 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2851: done (11 left) 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2852: done (10 left) 2023-08-27T20:54:25.246 INFO:tasks.rados.rados.0.smithi093.stdout:2853: done (9 left) 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2854: done (8 left) 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2855: done (7 left) 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2863: read oid 15 snap -1 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2863: expect (ObjNum 1576 snap 0 seq_num 1576) 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2864: write oid 206 current snap is 0 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2856: finishing write tid 2 to smithi093153278-137 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2856: finishing write tid 3 to smithi093153278-137 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2856: finishing write tid 4 to smithi093153278-137 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 137 v 450 (ObjNum 1828 snap 0 seq_num 1828) dirty exists 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2856: left oid 137 (ObjNum 1828 snap 0 seq_num 1828) 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2857: finishing write tid 1 to smithi093153278-340 2023-08-27T20:54:25.247 INFO:tasks.rados.rados.0.smithi093.stdout:2858: expect (ObjNum 969 snap 0 seq_num 969) 2023-08-27T20:54:25.332 INFO:tasks.rados.rados.0.smithi093.stdout:2860: expect (ObjNum 875 snap 0 seq_num 875) 2023-08-27T20:54:25.490 INFO:tasks.rados.rados.0.smithi093.stdout:2857: finishing write tid 2 to smithi093153278-340 2023-08-27T20:54:25.490 INFO:tasks.rados.rados.0.smithi093.stdout:2857: finishing write tid 3 to smithi093153278-340 2023-08-27T20:54:25.490 INFO:tasks.rados.rados.0.smithi093.stdout:2857: finishing write tid 4 to smithi093153278-340 2023-08-27T20:54:25.490 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 340 v 339 (ObjNum 1829 snap 0 seq_num 1829) dirty exists 2023-08-27T20:54:25.490 INFO:tasks.rados.rados.0.smithi093.stdout:2857: left oid 340 (ObjNum 1829 snap 0 seq_num 1829) 2023-08-27T20:54:25.490 INFO:tasks.rados.rados.0.smithi093.stdout:2861: finishing write tid 1 to smithi093153278-167 2023-08-27T20:54:25.490 INFO:tasks.rados.rados.0.smithi093.stdout:2864: seq_num 1831 ranges {773617=505639,2079118=657965} 2023-08-27T20:54:25.496 INFO:tasks.rados.rados.0.smithi093.stdout:2864: writing smithi093153278-206 from 773617 to 1279256 tid 1 2023-08-27T20:54:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:2864: writing smithi093153278-206 from 2079118 to 2737083 tid 2 2023-08-27T20:54:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:2856: done (8 left) 2023-08-27T20:54:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:2857: done (7 left) 2023-08-27T20:54:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:2858: done (6 left) 2023-08-27T20:54:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:2859: done (5 left) 2023-08-27T20:54:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:2860: done (4 left) 2023-08-27T20:54:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:2865: write oid 326 current snap is 0 2023-08-27T20:54:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:2861: finishing write tid 2 to smithi093153278-167 2023-08-27T20:54:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:2861: finishing write tid 3 to smithi093153278-167 2023-08-27T20:54:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:2861: finishing write tid 4 to smithi093153278-167 2023-08-27T20:54:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 167 v 448 (ObjNum 1830 snap 0 seq_num 1830) dirty exists 2023-08-27T20:54:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:2861: left oid 167 (ObjNum 1830 snap 0 seq_num 1830) 2023-08-27T20:54:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:2862: expect (ObjNum 1572 snap 0 seq_num 1572) 2023-08-27T20:54:25.612 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:54:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:54:25] "GET /metrics HTTP/1.1" 200 242534 "" "Prometheus/2.43.0" 2023-08-27T20:54:25.647 INFO:tasks.rados.rados.0.smithi093.stdout:2863: expect (ObjNum 1576 snap 0 seq_num 1576) 2023-08-27T20:54:25.823 INFO:tasks.rados.rados.0.smithi093.stdout:2864: finishing write tid 1 to smithi093153278-206 2023-08-27T20:54:25.823 INFO:tasks.rados.rados.0.smithi093.stdout:2864: finishing write tid 2 to smithi093153278-206 2023-08-27T20:54:25.823 INFO:tasks.rados.rados.0.smithi093.stdout:2864: finishing write tid 3 to smithi093153278-206 2023-08-27T20:54:25.824 INFO:tasks.rados.rados.0.smithi093.stdout:2864: finishing write tid 4 to smithi093153278-206 2023-08-27T20:54:25.824 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 206 v 341 (ObjNum 1831 snap 0 seq_num 1831) dirty exists 2023-08-27T20:54:25.824 INFO:tasks.rados.rados.0.smithi093.stdout:2864: left oid 206 (ObjNum 1831 snap 0 seq_num 1831) 2023-08-27T20:54:25.824 INFO:tasks.rados.rados.0.smithi093.stdout:2865: seq_num 1832 ranges {790424=580888,2137600=517976} 2023-08-27T20:54:25.830 INFO:tasks.rados.rados.0.smithi093.stdout:2865: writing smithi093153278-326 from 790424 to 1371312 tid 1 2023-08-27T20:54:25.838 INFO:tasks.rados.rados.0.smithi093.stdout:2865: writing smithi093153278-326 from 2137600 to 2655576 tid 2 2023-08-27T20:54:25.838 INFO:tasks.rados.rados.0.smithi093.stdout:2861: done (4 left) 2023-08-27T20:54:25.838 INFO:tasks.rados.rados.0.smithi093.stdout:2862: done (3 left) 2023-08-27T20:54:25.838 INFO:tasks.rados.rados.0.smithi093.stdout:2863: done (2 left) 2023-08-27T20:54:25.838 INFO:tasks.rados.rados.0.smithi093.stdout:2864: done (1 left) 2023-08-27T20:54:25.838 INFO:tasks.rados.rados.0.smithi093.stdout:2866: write oid 63 current snap is 0 2023-08-27T20:54:25.839 INFO:tasks.rados.rados.0.smithi093.stdout:2865: finishing write tid 1 to smithi093153278-326 2023-08-27T20:54:25.839 INFO:tasks.rados.rados.0.smithi093.stdout:2866: seq_num 1833 ranges {407231=772490,1912435=661634} 2023-08-27T20:54:25.844 INFO:tasks.rados.rados.0.smithi093.stdout:2866: writing smithi093153278-63 from 407231 to 1179721 tid 1 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2866: writing smithi093153278-63 from 1912435 to 2574069 tid 2 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2867: write oid 46 current snap is 0 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2865: finishing write tid 2 to smithi093153278-326 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2865: finishing write tid 3 to smithi093153278-326 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2865: finishing write tid 4 to smithi093153278-326 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 310 (ObjNum 1832 snap 0 seq_num 1832) dirty exists 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2865: left oid 326 (ObjNum 1832 snap 0 seq_num 1832) 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2866: finishing write tid 1 to smithi093153278-63 2023-08-27T20:54:25.852 INFO:tasks.rados.rados.0.smithi093.stdout:2867: seq_num 1834 ranges {424038=447739,1454564=521645,2630710=193328} 2023-08-27T20:54:25.857 INFO:tasks.rados.rados.0.smithi093.stdout:2867: writing smithi093153278-46 from 424038 to 871777 tid 1 2023-08-27T20:54:25.863 INFO:tasks.rados.rados.0.smithi093.stdout:2867: writing smithi093153278-46 from 1454564 to 1976209 tid 2 2023-08-27T20:54:25.868 INFO:tasks.rados.rados.0.smithi093.stdout:2867: writing smithi093153278-46 from 2630710 to 2824038 tid 3 2023-08-27T20:54:25.868 INFO:tasks.rados.rados.0.smithi093.stdout:2865: done (2 left) 2023-08-27T20:54:25.868 INFO:tasks.rados.rados.0.smithi093.stdout:2868: write oid 235 current snap is 0 2023-08-27T20:54:25.868 INFO:tasks.rados.rados.0.smithi093.stdout:2866: finishing write tid 2 to smithi093153278-63 2023-08-27T20:54:25.868 INFO:tasks.rados.rados.0.smithi093.stdout:2868: seq_num 1835 ranges {440845=522988,1513046=665303} 2023-08-27T20:54:25.873 INFO:tasks.rados.rados.0.smithi093.stdout:2868: writing smithi093153278-235 from 440845 to 963833 tid 1 2023-08-27T20:54:25.880 INFO:tasks.rados.rados.0.smithi093.stdout:2868: writing smithi093153278-235 from 1513046 to 2178349 tid 2 2023-08-27T20:54:25.880 INFO:tasks.rados.rados.0.smithi093.stdout:2869: read oid 195 snap -1 2023-08-27T20:54:25.880 INFO:tasks.rados.rados.0.smithi093.stdout:2869: expect (ObjNum 1596 snap 0 seq_num 1596) 2023-08-27T20:54:25.880 INFO:tasks.rados.rados.0.smithi093.stdout:2870: write oid 365 current snap is 0 2023-08-27T20:54:25.880 INFO:tasks.rados.rados.0.smithi093.stdout:2866: finishing write tid 3 to smithi093153278-63 2023-08-27T20:54:25.881 INFO:tasks.rados.rados.0.smithi093.stdout:2870: seq_num 1836 ranges {457652=598237,1571528=525314,2685556=172096} 2023-08-27T20:54:25.886 INFO:tasks.rados.rados.0.smithi093.stdout:2870: writing smithi093153278-365 from 457652 to 1055889 tid 1 2023-08-27T20:54:25.891 INFO:tasks.rados.rados.0.smithi093.stdout:2870: writing smithi093153278-365 from 1571528 to 2096842 tid 2 2023-08-27T20:54:25.896 INFO:tasks.rados.rados.0.smithi093.stdout:2870: writing smithi093153278-365 from 2685556 to 2857652 tid 3 2023-08-27T20:54:25.896 INFO:tasks.rados.rados.0.smithi093.stdout:2871: write oid 194 current snap is 0 2023-08-27T20:54:25.896 INFO:tasks.rados.rados.0.smithi093.stdout:2866: finishing write tid 4 to smithi093153278-63 2023-08-27T20:54:25.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 63 v 342 (ObjNum 1833 snap 0 seq_num 1833) dirty exists 2023-08-27T20:54:25.897 INFO:tasks.rados.rados.0.smithi093.stdout:2866: left oid 63 (ObjNum 1833 snap 0 seq_num 1833) 2023-08-27T20:54:25.897 INFO:tasks.rados.rados.0.smithi093.stdout:2867: finishing write tid 1 to smithi093153278-46 2023-08-27T20:54:25.897 INFO:tasks.rados.rados.0.smithi093.stdout:2871: seq_num 1837 ranges {474459=673486,1630010=668972} 2023-08-27T20:54:25.902 INFO:tasks.rados.rados.0.smithi093.stdout:2871: writing smithi093153278-194 from 474459 to 1147945 tid 1 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2871: writing smithi093153278-194 from 1630010 to 2298982 tid 2 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2866: done (5 left) 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2872: read oid 255 snap -1 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2872: expect (ObjNum 1658 snap 0 seq_num 1658) 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2873: read oid 189 snap -1 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2873: expect (ObjNum 1455 snap 0 seq_num 1455) 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2874: read oid 140 snap -1 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2874: expect (ObjNum 1751 snap 0 seq_num 1751) 2023-08-27T20:54:25.910 INFO:tasks.rados.rados.0.smithi093.stdout:2875: read oid 488 snap -1 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2875: expect (ObjNum 968 snap 0 seq_num 968) 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2876: delete oid 302 current snap is 0 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2867: finishing write tid 2 to smithi093153278-46 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2867: finishing write tid 3 to smithi093153278-46 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2867: finishing write tid 4 to smithi093153278-46 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2867: finishing write tid 5 to smithi093153278-46 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 46 v 507 (ObjNum 1834 snap 0 seq_num 1834) dirty exists 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2867: left oid 46 (ObjNum 1834 snap 0 seq_num 1834) 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2868: finishing write tid 1 to smithi093153278-235 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2868: finishing write tid 2 to smithi093153278-235 2023-08-27T20:54:25.911 INFO:tasks.rados.rados.0.smithi093.stdout:2868: finishing write tid 3 to smithi093153278-235 2023-08-27T20:54:25.912 INFO:tasks.rados.rados.0.smithi093.stdout:2868: finishing write tid 4 to smithi093153278-235 2023-08-27T20:54:25.912 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 235 v 341 (ObjNum 1835 snap 0 seq_num 1835) dirty exists 2023-08-27T20:54:25.912 INFO:tasks.rados.rados.0.smithi093.stdout:2868: left oid 235 (ObjNum 1835 snap 0 seq_num 1835) 2023-08-27T20:54:25.912 INFO:tasks.rados.rados.0.smithi093.stdout:2870: finishing write tid 1 to smithi093153278-365 2023-08-27T20:54:25.912 INFO:tasks.rados.rados.0.smithi093.stdout:2869: expect (ObjNum 1596 snap 0 seq_num 1596) 2023-08-27T20:54:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:25 smithi136 ceph-mon[113531]: pgmap v26: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 9.4 MiB/s wr, 86 op/s 2023-08-27T20:54:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:25 smithi093 ceph-mon[111335]: pgmap v26: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 9.4 MiB/s wr, 86 op/s 2023-08-27T20:54:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:25 smithi093 ceph-mon[118902]: pgmap v26: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 9.4 MiB/s wr, 86 op/s 2023-08-27T20:54:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:2870: finishing write tid 2 to smithi093153278-365 2023-08-27T20:54:26.064 INFO:tasks.rados.rados.0.smithi093.stdout:2870: finishing write tid 3 to smithi093153278-365 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2870: finishing write tid 4 to smithi093153278-365 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2870: finishing write tid 5 to smithi093153278-365 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 365 v 454 (ObjNum 1836 snap 0 seq_num 1836) dirty exists 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2870: left oid 365 (ObjNum 1836 snap 0 seq_num 1836) 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2867: done (9 left) 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2868: done (8 left) 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2869: done (7 left) 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2870: done (6 left) 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2876: done (5 left) 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2877: read oid 302 snap -1 2023-08-27T20:54:26.065 INFO:tasks.rados.rados.0.smithi093.stdout:2877: expect deleted 2023-08-27T20:54:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:2878: read oid 294 snap -1 2023-08-27T20:54:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:2878: expect (ObjNum 1410 snap 0 seq_num 1410) 2023-08-27T20:54:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:2879: read oid 6 snap -1 2023-08-27T20:54:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:2879: expect (ObjNum 1552 snap 0 seq_num 1552) 2023-08-27T20:54:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:2880: write oid 426 current snap is 0 2023-08-27T20:54:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:2871: finishing write tid 1 to smithi093153278-194 2023-08-27T20:54:26.066 INFO:tasks.rados.rados.0.smithi093.stdout:2880: seq_num 1839 ranges {508073=423984,1630621=672641} 2023-08-27T20:54:26.069 INFO:tasks.rados.rados.0.smithi093.stdout:2880: writing smithi093153278-426 from 508073 to 932057 tid 1 2023-08-27T20:54:26.077 INFO:tasks.rados.rados.0.smithi093.stdout:2880: writing smithi093153278-426 from 1630621 to 2303262 tid 2 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2881: read oid 157 snap -1 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2881: expect (ObjNum 156 snap 0 seq_num 156) 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2882: write oid 384 current snap is 0 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2871: finishing write tid 2 to smithi093153278-194 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2871: finishing write tid 3 to smithi093153278-194 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2871: finishing write tid 4 to smithi093153278-194 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 194 v 247 (ObjNum 1837 snap 0 seq_num 1837) dirty exists 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2871: left oid 194 (ObjNum 1837 snap 0 seq_num 1837) 2023-08-27T20:54:26.078 INFO:tasks.rados.rados.0.smithi093.stdout:2875: expect (ObjNum 968 snap 0 seq_num 968) 2023-08-27T20:54:26.173 INFO:tasks.rados.rados.0.smithi093.stdout:2874: expect (ObjNum 1751 snap 0 seq_num 1751) 2023-08-27T20:54:26.315 INFO:tasks.rados.rados.0.smithi093.stdout:2873: expect (ObjNum 1455 snap 0 seq_num 1455) 2023-08-27T20:54:26.439 INFO:tasks.rados.rados.0.smithi093.stdout:2872: expect (ObjNum 1658 snap 0 seq_num 1658) 2023-08-27T20:54:26.661 INFO:tasks.rados.rados.0.smithi093.stdout:2882: seq_num 1840 ranges {524880=615586,1805456=416299,2678895=245985} 2023-08-27T20:54:26.668 INFO:tasks.rados.rados.0.smithi093.stdout:2882: writing smithi093153278-384 from 524880 to 1140466 tid 1 2023-08-27T20:54:26.675 INFO:tasks.rados.rados.0.smithi093.stdout:2882: writing smithi093153278-384 from 1805456 to 2221755 tid 2 2023-08-27T20:54:26.679 INFO:tasks.rados.rados.0.smithi093.stdout:2882: writing smithi093153278-384 from 2678895 to 2924880 tid 3 2023-08-27T20:54:26.679 INFO:tasks.rados.rados.0.smithi093.stdout:2871: done (10 left) 2023-08-27T20:54:26.680 INFO:tasks.rados.rados.0.smithi093.stdout:2872: done (9 left) 2023-08-27T20:54:26.680 INFO:tasks.rados.rados.0.smithi093.stdout:2873: done (8 left) 2023-08-27T20:54:26.680 INFO:tasks.rados.rados.0.smithi093.stdout:2874: done (7 left) 2023-08-27T20:54:26.680 INFO:tasks.rados.rados.0.smithi093.stdout:2875: done (6 left) 2023-08-27T20:54:26.680 INFO:tasks.rados.rados.0.smithi093.stdout:2877: done (5 left) 2023-08-27T20:54:26.680 INFO:tasks.rados.rados.0.smithi093.stdout:2883: delete oid 27 current snap is 0 2023-08-27T20:54:26.680 INFO:tasks.rados.rados.0.smithi093.stdout:2878: expect (ObjNum 1410 snap 0 seq_num 1410) 2023-08-27T20:54:26.859 INFO:tasks.rados.rados.0.smithi093.stdout:2879: expect (ObjNum 1552 snap 0 seq_num 1552) 2023-08-27T20:54:26.926 INFO:tasks.rados.rados.0.smithi093.stdout:2880: finishing write tid 1 to smithi093153278-426 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2878: done (5 left) 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2879: done (4 left) 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2883: done (3 left) 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2884: read oid 61 snap -1 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2884: expect (ObjNum 1642 snap 0 seq_num 1642) 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2885: read oid 263 snap -1 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2885: expect (ObjNum 1741 snap 0 seq_num 1741) 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2886: read oid 309 snap -1 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2886: expect (ObjNum 1527 snap 0 seq_num 1527) 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2887: read oid 232 snap -1 2023-08-27T20:54:26.927 INFO:tasks.rados.rados.0.smithi093.stdout:2887: expect (ObjNum 1525 snap 0 seq_num 1525) 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2888: read oid 4 snap -1 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2888: expect (ObjNum 1587 snap 0 seq_num 1587) 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2889: read oid 446 snap -1 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2889: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2890: read oid 453 snap -1 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2890: expect (ObjNum 1187 snap 0 seq_num 1187) 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2891: read oid 408 snap -1 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2891: expect (ObjNum 1355 snap 0 seq_num 1355) 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2892: delete oid 300 current snap is 0 2023-08-27T20:54:26.928 INFO:tasks.rados.rados.0.smithi093.stdout:2881: expect (ObjNum 156 snap 0 seq_num 156) 2023-08-27T20:54:27.013 INFO:tasks.rados.rados.0.smithi093.stdout:2880: finishing write tid 2 to smithi093153278-426 2023-08-27T20:54:27.013 INFO:tasks.rados.rados.0.smithi093.stdout:2880: finishing write tid 3 to smithi093153278-426 2023-08-27T20:54:27.013 INFO:tasks.rados.rados.0.smithi093.stdout:2880: finishing write tid 4 to smithi093153278-426 2023-08-27T20:54:27.013 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 426 v 386 (ObjNum 1839 snap 0 seq_num 1839) dirty exists 2023-08-27T20:54:27.013 INFO:tasks.rados.rados.0.smithi093.stdout:2880: left oid 426 (ObjNum 1839 snap 0 seq_num 1839) 2023-08-27T20:54:27.013 INFO:tasks.rados.rados.0.smithi093.stdout:2880: done (11 left) 2023-08-27T20:54:27.013 INFO:tasks.rados.rados.0.smithi093.stdout:2881: done (10 left) 2023-08-27T20:54:27.014 INFO:tasks.rados.rados.0.smithi093.stdout:2892: done (9 left) 2023-08-27T20:54:27.014 INFO:tasks.rados.rados.0.smithi093.stdout:2893: write oid 99 current snap is 0 2023-08-27T20:54:27.014 INFO:tasks.rados.rados.0.smithi093.stdout:2882: finishing write tid 1 to smithi093153278-384 2023-08-27T20:54:27.014 INFO:tasks.rados.rados.0.smithi093.stdout:2882: finishing write tid 2 to smithi093153278-384 2023-08-27T20:54:27.014 INFO:tasks.rados.rados.0.smithi093.stdout:2882: finishing write tid 3 to smithi093153278-384 2023-08-27T20:54:27.014 INFO:tasks.rados.rados.0.smithi093.stdout:2893: seq_num 1843 ranges {575301=441333,1464549=679979,2644811=330490} 2023-08-27T20:54:27.019 INFO:tasks.rados.rados.0.smithi093.stdout:2893: writing smithi093153278-99 from 575301 to 1016634 tid 1 2023-08-27T20:54:27.026 INFO:tasks.rados.rados.0.smithi093.stdout:2893: writing smithi093153278-99 from 1464549 to 2144528 tid 2 2023-08-27T20:54:27.031 INFO:tasks.rados.rados.0.smithi093.stdout:2893: writing smithi093153278-99 from 2644811 to 2975301 tid 3 2023-08-27T20:54:27.031 INFO:tasks.rados.rados.0.smithi093.stdout:2894: read oid 267 snap -1 2023-08-27T20:54:27.031 INFO:tasks.rados.rados.0.smithi093.stdout:2894: expect (ObjNum 1771 snap 0 seq_num 1771) 2023-08-27T20:54:27.031 INFO:tasks.rados.rados.0.smithi093.stdout:2895: write oid 53 current snap is 0 2023-08-27T20:54:27.031 INFO:tasks.rados.rados.0.smithi093.stdout:2882: finishing write tid 4 to smithi093153278-384 2023-08-27T20:54:27.032 INFO:tasks.rados.rados.0.smithi093.stdout:2882: finishing write tid 5 to smithi093153278-384 2023-08-27T20:54:27.032 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 384 v 337 (ObjNum 1840 snap 0 seq_num 1840) dirty exists 2023-08-27T20:54:27.032 INFO:tasks.rados.rados.0.smithi093.stdout:2882: left oid 384 (ObjNum 1840 snap 0 seq_num 1840) 2023-08-27T20:54:27.032 INFO:tasks.rados.rados.0.smithi093.stdout:2887: expect (ObjNum 1525 snap 0 seq_num 1525) 2023-08-27T20:54:27.217 INFO:tasks.rados.rados.0.smithi093.stdout:2885: expect (ObjNum 1741 snap 0 seq_num 1741) 2023-08-27T20:54:27.415 INFO:tasks.rados.rados.0.smithi093.stdout:2888: expect (ObjNum 1587 snap 0 seq_num 1587) 2023-08-27T20:54:27.559 INFO:tasks.rados.rados.0.smithi093.stdout:2891: expect (ObjNum 1355 snap 0 seq_num 1355) 2023-08-27T20:54:27.647 INFO:tasks.rados.rados.0.smithi093.stdout:2884: expect (ObjNum 1642 snap 0 seq_num 1642) 2023-08-27T20:54:27.834 INFO:tasks.rados.rados.0.smithi093.stdout:2886: expect (ObjNum 1527 snap 0 seq_num 1527) 2023-08-27T20:54:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:27 smithi136 ceph-mon[113531]: pgmap v27: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 MiB/s rd, 12 MiB/s wr, 124 op/s 2023-08-27T20:54:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3025470174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:27 smithi093 ceph-mon[111335]: pgmap v27: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 MiB/s rd, 12 MiB/s wr, 124 op/s 2023-08-27T20:54:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3025470174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:27 smithi093 ceph-mon[118902]: pgmap v27: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 MiB/s rd, 12 MiB/s wr, 124 op/s 2023-08-27T20:54:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3025470174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:28.032 INFO:tasks.rados.rados.0.smithi093.stdout:2889: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:54:28.141 INFO:tasks.rados.rados.0.smithi093.stdout:2890: expect (ObjNum 1187 snap 0 seq_num 1187) 2023-08-27T20:54:28.312 INFO:tasks.rados.rados.0.smithi093.stdout:2893: finishing write tid 1 to smithi093153278-99 2023-08-27T20:54:28.312 INFO:tasks.rados.rados.0.smithi093.stdout:2893: finishing write tid 2 to smithi093153278-99 2023-08-27T20:54:28.313 INFO:tasks.rados.rados.0.smithi093.stdout:2895: seq_num 1844 ranges {592108=516582,1523031=423637,2672234=319874} 2023-08-27T20:54:28.318 INFO:tasks.rados.rados.0.smithi093.stdout:2895: writing smithi093153278-53 from 592108 to 1108690 tid 1 2023-08-27T20:54:28.323 INFO:tasks.rados.rados.0.smithi093.stdout:2895: writing smithi093153278-53 from 1523031 to 1946668 tid 2 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2895: writing smithi093153278-53 from 2672234 to 2992108 tid 3 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2882: done (11 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2884: done (10 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2885: done (9 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2886: done (8 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2887: done (7 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2888: done (6 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2889: done (5 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2890: done (4 left) 2023-08-27T20:54:28.330 INFO:tasks.rados.rados.0.smithi093.stdout:2891: done (3 left) 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2896: read oid 236 snap -1 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2896: expect (ObjNum 1562 snap 0 seq_num 1562) 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2897: delete oid 198 current snap is 0 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2893: finishing write tid 3 to smithi093153278-99 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2893: finishing write tid 4 to smithi093153278-99 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2893: finishing write tid 5 to smithi093153278-99 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 99 v 390 (ObjNum 1843 snap 0 seq_num 1843) dirty exists 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2893: left oid 99 (ObjNum 1843 snap 0 seq_num 1843) 2023-08-27T20:54:28.331 INFO:tasks.rados.rados.0.smithi093.stdout:2894: expect (ObjNum 1771 snap 0 seq_num 1771) 2023-08-27T20:54:28.593 INFO:tasks.rados.rados.0.smithi093.stdout:2895: finishing write tid 1 to smithi093153278-53 2023-08-27T20:54:28.593 INFO:tasks.rados.rados.0.smithi093.stdout:2895: finishing write tid 2 to smithi093153278-53 2023-08-27T20:54:28.594 INFO:tasks.rados.rados.0.smithi093.stdout:2895: finishing write tid 3 to smithi093153278-53 2023-08-27T20:54:28.594 INFO:tasks.rados.rados.0.smithi093.stdout:2895: finishing write tid 4 to smithi093153278-53 2023-08-27T20:54:28.594 INFO:tasks.rados.rados.0.smithi093.stdout:2895: finishing write tid 5 to smithi093153278-53 2023-08-27T20:54:28.594 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 53 v 452 (ObjNum 1844 snap 0 seq_num 1844) dirty exists 2023-08-27T20:54:28.594 INFO:tasks.rados.rados.0.smithi093.stdout:2895: left oid 53 (ObjNum 1844 snap 0 seq_num 1844) 2023-08-27T20:54:28.594 INFO:tasks.rados.rados.0.smithi093.stdout:2896: expect (ObjNum 1562 snap 0 seq_num 1562) 2023-08-27T20:54:28.667 INFO:tasks.rados.rados.0.smithi093.stdout:2893: done (4 left) 2023-08-27T20:54:28.667 INFO:tasks.rados.rados.0.smithi093.stdout:2894: done (3 left) 2023-08-27T20:54:28.667 INFO:tasks.rados.rados.0.smithi093.stdout:2895: done (2 left) 2023-08-27T20:54:28.667 INFO:tasks.rados.rados.0.smithi093.stdout:2896: done (1 left) 2023-08-27T20:54:28.667 INFO:tasks.rados.rados.0.smithi093.stdout:2897: done (0 left) 2023-08-27T20:54:28.667 INFO:tasks.rados.rados.0.smithi093.stdout:2898: write oid 84 current snap is 0 2023-08-27T20:54:28.667 INFO:tasks.rados.rados.0.smithi093.stdout:2898: seq_num 1846 ranges {625722=667080,1923642=427306,3010727=14995} 2023-08-27T20:54:28.673 INFO:tasks.rados.rados.0.smithi093.stdout:2898: writing smithi093153278-84 from 625722 to 1292802 tid 1 2023-08-27T20:54:28.679 INFO:tasks.rados.rados.0.smithi093.stdout:2898: writing smithi093153278-84 from 1923642 to 2350948 tid 2 2023-08-27T20:54:28.683 INFO:tasks.rados.rados.0.smithi093.stdout:2898: writing smithi093153278-84 from 3010727 to 3025722 tid 3 2023-08-27T20:54:28.683 INFO:tasks.rados.rados.0.smithi093.stdout:2899: write oid 36 current snap is 0 2023-08-27T20:54:28.683 INFO:tasks.rados.rados.0.smithi093.stdout:2899: seq_num 1847 ranges {642529=742329,1982124=687317} 2023-08-27T20:54:28.690 INFO:tasks.rados.rados.0.smithi093.stdout:2899: writing smithi093153278-36 from 642529 to 1384858 tid 1 2023-08-27T20:54:28.697 INFO:tasks.rados.rados.0.smithi093.stdout:2899: writing smithi093153278-36 from 1982124 to 2669441 tid 2 2023-08-27T20:54:28.697 INFO:tasks.rados.rados.0.smithi093.stdout:2900: write oid 43 current snap is 0 2023-08-27T20:54:28.698 INFO:tasks.rados.rados.0.smithi093.stdout:2898: finishing write tid 1 to smithi093153278-84 2023-08-27T20:54:28.698 INFO:tasks.rados.rados.0.smithi093.stdout:2900: seq_num 1848 ranges {659336=533931,1756959=430975,2781926=277410} 2023-08-27T20:54:28.704 INFO:tasks.rados.rados.0.smithi093.stdout:2900: writing smithi093153278-43 from 659336 to 1193267 tid 1 2023-08-27T20:54:28.709 INFO:tasks.rados.rados.0.smithi093.stdout:2900: writing smithi093153278-43 from 1756959 to 2187934 tid 2 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:2900: writing smithi093153278-43 from 2781926 to 3059336 tid 3 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:2901: write oid 274 current snap is 0 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:2898: finishing write tid 2 to smithi093153278-84 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:2898: finishing write tid 3 to smithi093153278-84 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:2898: finishing write tid 4 to smithi093153278-84 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:2898: finishing write tid 5 to smithi093153278-84 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 84 v 382 (ObjNum 1846 snap 0 seq_num 1846) dirty exists 2023-08-27T20:54:28.715 INFO:tasks.rados.rados.0.smithi093.stdout:2898: left oid 84 (ObjNum 1846 snap 0 seq_num 1846) 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2899: finishing write tid 1 to smithi093153278-36 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2899: finishing write tid 2 to smithi093153278-36 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2899: finishing write tid 3 to smithi093153278-36 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2899: finishing write tid 4 to smithi093153278-36 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 367 (ObjNum 1847 snap 0 seq_num 1847) dirty exists 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2899: left oid 36 (ObjNum 1847 snap 0 seq_num 1847) 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2900: finishing write tid 1 to smithi093153278-43 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2900: finishing write tid 2 to smithi093153278-43 2023-08-27T20:54:28.716 INFO:tasks.rados.rados.0.smithi093.stdout:2901: seq_num 1849 ranges {676143=609180,1815441=690986,2925702=150441} 2023-08-27T20:54:28.721 INFO:tasks.rados.rados.0.smithi093.stdout:2901: writing smithi093153278-274 from 676143 to 1285323 tid 1 2023-08-27T20:54:28.728 INFO:tasks.rados.rados.0.smithi093.stdout:2901: writing smithi093153278-274 from 1815441 to 2506427 tid 2 2023-08-27T20:54:28.731 INFO:tasks.rados.rados.0.smithi093.stdout:2901: writing smithi093153278-274 from 2925702 to 3076143 tid 3 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2898: done (3 left) 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2899: done (2 left) 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2902: write oid 105 current snap is 0 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2900: finishing write tid 3 to smithi093153278-43 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2900: finishing write tid 4 to smithi093153278-43 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2900: finishing write tid 5 to smithi093153278-43 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 43 v 458 (ObjNum 1848 snap 0 seq_num 1848) dirty exists 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2900: left oid 43 (ObjNum 1848 snap 0 seq_num 1848) 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2901: finishing write tid 1 to smithi093153278-274 2023-08-27T20:54:28.732 INFO:tasks.rados.rados.0.smithi093.stdout:2902: seq_num 1850 ranges {692950=684429,2157570=434644} 2023-08-27T20:54:28.738 INFO:tasks.rados.rados.0.smithi093.stdout:2902: writing smithi093153278-105 from 692950 to 1377379 tid 1 2023-08-27T20:54:28.745 INFO:tasks.rados.rados.0.smithi093.stdout:2902: writing smithi093153278-105 from 2157570 to 2592214 tid 2 2023-08-27T20:54:28.745 INFO:tasks.rados.rados.0.smithi093.stdout:2900: done (2 left) 2023-08-27T20:54:28.745 INFO:tasks.rados.rados.0.smithi093.stdout:2903: write oid 90 current snap is 0 2023-08-27T20:54:28.746 INFO:tasks.rados.rados.0.smithi093.stdout:2903: seq_num 1851 ranges {709757=759678,2216052=578302} 2023-08-27T20:54:28.753 INFO:tasks.rados.rados.0.smithi093.stdout:2903: writing smithi093153278-90 from 709757 to 1469435 tid 1 2023-08-27T20:54:28.761 INFO:tasks.rados.rados.0.smithi093.stdout:2903: writing smithi093153278-90 from 2216052 to 2794354 tid 2 2023-08-27T20:54:28.761 INFO:tasks.rados.rados.0.smithi093.stdout:2904: write oid 111 current snap is 0 2023-08-27T20:54:28.761 INFO:tasks.rados.rados.0.smithi093.stdout:2901: finishing write tid 2 to smithi093153278-274 2023-08-27T20:54:28.761 INFO:tasks.rados.rados.0.smithi093.stdout:2904: seq_num 1852 ranges {726564=434927,1874534=438313,2775265=351299} 2023-08-27T20:54:28.767 INFO:tasks.rados.rados.0.smithi093.stdout:2904: writing smithi093153278-111 from 726564 to 1161491 tid 1 2023-08-27T20:54:28.773 INFO:tasks.rados.rados.0.smithi093.stdout:2904: writing smithi093153278-111 from 1874534 to 2312847 tid 2 2023-08-27T20:54:28.778 INFO:tasks.rados.rados.0.smithi093.stdout:2904: writing smithi093153278-111 from 2775265 to 3126564 tid 3 2023-08-27T20:54:28.778 INFO:tasks.rados.rados.0.smithi093.stdout:2905: delete oid 285 current snap is 0 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2901: finishing write tid 3 to smithi093153278-274 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2901: finishing write tid 4 to smithi093153278-274 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2901: finishing write tid 5 to smithi093153278-274 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 274 v 394 (ObjNum 1849 snap 0 seq_num 1849) dirty exists 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2901: left oid 274 (ObjNum 1849 snap 0 seq_num 1849) 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2902: finishing write tid 1 to smithi093153278-105 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2902: finishing write tid 2 to smithi093153278-105 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2902: finishing write tid 3 to smithi093153278-105 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:2902: finishing write tid 4 to smithi093153278-105 2023-08-27T20:54:28.779 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 105 v 362 (ObjNum 1850 snap 0 seq_num 1850) dirty exists 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2902: left oid 105 (ObjNum 1850 snap 0 seq_num 1850) 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2903: finishing write tid 1 to smithi093153278-90 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2903: finishing write tid 2 to smithi093153278-90 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2903: finishing write tid 3 to smithi093153278-90 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2903: finishing write tid 4 to smithi093153278-90 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 90 v 455 (ObjNum 1851 snap 0 seq_num 1851) dirty exists 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2903: left oid 90 (ObjNum 1851 snap 0 seq_num 1851) 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2904: finishing write tid 1 to smithi093153278-111 2023-08-27T20:54:28.780 INFO:tasks.rados.rados.0.smithi093.stdout:2904: finishing write tid 2 to smithi093153278-111 2023-08-27T20:54:28.781 INFO:tasks.rados.rados.0.smithi093.stdout:2901: done (4 left) 2023-08-27T20:54:28.781 INFO:tasks.rados.rados.0.smithi093.stdout:2902: done (3 left) 2023-08-27T20:54:28.781 INFO:tasks.rados.rados.0.smithi093.stdout:2903: done (2 left) 2023-08-27T20:54:28.781 INFO:tasks.rados.rados.0.smithi093.stdout:2905: done (1 left) 2023-08-27T20:54:28.781 INFO:tasks.rados.rados.0.smithi093.stdout:2906: write oid 220 current snap is 0 2023-08-27T20:54:28.781 INFO:tasks.rados.rados.0.smithi093.stdout:2906: seq_num 1854 ranges {760178=585425,1875145=441982,3113758=46420} 2023-08-27T20:54:28.786 INFO:tasks.rados.rados.0.smithi093.stdout:2906: writing smithi093153278-220 from 760178 to 1345603 tid 1 2023-08-27T20:54:28.792 INFO:tasks.rados.rados.0.smithi093.stdout:2906: writing smithi093153278-220 from 1875145 to 2317127 tid 2 2023-08-27T20:54:28.797 INFO:tasks.rados.rados.0.smithi093.stdout:2906: writing smithi093153278-220 from 3113758 to 3160178 tid 3 2023-08-27T20:54:28.797 INFO:tasks.rados.rados.0.smithi093.stdout:2907: delete oid 26 current snap is 0 2023-08-27T20:54:28.797 INFO:tasks.rados.rados.0.smithi093.stdout:2904: finishing write tid 3 to smithi093153278-111 2023-08-27T20:54:28.797 INFO:tasks.rados.rados.0.smithi093.stdout:2904: finishing write tid 4 to smithi093153278-111 2023-08-27T20:54:28.797 INFO:tasks.rados.rados.0.smithi093.stdout:2904: finishing write tid 5 to smithi093153278-111 2023-08-27T20:54:28.797 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 111 v 403 (ObjNum 1852 snap 0 seq_num 1852) dirty exists 2023-08-27T20:54:28.797 INFO:tasks.rados.rados.0.smithi093.stdout:2904: left oid 111 (ObjNum 1852 snap 0 seq_num 1852) 2023-08-27T20:54:28.798 INFO:tasks.rados.rados.0.smithi093.stdout:2906: finishing write tid 1 to smithi093153278-220 2023-08-27T20:54:28.798 INFO:tasks.rados.rados.0.smithi093.stdout:2906: finishing write tid 2 to smithi093153278-220 2023-08-27T20:54:28.798 INFO:tasks.rados.rados.0.smithi093.stdout:2906: finishing write tid 3 to smithi093153278-220 2023-08-27T20:54:28.798 INFO:tasks.rados.rados.0.smithi093.stdout:2904: done (2 left) 2023-08-27T20:54:28.798 INFO:tasks.rados.rados.0.smithi093.stdout:2907: done (1 left) 2023-08-27T20:54:28.799 INFO:tasks.rados.rados.0.smithi093.stdout:2908: write oid 129 current snap is 0 2023-08-27T20:54:28.799 INFO:tasks.rados.rados.0.smithi093.stdout:2908: seq_num 1856 ranges {793792=452276,1708462=445651,2884957=308835} 2023-08-27T20:54:28.805 INFO:tasks.rados.rados.0.smithi093.stdout:2908: writing smithi093153278-129 from 793792 to 1246068 tid 1 2023-08-27T20:54:28.810 INFO:tasks.rados.rados.0.smithi093.stdout:2908: writing smithi093153278-129 from 1708462 to 2154113 tid 2 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:2908: writing smithi093153278-129 from 2884957 to 3193792 tid 3 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:2909: write oid 253 current snap is 0 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:2906: finishing write tid 4 to smithi093153278-220 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:2906: finishing write tid 5 to smithi093153278-220 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 220 v 459 (ObjNum 1854 snap 0 seq_num 1854) dirty exists 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:2906: left oid 220 (ObjNum 1854 snap 0 seq_num 1854) 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:2908: finishing write tid 1 to smithi093153278-129 2023-08-27T20:54:28.816 INFO:tasks.rados.rados.0.smithi093.stdout:2909: seq_num 1857 ranges {410599=527525,1366944=589309,2512380=698219} 2023-08-27T20:54:28.821 INFO:tasks.rados.rados.0.smithi093.stdout:2909: writing smithi093153278-253 from 410599 to 938124 tid 1 2023-08-27T20:54:28.826 INFO:tasks.rados.rados.0.smithi093.stdout:2909: writing smithi093153278-253 from 1366944 to 1956253 tid 2 2023-08-27T20:54:28.834 INFO:tasks.rados.rados.0.smithi093.stdout:2909: writing smithi093153278-253 from 2512380 to 3210599 tid 3 2023-08-27T20:54:28.834 INFO:tasks.rados.rados.0.smithi093.stdout:2906: done (2 left) 2023-08-27T20:54:28.834 INFO:tasks.rados.rados.0.smithi093.stdout:2910: read oid 142 snap -1 2023-08-27T20:54:28.834 INFO:tasks.rados.rados.0.smithi093.stdout:2910: expect deleted 2023-08-27T20:54:28.834 INFO:tasks.rados.rados.0.smithi093.stdout:2911: read oid 27 snap -1 2023-08-27T20:54:28.834 INFO:tasks.rados.rados.0.smithi093.stdout:2911: expect deleted 2023-08-27T20:54:28.834 INFO:tasks.rados.rados.0.smithi093.stdout:2912: delete oid 282 current snap is 0 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:2908: finishing write tid 2 to smithi093153278-129 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:2908: finishing write tid 3 to smithi093153278-129 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:2908: finishing write tid 4 to smithi093153278-129 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:2908: finishing write tid 5 to smithi093153278-129 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 129 v 325 (ObjNum 1856 snap 0 seq_num 1856) dirty exists 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:2908: left oid 129 (ObjNum 1856 snap 0 seq_num 1856) 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:2909: finishing write tid 1 to smithi093153278-253 2023-08-27T20:54:28.835 INFO:tasks.rados.rados.0.smithi093.stdout:2909: finishing write tid 2 to smithi093153278-253 2023-08-27T20:54:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:2908: done (4 left) 2023-08-27T20:54:28.836 INFO:tasks.rados.rados.0.smithi093.stdout:2910: done (3 left) 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2911: done (2 left) 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2912: done (1 left) 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2913: read oid 18 snap -1 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2913: expect (ObjNum 1545 snap 0 seq_num 1545) 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2914: read oid 58 snap -1 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2914: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2915: read oid 224 snap -1 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2915: expect (ObjNum 1358 snap 0 seq_num 1358) 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2916: read oid 163 snap -1 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2916: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2917: read oid 19 snap -1 2023-08-27T20:54:28.837 INFO:tasks.rados.rados.0.smithi093.stdout:2917: expect (ObjNum 18 snap 0 seq_num 18) 2023-08-27T20:54:28.838 INFO:tasks.rados.rados.0.smithi093.stdout:2918: read oid 81 snap -1 2023-08-27T20:54:28.838 INFO:tasks.rados.rados.0.smithi093.stdout:2918: expect (ObjNum 1433 snap 0 seq_num 1433) 2023-08-27T20:54:28.838 INFO:tasks.rados.rados.0.smithi093.stdout:2919: delete oid 46 current snap is 0 2023-08-27T20:54:28.842 INFO:tasks.rados.rados.0.smithi093.stdout:2909: finishing write tid 3 to smithi093153278-253 2023-08-27T20:54:28.842 INFO:tasks.rados.rados.0.smithi093.stdout:2909: finishing write tid 4 to smithi093153278-253 2023-08-27T20:54:28.842 INFO:tasks.rados.rados.0.smithi093.stdout:2909: finishing write tid 5 to smithi093153278-253 2023-08-27T20:54:28.842 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 398 (ObjNum 1857 snap 0 seq_num 1857) dirty exists 2023-08-27T20:54:28.842 INFO:tasks.rados.rados.0.smithi093.stdout:2909: left oid 253 (ObjNum 1857 snap 0 seq_num 1857) 2023-08-27T20:54:28.845 INFO:tasks.rados.rados.0.smithi093.stdout:2914: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:54:28.936 INFO:tasks.rados.rados.0.smithi093.stdout:2917: expect (ObjNum 18 snap 0 seq_num 18) 2023-08-27T20:54:29.004 INFO:tasks.rados.rados.0.smithi093.stdout:2913: expect (ObjNum 1545 snap 0 seq_num 1545) 2023-08-27T20:54:29.189 INFO:tasks.rados.rados.0.smithi093.stdout:2916: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:54:29.369 INFO:tasks.rados.rados.0.smithi093.stdout:2918: expect (ObjNum 1433 snap 0 seq_num 1433) 2023-08-27T20:54:29.541 INFO:tasks.rados.rados.0.smithi093.stdout:2915: expect (ObjNum 1358 snap 0 seq_num 1358) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2909: done (7 left) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2913: done (6 left) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2914: done (5 left) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2915: done (4 left) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2916: done (3 left) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2917: done (2 left) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2918: done (1 left) 2023-08-27T20:54:29.695 INFO:tasks.rados.rados.0.smithi093.stdout:2919: done (0 left) 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2920: read oid 273 snap -1 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2920: expect deleted 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2921: read oid 272 snap -1 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2921: expect (ObjNum 1388 snap 0 seq_num 1388) 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2922: read oid 135 snap -1 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2922: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2923: read oid 417 snap -1 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2923: expect (ObjNum 1385 snap 0 seq_num 1385) 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2924: read oid 144 snap -1 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2924: expect (ObjNum 1672 snap 0 seq_num 1672) 2023-08-27T20:54:29.696 INFO:tasks.rados.rados.0.smithi093.stdout:2925: read oid 290 snap -1 2023-08-27T20:54:29.697 INFO:tasks.rados.rados.0.smithi093.stdout:2925: expect (ObjNum 1588 snap 0 seq_num 1588) 2023-08-27T20:54:29.697 INFO:tasks.rados.rados.0.smithi093.stdout:2926: read oid 63 snap -1 2023-08-27T20:54:29.697 INFO:tasks.rados.rados.0.smithi093.stdout:2926: expect (ObjNum 1833 snap 0 seq_num 1833) 2023-08-27T20:54:29.697 INFO:tasks.rados.rados.0.smithi093.stdout:2927: read oid 444 snap -1 2023-08-27T20:54:29.697 INFO:tasks.rados.rados.0.smithi093.stdout:2927: expect (ObjNum 1699 snap 0 seq_num 1699) 2023-08-27T20:54:29.697 INFO:tasks.rados.rados.0.smithi093.stdout:2928: write oid 300 current snap is 0 2023-08-27T20:54:29.697 INFO:tasks.rados.rados.0.smithi093.stdout:2928: seq_num 1860 ranges {461020=753272,1826037=452989,2878296=382724} 2023-08-27T20:54:29.700 INFO:tasks.rados.rados.0.smithi093.stdout:2928: writing smithi093153278-300 from 461020 to 1214292 tid 1 2023-08-27T20:54:29.707 INFO:tasks.rados.rados.0.smithi093.stdout:2928: writing smithi093153278-300 from 1826037 to 2279026 tid 2 2023-08-27T20:54:29.714 INFO:tasks.rados.rados.0.smithi093.stdout:2928: writing smithi093153278-300 from 2878296 to 3261020 tid 3 2023-08-27T20:54:29.714 INFO:tasks.rados.rados.0.smithi093.stdout:2929: write oid 430 current snap is 0 2023-08-27T20:54:29.714 INFO:tasks.rados.rados.0.smithi093.stdout:2929: seq_num 1861 ranges {477827=428521,1484519=596647,2505719=532863} 2023-08-27T20:54:29.719 INFO:tasks.rados.rados.0.smithi093.stdout:2929: writing smithi093153278-430 from 477827 to 906348 tid 1 2023-08-27T20:54:29.725 INFO:tasks.rados.rados.0.smithi093.stdout:2929: writing smithi093153278-430 from 1484519 to 2081166 tid 2 2023-08-27T20:54:29.731 INFO:tasks.rados.rados.0.smithi093.stdout:2929: writing smithi093153278-430 from 2505719 to 3038582 tid 3 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2930: read oid 390 snap -1 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2930: expect deleted 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2931: read oid 429 snap -1 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2931: expect (ObjNum 1574 snap 0 seq_num 1574) 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2932: read oid 175 snap -1 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2932: expect (ObjNum 1144 snap 0 seq_num 1144) 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2920: done (12 left) 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2933: write oid 132 current snap is 0 2023-08-27T20:54:29.732 INFO:tasks.rados.rados.0.smithi093.stdout:2924: expect (ObjNum 1672 snap 0 seq_num 1672) 2023-08-27T20:54:29.799 INFO:tasks.rados.rados.0.smithi093.stdout:2927: expect (ObjNum 1699 snap 0 seq_num 1699) 2023-08-27T20:54:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:29 smithi136 ceph-mon[113531]: pgmap v28: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 7.9 MiB/s wr, 79 op/s 2023-08-27T20:54:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2620460947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:29.942 INFO:tasks.rados.rados.0.smithi093.stdout:2921: expect (ObjNum 1388 snap 0 seq_num 1388) 2023-08-27T20:54:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:29 smithi093 ceph-mon[111335]: pgmap v28: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 7.9 MiB/s wr, 79 op/s 2023-08-27T20:54:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2620460947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:29 smithi093 ceph-mon[118902]: pgmap v28: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 7.9 MiB/s wr, 79 op/s 2023-08-27T20:54:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2620460947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:30.085 INFO:tasks.rados.rados.0.smithi093.stdout:2923: expect (ObjNum 1385 snap 0 seq_num 1385) 2023-08-27T20:54:30.260 INFO:tasks.rados.rados.0.smithi093.stdout:2928: finishing write tid 1 to smithi093153278-300 2023-08-27T20:54:30.260 INFO:tasks.rados.rados.0.smithi093.stdout:2922: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:54:30.409 INFO:tasks.rados.rados.0.smithi093.stdout:2928: finishing write tid 2 to smithi093153278-300 2023-08-27T20:54:30.409 INFO:tasks.rados.rados.0.smithi093.stdout:2925: expect (ObjNum 1588 snap 0 seq_num 1588) 2023-08-27T20:54:30.531 INFO:tasks.rados.rados.0.smithi093.stdout:2928: finishing write tid 3 to smithi093153278-300 2023-08-27T20:54:30.531 INFO:tasks.rados.rados.0.smithi093.stdout:2928: finishing write tid 4 to smithi093153278-300 2023-08-27T20:54:30.531 INFO:tasks.rados.rados.0.smithi093.stdout:2926: expect (ObjNum 1833 snap 0 seq_num 1833) 2023-08-27T20:54:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:2928: finishing write tid 5 to smithi093153278-300 2023-08-27T20:54:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 300 v 329 (ObjNum 1860 snap 0 seq_num 1860) dirty exists 2023-08-27T20:54:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:2928: left oid 300 (ObjNum 1860 snap 0 seq_num 1860) 2023-08-27T20:54:30.616 INFO:tasks.rados.rados.0.smithi093.stdout:2929: finishing write tid 1 to smithi093153278-430 2023-08-27T20:54:30.616 INFO:tasks.rados.rados.0.smithi093.stdout:2933: seq_num 1862 ranges {494634=503770,1426648=456658,2533142=744135} 2023-08-27T20:54:30.621 INFO:tasks.rados.rados.0.smithi093.stdout:2933: writing smithi093153278-132 from 494634 to 998404 tid 1 2023-08-27T20:54:30.626 INFO:tasks.rados.rados.0.smithi093.stdout:2933: writing smithi093153278-132 from 1426648 to 1883306 tid 2 2023-08-27T20:54:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:2933: writing smithi093153278-132 from 2533142 to 3277277 tid 3 2023-08-27T20:54:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:2921: done (12 left) 2023-08-27T20:54:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:2922: done (11 left) 2023-08-27T20:54:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:2923: done (10 left) 2023-08-27T20:54:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:2924: done (9 left) 2023-08-27T20:54:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:2925: done (8 left) 2023-08-27T20:54:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:2926: done (7 left) 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2927: done (6 left) 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2928: done (5 left) 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2934: delete oid 418 current snap is 0 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2929: finishing write tid 2 to smithi093153278-430 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2929: finishing write tid 3 to smithi093153278-430 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2929: finishing write tid 4 to smithi093153278-430 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2929: finishing write tid 5 to smithi093153278-430 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 430 v 513 (ObjNum 1861 snap 0 seq_num 1861) dirty exists 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2929: left oid 430 (ObjNum 1861 snap 0 seq_num 1861) 2023-08-27T20:54:30.635 INFO:tasks.rados.rados.0.smithi093.stdout:2931: expect (ObjNum 1574 snap 0 seq_num 1574) 2023-08-27T20:54:30.745 INFO:tasks.rados.rados.0.smithi093.stdout:2932: expect (ObjNum 1144 snap 0 seq_num 1144) 2023-08-27T20:54:30.926 INFO:tasks.rados.rados.0.smithi093.stdout:2933: finishing write tid 1 to smithi093153278-132 2023-08-27T20:54:30.926 INFO:tasks.rados.rados.0.smithi093.stdout:2933: finishing write tid 2 to smithi093153278-132 2023-08-27T20:54:30.926 INFO:tasks.rados.rados.0.smithi093.stdout:2933: finishing write tid 3 to smithi093153278-132 2023-08-27T20:54:30.926 INFO:tasks.rados.rados.0.smithi093.stdout:2933: finishing write tid 4 to smithi093153278-132 2023-08-27T20:54:30.926 INFO:tasks.rados.rados.0.smithi093.stdout:2933: finishing write tid 5 to smithi093153278-132 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 132 v 367 (ObjNum 1862 snap 0 seq_num 1862) dirty exists 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2933: left oid 132 (ObjNum 1862 snap 0 seq_num 1862) 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2929: done (5 left) 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2930: done (4 left) 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2931: done (3 left) 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2932: done (2 left) 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2933: done (1 left) 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2934: done (0 left) 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2935: write oid 162 current snap is 0 2023-08-27T20:54:30.927 INFO:tasks.rados.rados.0.smithi093.stdout:2935: seq_num 1864 ranges {528248=770621,2059965=743974} 2023-08-27T20:54:30.930 INFO:tasks.rados.rados.0.smithi093.stdout:2935: writing smithi093153278-162 from 528248 to 1298869 tid 1 2023-08-27T20:54:30.938 INFO:tasks.rados.rados.0.smithi093.stdout:2935: writing smithi093153278-162 from 2059965 to 2803939 tid 2 2023-08-27T20:54:30.938 INFO:tasks.rados.rados.0.smithi093.stdout:2936: write oid 56 current snap is 0 2023-08-27T20:54:30.938 INFO:tasks.rados.rados.0.smithi093.stdout:2935: finishing write tid 1 to smithi093153278-162 2023-08-27T20:54:30.938 INFO:tasks.rados.rados.0.smithi093.stdout:2936: seq_num 1865 ranges {545055=445870,1718447=603985,3015411=329644} 2023-08-27T20:54:30.943 INFO:tasks.rados.rados.0.smithi093.stdout:2936: writing smithi093153278-56 from 545055 to 990925 tid 1 2023-08-27T20:54:30.950 INFO:tasks.rados.rados.0.smithi093.stdout:2936: writing smithi093153278-56 from 1718447 to 2322432 tid 2 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2936: writing smithi093153278-56 from 3015411 to 3345055 tid 3 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2937: delete oid 358 current snap is 0 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2935: finishing write tid 2 to smithi093153278-162 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2935: finishing write tid 3 to smithi093153278-162 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2935: finishing write tid 4 to smithi093153278-162 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 162 v 332 (ObjNum 1864 snap 0 seq_num 1864) dirty exists 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2935: left oid 162 (ObjNum 1864 snap 0 seq_num 1864) 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2936: finishing write tid 1 to smithi093153278-56 2023-08-27T20:54:30.956 INFO:tasks.rados.rados.0.smithi093.stdout:2936: finishing write tid 2 to smithi093153278-56 2023-08-27T20:54:30.957 INFO:tasks.rados.rados.0.smithi093.stdout:2935: done (2 left) 2023-08-27T20:54:30.957 INFO:tasks.rados.rados.0.smithi093.stdout:2937: done (1 left) 2023-08-27T20:54:30.957 INFO:tasks.rados.rados.0.smithi093.stdout:2938: read oid 16 snap -1 2023-08-27T20:54:30.957 INFO:tasks.rados.rados.0.smithi093.stdout:2938: expect (ObjNum 1491 snap 0 seq_num 1491) 2023-08-27T20:54:30.958 INFO:tasks.rados.rados.0.smithi093.stdout:2939: read oid 396 snap -1 2023-08-27T20:54:30.958 INFO:tasks.rados.rados.0.smithi093.stdout:2939: expect deleted 2023-08-27T20:54:30.958 INFO:tasks.rados.rados.0.smithi093.stdout:2940: write oid 433 current snap is 0 2023-08-27T20:54:30.958 INFO:tasks.rados.rados.0.smithi093.stdout:2940: seq_num 1867 ranges {578669=596368,1719058=607654,2953904=424765} 2023-08-27T20:54:30.964 INFO:tasks.rados.rados.0.smithi093.stdout:2940: writing smithi093153278-433 from 578669 to 1175037 tid 1 2023-08-27T20:54:30.970 INFO:tasks.rados.rados.0.smithi093.stdout:2940: writing smithi093153278-433 from 1719058 to 2326712 tid 2 2023-08-27T20:54:30.976 INFO:tasks.rados.rados.0.smithi093.stdout:2940: writing smithi093153278-433 from 2953904 to 3378669 tid 3 2023-08-27T20:54:30.976 INFO:tasks.rados.rados.0.smithi093.stdout:2941: write oid 88 current snap is 0 2023-08-27T20:54:30.977 INFO:tasks.rados.rados.0.smithi093.stdout:2941: seq_num 1868 ranges {595476=671617,1777540=751312,2981327=414149} 2023-08-27T20:54:30.983 INFO:tasks.rados.rados.0.smithi093.stdout:2941: writing smithi093153278-88 from 595476 to 1267093 tid 1 2023-08-27T20:54:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:2941: writing smithi093153278-88 from 1777540 to 2528852 tid 2 2023-08-27T20:54:30.995 INFO:tasks.rados.rados.0.smithi093.stdout:2941: writing smithi093153278-88 from 2981327 to 3395476 tid 3 2023-08-27T20:54:30.995 INFO:tasks.rados.rados.0.smithi093.stdout:2942: write oid 490 current snap is 0 2023-08-27T20:54:30.995 INFO:tasks.rados.rados.0.smithi093.stdout:2942: seq_num 1869 ranges {612283=746866,1836022=611323,3008750=403533} 2023-08-27T20:54:31.003 INFO:tasks.rados.rados.0.smithi093.stdout:2942: writing smithi093153278-490 from 612283 to 1359149 tid 1 2023-08-27T20:54:31.009 INFO:tasks.rados.rados.0.smithi093.stdout:2942: writing smithi093153278-490 from 1836022 to 2447345 tid 2 2023-08-27T20:54:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:2942: writing smithi093153278-490 from 3008750 to 3412283 tid 3 2023-08-27T20:54:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:2943: write oid 453 current snap is 0 2023-08-27T20:54:31.014 INFO:tasks.rados.rados.0.smithi093.stdout:2943: seq_num 1870 ranges {629090=422115,1778151=754981,3319820=109270} 2023-08-27T20:54:31.020 INFO:tasks.rados.rados.0.smithi093.stdout:2943: writing smithi093153278-453 from 629090 to 1051205 tid 1 2023-08-27T20:54:31.028 INFO:tasks.rados.rados.0.smithi093.stdout:2943: writing smithi093153278-453 from 1778151 to 2533132 tid 2 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:2943: writing smithi093153278-453 from 3319820 to 3429090 tid 3 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:2944: write oid 414 current snap is 0 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:2936: finishing write tid 3 to smithi093153278-56 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:2936: finishing write tid 4 to smithi093153278-56 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:2936: finishing write tid 5 to smithi093153278-56 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 56 v 336 (ObjNum 1865 snap 0 seq_num 1865) dirty exists 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:2936: left oid 56 (ObjNum 1865 snap 0 seq_num 1865) 2023-08-27T20:54:31.034 INFO:tasks.rados.rados.0.smithi093.stdout:2938: expect (ObjNum 1491 snap 0 seq_num 1491) 2023-08-27T20:54:31.174 INFO:tasks.rados.rados.0.smithi093.stdout:2940: finishing write tid 1 to smithi093153278-433 2023-08-27T20:54:31.174 INFO:tasks.rados.rados.0.smithi093.stdout:2940: finishing write tid 2 to smithi093153278-433 2023-08-27T20:54:31.175 INFO:tasks.rados.rados.0.smithi093.stdout:2940: finishing write tid 3 to smithi093153278-433 2023-08-27T20:54:31.175 INFO:tasks.rados.rados.0.smithi093.stdout:2940: finishing write tid 4 to smithi093153278-433 2023-08-27T20:54:31.175 INFO:tasks.rados.rados.0.smithi093.stdout:2940: finishing write tid 5 to smithi093153278-433 2023-08-27T20:54:31.175 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 433 v 407 (ObjNum 1867 snap 0 seq_num 1867) dirty exists 2023-08-27T20:54:31.175 INFO:tasks.rados.rados.0.smithi093.stdout:2940: left oid 433 (ObjNum 1867 snap 0 seq_num 1867) 2023-08-27T20:54:31.175 INFO:tasks.rados.rados.0.smithi093.stdout:2941: finishing write tid 1 to smithi093153278-88 2023-08-27T20:54:31.175 INFO:tasks.rados.rados.0.smithi093.stdout:2944: seq_num 1871 ranges {645897=613717,1952986=614992,3063596=382301} 2023-08-27T20:54:31.181 INFO:tasks.rados.rados.0.smithi093.stdout:2944: writing smithi093153278-414 from 645897 to 1259614 tid 1 2023-08-27T20:54:31.188 INFO:tasks.rados.rados.0.smithi093.stdout:2944: writing smithi093153278-414 from 1952986 to 2567978 tid 2 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2944: writing smithi093153278-414 from 3063596 to 3445897 tid 3 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2936: done (7 left) 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2938: done (6 left) 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2939: done (5 left) 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2940: done (4 left) 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2945: delete oid 342 current snap is 0 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2941: finishing write tid 2 to smithi093153278-88 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2941: finishing write tid 3 to smithi093153278-88 2023-08-27T20:54:31.194 INFO:tasks.rados.rados.0.smithi093.stdout:2941: finishing write tid 4 to smithi093153278-88 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2941: finishing write tid 5 to smithi093153278-88 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 462 (ObjNum 1868 snap 0 seq_num 1868) dirty exists 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2941: left oid 88 (ObjNum 1868 snap 0 seq_num 1868) 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2942: finishing write tid 1 to smithi093153278-490 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2942: finishing write tid 2 to smithi093153278-490 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2942: finishing write tid 3 to smithi093153278-490 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2942: finishing write tid 4 to smithi093153278-490 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2942: finishing write tid 5 to smithi093153278-490 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 490 v 371 (ObjNum 1869 snap 0 seq_num 1869) dirty exists 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2942: left oid 490 (ObjNum 1869 snap 0 seq_num 1869) 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2943: finishing write tid 1 to smithi093153278-453 2023-08-27T20:54:31.195 INFO:tasks.rados.rados.0.smithi093.stdout:2943: finishing write tid 2 to smithi093153278-453 2023-08-27T20:54:31.196 INFO:tasks.rados.rados.0.smithi093.stdout:2943: finishing write tid 3 to smithi093153278-453 2023-08-27T20:54:31.196 INFO:tasks.rados.rados.0.smithi093.stdout:2943: finishing write tid 4 to smithi093153278-453 2023-08-27T20:54:31.196 INFO:tasks.rados.rados.0.smithi093.stdout:2943: finishing write tid 5 to smithi093153278-453 2023-08-27T20:54:31.196 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 453 v 345 (ObjNum 1870 snap 0 seq_num 1870) dirty exists 2023-08-27T20:54:31.196 INFO:tasks.rados.rados.0.smithi093.stdout:2943: left oid 453 (ObjNum 1870 snap 0 seq_num 1870) 2023-08-27T20:54:31.196 INFO:tasks.rados.rados.0.smithi093.stdout:2944: finishing write tid 1 to smithi093153278-414 2023-08-27T20:54:31.196 INFO:tasks.rados.rados.0.smithi093.stdout:2944: finishing write tid 2 to smithi093153278-414 2023-08-27T20:54:31.197 INFO:tasks.rados.rados.0.smithi093.stdout:2941: done (4 left) 2023-08-27T20:54:31.197 INFO:tasks.rados.rados.0.smithi093.stdout:2942: done (3 left) 2023-08-27T20:54:31.197 INFO:tasks.rados.rados.0.smithi093.stdout:2943: done (2 left) 2023-08-27T20:54:31.197 INFO:tasks.rados.rados.0.smithi093.stdout:2945: done (1 left) 2023-08-27T20:54:31.197 INFO:tasks.rados.rados.0.smithi093.stdout:2946: write oid 392 current snap is 0 2023-08-27T20:54:31.197 INFO:tasks.rados.rados.0.smithi093.stdout:2946: seq_num 1873 ranges {679511=764215,2069950=618661,3118442=361069} 2023-08-27T20:54:31.202 INFO:tasks.rados.rados.0.smithi093.stdout:2946: writing smithi093153278-392 from 679511 to 1443726 tid 1 2023-08-27T20:54:31.209 INFO:tasks.rados.rados.0.smithi093.stdout:2946: writing smithi093153278-392 from 2069950 to 2688611 tid 2 2023-08-27T20:54:31.214 INFO:tasks.rados.rados.0.smithi093.stdout:2946: writing smithi093153278-392 from 3118442 to 3479511 tid 3 2023-08-27T20:54:31.214 INFO:tasks.rados.rados.0.smithi093.stdout:2947: write oid 496 current snap is 0 2023-08-27T20:54:31.214 INFO:tasks.rados.rados.0.smithi093.stdout:2944: finishing write tid 3 to smithi093153278-414 2023-08-27T20:54:31.214 INFO:tasks.rados.rados.0.smithi093.stdout:2947: seq_num 1874 ranges {696318=439464,1728432=762319,3145865=350453} 2023-08-27T20:54:31.220 INFO:tasks.rados.rados.0.smithi093.stdout:2947: writing smithi093153278-496 from 696318 to 1135782 tid 1 2023-08-27T20:54:31.228 INFO:tasks.rados.rados.0.smithi093.stdout:2947: writing smithi093153278-496 from 1728432 to 2490751 tid 2 2023-08-27T20:54:31.234 INFO:tasks.rados.rados.0.smithi093.stdout:2947: writing smithi093153278-496 from 3145865 to 3496318 tid 3 2023-08-27T20:54:31.234 INFO:tasks.rados.rados.0.smithi093.stdout:2948: delete oid 257 current snap is 0 2023-08-27T20:54:31.234 INFO:tasks.rados.rados.0.smithi093.stdout:2944: finishing write tid 4 to smithi093153278-414 2023-08-27T20:54:31.234 INFO:tasks.rados.rados.0.smithi093.stdout:2944: finishing write tid 5 to smithi093153278-414 2023-08-27T20:54:31.234 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 414 v 466 (ObjNum 1871 snap 0 seq_num 1871) dirty exists 2023-08-27T20:54:31.234 INFO:tasks.rados.rados.0.smithi093.stdout:2944: left oid 414 (ObjNum 1871 snap 0 seq_num 1871) 2023-08-27T20:54:31.234 INFO:tasks.rados.rados.0.smithi093.stdout:2946: finishing write tid 1 to smithi093153278-392 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:2946: finishing write tid 2 to smithi093153278-392 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:2946: finishing write tid 3 to smithi093153278-392 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:2946: finishing write tid 4 to smithi093153278-392 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:2946: finishing write tid 5 to smithi093153278-392 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 392 v 402 (ObjNum 1873 snap 0 seq_num 1873) dirty exists 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:2946: left oid 392 (ObjNum 1873 snap 0 seq_num 1873) 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:2947: finishing write tid 1 to smithi093153278-496 2023-08-27T20:54:31.235 INFO:tasks.rados.rados.0.smithi093.stdout:2947: finishing write tid 2 to smithi093153278-496 2023-08-27T20:54:31.236 INFO:tasks.rados.rados.0.smithi093.stdout:2944: done (3 left) 2023-08-27T20:54:31.236 INFO:tasks.rados.rados.0.smithi093.stdout:2946: done (2 left) 2023-08-27T20:54:31.236 INFO:tasks.rados.rados.0.smithi093.stdout:2948: done (1 left) 2023-08-27T20:54:31.236 INFO:tasks.rados.rados.0.smithi093.stdout:2949: write oid 101 current snap is 0 2023-08-27T20:54:31.236 INFO:tasks.rados.rados.0.smithi093.stdout:2949: seq_num 1876 ranges {729932=589962,1729043=765988,3084358=445574} 2023-08-27T20:54:31.242 INFO:tasks.rados.rados.0.smithi093.stdout:2949: writing smithi093153278-101 from 729932 to 1319894 tid 1 2023-08-27T20:54:31.248 INFO:tasks.rados.rados.0.smithi093.stdout:2949: writing smithi093153278-101 from 1729043 to 2495031 tid 2 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2949: writing smithi093153278-101 from 3084358 to 3529932 tid 3 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2950: delete oid 322 current snap is 0 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2947: finishing write tid 3 to smithi093153278-496 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2947: finishing write tid 4 to smithi093153278-496 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2947: finishing write tid 5 to smithi093153278-496 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 496 v 371 (ObjNum 1874 snap 0 seq_num 1874) dirty exists 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2947: left oid 496 (ObjNum 1874 snap 0 seq_num 1874) 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2949: finishing write tid 1 to smithi093153278-101 2023-08-27T20:54:31.254 INFO:tasks.rados.rados.0.smithi093.stdout:2949: finishing write tid 2 to smithi093153278-101 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2947: done (2 left) 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2950: done (1 left) 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2951: read oid 218 snap -1 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2951: expect (ObjNum 1674 snap 0 seq_num 1674) 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2952: read oid 279 snap -1 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2952: expect (ObjNum 841 snap 0 seq_num 841) 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2953: read oid 288 snap -1 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2953: expect (ObjNum 1511 snap 0 seq_num 1511) 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2954: read oid 145 snap -1 2023-08-27T20:54:31.256 INFO:tasks.rados.rados.0.smithi093.stdout:2954: expect (ObjNum 144 snap 0 seq_num 144) 2023-08-27T20:54:31.257 INFO:tasks.rados.rados.0.smithi093.stdout:2955: read oid 112 snap -1 2023-08-27T20:54:31.257 INFO:tasks.rados.rados.0.smithi093.stdout:2955: expect (ObjNum 1010 snap 0 seq_num 1010) 2023-08-27T20:54:31.257 INFO:tasks.rados.rados.0.smithi093.stdout:2956: read oid 88 snap -1 2023-08-27T20:54:31.257 INFO:tasks.rados.rados.0.smithi093.stdout:2956: expect (ObjNum 1868 snap 0 seq_num 1868) 2023-08-27T20:54:31.257 INFO:tasks.rados.rados.0.smithi093.stdout:2957: write oid 378 current snap is 0 2023-08-27T20:54:31.257 INFO:tasks.rados.rados.0.smithi093.stdout:2957: seq_num 1878 ranges {763546=456813,1962360=769657,3255557=307989} 2023-08-27T20:54:31.262 INFO:tasks.rados.rados.0.smithi093.stdout:2957: writing smithi093153278-378 from 763546 to 1220359 tid 1 2023-08-27T20:54:31.271 INFO:tasks.rados.rados.0.smithi093.stdout:2957: writing smithi093153278-378 from 1962360 to 2732017 tid 2 2023-08-27T20:54:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:2957: writing smithi093153278-378 from 3255557 to 3563546 tid 3 2023-08-27T20:54:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:2958: read oid 162 snap -1 2023-08-27T20:54:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:2958: expect (ObjNum 1864 snap 0 seq_num 1864) 2023-08-27T20:54:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:2959: read oid 215 snap -1 2023-08-27T20:54:31.276 INFO:tasks.rados.rados.0.smithi093.stdout:2959: expect (ObjNum 1565 snap 0 seq_num 1565) 2023-08-27T20:54:31.277 INFO:tasks.rados.rados.0.smithi093.stdout:2960: write oid 362 current snap is 0 2023-08-27T20:54:31.277 INFO:tasks.rados.rados.0.smithi093.stdout:2951: expect (ObjNum 1674 snap 0 seq_num 1674) 2023-08-27T20:54:31.412 INFO:tasks.rados.rados.0.smithi093.stdout:2952: expect (ObjNum 841 snap 0 seq_num 841) 2023-08-27T20:54:31.500 INFO:tasks.rados.rados.0.smithi093.stdout:2954: expect (ObjNum 144 snap 0 seq_num 144) 2023-08-27T20:54:31.579 INFO:tasks.rados.rados.0.smithi093.stdout:2956: expect (ObjNum 1868 snap 0 seq_num 1868) 2023-08-27T20:54:31.827 INFO:tasks.rados.rados.0.smithi093.stdout:2957: finishing write tid 1 to smithi093153278-378 2023-08-27T20:54:31.827 INFO:tasks.rados.rados.0.smithi093.stdout:2949: finishing write tid 3 to smithi093153278-101 2023-08-27T20:54:31.827 INFO:tasks.rados.rados.0.smithi093.stdout:2949: finishing write tid 4 to smithi093153278-101 2023-08-27T20:54:31.827 INFO:tasks.rados.rados.0.smithi093.stdout:2949: finishing write tid 5 to smithi093153278-101 2023-08-27T20:54:31.827 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 101 v 340 (ObjNum 1876 snap 0 seq_num 1876) dirty exists 2023-08-27T20:54:31.827 INFO:tasks.rados.rados.0.smithi093.stdout:2949: left oid 101 (ObjNum 1876 snap 0 seq_num 1876) 2023-08-27T20:54:31.827 INFO:tasks.rados.rados.0.smithi093.stdout:2953: expect (ObjNum 1511 snap 0 seq_num 1511) 2023-08-27T20:54:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:31 smithi136 ceph-mon[113531]: pgmap v29: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 10 MiB/s wr, 108 op/s 2023-08-27T20:54:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:31 smithi093 ceph-mon[111335]: pgmap v29: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 10 MiB/s wr, 108 op/s 2023-08-27T20:54:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:31 smithi093 ceph-mon[118902]: pgmap v29: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 10 MiB/s wr, 108 op/s 2023-08-27T20:54:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:31.625Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:31.626Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:32.012 INFO:tasks.rados.rados.0.smithi093.stdout:2957: finishing write tid 2 to smithi093153278-378 2023-08-27T20:54:32.012 INFO:tasks.rados.rados.0.smithi093.stdout:2960: seq_num 1879 ranges {780353=532062,1904489=513315,3166627=401171} 2023-08-27T20:54:32.019 INFO:tasks.rados.rados.0.smithi093.stdout:2960: writing smithi093153278-362 from 780353 to 1312415 tid 1 2023-08-27T20:54:32.025 INFO:tasks.rados.rados.0.smithi093.stdout:2960: writing smithi093153278-362 from 1904489 to 2417804 tid 2 2023-08-27T20:54:32.031 INFO:tasks.rados.rados.0.smithi093.stdout:2960: writing smithi093153278-362 from 3166627 to 3567798 tid 3 2023-08-27T20:54:32.031 INFO:tasks.rados.rados.0.smithi093.stdout:2949: done (10 left) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2951: done (9 left) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2952: done (8 left) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2953: done (7 left) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2954: done (6 left) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2956: done (5 left) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2961: read oid 173 snap -1 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2961: expect (ObjNum 1331 snap 0 seq_num 1331) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2962: read oid 242 snap -1 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2962: expect (ObjNum 1557 snap 0 seq_num 1557) 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2963: write oid 394 current snap is 0 2023-08-27T20:54:32.032 INFO:tasks.rados.rados.0.smithi093.stdout:2957: finishing write tid 3 to smithi093153278-378 2023-08-27T20:54:32.033 INFO:tasks.rados.rados.0.smithi093.stdout:2957: finishing write tid 4 to smithi093153278-378 2023-08-27T20:54:32.033 INFO:tasks.rados.rados.0.smithi093.stdout:2963: seq_num 1880 ranges {797160=607311,1962971=773326,3194050=403110} 2023-08-27T20:54:32.039 INFO:tasks.rados.rados.0.smithi093.stdout:2963: writing smithi093153278-394 from 797160 to 1404471 tid 1 2023-08-27T20:54:32.046 INFO:tasks.rados.rados.0.smithi093.stdout:2963: writing smithi093153278-394 from 1962971 to 2736297 tid 2 2023-08-27T20:54:32.051 INFO:tasks.rados.rados.0.smithi093.stdout:2963: writing smithi093153278-394 from 3194050 to 3597160 tid 3 2023-08-27T20:54:32.051 INFO:tasks.rados.rados.0.smithi093.stdout:2964: write oid 309 current snap is 0 2023-08-27T20:54:32.051 INFO:tasks.rados.rados.0.smithi093.stdout:2955: expect (ObjNum 1010 snap 0 seq_num 1010) 2023-08-27T20:54:32.199 INFO:tasks.rados.rados.0.smithi093.stdout:2957: finishing write tid 5 to smithi093153278-378 2023-08-27T20:54:32.199 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 378 v 407 (ObjNum 1878 snap 0 seq_num 1878) dirty exists 2023-08-27T20:54:32.199 INFO:tasks.rados.rados.0.smithi093.stdout:2957: left oid 378 (ObjNum 1878 snap 0 seq_num 1878) 2023-08-27T20:54:32.199 INFO:tasks.rados.rados.0.smithi093.stdout:2958: expect (ObjNum 1864 snap 0 seq_num 1864) 2023-08-27T20:54:32.390 INFO:tasks.rados.rados.0.smithi093.stdout:2959: expect (ObjNum 1565 snap 0 seq_num 1565) 2023-08-27T20:54:32.493 INFO:tasks.rados.rados.0.smithi093.stdout:2960: finishing write tid 1 to smithi093153278-362 2023-08-27T20:54:32.493 INFO:tasks.rados.rados.0.smithi093.stdout:2960: finishing write tid 2 to smithi093153278-362 2023-08-27T20:54:32.493 INFO:tasks.rados.rados.0.smithi093.stdout:2964: seq_num 1881 ranges {413967=682560,1621453=516984,2821473=423715} 2023-08-27T20:54:32.499 INFO:tasks.rados.rados.0.smithi093.stdout:2964: writing smithi093153278-309 from 413967 to 1096527 tid 1 2023-08-27T20:54:32.504 INFO:tasks.rados.rados.0.smithi093.stdout:2964: writing smithi093153278-309 from 1621453 to 2138437 tid 2 2023-08-27T20:54:32.511 INFO:tasks.rados.rados.0.smithi093.stdout:2964: writing smithi093153278-309 from 2821473 to 3245188 tid 3 2023-08-27T20:54:32.511 INFO:tasks.rados.rados.0.smithi093.stdout:2955: done (8 left) 2023-08-27T20:54:32.511 INFO:tasks.rados.rados.0.smithi093.stdout:2957: done (7 left) 2023-08-27T20:54:32.511 INFO:tasks.rados.rados.0.smithi093.stdout:2958: done (6 left) 2023-08-27T20:54:32.511 INFO:tasks.rados.rados.0.smithi093.stdout:2959: done (5 left) 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2965: read oid 440 snap -1 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2965: expect (ObjNum 1444 snap 0 seq_num 1444) 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2966: write oid 237 current snap is 0 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2960: finishing write tid 3 to smithi093153278-362 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2960: finishing write tid 4 to smithi093153278-362 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2960: finishing write tid 5 to smithi093153278-362 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 362 v 345 (ObjNum 1879 snap 0 seq_num 1879) dirty exists 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2960: left oid 362 (ObjNum 1879 snap 0 seq_num 1879) 2023-08-27T20:54:32.512 INFO:tasks.rados.rados.0.smithi093.stdout:2961: expect (ObjNum 1331 snap 0 seq_num 1331) 2023-08-27T20:54:32.635 INFO:tasks.rados.rados.0.smithi093.stdout:2962: expect (ObjNum 1557 snap 0 seq_num 1557) 2023-08-27T20:54:32.762 INFO:tasks.rados.rados.0.smithi093.stdout:2963: finishing write tid 1 to smithi093153278-394 2023-08-27T20:54:32.762 INFO:tasks.rados.rados.0.smithi093.stdout:2963: finishing write tid 2 to smithi093153278-394 2023-08-27T20:54:32.762 INFO:tasks.rados.rados.0.smithi093.stdout:2963: finishing write tid 3 to smithi093153278-394 2023-08-27T20:54:32.762 INFO:tasks.rados.rados.0.smithi093.stdout:2966: seq_num 1882 ranges {430774=757809,1679935=776995,3248896=381878} 2023-08-27T20:54:32.768 INFO:tasks.rados.rados.0.smithi093.stdout:2966: writing smithi093153278-237 from 430774 to 1188583 tid 1 2023-08-27T20:54:32.775 INFO:tasks.rados.rados.0.smithi093.stdout:2966: writing smithi093153278-237 from 1679935 to 2456930 tid 2 2023-08-27T20:54:32.782 INFO:tasks.rados.rados.0.smithi093.stdout:2966: writing smithi093153278-237 from 3248896 to 3630774 tid 3 2023-08-27T20:54:32.782 INFO:tasks.rados.rados.0.smithi093.stdout:2960: done (6 left) 2023-08-27T20:54:32.782 INFO:tasks.rados.rados.0.smithi093.stdout:2961: done (5 left) 2023-08-27T20:54:32.782 INFO:tasks.rados.rados.0.smithi093.stdout:2962: done (4 left) 2023-08-27T20:54:32.782 INFO:tasks.rados.rados.0.smithi093.stdout:2967: write oid 203 current snap is 0 2023-08-27T20:54:32.782 INFO:tasks.rados.rados.0.smithi093.stdout:2963: finishing write tid 4 to smithi093153278-394 2023-08-27T20:54:32.783 INFO:tasks.rados.rados.0.smithi093.stdout:2963: finishing write tid 5 to smithi093153278-394 2023-08-27T20:54:32.783 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 394 v 388 (ObjNum 1880 snap 0 seq_num 1880) dirty exists 2023-08-27T20:54:32.783 INFO:tasks.rados.rados.0.smithi093.stdout:2963: left oid 394 (ObjNum 1880 snap 0 seq_num 1880) 2023-08-27T20:54:32.783 INFO:tasks.rados.rados.0.smithi093.stdout:2964: finishing write tid 1 to smithi093153278-309 2023-08-27T20:54:32.783 INFO:tasks.rados.rados.0.smithi093.stdout:2967: seq_num 1883 ranges {447581=433058,1622064=520653,2759966=562612} 2023-08-27T20:54:32.786 INFO:tasks.rados.rados.0.smithi093.stdout:2967: writing smithi093153278-203 from 447581 to 880639 tid 1 2023-08-27T20:54:32.793 INFO:tasks.rados.rados.0.smithi093.stdout:2967: writing smithi093153278-203 from 1622064 to 2142717 tid 2 2023-08-27T20:54:32.800 INFO:tasks.rados.rados.0.smithi093.stdout:2967: writing smithi093153278-203 from 2759966 to 3322578 tid 3 2023-08-27T20:54:32.800 INFO:tasks.rados.rados.0.smithi093.stdout:2963: done (4 left) 2023-08-27T20:54:32.801 INFO:tasks.rados.rados.0.smithi093.stdout:2968: write oid 277 current snap is 0 2023-08-27T20:54:32.801 INFO:tasks.rados.rados.0.smithi093.stdout:2964: finishing write tid 2 to smithi093153278-309 2023-08-27T20:54:32.801 INFO:tasks.rados.rados.0.smithi093.stdout:2964: finishing write tid 3 to smithi093153278-309 2023-08-27T20:54:32.801 INFO:tasks.rados.rados.0.smithi093.stdout:2964: finishing write tid 4 to smithi093153278-309 2023-08-27T20:54:32.801 INFO:tasks.rados.rados.0.smithi093.stdout:2968: seq_num 1884 ranges {464388=508307,1680546=780664,3187389=476999} 2023-08-27T20:54:32.805 INFO:tasks.rados.rados.0.smithi093.stdout:2968: writing smithi093153278-277 from 464388 to 972695 tid 1 2023-08-27T20:54:32.814 INFO:tasks.rados.rados.0.smithi093.stdout:2968: writing smithi093153278-277 from 1680546 to 2461210 tid 2 2023-08-27T20:54:32.821 INFO:tasks.rados.rados.0.smithi093.stdout:2968: writing smithi093153278-277 from 3187389 to 3664388 tid 3 2023-08-27T20:54:32.821 INFO:tasks.rados.rados.0.smithi093.stdout:2969: write oid 72 current snap is 0 2023-08-27T20:54:32.821 INFO:tasks.rados.rados.0.smithi093.stdout:2964: finishing write tid 5 to smithi093153278-309 2023-08-27T20:54:32.821 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 309 v 344 (ObjNum 1881 snap 0 seq_num 1881) dirty exists 2023-08-27T20:54:32.821 INFO:tasks.rados.rados.0.smithi093.stdout:2964: left oid 309 (ObjNum 1881 snap 0 seq_num 1881) 2023-08-27T20:54:32.821 INFO:tasks.rados.rados.0.smithi093.stdout:2965: expect (ObjNum 1444 snap 0 seq_num 1444) 2023-08-27T20:54:32.890 INFO:tasks.rados.rados.0.smithi093.stdout:2966: finishing write tid 1 to smithi093153278-237 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:2966: finishing write tid 2 to smithi093153278-237 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:2966: finishing write tid 3 to smithi093153278-237 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:2966: finishing write tid 4 to smithi093153278-237 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:2966: finishing write tid 5 to smithi093153278-237 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 237 v 392 (ObjNum 1882 snap 0 seq_num 1882) dirty exists 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:2966: left oid 237 (ObjNum 1882 snap 0 seq_num 1882) 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:2967: finishing write tid 1 to smithi093153278-203 2023-08-27T20:54:32.891 INFO:tasks.rados.rados.0.smithi093.stdout:2969: seq_num 1885 ranges {481195=583556,1739028=524322,2814812=585156} 2023-08-27T20:54:32.896 INFO:tasks.rados.rados.0.smithi093.stdout:2969: writing smithi093153278-72 from 481195 to 1064751 tid 1 2023-08-27T20:54:32.902 INFO:tasks.rados.rados.0.smithi093.stdout:2969: writing smithi093153278-72 from 1739028 to 2263350 tid 2 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2969: writing smithi093153278-72 from 2814812 to 3399968 tid 3 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2964: done (5 left) 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2965: done (4 left) 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2966: done (3 left) 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2970: read oid 226 snap -1 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2970: expect (ObjNum 1823 snap 0 seq_num 1823) 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2971: read oid 195 snap -1 2023-08-27T20:54:32.910 INFO:tasks.rados.rados.0.smithi093.stdout:2971: expect (ObjNum 1596 snap 0 seq_num 1596) 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2972: read oid 403 snap -1 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2972: expect (ObjNum 1285 snap 0 seq_num 1285) 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2973: read oid 223 snap -1 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2973: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2974: read oid 34 snap -1 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2974: expect (ObjNum 1707 snap 0 seq_num 1707) 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2975: read oid 389 snap -1 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2975: expect (ObjNum 1533 snap 0 seq_num 1533) 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2976: write oid 138 current snap is 0 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2967: finishing write tid 2 to smithi093153278-203 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2967: finishing write tid 3 to smithi093153278-203 2023-08-27T20:54:32.911 INFO:tasks.rados.rados.0.smithi093.stdout:2967: finishing write tid 4 to smithi093153278-203 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2967: finishing write tid 5 to smithi093153278-203 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 203 v 314 (ObjNum 1883 snap 0 seq_num 1883) dirty exists 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2967: left oid 203 (ObjNum 1883 snap 0 seq_num 1883) 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2968: finishing write tid 1 to smithi093153278-277 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2968: finishing write tid 2 to smithi093153278-277 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2968: finishing write tid 3 to smithi093153278-277 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2968: finishing write tid 4 to smithi093153278-277 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2968: finishing write tid 5 to smithi093153278-277 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 277 v 412 (ObjNum 1884 snap 0 seq_num 1884) dirty exists 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2968: left oid 277 (ObjNum 1884 snap 0 seq_num 1884) 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2969: finishing write tid 1 to smithi093153278-72 2023-08-27T20:54:32.912 INFO:tasks.rados.rados.0.smithi093.stdout:2976: seq_num 1886 ranges {498002=775158,1913863=784333,3358588=339414} 2023-08-27T20:54:32.916 INFO:tasks.rados.rados.0.smithi093.stdout:2976: writing smithi093153278-138 from 498002 to 1273160 tid 1 2023-08-27T20:54:32.924 INFO:tasks.rados.rados.0.smithi093.stdout:2976: writing smithi093153278-138 from 1913863 to 2698196 tid 2 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2976: writing smithi093153278-138 from 3358588 to 3698002 tid 3 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2967: done (9 left) 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2968: done (8 left) 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2977: read oid 29 snap -1 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2977: expect (ObjNum 1127 snap 0 seq_num 1127) 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2978: write oid 489 current snap is 0 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2969: finishing write tid 2 to smithi093153278-72 2023-08-27T20:54:32.931 INFO:tasks.rados.rados.0.smithi093.stdout:2975: expect (ObjNum 1533 snap 0 seq_num 1533) 2023-08-27T20:54:32.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1842398974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1842398974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1842398974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:33.050 INFO:tasks.rados.rados.0.smithi093.stdout:2976: finishing write tid 1 to smithi093153278-138 2023-08-27T20:54:33.051 INFO:tasks.rados.rados.0.smithi093.stdout:2973: expect (ObjNum 842 snap 0 seq_num 842) 2023-08-27T20:54:33.139 INFO:tasks.rados.rados.0.smithi093.stdout:2976: finishing write tid 2 to smithi093153278-138 2023-08-27T20:54:33.140 INFO:tasks.rados.rados.0.smithi093.stdout:2969: finishing write tid 3 to smithi093153278-72 2023-08-27T20:54:33.140 INFO:tasks.rados.rados.0.smithi093.stdout:2969: finishing write tid 4 to smithi093153278-72 2023-08-27T20:54:33.140 INFO:tasks.rados.rados.0.smithi093.stdout:2969: finishing write tid 5 to smithi093153278-72 2023-08-27T20:54:33.140 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 72 v 396 (ObjNum 1885 snap 0 seq_num 1885) dirty exists 2023-08-27T20:54:33.140 INFO:tasks.rados.rados.0.smithi093.stdout:2969: left oid 72 (ObjNum 1885 snap 0 seq_num 1885) 2023-08-27T20:54:33.140 INFO:tasks.rados.rados.0.smithi093.stdout:2971: expect (ObjNum 1596 snap 0 seq_num 1596) 2023-08-27T20:54:33.292 INFO:tasks.rados.rados.0.smithi093.stdout:2976: finishing write tid 3 to smithi093153278-138 2023-08-27T20:54:33.292 INFO:tasks.rados.rados.0.smithi093.stdout:2976: finishing write tid 4 to smithi093153278-138 2023-08-27T20:54:33.293 INFO:tasks.rados.rados.0.smithi093.stdout:2976: finishing write tid 5 to smithi093153278-138 2023-08-27T20:54:33.293 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 138 v 470 (ObjNum 1886 snap 0 seq_num 1886) dirty exists 2023-08-27T20:54:33.293 INFO:tasks.rados.rados.0.smithi093.stdout:2976: left oid 138 (ObjNum 1886 snap 0 seq_num 1886) 2023-08-27T20:54:33.293 INFO:tasks.rados.rados.0.smithi093.stdout:2977: expect (ObjNum 1127 snap 0 seq_num 1127) 2023-08-27T20:54:33.418 INFO:tasks.rados.rados.0.smithi093.stdout:2970: expect (ObjNum 1823 snap 0 seq_num 1823) 2023-08-27T20:54:33.553 INFO:tasks.rados.rados.0.smithi093.stdout:2972: expect (ObjNum 1285 snap 0 seq_num 1285) 2023-08-27T20:54:33.757 INFO:tasks.rados.rados.0.smithi093.stdout:2974: expect (ObjNum 1707 snap 0 seq_num 1707) 2023-08-27T20:54:33.887 INFO:tasks.rados.rados.0.smithi093.stdout:2978: seq_num 1887 ranges {514809=450407,1455992=527991,2469658=724053,3608303=106506} 2023-08-27T20:54:33.892 INFO:tasks.rados.rados.0.smithi093.stdout:2978: writing smithi093153278-489 from 514809 to 965216 tid 1 2023-08-27T20:54:33.898 INFO:tasks.rados.rados.0.smithi093.stdout:2978: writing smithi093153278-489 from 1455992 to 1983983 tid 2 2023-08-27T20:54:33.905 INFO:tasks.rados.rados.0.smithi093.stdout:2978: writing smithi093153278-489 from 2469658 to 3193711 tid 3 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2978: writing smithi093153278-489 from 3608303 to 3714809 tid 4 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2969: done (9 left) 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2970: done (8 left) 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2971: done (7 left) 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2972: done (6 left) 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2973: done (5 left) 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2974: done (4 left) 2023-08-27T20:54:33.909 INFO:tasks.rados.rados.0.smithi093.stdout:2975: done (3 left) 2023-08-27T20:54:33.910 INFO:tasks.rados.rados.0.smithi093.stdout:2976: done (2 left) 2023-08-27T20:54:33.910 INFO:tasks.rados.rados.0.smithi093.stdout:2977: done (1 left) 2023-08-27T20:54:33.910 INFO:tasks.rados.rados.0.smithi093.stdout:2979: write oid 396 current snap is 0 2023-08-27T20:54:33.910 INFO:tasks.rados.rados.0.smithi093.stdout:2978: finishing write tid 1 to smithi093153278-489 2023-08-27T20:54:33.910 INFO:tasks.rados.rados.0.smithi093.stdout:2978: finishing write tid 2 to smithi093153278-489 2023-08-27T20:54:33.910 INFO:tasks.rados.rados.0.smithi093.stdout:2979: seq_num 1888 ranges {531616=525656,1514474=671649,2780728=535325} 2023-08-27T20:54:33.914 INFO:tasks.rados.rados.0.smithi093.stdout:2979: writing smithi093153278-396 from 531616 to 1057272 tid 1 2023-08-27T20:54:33.920 INFO:tasks.rados.rados.0.smithi093.stdout:2979: writing smithi093153278-396 from 1514474 to 2186123 tid 2 2023-08-27T20:54:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:2979: writing smithi093153278-396 from 2780728 to 3316053 tid 3 2023-08-27T20:54:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:2980: write oid 272 current snap is 0 2023-08-27T20:54:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:2978: finishing write tid 3 to smithi093153278-489 2023-08-27T20:54:33.927 INFO:tasks.rados.rados.0.smithi093.stdout:2980: seq_num 1889 ranges {548423=600905,1572956=531660,2524504=746597,3740781=7642} 2023-08-27T20:54:33.933 INFO:tasks.rados.rados.0.smithi093.stdout:2980: writing smithi093153278-272 from 548423 to 1149328 tid 1 2023-08-27T20:54:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:33 smithi136 ceph-mon[113531]: pgmap v30: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 4.9 MiB/s wr, 66 op/s 2023-08-27T20:54:33.938 INFO:tasks.rados.rados.0.smithi093.stdout:2980: writing smithi093153278-272 from 1572956 to 2104616 tid 2 2023-08-27T20:54:33.945 INFO:tasks.rados.rados.0.smithi093.stdout:2980: writing smithi093153278-272 from 2524504 to 3271101 tid 3 2023-08-27T20:54:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2980: writing smithi093153278-272 from 3740781 to 3748423 tid 4 2023-08-27T20:54:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2981: write oid 149 current snap is 0 2023-08-27T20:54:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2978: finishing write tid 4 to smithi093153278-489 2023-08-27T20:54:33.948 INFO:tasks.rados.rados.0.smithi093.stdout:2981: seq_num 1890 ranges {565230=676154,2031438=675318,3235574=529656} 2023-08-27T20:54:33.954 INFO:tasks.rados.rados.0.smithi093.stdout:2981: writing smithi093153278-149 from 565230 to 1241384 tid 1 2023-08-27T20:54:33.963 INFO:tasks.rados.rados.0.smithi093.stdout:2981: writing smithi093153278-149 from 2031438 to 2706756 tid 2 2023-08-27T20:54:33.969 INFO:tasks.rados.rados.0.smithi093.stdout:2981: writing smithi093153278-149 from 3235574 to 3765230 tid 3 2023-08-27T20:54:33.969 INFO:tasks.rados.rados.0.smithi093.stdout:2982: delete oid 377 current snap is 0 2023-08-27T20:54:33.969 INFO:tasks.rados.rados.0.smithi093.stdout:2978: finishing write tid 5 to smithi093153278-489 2023-08-27T20:54:33.969 INFO:tasks.rados.rados.0.smithi093.stdout:2978: finishing write tid 6 to smithi093153278-489 2023-08-27T20:54:33.969 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 489 v 376 (ObjNum 1887 snap 0 seq_num 1887) dirty exists 2023-08-27T20:54:33.969 INFO:tasks.rados.rados.0.smithi093.stdout:2978: left oid 489 (ObjNum 1887 snap 0 seq_num 1887) 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2979: finishing write tid 1 to smithi093153278-396 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2979: finishing write tid 2 to smithi093153278-396 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2979: finishing write tid 3 to smithi093153278-396 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2979: finishing write tid 4 to smithi093153278-396 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2979: finishing write tid 5 to smithi093153278-396 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 396 v 474 (ObjNum 1888 snap 0 seq_num 1888) dirty exists 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2979: left oid 396 (ObjNum 1888 snap 0 seq_num 1888) 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2980: finishing write tid 1 to smithi093153278-272 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2980: finishing write tid 2 to smithi093153278-272 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2980: finishing write tid 3 to smithi093153278-272 2023-08-27T20:54:33.970 INFO:tasks.rados.rados.0.smithi093.stdout:2980: finishing write tid 4 to smithi093153278-272 2023-08-27T20:54:33.971 INFO:tasks.rados.rados.0.smithi093.stdout:2980: finishing write tid 5 to smithi093153278-272 2023-08-27T20:54:33.971 INFO:tasks.rados.rados.0.smithi093.stdout:2980: finishing write tid 6 to smithi093153278-272 2023-08-27T20:54:33.971 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 272 v 417 (ObjNum 1889 snap 0 seq_num 1889) dirty exists 2023-08-27T20:54:33.971 INFO:tasks.rados.rados.0.smithi093.stdout:2980: left oid 272 (ObjNum 1889 snap 0 seq_num 1889) 2023-08-27T20:54:33.971 INFO:tasks.rados.rados.0.smithi093.stdout:2981: finishing write tid 1 to smithi093153278-149 2023-08-27T20:54:33.971 INFO:tasks.rados.rados.0.smithi093.stdout:2981: finishing write tid 2 to smithi093153278-149 2023-08-27T20:54:33.971 INFO:tasks.rados.rados.0.smithi093.stdout:2978: done (4 left) 2023-08-27T20:54:33.972 INFO:tasks.rados.rados.0.smithi093.stdout:2979: done (3 left) 2023-08-27T20:54:33.972 INFO:tasks.rados.rados.0.smithi093.stdout:2980: done (2 left) 2023-08-27T20:54:33.972 INFO:tasks.rados.rados.0.smithi093.stdout:2982: done (1 left) 2023-08-27T20:54:33.972 INFO:tasks.rados.rados.0.smithi093.stdout:2983: write oid 425 current snap is 0 2023-08-27T20:54:33.972 INFO:tasks.rados.rados.0.smithi093.stdout:2983: seq_num 1892 ranges {598844=426652,1632049=678987,2774067=696766} 2023-08-27T20:54:33.976 INFO:tasks.rados.rados.0.smithi093.stdout:2983: writing smithi093153278-425 from 598844 to 1025496 tid 1 2023-08-27T20:54:33.983 INFO:tasks.rados.rados.0.smithi093.stdout:2983: writing smithi093153278-425 from 1632049 to 2311036 tid 2 2023-08-27T20:54:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:33 smithi093 ceph-mon[111335]: pgmap v30: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 4.9 MiB/s wr, 66 op/s 2023-08-27T20:54:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:33 smithi093 ceph-mon[118902]: pgmap v30: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 4.9 MiB/s wr, 66 op/s 2023-08-27T20:54:33.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:33.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:33.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:33.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:33.991 INFO:tasks.rados.rados.0.smithi093.stdout:2983: writing smithi093153278-425 from 2774067 to 3470833 tid 3 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2984: read oid 62 snap -1 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2984: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2985: read oid 84 snap -1 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2985: expect (ObjNum 1846 snap 0 seq_num 1846) 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2986: read oid 231 snap -1 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2986: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2987: write oid 320 current snap is 0 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2981: finishing write tid 3 to smithi093153278-149 2023-08-27T20:54:33.992 INFO:tasks.rados.rados.0.smithi093.stdout:2987: seq_num 1893 ranges {615651=618254,1806884=538998,3034196=508038} 2023-08-27T20:54:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:2987: writing smithi093153278-320 from 615651 to 1233905 tid 1 2023-08-27T20:54:34.004 INFO:tasks.rados.rados.0.smithi093.stdout:2987: writing smithi093153278-320 from 1806884 to 2345882 tid 2 2023-08-27T20:54:34.011 INFO:tasks.rados.rados.0.smithi093.stdout:2987: writing smithi093153278-320 from 3034196 to 3542234 tid 3 2023-08-27T20:54:34.011 INFO:tasks.rados.rados.0.smithi093.stdout:2988: read oid 277 snap -1 2023-08-27T20:54:34.011 INFO:tasks.rados.rados.0.smithi093.stdout:2988: expect (ObjNum 1884 snap 0 seq_num 1884) 2023-08-27T20:54:34.011 INFO:tasks.rados.rados.0.smithi093.stdout:2989: read oid 389 snap -1 2023-08-27T20:54:34.011 INFO:tasks.rados.rados.0.smithi093.stdout:2989: expect (ObjNum 1533 snap 0 seq_num 1533) 2023-08-27T20:54:34.012 INFO:tasks.rados.rados.0.smithi093.stdout:2990: read oid 459 snap -1 2023-08-27T20:54:34.012 INFO:tasks.rados.rados.0.smithi093.stdout:2990: expect (ObjNum 1473 snap 0 seq_num 1473) 2023-08-27T20:54:34.012 INFO:tasks.rados.rados.0.smithi093.stdout:2991: write oid 254 current snap is 0 2023-08-27T20:54:34.012 INFO:tasks.rados.rados.0.smithi093.stdout:2981: finishing write tid 4 to smithi093153278-149 2023-08-27T20:54:34.012 INFO:tasks.rados.rados.0.smithi093.stdout:2991: seq_num 1894 ranges {632458=693503,1865366=682656,3061619=719310} 2023-08-27T20:54:34.018 INFO:tasks.rados.rados.0.smithi093.stdout:2991: writing smithi093153278-254 from 632458 to 1325961 tid 1 2023-08-27T20:54:34.025 INFO:tasks.rados.rados.0.smithi093.stdout:2991: writing smithi093153278-254 from 1865366 to 2548022 tid 2 2023-08-27T20:54:34.032 INFO:tasks.rados.rados.0.smithi093.stdout:2991: writing smithi093153278-254 from 3061619 to 3780929 tid 3 2023-08-27T20:54:34.032 INFO:tasks.rados.rados.0.smithi093.stdout:2992: write oid 78 current snap is 0 2023-08-27T20:54:34.032 INFO:tasks.rados.rados.0.smithi093.stdout:2981: finishing write tid 5 to smithi093153278-149 2023-08-27T20:54:34.033 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 149 v 375 (ObjNum 1890 snap 0 seq_num 1890) dirty exists 2023-08-27T20:54:34.033 INFO:tasks.rados.rados.0.smithi093.stdout:2981: left oid 149 (ObjNum 1890 snap 0 seq_num 1890) 2023-08-27T20:54:34.033 INFO:tasks.rados.rados.0.smithi093.stdout:2992: seq_num 1895 ranges {649265=768752,2207495=542667,3372689=476576} 2023-08-27T20:54:34.040 INFO:tasks.rados.rados.0.smithi093.stdout:2992: writing smithi093153278-78 from 649265 to 1418017 tid 1 2023-08-27T20:54:34.047 INFO:tasks.rados.rados.0.smithi093.stdout:2992: writing smithi093153278-78 from 2207495 to 2750162 tid 2 2023-08-27T20:54:34.053 INFO:tasks.rados.rados.0.smithi093.stdout:2992: writing smithi093153278-78 from 3372689 to 3849265 tid 3 2023-08-27T20:54:34.054 INFO:tasks.rados.rados.0.smithi093.stdout:2981: done (10 left) 2023-08-27T20:54:34.054 INFO:tasks.rados.rados.0.smithi093.stdout:2993: read oid 167 snap -1 2023-08-27T20:54:34.054 INFO:tasks.rados.rados.0.smithi093.stdout:2993: expect (ObjNum 1830 snap 0 seq_num 1830) 2023-08-27T20:54:34.054 INFO:tasks.rados.rados.0.smithi093.stdout:2994: write oid 283 current snap is 0 2023-08-27T20:54:34.054 INFO:tasks.rados.rados.0.smithi093.stdout:2983: finishing write tid 1 to smithi093153278-425 2023-08-27T20:54:34.054 INFO:tasks.rados.rados.0.smithi093.stdout:2983: finishing write tid 2 to smithi093153278-425 2023-08-27T20:54:34.054 INFO:tasks.rados.rados.0.smithi093.stdout:2994: seq_num 1896 ranges {666072=444001,1865977=686325,3000112=458207} 2023-08-27T20:54:34.059 INFO:tasks.rados.rados.0.smithi093.stdout:2994: writing smithi093153278-283 from 666072 to 1110073 tid 1 2023-08-27T20:54:34.067 INFO:tasks.rados.rados.0.smithi093.stdout:2994: writing smithi093153278-283 from 1865977 to 2552302 tid 2 2023-08-27T20:54:34.073 INFO:tasks.rados.rados.0.smithi093.stdout:2994: writing smithi093153278-283 from 3000112 to 3458319 tid 3 2023-08-27T20:54:34.073 INFO:tasks.rados.rados.0.smithi093.stdout:2995: write oid 24 current snap is 0 2023-08-27T20:54:34.073 INFO:tasks.rados.rados.0.smithi093.stdout:2995: seq_num 1897 ranges {682879=519250,1924459=546336,3027535=669479} 2023-08-27T20:54:34.079 INFO:tasks.rados.rados.0.smithi093.stdout:2995: writing smithi093153278-24 from 682879 to 1202129 tid 1 2023-08-27T20:54:34.086 INFO:tasks.rados.rados.0.smithi093.stdout:2995: writing smithi093153278-24 from 1924459 to 2470795 tid 2 2023-08-27T20:54:34.093 INFO:tasks.rados.rados.0.smithi093.stdout:2995: writing smithi093153278-24 from 3027535 to 3697014 tid 3 2023-08-27T20:54:34.093 INFO:tasks.rados.rados.0.smithi093.stdout:2996: write oid 474 current snap is 0 2023-08-27T20:54:34.093 INFO:tasks.rados.rados.0.smithi093.stdout:2996: seq_num 1898 ranges {699686=594499,1982941=689994,3454958=444728} 2023-08-27T20:54:34.100 INFO:tasks.rados.rados.0.smithi093.stdout:2996: writing smithi093153278-474 from 699686 to 1294185 tid 1 2023-08-27T20:54:34.107 INFO:tasks.rados.rados.0.smithi093.stdout:2996: writing smithi093153278-474 from 1982941 to 2672935 tid 2 2023-08-27T20:54:34.114 INFO:tasks.rados.rados.0.smithi093.stdout:2996: writing smithi093153278-474 from 3454958 to 3899686 tid 3 2023-08-27T20:54:34.115 INFO:tasks.rados.rados.0.smithi093.stdout:2997: read oid 315 snap -1 2023-08-27T20:54:34.115 INFO:tasks.rados.rados.0.smithi093.stdout:2997: expect (ObjNum 1558 snap 0 seq_num 1558) 2023-08-27T20:54:34.115 INFO:tasks.rados.rados.0.smithi093.stdout:2998: write oid 306 current snap is 0 2023-08-27T20:54:34.115 INFO:tasks.rados.rados.0.smithi093.stdout:2986: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:54:34.239 INFO:tasks.rados.rados.0.smithi093.stdout:2985: expect (ObjNum 1846 snap 0 seq_num 1846) 2023-08-27T20:54:34.379 INFO:tasks.rados.rados.0.smithi093.stdout:2983: finishing write tid 3 to smithi093153278-425 2023-08-27T20:54:34.379 INFO:tasks.rados.rados.0.smithi093.stdout:2983: finishing write tid 4 to smithi093153278-425 2023-08-27T20:54:34.379 INFO:tasks.rados.rados.0.smithi093.stdout:2983: finishing write tid 5 to smithi093153278-425 2023-08-27T20:54:34.380 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 425 v 379 (ObjNum 1892 snap 0 seq_num 1892) dirty exists 2023-08-27T20:54:34.380 INFO:tasks.rados.rados.0.smithi093.stdout:2983: left oid 425 (ObjNum 1892 snap 0 seq_num 1892) 2023-08-27T20:54:34.380 INFO:tasks.rados.rados.0.smithi093.stdout:2984: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:54:34.583 INFO:tasks.rados.rados.0.smithi093.stdout:2987: finishing write tid 1 to smithi093153278-320 2023-08-27T20:54:34.584 INFO:tasks.rados.rados.0.smithi093.stdout:2987: finishing write tid 2 to smithi093153278-320 2023-08-27T20:54:34.584 INFO:tasks.rados.rados.0.smithi093.stdout:2998: seq_num 1899 ranges {716493=669748,1925070=433652,2849675=692023} 2023-08-27T20:54:34.590 INFO:tasks.rados.rados.0.smithi093.stdout:2998: writing smithi093153278-306 from 716493 to 1386241 tid 1 2023-08-27T20:54:34.596 INFO:tasks.rados.rados.0.smithi093.stdout:2998: writing smithi093153278-306 from 1925070 to 2358722 tid 2 2023-08-27T20:54:34.603 INFO:tasks.rados.rados.0.smithi093.stdout:2998: writing smithi093153278-306 from 2849675 to 3541698 tid 3 2023-08-27T20:54:34.604 INFO:tasks.rados.rados.0.smithi093.stdout:2983: done (15 left) 2023-08-27T20:54:34.604 INFO:tasks.rados.rados.0.smithi093.stdout:2984: done (14 left) 2023-08-27T20:54:34.604 INFO:tasks.rados.rados.0.smithi093.stdout:2985: done (13 left) 2023-08-27T20:54:34.604 INFO:tasks.rados.rados.0.smithi093.stdout:2986: done (12 left) 2023-08-27T20:54:34.604 INFO:tasks.rados.rados.0.smithi093.stdout:2999: write oid 263 current snap is 0 2023-08-27T20:54:34.604 INFO:tasks.rados.rados.0.smithi093.stdout:2999: seq_num 1900 ranges {733300=744997,1983552=693663,3393451=539849} 2023-08-27T20:54:34.610 INFO:tasks.rados.rados.0.smithi093.stdout:2999: writing smithi093153278-263 from 733300 to 1478297 tid 1 2023-08-27T20:54:34.617 INFO:tasks.rados.rados.0.smithi093.stdout:2999: writing smithi093153278-263 from 1983552 to 2677215 tid 2 2023-08-27T20:54:34.624 INFO:tasks.rados.rados.0.smithi093.stdout:2999: writing smithi093153278-263 from 3393451 to 3933300 tid 3 2023-08-27T20:54:34.624 INFO:tasks.rados.rados.0.smithi093.stdout:3000: read oid 388 snap -1 2023-08-27T20:54:34.625 INFO:tasks.rados.rados.0.smithi093.stdout:3000: expect deleted 2023-08-27T20:54:34.625 INFO:tasks.rados.rados.0.smithi093.stdout:3001: write oid 89 current snap is 0 2023-08-27T20:54:34.625 INFO:tasks.rados.rados.0.smithi093.stdout:2990: expect (ObjNum 1473 snap 0 seq_num 1473) 2023-08-27T20:54:34.759 INFO:tasks.rados.rados.0.smithi093.stdout:2989: expect (ObjNum 1533 snap 0 seq_num 1533) 2023-08-27T20:54:34.876 INFO:tasks.rados.rados.0.smithi093.stdout:2988: expect (ObjNum 1884 snap 0 seq_num 1884) 2023-08-27T20:54:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1384757074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1384757074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1384757074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:35.038 INFO:tasks.rados.rados.0.smithi093.stdout:2987: finishing write tid 3 to smithi093153278-320 2023-08-27T20:54:35.038 INFO:tasks.rados.rados.0.smithi093.stdout:2987: finishing write tid 4 to smithi093153278-320 2023-08-27T20:54:35.038 INFO:tasks.rados.rados.0.smithi093.stdout:2987: finishing write tid 5 to smithi093153278-320 2023-08-27T20:54:35.038 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 320 v 518 (ObjNum 1893 snap 0 seq_num 1893) dirty exists 2023-08-27T20:54:35.038 INFO:tasks.rados.rados.0.smithi093.stdout:2987: left oid 320 (ObjNum 1893 snap 0 seq_num 1893) 2023-08-27T20:54:35.038 INFO:tasks.rados.rados.0.smithi093.stdout:2991: finishing write tid 1 to smithi093153278-254 2023-08-27T20:54:35.038 INFO:tasks.rados.rados.0.smithi093.stdout:3001: seq_num 1901 ranges {750107=536599,1758387=437321,2620874=430920,3568355=381752} 2023-08-27T20:54:35.045 INFO:tasks.rados.rados.0.smithi093.stdout:3001: writing smithi093153278-89 from 750107 to 1286706 tid 1 2023-08-27T20:54:35.050 INFO:tasks.rados.rados.0.smithi093.stdout:3001: writing smithi093153278-89 from 1758387 to 2195708 tid 2 2023-08-27T20:54:35.054 INFO:tasks.rados.rados.0.smithi093.stdout:3001: writing smithi093153278-89 from 2620874 to 3051794 tid 3 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:3001: writing smithi093153278-89 from 3568355 to 3950107 tid 4 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:2987: done (14 left) 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:2988: done (13 left) 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:2989: done (12 left) 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:2990: done (11 left) 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:3002: read oid 446 snap -1 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:3002: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:3003: write oid 478 current snap is 0 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:2991: finishing write tid 2 to smithi093153278-254 2023-08-27T20:54:35.060 INFO:tasks.rados.rados.0.smithi093.stdout:2991: finishing write tid 3 to smithi093153278-254 2023-08-27T20:54:35.061 INFO:tasks.rados.rados.0.smithi093.stdout:3003: seq_num 1902 ranges {766914=611848,1816869=697332,3164650=642192} 2023-08-27T20:54:35.067 INFO:tasks.rados.rados.0.smithi093.stdout:3003: writing smithi093153278-478 from 766914 to 1378762 tid 1 2023-08-27T20:54:35.073 INFO:tasks.rados.rados.0.smithi093.stdout:3003: writing smithi093153278-478 from 1816869 to 2514201 tid 2 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:3003: writing smithi093153278-478 from 3164650 to 3806842 tid 3 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:3004: read oid 488 snap -1 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:3004: expect (ObjNum 968 snap 0 seq_num 968) 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:3005: read oid 402 snap -1 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:3005: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:3006: delete oid 207 current snap is 0 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:2991: finishing write tid 4 to smithi093153278-254 2023-08-27T20:54:35.081 INFO:tasks.rados.rados.0.smithi093.stdout:2991: finishing write tid 5 to smithi093153278-254 2023-08-27T20:54:35.082 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 254 v 380 (ObjNum 1894 snap 0 seq_num 1894) dirty exists 2023-08-27T20:54:35.082 INFO:tasks.rados.rados.0.smithi093.stdout:2991: left oid 254 (ObjNum 1894 snap 0 seq_num 1894) 2023-08-27T20:54:35.082 INFO:tasks.rados.rados.0.smithi093.stdout:2992: finishing write tid 1 to smithi093153278-78 2023-08-27T20:54:35.082 INFO:tasks.rados.rados.0.smithi093.stdout:2992: finishing write tid 2 to smithi093153278-78 2023-08-27T20:54:35.082 INFO:tasks.rados.rados.0.smithi093.stdout:2993: expect (ObjNum 1830 snap 0 seq_num 1830) 2023-08-27T20:54:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:2992: finishing write tid 3 to smithi093153278-78 2023-08-27T20:54:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:2991: done (15 left) 2023-08-27T20:54:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:2993: done (14 left) 2023-08-27T20:54:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:3006: done (13 left) 2023-08-27T20:54:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:3007: write oid 376 current snap is 0 2023-08-27T20:54:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:2992: finishing write tid 4 to smithi093153278-78 2023-08-27T20:54:35.258 INFO:tasks.rados.rados.0.smithi093.stdout:2992: finishing write tid 5 to smithi093153278-78 2023-08-27T20:54:35.259 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 346 (ObjNum 1895 snap 0 seq_num 1895) dirty exists 2023-08-27T20:54:35.259 INFO:tasks.rados.rados.0.smithi093.stdout:2992: left oid 78 (ObjNum 1895 snap 0 seq_num 1895) 2023-08-27T20:54:35.259 INFO:tasks.rados.rados.0.smithi093.stdout:2994: finishing write tid 1 to smithi093153278-283 2023-08-27T20:54:35.259 INFO:tasks.rados.rados.0.smithi093.stdout:2994: finishing write tid 2 to smithi093153278-283 2023-08-27T20:54:35.259 INFO:tasks.rados.rados.0.smithi093.stdout:2994: finishing write tid 3 to smithi093153278-283 2023-08-27T20:54:35.259 INFO:tasks.rados.rados.0.smithi093.stdout:2994: finishing write tid 4 to smithi093153278-283 2023-08-27T20:54:35.259 INFO:tasks.rados.rados.0.smithi093.stdout:3007: seq_num 1904 ranges {400528=762346,1817480=183048} 2023-08-27T20:54:35.263 INFO:tasks.rados.rados.0.smithi093.stdout:3007: writing smithi093153278-376 from 400528 to 1162874 tid 1 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:3007: writing smithi093153278-376 from 1817480 to 2000528 tid 2 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:2992: done (13 left) 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:3008: write oid 61 current snap is 0 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:2994: finishing write tid 5 to smithi093153278-283 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 283 v 400 (ObjNum 1896 snap 0 seq_num 1896) dirty exists 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:2994: left oid 283 (ObjNum 1896 snap 0 seq_num 1896) 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:2995: finishing write tid 1 to smithi093153278-24 2023-08-27T20:54:35.269 INFO:tasks.rados.rados.0.smithi093.stdout:3008: seq_num 1905 ranges {417335=437595,1475962=444659} 2023-08-27T20:54:35.273 INFO:tasks.rados.rados.0.smithi093.stdout:3008: writing smithi093153278-61 from 417335 to 854930 tid 1 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:3008: writing smithi093153278-61 from 1475962 to 1920621 tid 2 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:2994: done (13 left) 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:3009: read oid 240 snap -1 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:3009: expect deleted 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:3010: write oid 269 current snap is 0 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:2995: finishing write tid 2 to smithi093153278-24 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:2995: finishing write tid 3 to smithi093153278-24 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:2995: finishing write tid 4 to smithi093153278-24 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:2995: finishing write tid 5 to smithi093153278-24 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 24 v 348 (ObjNum 1897 snap 0 seq_num 1897) dirty exists 2023-08-27T20:54:35.280 INFO:tasks.rados.rados.0.smithi093.stdout:2995: left oid 24 (ObjNum 1897 snap 0 seq_num 1897) 2023-08-27T20:54:35.281 INFO:tasks.rados.rados.0.smithi093.stdout:2996: finishing write tid 1 to smithi093153278-474 2023-08-27T20:54:35.281 INFO:tasks.rados.rados.0.smithi093.stdout:3010: seq_num 1906 ranges {434142=512844,1534444=499698} 2023-08-27T20:54:35.284 INFO:tasks.rados.rados.0.smithi093.stdout:3010: writing smithi093153278-269 from 434142 to 946986 tid 1 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:3010: writing smithi093153278-269 from 1534444 to 2034142 tid 2 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:2995: done (14 left) 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:3011: read oid 298 snap -1 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:3011: expect (ObjNum 1651 snap 0 seq_num 1651) 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:3012: write oid 317 current snap is 0 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:2996: finishing write tid 2 to smithi093153278-474 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:2996: finishing write tid 3 to smithi093153278-474 2023-08-27T20:54:35.291 INFO:tasks.rados.rados.0.smithi093.stdout:3012: seq_num 1907 ranges {450949=588093,1476573=448328} 2023-08-27T20:54:35.296 INFO:tasks.rados.rados.0.smithi093.stdout:3012: writing smithi093153278-317 from 450949 to 1039042 tid 1 2023-08-27T20:54:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:3012: writing smithi093153278-317 from 1476573 to 1924901 tid 2 2023-08-27T20:54:35.301 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:54:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:2996: finishing write tid 4 to smithi093153278-474 2023-08-27T20:54:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:2996: finishing write tid 5 to smithi093153278-474 2023-08-27T20:54:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 474 v 349 (ObjNum 1898 snap 0 seq_num 1898) dirty exists 2023-08-27T20:54:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:2996: left oid 474 (ObjNum 1898 snap 0 seq_num 1898) 2023-08-27T20:54:35.301 INFO:tasks.rados.rados.0.smithi093.stdout:2997: expect (ObjNum 1558 snap 0 seq_num 1558) 2023-08-27T20:54:35.409 INFO:tasks.rados.rados.0.smithi093.stdout:2998: finishing write tid 1 to smithi093153278-306 2023-08-27T20:54:35.409 INFO:tasks.rados.rados.0.smithi093.stdout:2998: finishing write tid 2 to smithi093153278-306 2023-08-27T20:54:35.409 INFO:tasks.rados.rados.0.smithi093.stdout:2998: finishing write tid 3 to smithi093153278-306 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2998: finishing write tid 4 to smithi093153278-306 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2998: finishing write tid 5 to smithi093153278-306 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 306 v 350 (ObjNum 1899 snap 0 seq_num 1899) dirty exists 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2998: left oid 306 (ObjNum 1899 snap 0 seq_num 1899) 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2999: finishing write tid 1 to smithi093153278-263 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2999: finishing write tid 2 to smithi093153278-263 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2996: done (15 left) 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2997: done (14 left) 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:2998: done (13 left) 2023-08-27T20:54:35.410 INFO:tasks.rados.rados.0.smithi093.stdout:3013: read oid 440 snap -1 2023-08-27T20:54:35.411 INFO:tasks.rados.rados.0.smithi093.stdout:3013: expect (ObjNum 1444 snap 0 seq_num 1444) 2023-08-27T20:54:35.411 INFO:tasks.rados.rados.0.smithi093.stdout:3014: read oid 318 snap -1 2023-08-27T20:54:35.411 INFO:tasks.rados.rados.0.smithi093.stdout:3014: expect (ObjNum 1678 snap 0 seq_num 1678) 2023-08-27T20:54:35.411 INFO:tasks.rados.rados.0.smithi093.stdout:3015: write oid 287 current snap is 0 2023-08-27T20:54:35.411 INFO:tasks.rados.rados.0.smithi093.stdout:3015: seq_num 1908 ranges {467756=663342,1535055=532701} 2023-08-27T20:54:35.415 INFO:tasks.rados.rados.0.smithi093.stdout:3015: writing smithi093153278-287 from 467756 to 1131098 tid 1 2023-08-27T20:54:35.421 INFO:tasks.rados.rados.0.smithi093.stdout:3015: writing smithi093153278-287 from 1535055 to 2067756 tid 2 2023-08-27T20:54:35.421 INFO:tasks.rados.rados.0.smithi093.stdout:3000: done (15 left) 2023-08-27T20:54:35.421 INFO:tasks.rados.rados.0.smithi093.stdout:3016: delete oid 196 current snap is 0 2023-08-27T20:54:35.421 INFO:tasks.rados.rados.0.smithi093.stdout:2999: finishing write tid 3 to smithi093153278-263 2023-08-27T20:54:35.421 INFO:tasks.rados.rados.0.smithi093.stdout:2999: finishing write tid 4 to smithi093153278-263 2023-08-27T20:54:35.421 INFO:tasks.rados.rados.0.smithi093.stdout:2999: finishing write tid 5 to smithi093153278-263 2023-08-27T20:54:35.421 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 263 v 463 (ObjNum 1900 snap 0 seq_num 1900) dirty exists 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:2999: left oid 263 (ObjNum 1900 snap 0 seq_num 1900) 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3001: finishing write tid 1 to smithi093153278-89 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3001: finishing write tid 2 to smithi093153278-89 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3001: finishing write tid 3 to smithi093153278-89 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3001: finishing write tid 4 to smithi093153278-89 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3001: finishing write tid 5 to smithi093153278-89 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3001: finishing write tid 6 to smithi093153278-89 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 89 v 332 (ObjNum 1901 snap 0 seq_num 1901) dirty exists 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3001: left oid 89 (ObjNum 1901 snap 0 seq_num 1901) 2023-08-27T20:54:35.422 INFO:tasks.rados.rados.0.smithi093.stdout:3002: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:54:35.530 INFO:tasks.rados.rados.0.smithi093.stdout:3003: finishing write tid 1 to smithi093153278-478 2023-08-27T20:54:35.530 INFO:tasks.rados.rados.0.smithi093.stdout:3003: finishing write tid 2 to smithi093153278-478 2023-08-27T20:54:35.530 INFO:tasks.rados.rados.0.smithi093.stdout:3003: finishing write tid 3 to smithi093153278-478 2023-08-27T20:54:35.530 INFO:tasks.rados.rados.0.smithi093.stdout:3003: finishing write tid 4 to smithi093153278-478 2023-08-27T20:54:35.530 INFO:tasks.rados.rados.0.smithi093.stdout:3004: expect (ObjNum 968 snap 0 seq_num 968) 2023-08-27T20:54:35.625 INFO:tasks.rados.rados.0.smithi093.stdout:3003: finishing write tid 5 to smithi093153278-478 2023-08-27T20:54:35.625 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 404 (ObjNum 1902 snap 0 seq_num 1902) dirty exists 2023-08-27T20:54:35.625 INFO:tasks.rados.rados.0.smithi093.stdout:3003: left oid 478 (ObjNum 1902 snap 0 seq_num 1902) 2023-08-27T20:54:35.626 INFO:tasks.rados.rados.0.smithi093.stdout:3005: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:54:35.633 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:54:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:54:35] "GET /metrics HTTP/1.1" 200 242534 "" "Prometheus/2.43.0" 2023-08-27T20:54:35.789 INFO:tasks.rados.rados.0.smithi093.stdout:3007: finishing write tid 1 to smithi093153278-376 2023-08-27T20:54:35.789 INFO:tasks.rados.rados.0.smithi093.stdout:3007: finishing write tid 2 to smithi093153278-376 2023-08-27T20:54:35.789 INFO:tasks.rados.rados.0.smithi093.stdout:3007: finishing write tid 3 to smithi093153278-376 2023-08-27T20:54:35.789 INFO:tasks.rados.rados.0.smithi093.stdout:3007: finishing write tid 4 to smithi093153278-376 2023-08-27T20:54:35.789 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 376 v 521 (ObjNum 1904 snap 0 seq_num 1904) dirty exists 2023-08-27T20:54:35.789 INFO:tasks.rados.rados.0.smithi093.stdout:3007: left oid 376 (ObjNum 1904 snap 0 seq_num 1904) 2023-08-27T20:54:35.789 INFO:tasks.rados.rados.0.smithi093.stdout:3008: finishing write tid 1 to smithi093153278-61 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:2999: done (15 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3001: done (14 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3002: done (13 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3003: done (12 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3004: done (11 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3005: done (10 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3007: done (9 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3016: done (8 left) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3017: read oid 293 snap -1 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3017: expect (ObjNum 292 snap 0 seq_num 292) 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3018: read oid 275 snap -1 2023-08-27T20:54:35.790 INFO:tasks.rados.rados.0.smithi093.stdout:3018: expect (ObjNum 1556 snap 0 seq_num 1556) 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3019: write oid 328 current snap is 0 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3008: finishing write tid 2 to smithi093153278-61 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3008: finishing write tid 3 to smithi093153278-61 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3008: finishing write tid 4 to smithi093153278-61 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 61 v 407 (ObjNum 1905 snap 0 seq_num 1905) dirty exists 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3008: left oid 61 (ObjNum 1905 snap 0 seq_num 1905) 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3010: finishing write tid 1 to smithi093153278-269 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3012: finishing write tid 1 to smithi093153278-317 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3010: finishing write tid 2 to smithi093153278-269 2023-08-27T20:54:35.791 INFO:tasks.rados.rados.0.smithi093.stdout:3019: seq_num 1910 ranges {501370=530193,1768372=332998} 2023-08-27T20:54:35.794 INFO:tasks.rados.rados.0.smithi093.stdout:3019: writing smithi093153278-328 from 501370 to 1031563 tid 1 2023-08-27T20:54:35.800 INFO:tasks.rados.rados.0.smithi093.stdout:3019: writing smithi093153278-328 from 1768372 to 2101370 tid 2 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3008: done (10 left) 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3009: done (9 left) 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3020: read oid 472 snap -1 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3020: expect deleted 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3021: read oid 395 snap -1 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3021: expect (ObjNum 1794 snap 0 seq_num 1794) 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3022: write oid 29 current snap is 0 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3010: finishing write tid 3 to smithi093153278-269 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3012: finishing write tid 2 to smithi093153278-317 2023-08-27T20:54:35.801 INFO:tasks.rados.rados.0.smithi093.stdout:3022: seq_num 1911 ranges {518177=605442,1710501=407676} 2023-08-27T20:54:35.806 INFO:tasks.rados.rados.0.smithi093.stdout:3022: writing smithi093153278-29 from 518177 to 1123619 tid 1 2023-08-27T20:54:35.812 INFO:tasks.rados.rados.0.smithi093.stdout:3022: writing smithi093153278-29 from 1710501 to 2118177 tid 2 2023-08-27T20:54:35.813 INFO:tasks.rados.rados.0.smithi093.stdout:3023: read oid 432 snap -1 2023-08-27T20:54:35.813 INFO:tasks.rados.rados.0.smithi093.stdout:3023: expect (ObjNum 1738 snap 0 seq_num 1738) 2023-08-27T20:54:35.813 INFO:tasks.rados.rados.0.smithi093.stdout:3024: read oid 338 snap -1 2023-08-27T20:54:35.813 INFO:tasks.rados.rados.0.smithi093.stdout:3024: expect (ObjNum 1472 snap 0 seq_num 1472) 2023-08-27T20:54:35.813 INFO:tasks.rados.rados.0.smithi093.stdout:3012: finishing write tid 3 to smithi093153278-317 2023-08-27T20:54:35.813 INFO:tasks.rados.rados.0.smithi093.stdout:3012: finishing write tid 4 to smithi093153278-317 2023-08-27T20:54:35.813 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 317 v 352 (ObjNum 1907 snap 0 seq_num 1907) dirty exists 2023-08-27T20:54:35.814 INFO:tasks.rados.rados.0.smithi093.stdout:3012: left oid 317 (ObjNum 1907 snap 0 seq_num 1907) 2023-08-27T20:54:35.814 INFO:tasks.rados.rados.0.smithi093.stdout:3011: expect (ObjNum 1651 snap 0 seq_num 1651) 2023-08-27T20:54:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:35 smithi136 ceph-mon[113531]: pgmap v31: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 4.9 MiB/s wr, 66 op/s 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3010: finishing write tid 4 to smithi093153278-269 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 269 v 351 (ObjNum 1906 snap 0 seq_num 1906) dirty exists 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3010: left oid 269 (ObjNum 1906 snap 0 seq_num 1906) 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3025: read oid 132 snap -1 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3025: expect (ObjNum 1862 snap 0 seq_num 1862) 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3010: done (14 left) 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3011: done (13 left) 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3012: done (12 left) 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3026: delete oid 453 current snap is 0 2023-08-27T20:54:35.981 INFO:tasks.rados.rados.0.smithi093.stdout:3013: expect (ObjNum 1444 snap 0 seq_num 1444) 2023-08-27T20:54:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:35 smithi093 ceph-mon[111335]: pgmap v31: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 4.9 MiB/s wr, 66 op/s 2023-08-27T20:54:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:35 smithi093 ceph-mon[118902]: pgmap v31: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 4.9 MiB/s wr, 66 op/s 2023-08-27T20:54:36.050 INFO:tasks.rados.rados.0.smithi093.stdout:3014: expect (ObjNum 1678 snap 0 seq_num 1678) 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:3015: finishing write tid 1 to smithi093153278-287 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:3015: finishing write tid 2 to smithi093153278-287 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:3015: finishing write tid 3 to smithi093153278-287 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:3015: finishing write tid 4 to smithi093153278-287 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 287 v 383 (ObjNum 1908 snap 0 seq_num 1908) dirty exists 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:3015: left oid 287 (ObjNum 1908 snap 0 seq_num 1908) 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:3013: done (12 left) 2023-08-27T20:54:36.119 INFO:tasks.rados.rados.0.smithi093.stdout:3014: done (11 left) 2023-08-27T20:54:36.120 INFO:tasks.rados.rados.0.smithi093.stdout:3015: done (10 left) 2023-08-27T20:54:36.120 INFO:tasks.rados.rados.0.smithi093.stdout:3026: done (9 left) 2023-08-27T20:54:36.120 INFO:tasks.rados.rados.0.smithi093.stdout:3027: write oid 5 current snap is 0 2023-08-27T20:54:36.120 INFO:tasks.rados.rados.0.smithi093.stdout:3018: expect (ObjNum 1556 snap 0 seq_num 1556) 2023-08-27T20:54:36.243 INFO:tasks.rados.rados.0.smithi093.stdout:3017: expect (ObjNum 292 snap 0 seq_num 292) 2023-08-27T20:54:36.337 INFO:tasks.rados.rados.0.smithi093.stdout:3019: finishing write tid 1 to smithi093153278-328 2023-08-27T20:54:36.337 INFO:tasks.rados.rados.0.smithi093.stdout:3019: finishing write tid 2 to smithi093153278-328 2023-08-27T20:54:36.337 INFO:tasks.rados.rados.0.smithi093.stdout:3019: finishing write tid 3 to smithi093153278-328 2023-08-27T20:54:36.337 INFO:tasks.rados.rados.0.smithi093.stdout:3019: finishing write tid 4 to smithi093153278-328 2023-08-27T20:54:36.337 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 410 (ObjNum 1910 snap 0 seq_num 1910) dirty exists 2023-08-27T20:54:36.338 INFO:tasks.rados.rados.0.smithi093.stdout:3019: left oid 328 (ObjNum 1910 snap 0 seq_num 1910) 2023-08-27T20:54:36.338 INFO:tasks.rados.rados.0.smithi093.stdout:3021: expect (ObjNum 1794 snap 0 seq_num 1794) 2023-08-27T20:54:36.505 INFO:tasks.rados.rados.0.smithi093.stdout:3022: finishing write tid 1 to smithi093153278-29 2023-08-27T20:54:36.505 INFO:tasks.rados.rados.0.smithi093.stdout:3022: finishing write tid 2 to smithi093153278-29 2023-08-27T20:54:36.505 INFO:tasks.rados.rados.0.smithi093.stdout:3022: finishing write tid 3 to smithi093153278-29 2023-08-27T20:54:36.505 INFO:tasks.rados.rados.0.smithi093.stdout:3022: finishing write tid 4 to smithi093153278-29 2023-08-27T20:54:36.505 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 29 v 420 (ObjNum 1911 snap 0 seq_num 1911) dirty exists 2023-08-27T20:54:36.505 INFO:tasks.rados.rados.0.smithi093.stdout:3022: left oid 29 (ObjNum 1911 snap 0 seq_num 1911) 2023-08-27T20:54:36.505 INFO:tasks.rados.rados.0.smithi093.stdout:3023: expect (ObjNum 1738 snap 0 seq_num 1738) 2023-08-27T20:54:36.653 INFO:tasks.rados.rados.0.smithi093.stdout:3024: expect (ObjNum 1472 snap 0 seq_num 1472) 2023-08-27T20:54:36.771 INFO:tasks.rados.rados.0.smithi093.stdout:3025: expect (ObjNum 1862 snap 0 seq_num 1862) 2023-08-27T20:54:36.934 INFO:tasks.rados.rados.0.smithi093.stdout:3027: seq_num 1913 ranges {551791=755940,1827465=324326} 2023-08-27T20:54:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2761801715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:36.941 INFO:tasks.rados.rados.0.smithi093.stdout:3027: writing smithi093153278-5 from 551791 to 1307731 tid 1 2023-08-27T20:54:36.947 INFO:tasks.rados.rados.0.smithi093.stdout:3027: writing smithi093153278-5 from 1827465 to 2151791 tid 2 2023-08-27T20:54:36.947 INFO:tasks.rados.rados.0.smithi093.stdout:3017: done (9 left) 2023-08-27T20:54:36.947 INFO:tasks.rados.rados.0.smithi093.stdout:3018: done (8 left) 2023-08-27T20:54:36.947 INFO:tasks.rados.rados.0.smithi093.stdout:3019: done (7 left) 2023-08-27T20:54:36.947 INFO:tasks.rados.rados.0.smithi093.stdout:3020: done (6 left) 2023-08-27T20:54:36.947 INFO:tasks.rados.rados.0.smithi093.stdout:3021: done (5 left) 2023-08-27T20:54:36.947 INFO:tasks.rados.rados.0.smithi093.stdout:3022: done (4 left) 2023-08-27T20:54:36.948 INFO:tasks.rados.rados.0.smithi093.stdout:3023: done (3 left) 2023-08-27T20:54:36.948 INFO:tasks.rados.rados.0.smithi093.stdout:3024: done (2 left) 2023-08-27T20:54:36.948 INFO:tasks.rados.rados.0.smithi093.stdout:3025: done (1 left) 2023-08-27T20:54:36.948 INFO:tasks.rados.rados.0.smithi093.stdout:3028: write oid 313 current snap is 0 2023-08-27T20:54:36.948 INFO:tasks.rados.rados.0.smithi093.stdout:3027: finishing write tid 1 to smithi093153278-5 2023-08-27T20:54:36.948 INFO:tasks.rados.rados.0.smithi093.stdout:3028: seq_num 1914 ranges {568598=431189,1485947=602993} 2023-08-27T20:54:36.952 INFO:tasks.rados.rados.0.smithi093.stdout:3028: writing smithi093153278-313 from 568598 to 999787 tid 1 2023-08-27T20:54:36.958 INFO:tasks.rados.rados.0.smithi093.stdout:3028: writing smithi093153278-313 from 1485947 to 2088940 tid 2 2023-08-27T20:54:36.958 INFO:tasks.rados.rados.0.smithi093.stdout:3029: read oid 406 snap -1 2023-08-27T20:54:36.958 INFO:tasks.rados.rados.0.smithi093.stdout:3029: expect (ObjNum 1661 snap 0 seq_num 1661) 2023-08-27T20:54:36.958 INFO:tasks.rados.rados.0.smithi093.stdout:3030: delete oid 137 current snap is 0 2023-08-27T20:54:36.958 INFO:tasks.rados.rados.0.smithi093.stdout:3027: finishing write tid 2 to smithi093153278-5 2023-08-27T20:54:36.959 INFO:tasks.rados.rados.0.smithi093.stdout:3027: finishing write tid 3 to smithi093153278-5 2023-08-27T20:54:36.959 INFO:tasks.rados.rados.0.smithi093.stdout:3027: finishing write tid 4 to smithi093153278-5 2023-08-27T20:54:36.959 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 5 v 356 (ObjNum 1913 snap 0 seq_num 1913) dirty exists 2023-08-27T20:54:36.959 INFO:tasks.rados.rados.0.smithi093.stdout:3027: left oid 5 (ObjNum 1913 snap 0 seq_num 1913) 2023-08-27T20:54:36.959 INFO:tasks.rados.rados.0.smithi093.stdout:3028: finishing write tid 1 to smithi093153278-313 2023-08-27T20:54:36.960 INFO:tasks.rados.rados.0.smithi093.stdout:3027: done (3 left) 2023-08-27T20:54:36.960 INFO:tasks.rados.rados.0.smithi093.stdout:3030: done (2 left) 2023-08-27T20:54:36.960 INFO:tasks.rados.rados.0.smithi093.stdout:3031: write oid 315 current snap is 0 2023-08-27T20:54:36.960 INFO:tasks.rados.rados.0.smithi093.stdout:3031: seq_num 1916 ranges {602212=698040,2002911=199301} 2023-08-27T20:54:36.967 INFO:tasks.rados.rados.0.smithi093.stdout:3031: writing smithi093153278-315 from 602212 to 1300252 tid 1 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3031: writing smithi093153278-315 from 2002911 to 2202212 tid 2 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3032: read oid 53 snap -1 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3032: expect (ObjNum 1844 snap 0 seq_num 1844) 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3033: write oid 182 current snap is 0 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3028: finishing write tid 2 to smithi093153278-313 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3028: finishing write tid 3 to smithi093153278-313 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3028: finishing write tid 4 to smithi093153278-313 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 313 v 251 (ObjNum 1914 snap 0 seq_num 1914) dirty exists 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3028: left oid 313 (ObjNum 1914 snap 0 seq_num 1914) 2023-08-27T20:54:36.973 INFO:tasks.rados.rados.0.smithi093.stdout:3033: seq_num 1917 ranges {619019=773289,2061393=157626} 2023-08-27T20:54:36.980 INFO:tasks.rados.rados.0.smithi093.stdout:3033: writing smithi093153278-182 from 619019 to 1392308 tid 1 2023-08-27T20:54:36.985 INFO:tasks.rados.rados.0.smithi093.stdout:3033: writing smithi093153278-182 from 2061393 to 2219019 tid 2 2023-08-27T20:54:36.985 INFO:tasks.rados.rados.0.smithi093.stdout:3028: done (4 left) 2023-08-27T20:54:36.985 INFO:tasks.rados.rados.0.smithi093.stdout:3034: read oid 57 snap -1 2023-08-27T20:54:36.985 INFO:tasks.rados.rados.0.smithi093.stdout:3034: expect (ObjNum 1539 snap 0 seq_num 1539) 2023-08-27T20:54:36.985 INFO:tasks.rados.rados.0.smithi093.stdout:3035: write oid 142 current snap is 0 2023-08-27T20:54:36.985 INFO:tasks.rados.rados.0.smithi093.stdout:3029: expect (ObjNum 1661 snap 0 seq_num 1661) 2023-08-27T20:54:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2761801715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2761801715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:37.103 INFO:tasks.rados.rados.0.smithi093.stdout:3031: finishing write tid 1 to smithi093153278-315 2023-08-27T20:54:37.103 INFO:tasks.rados.rados.0.smithi093.stdout:3031: finishing write tid 2 to smithi093153278-315 2023-08-27T20:54:37.103 INFO:tasks.rados.rados.0.smithi093.stdout:3031: finishing write tid 3 to smithi093153278-315 2023-08-27T20:54:37.103 INFO:tasks.rados.rados.0.smithi093.stdout:3031: finishing write tid 4 to smithi093153278-315 2023-08-27T20:54:37.104 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 413 (ObjNum 1916 snap 0 seq_num 1916) dirty exists 2023-08-27T20:54:37.104 INFO:tasks.rados.rados.0.smithi093.stdout:3031: left oid 315 (ObjNum 1916 snap 0 seq_num 1916) 2023-08-27T20:54:37.104 INFO:tasks.rados.rados.0.smithi093.stdout:3032: expect (ObjNum 1844 snap 0 seq_num 1844) 2023-08-27T20:54:37.263 INFO:tasks.rados.rados.0.smithi093.stdout:3033: finishing write tid 1 to smithi093153278-182 2023-08-27T20:54:37.263 INFO:tasks.rados.rados.0.smithi093.stdout:3033: finishing write tid 2 to smithi093153278-182 2023-08-27T20:54:37.263 INFO:tasks.rados.rados.0.smithi093.stdout:3033: finishing write tid 3 to smithi093153278-182 2023-08-27T20:54:37.263 INFO:tasks.rados.rados.0.smithi093.stdout:3035: seq_num 1918 ranges {635826=448538,1719875=515951} 2023-08-27T20:54:37.269 INFO:tasks.rados.rados.0.smithi093.stdout:3035: writing smithi093153278-142 from 635826 to 1084364 tid 1 2023-08-27T20:54:37.275 INFO:tasks.rados.rados.0.smithi093.stdout:3035: writing smithi093153278-142 from 1719875 to 2235826 tid 2 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:3029: done (5 left) 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:3031: done (4 left) 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:3032: done (3 left) 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:3036: delete oid 12 current snap is 0 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:3033: finishing write tid 4 to smithi093153278-182 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 182 v 412 (ObjNum 1917 snap 0 seq_num 1917) dirty exists 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:3033: left oid 182 (ObjNum 1917 snap 0 seq_num 1917) 2023-08-27T20:54:37.276 INFO:tasks.rados.rados.0.smithi093.stdout:3034: expect (ObjNum 1539 snap 0 seq_num 1539) 2023-08-27T20:54:37.393 INFO:tasks.rados.rados.0.smithi093.stdout:3035: finishing write tid 1 to smithi093153278-142 2023-08-27T20:54:37.393 INFO:tasks.rados.rados.0.smithi093.stdout:3035: finishing write tid 2 to smithi093153278-142 2023-08-27T20:54:37.393 INFO:tasks.rados.rados.0.smithi093.stdout:3033: done (3 left) 2023-08-27T20:54:37.393 INFO:tasks.rados.rados.0.smithi093.stdout:3034: done (2 left) 2023-08-27T20:54:37.393 INFO:tasks.rados.rados.0.smithi093.stdout:3036: done (1 left) 2023-08-27T20:54:37.393 INFO:tasks.rados.rados.0.smithi093.stdout:3037: write oid 27 current snap is 0 2023-08-27T20:54:37.394 INFO:tasks.rados.rados.0.smithi093.stdout:3035: finishing write tid 3 to smithi093153278-142 2023-08-27T20:54:37.394 INFO:tasks.rados.rados.0.smithi093.stdout:3035: finishing write tid 4 to smithi093153278-142 2023-08-27T20:54:37.394 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 142 v 386 (ObjNum 1918 snap 0 seq_num 1918) dirty exists 2023-08-27T20:54:37.394 INFO:tasks.rados.rados.0.smithi093.stdout:3035: left oid 142 (ObjNum 1918 snap 0 seq_num 1918) 2023-08-27T20:54:37.394 INFO:tasks.rados.rados.0.smithi093.stdout:3037: seq_num 1920 ranges {669440=599036,1720486=548954} 2023-08-27T20:54:37.399 INFO:tasks.rados.rados.0.smithi093.stdout:3037: writing smithi093153278-27 from 669440 to 1268476 tid 1 2023-08-27T20:54:37.405 INFO:tasks.rados.rados.0.smithi093.stdout:3037: writing smithi093153278-27 from 1720486 to 2269440 tid 2 2023-08-27T20:54:37.406 INFO:tasks.rados.rados.0.smithi093.stdout:3035: done (1 left) 2023-08-27T20:54:37.406 INFO:tasks.rados.rados.0.smithi093.stdout:3038: read oid 148 snap -1 2023-08-27T20:54:37.406 INFO:tasks.rados.rados.0.smithi093.stdout:3038: expect (ObjNum 1827 snap 0 seq_num 1827) 2023-08-27T20:54:37.406 INFO:tasks.rados.rados.0.smithi093.stdout:3039: read oid 393 snap -1 2023-08-27T20:54:37.406 INFO:tasks.rados.rados.0.smithi093.stdout:3039: expect (ObjNum 1186 snap 0 seq_num 1186) 2023-08-27T20:54:37.406 INFO:tasks.rados.rados.0.smithi093.stdout:3040: read oid 213 snap -1 2023-08-27T20:54:37.406 INFO:tasks.rados.rados.0.smithi093.stdout:3040: expect (ObjNum 981 snap 0 seq_num 981) 2023-08-27T20:54:37.407 INFO:tasks.rados.rados.0.smithi093.stdout:3041: read oid 453 snap -1 2023-08-27T20:54:37.407 INFO:tasks.rados.rados.0.smithi093.stdout:3041: expect deleted 2023-08-27T20:54:37.407 INFO:tasks.rados.rados.0.smithi093.stdout:3042: read oid 447 snap -1 2023-08-27T20:54:37.407 INFO:tasks.rados.rados.0.smithi093.stdout:3042: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:54:37.407 INFO:tasks.rados.rados.0.smithi093.stdout:3043: write oid 29 current snap is 0 2023-08-27T20:54:37.407 INFO:tasks.rados.rados.0.smithi093.stdout:3037: finishing write tid 1 to smithi093153278-27 2023-08-27T20:54:37.408 INFO:tasks.rados.rados.0.smithi093.stdout:3043: seq_num 1921 ranges {686247=674285,1778968=474011} 2023-08-27T20:54:37.413 INFO:tasks.rados.rados.0.smithi093.stdout:3043: writing smithi093153278-29 from 686247 to 1360532 tid 1 2023-08-27T20:54:37.419 INFO:tasks.rados.rados.0.smithi093.stdout:3043: writing smithi093153278-29 from 1778968 to 2252979 tid 2 2023-08-27T20:54:37.419 INFO:tasks.rados.rados.0.smithi093.stdout:3044: write oid 282 current snap is 0 2023-08-27T20:54:37.419 INFO:tasks.rados.rados.0.smithi093.stdout:3037: finishing write tid 2 to smithi093153278-27 2023-08-27T20:54:37.419 INFO:tasks.rados.rados.0.smithi093.stdout:3037: finishing write tid 3 to smithi093153278-27 2023-08-27T20:54:37.419 INFO:tasks.rados.rados.0.smithi093.stdout:3044: seq_num 1922 ranges {703054=749534,2237450=65604} 2023-08-27T20:54:37.427 INFO:tasks.rados.rados.0.smithi093.stdout:3044: writing smithi093153278-282 from 703054 to 1452588 tid 1 2023-08-27T20:54:37.432 INFO:tasks.rados.rados.0.smithi093.stdout:3044: writing smithi093153278-282 from 2237450 to 2303054 tid 2 2023-08-27T20:54:37.432 INFO:tasks.rados.rados.0.smithi093.stdout:3041: done (7 left) 2023-08-27T20:54:37.432 INFO:tasks.rados.rados.0.smithi093.stdout:3045: delete oid 381 current snap is 0 2023-08-27T20:54:37.432 INFO:tasks.rados.rados.0.smithi093.stdout:3037: finishing write tid 4 to smithi093153278-27 2023-08-27T20:54:37.433 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 27 v 342 (ObjNum 1920 snap 0 seq_num 1920) dirty exists 2023-08-27T20:54:37.433 INFO:tasks.rados.rados.0.smithi093.stdout:3037: left oid 27 (ObjNum 1920 snap 0 seq_num 1920) 2023-08-27T20:54:37.433 INFO:tasks.rados.rados.0.smithi093.stdout:3040: expect (ObjNum 981 snap 0 seq_num 981) 2023-08-27T20:54:37.585 INFO:tasks.rados.rados.0.smithi093.stdout:3042: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:54:37.702 INFO:tasks.rados.rados.0.smithi093.stdout:3043: finishing write tid 1 to smithi093153278-29 2023-08-27T20:54:37.702 INFO:tasks.rados.rados.0.smithi093.stdout:3038: expect (ObjNum 1827 snap 0 seq_num 1827) 2023-08-27T20:54:37.825 INFO:tasks.rados.rados.0.smithi093.stdout:3043: finishing write tid 2 to smithi093153278-29 2023-08-27T20:54:37.825 INFO:tasks.rados.rados.0.smithi093.stdout:3039: expect (ObjNum 1186 snap 0 seq_num 1186) 2023-08-27T20:54:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:37 smithi136 ceph-mon[113531]: pgmap v32: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 89 MiB/s rd, 9.3 MiB/s wr, 109 op/s 2023-08-27T20:54:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:37 smithi093 ceph-mon[111335]: pgmap v32: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 89 MiB/s rd, 9.3 MiB/s wr, 109 op/s 2023-08-27T20:54:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:37 smithi093 ceph-mon[118902]: pgmap v32: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 89 MiB/s rd, 9.3 MiB/s wr, 109 op/s 2023-08-27T20:54:38.040 INFO:tasks.rados.rados.0.smithi093.stdout:3043: finishing write tid 3 to smithi093153278-29 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3043: finishing write tid 4 to smithi093153278-29 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 29 v 423 (ObjNum 1921 snap 0 seq_num 1921) dirty exists 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3043: left oid 29 (ObjNum 1921 snap 0 seq_num 1921) 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3044: finishing write tid 1 to smithi093153278-282 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3044: finishing write tid 2 to smithi093153278-282 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3037: done (7 left) 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3038: done (6 left) 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3039: done (5 left) 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3040: done (4 left) 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3042: done (3 left) 2023-08-27T20:54:38.041 INFO:tasks.rados.rados.0.smithi093.stdout:3043: done (2 left) 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:3045: done (1 left) 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:3046: read oid 151 snap -1 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:3046: expect (ObjNum 1783 snap 0 seq_num 1783) 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:3047: delete oid 200 current snap is 0 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:3044: finishing write tid 3 to smithi093153278-282 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:3044: finishing write tid 4 to smithi093153278-282 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 282 v 254 (ObjNum 1922 snap 0 seq_num 1922) dirty exists 2023-08-27T20:54:38.042 INFO:tasks.rados.rados.0.smithi093.stdout:3044: left oid 282 (ObjNum 1922 snap 0 seq_num 1922) 2023-08-27T20:54:38.043 INFO:tasks.rados.rados.0.smithi093.stdout:3044: done (2 left) 2023-08-27T20:54:38.043 INFO:tasks.rados.rados.0.smithi093.stdout:3047: done (1 left) 2023-08-27T20:54:38.043 INFO:tasks.rados.rados.0.smithi093.stdout:3048: write oid 6 current snap is 0 2023-08-27T20:54:38.043 INFO:tasks.rados.rados.0.smithi093.stdout:3048: seq_num 1925 ranges {753475=691634,2012896=340579} 2023-08-27T20:54:38.050 INFO:tasks.rados.rados.0.smithi093.stdout:3048: writing smithi093153278-6 from 753475 to 1445109 tid 1 2023-08-27T20:54:38.055 INFO:tasks.rados.rados.0.smithi093.stdout:3048: writing smithi093153278-6 from 2012896 to 2353475 tid 2 2023-08-27T20:54:38.055 INFO:tasks.rados.rados.0.smithi093.stdout:3049: write oid 385 current snap is 0 2023-08-27T20:54:38.055 INFO:tasks.rados.rados.0.smithi093.stdout:3049: seq_num 1926 ranges {770282=766883,2071378=298904} 2023-08-27T20:54:38.063 INFO:tasks.rados.rados.0.smithi093.stdout:3049: writing smithi093153278-385 from 770282 to 1537165 tid 1 2023-08-27T20:54:38.068 INFO:tasks.rados.rados.0.smithi093.stdout:3049: writing smithi093153278-385 from 2071378 to 2370282 tid 2 2023-08-27T20:54:38.068 INFO:tasks.rados.rados.0.smithi093.stdout:3050: write oid 481 current snap is 0 2023-08-27T20:54:38.068 INFO:tasks.rados.rados.0.smithi093.stdout:3050: seq_num 1927 ranges {787089=442132,1729860=657229} 2023-08-27T20:54:38.074 INFO:tasks.rados.rados.0.smithi093.stdout:3050: writing smithi093153278-481 from 787089 to 1229221 tid 1 2023-08-27T20:54:38.081 INFO:tasks.rados.rados.0.smithi093.stdout:3050: writing smithi093153278-481 from 1729860 to 2387089 tid 2 2023-08-27T20:54:38.081 INFO:tasks.rados.rados.0.smithi093.stdout:3051: delete oid 320 current snap is 0 2023-08-27T20:54:38.081 INFO:tasks.rados.rados.0.smithi093.stdout:3046: expect (ObjNum 1783 snap 0 seq_num 1783) 2023-08-27T20:54:38.379 INFO:tasks.rados.rados.0.smithi093.stdout:3048: finishing write tid 1 to smithi093153278-6 2023-08-27T20:54:38.379 INFO:tasks.rados.rados.0.smithi093.stdout:3046: done (4 left) 2023-08-27T20:54:38.379 INFO:tasks.rados.rados.0.smithi093.stdout:3051: done (3 left) 2023-08-27T20:54:38.379 INFO:tasks.rados.rados.0.smithi093.stdout:3052: read oid 189 snap -1 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3052: expect (ObjNum 1455 snap 0 seq_num 1455) 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3053: read oid 316 snap -1 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3053: expect (ObjNum 1772 snap 0 seq_num 1772) 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3054: read oid 451 snap -1 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3054: expect deleted 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3055: write oid 297 current snap is 0 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3048: finishing write tid 2 to smithi093153278-6 2023-08-27T20:54:38.380 INFO:tasks.rados.rados.0.smithi093.stdout:3055: seq_num 1929 ranges {420703=592630,1730471=690232} 2023-08-27T20:54:38.384 INFO:tasks.rados.rados.0.smithi093.stdout:3055: writing smithi093153278-297 from 420703 to 1013333 tid 1 2023-08-27T20:54:38.393 INFO:tasks.rados.rados.0.smithi093.stdout:3055: writing smithi093153278-297 from 1730471 to 2420703 tid 2 2023-08-27T20:54:38.393 INFO:tasks.rados.rados.0.smithi093.stdout:3056: delete oid 490 current snap is 0 2023-08-27T20:54:38.393 INFO:tasks.rados.rados.0.smithi093.stdout:3048: finishing write tid 3 to smithi093153278-6 2023-08-27T20:54:38.393 INFO:tasks.rados.rados.0.smithi093.stdout:3048: finishing write tid 4 to smithi093153278-6 2023-08-27T20:54:38.393 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 6 v 359 (ObjNum 1925 snap 0 seq_num 1925) dirty exists 2023-08-27T20:54:38.393 INFO:tasks.rados.rados.0.smithi093.stdout:3048: left oid 6 (ObjNum 1925 snap 0 seq_num 1925) 2023-08-27T20:54:38.393 INFO:tasks.rados.rados.0.smithi093.stdout:3049: finishing write tid 1 to smithi093153278-385 2023-08-27T20:54:38.394 INFO:tasks.rados.rados.0.smithi093.stdout:3049: finishing write tid 2 to smithi093153278-385 2023-08-27T20:54:38.394 INFO:tasks.rados.rados.0.smithi093.stdout:3049: finishing write tid 3 to smithi093153278-385 2023-08-27T20:54:38.394 INFO:tasks.rados.rados.0.smithi093.stdout:3049: finishing write tid 4 to smithi093153278-385 2023-08-27T20:54:38.394 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 385 v 416 (ObjNum 1926 snap 0 seq_num 1926) dirty exists 2023-08-27T20:54:38.394 INFO:tasks.rados.rados.0.smithi093.stdout:3049: left oid 385 (ObjNum 1926 snap 0 seq_num 1926) 2023-08-27T20:54:38.395 INFO:tasks.rados.rados.0.smithi093.stdout:3050: finishing write tid 1 to smithi093153278-481 2023-08-27T20:54:38.395 INFO:tasks.rados.rados.0.smithi093.stdout:3050: finishing write tid 2 to smithi093153278-481 2023-08-27T20:54:38.395 INFO:tasks.rados.rados.0.smithi093.stdout:3050: finishing write tid 3 to smithi093153278-481 2023-08-27T20:54:38.395 INFO:tasks.rados.rados.0.smithi093.stdout:3050: finishing write tid 4 to smithi093153278-481 2023-08-27T20:54:38.395 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 481 v 349 (ObjNum 1927 snap 0 seq_num 1927) dirty exists 2023-08-27T20:54:38.395 INFO:tasks.rados.rados.0.smithi093.stdout:3050: left oid 481 (ObjNum 1927 snap 0 seq_num 1927) 2023-08-27T20:54:38.395 INFO:tasks.rados.rados.0.smithi093.stdout:3053: expect (ObjNum 1772 snap 0 seq_num 1772) 2023-08-27T20:54:38.569 INFO:tasks.rados.rados.0.smithi093.stdout:3052: expect (ObjNum 1455 snap 0 seq_num 1455) 2023-08-27T20:54:38.702 INFO:tasks.rados.rados.0.smithi093.stdout:3055: finishing write tid 1 to smithi093153278-297 2023-08-27T20:54:38.702 INFO:tasks.rados.rados.0.smithi093.stdout:3055: finishing write tid 2 to smithi093153278-297 2023-08-27T20:54:38.702 INFO:tasks.rados.rados.0.smithi093.stdout:3048: done (7 left) 2023-08-27T20:54:38.702 INFO:tasks.rados.rados.0.smithi093.stdout:3049: done (6 left) 2023-08-27T20:54:38.702 INFO:tasks.rados.rados.0.smithi093.stdout:3050: done (5 left) 2023-08-27T20:54:38.702 INFO:tasks.rados.rados.0.smithi093.stdout:3052: done (4 left) 2023-08-27T20:54:38.702 INFO:tasks.rados.rados.0.smithi093.stdout:3053: done (3 left) 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3054: done (2 left) 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3056: done (1 left) 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3057: read oid 214 snap -1 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3057: expect (ObjNum 1517 snap 0 seq_num 1517) 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3058: delete oid 137 current snap is 0 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3055: finishing write tid 3 to smithi093153278-297 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3055: finishing write tid 4 to smithi093153278-297 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 297 v 345 (ObjNum 1929 snap 0 seq_num 1929) dirty exists 2023-08-27T20:54:38.703 INFO:tasks.rados.rados.0.smithi093.stdout:3055: left oid 297 (ObjNum 1929 snap 0 seq_num 1929) 2023-08-27T20:54:38.704 INFO:tasks.rados.rados.0.smithi093.stdout:3055: done (2 left) 2023-08-27T20:54:38.704 INFO:tasks.rados.rados.0.smithi093.stdout:3058: done (1 left) 2023-08-27T20:54:38.704 INFO:tasks.rados.rados.0.smithi093.stdout:3059: write oid 5 current snap is 0 2023-08-27T20:54:38.704 INFO:tasks.rados.rados.0.smithi093.stdout:3059: seq_num 1932 ranges {471124=534730,1505917=636014} 2023-08-27T20:54:38.708 INFO:tasks.rados.rados.0.smithi093.stdout:3059: writing smithi093153278-5 from 471124 to 1005854 tid 1 2023-08-27T20:54:38.715 INFO:tasks.rados.rados.0.smithi093.stdout:3059: writing smithi093153278-5 from 1505917 to 2141931 tid 2 2023-08-27T20:54:38.715 INFO:tasks.rados.rados.0.smithi093.stdout:3060: write oid 286 current snap is 0 2023-08-27T20:54:38.715 INFO:tasks.rados.rados.0.smithi093.stdout:3060: seq_num 1933 ranges {487931=609979,1564399=779672} 2023-08-27T20:54:38.721 INFO:tasks.rados.rados.0.smithi093.stdout:3060: writing smithi093153278-286 from 487931 to 1097910 tid 1 2023-08-27T20:54:38.728 INFO:tasks.rados.rados.0.smithi093.stdout:3060: writing smithi093153278-286 from 1564399 to 2344071 tid 2 2023-08-27T20:54:38.728 INFO:tasks.rados.rados.0.smithi093.stdout:3061: write oid 399 current snap is 0 2023-08-27T20:54:38.728 INFO:tasks.rados.rados.0.smithi093.stdout:3061: seq_num 1934 ranges {504738=685228,1622881=639683} 2023-08-27T20:54:38.734 INFO:tasks.rados.rados.0.smithi093.stdout:3061: writing smithi093153278-399 from 504738 to 1189966 tid 1 2023-08-27T20:54:38.740 INFO:tasks.rados.rados.0.smithi093.stdout:3061: writing smithi093153278-399 from 1622881 to 2262564 tid 2 2023-08-27T20:54:38.741 INFO:tasks.rados.rados.0.smithi093.stdout:3062: read oid 491 snap -1 2023-08-27T20:54:38.741 INFO:tasks.rados.rados.0.smithi093.stdout:3062: expect (ObjNum 1110 snap 0 seq_num 1110) 2023-08-27T20:54:38.741 INFO:tasks.rados.rados.0.smithi093.stdout:3063: write oid 482 current snap is 0 2023-08-27T20:54:38.741 INFO:tasks.rados.rados.0.smithi093.stdout:3057: expect (ObjNum 1517 snap 0 seq_num 1517) 2023-08-27T20:54:38.884 INFO:tasks.rados.rados.0.smithi093.stdout:3059: finishing write tid 1 to smithi093153278-5 2023-08-27T20:54:38.884 INFO:tasks.rados.rados.0.smithi093.stdout:3063: seq_num 1935 ranges {521545=760477,2081363=440182} 2023-08-27T20:54:38.891 INFO:tasks.rados.rados.0.smithi093.stdout:3063: writing smithi093153278-482 from 521545 to 1282022 tid 1 2023-08-27T20:54:38.898 INFO:tasks.rados.rados.0.smithi093.stdout:3063: writing smithi093153278-482 from 2081363 to 2521545 tid 2 2023-08-27T20:54:38.898 INFO:tasks.rados.rados.0.smithi093.stdout:3057: done (5 left) 2023-08-27T20:54:38.898 INFO:tasks.rados.rados.0.smithi093.stdout:3064: write oid 236 current snap is 0 2023-08-27T20:54:38.898 INFO:tasks.rados.rados.0.smithi093.stdout:3059: finishing write tid 2 to smithi093153278-5 2023-08-27T20:54:38.899 INFO:tasks.rados.rados.0.smithi093.stdout:3059: finishing write tid 3 to smithi093153278-5 2023-08-27T20:54:38.899 INFO:tasks.rados.rados.0.smithi093.stdout:3064: seq_num 1936 ranges {538352=435726,1623492=526999} 2023-08-27T20:54:38.903 INFO:tasks.rados.rados.0.smithi093.stdout:3064: writing smithi093153278-236 from 538352 to 974078 tid 1 2023-08-27T20:54:38.910 INFO:tasks.rados.rados.0.smithi093.stdout:3064: writing smithi093153278-236 from 1623492 to 2150491 tid 2 2023-08-27T20:54:38.910 INFO:tasks.rados.rados.0.smithi093.stdout:3065: write oid 264 current snap is 0 2023-08-27T20:54:38.910 INFO:tasks.rados.rados.0.smithi093.stdout:3059: finishing write tid 4 to smithi093153278-5 2023-08-27T20:54:38.910 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 5 v 362 (ObjNum 1932 snap 0 seq_num 1932) dirty exists 2023-08-27T20:54:38.910 INFO:tasks.rados.rados.0.smithi093.stdout:3059: left oid 5 (ObjNum 1932 snap 0 seq_num 1932) 2023-08-27T20:54:38.911 INFO:tasks.rados.rados.0.smithi093.stdout:3060: finishing write tid 1 to smithi093153278-286 2023-08-27T20:54:38.911 INFO:tasks.rados.rados.0.smithi093.stdout:3065: seq_num 1937 ranges {555159=510975,1681974=787010} 2023-08-27T20:54:38.916 INFO:tasks.rados.rados.0.smithi093.stdout:3065: writing smithi093153278-264 from 555159 to 1066134 tid 1 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3065: writing smithi093153278-264 from 1681974 to 2468984 tid 2 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3059: done (6 left) 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3066: read oid 329 snap -1 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3066: expect (ObjNum 1811 snap 0 seq_num 1811) 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3067: delete oid 385 current snap is 0 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3060: finishing write tid 2 to smithi093153278-286 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3060: finishing write tid 3 to smithi093153278-286 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:3060: finishing write tid 4 to smithi093153278-286 2023-08-27T20:54:38.925 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 286 v 415 (ObjNum 1933 snap 0 seq_num 1933) dirty exists 2023-08-27T20:54:38.926 INFO:tasks.rados.rados.0.smithi093.stdout:3060: left oid 286 (ObjNum 1933 snap 0 seq_num 1933) 2023-08-27T20:54:38.926 INFO:tasks.rados.rados.0.smithi093.stdout:3061: finishing write tid 1 to smithi093153278-399 2023-08-27T20:54:38.926 INFO:tasks.rados.rados.0.smithi093.stdout:3062: expect (ObjNum 1110 snap 0 seq_num 1110) 2023-08-27T20:54:39.090 INFO:tasks.rados.rados.0.smithi093.stdout:3061: finishing write tid 2 to smithi093153278-399 2023-08-27T20:54:39.090 INFO:tasks.rados.rados.0.smithi093.stdout:3061: finishing write tid 3 to smithi093153278-399 2023-08-27T20:54:39.090 INFO:tasks.rados.rados.0.smithi093.stdout:3061: finishing write tid 4 to smithi093153278-399 2023-08-27T20:54:39.091 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 399 v 348 (ObjNum 1934 snap 0 seq_num 1934) dirty exists 2023-08-27T20:54:39.091 INFO:tasks.rados.rados.0.smithi093.stdout:3061: left oid 399 (ObjNum 1934 snap 0 seq_num 1934) 2023-08-27T20:54:39.091 INFO:tasks.rados.rados.0.smithi093.stdout:3060: done (7 left) 2023-08-27T20:54:39.091 INFO:tasks.rados.rados.0.smithi093.stdout:3061: done (6 left) 2023-08-27T20:54:39.091 INFO:tasks.rados.rados.0.smithi093.stdout:3062: done (5 left) 2023-08-27T20:54:39.091 INFO:tasks.rados.rados.0.smithi093.stdout:3067: done (4 left) 2023-08-27T20:54:39.092 INFO:tasks.rados.rados.0.smithi093.stdout:3068: read oid 255 snap -1 2023-08-27T20:54:39.092 INFO:tasks.rados.rados.0.smithi093.stdout:3068: expect (ObjNum 1658 snap 0 seq_num 1658) 2023-08-27T20:54:39.092 INFO:tasks.rados.rados.0.smithi093.stdout:3069: write oid 438 current snap is 0 2023-08-27T20:54:39.092 INFO:tasks.rados.rados.0.smithi093.stdout:3063: finishing write tid 1 to smithi093153278-482 2023-08-27T20:54:39.092 INFO:tasks.rados.rados.0.smithi093.stdout:3063: finishing write tid 2 to smithi093153278-482 2023-08-27T20:54:39.092 INFO:tasks.rados.rados.0.smithi093.stdout:3063: finishing write tid 3 to smithi093153278-482 2023-08-27T20:54:39.093 INFO:tasks.rados.rados.0.smithi093.stdout:3063: finishing write tid 4 to smithi093153278-482 2023-08-27T20:54:39.093 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 482 v 355 (ObjNum 1935 snap 0 seq_num 1935) dirty exists 2023-08-27T20:54:39.093 INFO:tasks.rados.rados.0.smithi093.stdout:3063: left oid 482 (ObjNum 1935 snap 0 seq_num 1935) 2023-08-27T20:54:39.093 INFO:tasks.rados.rados.0.smithi093.stdout:3064: finishing write tid 1 to smithi093153278-236 2023-08-27T20:54:39.093 INFO:tasks.rados.rados.0.smithi093.stdout:3064: finishing write tid 2 to smithi093153278-236 2023-08-27T20:54:39.094 INFO:tasks.rados.rados.0.smithi093.stdout:3069: seq_num 1939 ranges {588773=777826,1915291=673482} 2023-08-27T20:54:39.097 INFO:tasks.rados.rados.0.smithi093.stdout:3069: writing smithi093153278-438 from 588773 to 1366599 tid 1 2023-08-27T20:54:39.105 INFO:tasks.rados.rados.0.smithi093.stdout:3069: writing smithi093153278-438 from 1915291 to 2588773 tid 2 2023-08-27T20:54:39.106 INFO:tasks.rados.rados.0.smithi093.stdout:3063: done (5 left) 2023-08-27T20:54:39.106 INFO:tasks.rados.rados.0.smithi093.stdout:3070: read oid 106 snap -1 2023-08-27T20:54:39.106 INFO:tasks.rados.rados.0.smithi093.stdout:3070: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:54:39.106 INFO:tasks.rados.rados.0.smithi093.stdout:3071: delete oid 99 current snap is 0 2023-08-27T20:54:39.107 INFO:tasks.rados.rados.0.smithi093.stdout:3064: finishing write tid 3 to smithi093153278-236 2023-08-27T20:54:39.107 INFO:tasks.rados.rados.0.smithi093.stdout:3064: finishing write tid 4 to smithi093153278-236 2023-08-27T20:54:39.107 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 236 v 352 (ObjNum 1936 snap 0 seq_num 1936) dirty exists 2023-08-27T20:54:39.107 INFO:tasks.rados.rados.0.smithi093.stdout:3064: left oid 236 (ObjNum 1936 snap 0 seq_num 1936) 2023-08-27T20:54:39.107 INFO:tasks.rados.rados.0.smithi093.stdout:3065: finishing write tid 1 to smithi093153278-264 2023-08-27T20:54:39.108 INFO:tasks.rados.rados.0.smithi093.stdout:3065: finishing write tid 2 to smithi093153278-264 2023-08-27T20:54:39.108 INFO:tasks.rados.rados.0.smithi093.stdout:3065: finishing write tid 3 to smithi093153278-264 2023-08-27T20:54:39.108 INFO:tasks.rados.rados.0.smithi093.stdout:3065: finishing write tid 4 to smithi093153278-264 2023-08-27T20:54:39.108 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 264 v 353 (ObjNum 1937 snap 0 seq_num 1937) dirty exists 2023-08-27T20:54:39.108 INFO:tasks.rados.rados.0.smithi093.stdout:3065: left oid 264 (ObjNum 1937 snap 0 seq_num 1937) 2023-08-27T20:54:39.109 INFO:tasks.rados.rados.0.smithi093.stdout:3066: expect (ObjNum 1811 snap 0 seq_num 1811) 2023-08-27T20:54:39.214 INFO:tasks.rados.rados.0.smithi093.stdout:3068: expect (ObjNum 1658 snap 0 seq_num 1658) 2023-08-27T20:54:39.433 INFO:tasks.rados.rados.0.smithi093.stdout:3069: finishing write tid 1 to smithi093153278-438 2023-08-27T20:54:40.105 INFO:tasks.rados.rados.0.smithi093.stdout:3069: finishing write tid 2 to smithi093153278-438 2023-08-27T20:54:40.105 INFO:tasks.rados.rados.0.smithi093.stdout:3069: finishing write tid 3 to smithi093153278-438 2023-08-27T20:54:40.105 INFO:tasks.rados.rados.0.smithi093.stdout:3069: finishing write tid 4 to smithi093153278-438 2023-08-27T20:54:40.105 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 438 v 335 (ObjNum 1939 snap 0 seq_num 1939) dirty exists 2023-08-27T20:54:40.105 INFO:tasks.rados.rados.0.smithi093.stdout:3069: left oid 438 (ObjNum 1939 snap 0 seq_num 1939) 2023-08-27T20:54:40.105 INFO:tasks.rados.rados.0.smithi093.stdout:3070: expect (ObjNum 540 snap 0 seq_num 540) 2023-08-27T20:54:40.106 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:39 smithi136 ceph-mon[113531]: pgmap v33: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 6.6 MiB/s wr, 71 op/s 2023-08-27T20:54:40.106 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:40.106 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/332373636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:40.111 INFO:tasks.rados.rados.0.smithi093.stdout:3064: done (6 left) 2023-08-27T20:54:40.111 INFO:tasks.rados.rados.0.smithi093.stdout:3065: done (5 left) 2023-08-27T20:54:40.111 INFO:tasks.rados.rados.0.smithi093.stdout:3066: done (4 left) 2023-08-27T20:54:40.111 INFO:tasks.rados.rados.0.smithi093.stdout:3068: done (3 left) 2023-08-27T20:54:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:3069: done (2 left) 2023-08-27T20:54:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:3070: done (1 left) 2023-08-27T20:54:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:3071: done (0 left) 2023-08-27T20:54:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:3072: read oid 338 snap -1 2023-08-27T20:54:40.112 INFO:tasks.rados.rados.0.smithi093.stdout:3072: expect (ObjNum 1472 snap 0 seq_num 1472) 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3073: read oid 68 snap -1 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3073: expect (ObjNum 1763 snap 0 seq_num 1763) 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3074: write oid 160 current snap is 0 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3074: seq_num 1941 ranges {622387=528324,1915902=706485} 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3074: writing smithi093153278-160 from 622387 to 1150711 tid 1 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3074: writing smithi093153278-160 from 1915902 to 2622387 tid 2 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3075: read oid 234 snap -1 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3075: expect (ObjNum 1202 snap 0 seq_num 1202) 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3076: write oid 146 current snap is 0 2023-08-27T20:54:40.113 INFO:tasks.rados.rados.0.smithi093.stdout:3072: expect (ObjNum 1472 snap 0 seq_num 1472) 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3073: expect (ObjNum 1763 snap 0 seq_num 1763) 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3074: finishing write tid 1 to smithi093153278-160 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3076: seq_num 1942 ranges {639194=603573,1974384=538006} 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3076: writing smithi093153278-146 from 639194 to 1242767 tid 1 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3076: writing smithi093153278-146 from 1974384 to 2512390 tid 2 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3072: done (4 left) 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3073: done (3 left) 2023-08-27T20:54:40.114 INFO:tasks.rados.rados.0.smithi093.stdout:3077: write oid 338 current snap is 0 2023-08-27T20:54:40.115 INFO:tasks.rados.rados.0.smithi093.stdout:3074: finishing write tid 2 to smithi093153278-160 2023-08-27T20:54:40.115 INFO:tasks.rados.rados.0.smithi093.stdout:3077: seq_num 1943 ranges {656001=678822,2032866=623135} 2023-08-27T20:54:40.115 INFO:tasks.rados.rados.0.smithi093.stdout:3077: writing smithi093153278-338 from 656001 to 1334823 tid 1 2023-08-27T20:54:40.115 INFO:tasks.rados.rados.0.smithi093.stdout:3077: writing smithi093153278-338 from 2032866 to 2656001 tid 2 2023-08-27T20:54:40.116 INFO:tasks.rados.rados.0.smithi093.stdout:3078: read oid 237 snap -1 2023-08-27T20:54:40.116 INFO:tasks.rados.rados.0.smithi093.stdout:3078: expect (ObjNum 1882 snap 0 seq_num 1882) 2023-08-27T20:54:40.116 INFO:tasks.rados.rados.0.smithi093.stdout:3079: read oid 303 snap -1 2023-08-27T20:54:40.116 INFO:tasks.rados.rados.0.smithi093.stdout:3079: expect deleted 2023-08-27T20:54:40.116 INFO:tasks.rados.rados.0.smithi093.stdout:3080: write oid 179 current snap is 0 2023-08-27T20:54:40.116 INFO:tasks.rados.rados.0.smithi093.stdout:3074: finishing write tid 3 to smithi093153278-160 2023-08-27T20:54:40.117 INFO:tasks.rados.rados.0.smithi093.stdout:3074: finishing write tid 4 to smithi093153278-160 2023-08-27T20:54:40.121 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 160 v 479 (ObjNum 1941 snap 0 seq_num 1941) dirty exists 2023-08-27T20:54:40.121 INFO:tasks.rados.rados.0.smithi093.stdout:3074: left oid 160 (ObjNum 1941 snap 0 seq_num 1941) 2023-08-27T20:54:40.121 INFO:tasks.rados.rados.0.smithi093.stdout:3075: expect (ObjNum 1202 snap 0 seq_num 1202) 2023-08-27T20:54:40.122 INFO:tasks.rados.rados.0.smithi093.stdout:3076: finishing write tid 1 to smithi093153278-146 2023-08-27T20:54:40.122 INFO:tasks.rados.rados.0.smithi093.stdout:3076: finishing write tid 2 to smithi093153278-146 2023-08-27T20:54:40.122 INFO:tasks.rados.rados.0.smithi093.stdout:3076: finishing write tid 3 to smithi093153278-146 2023-08-27T20:54:40.122 INFO:tasks.rados.rados.0.smithi093.stdout:3076: finishing write tid 4 to smithi093153278-146 2023-08-27T20:54:40.122 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 146 v 365 (ObjNum 1942 snap 0 seq_num 1942) dirty exists 2023-08-27T20:54:40.123 INFO:tasks.rados.rados.0.smithi093.stdout:3076: left oid 146 (ObjNum 1942 snap 0 seq_num 1942) 2023-08-27T20:54:40.123 INFO:tasks.rados.rados.0.smithi093.stdout:3077: finishing write tid 1 to smithi093153278-338 2023-08-27T20:54:40.123 INFO:tasks.rados.rados.0.smithi093.stdout:3080: seq_num 1944 ranges {672808=754071,1974995=541675} 2023-08-27T20:54:40.123 INFO:tasks.rados.rados.0.smithi093.stdout:3080: writing smithi093153278-179 from 672808 to 1426879 tid 1 2023-08-27T20:54:40.123 INFO:tasks.rados.rados.0.smithi093.stdout:3080: writing smithi093153278-179 from 1974995 to 2516670 tid 2 2023-08-27T20:54:40.123 INFO:tasks.rados.rados.0.smithi093.stdout:3074: done (6 left) 2023-08-27T20:54:40.124 INFO:tasks.rados.rados.0.smithi093.stdout:3075: done (5 left) 2023-08-27T20:54:40.124 INFO:tasks.rados.rados.0.smithi093.stdout:3076: done (4 left) 2023-08-27T20:54:40.124 INFO:tasks.rados.rados.0.smithi093.stdout:3081: write oid 264 current snap is 0 2023-08-27T20:54:40.124 INFO:tasks.rados.rados.0.smithi093.stdout:3077: finishing write tid 2 to smithi093153278-338 2023-08-27T20:54:40.124 INFO:tasks.rados.rados.0.smithi093.stdout:3077: finishing write tid 3 to smithi093153278-338 2023-08-27T20:54:40.124 INFO:tasks.rados.rados.0.smithi093.stdout:3077: finishing write tid 4 to smithi093153278-338 2023-08-27T20:54:40.125 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 338 v 383 (ObjNum 1943 snap 0 seq_num 1943) dirty exists 2023-08-27T20:54:40.125 INFO:tasks.rados.rados.0.smithi093.stdout:3077: left oid 338 (ObjNum 1943 snap 0 seq_num 1943) 2023-08-27T20:54:40.125 INFO:tasks.rados.rados.0.smithi093.stdout:3078: expect (ObjNum 1882 snap 0 seq_num 1882) 2023-08-27T20:54:40.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:39 smithi093 ceph-mon[111335]: pgmap v33: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 6.6 MiB/s wr, 71 op/s 2023-08-27T20:54:40.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:40.126 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/332373636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:40.126 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:39 smithi093 ceph-mon[118902]: pgmap v33: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 6.6 MiB/s wr, 71 op/s 2023-08-27T20:54:40.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:40.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/332373636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:3080: finishing write tid 1 to smithi093153278-179 2023-08-27T20:54:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:3080: finishing write tid 2 to smithi093153278-179 2023-08-27T20:54:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:3080: finishing write tid 3 to smithi093153278-179 2023-08-27T20:54:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:3080: finishing write tid 4 to smithi093153278-179 2023-08-27T20:54:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 179 v 386 (ObjNum 1944 snap 0 seq_num 1944) dirty exists 2023-08-27T20:54:40.128 INFO:tasks.rados.rados.0.smithi093.stdout:3080: left oid 179 (ObjNum 1944 snap 0 seq_num 1944) 2023-08-27T20:54:40.129 INFO:tasks.rados.rados.0.smithi093.stdout:3081: seq_num 1945 ranges {689615=429320,1633477=401686,2445721=243894} 2023-08-27T20:54:40.129 INFO:tasks.rados.rados.0.smithi093.stdout:3081: writing smithi093153278-264 from 689615 to 1118935 tid 1 2023-08-27T20:54:40.132 INFO:tasks.rados.rados.0.smithi093.stdout:3081: writing smithi093153278-264 from 1633477 to 2035163 tid 2 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3081: writing smithi093153278-264 from 2445721 to 2689615 tid 3 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3077: done (4 left) 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3078: done (3 left) 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3079: done (2 left) 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3080: done (1 left) 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3082: write oid 336 current snap is 0 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3081: finishing write tid 1 to smithi093153278-264 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3081: finishing write tid 2 to smithi093153278-264 2023-08-27T20:54:40.136 INFO:tasks.rados.rados.0.smithi093.stdout:3082: seq_num 1946 ranges {706422=504569,1691959=545344} 2023-08-27T20:54:40.142 INFO:tasks.rados.rados.0.smithi093.stdout:3082: writing smithi093153278-336 from 706422 to 1210991 tid 1 2023-08-27T20:54:40.148 INFO:tasks.rados.rados.0.smithi093.stdout:3082: writing smithi093153278-336 from 1691959 to 2237303 tid 2 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3083: read oid 414 snap -1 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3083: expect (ObjNum 1871 snap 0 seq_num 1871) 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3084: read oid 487 snap -1 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3084: expect (ObjNum 1594 snap 0 seq_num 1594) 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3085: read oid 203 snap -1 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3085: expect (ObjNum 1883 snap 0 seq_num 1883) 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3086: write oid 302 current snap is 0 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3081: finishing write tid 3 to smithi093153278-264 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3081: finishing write tid 4 to smithi093153278-264 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:3081: finishing write tid 5 to smithi093153278-264 2023-08-27T20:54:40.149 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 264 v 357 (ObjNum 1945 snap 0 seq_num 1945) dirty exists 2023-08-27T20:54:40.150 INFO:tasks.rados.rados.0.smithi093.stdout:3081: left oid 264 (ObjNum 1945 snap 0 seq_num 1945) 2023-08-27T20:54:40.150 INFO:tasks.rados.rados.0.smithi093.stdout:3082: finishing write tid 1 to smithi093153278-336 2023-08-27T20:54:40.150 INFO:tasks.rados.rados.0.smithi093.stdout:3086: seq_num 1947 ranges {723229=696171,1866794=689002} 2023-08-27T20:54:40.156 INFO:tasks.rados.rados.0.smithi093.stdout:3086: writing smithi093153278-302 from 723229 to 1419400 tid 1 2023-08-27T20:54:40.273 INFO:tasks.rados.rados.0.smithi093.stdout:3086: writing smithi093153278-302 from 1866794 to 2555796 tid 2 2023-08-27T20:54:40.273 INFO:tasks.rados.rados.0.smithi093.stdout:3081: done (5 left) 2023-08-27T20:54:40.273 INFO:tasks.rados.rados.0.smithi093.stdout:3087: read oid 379 snap -1 2023-08-27T20:54:40.273 INFO:tasks.rados.rados.0.smithi093.stdout:3087: expect (ObjNum 378 snap 0 seq_num 378) 2023-08-27T20:54:40.273 INFO:tasks.rados.rados.0.smithi093.stdout:3088: write oid 175 current snap is 0 2023-08-27T20:54:40.274 INFO:tasks.rados.rados.0.smithi093.stdout:3082: finishing write tid 2 to smithi093153278-336 2023-08-27T20:54:40.274 INFO:tasks.rados.rados.0.smithi093.stdout:3082: finishing write tid 3 to smithi093153278-336 2023-08-27T20:54:40.274 INFO:tasks.rados.rados.0.smithi093.stdout:3082: finishing write tid 4 to smithi093153278-336 2023-08-27T20:54:40.274 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 336 v 338 (ObjNum 1946 snap 0 seq_num 1946) dirty exists 2023-08-27T20:54:40.274 INFO:tasks.rados.rados.0.smithi093.stdout:3082: left oid 336 (ObjNum 1946 snap 0 seq_num 1946) 2023-08-27T20:54:40.274 INFO:tasks.rados.rados.0.smithi093.stdout:3088: seq_num 1948 ranges {740036=771420,2208923=531113} 2023-08-27T20:54:40.275 INFO:tasks.rados.rados.0.smithi093.stdout:3088: writing smithi093153278-175 from 740036 to 1511456 tid 1 2023-08-27T20:54:40.275 INFO:tasks.rados.rados.0.smithi093.stdout:3088: writing smithi093153278-175 from 2208923 to 2740036 tid 2 2023-08-27T20:54:40.275 INFO:tasks.rados.rados.0.smithi093.stdout:3082: done (6 left) 2023-08-27T20:54:40.275 INFO:tasks.rados.rados.0.smithi093.stdout:3089: read oid 335 snap -1 2023-08-27T20:54:40.275 INFO:tasks.rados.rados.0.smithi093.stdout:3089: expect deleted 2023-08-27T20:54:40.276 INFO:tasks.rados.rados.0.smithi093.stdout:3090: read oid 52 snap -1 2023-08-27T20:54:40.276 INFO:tasks.rados.rados.0.smithi093.stdout:3090: expect deleted 2023-08-27T20:54:40.276 INFO:tasks.rados.rados.0.smithi093.stdout:3091: write oid 486 current snap is 0 2023-08-27T20:54:40.276 INFO:tasks.rados.rados.0.smithi093.stdout:3084: expect (ObjNum 1594 snap 0 seq_num 1594) 2023-08-27T20:54:40.276 INFO:tasks.rados.rados.0.smithi093.stdout:3085: expect (ObjNum 1883 snap 0 seq_num 1883) 2023-08-27T20:54:40.462 INFO:tasks.rados.rados.0.smithi093.stdout:3083: expect (ObjNum 1871 snap 0 seq_num 1871) 2023-08-27T20:54:41.259 INFO:tasks.rados.rados.0.smithi093.stdout:3086: finishing write tid 1 to smithi093153278-302 2023-08-27T20:54:41.408 INFO:tasks.rados.rados.0.smithi093.stdout:3091: seq_num 1949 ranges {756843=446669,1867405=692671} 2023-08-27T20:54:41.408 INFO:tasks.rados.rados.0.smithi093.stdout:3091: writing smithi093153278-486 from 756843 to 1203512 tid 1 2023-08-27T20:54:41.408 INFO:tasks.rados.rados.0.smithi093.stdout:3091: writing smithi093153278-486 from 1867405 to 2560076 tid 2 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3083: done (8 left) 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3084: done (7 left) 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3085: done (6 left) 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3092: read oid 414 snap -1 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3092: expect (ObjNum 1871 snap 0 seq_num 1871) 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3093: write oid 357 current snap is 0 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3087: expect (ObjNum 378 snap 0 seq_num 378) 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3086: finishing write tid 2 to smithi093153278-302 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3086: finishing write tid 3 to smithi093153278-302 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:3086: finishing write tid 4 to smithi093153278-302 2023-08-27T20:54:41.409 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 302 v 419 (ObjNum 1947 snap 0 seq_num 1947) dirty exists 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3086: left oid 302 (ObjNum 1947 snap 0 seq_num 1947) 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3088: finishing write tid 1 to smithi093153278-175 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3088: finishing write tid 2 to smithi093153278-175 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3088: finishing write tid 3 to smithi093153278-175 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3088: finishing write tid 4 to smithi093153278-175 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 175 v 317 (ObjNum 1948 snap 0 seq_num 1948) dirty exists 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3088: left oid 175 (ObjNum 1948 snap 0 seq_num 1948) 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3093: seq_num 1950 ranges {773650=521918,1925887=552682} 2023-08-27T20:54:41.410 INFO:tasks.rados.rados.0.smithi093.stdout:3093: writing smithi093153278-357 from 773650 to 1295568 tid 1 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3093: writing smithi093153278-357 from 1925887 to 2478569 tid 2 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3086: done (7 left) 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3087: done (6 left) 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3088: done (5 left) 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3089: done (4 left) 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3090: done (3 left) 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3094: write oid 34 current snap is 0 2023-08-27T20:54:41.411 INFO:tasks.rados.rados.0.smithi093.stdout:3091: finishing write tid 1 to smithi093153278-486 2023-08-27T20:54:41.415 INFO:tasks.rados.rados.0.smithi093.stdout:3094: seq_num 1951 ranges {790457=597167,1984369=696340} 2023-08-27T20:54:41.415 INFO:tasks.rados.rados.0.smithi093.stdout:3094: writing smithi093153278-34 from 790457 to 1387624 tid 1 2023-08-27T20:54:41.415 INFO:tasks.rados.rados.0.smithi093.stdout:3094: writing smithi093153278-34 from 1984369 to 2680709 tid 2 2023-08-27T20:54:41.416 INFO:tasks.rados.rados.0.smithi093.stdout:3095: write oid 35 current snap is 0 2023-08-27T20:54:41.416 INFO:tasks.rados.rados.0.smithi093.stdout:3092: expect (ObjNum 1871 snap 0 seq_num 1871) 2023-08-27T20:54:41.416 INFO:tasks.rados.rados.0.smithi093.stdout:3091: finishing write tid 2 to smithi093153278-486 2023-08-27T20:54:41.416 INFO:tasks.rados.rados.0.smithi093.stdout:3091: finishing write tid 3 to smithi093153278-486 2023-08-27T20:54:41.416 INFO:tasks.rados.rados.0.smithi093.stdout:3091: finishing write tid 4 to smithi093153278-486 2023-08-27T20:54:41.417 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 420 (ObjNum 1949 snap 0 seq_num 1949) dirty exists 2023-08-27T20:54:41.417 INFO:tasks.rados.rados.0.smithi093.stdout:3091: left oid 486 (ObjNum 1949 snap 0 seq_num 1949) 2023-08-27T20:54:41.418 INFO:tasks.rados.rados.0.smithi093.stdout:3095: seq_num 1952 ranges {407264=672416,1526498=556351,2804976=2288} 2023-08-27T20:54:41.419 INFO:tasks.rados.rados.0.smithi093.stdout:3095: writing smithi093153278-35 from 407264 to 1079680 tid 1 2023-08-27T20:54:41.419 INFO:tasks.rados.rados.0.smithi093.stdout:3095: writing smithi093153278-35 from 1526498 to 2082849 tid 2 2023-08-27T20:54:41.419 INFO:tasks.rados.rados.0.smithi093.stdout:3095: writing smithi093153278-35 from 2804976 to 2807264 tid 3 2023-08-27T20:54:41.419 INFO:tasks.rados.rados.0.smithi093.stdout:3091: done (4 left) 2023-08-27T20:54:41.419 INFO:tasks.rados.rados.0.smithi093.stdout:3092: done (3 left) 2023-08-27T20:54:41.420 INFO:tasks.rados.rados.0.smithi093.stdout:3096: read oid 440 snap -1 2023-08-27T20:54:41.420 INFO:tasks.rados.rados.0.smithi093.stdout:3096: expect (ObjNum 1444 snap 0 seq_num 1444) 2023-08-27T20:54:41.420 INFO:tasks.rados.rados.0.smithi093.stdout:3097: delete oid 198 current snap is 0 2023-08-27T20:54:41.420 INFO:tasks.rados.rados.0.smithi093.stdout:3093: finishing write tid 1 to smithi093153278-357 2023-08-27T20:54:41.420 INFO:tasks.rados.rados.0.smithi093.stdout:3093: finishing write tid 2 to smithi093153278-357 2023-08-27T20:54:41.421 INFO:tasks.rados.rados.0.smithi093.stdout:3093: finishing write tid 3 to smithi093153278-357 2023-08-27T20:54:41.421 INFO:tasks.rados.rados.0.smithi093.stdout:3093: finishing write tid 4 to smithi093153278-357 2023-08-27T20:54:41.421 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 357 v 320 (ObjNum 1950 snap 0 seq_num 1950) dirty exists 2023-08-27T20:54:41.421 INFO:tasks.rados.rados.0.smithi093.stdout:3093: left oid 357 (ObjNum 1950 snap 0 seq_num 1950) 2023-08-27T20:54:41.421 INFO:tasks.rados.rados.0.smithi093.stdout:3094: finishing write tid 1 to smithi093153278-34 2023-08-27T20:54:41.421 INFO:tasks.rados.rados.0.smithi093.stdout:3094: finishing write tid 2 to smithi093153278-34 2023-08-27T20:54:41.422 INFO:tasks.rados.rados.0.smithi093.stdout:3094: finishing write tid 3 to smithi093153278-34 2023-08-27T20:54:41.422 INFO:tasks.rados.rados.0.smithi093.stdout:3094: finishing write tid 4 to smithi093153278-34 2023-08-27T20:54:41.422 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 34 v 355 (ObjNum 1951 snap 0 seq_num 1951) dirty exists 2023-08-27T20:54:41.422 INFO:tasks.rados.rados.0.smithi093.stdout:3094: left oid 34 (ObjNum 1951 snap 0 seq_num 1951) 2023-08-27T20:54:41.422 INFO:tasks.rados.rados.0.smithi093.stdout:3095: finishing write tid 1 to smithi093153278-35 2023-08-27T20:54:41.423 INFO:tasks.rados.rados.0.smithi093.stdout:3095: finishing write tid 2 to smithi093153278-35 2023-08-27T20:54:41.423 INFO:tasks.rados.rados.0.smithi093.stdout:3093: done (4 left) 2023-08-27T20:54:41.423 INFO:tasks.rados.rados.0.smithi093.stdout:3094: done (3 left) 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3097: done (2 left) 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3098: read oid 124 snap -1 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3098: expect (ObjNum 1747 snap 0 seq_num 1747) 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3099: write oid 174 current snap is 0 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3099: seq_num 1954 ranges {440878=539267,1759815=560020} 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3099: writing smithi093153278-174 from 440878 to 980145 tid 1 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3099: writing smithi093153278-174 from 1759815 to 2319835 tid 2 2023-08-27T20:54:41.460 INFO:tasks.rados.rados.0.smithi093.stdout:3100: write oid 450 current snap is 0 2023-08-27T20:54:41.461 INFO:tasks.rados.rados.0.smithi093.stdout:3100: seq_num 1955 ranges {457685=614516,1818297=703678} 2023-08-27T20:54:41.461 INFO:tasks.rados.rados.0.smithi093.stdout:3100: writing smithi093153278-450 from 457685 to 1072201 tid 1 2023-08-27T20:54:41.461 INFO:tasks.rados.rados.0.smithi093.stdout:3100: writing smithi093153278-450 from 1818297 to 2521975 tid 2 2023-08-27T20:54:41.461 INFO:tasks.rados.rados.0.smithi093.stdout:3101: read oid 371 snap -1 2023-08-27T20:54:41.462 INFO:tasks.rados.rados.0.smithi093.stdout:3101: expect (ObjNum 1396 snap 0 seq_num 1396) 2023-08-27T20:54:41.462 INFO:tasks.rados.rados.0.smithi093.stdout:3102: write oid 338 current snap is 0 2023-08-27T20:54:41.462 INFO:tasks.rados.rados.0.smithi093.stdout:3102: seq_num 1956 ranges {474492=689765,1760426=563689} 2023-08-27T20:54:41.462 INFO:tasks.rados.rados.0.smithi093.stdout:3102: writing smithi093153278-338 from 474492 to 1164257 tid 1 2023-08-27T20:54:41.462 INFO:tasks.rados.rados.0.smithi093.stdout:3102: writing smithi093153278-338 from 1760426 to 2324115 tid 2 2023-08-27T20:54:41.462 INFO:tasks.rados.rados.0.smithi093.stdout:3103: read oid 165 snap -1 2023-08-27T20:54:41.462 INFO:tasks.rados.rados.0.smithi093.stdout:3103: expect (ObjNum 1138 snap 0 seq_num 1138) 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3104: write oid 257 current snap is 0 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3096: expect (ObjNum 1444 snap 0 seq_num 1444) 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3095: finishing write tid 3 to smithi093153278-35 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3095: finishing write tid 4 to smithi093153278-35 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3095: finishing write tid 5 to smithi093153278-35 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 35 v 352 (ObjNum 1952 snap 0 seq_num 1952) dirty exists 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3095: left oid 35 (ObjNum 1952 snap 0 seq_num 1952) 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3104: seq_num 1957 ranges {491299=765014,1818908=707347} 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3104: writing smithi093153278-257 from 491299 to 1256313 tid 1 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3104: writing smithi093153278-257 from 1818908 to 2526255 tid 2 2023-08-27T20:54:41.463 INFO:tasks.rados.rados.0.smithi093.stdout:3095: done (8 left) 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3096: done (7 left) 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3105: read oid 368 snap -1 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3105: expect (ObjNum 1688 snap 0 seq_num 1688) 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3106: write oid 300 current snap is 0 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3098: expect (ObjNum 1747 snap 0 seq_num 1747) 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3099: finishing write tid 1 to smithi093153278-174 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3099: finishing write tid 2 to smithi093153278-174 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3099: finishing write tid 3 to smithi093153278-174 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3099: finishing write tid 4 to smithi093153278-174 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 174 v 358 (ObjNum 1954 snap 0 seq_num 1954) dirty exists 2023-08-27T20:54:41.464 INFO:tasks.rados.rados.0.smithi093.stdout:3099: left oid 174 (ObjNum 1954 snap 0 seq_num 1954) 2023-08-27T20:54:41.465 INFO:tasks.rados.rados.0.smithi093.stdout:3100: finishing write tid 1 to smithi093153278-450 2023-08-27T20:54:41.562 INFO:tasks.rados.rados.0.smithi093.stdout:3100: finishing write tid 2 to smithi093153278-450 2023-08-27T20:54:41.562 INFO:tasks.rados.rados.0.smithi093.stdout:3106: seq_num 1958 ranges {508106=440263,1477390=567358,2685867=222239} 2023-08-27T20:54:41.565 INFO:tasks.rados.rados.0.smithi093.stdout:3106: writing smithi093153278-300 from 508106 to 948369 tid 1 2023-08-27T20:54:41.565 INFO:tasks.rados.rados.0.smithi093.stdout:3106: writing smithi093153278-300 from 1477390 to 2044748 tid 2 2023-08-27T20:54:41.565 INFO:tasks.rados.rados.0.smithi093.stdout:3106: writing smithi093153278-300 from 2685867 to 2908106 tid 3 2023-08-27T20:54:41.565 INFO:tasks.rados.rados.0.smithi093.stdout:3098: done (8 left) 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:3099: done (7 left) 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:3107: delete oid 40 current snap is 0 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:3100: finishing write tid 3 to smithi093153278-450 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:3100: finishing write tid 4 to smithi093153278-450 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 450 v 323 (ObjNum 1955 snap 0 seq_num 1955) dirty exists 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:3100: left oid 450 (ObjNum 1955 snap 0 seq_num 1955) 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:3102: finishing write tid 1 to smithi093153278-338 2023-08-27T20:54:41.566 INFO:tasks.rados.rados.0.smithi093.stdout:3101: expect (ObjNum 1396 snap 0 seq_num 1396) 2023-08-27T20:54:41.567 INFO:tasks.rados.rados.0.smithi093.stdout:3102: finishing write tid 2 to smithi093153278-338 2023-08-27T20:54:41.567 INFO:tasks.rados.rados.0.smithi093.stdout:3100: done (7 left) 2023-08-27T20:54:41.567 INFO:tasks.rados.rados.0.smithi093.stdout:3101: done (6 left) 2023-08-27T20:54:41.567 INFO:tasks.rados.rados.0.smithi093.stdout:3107: done (5 left) 2023-08-27T20:54:41.567 INFO:tasks.rados.rados.0.smithi093.stdout:3108: delete oid 249 current snap is 0 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3102: finishing write tid 3 to smithi093153278-338 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3102: finishing write tid 4 to smithi093153278-338 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 338 v 389 (ObjNum 1956 snap 0 seq_num 1956) dirty exists 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3102: left oid 338 (ObjNum 1956 snap 0 seq_num 1956) 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3103: expect (ObjNum 1138 snap 0 seq_num 1138) 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3104: finishing write tid 1 to smithi093153278-257 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3104: finishing write tid 2 to smithi093153278-257 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3104: finishing write tid 3 to smithi093153278-257 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3104: finishing write tid 4 to smithi093153278-257 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 257 v 526 (ObjNum 1957 snap 0 seq_num 1957) dirty exists 2023-08-27T20:54:41.568 INFO:tasks.rados.rados.0.smithi093.stdout:3104: left oid 257 (ObjNum 1957 snap 0 seq_num 1957) 2023-08-27T20:54:41.573 INFO:tasks.rados.rados.0.smithi093.stdout:3105: expect (ObjNum 1688 snap 0 seq_num 1688) 2023-08-27T20:54:41.646 INFO:tasks.rados.rados.0.smithi093.stdout:3106: finishing write tid 1 to smithi093153278-300 2023-08-27T20:54:41.882 INFO:tasks.rados.rados.0.smithi093.stdout:3106: finishing write tid 2 to smithi093153278-300 2023-08-27T20:54:41.882 INFO:tasks.rados.rados.0.smithi093.stdout:3106: finishing write tid 3 to smithi093153278-300 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3106: finishing write tid 4 to smithi093153278-300 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3106: finishing write tid 5 to smithi093153278-300 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 300 v 359 (ObjNum 1958 snap 0 seq_num 1958) dirty exists 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3106: left oid 300 (ObjNum 1958 snap 0 seq_num 1958) 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3102: done (5 left) 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3103: done (4 left) 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3104: done (3 left) 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3105: done (2 left) 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3106: done (1 left) 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3108: done (0 left) 2023-08-27T20:54:41.883 INFO:tasks.rados.rados.0.smithi093.stdout:3109: write oid 433 current snap is 0 2023-08-27T20:54:41.884 INFO:tasks.rados.rados.0.smithi093.stdout:3109: seq_num 1961 ranges {558527=666010,1936483=714685} 2023-08-27T20:54:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:3109: writing smithi093153278-433 from 558527 to 1224537 tid 1 2023-08-27T20:54:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:3109: writing smithi093153278-433 from 1936483 to 2651168 tid 2 2023-08-27T20:54:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:3110: write oid 28 current snap is 0 2023-08-27T20:54:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:3109: finishing write tid 1 to smithi093153278-433 2023-08-27T20:54:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:3110: seq_num 1962 ranges {575334=457612,1711318=458343,2679206=296128} 2023-08-27T20:54:41.887 INFO:tasks.rados.rados.0.smithi093.stdout:3110: writing smithi093153278-28 from 575334 to 1032946 tid 1 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3110: writing smithi093153278-28 from 1711318 to 2169661 tid 2 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3110: writing smithi093153278-28 from 2679206 to 2975334 tid 3 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3111: write oid 281 current snap is 0 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3109: finishing write tid 2 to smithi093153278-433 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3109: finishing write tid 3 to smithi093153278-433 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3109: finishing write tid 4 to smithi093153278-433 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 433 v 426 (ObjNum 1961 snap 0 seq_num 1961) dirty exists 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3109: left oid 433 (ObjNum 1961 snap 0 seq_num 1961) 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3110: finishing write tid 1 to smithi093153278-28 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3110: finishing write tid 2 to smithi093153278-28 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3111: seq_num 1963 ranges {592141=532861,1769800=718354} 2023-08-27T20:54:41.888 INFO:tasks.rados.rados.0.smithi093.stdout:3111: writing smithi093153278-281 from 592141 to 1125002 tid 1 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3111: writing smithi093153278-281 from 1769800 to 2488154 tid 2 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3109: done (2 left) 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3112: read oid 91 snap -1 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3112: expect (ObjNum 826 snap 0 seq_num 826) 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3113: write oid 362 current snap is 0 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3110: finishing write tid 3 to smithi093153278-28 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3110: finishing write tid 4 to smithi093153278-28 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3110: finishing write tid 5 to smithi093153278-28 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 28 v 363 (ObjNum 1962 snap 0 seq_num 1962) dirty exists 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3110: left oid 28 (ObjNum 1962 snap 0 seq_num 1962) 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3111: finishing write tid 1 to smithi093153278-281 2023-08-27T20:54:41.889 INFO:tasks.rados.rados.0.smithi093.stdout:3113: seq_num 1964 ranges {608948=608110,1828282=462012,2734052=274896} 2023-08-27T20:54:41.890 INFO:tasks.rados.rados.0.smithi093.stdout:3113: writing smithi093153278-362 from 608948 to 1217058 tid 1 2023-08-27T20:54:41.890 INFO:tasks.rados.rados.0.smithi093.stdout:3113: writing smithi093153278-362 from 1828282 to 2290294 tid 2 2023-08-27T20:54:41.890 INFO:tasks.rados.rados.0.smithi093.stdout:3113: writing smithi093153278-362 from 2734052 to 3008948 tid 3 2023-08-27T20:54:41.894 INFO:tasks.rados.rados.0.smithi093.stdout:3110: done (3 left) 2023-08-27T20:54:41.894 INFO:tasks.rados.rados.0.smithi093.stdout:3114: read oid 471 snap -1 2023-08-27T20:54:41.894 INFO:tasks.rados.rados.0.smithi093.stdout:3114: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3115: read oid 277 snap -1 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3115: expect (ObjNum 1884 snap 0 seq_num 1884) 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3116: write oid 239 current snap is 0 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3116: seq_num 1965 ranges {625755=683359,1770411=722023} 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3116: writing smithi093153278-239 from 625755 to 1309114 tid 1 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3116: writing smithi093153278-239 from 1770411 to 2492434 tid 2 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3117: read oid 498 snap -1 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3117: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3118: read oid 135 snap -1 2023-08-27T20:54:41.895 INFO:tasks.rados.rados.0.smithi093.stdout:3118: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:54:41.896 INFO:tasks.rados.rados.0.smithi093.stdout:3119: write oid 398 current snap is 0 2023-08-27T20:54:41.896 INFO:tasks.rados.rados.0.smithi093.stdout:3111: finishing write tid 2 to smithi093153278-281 2023-08-27T20:54:41.896 INFO:tasks.rados.rados.0.smithi093.stdout:3111: finishing write tid 3 to smithi093153278-281 2023-08-27T20:54:41.896 INFO:tasks.rados.rados.0.smithi093.stdout:3111: finishing write tid 4 to smithi093153278-281 2023-08-27T20:54:41.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 281 v 468 (ObjNum 1963 snap 0 seq_num 1963) dirty exists 2023-08-27T20:54:41.896 INFO:tasks.rados.rados.0.smithi093.stdout:3111: left oid 281 (ObjNum 1963 snap 0 seq_num 1963) 2023-08-27T20:54:41.896 INFO:tasks.rados.rados.0.smithi093.stdout:3112: expect (ObjNum 826 snap 0 seq_num 826) 2023-08-27T20:54:41.907 INFO:tasks.rados.rados.0.smithi093.stdout:3113: finishing write tid 1 to smithi093153278-362 2023-08-27T20:54:41.907 INFO:tasks.rados.rados.0.smithi093.stdout:3113: finishing write tid 2 to smithi093153278-362 2023-08-27T20:54:41.907 INFO:tasks.rados.rados.0.smithi093.stdout:3113: finishing write tid 3 to smithi093153278-362 2023-08-27T20:54:41.908 INFO:tasks.rados.rados.0.smithi093.stdout:3113: finishing write tid 4 to smithi093153278-362 2023-08-27T20:54:41.908 INFO:tasks.rados.rados.0.smithi093.stdout:3113: finishing write tid 5 to smithi093153278-362 2023-08-27T20:54:41.908 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 362 v 369 (ObjNum 1964 snap 0 seq_num 1964) dirty exists 2023-08-27T20:54:41.908 INFO:tasks.rados.rados.0.smithi093.stdout:3113: left oid 362 (ObjNum 1964 snap 0 seq_num 1964) 2023-08-27T20:54:41.908 INFO:tasks.rados.rados.0.smithi093.stdout:3115: expect (ObjNum 1884 snap 0 seq_num 1884) 2023-08-27T20:54:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:41 smithi136 ceph-mon[113531]: pgmap v34: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s rd, 9.4 MiB/s wr, 104 op/s 2023-08-27T20:54:42.025 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2367944942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:42.026 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:41 smithi093 ceph-mon[111335]: pgmap v34: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s rd, 9.4 MiB/s wr, 104 op/s 2023-08-27T20:54:42.026 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2367944942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:42.026 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:41 smithi093 ceph-mon[118902]: pgmap v34: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 82 MiB/s rd, 9.4 MiB/s wr, 104 op/s 2023-08-27T20:54:42.026 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2367944942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:42.026 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:41.626Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:42.027 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:41.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:42.069 INFO:tasks.rados.rados.0.smithi093.stdout:3116: finishing write tid 1 to smithi093153278-239 2023-08-27T20:54:42.069 INFO:tasks.rados.rados.0.smithi093.stdout:3114: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:54:42.189 INFO:tasks.rados.rados.0.smithi093.stdout:3116: finishing write tid 2 to smithi093153278-239 2023-08-27T20:54:42.205 INFO:tasks.rados.rados.0.smithi093.stdout:3116: finishing write tid 3 to smithi093153278-239 2023-08-27T20:54:42.205 INFO:tasks.rados.rados.0.smithi093.stdout:3116: finishing write tid 4 to smithi093153278-239 2023-08-27T20:54:42.205 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 239 v 361 (ObjNum 1965 snap 0 seq_num 1965) dirty exists 2023-08-27T20:54:42.205 INFO:tasks.rados.rados.0.smithi093.stdout:3116: left oid 239 (ObjNum 1965 snap 0 seq_num 1965) 2023-08-27T20:54:42.205 INFO:tasks.rados.rados.0.smithi093.stdout:3119: seq_num 1966 ranges {642562=758608,1828893=465681} 2023-08-27T20:54:42.206 INFO:tasks.rados.rados.0.smithi093.stdout:3119: writing smithi093153278-398 from 642562 to 1401170 tid 1 2023-08-27T20:54:42.206 INFO:tasks.rados.rados.0.smithi093.stdout:3119: writing smithi093153278-398 from 1828893 to 2294574 tid 2 2023-08-27T20:54:42.206 INFO:tasks.rados.rados.0.smithi093.stdout:3111: done (8 left) 2023-08-27T20:54:42.207 INFO:tasks.rados.rados.0.smithi093.stdout:3112: done (7 left) 2023-08-27T20:54:42.207 INFO:tasks.rados.rados.0.smithi093.stdout:3113: done (6 left) 2023-08-27T20:54:42.207 INFO:tasks.rados.rados.0.smithi093.stdout:3114: done (5 left) 2023-08-27T20:54:42.207 INFO:tasks.rados.rados.0.smithi093.stdout:3115: done (4 left) 2023-08-27T20:54:42.207 INFO:tasks.rados.rados.0.smithi093.stdout:3116: done (3 left) 2023-08-27T20:54:42.207 INFO:tasks.rados.rados.0.smithi093.stdout:3120: delete oid 385 current snap is 0 2023-08-27T20:54:42.207 INFO:tasks.rados.rados.0.smithi093.stdout:3117: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:54:42.362 INFO:tasks.rados.rados.0.smithi093.stdout:3118: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:54:42.512 INFO:tasks.rados.rados.0.smithi093.stdout:3119: finishing write tid 1 to smithi093153278-398 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3119: finishing write tid 2 to smithi093153278-398 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3117: done (3 left) 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3118: done (2 left) 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3120: done (1 left) 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3121: read oid 246 snap -1 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3121: expect deleted 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3122: write oid 492 current snap is 0 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3119: finishing write tid 3 to smithi093153278-398 2023-08-27T20:54:42.513 INFO:tasks.rados.rados.0.smithi093.stdout:3122: seq_num 1968 ranges {676176=509106,1945857=469350} 2023-08-27T20:54:42.518 INFO:tasks.rados.rados.0.smithi093.stdout:3122: writing smithi093153278-492 from 676176 to 1185282 tid 1 2023-08-27T20:54:42.525 INFO:tasks.rados.rados.0.smithi093.stdout:3122: writing smithi093153278-492 from 1945857 to 2415207 tid 2 2023-08-27T20:54:42.525 INFO:tasks.rados.rados.0.smithi093.stdout:3123: read oid 467 snap -1 2023-08-27T20:54:42.525 INFO:tasks.rados.rados.0.smithi093.stdout:3123: expect deleted 2023-08-27T20:54:42.526 INFO:tasks.rados.rados.0.smithi093.stdout:3124: write oid 43 current snap is 0 2023-08-27T20:54:42.526 INFO:tasks.rados.rados.0.smithi093.stdout:3124: seq_num 1969 ranges {692983=584355,1887986=729361} 2023-08-27T20:54:42.532 INFO:tasks.rados.rados.0.smithi093.stdout:3124: writing smithi093153278-43 from 692983 to 1277338 tid 1 2023-08-27T20:54:42.540 INFO:tasks.rados.rados.0.smithi093.stdout:3124: writing smithi093153278-43 from 1887986 to 2617347 tid 2 2023-08-27T20:54:42.540 INFO:tasks.rados.rados.0.smithi093.stdout:3125: read oid 153 snap -1 2023-08-27T20:54:42.540 INFO:tasks.rados.rados.0.smithi093.stdout:3125: expect (ObjNum 1775 snap 0 seq_num 1775) 2023-08-27T20:54:42.540 INFO:tasks.rados.rados.0.smithi093.stdout:3126: write oid 55 current snap is 0 2023-08-27T20:54:42.540 INFO:tasks.rados.rados.0.smithi093.stdout:3119: finishing write tid 4 to smithi093153278-398 2023-08-27T20:54:42.540 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 398 v 424 (ObjNum 1966 snap 0 seq_num 1966) dirty exists 2023-08-27T20:54:42.540 INFO:tasks.rados.rados.0.smithi093.stdout:3119: left oid 398 (ObjNum 1966 snap 0 seq_num 1966) 2023-08-27T20:54:42.541 INFO:tasks.rados.rados.0.smithi093.stdout:3126: seq_num 1970 ranges {709790=775957,2062821=473019} 2023-08-27T20:54:42.548 INFO:tasks.rados.rados.0.smithi093.stdout:3126: writing smithi093153278-55 from 709790 to 1485747 tid 1 2023-08-27T20:54:42.555 INFO:tasks.rados.rados.0.smithi093.stdout:3126: writing smithi093153278-55 from 2062821 to 2535840 tid 2 2023-08-27T20:54:42.555 INFO:tasks.rados.rados.0.smithi093.stdout:3119: done (6 left) 2023-08-27T20:54:42.555 INFO:tasks.rados.rados.0.smithi093.stdout:3127: write oid 254 current snap is 0 2023-08-27T20:54:42.555 INFO:tasks.rados.rados.0.smithi093.stdout:3122: finishing write tid 1 to smithi093153278-492 2023-08-27T20:54:42.555 INFO:tasks.rados.rados.0.smithi093.stdout:3122: finishing write tid 2 to smithi093153278-492 2023-08-27T20:54:42.555 INFO:tasks.rados.rados.0.smithi093.stdout:3127: seq_num 1971 ranges {726597=451206,1721303=616677,2809660=316937} 2023-08-27T20:54:42.561 INFO:tasks.rados.rados.0.smithi093.stdout:3127: writing smithi093153278-254 from 726597 to 1177803 tid 1 2023-08-27T20:54:42.568 INFO:tasks.rados.rados.0.smithi093.stdout:3127: writing smithi093153278-254 from 1721303 to 2337980 tid 2 2023-08-27T20:54:42.572 INFO:tasks.rados.rados.0.smithi093.stdout:3127: writing smithi093153278-254 from 2809660 to 3126597 tid 3 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3121: done (6 left) 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3123: done (5 left) 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3128: write oid 478 current snap is 0 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3122: finishing write tid 3 to smithi093153278-492 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3122: finishing write tid 4 to smithi093153278-492 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 492 v 341 (ObjNum 1968 snap 0 seq_num 1968) dirty exists 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3122: left oid 492 (ObjNum 1968 snap 0 seq_num 1968) 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3124: finishing write tid 1 to smithi093153278-43 2023-08-27T20:54:42.573 INFO:tasks.rados.rados.0.smithi093.stdout:3128: seq_num 1972 ranges {743404=526455,1779785=476688,2837083=306321} 2023-08-27T20:54:42.579 INFO:tasks.rados.rados.0.smithi093.stdout:3128: writing smithi093153278-478 from 743404 to 1269859 tid 1 2023-08-27T20:54:42.584 INFO:tasks.rados.rados.0.smithi093.stdout:3128: writing smithi093153278-478 from 1779785 to 2256473 tid 2 2023-08-27T20:54:42.590 INFO:tasks.rados.rados.0.smithi093.stdout:3128: writing smithi093153278-478 from 2837083 to 3143404 tid 3 2023-08-27T20:54:42.590 INFO:tasks.rados.rados.0.smithi093.stdout:3122: done (5 left) 2023-08-27T20:54:42.590 INFO:tasks.rados.rados.0.smithi093.stdout:3129: write oid 228 current snap is 0 2023-08-27T20:54:42.590 INFO:tasks.rados.rados.0.smithi093.stdout:3124: finishing write tid 2 to smithi093153278-43 2023-08-27T20:54:42.590 INFO:tasks.rados.rados.0.smithi093.stdout:3124: finishing write tid 3 to smithi093153278-43 2023-08-27T20:54:42.590 INFO:tasks.rados.rados.0.smithi093.stdout:3124: finishing write tid 4 to smithi093153278-43 2023-08-27T20:54:42.591 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 43 v 482 (ObjNum 1969 snap 0 seq_num 1969) dirty exists 2023-08-27T20:54:42.591 INFO:tasks.rados.rados.0.smithi093.stdout:3124: left oid 43 (ObjNum 1969 snap 0 seq_num 1969) 2023-08-27T20:54:42.591 INFO:tasks.rados.rados.0.smithi093.stdout:3125: expect (ObjNum 1775 snap 0 seq_num 1775) 2023-08-27T20:54:42.708 INFO:tasks.rados.rados.0.smithi093.stdout:3126: finishing write tid 1 to smithi093153278-55 2023-08-27T20:54:42.709 INFO:tasks.rados.rados.0.smithi093.stdout:3126: finishing write tid 2 to smithi093153278-55 2023-08-27T20:54:42.709 INFO:tasks.rados.rados.0.smithi093.stdout:3129: seq_num 1973 ranges {760211=601704,2121914=620346,3148153=12058} 2023-08-27T20:54:42.716 INFO:tasks.rados.rados.0.smithi093.stdout:3129: writing smithi093153278-228 from 760211 to 1361915 tid 1 2023-08-27T20:54:42.723 INFO:tasks.rados.rados.0.smithi093.stdout:3129: writing smithi093153278-228 from 2121914 to 2742260 tid 2 2023-08-27T20:54:42.726 INFO:tasks.rados.rados.0.smithi093.stdout:3129: writing smithi093153278-228 from 3148153 to 3160211 tid 3 2023-08-27T20:54:42.726 INFO:tasks.rados.rados.0.smithi093.stdout:3124: done (5 left) 2023-08-27T20:54:42.726 INFO:tasks.rados.rados.0.smithi093.stdout:3125: done (4 left) 2023-08-27T20:54:42.726 INFO:tasks.rados.rados.0.smithi093.stdout:3130: read oid 62 snap -1 2023-08-27T20:54:42.726 INFO:tasks.rados.rados.0.smithi093.stdout:3130: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3131: read oid 47 snap -1 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3131: expect (ObjNum 1826 snap 0 seq_num 1826) 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3132: read oid 20 snap -1 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3132: expect (ObjNum 1572 snap 0 seq_num 1572) 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3133: write oid 177 current snap is 0 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3126: finishing write tid 3 to smithi093153278-55 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3126: finishing write tid 4 to smithi093153278-55 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 55 v 364 (ObjNum 1970 snap 0 seq_num 1970) dirty exists 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3126: left oid 55 (ObjNum 1970 snap 0 seq_num 1970) 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3127: finishing write tid 1 to smithi093153278-254 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3127: finishing write tid 2 to smithi093153278-254 2023-08-27T20:54:42.727 INFO:tasks.rados.rados.0.smithi093.stdout:3127: finishing write tid 3 to smithi093153278-254 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3127: finishing write tid 4 to smithi093153278-254 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3127: finishing write tid 5 to smithi093153278-254 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 254 v 391 (ObjNum 1971 snap 0 seq_num 1971) dirty exists 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3127: left oid 254 (ObjNum 1971 snap 0 seq_num 1971) 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3128: finishing write tid 1 to smithi093153278-478 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3128: finishing write tid 2 to smithi093153278-478 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3128: finishing write tid 3 to smithi093153278-478 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3128: finishing write tid 4 to smithi093153278-478 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3128: finishing write tid 5 to smithi093153278-478 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 429 (ObjNum 1972 snap 0 seq_num 1972) dirty exists 2023-08-27T20:54:42.728 INFO:tasks.rados.rados.0.smithi093.stdout:3128: left oid 478 (ObjNum 1972 snap 0 seq_num 1972) 2023-08-27T20:54:42.729 INFO:tasks.rados.rados.0.smithi093.stdout:3129: finishing write tid 1 to smithi093153278-228 2023-08-27T20:54:42.729 INFO:tasks.rados.rados.0.smithi093.stdout:3133: seq_num 1974 ranges {777018=676953,2180396=480357,3175576=1442} 2023-08-27T20:54:42.734 INFO:tasks.rados.rados.0.smithi093.stdout:3133: writing smithi093153278-177 from 777018 to 1453971 tid 1 2023-08-27T20:54:42.741 INFO:tasks.rados.rados.0.smithi093.stdout:3133: writing smithi093153278-177 from 2180396 to 2660753 tid 2 2023-08-27T20:54:42.744 INFO:tasks.rados.rados.0.smithi093.stdout:3133: writing smithi093153278-177 from 3175576 to 3177018 tid 3 2023-08-27T20:54:42.744 INFO:tasks.rados.rados.0.smithi093.stdout:3126: done (7 left) 2023-08-27T20:54:42.744 INFO:tasks.rados.rados.0.smithi093.stdout:3127: done (6 left) 2023-08-27T20:54:42.744 INFO:tasks.rados.rados.0.smithi093.stdout:3128: done (5 left) 2023-08-27T20:54:42.744 INFO:tasks.rados.rados.0.smithi093.stdout:3134: read oid 186 snap -1 2023-08-27T20:54:42.744 INFO:tasks.rados.rados.0.smithi093.stdout:3134: expect (ObjNum 934 snap 0 seq_num 934) 2023-08-27T20:54:42.744 INFO:tasks.rados.rados.0.smithi093.stdout:3135: write oid 133 current snap is 0 2023-08-27T20:54:42.745 INFO:tasks.rados.rados.0.smithi093.stdout:3135: seq_num 1975 ranges {793825=752202,2238878=624015} 2023-08-27T20:54:42.752 INFO:tasks.rados.rados.0.smithi093.stdout:3135: writing smithi093153278-133 from 793825 to 1546027 tid 1 2023-08-27T20:54:42.760 INFO:tasks.rados.rados.0.smithi093.stdout:3135: writing smithi093153278-133 from 2238878 to 2862893 tid 2 2023-08-27T20:54:42.760 INFO:tasks.rados.rados.0.smithi093.stdout:3136: write oid 261 current snap is 0 2023-08-27T20:54:42.760 INFO:tasks.rados.rados.0.smithi093.stdout:3129: finishing write tid 2 to smithi093153278-228 2023-08-27T20:54:42.760 INFO:tasks.rados.rados.0.smithi093.stdout:3130: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:54:42.961 INFO:tasks.rados.rados.0.smithi093.stdout:3129: finishing write tid 3 to smithi093153278-228 2023-08-27T20:54:43.161 INFO:tasks.rados.rados.0.smithi093.stdout:3129: finishing write tid 4 to smithi093153278-228 2023-08-27T20:54:43.161 INFO:tasks.rados.rados.0.smithi093.stdout:3129: finishing write tid 5 to smithi093153278-228 2023-08-27T20:54:43.161 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 228 v 367 (ObjNum 1973 snap 0 seq_num 1973) dirty exists 2023-08-27T20:54:43.161 INFO:tasks.rados.rados.0.smithi093.stdout:3129: left oid 228 (ObjNum 1973 snap 0 seq_num 1973) 2023-08-27T20:54:43.161 INFO:tasks.rados.rados.0.smithi093.stdout:3136: seq_num 1976 ranges {410632=427451,1497360=484026,2430422=537968} 2023-08-27T20:54:43.162 INFO:tasks.rados.rados.0.smithi093.stdout:3136: writing smithi093153278-261 from 410632 to 838083 tid 1 2023-08-27T20:54:43.162 INFO:tasks.rados.rados.0.smithi093.stdout:3136: writing smithi093153278-261 from 1497360 to 1981386 tid 2 2023-08-27T20:54:43.163 INFO:tasks.rados.rados.0.smithi093.stdout:3136: writing smithi093153278-261 from 2430422 to 2968390 tid 3 2023-08-27T20:54:43.163 INFO:tasks.rados.rados.0.smithi093.stdout:3129: done (7 left) 2023-08-27T20:54:43.163 INFO:tasks.rados.rados.0.smithi093.stdout:3130: done (6 left) 2023-08-27T20:54:43.163 INFO:tasks.rados.rados.0.smithi093.stdout:3137: write oid 159 current snap is 0 2023-08-27T20:54:43.163 INFO:tasks.rados.rados.0.smithi093.stdout:3132: expect (ObjNum 1572 snap 0 seq_num 1572) 2023-08-27T20:54:43.163 INFO:tasks.rados.rados.0.smithi093.stdout:3131: expect (ObjNum 1826 snap 0 seq_num 1826) 2023-08-27T20:54:43.239 INFO:tasks.rados.rados.0.smithi093.stdout:3133: finishing write tid 1 to smithi093153278-177 2023-08-27T20:54:43.239 INFO:tasks.rados.rados.0.smithi093.stdout:3133: finishing write tid 2 to smithi093153278-177 2023-08-27T20:54:43.240 INFO:tasks.rados.rados.0.smithi093.stdout:3133: finishing write tid 3 to smithi093153278-177 2023-08-27T20:54:43.240 INFO:tasks.rados.rados.0.smithi093.stdout:3133: finishing write tid 4 to smithi093153278-177 2023-08-27T20:54:43.240 INFO:tasks.rados.rados.0.smithi093.stdout:3133: finishing write tid 5 to smithi093153278-177 2023-08-27T20:54:43.240 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 177 v 345 (ObjNum 1974 snap 0 seq_num 1974) dirty exists 2023-08-27T20:54:43.240 INFO:tasks.rados.rados.0.smithi093.stdout:3133: left oid 177 (ObjNum 1974 snap 0 seq_num 1974) 2023-08-27T20:54:43.240 INFO:tasks.rados.rados.0.smithi093.stdout:3134: expect (ObjNum 934 snap 0 seq_num 934) 2023-08-27T20:54:43.426 INFO:tasks.rados.rados.0.smithi093.stdout:3135: finishing write tid 1 to smithi093153278-133 2023-08-27T20:54:43.427 INFO:tasks.rados.rados.0.smithi093.stdout:3135: finishing write tid 2 to smithi093153278-133 2023-08-27T20:54:43.427 INFO:tasks.rados.rados.0.smithi093.stdout:3135: finishing write tid 3 to smithi093153278-133 2023-08-27T20:54:43.427 INFO:tasks.rados.rados.0.smithi093.stdout:3137: seq_num 1977 ranges {427439=619053,1555842=627684,2857845=369594} 2023-08-27T20:54:43.432 INFO:tasks.rados.rados.0.smithi093.stdout:3137: writing smithi093153278-159 from 427439 to 1046492 tid 1 2023-08-27T20:54:43.438 INFO:tasks.rados.rados.0.smithi093.stdout:3137: writing smithi093153278-159 from 1555842 to 2183526 tid 2 2023-08-27T20:54:43.445 INFO:tasks.rados.rados.0.smithi093.stdout:3137: writing smithi093153278-159 from 2857845 to 3227439 tid 3 2023-08-27T20:54:43.445 INFO:tasks.rados.rados.0.smithi093.stdout:3131: done (6 left) 2023-08-27T20:54:43.445 INFO:tasks.rados.rados.0.smithi093.stdout:3132: done (5 left) 2023-08-27T20:54:43.445 INFO:tasks.rados.rados.0.smithi093.stdout:3133: done (4 left) 2023-08-27T20:54:43.445 INFO:tasks.rados.rados.0.smithi093.stdout:3134: done (3 left) 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3138: read oid 247 snap -1 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3138: expect (ObjNum 1438 snap 0 seq_num 1438) 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3139: write oid 431 current snap is 0 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3135: finishing write tid 4 to smithi093153278-133 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 133 v 485 (ObjNum 1975 snap 0 seq_num 1975) dirty exists 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3135: left oid 133 (ObjNum 1975 snap 0 seq_num 1975) 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3136: finishing write tid 1 to smithi093153278-261 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3136: finishing write tid 2 to smithi093153278-261 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3136: finishing write tid 3 to smithi093153278-261 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3136: finishing write tid 4 to smithi093153278-261 2023-08-27T20:54:43.446 INFO:tasks.rados.rados.0.smithi093.stdout:3136: finishing write tid 5 to smithi093153278-261 2023-08-27T20:54:43.447 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 261 v 423 (ObjNum 1976 snap 0 seq_num 1976) dirty exists 2023-08-27T20:54:43.447 INFO:tasks.rados.rados.0.smithi093.stdout:3136: left oid 261 (ObjNum 1976 snap 0 seq_num 1976) 2023-08-27T20:54:43.447 INFO:tasks.rados.rados.0.smithi093.stdout:3139: seq_num 1978 ranges {444246=694302,1614324=487695,2885268=358978} 2023-08-27T20:54:43.451 INFO:tasks.rados.rados.0.smithi093.stdout:3139: writing smithi093153278-431 from 444246 to 1138548 tid 1 2023-08-27T20:54:43.457 INFO:tasks.rados.rados.0.smithi093.stdout:3139: writing smithi093153278-431 from 1614324 to 2102019 tid 2 2023-08-27T20:54:43.463 INFO:tasks.rados.rados.0.smithi093.stdout:3139: writing smithi093153278-431 from 2885268 to 3244246 tid 3 2023-08-27T20:54:43.463 INFO:tasks.rados.rados.0.smithi093.stdout:3135: done (4 left) 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3136: done (3 left) 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3140: read oid 301 snap -1 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3140: expect (ObjNum 1723 snap 0 seq_num 1723) 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3141: read oid 183 snap -1 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3141: expect (ObjNum 1569 snap 0 seq_num 1569) 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3142: write oid 490 current snap is 0 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3137: finishing write tid 1 to smithi093153278-159 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3137: finishing write tid 2 to smithi093153278-159 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3137: finishing write tid 3 to smithi093153278-159 2023-08-27T20:54:43.464 INFO:tasks.rados.rados.0.smithi093.stdout:3142: seq_num 1979 ranges {461053=769551,1672806=631353,2912691=348362} 2023-08-27T20:54:43.470 INFO:tasks.rados.rados.0.smithi093.stdout:3142: writing smithi093153278-490 from 461053 to 1230604 tid 1 2023-08-27T20:54:43.476 INFO:tasks.rados.rados.0.smithi093.stdout:3142: writing smithi093153278-490 from 1672806 to 2304159 tid 2 2023-08-27T20:54:43.482 INFO:tasks.rados.rados.0.smithi093.stdout:3142: writing smithi093153278-490 from 2912691 to 3261053 tid 3 2023-08-27T20:54:43.482 INFO:tasks.rados.rados.0.smithi093.stdout:3143: write oid 22 current snap is 0 2023-08-27T20:54:43.482 INFO:tasks.rados.rados.0.smithi093.stdout:3137: finishing write tid 4 to smithi093153278-159 2023-08-27T20:54:43.482 INFO:tasks.rados.rados.0.smithi093.stdout:3143: seq_num 1980 ranges {477860=444800,1331288=491364,2540114=699409} 2023-08-27T20:54:43.486 INFO:tasks.rados.rados.0.smithi093.stdout:3143: writing smithi093153278-22 from 477860 to 922660 tid 1 2023-08-27T20:54:43.491 INFO:tasks.rados.rados.0.smithi093.stdout:3143: writing smithi093153278-22 from 1331288 to 1822652 tid 2 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:3143: writing smithi093153278-22 from 2540114 to 3239523 tid 3 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:3144: read oid 71 snap -1 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:3144: expect (ObjNum 1099 snap 0 seq_num 1099) 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:3145: write oid 435 current snap is 0 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:3137: finishing write tid 5 to smithi093153278-159 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 159 v 327 (ObjNum 1977 snap 0 seq_num 1977) dirty exists 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:3137: left oid 159 (ObjNum 1977 snap 0 seq_num 1977) 2023-08-27T20:54:43.500 INFO:tasks.rados.rados.0.smithi093.stdout:3138: expect (ObjNum 1438 snap 0 seq_num 1438) 2023-08-27T20:54:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:3139: finishing write tid 1 to smithi093153278-431 2023-08-27T20:54:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:3139: finishing write tid 2 to smithi093153278-431 2023-08-27T20:54:43.620 INFO:tasks.rados.rados.0.smithi093.stdout:3145: seq_num 1981 ranges {494667=520049,1673417=635022,2851184=443483} 2023-08-27T20:54:43.626 INFO:tasks.rados.rados.0.smithi093.stdout:3145: writing smithi093153278-435 from 494667 to 1014716 tid 1 2023-08-27T20:54:43.633 INFO:tasks.rados.rados.0.smithi093.stdout:3145: writing smithi093153278-435 from 1673417 to 2308439 tid 2 2023-08-27T20:54:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:3145: writing smithi093153278-435 from 2851184 to 3294667 tid 3 2023-08-27T20:54:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:3137: done (8 left) 2023-08-27T20:54:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:3138: done (7 left) 2023-08-27T20:54:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:3146: read oid 66 snap -1 2023-08-27T20:54:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:3146: expect (ObjNum 1704 snap 0 seq_num 1704) 2023-08-27T20:54:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:3147: read oid 151 snap -1 2023-08-27T20:54:43.639 INFO:tasks.rados.rados.0.smithi093.stdout:3147: expect (ObjNum 1783 snap 0 seq_num 1783) 2023-08-27T20:54:43.640 INFO:tasks.rados.rados.0.smithi093.stdout:3148: write oid 212 current snap is 0 2023-08-27T20:54:43.640 INFO:tasks.rados.rados.0.smithi093.stdout:3139: finishing write tid 3 to smithi093153278-431 2023-08-27T20:54:43.640 INFO:tasks.rados.rados.0.smithi093.stdout:3139: finishing write tid 4 to smithi093153278-431 2023-08-27T20:54:43.640 INFO:tasks.rados.rados.0.smithi093.stdout:3139: finishing write tid 5 to smithi093153278-431 2023-08-27T20:54:43.640 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 431 v 395 (ObjNum 1978 snap 0 seq_num 1978) dirty exists 2023-08-27T20:54:43.640 INFO:tasks.rados.rados.0.smithi093.stdout:3139: left oid 431 (ObjNum 1978 snap 0 seq_num 1978) 2023-08-27T20:54:43.640 INFO:tasks.rados.rados.0.smithi093.stdout:3140: expect (ObjNum 1723 snap 0 seq_num 1723) 2023-08-27T20:54:43.733 INFO:tasks.rados.rados.0.smithi093.stdout:3141: expect (ObjNum 1569 snap 0 seq_num 1569) 2023-08-27T20:54:43.857 INFO:tasks.rados.rados.0.smithi093.stdout:3142: finishing write tid 1 to smithi093153278-490 2023-08-27T20:54:43.857 INFO:tasks.rados.rados.0.smithi093.stdout:3142: finishing write tid 2 to smithi093153278-490 2023-08-27T20:54:43.857 INFO:tasks.rados.rados.0.smithi093.stdout:3142: finishing write tid 3 to smithi093153278-490 2023-08-27T20:54:43.858 INFO:tasks.rados.rados.0.smithi093.stdout:3142: finishing write tid 4 to smithi093153278-490 2023-08-27T20:54:43.858 INFO:tasks.rados.rados.0.smithi093.stdout:3148: seq_num 1982 ranges {511474=595298,1731899=495033,2878607=432867} 2023-08-27T20:54:43.863 INFO:tasks.rados.rados.0.smithi093.stdout:3148: writing smithi093153278-212 from 511474 to 1106772 tid 1 2023-08-27T20:54:43.869 INFO:tasks.rados.rados.0.smithi093.stdout:3148: writing smithi093153278-212 from 1731899 to 2226932 tid 2 2023-08-27T20:54:43.875 INFO:tasks.rados.rados.0.smithi093.stdout:3148: writing smithi093153278-212 from 2878607 to 3311474 tid 3 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3139: done (9 left) 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3140: done (8 left) 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3141: done (7 left) 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3149: delete oid 208 current snap is 0 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3142: finishing write tid 5 to smithi093153278-490 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 490 v 399 (ObjNum 1979 snap 0 seq_num 1979) dirty exists 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3142: left oid 490 (ObjNum 1979 snap 0 seq_num 1979) 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3143: finishing write tid 1 to smithi093153278-22 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3143: finishing write tid 2 to smithi093153278-22 2023-08-27T20:54:43.876 INFO:tasks.rados.rados.0.smithi093.stdout:3143: finishing write tid 3 to smithi093153278-22 2023-08-27T20:54:43.877 INFO:tasks.rados.rados.0.smithi093.stdout:3143: finishing write tid 4 to smithi093153278-22 2023-08-27T20:54:43.877 INFO:tasks.rados.rados.0.smithi093.stdout:3143: finishing write tid 5 to smithi093153278-22 2023-08-27T20:54:43.877 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 22 v 403 (ObjNum 1980 snap 0 seq_num 1980) dirty exists 2023-08-27T20:54:43.877 INFO:tasks.rados.rados.0.smithi093.stdout:3143: left oid 22 (ObjNum 1980 snap 0 seq_num 1980) 2023-08-27T20:54:43.877 INFO:tasks.rados.rados.0.smithi093.stdout:3144: expect (ObjNum 1099 snap 0 seq_num 1099) 2023-08-27T20:54:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:43 smithi136 ceph-mon[113531]: pgmap v35: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.3 MiB/s wr, 75 op/s 2023-08-27T20:54:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:43 smithi093 ceph-mon[111335]: pgmap v35: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.3 MiB/s wr, 75 op/s 2023-08-27T20:54:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:43 smithi093 ceph-mon[118902]: pgmap v35: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.3 MiB/s wr, 75 op/s 2023-08-27T20:54:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:43.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:43.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:3145: finishing write tid 1 to smithi093153278-435 2023-08-27T20:54:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:3145: finishing write tid 2 to smithi093153278-435 2023-08-27T20:54:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:3145: finishing write tid 3 to smithi093153278-435 2023-08-27T20:54:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:3145: finishing write tid 4 to smithi093153278-435 2023-08-27T20:54:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:3145: finishing write tid 5 to smithi093153278-435 2023-08-27T20:54:44.020 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 435 v 433 (ObjNum 1981 snap 0 seq_num 1981) dirty exists 2023-08-27T20:54:44.021 INFO:tasks.rados.rados.0.smithi093.stdout:3145: left oid 435 (ObjNum 1981 snap 0 seq_num 1981) 2023-08-27T20:54:44.021 INFO:tasks.rados.rados.0.smithi093.stdout:3146: expect (ObjNum 1704 snap 0 seq_num 1704) 2023-08-27T20:54:44.101 INFO:tasks.rados.rados.0.smithi093.stdout:3147: expect (ObjNum 1783 snap 0 seq_num 1783) 2023-08-27T20:54:44.395 INFO:tasks.rados.rados.0.smithi093.stdout:3148: finishing write tid 1 to smithi093153278-212 2023-08-27T20:54:44.395 INFO:tasks.rados.rados.0.smithi093.stdout:3148: finishing write tid 2 to smithi093153278-212 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3148: finishing write tid 3 to smithi093153278-212 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3148: finishing write tid 4 to smithi093153278-212 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3148: finishing write tid 5 to smithi093153278-212 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 212 v 356 (ObjNum 1982 snap 0 seq_num 1982) dirty exists 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3148: left oid 212 (ObjNum 1982 snap 0 seq_num 1982) 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3142: done (7 left) 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3143: done (6 left) 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3144: done (5 left) 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3145: done (4 left) 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3146: done (3 left) 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3147: done (2 left) 2023-08-27T20:54:44.396 INFO:tasks.rados.rados.0.smithi093.stdout:3148: done (1 left) 2023-08-27T20:54:44.397 INFO:tasks.rados.rados.0.smithi093.stdout:3149: done (0 left) 2023-08-27T20:54:44.397 INFO:tasks.rados.rados.0.smithi093.stdout:3150: delete oid 78 current snap is 0 2023-08-27T20:54:44.397 INFO:tasks.rados.rados.0.smithi093.stdout:3150: done (0 left) 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3151: read oid 154 snap -1 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3151: expect (ObjNum 1263 snap 0 seq_num 1263) 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3152: read oid 60 snap -1 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3152: expect (ObjNum 1768 snap 0 seq_num 1768) 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3153: read oid 446 snap -1 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3153: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3154: write oid 199 current snap is 0 2023-08-27T20:54:44.398 INFO:tasks.rados.rados.0.smithi093.stdout:3154: seq_num 1985 ranges {561895=537398,1507345=642360,2560876=672122} 2023-08-27T20:54:44.404 INFO:tasks.rados.rados.0.smithi093.stdout:3154: writing smithi093153278-199 from 561895 to 1099293 tid 1 2023-08-27T20:54:44.411 INFO:tasks.rados.rados.0.smithi093.stdout:3154: writing smithi093153278-199 from 1507345 to 2149705 tid 2 2023-08-27T20:54:44.417 INFO:tasks.rados.rados.0.smithi093.stdout:3154: writing smithi093153278-199 from 2560876 to 3232998 tid 3 2023-08-27T20:54:44.417 INFO:tasks.rados.rados.0.smithi093.stdout:3155: read oid 95 snap -1 2023-08-27T20:54:44.417 INFO:tasks.rados.rados.0.smithi093.stdout:3155: expect (ObjNum 1227 snap 0 seq_num 1227) 2023-08-27T20:54:44.417 INFO:tasks.rados.rados.0.smithi093.stdout:3156: delete oid 64 current snap is 0 2023-08-27T20:54:44.417 INFO:tasks.rados.rados.0.smithi093.stdout:3151: expect (ObjNum 1263 snap 0 seq_num 1263) 2023-08-27T20:54:44.604 INFO:tasks.rados.rados.0.smithi093.stdout:3154: finishing write tid 1 to smithi093153278-199 2023-08-27T20:54:44.604 INFO:tasks.rados.rados.0.smithi093.stdout:3153: expect (ObjNum 1043 snap 0 seq_num 1043) 2023-08-27T20:54:44.712 INFO:tasks.rados.rados.0.smithi093.stdout:3154: finishing write tid 2 to smithi093153278-199 2023-08-27T20:54:44.713 INFO:tasks.rados.rados.0.smithi093.stdout:3152: expect (ObjNum 1768 snap 0 seq_num 1768) 2023-08-27T20:54:44.906 INFO:tasks.rados.rados.0.smithi093.stdout:3154: finishing write tid 3 to smithi093153278-199 2023-08-27T20:54:44.906 INFO:tasks.rados.rados.0.smithi093.stdout:3154: finishing write tid 4 to smithi093153278-199 2023-08-27T20:54:44.907 INFO:tasks.rados.rados.0.smithi093.stdout:3154: finishing write tid 5 to smithi093153278-199 2023-08-27T20:54:44.907 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 199 v 531 (ObjNum 1985 snap 0 seq_num 1985) dirty exists 2023-08-27T20:54:44.907 INFO:tasks.rados.rados.0.smithi093.stdout:3154: left oid 199 (ObjNum 1985 snap 0 seq_num 1985) 2023-08-27T20:54:44.907 INFO:tasks.rados.rados.0.smithi093.stdout:3155: expect (ObjNum 1227 snap 0 seq_num 1227) 2023-08-27T20:54:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3060858068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3060858068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3060858068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:45.016 INFO:tasks.rados.rados.0.smithi093.stdout:3151: done (5 left) 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3152: done (4 left) 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3153: done (3 left) 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3154: done (2 left) 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3155: done (1 left) 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3156: done (0 left) 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3157: read oid 90 snap -1 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3157: expect (ObjNum 1851 snap 0 seq_num 1851) 2023-08-27T20:54:45.017 INFO:tasks.rados.rados.0.smithi093.stdout:3158: delete oid 451 current snap is 0 2023-08-27T20:54:45.018 INFO:tasks.rados.rados.0.smithi093.stdout:3158: done (1 left) 2023-08-27T20:54:45.018 INFO:tasks.rados.rados.0.smithi093.stdout:3159: write oid 174 current snap is 0 2023-08-27T20:54:45.018 INFO:tasks.rados.rados.0.smithi093.stdout:3159: seq_num 1988 ranges {612316=763145,2082791=789687} 2023-08-27T20:54:45.026 INFO:tasks.rados.rados.0.smithi093.stdout:3159: writing smithi093153278-174 from 612316 to 1375461 tid 1 2023-08-27T20:54:45.034 INFO:tasks.rados.rados.0.smithi093.stdout:3159: writing smithi093153278-174 from 2082791 to 2872478 tid 2 2023-08-27T20:54:45.035 INFO:tasks.rados.rados.0.smithi093.stdout:3160: read oid 335 snap -1 2023-08-27T20:54:45.035 INFO:tasks.rados.rados.0.smithi093.stdout:3160: expect deleted 2023-08-27T20:54:45.035 INFO:tasks.rados.rados.0.smithi093.stdout:3161: delete oid 475 current snap is 0 2023-08-27T20:54:45.035 INFO:tasks.rados.rados.0.smithi093.stdout:3157: expect (ObjNum 1851 snap 0 seq_num 1851) 2023-08-27T20:54:45.132 INFO:tasks.rados.rados.0.smithi093.stdout:3159: finishing write tid 1 to smithi093153278-174 2023-08-27T20:54:45.133 INFO:tasks.rados.rados.0.smithi093.stdout:3157: done (3 left) 2023-08-27T20:54:45.133 INFO:tasks.rados.rados.0.smithi093.stdout:3161: done (2 left) 2023-08-27T20:54:45.133 INFO:tasks.rados.rados.0.smithi093.stdout:3162: read oid 180 snap -1 2023-08-27T20:54:45.133 INFO:tasks.rados.rados.0.smithi093.stdout:3162: expect (ObjNum 1653 snap 0 seq_num 1653) 2023-08-27T20:54:45.133 INFO:tasks.rados.rados.0.smithi093.stdout:3163: write oid 234 current snap is 0 2023-08-27T20:54:45.133 INFO:tasks.rados.rados.0.smithi093.stdout:3163: seq_num 1990 ranges {645930=513643,1683402=793356,2981638=464292} 2023-08-27T20:54:45.138 INFO:tasks.rados.rados.0.smithi093.stdout:3163: writing smithi093153278-234 from 645930 to 1159573 tid 1 2023-08-27T20:54:45.146 INFO:tasks.rados.rados.0.smithi093.stdout:3163: writing smithi093153278-234 from 1683402 to 2476758 tid 2 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3163: writing smithi093153278-234 from 2981638 to 3445930 tid 3 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3160: done (3 left) 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3164: write oid 239 current snap is 0 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3159: finishing write tid 2 to smithi093153278-174 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3159: finishing write tid 3 to smithi093153278-174 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3159: finishing write tid 4 to smithi093153278-174 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 174 v 367 (ObjNum 1988 snap 0 seq_num 1988) dirty exists 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3159: left oid 174 (ObjNum 1988 snap 0 seq_num 1988) 2023-08-27T20:54:45.152 INFO:tasks.rados.rados.0.smithi093.stdout:3162: expect (ObjNum 1653 snap 0 seq_num 1653) 2023-08-27T20:54:45.448 INFO:tasks.rados.rados.0.smithi093.stdout:3163: finishing write tid 1 to smithi093153278-234 2023-08-27T20:54:45.448 INFO:tasks.rados.rados.0.smithi093.stdout:3163: finishing write tid 2 to smithi093153278-234 2023-08-27T20:54:45.448 INFO:tasks.rados.rados.0.smithi093.stdout:3163: finishing write tid 3 to smithi093153278-234 2023-08-27T20:54:45.449 INFO:tasks.rados.rados.0.smithi093.stdout:3163: finishing write tid 4 to smithi093153278-234 2023-08-27T20:54:45.449 INFO:tasks.rados.rados.0.smithi093.stdout:3163: finishing write tid 5 to smithi093153278-234 2023-08-27T20:54:45.449 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 234 v 373 (ObjNum 1990 snap 0 seq_num 1990) dirty exists 2023-08-27T20:54:45.449 INFO:tasks.rados.rados.0.smithi093.stdout:3163: left oid 234 (ObjNum 1990 snap 0 seq_num 1990) 2023-08-27T20:54:45.449 INFO:tasks.rados.rados.0.smithi093.stdout:3164: seq_num 1991 ranges {662737=588892,1741884=653367,3009061=453676} 2023-08-27T20:54:45.454 INFO:tasks.rados.rados.0.smithi093.stdout:3164: writing smithi093153278-239 from 662737 to 1251629 tid 1 2023-08-27T20:54:45.461 INFO:tasks.rados.rados.0.smithi093.stdout:3164: writing smithi093153278-239 from 1741884 to 2395251 tid 2 2023-08-27T20:54:45.467 INFO:tasks.rados.rados.0.smithi093.stdout:3164: writing smithi093153278-239 from 3009061 to 3462737 tid 3 2023-08-27T20:54:45.468 INFO:tasks.rados.rados.0.smithi093.stdout:3159: done (3 left) 2023-08-27T20:54:45.468 INFO:tasks.rados.rados.0.smithi093.stdout:3162: done (2 left) 2023-08-27T20:54:45.468 INFO:tasks.rados.rados.0.smithi093.stdout:3163: done (1 left) 2023-08-27T20:54:45.468 INFO:tasks.rados.rados.0.smithi093.stdout:3165: read oid 286 snap -1 2023-08-27T20:54:45.468 INFO:tasks.rados.rados.0.smithi093.stdout:3165: expect (ObjNum 1933 snap 0 seq_num 1933) 2023-08-27T20:54:45.468 INFO:tasks.rados.rados.0.smithi093.stdout:3166: delete oid 284 current snap is 0 2023-08-27T20:54:45.468 INFO:tasks.rados.rados.0.smithi093.stdout:3164: finishing write tid 1 to smithi093153278-239 2023-08-27T20:54:45.469 INFO:tasks.rados.rados.0.smithi093.stdout:3164: finishing write tid 2 to smithi093153278-239 2023-08-27T20:54:45.471 INFO:tasks.rados.rados.0.smithi093.stdout:3165: expect (ObjNum 1933 snap 0 seq_num 1933) 2023-08-27T20:54:45.579 INFO:tasks.rados.rados.0.smithi093.stdout:3164: finishing write tid 3 to smithi093153278-239 2023-08-27T20:54:45.579 INFO:tasks.rados.rados.0.smithi093.stdout:3164: finishing write tid 4 to smithi093153278-239 2023-08-27T20:54:45.579 INFO:tasks.rados.rados.0.smithi093.stdout:3164: finishing write tid 5 to smithi093153278-239 2023-08-27T20:54:45.580 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 239 v 371 (ObjNum 1991 snap 0 seq_num 1991) dirty exists 2023-08-27T20:54:45.580 INFO:tasks.rados.rados.0.smithi093.stdout:3164: left oid 239 (ObjNum 1991 snap 0 seq_num 1991) 2023-08-27T20:54:45.580 INFO:tasks.rados.rados.0.smithi093.stdout:3164: done (2 left) 2023-08-27T20:54:45.580 INFO:tasks.rados.rados.0.smithi093.stdout:3165: done (1 left) 2023-08-27T20:54:45.580 INFO:tasks.rados.rados.0.smithi093.stdout:3166: done (0 left) 2023-08-27T20:54:45.580 INFO:tasks.rados.rados.0.smithi093.stdout:3167: delete oid 166 current snap is 0 2023-08-27T20:54:45.581 INFO:tasks.rados.rados.0.smithi093.stdout:3167: done (0 left) 2023-08-27T20:54:45.581 INFO:tasks.rados.rados.0.smithi093.stdout:3168: write oid 375 current snap is 0 2023-08-27T20:54:45.581 INFO:tasks.rados.rados.0.smithi093.stdout:3168: seq_num 1994 ranges {713158=530992,1917330=400694,3091330=406276} 2023-08-27T20:54:45.587 INFO:tasks.rados.rados.0.smithi093.stdout:3168: writing smithi093153278-375 from 713158 to 1244150 tid 1 2023-08-27T20:54:45.593 INFO:tasks.rados.rados.0.smithi093.stdout:3168: writing smithi093153278-375 from 1917330 to 2318024 tid 2 2023-08-27T20:54:45.600 INFO:tasks.rados.rados.0.smithi093.stdout:3168: writing smithi093153278-375 from 3091330 to 3497606 tid 3 2023-08-27T20:54:45.600 INFO:tasks.rados.rados.0.smithi093.stdout:3169: write oid 84 current snap is 0 2023-08-27T20:54:45.600 INFO:tasks.rados.rados.0.smithi093.stdout:3168: finishing write tid 1 to smithi093153278-375 2023-08-27T20:54:45.600 INFO:tasks.rados.rados.0.smithi093.stdout:3168: finishing write tid 2 to smithi093153278-375 2023-08-27T20:54:45.601 INFO:tasks.rados.rados.0.smithi093.stdout:3169: seq_num 1995 ranges {729965=606241,1975812=660705,3118753=411212} 2023-08-27T20:54:45.607 INFO:tasks.rados.rados.0.smithi093.stdout:3169: writing smithi093153278-84 from 729965 to 1336206 tid 1 2023-08-27T20:54:45.614 INFO:tasks.rados.rados.0.smithi093.stdout:3169: writing smithi093153278-84 from 1975812 to 2636517 tid 2 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3169: writing smithi093153278-84 from 3118753 to 3529965 tid 3 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3170: read oid 382 snap -1 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3170: expect (ObjNum 1502 snap 0 seq_num 1502) 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3171: write oid 349 current snap is 0 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3168: finishing write tid 3 to smithi093153278-375 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3168: finishing write tid 4 to smithi093153278-375 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3168: finishing write tid 5 to smithi093153278-375 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 375 v 393 (ObjNum 1994 snap 0 seq_num 1994) dirty exists 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3168: left oid 375 (ObjNum 1994 snap 0 seq_num 1994) 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3169: finishing write tid 1 to smithi093153278-84 2023-08-27T20:54:45.620 INFO:tasks.rados.rados.0.smithi093.stdout:3169: finishing write tid 2 to smithi093153278-84 2023-08-27T20:54:45.621 INFO:tasks.rados.rados.0.smithi093.stdout:3171: seq_num 1996 ranges {746772=681490,2034294=404363,3146176=400596} 2023-08-27T20:54:45.627 INFO:tasks.rados.rados.0.smithi093.stdout:3171: writing smithi093153278-349 from 746772 to 1428262 tid 1 2023-08-27T20:54:45.633 INFO:tasks.rados.rados.0.smithi093.stdout:3171: writing smithi093153278-349 from 2034294 to 2438657 tid 2 2023-08-27T20:54:45.639 INFO:tasks.rados.rados.0.smithi093.stdout:3171: writing smithi093153278-349 from 3146176 to 3546772 tid 3 2023-08-27T20:54:45.639 INFO:tasks.rados.rados.0.smithi093.stdout:3168: done (3 left) 2023-08-27T20:54:45.639 INFO:tasks.rados.rados.0.smithi093.stdout:3172: write oid 81 current snap is 0 2023-08-27T20:54:45.639 INFO:tasks.rados.rados.0.smithi093.stdout:3169: finishing write tid 3 to smithi093153278-84 2023-08-27T20:54:45.639 INFO:tasks.rados.rados.0.smithi093.stdout:3172: seq_num 1997 ranges {763579=756739,1976423=548021,2940893=622686} 2023-08-27T20:54:45.647 INFO:tasks.rados.rados.0.smithi093.stdout:3172: writing smithi093153278-81 from 763579 to 1520318 tid 1 2023-08-27T20:54:45.653 INFO:tasks.rados.rados.0.smithi093.stdout:3172: writing smithi093153278-81 from 1976423 to 2524444 tid 2 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3172: writing smithi093153278-81 from 2940893 to 3563579 tid 3 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3173: read oid 424 snap -1 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3173: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3174: read oid 61 snap -1 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3174: expect (ObjNum 1905 snap 0 seq_num 1905) 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3175: write oid 249 current snap is 0 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3169: finishing write tid 4 to smithi093153278-84 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:3169: finishing write tid 5 to smithi093153278-84 2023-08-27T20:54:45.660 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 84 v 437 (ObjNum 1995 snap 0 seq_num 1995) dirty exists 2023-08-27T20:54:45.661 INFO:tasks.rados.rados.0.smithi093.stdout:3169: left oid 84 (ObjNum 1995 snap 0 seq_num 1995) 2023-08-27T20:54:45.661 INFO:tasks.rados.rados.0.smithi093.stdout:3170: expect (ObjNum 1502 snap 0 seq_num 1502) 2023-08-27T20:54:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:45 smithi093 ceph-mon[111335]: pgmap v36: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.3 MiB/s wr, 75 op/s 2023-08-27T20:54:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:45 smithi093 ceph-mon[118902]: pgmap v36: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.3 MiB/s wr, 75 op/s 2023-08-27T20:54:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:54:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:54:45] "GET /metrics HTTP/1.1" 200 242576 "" "Prometheus/2.43.0" 2023-08-27T20:54:45.838 INFO:tasks.rados.rados.0.smithi093.stdout:3171: finishing write tid 1 to smithi093153278-349 2023-08-27T20:54:45.838 INFO:tasks.rados.rados.0.smithi093.stdout:3171: finishing write tid 2 to smithi093153278-349 2023-08-27T20:54:45.838 INFO:tasks.rados.rados.0.smithi093.stdout:3171: finishing write tid 3 to smithi093153278-349 2023-08-27T20:54:45.838 INFO:tasks.rados.rados.0.smithi093.stdout:3175: seq_num 1998 ranges {780386=431988,1634905=408032,2684669=567717} 2023-08-27T20:54:45.845 INFO:tasks.rados.rados.0.smithi093.stdout:3175: writing smithi093153278-249 from 780386 to 1212374 tid 1 2023-08-27T20:54:45.849 INFO:tasks.rados.rados.0.smithi093.stdout:3175: writing smithi093153278-249 from 1634905 to 2042937 tid 2 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3175: writing smithi093153278-249 from 2684669 to 3252386 tid 3 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3169: done (6 left) 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3170: done (5 left) 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3176: read oid 338 snap -1 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3176: expect (ObjNum 1956 snap 0 seq_num 1956) 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3177: write oid 126 current snap is 0 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3171: finishing write tid 4 to smithi093153278-349 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:3171: finishing write tid 5 to smithi093153278-349 2023-08-27T20:54:45.857 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 349 v 472 (ObjNum 1996 snap 0 seq_num 1996) dirty exists 2023-08-27T20:54:45.858 INFO:tasks.rados.rados.0.smithi093.stdout:3171: left oid 349 (ObjNum 1996 snap 0 seq_num 1996) 2023-08-27T20:54:45.858 INFO:tasks.rados.rados.0.smithi093.stdout:3172: finishing write tid 1 to smithi093153278-81 2023-08-27T20:54:45.858 INFO:tasks.rados.rados.0.smithi093.stdout:3172: finishing write tid 2 to smithi093153278-81 2023-08-27T20:54:45.858 INFO:tasks.rados.rados.0.smithi093.stdout:3172: finishing write tid 3 to smithi093153278-81 2023-08-27T20:54:45.858 INFO:tasks.rados.rados.0.smithi093.stdout:3172: finishing write tid 4 to smithi093153278-81 2023-08-27T20:54:45.858 INFO:tasks.rados.rados.0.smithi093.stdout:3177: seq_num 1999 ranges {797193=507237,2093387=551690,3112092=485101} 2023-08-27T20:54:45.864 INFO:tasks.rados.rados.0.smithi093.stdout:3177: writing smithi093153278-126 from 797193 to 1304430 tid 1 2023-08-27T20:54:45.871 INFO:tasks.rados.rados.0.smithi093.stdout:3177: writing smithi093153278-126 from 2093387 to 2645077 tid 2 2023-08-27T20:54:45.877 INFO:tasks.rados.rados.0.smithi093.stdout:3177: writing smithi093153278-126 from 3112092 to 3597193 tid 3 2023-08-27T20:54:45.877 INFO:tasks.rados.rados.0.smithi093.stdout:3171: done (6 left) 2023-08-27T20:54:45.877 INFO:tasks.rados.rados.0.smithi093.stdout:3178: delete oid 356 current snap is 0 2023-08-27T20:54:45.877 INFO:tasks.rados.rados.0.smithi093.stdout:3172: finishing write tid 5 to smithi093153278-81 2023-08-27T20:54:45.877 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 535 (ObjNum 1997 snap 0 seq_num 1997) dirty exists 2023-08-27T20:54:45.877 INFO:tasks.rados.rados.0.smithi093.stdout:3172: left oid 81 (ObjNum 1997 snap 0 seq_num 1997) 2023-08-27T20:54:45.878 INFO:tasks.rados.rados.0.smithi093.stdout:3173: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:54:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:45 smithi136 ceph-mon[113531]: pgmap v36: 89 pgs: 89 active+clean; 1.2 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 7.3 MiB/s wr, 75 op/s 2023-08-27T20:54:46.056 INFO:tasks.rados.rados.0.smithi093.stdout:3174: expect (ObjNum 1905 snap 0 seq_num 1905) 2023-08-27T20:54:46.148 INFO:tasks.rados.rados.0.smithi093.stdout:3175: finishing write tid 1 to smithi093153278-249 2023-08-27T20:54:46.148 INFO:tasks.rados.rados.0.smithi093.stdout:3172: done (6 left) 2023-08-27T20:54:46.148 INFO:tasks.rados.rados.0.smithi093.stdout:3173: done (5 left) 2023-08-27T20:54:46.148 INFO:tasks.rados.rados.0.smithi093.stdout:3174: done (4 left) 2023-08-27T20:54:46.148 INFO:tasks.rados.rados.0.smithi093.stdout:3178: done (3 left) 2023-08-27T20:54:46.148 INFO:tasks.rados.rados.0.smithi093.stdout:3179: write oid 291 current snap is 0 2023-08-27T20:54:46.149 INFO:tasks.rados.rados.0.smithi093.stdout:3175: finishing write tid 2 to smithi093153278-249 2023-08-27T20:54:46.149 INFO:tasks.rados.rados.0.smithi093.stdout:3176: expect (ObjNum 1956 snap 0 seq_num 1956) 2023-08-27T20:54:46.300 INFO:tasks.rados.rados.0.smithi093.stdout:3175: finishing write tid 3 to smithi093153278-249 2023-08-27T20:54:46.301 INFO:tasks.rados.rados.0.smithi093.stdout:3175: finishing write tid 4 to smithi093153278-249 2023-08-27T20:54:46.301 INFO:tasks.rados.rados.0.smithi093.stdout:3175: finishing write tid 5 to smithi093153278-249 2023-08-27T20:54:46.301 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 249 v 441 (ObjNum 1998 snap 0 seq_num 1998) dirty exists 2023-08-27T20:54:46.301 INFO:tasks.rados.rados.0.smithi093.stdout:3175: left oid 249 (ObjNum 1998 snap 0 seq_num 1998) 2023-08-27T20:54:46.301 INFO:tasks.rados.rados.0.smithi093.stdout:3177: finishing write tid 1 to smithi093153278-126 2023-08-27T20:54:46.301 INFO:tasks.rados.rados.0.smithi093.stdout:3179: seq_num 2001 ranges {430807=774088,1810351=555359,2766938=517886} 2023-08-27T20:54:46.307 INFO:tasks.rados.rados.0.smithi093.stdout:3179: writing smithi093153278-291 from 430807 to 1204895 tid 1 2023-08-27T20:54:46.313 INFO:tasks.rados.rados.0.smithi093.stdout:3179: writing smithi093153278-291 from 1810351 to 2365710 tid 2 2023-08-27T20:54:46.319 INFO:tasks.rados.rados.0.smithi093.stdout:3179: writing smithi093153278-291 from 2766938 to 3284824 tid 3 2023-08-27T20:54:46.319 INFO:tasks.rados.rados.0.smithi093.stdout:3175: done (3 left) 2023-08-27T20:54:46.319 INFO:tasks.rados.rados.0.smithi093.stdout:3176: done (2 left) 2023-08-27T20:54:46.319 INFO:tasks.rados.rados.0.smithi093.stdout:3180: write oid 258 current snap is 0 2023-08-27T20:54:46.319 INFO:tasks.rados.rados.0.smithi093.stdout:3177: finishing write tid 2 to smithi093153278-126 2023-08-27T20:54:46.319 INFO:tasks.rados.rados.0.smithi093.stdout:3177: finishing write tid 3 to smithi093153278-126 2023-08-27T20:54:46.319 INFO:tasks.rados.rados.0.smithi093.stdout:3177: finishing write tid 4 to smithi093153278-126 2023-08-27T20:54:46.320 INFO:tasks.rados.rados.0.smithi093.stdout:3177: finishing write tid 5 to smithi093153278-126 2023-08-27T20:54:46.320 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 377 (ObjNum 1999 snap 0 seq_num 1999) dirty exists 2023-08-27T20:54:46.320 INFO:tasks.rados.rados.0.smithi093.stdout:3177: left oid 126 (ObjNum 1999 snap 0 seq_num 1999) 2023-08-27T20:54:46.320 INFO:tasks.rados.rados.0.smithi093.stdout:3180: seq_num 2002 ranges {447614=449337,1468833=415370,2394361=729158,3603789=43825} 2023-08-27T20:54:46.323 INFO:tasks.rados.rados.0.smithi093.stdout:3180: writing smithi093153278-258 from 447614 to 896951 tid 1 2023-08-27T20:54:46.329 INFO:tasks.rados.rados.0.smithi093.stdout:3180: writing smithi093153278-258 from 1468833 to 1884203 tid 2 2023-08-27T20:54:46.336 INFO:tasks.rados.rados.0.smithi093.stdout:3180: writing smithi093153278-258 from 2394361 to 3123519 tid 3 2023-08-27T20:54:46.339 INFO:tasks.rados.rados.0.smithi093.stdout:3180: writing smithi093153278-258 from 3603789 to 3647614 tid 4 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3177: done (2 left) 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3181: delete oid 247 current snap is 0 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3179: finishing write tid 1 to smithi093153278-291 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3179: finishing write tid 2 to smithi093153278-291 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3179: finishing write tid 3 to smithi093153278-291 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3179: finishing write tid 4 to smithi093153278-291 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3179: finishing write tid 5 to smithi093153278-291 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 291 v 427 (ObjNum 2001 snap 0 seq_num 2001) dirty exists 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3179: left oid 291 (ObjNum 2001 snap 0 seq_num 2001) 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3180: finishing write tid 1 to smithi093153278-258 2023-08-27T20:54:46.340 INFO:tasks.rados.rados.0.smithi093.stdout:3180: finishing write tid 2 to smithi093153278-258 2023-08-27T20:54:46.341 INFO:tasks.rados.rados.0.smithi093.stdout:3180: finishing write tid 3 to smithi093153278-258 2023-08-27T20:54:46.341 INFO:tasks.rados.rados.0.smithi093.stdout:3179: done (2 left) 2023-08-27T20:54:46.341 INFO:tasks.rados.rados.0.smithi093.stdout:3181: done (1 left) 2023-08-27T20:54:46.342 INFO:tasks.rados.rados.0.smithi093.stdout:3182: write oid 338 current snap is 0 2023-08-27T20:54:46.342 INFO:tasks.rados.rados.0.smithi093.stdout:3182: seq_num 2004 ranges {481228=599835,1585797=419039,2449207=751702} 2023-08-27T20:54:46.346 INFO:tasks.rados.rados.0.smithi093.stdout:3182: writing smithi093153278-338 from 481228 to 1081063 tid 1 2023-08-27T20:54:46.351 INFO:tasks.rados.rados.0.smithi093.stdout:3182: writing smithi093153278-338 from 1585797 to 2004836 tid 2 2023-08-27T20:54:46.358 INFO:tasks.rados.rados.0.smithi093.stdout:3182: writing smithi093153278-338 from 2449207 to 3200909 tid 3 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3183: read oid 30 snap -1 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3183: expect deleted 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3184: delete oid 266 current snap is 0 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3180: finishing write tid 4 to smithi093153278-258 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3180: finishing write tid 5 to smithi093153278-258 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3180: finishing write tid 6 to smithi093153278-258 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 258 v 351 (ObjNum 2002 snap 0 seq_num 2002) dirty exists 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3180: left oid 258 (ObjNum 2002 snap 0 seq_num 2002) 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3182: finishing write tid 1 to smithi093153278-338 2023-08-27T20:54:46.359 INFO:tasks.rados.rados.0.smithi093.stdout:3182: finishing write tid 2 to smithi093153278-338 2023-08-27T20:54:46.360 INFO:tasks.rados.rados.0.smithi093.stdout:3180: done (3 left) 2023-08-27T20:54:46.360 INFO:tasks.rados.rados.0.smithi093.stdout:3183: done (2 left) 2023-08-27T20:54:46.360 INFO:tasks.rados.rados.0.smithi093.stdout:3184: done (1 left) 2023-08-27T20:54:46.361 INFO:tasks.rados.rados.0.smithi093.stdout:3185: read oid 300 snap -1 2023-08-27T20:54:46.361 INFO:tasks.rados.rados.0.smithi093.stdout:3185: expect (ObjNum 1958 snap 0 seq_num 1958) 2023-08-27T20:54:46.361 INFO:tasks.rados.rados.0.smithi093.stdout:3186: read oid 280 snap -1 2023-08-27T20:54:46.361 INFO:tasks.rados.rados.0.smithi093.stdout:3186: expect (ObjNum 1801 snap 0 seq_num 1801) 2023-08-27T20:54:46.361 INFO:tasks.rados.rados.0.smithi093.stdout:3187: write oid 271 current snap is 0 2023-08-27T20:54:46.361 INFO:tasks.rados.rados.0.smithi093.stdout:3187: seq_num 2006 ranges {514842=750333,1986408=422708,3187700=490599} 2023-08-27T20:54:46.367 INFO:tasks.rados.rados.0.smithi093.stdout:3187: writing smithi093153278-271 from 514842 to 1265175 tid 1 2023-08-27T20:54:46.373 INFO:tasks.rados.rados.0.smithi093.stdout:3187: writing smithi093153278-271 from 1986408 to 2409116 tid 2 2023-08-27T20:54:46.380 INFO:tasks.rados.rados.0.smithi093.stdout:3187: writing smithi093153278-271 from 3187700 to 3678299 tid 3 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3188: read oid 135 snap -1 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3188: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3189: read oid 132 snap -1 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3189: expect (ObjNum 1862 snap 0 seq_num 1862) 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3190: delete oid 309 current snap is 0 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3182: finishing write tid 3 to smithi093153278-338 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3182: finishing write tid 4 to smithi093153278-338 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3182: finishing write tid 5 to smithi093153278-338 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 338 v 397 (ObjNum 2004 snap 0 seq_num 2004) dirty exists 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3182: left oid 338 (ObjNum 2004 snap 0 seq_num 2004) 2023-08-27T20:54:46.381 INFO:tasks.rados.rados.0.smithi093.stdout:3185: expect (ObjNum 1958 snap 0 seq_num 1958) 2023-08-27T20:54:46.501 INFO:tasks.rados.rados.0.smithi093.stdout:3187: finishing write tid 1 to smithi093153278-271 2023-08-27T20:54:46.501 INFO:tasks.rados.rados.0.smithi093.stdout:3186: expect (ObjNum 1801 snap 0 seq_num 1801) 2023-08-27T20:54:46.619 INFO:tasks.rados.rados.0.smithi093.stdout:3187: finishing write tid 2 to smithi093153278-271 2023-08-27T20:54:46.619 INFO:tasks.rados.rados.0.smithi093.stdout:3182: done (6 left) 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3185: done (5 left) 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3186: done (4 left) 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3190: done (3 left) 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3191: delete oid 242 current snap is 0 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3187: finishing write tid 3 to smithi093153278-271 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3187: finishing write tid 4 to smithi093153278-271 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3187: finishing write tid 5 to smithi093153278-271 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 271 v 331 (ObjNum 2006 snap 0 seq_num 2006) dirty exists 2023-08-27T20:54:46.620 INFO:tasks.rados.rados.0.smithi093.stdout:3187: left oid 271 (ObjNum 2006 snap 0 seq_num 2006) 2023-08-27T20:54:46.621 INFO:tasks.rados.rados.0.smithi093.stdout:3189: expect (ObjNum 1862 snap 0 seq_num 1862) 2023-08-27T20:54:46.782 INFO:tasks.rados.rados.0.smithi093.stdout:3188: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:54:46.932 INFO:tasks.rados.rados.0.smithi093.stdout:3187: done (3 left) 2023-08-27T20:54:46.932 INFO:tasks.rados.rados.0.smithi093.stdout:3188: done (2 left) 2023-08-27T20:54:46.932 INFO:tasks.rados.rados.0.smithi093.stdout:3189: done (1 left) 2023-08-27T20:54:46.932 INFO:tasks.rados.rados.0.smithi093.stdout:3191: done (0 left) 2023-08-27T20:54:46.933 INFO:tasks.rados.rados.0.smithi093.stdout:3192: write oid 427 current snap is 0 2023-08-27T20:54:46.933 INFO:tasks.rados.rados.0.smithi093.stdout:3192: seq_num 2009 ranges {565263=692433,1878207=570035,2986322=724415} 2023-08-27T20:54:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1270897169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:46.939 INFO:tasks.rados.rados.0.smithi093.stdout:3192: writing smithi093153278-427 from 565263 to 1257696 tid 1 2023-08-27T20:54:46.944 INFO:tasks.rados.rados.0.smithi093.stdout:3192: writing smithi093153278-427 from 1878207 to 2448242 tid 2 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3192: writing smithi093153278-427 from 2986322 to 3710737 tid 3 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3193: read oid 269 snap -1 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3193: expect (ObjNum 1906 snap 0 seq_num 1906) 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3194: read oid 331 snap -1 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3194: expect (ObjNum 1779 snap 0 seq_num 1779) 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3195: write oid 454 current snap is 0 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3192: finishing write tid 1 to smithi093153278-427 2023-08-27T20:54:46.952 INFO:tasks.rados.rados.0.smithi093.stdout:3192: finishing write tid 2 to smithi093153278-427 2023-08-27T20:54:46.953 INFO:tasks.rados.rados.0.smithi093.stdout:3195: seq_num 2010 ranges {582070=767682,1820336=713693,3181039=601031} 2023-08-27T20:54:46.959 INFO:tasks.rados.rados.0.smithi093.stdout:3195: writing smithi093153278-454 from 582070 to 1349752 tid 1 2023-08-27T20:54:46.966 INFO:tasks.rados.rados.0.smithi093.stdout:3195: writing smithi093153278-454 from 1820336 to 2534029 tid 2 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:3195: writing smithi093153278-454 from 3181039 to 3782070 tid 3 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:3196: write oid 106 current snap is 0 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:3192: finishing write tid 3 to smithi093153278-427 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:3192: finishing write tid 4 to smithi093153278-427 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:3192: finishing write tid 5 to smithi093153278-427 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 427 v 363 (ObjNum 2009 snap 0 seq_num 2009) dirty exists 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:3192: left oid 427 (ObjNum 2009 snap 0 seq_num 2009) 2023-08-27T20:54:46.974 INFO:tasks.rados.rados.0.smithi093.stdout:3194: expect (ObjNum 1779 snap 0 seq_num 1779) 2023-08-27T20:54:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1270897169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1270897169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:47.222 INFO:tasks.rados.rados.0.smithi093.stdout:3193: expect (ObjNum 1906 snap 0 seq_num 1906) 2023-08-27T20:54:47.315 INFO:tasks.rados.rados.0.smithi093.stdout:3195: finishing write tid 1 to smithi093153278-454 2023-08-27T20:54:47.315 INFO:tasks.rados.rados.0.smithi093.stdout:3195: finishing write tid 2 to smithi093153278-454 2023-08-27T20:54:47.315 INFO:tasks.rados.rados.0.smithi093.stdout:3196: seq_num 2011 ranges {598877=442931,1478818=573704,2524815=463312,3516293=282584} 2023-08-27T20:54:47.320 INFO:tasks.rados.rados.0.smithi093.stdout:3196: writing smithi093153278-106 from 598877 to 1041808 tid 1 2023-08-27T20:54:47.326 INFO:tasks.rados.rados.0.smithi093.stdout:3196: writing smithi093153278-106 from 1478818 to 2052522 tid 2 2023-08-27T20:54:47.331 INFO:tasks.rados.rados.0.smithi093.stdout:3196: writing smithi093153278-106 from 2524815 to 2988127 tid 3 2023-08-27T20:54:47.336 INFO:tasks.rados.rados.0.smithi093.stdout:3196: writing smithi093153278-106 from 3516293 to 3798877 tid 4 2023-08-27T20:54:47.336 INFO:tasks.rados.rados.0.smithi093.stdout:3192: done (4 left) 2023-08-27T20:54:47.336 INFO:tasks.rados.rados.0.smithi093.stdout:3193: done (3 left) 2023-08-27T20:54:47.336 INFO:tasks.rados.rados.0.smithi093.stdout:3194: done (2 left) 2023-08-27T20:54:47.336 INFO:tasks.rados.rados.0.smithi093.stdout:3197: read oid 309 snap -1 2023-08-27T20:54:47.337 INFO:tasks.rados.rados.0.smithi093.stdout:3197: expect deleted 2023-08-27T20:54:47.337 INFO:tasks.rados.rados.0.smithi093.stdout:3198: read oid 72 snap -1 2023-08-27T20:54:47.337 INFO:tasks.rados.rados.0.smithi093.stdout:3198: expect (ObjNum 1885 snap 0 seq_num 1885) 2023-08-27T20:54:47.337 INFO:tasks.rados.rados.0.smithi093.stdout:3199: read oid 455 snap -1 2023-08-27T20:54:47.337 INFO:tasks.rados.rados.0.smithi093.stdout:3199: expect (ObjNum 1670 snap 0 seq_num 1670) 2023-08-27T20:54:47.337 INFO:tasks.rados.rados.0.smithi093.stdout:3200: write oid 365 current snap is 0 2023-08-27T20:54:47.337 INFO:tasks.rados.rados.0.smithi093.stdout:3195: finishing write tid 3 to smithi093153278-454 2023-08-27T20:54:47.338 INFO:tasks.rados.rados.0.smithi093.stdout:3195: finishing write tid 4 to smithi093153278-454 2023-08-27T20:54:47.338 INFO:tasks.rados.rados.0.smithi093.stdout:3195: finishing write tid 5 to smithi093153278-454 2023-08-27T20:54:47.338 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 454 v 430 (ObjNum 2010 snap 0 seq_num 2010) dirty exists 2023-08-27T20:54:47.338 INFO:tasks.rados.rados.0.smithi093.stdout:3195: left oid 454 (ObjNum 2010 snap 0 seq_num 2010) 2023-08-27T20:54:47.338 INFO:tasks.rados.rados.0.smithi093.stdout:3196: finishing write tid 1 to smithi093153278-106 2023-08-27T20:54:47.338 INFO:tasks.rados.rados.0.smithi093.stdout:3196: finishing write tid 2 to smithi093153278-106 2023-08-27T20:54:47.338 INFO:tasks.rados.rados.0.smithi093.stdout:3196: finishing write tid 3 to smithi093153278-106 2023-08-27T20:54:47.339 INFO:tasks.rados.rados.0.smithi093.stdout:3200: seq_num 2012 ranges {615684=518180,1537300=717362,2835885=674584} 2023-08-27T20:54:47.342 INFO:tasks.rados.rados.0.smithi093.stdout:3200: writing smithi093153278-365 from 615684 to 1133864 tid 1 2023-08-27T20:54:47.348 INFO:tasks.rados.rados.0.smithi093.stdout:3200: writing smithi093153278-365 from 1537300 to 2254662 tid 2 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3200: writing smithi093153278-365 from 2835885 to 3510469 tid 3 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3195: done (5 left) 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3201: delete oid 398 current snap is 0 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3196: finishing write tid 4 to smithi093153278-106 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3196: finishing write tid 5 to smithi093153278-106 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3196: finishing write tid 6 to smithi093153278-106 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 106 v 382 (ObjNum 2011 snap 0 seq_num 2011) dirty exists 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3196: left oid 106 (ObjNum 2011 snap 0 seq_num 2011) 2023-08-27T20:54:47.356 INFO:tasks.rados.rados.0.smithi093.stdout:3199: expect (ObjNum 1670 snap 0 seq_num 1670) 2023-08-27T20:54:47.481 INFO:tasks.rados.rados.0.smithi093.stdout:3200: finishing write tid 1 to smithi093153278-365 2023-08-27T20:54:47.481 INFO:tasks.rados.rados.0.smithi093.stdout:3198: expect (ObjNum 1885 snap 0 seq_num 1885) 2023-08-27T20:54:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:3200: finishing write tid 2 to smithi093153278-365 2023-08-27T20:54:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:3200: finishing write tid 3 to smithi093153278-365 2023-08-27T20:54:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:3200: finishing write tid 4 to smithi093153278-365 2023-08-27T20:54:47.721 INFO:tasks.rados.rados.0.smithi093.stdout:3200: finishing write tid 5 to smithi093153278-365 2023-08-27T20:54:47.722 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 365 v 489 (ObjNum 2012 snap 0 seq_num 2012) dirty exists 2023-08-27T20:54:47.722 INFO:tasks.rados.rados.0.smithi093.stdout:3200: left oid 365 (ObjNum 2012 snap 0 seq_num 2012) 2023-08-27T20:54:47.722 INFO:tasks.rados.rados.0.smithi093.stdout:3196: done (5 left) 2023-08-27T20:54:47.722 INFO:tasks.rados.rados.0.smithi093.stdout:3197: done (4 left) 2023-08-27T20:54:47.722 INFO:tasks.rados.rados.0.smithi093.stdout:3198: done (3 left) 2023-08-27T20:54:47.722 INFO:tasks.rados.rados.0.smithi093.stdout:3199: done (2 left) 2023-08-27T20:54:47.723 INFO:tasks.rados.rados.0.smithi093.stdout:3200: done (1 left) 2023-08-27T20:54:47.723 INFO:tasks.rados.rados.0.smithi093.stdout:3201: done (0 left) 2023-08-27T20:54:47.723 INFO:tasks.rados.rados.0.smithi093.stdout:3202: write oid 182 current snap is 0 2023-08-27T20:54:47.723 INFO:tasks.rados.rados.0.smithi093.stdout:3202: seq_num 2014 ranges {649298=668678,1937911=721031,3174378=674920} 2023-08-27T20:54:47.727 INFO:tasks.rados.rados.0.smithi093.stdout:3202: writing smithi093153278-182 from 649298 to 1317976 tid 1 2023-08-27T20:54:47.735 INFO:tasks.rados.rados.0.smithi093.stdout:3202: writing smithi093153278-182 from 1937911 to 2658942 tid 2 2023-08-27T20:54:47.742 INFO:tasks.rados.rados.0.smithi093.stdout:3202: writing smithi093153278-182 from 3174378 to 3849298 tid 3 2023-08-27T20:54:47.742 INFO:tasks.rados.rados.0.smithi093.stdout:3203: write oid 285 current snap is 0 2023-08-27T20:54:47.742 INFO:tasks.rados.rados.0.smithi093.stdout:3202: finishing write tid 1 to smithi093153278-182 2023-08-27T20:54:47.742 INFO:tasks.rados.rados.0.smithi093.stdout:3202: finishing write tid 2 to smithi093153278-182 2023-08-27T20:54:47.742 INFO:tasks.rados.rados.0.smithi093.stdout:3203: seq_num 2015 ranges {666105=460280,1712746=581042,3034507=624753} 2023-08-27T20:54:47.748 INFO:tasks.rados.rados.0.smithi093.stdout:3203: writing smithi093153278-285 from 666105 to 1126385 tid 1 2023-08-27T20:54:47.754 INFO:tasks.rados.rados.0.smithi093.stdout:3203: writing smithi093153278-285 from 1712746 to 2293788 tid 2 2023-08-27T20:54:47.762 INFO:tasks.rados.rados.0.smithi093.stdout:3203: writing smithi093153278-285 from 3034507 to 3659260 tid 3 2023-08-27T20:54:47.762 INFO:tasks.rados.rados.0.smithi093.stdout:3204: write oid 324 current snap is 0 2023-08-27T20:54:47.762 INFO:tasks.rados.rados.0.smithi093.stdout:3202: finishing write tid 3 to smithi093153278-182 2023-08-27T20:54:47.762 INFO:tasks.rados.rados.0.smithi093.stdout:3202: finishing write tid 4 to smithi093153278-182 2023-08-27T20:54:47.762 INFO:tasks.rados.rados.0.smithi093.stdout:3202: finishing write tid 5 to smithi093153278-182 2023-08-27T20:54:47.762 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 182 v 431 (ObjNum 2014 snap 0 seq_num 2014) dirty exists 2023-08-27T20:54:47.763 INFO:tasks.rados.rados.0.smithi093.stdout:3202: left oid 182 (ObjNum 2014 snap 0 seq_num 2014) 2023-08-27T20:54:47.763 INFO:tasks.rados.rados.0.smithi093.stdout:3203: finishing write tid 1 to smithi093153278-285 2023-08-27T20:54:47.763 INFO:tasks.rados.rados.0.smithi093.stdout:3203: finishing write tid 2 to smithi093153278-285 2023-08-27T20:54:47.763 INFO:tasks.rados.rados.0.smithi093.stdout:3204: seq_num 2016 ranges {682912=535529,1771228=724700,3061930=436025} 2023-08-27T20:54:47.768 INFO:tasks.rados.rados.0.smithi093.stdout:3204: writing smithi093153278-324 from 682912 to 1218441 tid 1 2023-08-27T20:54:47.775 INFO:tasks.rados.rados.0.smithi093.stdout:3204: writing smithi093153278-324 from 1771228 to 2495928 tid 2 2023-08-27T20:54:47.781 INFO:tasks.rados.rados.0.smithi093.stdout:3204: writing smithi093153278-324 from 3061930 to 3497955 tid 3 2023-08-27T20:54:47.781 INFO:tasks.rados.rados.0.smithi093.stdout:3202: done (2 left) 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:3205: read oid 70 snap -1 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:3205: expect deleted 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:3206: delete oid 100 current snap is 0 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:3203: finishing write tid 3 to smithi093153278-285 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:3203: finishing write tid 4 to smithi093153278-285 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:3203: finishing write tid 5 to smithi093153278-285 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 285 v 446 (ObjNum 2015 snap 0 seq_num 2015) dirty exists 2023-08-27T20:54:47.782 INFO:tasks.rados.rados.0.smithi093.stdout:3203: left oid 285 (ObjNum 2015 snap 0 seq_num 2015) 2023-08-27T20:54:47.783 INFO:tasks.rados.rados.0.smithi093.stdout:3204: finishing write tid 1 to smithi093153278-324 2023-08-27T20:54:47.783 INFO:tasks.rados.rados.0.smithi093.stdout:3204: finishing write tid 2 to smithi093153278-324 2023-08-27T20:54:47.783 INFO:tasks.rados.rados.0.smithi093.stdout:3203: done (3 left) 2023-08-27T20:54:47.783 INFO:tasks.rados.rados.0.smithi093.stdout:3205: done (2 left) 2023-08-27T20:54:47.783 INFO:tasks.rados.rados.0.smithi093.stdout:3206: done (1 left) 2023-08-27T20:54:47.784 INFO:tasks.rados.rados.0.smithi093.stdout:3207: write oid 59 current snap is 0 2023-08-27T20:54:47.784 INFO:tasks.rados.rados.0.smithi093.stdout:3207: seq_num 2018 ranges {716526=686027,2171839=728369,3400423=458569} 2023-08-27T20:54:47.790 INFO:tasks.rados.rados.0.smithi093.stdout:3207: writing smithi093153278-59 from 716526 to 1402553 tid 1 2023-08-27T20:54:47.799 INFO:tasks.rados.rados.0.smithi093.stdout:3207: writing smithi093153278-59 from 2171839 to 2900208 tid 2 2023-08-27T20:54:47.804 INFO:tasks.rados.rados.0.smithi093.stdout:3207: writing smithi093153278-59 from 3400423 to 3858992 tid 3 2023-08-27T20:54:47.804 INFO:tasks.rados.rados.0.smithi093.stdout:3208: read oid 134 snap -1 2023-08-27T20:54:47.804 INFO:tasks.rados.rados.0.smithi093.stdout:3208: expect (ObjNum 1580 snap 0 seq_num 1580) 2023-08-27T20:54:47.805 INFO:tasks.rados.rados.0.smithi093.stdout:3209: write oid 446 current snap is 0 2023-08-27T20:54:47.805 INFO:tasks.rados.rados.0.smithi093.stdout:3204: finishing write tid 3 to smithi093153278-324 2023-08-27T20:54:47.805 INFO:tasks.rados.rados.0.smithi093.stdout:3209: seq_num 2019 ranges {733333=761276,2230321=588380,3427846=505487} 2023-08-27T20:54:47.812 INFO:tasks.rados.rados.0.smithi093.stdout:3209: writing smithi093153278-446 from 733333 to 1494609 tid 1 2023-08-27T20:54:47.819 INFO:tasks.rados.rados.0.smithi093.stdout:3209: writing smithi093153278-446 from 2230321 to 2818701 tid 2 2023-08-27T20:54:47.826 INFO:tasks.rados.rados.0.smithi093.stdout:3209: writing smithi093153278-446 from 3427846 to 3933333 tid 3 2023-08-27T20:54:47.826 INFO:tasks.rados.rados.0.smithi093.stdout:3210: read oid 474 snap -1 2023-08-27T20:54:47.826 INFO:tasks.rados.rados.0.smithi093.stdout:3210: expect (ObjNum 1898 snap 0 seq_num 1898) 2023-08-27T20:54:47.826 INFO:tasks.rados.rados.0.smithi093.stdout:3211: read oid 444 snap -1 2023-08-27T20:54:47.826 INFO:tasks.rados.rados.0.smithi093.stdout:3211: expect (ObjNum 1699 snap 0 seq_num 1699) 2023-08-27T20:54:47.827 INFO:tasks.rados.rados.0.smithi093.stdout:3212: write oid 230 current snap is 0 2023-08-27T20:54:47.827 INFO:tasks.rados.rados.0.smithi093.stdout:3204: finishing write tid 4 to smithi093153278-324 2023-08-27T20:54:47.827 INFO:tasks.rados.rados.0.smithi093.stdout:3212: seq_num 2020 ranges {750140=436525,1888803=732038,3055269=597466} 2023-08-27T20:54:47.832 INFO:tasks.rados.rados.0.smithi093.stdout:3212: writing smithi093153278-230 from 750140 to 1186665 tid 1 2023-08-27T20:54:47.840 INFO:tasks.rados.rados.0.smithi093.stdout:3212: writing smithi093153278-230 from 1888803 to 2620841 tid 2 2023-08-27T20:54:47.847 INFO:tasks.rados.rados.0.smithi093.stdout:3212: writing smithi093153278-230 from 3055269 to 3652735 tid 3 2023-08-27T20:54:47.847 INFO:tasks.rados.rados.0.smithi093.stdout:3213: read oid 194 snap -1 2023-08-27T20:54:47.847 INFO:tasks.rados.rados.0.smithi093.stdout:3213: expect (ObjNum 1837 snap 0 seq_num 1837) 2023-08-27T20:54:47.847 INFO:tasks.rados.rados.0.smithi093.stdout:3214: read oid 361 snap -1 2023-08-27T20:54:47.847 INFO:tasks.rados.rados.0.smithi093.stdout:3214: expect (ObjNum 360 snap 0 seq_num 360) 2023-08-27T20:54:47.848 INFO:tasks.rados.rados.0.smithi093.stdout:3215: write oid 332 current snap is 0 2023-08-27T20:54:47.848 INFO:tasks.rados.rados.0.smithi093.stdout:3204: finishing write tid 5 to smithi093153278-324 2023-08-27T20:54:47.848 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 324 v 401 (ObjNum 2016 snap 0 seq_num 2016) dirty exists 2023-08-27T20:54:47.848 INFO:tasks.rados.rados.0.smithi093.stdout:3204: left oid 324 (ObjNum 2016 snap 0 seq_num 2016) 2023-08-27T20:54:47.848 INFO:tasks.rados.rados.0.smithi093.stdout:3207: finishing write tid 1 to smithi093153278-59 2023-08-27T20:54:47.848 INFO:tasks.rados.rados.0.smithi093.stdout:3207: finishing write tid 2 to smithi093153278-59 2023-08-27T20:54:47.848 INFO:tasks.rados.rados.0.smithi093.stdout:3215: seq_num 2021 ranges {766947=511774,1947285=475696,2966339=408738,3895036=71911} 2023-08-27T20:54:47.853 INFO:tasks.rados.rados.0.smithi093.stdout:3215: writing smithi093153278-332 from 766947 to 1278721 tid 1 2023-08-27T20:54:47.860 INFO:tasks.rados.rados.0.smithi093.stdout:3215: writing smithi093153278-332 from 1947285 to 2422981 tid 2 2023-08-27T20:54:47.865 INFO:tasks.rados.rados.0.smithi093.stdout:3215: writing smithi093153278-332 from 2966339 to 3375077 tid 3 2023-08-27T20:54:47.869 INFO:tasks.rados.rados.0.smithi093.stdout:3215: writing smithi093153278-332 from 3895036 to 3966947 tid 4 2023-08-27T20:54:47.869 INFO:tasks.rados.rados.0.smithi093.stdout:3204: done (9 left) 2023-08-27T20:54:47.869 INFO:tasks.rados.rados.0.smithi093.stdout:3216: write oid 36 current snap is 0 2023-08-27T20:54:47.869 INFO:tasks.rados.rados.0.smithi093.stdout:3207: finishing write tid 3 to smithi093153278-59 2023-08-27T20:54:47.869 INFO:tasks.rados.rados.0.smithi093.stdout:3216: seq_num 2022 ranges {783754=587023,1889414=735707,3393762=589992} 2023-08-27T20:54:47.876 INFO:tasks.rados.rados.0.smithi093.stdout:3216: writing smithi093153278-36 from 783754 to 1370777 tid 1 2023-08-27T20:54:47.883 INFO:tasks.rados.rados.0.smithi093.stdout:3216: writing smithi093153278-36 from 1889414 to 2625121 tid 2 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3216: writing smithi093153278-36 from 3393762 to 3983754 tid 3 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3217: read oid 287 snap -1 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3217: expect (ObjNum 1908 snap 0 seq_num 1908) 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3218: write oid 397 current snap is 0 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3207: finishing write tid 4 to smithi093153278-59 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3207: finishing write tid 5 to smithi093153278-59 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 59 v 367 (ObjNum 2018 snap 0 seq_num 2018) dirty exists 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3207: left oid 59 (ObjNum 2018 snap 0 seq_num 2018) 2023-08-27T20:54:47.891 INFO:tasks.rados.rados.0.smithi093.stdout:3208: expect (ObjNum 1580 snap 0 seq_num 1580) 2023-08-27T20:54:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:47 smithi136 ceph-mon[113531]: pgmap v37: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 11 MiB/s wr, 118 op/s 2023-08-27T20:54:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:47 smithi093 ceph-mon[111335]: pgmap v37: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 11 MiB/s wr, 118 op/s 2023-08-27T20:54:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:47 smithi093 ceph-mon[118902]: pgmap v37: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 91 MiB/s rd, 11 MiB/s wr, 118 op/s 2023-08-27T20:54:48.061 INFO:tasks.rados.rados.0.smithi093.stdout:3209: finishing write tid 1 to smithi093153278-446 2023-08-27T20:54:48.061 INFO:tasks.rados.rados.0.smithi093.stdout:3209: finishing write tid 2 to smithi093153278-446 2023-08-27T20:54:48.061 INFO:tasks.rados.rados.0.smithi093.stdout:3209: finishing write tid 3 to smithi093153278-446 2023-08-27T20:54:48.061 INFO:tasks.rados.rados.0.smithi093.stdout:3209: finishing write tid 4 to smithi093153278-446 2023-08-27T20:54:48.061 INFO:tasks.rados.rados.0.smithi093.stdout:3209: finishing write tid 5 to smithi093153278-446 2023-08-27T20:54:48.061 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 446 v 377 (ObjNum 2019 snap 0 seq_num 2019) dirty exists 2023-08-27T20:54:48.062 INFO:tasks.rados.rados.0.smithi093.stdout:3209: left oid 446 (ObjNum 2019 snap 0 seq_num 2019) 2023-08-27T20:54:48.062 INFO:tasks.rados.rados.0.smithi093.stdout:3211: expect (ObjNum 1699 snap 0 seq_num 1699) 2023-08-27T20:54:48.205 INFO:tasks.rados.rados.0.smithi093.stdout:3212: finishing write tid 1 to smithi093153278-230 2023-08-27T20:54:48.205 INFO:tasks.rados.rados.0.smithi093.stdout:3210: expect (ObjNum 1898 snap 0 seq_num 1898) 2023-08-27T20:54:48.420 INFO:tasks.rados.rados.0.smithi093.stdout:3212: finishing write tid 2 to smithi093153278-230 2023-08-27T20:54:48.421 INFO:tasks.rados.rados.0.smithi093.stdout:3218: seq_num 2023 ranges {400561=778625,1664249=336312} 2023-08-27T20:54:48.427 INFO:tasks.rados.rados.0.smithi093.stdout:3218: writing smithi093153278-397 from 400561 to 1179186 tid 1 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3218: writing smithi093153278-397 from 1664249 to 2000561 tid 2 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3207: done (11 left) 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3208: done (10 left) 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3209: done (9 left) 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3210: done (8 left) 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3211: done (7 left) 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3219: write oid 313 current snap is 0 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3212: finishing write tid 3 to smithi093153278-230 2023-08-27T20:54:48.432 INFO:tasks.rados.rados.0.smithi093.stdout:3219: seq_num 2024 ranges {417368=453874,1322731=694637} 2023-08-27T20:54:48.436 INFO:tasks.rados.rados.0.smithi093.stdout:3219: writing smithi093153278-313 from 417368 to 871242 tid 1 2023-08-27T20:54:48.443 INFO:tasks.rados.rados.0.smithi093.stdout:3219: writing smithi093153278-313 from 1322731 to 2017368 tid 2 2023-08-27T20:54:48.443 INFO:tasks.rados.rados.0.smithi093.stdout:3220: write oid 450 current snap is 0 2023-08-27T20:54:48.443 INFO:tasks.rados.rados.0.smithi093.stdout:3212: finishing write tid 4 to smithi093153278-230 2023-08-27T20:54:48.443 INFO:tasks.rados.rados.0.smithi093.stdout:3212: finishing write tid 5 to smithi093153278-230 2023-08-27T20:54:48.443 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 230 v 405 (ObjNum 2020 snap 0 seq_num 2020) dirty exists 2023-08-27T20:54:48.443 INFO:tasks.rados.rados.0.smithi093.stdout:3212: left oid 230 (ObjNum 2020 snap 0 seq_num 2020) 2023-08-27T20:54:48.443 INFO:tasks.rados.rados.0.smithi093.stdout:3214: expect (ObjNum 360 snap 0 seq_num 360) 2023-08-27T20:54:48.507 INFO:tasks.rados.rados.0.smithi093.stdout:3213: expect (ObjNum 1837 snap 0 seq_num 1837) 2023-08-27T20:54:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:3215: finishing write tid 1 to smithi093153278-332 2023-08-27T20:54:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:3215: finishing write tid 2 to smithi093153278-332 2023-08-27T20:54:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:3215: finishing write tid 3 to smithi093153278-332 2023-08-27T20:54:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:3215: finishing write tid 4 to smithi093153278-332 2023-08-27T20:54:48.637 INFO:tasks.rados.rados.0.smithi093.stdout:3215: finishing write tid 5 to smithi093153278-332 2023-08-27T20:54:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:3215: finishing write tid 6 to smithi093153278-332 2023-08-27T20:54:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 332 v 363 (ObjNum 2021 snap 0 seq_num 2021) dirty exists 2023-08-27T20:54:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:3215: left oid 332 (ObjNum 2021 snap 0 seq_num 2021) 2023-08-27T20:54:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:3216: finishing write tid 1 to smithi093153278-36 2023-08-27T20:54:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:3216: finishing write tid 2 to smithi093153278-36 2023-08-27T20:54:48.638 INFO:tasks.rados.rados.0.smithi093.stdout:3220: seq_num 2025 ranges {434175=529123,1381213=483034} 2023-08-27T20:54:48.642 INFO:tasks.rados.rados.0.smithi093.stdout:3220: writing smithi093153278-450 from 434175 to 963298 tid 1 2023-08-27T20:54:48.647 INFO:tasks.rados.rados.0.smithi093.stdout:3220: writing smithi093153278-450 from 1381213 to 1864247 tid 2 2023-08-27T20:54:48.647 INFO:tasks.rados.rados.0.smithi093.stdout:3212: done (8 left) 2023-08-27T20:54:48.647 INFO:tasks.rados.rados.0.smithi093.stdout:3213: done (7 left) 2023-08-27T20:54:48.647 INFO:tasks.rados.rados.0.smithi093.stdout:3214: done (6 left) 2023-08-27T20:54:48.648 INFO:tasks.rados.rados.0.smithi093.stdout:3215: done (5 left) 2023-08-27T20:54:48.648 INFO:tasks.rados.rados.0.smithi093.stdout:3221: write oid 59 current snap is 0 2023-08-27T20:54:48.648 INFO:tasks.rados.rados.0.smithi093.stdout:3221: seq_num 2026 ranges {450982=604372,1723342=327640} 2023-08-27T20:54:48.653 INFO:tasks.rados.rados.0.smithi093.stdout:3221: writing smithi093153278-59 from 450982 to 1055354 tid 1 2023-08-27T20:54:48.659 INFO:tasks.rados.rados.0.smithi093.stdout:3221: writing smithi093153278-59 from 1723342 to 2050982 tid 2 2023-08-27T20:54:48.659 INFO:tasks.rados.rados.0.smithi093.stdout:3222: write oid 100 current snap is 0 2023-08-27T20:54:48.659 INFO:tasks.rados.rados.0.smithi093.stdout:3222: seq_num 2027 ranges {467789=679621,1781824=285965} 2023-08-27T20:54:48.665 INFO:tasks.rados.rados.0.smithi093.stdout:3222: writing smithi093153278-100 from 467789 to 1147410 tid 1 2023-08-27T20:54:48.671 INFO:tasks.rados.rados.0.smithi093.stdout:3222: writing smithi093153278-100 from 1781824 to 2067789 tid 2 2023-08-27T20:54:48.671 INFO:tasks.rados.rados.0.smithi093.stdout:3223: write oid 443 current snap is 0 2023-08-27T20:54:48.671 INFO:tasks.rados.rados.0.smithi093.stdout:3216: finishing write tid 3 to smithi093153278-36 2023-08-27T20:54:48.671 INFO:tasks.rados.rados.0.smithi093.stdout:3216: finishing write tid 4 to smithi093153278-36 2023-08-27T20:54:48.671 INFO:tasks.rados.rados.0.smithi093.stdout:3216: finishing write tid 5 to smithi093153278-36 2023-08-27T20:54:48.672 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 409 (ObjNum 2022 snap 0 seq_num 2022) dirty exists 2023-08-27T20:54:48.672 INFO:tasks.rados.rados.0.smithi093.stdout:3216: left oid 36 (ObjNum 2022 snap 0 seq_num 2022) 2023-08-27T20:54:48.672 INFO:tasks.rados.rados.0.smithi093.stdout:3217: expect (ObjNum 1908 snap 0 seq_num 1908) 2023-08-27T20:54:48.731 INFO:tasks.rados.rados.0.smithi093.stdout:3218: finishing write tid 1 to smithi093153278-397 2023-08-27T20:54:48.731 INFO:tasks.rados.rados.0.smithi093.stdout:3218: finishing write tid 2 to smithi093153278-397 2023-08-27T20:54:48.732 INFO:tasks.rados.rados.0.smithi093.stdout:3218: finishing write tid 3 to smithi093153278-397 2023-08-27T20:54:48.732 INFO:tasks.rados.rados.0.smithi093.stdout:3218: finishing write tid 4 to smithi093153278-397 2023-08-27T20:54:48.732 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 397 v 492 (ObjNum 2023 snap 0 seq_num 2023) dirty exists 2023-08-27T20:54:48.732 INFO:tasks.rados.rados.0.smithi093.stdout:3218: left oid 397 (ObjNum 2023 snap 0 seq_num 2023) 2023-08-27T20:54:48.732 INFO:tasks.rados.rados.0.smithi093.stdout:3219: finishing write tid 1 to smithi093153278-313 2023-08-27T20:54:48.732 INFO:tasks.rados.rados.0.smithi093.stdout:3223: seq_num 2028 ranges {484596=754870,1840306=244290} 2023-08-27T20:54:48.737 INFO:tasks.rados.rados.0.smithi093.stdout:3223: writing smithi093153278-443 from 484596 to 1239466 tid 1 2023-08-27T20:54:48.743 INFO:tasks.rados.rados.0.smithi093.stdout:3223: writing smithi093153278-443 from 1840306 to 2084596 tid 2 2023-08-27T20:54:48.743 INFO:tasks.rados.rados.0.smithi093.stdout:3216: done (7 left) 2023-08-27T20:54:48.743 INFO:tasks.rados.rados.0.smithi093.stdout:3217: done (6 left) 2023-08-27T20:54:48.743 INFO:tasks.rados.rados.0.smithi093.stdout:3218: done (5 left) 2023-08-27T20:54:48.744 INFO:tasks.rados.rados.0.smithi093.stdout:3224: write oid 379 current snap is 0 2023-08-27T20:54:48.744 INFO:tasks.rados.rados.0.smithi093.stdout:3219: finishing write tid 2 to smithi093153278-313 2023-08-27T20:54:48.744 INFO:tasks.rados.rados.0.smithi093.stdout:3224: seq_num 2029 ranges {501403=430119,1498788=490372} 2023-08-27T20:54:48.747 INFO:tasks.rados.rados.0.smithi093.stdout:3224: writing smithi093153278-379 from 501403 to 931522 tid 1 2023-08-27T20:54:48.754 INFO:tasks.rados.rados.0.smithi093.stdout:3224: writing smithi093153278-379 from 1498788 to 1989160 tid 2 2023-08-27T20:54:48.754 INFO:tasks.rados.rados.0.smithi093.stdout:3225: write oid 409 current snap is 0 2023-08-27T20:54:48.755 INFO:tasks.rados.rados.0.smithi093.stdout:3219: finishing write tid 3 to smithi093153278-313 2023-08-27T20:54:48.755 INFO:tasks.rados.rados.0.smithi093.stdout:3219: finishing write tid 4 to smithi093153278-313 2023-08-27T20:54:48.755 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 313 v 258 (ObjNum 2024 snap 0 seq_num 2024) dirty exists 2023-08-27T20:54:48.755 INFO:tasks.rados.rados.0.smithi093.stdout:3219: left oid 313 (ObjNum 2024 snap 0 seq_num 2024) 2023-08-27T20:54:48.755 INFO:tasks.rados.rados.0.smithi093.stdout:3220: finishing write tid 1 to smithi093153278-450 2023-08-27T20:54:48.755 INFO:tasks.rados.rados.0.smithi093.stdout:3220: finishing write tid 2 to smithi093153278-450 2023-08-27T20:54:48.756 INFO:tasks.rados.rados.0.smithi093.stdout:3220: finishing write tid 3 to smithi093153278-450 2023-08-27T20:54:48.756 INFO:tasks.rados.rados.0.smithi093.stdout:3220: finishing write tid 4 to smithi093153278-450 2023-08-27T20:54:48.756 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 450 v 334 (ObjNum 2025 snap 0 seq_num 2025) dirty exists 2023-08-27T20:54:48.756 INFO:tasks.rados.rados.0.smithi093.stdout:3220: left oid 450 (ObjNum 2025 snap 0 seq_num 2025) 2023-08-27T20:54:48.756 INFO:tasks.rados.rados.0.smithi093.stdout:3225: seq_num 2030 ranges {518210=621721,1557270=560940} 2023-08-27T20:54:48.760 INFO:tasks.rados.rados.0.smithi093.stdout:3225: writing smithi093153278-409 from 518210 to 1139931 tid 1 2023-08-27T20:54:48.766 INFO:tasks.rados.rados.0.smithi093.stdout:3225: writing smithi093153278-409 from 1557270 to 2118210 tid 2 2023-08-27T20:54:48.766 INFO:tasks.rados.rados.0.smithi093.stdout:3219: done (6 left) 2023-08-27T20:54:48.766 INFO:tasks.rados.rados.0.smithi093.stdout:3220: done (5 left) 2023-08-27T20:54:48.766 INFO:tasks.rados.rados.0.smithi093.stdout:3226: write oid 68 current snap is 0 2023-08-27T20:54:48.766 INFO:tasks.rados.rados.0.smithi093.stdout:3221: finishing write tid 1 to smithi093153278-59 2023-08-27T20:54:48.766 INFO:tasks.rados.rados.0.smithi093.stdout:3221: finishing write tid 2 to smithi093153278-59 2023-08-27T20:54:48.767 INFO:tasks.rados.rados.0.smithi093.stdout:3221: finishing write tid 3 to smithi093153278-59 2023-08-27T20:54:48.767 INFO:tasks.rados.rados.0.smithi093.stdout:3221: finishing write tid 4 to smithi093153278-59 2023-08-27T20:54:48.767 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 59 v 370 (ObjNum 2026 snap 0 seq_num 2026) dirty exists 2023-08-27T20:54:48.767 INFO:tasks.rados.rados.0.smithi093.stdout:3221: left oid 59 (ObjNum 2026 snap 0 seq_num 2026) 2023-08-27T20:54:48.767 INFO:tasks.rados.rados.0.smithi093.stdout:3222: finishing write tid 1 to smithi093153278-100 2023-08-27T20:54:48.767 INFO:tasks.rados.rados.0.smithi093.stdout:3222: finishing write tid 2 to smithi093153278-100 2023-08-27T20:54:48.768 INFO:tasks.rados.rados.0.smithi093.stdout:3226: seq_num 2031 ranges {535017=696970,2015752=119265} 2023-08-27T20:54:48.772 INFO:tasks.rados.rados.0.smithi093.stdout:3226: writing smithi093153278-68 from 535017 to 1231987 tid 1 2023-08-27T20:54:48.777 INFO:tasks.rados.rados.0.smithi093.stdout:3226: writing smithi093153278-68 from 2015752 to 2135017 tid 2 2023-08-27T20:54:48.777 INFO:tasks.rados.rados.0.smithi093.stdout:3221: done (5 left) 2023-08-27T20:54:48.777 INFO:tasks.rados.rados.0.smithi093.stdout:3227: write oid 72 current snap is 0 2023-08-27T20:54:48.777 INFO:tasks.rados.rados.0.smithi093.stdout:3222: finishing write tid 3 to smithi093153278-100 2023-08-27T20:54:48.777 INFO:tasks.rados.rados.0.smithi093.stdout:3222: finishing write tid 4 to smithi093153278-100 2023-08-27T20:54:48.778 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 100 v 380 (ObjNum 2027 snap 0 seq_num 2027) dirty exists 2023-08-27T20:54:48.778 INFO:tasks.rados.rados.0.smithi093.stdout:3222: left oid 100 (ObjNum 2027 snap 0 seq_num 2027) 2023-08-27T20:54:48.778 INFO:tasks.rados.rados.0.smithi093.stdout:3227: seq_num 2032 ranges {551824=772219,2074234=77590} 2023-08-27T20:54:48.784 INFO:tasks.rados.rados.0.smithi093.stdout:3227: writing smithi093153278-72 from 551824 to 1324043 tid 1 2023-08-27T20:54:48.789 INFO:tasks.rados.rados.0.smithi093.stdout:3227: writing smithi093153278-72 from 2074234 to 2151824 tid 2 2023-08-27T20:54:48.789 INFO:tasks.rados.rados.0.smithi093.stdout:3222: done (5 left) 2023-08-27T20:54:48.789 INFO:tasks.rados.rados.0.smithi093.stdout:3228: write oid 315 current snap is 0 2023-08-27T20:54:48.789 INFO:tasks.rados.rados.0.smithi093.stdout:3223: finishing write tid 1 to smithi093153278-443 2023-08-27T20:54:48.789 INFO:tasks.rados.rados.0.smithi093.stdout:3223: finishing write tid 2 to smithi093153278-443 2023-08-27T20:54:48.790 INFO:tasks.rados.rados.0.smithi093.stdout:3223: finishing write tid 3 to smithi093153278-443 2023-08-27T20:54:48.790 INFO:tasks.rados.rados.0.smithi093.stdout:3228: seq_num 2033 ranges {568631=447468,1732716=435915} 2023-08-27T20:54:48.794 INFO:tasks.rados.rados.0.smithi093.stdout:3228: writing smithi093153278-315 from 568631 to 1016099 tid 1 2023-08-27T20:54:48.801 INFO:tasks.rados.rados.0.smithi093.stdout:3228: writing smithi093153278-315 from 1732716 to 2168631 tid 2 2023-08-27T20:54:48.801 INFO:tasks.rados.rados.0.smithi093.stdout:3229: write oid 180 current snap is 0 2023-08-27T20:54:48.801 INFO:tasks.rados.rados.0.smithi093.stdout:3229: seq_num 2034 ranges {585438=522717,1674845=510593} 2023-08-27T20:54:48.807 INFO:tasks.rados.rados.0.smithi093.stdout:3229: writing smithi093153278-180 from 585438 to 1108155 tid 1 2023-08-27T20:54:48.813 INFO:tasks.rados.rados.0.smithi093.stdout:3229: writing smithi093153278-180 from 1674845 to 2185438 tid 2 2023-08-27T20:54:48.813 INFO:tasks.rados.rados.0.smithi093.stdout:3230: write oid 346 current snap is 0 2023-08-27T20:54:48.813 INFO:tasks.rados.rados.0.smithi093.stdout:3223: finishing write tid 4 to smithi093153278-443 2023-08-27T20:54:48.813 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 433 (ObjNum 2028 snap 0 seq_num 2028) dirty exists 2023-08-27T20:54:48.813 INFO:tasks.rados.rados.0.smithi093.stdout:3223: left oid 443 (ObjNum 2028 snap 0 seq_num 2028) 2023-08-27T20:54:48.814 INFO:tasks.rados.rados.0.smithi093.stdout:3224: finishing write tid 1 to smithi093153278-379 2023-08-27T20:54:48.814 INFO:tasks.rados.rados.0.smithi093.stdout:3224: finishing write tid 2 to smithi093153278-379 2023-08-27T20:54:48.814 INFO:tasks.rados.rados.0.smithi093.stdout:3224: finishing write tid 3 to smithi093153278-379 2023-08-27T20:54:48.814 INFO:tasks.rados.rados.0.smithi093.stdout:3230: seq_num 2035 ranges {602245=597966,1733327=468918} 2023-08-27T20:54:48.819 INFO:tasks.rados.rados.0.smithi093.stdout:3230: writing smithi093153278-346 from 602245 to 1200211 tid 1 2023-08-27T20:54:48.826 INFO:tasks.rados.rados.0.smithi093.stdout:3230: writing smithi093153278-346 from 1733327 to 2202245 tid 2 2023-08-27T20:54:48.826 INFO:tasks.rados.rados.0.smithi093.stdout:3223: done (7 left) 2023-08-27T20:54:48.826 INFO:tasks.rados.rados.0.smithi093.stdout:3231: read oid 79 snap -1 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3231: expect (ObjNum 1804 snap 0 seq_num 1804) 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3232: read oid 297 snap -1 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3232: expect (ObjNum 1929 snap 0 seq_num 1929) 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3233: write oid 16 current snap is 0 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3224: finishing write tid 4 to smithi093153278-379 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 379 v 366 (ObjNum 2029 snap 0 seq_num 2029) dirty exists 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3224: left oid 379 (ObjNum 2029 snap 0 seq_num 2029) 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3225: finishing write tid 1 to smithi093153278-409 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3225: finishing write tid 2 to smithi093153278-409 2023-08-27T20:54:48.827 INFO:tasks.rados.rados.0.smithi093.stdout:3225: finishing write tid 3 to smithi093153278-409 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3225: finishing write tid 4 to smithi093153278-409 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 409 v 373 (ObjNum 2030 snap 0 seq_num 2030) dirty exists 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3225: left oid 409 (ObjNum 2030 snap 0 seq_num 2030) 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3226: finishing write tid 1 to smithi093153278-68 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3226: finishing write tid 2 to smithi093153278-68 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3226: finishing write tid 3 to smithi093153278-68 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3226: finishing write tid 4 to smithi093153278-68 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 68 v 434 (ObjNum 2031 snap 0 seq_num 2031) dirty exists 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3226: left oid 68 (ObjNum 2031 snap 0 seq_num 2031) 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3227: finishing write tid 1 to smithi093153278-72 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3227: finishing write tid 2 to smithi093153278-72 2023-08-27T20:54:48.828 INFO:tasks.rados.rados.0.smithi093.stdout:3227: finishing write tid 3 to smithi093153278-72 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3227: finishing write tid 4 to smithi093153278-72 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 72 v 449 (ObjNum 2032 snap 0 seq_num 2032) dirty exists 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3227: left oid 72 (ObjNum 2032 snap 0 seq_num 2032) 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3228: finishing write tid 1 to smithi093153278-315 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3228: finishing write tid 2 to smithi093153278-315 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3228: finishing write tid 3 to smithi093153278-315 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3228: finishing write tid 4 to smithi093153278-315 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 452 (ObjNum 2033 snap 0 seq_num 2033) dirty exists 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3228: left oid 315 (ObjNum 2033 snap 0 seq_num 2033) 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3229: finishing write tid 1 to smithi093153278-180 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3229: finishing write tid 2 to smithi093153278-180 2023-08-27T20:54:48.829 INFO:tasks.rados.rados.0.smithi093.stdout:3229: finishing write tid 3 to smithi093153278-180 2023-08-27T20:54:48.830 INFO:tasks.rados.rados.0.smithi093.stdout:3229: finishing write tid 4 to smithi093153278-180 2023-08-27T20:54:48.830 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 180 v 355 (ObjNum 2034 snap 0 seq_num 2034) dirty exists 2023-08-27T20:54:48.830 INFO:tasks.rados.rados.0.smithi093.stdout:3229: left oid 180 (ObjNum 2034 snap 0 seq_num 2034) 2023-08-27T20:54:48.830 INFO:tasks.rados.rados.0.smithi093.stdout:3230: finishing write tid 1 to smithi093153278-346 2023-08-27T20:54:48.830 INFO:tasks.rados.rados.0.smithi093.stdout:3233: seq_num 2036 ranges {619052=673215,1791809=427243} 2023-08-27T20:54:48.832 INFO:tasks.rados.rados.0.smithi093.stdout:3233: writing smithi093153278-16 from 619052 to 1292267 tid 1 2023-08-27T20:54:48.838 INFO:tasks.rados.rados.0.smithi093.stdout:3233: writing smithi093153278-16 from 1791809 to 2219052 tid 2 2023-08-27T20:54:48.838 INFO:tasks.rados.rados.0.smithi093.stdout:3224: done (9 left) 2023-08-27T20:54:48.838 INFO:tasks.rados.rados.0.smithi093.stdout:3225: done (8 left) 2023-08-27T20:54:48.838 INFO:tasks.rados.rados.0.smithi093.stdout:3226: done (7 left) 2023-08-27T20:54:48.838 INFO:tasks.rados.rados.0.smithi093.stdout:3227: done (6 left) 2023-08-27T20:54:48.838 INFO:tasks.rados.rados.0.smithi093.stdout:3228: done (5 left) 2023-08-27T20:54:48.838 INFO:tasks.rados.rados.0.smithi093.stdout:3229: done (4 left) 2023-08-27T20:54:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:3234: delete oid 427 current snap is 0 2023-08-27T20:54:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:3230: finishing write tid 2 to smithi093153278-346 2023-08-27T20:54:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:3230: finishing write tid 3 to smithi093153278-346 2023-08-27T20:54:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:3230: finishing write tid 4 to smithi093153278-346 2023-08-27T20:54:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 346 v 475 (ObjNum 2035 snap 0 seq_num 2035) dirty exists 2023-08-27T20:54:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:3230: left oid 346 (ObjNum 2035 snap 0 seq_num 2035) 2023-08-27T20:54:48.839 INFO:tasks.rados.rados.0.smithi093.stdout:3232: expect (ObjNum 1929 snap 0 seq_num 1929) 2023-08-27T20:54:48.910 INFO:tasks.rados.rados.0.smithi093.stdout:3231: expect (ObjNum 1804 snap 0 seq_num 1804) 2023-08-27T20:54:49.016 INFO:tasks.rados.rados.0.smithi093.stdout:3233: finishing write tid 1 to smithi093153278-16 2023-08-27T20:54:49.016 INFO:tasks.rados.rados.0.smithi093.stdout:3233: finishing write tid 2 to smithi093153278-16 2023-08-27T20:54:49.016 INFO:tasks.rados.rados.0.smithi093.stdout:3233: finishing write tid 3 to smithi093153278-16 2023-08-27T20:54:49.016 INFO:tasks.rados.rados.0.smithi093.stdout:3233: finishing write tid 4 to smithi093153278-16 2023-08-27T20:54:49.016 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 16 v 385 (ObjNum 2036 snap 0 seq_num 2036) dirty exists 2023-08-27T20:54:49.016 INFO:tasks.rados.rados.0.smithi093.stdout:3233: left oid 16 (ObjNum 2036 snap 0 seq_num 2036) 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3230: done (4 left) 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3231: done (3 left) 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3232: done (2 left) 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3233: done (1 left) 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3234: done (0 left) 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3235: read oid 310 snap -1 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3235: expect deleted 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3236: write oid 78 current snap is 0 2023-08-27T20:54:49.017 INFO:tasks.rados.rados.0.smithi093.stdout:3236: seq_num 2038 ranges {652666=540066,1908773=343893} 2023-08-27T20:54:49.022 INFO:tasks.rados.rados.0.smithi093.stdout:3236: writing smithi093153278-78 from 652666 to 1192732 tid 1 2023-08-27T20:54:49.028 INFO:tasks.rados.rados.0.smithi093.stdout:3236: writing smithi093153278-78 from 1908773 to 2252666 tid 2 2023-08-27T20:54:49.028 INFO:tasks.rados.rados.0.smithi093.stdout:3237: write oid 101 current snap is 0 2023-08-27T20:54:49.028 INFO:tasks.rados.rados.0.smithi093.stdout:3237: seq_num 2039 ranges {669473=615315,1967255=302218} 2023-08-27T20:54:49.035 INFO:tasks.rados.rados.0.smithi093.stdout:3237: writing smithi093153278-101 from 669473 to 1284788 tid 1 2023-08-27T20:54:49.040 INFO:tasks.rados.rados.0.smithi093.stdout:3237: writing smithi093153278-101 from 1967255 to 2269473 tid 2 2023-08-27T20:54:49.040 INFO:tasks.rados.rados.0.smithi093.stdout:3238: read oid 346 snap -1 2023-08-27T20:54:49.041 INFO:tasks.rados.rados.0.smithi093.stdout:3238: expect (ObjNum 2035 snap 0 seq_num 2035) 2023-08-27T20:54:49.041 INFO:tasks.rados.rados.0.smithi093.stdout:3239: read oid 237 snap -1 2023-08-27T20:54:49.041 INFO:tasks.rados.rados.0.smithi093.stdout:3239: expect (ObjNum 1882 snap 0 seq_num 1882) 2023-08-27T20:54:49.041 INFO:tasks.rados.rados.0.smithi093.stdout:3240: write oid 246 current snap is 0 2023-08-27T20:54:49.041 INFO:tasks.rados.rados.0.smithi093.stdout:3240: seq_num 2040 ranges {686280=690564,2025737=260543} 2023-08-27T20:54:49.048 INFO:tasks.rados.rados.0.smithi093.stdout:3240: writing smithi093153278-246 from 686280 to 1376844 tid 1 2023-08-27T20:54:49.053 INFO:tasks.rados.rados.0.smithi093.stdout:3240: writing smithi093153278-246 from 2025737 to 2286280 tid 2 2023-08-27T20:54:49.053 INFO:tasks.rados.rados.0.smithi093.stdout:3241: read oid 94 snap -1 2023-08-27T20:54:49.053 INFO:tasks.rados.rados.0.smithi093.stdout:3241: expect deleted 2023-08-27T20:54:49.054 INFO:tasks.rados.rados.0.smithi093.stdout:3242: write oid 405 current snap is 0 2023-08-27T20:54:49.054 INFO:tasks.rados.rados.0.smithi093.stdout:3236: finishing write tid 1 to smithi093153278-78 2023-08-27T20:54:49.054 INFO:tasks.rados.rados.0.smithi093.stdout:3236: finishing write tid 2 to smithi093153278-78 2023-08-27T20:54:49.054 INFO:tasks.rados.rados.0.smithi093.stdout:3242: seq_num 2041 ranges {703087=765813,2084219=218868} 2023-08-27T20:54:49.061 INFO:tasks.rados.rados.0.smithi093.stdout:3242: writing smithi093153278-405 from 703087 to 1468900 tid 1 2023-08-27T20:54:49.066 INFO:tasks.rados.rados.0.smithi093.stdout:3242: writing smithi093153278-405 from 2084219 to 2303087 tid 2 2023-08-27T20:54:49.066 INFO:tasks.rados.rados.0.smithi093.stdout:3235: done (7 left) 2023-08-27T20:54:49.067 INFO:tasks.rados.rados.0.smithi093.stdout:3243: read oid 179 snap -1 2023-08-27T20:54:49.067 INFO:tasks.rados.rados.0.smithi093.stdout:3243: expect (ObjNum 1944 snap 0 seq_num 1944) 2023-08-27T20:54:49.067 INFO:tasks.rados.rados.0.smithi093.stdout:3244: delete oid 190 current snap is 0 2023-08-27T20:54:49.068 INFO:tasks.rados.rados.0.smithi093.stdout:3236: finishing write tid 3 to smithi093153278-78 2023-08-27T20:54:49.068 INFO:tasks.rados.rados.0.smithi093.stdout:3236: finishing write tid 4 to smithi093153278-78 2023-08-27T20:54:49.068 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 78 v 377 (ObjNum 2038 snap 0 seq_num 2038) dirty exists 2023-08-27T20:54:49.068 INFO:tasks.rados.rados.0.smithi093.stdout:3236: left oid 78 (ObjNum 2038 snap 0 seq_num 2038) 2023-08-27T20:54:49.068 INFO:tasks.rados.rados.0.smithi093.stdout:3237: finishing write tid 1 to smithi093153278-101 2023-08-27T20:54:49.068 INFO:tasks.rados.rados.0.smithi093.stdout:3238: expect (ObjNum 2035 snap 0 seq_num 2035) 2023-08-27T20:54:49.153 INFO:tasks.rados.rados.0.smithi093.stdout:3237: finishing write tid 2 to smithi093153278-101 2023-08-27T20:54:49.153 INFO:tasks.rados.rados.0.smithi093.stdout:3237: finishing write tid 3 to smithi093153278-101 2023-08-27T20:54:49.153 INFO:tasks.rados.rados.0.smithi093.stdout:3237: finishing write tid 4 to smithi093153278-101 2023-08-27T20:54:49.153 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 101 v 372 (ObjNum 2039 snap 0 seq_num 2039) dirty exists 2023-08-27T20:54:49.154 INFO:tasks.rados.rados.0.smithi093.stdout:3237: left oid 101 (ObjNum 2039 snap 0 seq_num 2039) 2023-08-27T20:54:49.154 INFO:tasks.rados.rados.0.smithi093.stdout:3239: expect (ObjNum 1882 snap 0 seq_num 1882) 2023-08-27T20:54:49.315 INFO:tasks.rados.rados.0.smithi093.stdout:3240: finishing write tid 1 to smithi093153278-246 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3240: finishing write tid 2 to smithi093153278-246 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3240: finishing write tid 3 to smithi093153278-246 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3240: finishing write tid 4 to smithi093153278-246 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 246 v 380 (ObjNum 2040 snap 0 seq_num 2040) dirty exists 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3240: left oid 246 (ObjNum 2040 snap 0 seq_num 2040) 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3242: finishing write tid 1 to smithi093153278-405 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3236: done (8 left) 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3237: done (7 left) 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3238: done (6 left) 2023-08-27T20:54:49.316 INFO:tasks.rados.rados.0.smithi093.stdout:3239: done (5 left) 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3240: done (4 left) 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3241: done (3 left) 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3244: done (2 left) 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3245: write oid 246 current snap is 0 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3242: finishing write tid 2 to smithi093153278-405 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3242: finishing write tid 3 to smithi093153278-405 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3242: finishing write tid 4 to smithi093153278-405 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 405 v 437 (ObjNum 2041 snap 0 seq_num 2041) dirty exists 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3242: left oid 405 (ObjNum 2041 snap 0 seq_num 2041) 2023-08-27T20:54:49.317 INFO:tasks.rados.rados.0.smithi093.stdout:3243: expect (ObjNum 1944 snap 0 seq_num 1944) 2023-08-27T20:54:49.485 INFO:tasks.rados.rados.0.smithi093.stdout:3245: seq_num 2043 ranges {736701=516311,1684830=516055} 2023-08-27T20:54:49.491 INFO:tasks.rados.rados.0.smithi093.stdout:3245: writing smithi093153278-246 from 736701 to 1253012 tid 1 2023-08-27T20:54:49.496 INFO:tasks.rados.rados.0.smithi093.stdout:3245: writing smithi093153278-246 from 1684830 to 2200885 tid 2 2023-08-27T20:54:49.497 INFO:tasks.rados.rados.0.smithi093.stdout:3242: done (2 left) 2023-08-27T20:54:49.497 INFO:tasks.rados.rados.0.smithi093.stdout:3243: done (1 left) 2023-08-27T20:54:49.497 INFO:tasks.rados.rados.0.smithi093.stdout:3246: write oid 330 current snap is 0 2023-08-27T20:54:49.497 INFO:tasks.rados.rados.0.smithi093.stdout:3245: finishing write tid 1 to smithi093153278-246 2023-08-27T20:54:49.497 INFO:tasks.rados.rados.0.smithi093.stdout:3246: seq_num 2044 ranges {753508=591560,2143312=210196} 2023-08-27T20:54:49.504 INFO:tasks.rados.rados.0.smithi093.stdout:3246: writing smithi093153278-330 from 753508 to 1345068 tid 1 2023-08-27T20:54:49.510 INFO:tasks.rados.rados.0.smithi093.stdout:3246: writing smithi093153278-330 from 2143312 to 2353508 tid 2 2023-08-27T20:54:49.510 INFO:tasks.rados.rados.0.smithi093.stdout:3247: read oid 470 snap -1 2023-08-27T20:54:49.510 INFO:tasks.rados.rados.0.smithi093.stdout:3247: expect (ObjNum 1796 snap 0 seq_num 1796) 2023-08-27T20:54:49.511 INFO:tasks.rados.rados.0.smithi093.stdout:3248: delete oid 359 current snap is 0 2023-08-27T20:54:49.511 INFO:tasks.rados.rados.0.smithi093.stdout:3245: finishing write tid 2 to smithi093153278-246 2023-08-27T20:54:49.511 INFO:tasks.rados.rados.0.smithi093.stdout:3245: finishing write tid 3 to smithi093153278-246 2023-08-27T20:54:49.511 INFO:tasks.rados.rados.0.smithi093.stdout:3245: finishing write tid 4 to smithi093153278-246 2023-08-27T20:54:49.511 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 246 v 383 (ObjNum 2043 snap 0 seq_num 2043) dirty exists 2023-08-27T20:54:49.511 INFO:tasks.rados.rados.0.smithi093.stdout:3245: left oid 246 (ObjNum 2043 snap 0 seq_num 2043) 2023-08-27T20:54:49.511 INFO:tasks.rados.rados.0.smithi093.stdout:3246: finishing write tid 1 to smithi093153278-330 2023-08-27T20:54:49.512 INFO:tasks.rados.rados.0.smithi093.stdout:3246: finishing write tid 2 to smithi093153278-330 2023-08-27T20:54:49.512 INFO:tasks.rados.rados.0.smithi093.stdout:3246: finishing write tid 3 to smithi093153278-330 2023-08-27T20:54:49.512 INFO:tasks.rados.rados.0.smithi093.stdout:3246: finishing write tid 4 to smithi093153278-330 2023-08-27T20:54:49.512 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 330 v 369 (ObjNum 2044 snap 0 seq_num 2044) dirty exists 2023-08-27T20:54:49.512 INFO:tasks.rados.rados.0.smithi093.stdout:3246: left oid 330 (ObjNum 2044 snap 0 seq_num 2044) 2023-08-27T20:54:49.515 INFO:tasks.rados.rados.0.smithi093.stdout:3247: expect (ObjNum 1796 snap 0 seq_num 1796) 2023-08-27T20:54:49.649 INFO:tasks.rados.rados.0.smithi093.stdout:3245: done (3 left) 2023-08-27T20:54:49.649 INFO:tasks.rados.rados.0.smithi093.stdout:3246: done (2 left) 2023-08-27T20:54:49.649 INFO:tasks.rados.rados.0.smithi093.stdout:3247: done (1 left) 2023-08-27T20:54:49.650 INFO:tasks.rados.rados.0.smithi093.stdout:3248: done (0 left) 2023-08-27T20:54:49.650 INFO:tasks.rados.rados.0.smithi093.stdout:3249: read oid 435 snap -1 2023-08-27T20:54:49.650 INFO:tasks.rados.rados.0.smithi093.stdout:3249: expect (ObjNum 1981 snap 0 seq_num 1981) 2023-08-27T20:54:49.650 INFO:tasks.rados.rados.0.smithi093.stdout:3250: write oid 427 current snap is 0 2023-08-27T20:54:49.650 INFO:tasks.rados.rados.0.smithi093.stdout:3250: seq_num 2046 ranges {787122=458411,1860276=526846} 2023-08-27T20:54:49.655 INFO:tasks.rados.rados.0.smithi093.stdout:3250: writing smithi093153278-427 from 787122 to 1245533 tid 1 2023-08-27T20:54:49.662 INFO:tasks.rados.rados.0.smithi093.stdout:3250: writing smithi093153278-427 from 1860276 to 2387122 tid 2 2023-08-27T20:54:49.662 INFO:tasks.rados.rados.0.smithi093.stdout:3251: read oid 244 snap -1 2023-08-27T20:54:49.662 INFO:tasks.rados.rados.0.smithi093.stdout:3251: expect (ObjNum 857 snap 0 seq_num 857) 2023-08-27T20:54:49.662 INFO:tasks.rados.rados.0.smithi093.stdout:3252: read oid 157 snap -1 2023-08-27T20:54:49.662 INFO:tasks.rados.rados.0.smithi093.stdout:3252: expect (ObjNum 156 snap 0 seq_num 156) 2023-08-27T20:54:49.663 INFO:tasks.rados.rados.0.smithi093.stdout:3253: write oid 11 current snap is 0 2023-08-27T20:54:49.663 INFO:tasks.rados.rados.0.smithi093.stdout:3253: seq_num 2047 ranges {403929=533660,1518758=407040} 2023-08-27T20:54:49.667 INFO:tasks.rados.rados.0.smithi093.stdout:3253: writing smithi093153278-11 from 403929 to 937589 tid 1 2023-08-27T20:54:49.673 INFO:tasks.rados.rados.0.smithi093.stdout:3253: writing smithi093153278-11 from 1518758 to 1925798 tid 2 2023-08-27T20:54:49.673 INFO:tasks.rados.rados.0.smithi093.stdout:3254: read oid 189 snap -1 2023-08-27T20:54:49.673 INFO:tasks.rados.rados.0.smithi093.stdout:3254: expect (ObjNum 1455 snap 0 seq_num 1455) 2023-08-27T20:54:49.673 INFO:tasks.rados.rados.0.smithi093.stdout:3255: write oid 441 current snap is 0 2023-08-27T20:54:49.673 INFO:tasks.rados.rados.0.smithi093.stdout:3255: seq_num 2048 ranges {420736=608909,1577240=667051} 2023-08-27T20:54:49.678 INFO:tasks.rados.rados.0.smithi093.stdout:3255: writing smithi093153278-441 from 420736 to 1029645 tid 1 2023-08-27T20:54:49.686 INFO:tasks.rados.rados.0.smithi093.stdout:3255: writing smithi093153278-441 from 1577240 to 2244291 tid 2 2023-08-27T20:54:49.686 INFO:tasks.rados.rados.0.smithi093.stdout:3256: read oid 351 snap -1 2023-08-27T20:54:49.686 INFO:tasks.rados.rados.0.smithi093.stdout:3256: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:54:49.686 INFO:tasks.rados.rados.0.smithi093.stdout:3257: read oid 374 snap -1 2023-08-27T20:54:49.686 INFO:tasks.rados.rados.0.smithi093.stdout:3257: expect deleted 2023-08-27T20:54:49.686 INFO:tasks.rados.rados.0.smithi093.stdout:3258: read oid 84 snap -1 2023-08-27T20:54:49.686 INFO:tasks.rados.rados.0.smithi093.stdout:3258: expect (ObjNum 1995 snap 0 seq_num 1995) 2023-08-27T20:54:49.687 INFO:tasks.rados.rados.0.smithi093.stdout:3259: write oid 333 current snap is 0 2023-08-27T20:54:49.687 INFO:tasks.rados.rados.0.smithi093.stdout:3250: finishing write tid 1 to smithi093153278-427 2023-08-27T20:54:49.687 INFO:tasks.rados.rados.0.smithi093.stdout:3249: expect (ObjNum 1981 snap 0 seq_num 1981) 2023-08-27T20:54:49.788 INFO:tasks.rados.rados.0.smithi093.stdout:3250: finishing write tid 2 to smithi093153278-427 2023-08-27T20:54:49.788 INFO:tasks.rados.rados.0.smithi093.stdout:3250: finishing write tid 3 to smithi093153278-427 2023-08-27T20:54:49.788 INFO:tasks.rados.rados.0.smithi093.stdout:3250: finishing write tid 4 to smithi093153278-427 2023-08-27T20:54:49.788 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 427 v 386 (ObjNum 2046 snap 0 seq_num 2046) dirty exists 2023-08-27T20:54:49.789 INFO:tasks.rados.rados.0.smithi093.stdout:3250: left oid 427 (ObjNum 2046 snap 0 seq_num 2046) 2023-08-27T20:54:49.789 INFO:tasks.rados.rados.0.smithi093.stdout:3252: expect (ObjNum 156 snap 0 seq_num 156) 2023-08-27T20:54:49.869 INFO:tasks.rados.rados.0.smithi093.stdout:3251: expect (ObjNum 857 snap 0 seq_num 857) 2023-08-27T20:54:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:49 smithi136 ceph-mon[113531]: pgmap v38: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 7.0 MiB/s wr, 75 op/s 2023-08-27T20:54:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3201733093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:49 smithi093 ceph-mon[111335]: pgmap v38: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 7.0 MiB/s wr, 75 op/s 2023-08-27T20:54:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3201733093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:49 smithi093 ceph-mon[118902]: pgmap v38: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 7.0 MiB/s wr, 75 op/s 2023-08-27T20:54:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3201733093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:50.005 INFO:tasks.rados.rados.0.smithi093.stdout:3253: finishing write tid 1 to smithi093153278-11 2023-08-27T20:54:50.005 INFO:tasks.rados.rados.0.smithi093.stdout:3253: finishing write tid 2 to smithi093153278-11 2023-08-27T20:54:50.005 INFO:tasks.rados.rados.0.smithi093.stdout:3253: finishing write tid 3 to smithi093153278-11 2023-08-27T20:54:50.005 INFO:tasks.rados.rados.0.smithi093.stdout:3259: seq_num 2049 ranges {437543=684158,1635722=410709} 2023-08-27T20:54:50.011 INFO:tasks.rados.rados.0.smithi093.stdout:3259: writing smithi093153278-333 from 437543 to 1121701 tid 1 2023-08-27T20:54:50.016 INFO:tasks.rados.rados.0.smithi093.stdout:3259: writing smithi093153278-333 from 1635722 to 2046431 tid 2 2023-08-27T20:54:50.016 INFO:tasks.rados.rados.0.smithi093.stdout:3249: done (10 left) 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3250: done (9 left) 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3251: done (8 left) 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3252: done (7 left) 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3260: read oid 37 snap -1 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3260: expect (ObjNum 1755 snap 0 seq_num 1755) 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3261: read oid 349 snap -1 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3261: expect (ObjNum 1996 snap 0 seq_num 1996) 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3262: read oid 465 snap -1 2023-08-27T20:54:50.017 INFO:tasks.rados.rados.0.smithi093.stdout:3262: expect (ObjNum 789 snap 0 seq_num 789) 2023-08-27T20:54:50.018 INFO:tasks.rados.rados.0.smithi093.stdout:3263: write oid 176 current snap is 0 2023-08-27T20:54:50.018 INFO:tasks.rados.rados.0.smithi093.stdout:3253: finishing write tid 4 to smithi093153278-11 2023-08-27T20:54:50.018 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 11 v 337 (ObjNum 2047 snap 0 seq_num 2047) dirty exists 2023-08-27T20:54:50.018 INFO:tasks.rados.rados.0.smithi093.stdout:3253: left oid 11 (ObjNum 2047 snap 0 seq_num 2047) 2023-08-27T20:54:50.018 INFO:tasks.rados.rados.0.smithi093.stdout:3254: expect (ObjNum 1455 snap 0 seq_num 1455) 2023-08-27T20:54:50.142 INFO:tasks.rados.rados.0.smithi093.stdout:3255: finishing write tid 1 to smithi093153278-441 2023-08-27T20:54:50.142 INFO:tasks.rados.rados.0.smithi093.stdout:3256: expect (ObjNum 350 snap 0 seq_num 350) 2023-08-27T20:54:50.261 INFO:tasks.rados.rados.0.smithi093.stdout:3255: finishing write tid 2 to smithi093153278-441 2023-08-27T20:54:50.262 INFO:tasks.rados.rados.0.smithi093.stdout:3255: finishing write tid 3 to smithi093153278-441 2023-08-27T20:54:50.262 INFO:tasks.rados.rados.0.smithi093.stdout:3255: finishing write tid 4 to smithi093153278-441 2023-08-27T20:54:50.262 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 358 (ObjNum 2048 snap 0 seq_num 2048) dirty exists 2023-08-27T20:54:50.262 INFO:tasks.rados.rados.0.smithi093.stdout:3255: left oid 441 (ObjNum 2048 snap 0 seq_num 2048) 2023-08-27T20:54:50.262 INFO:tasks.rados.rados.0.smithi093.stdout:3258: expect (ObjNum 1995 snap 0 seq_num 1995) 2023-08-27T20:54:50.459 INFO:tasks.rados.rados.0.smithi093.stdout:3259: finishing write tid 1 to smithi093153278-333 2023-08-27T20:54:50.459 INFO:tasks.rados.rados.0.smithi093.stdout:3263: seq_num 2050 ranges {454350=759407,1977851=476499} 2023-08-27T20:54:50.465 INFO:tasks.rados.rados.0.smithi093.stdout:3263: writing smithi093153278-176 from 454350 to 1213757 tid 1 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3263: writing smithi093153278-176 from 1977851 to 2454350 tid 2 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3253: done (10 left) 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3254: done (9 left) 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3255: done (8 left) 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3256: done (7 left) 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3257: done (6 left) 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3258: done (5 left) 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3264: read oid 216 snap -1 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3264: expect (ObjNum 1773 snap 0 seq_num 1773) 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3265: write oid 418 current snap is 0 2023-08-27T20:54:50.473 INFO:tasks.rados.rados.0.smithi093.stdout:3260: expect (ObjNum 1755 snap 0 seq_num 1755) 2023-08-27T20:54:50.684 INFO:tasks.rados.rados.0.smithi093.stdout:3261: expect (ObjNum 1996 snap 0 seq_num 1996) 2023-08-27T20:54:50.966 INFO:tasks.rados.rados.0.smithi093.stdout:3259: finishing write tid 2 to smithi093153278-333 2023-08-27T20:54:50.966 INFO:tasks.rados.rados.0.smithi093.stdout:3259: finishing write tid 3 to smithi093153278-333 2023-08-27T20:54:50.966 INFO:tasks.rados.rados.0.smithi093.stdout:3259: finishing write tid 4 to smithi093153278-333 2023-08-27T20:54:50.966 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 333 v 361 (ObjNum 2049 snap 0 seq_num 2049) dirty exists 2023-08-27T20:54:50.966 INFO:tasks.rados.rados.0.smithi093.stdout:3259: left oid 333 (ObjNum 2049 snap 0 seq_num 2049) 2023-08-27T20:54:50.966 INFO:tasks.rados.rados.0.smithi093.stdout:3262: expect (ObjNum 789 snap 0 seq_num 789) 2023-08-27T20:54:51.118 INFO:tasks.rados.rados.0.smithi093.stdout:3263: finishing write tid 1 to smithi093153278-176 2023-08-27T20:54:51.118 INFO:tasks.rados.rados.0.smithi093.stdout:3263: finishing write tid 2 to smithi093153278-176 2023-08-27T20:54:51.118 INFO:tasks.rados.rados.0.smithi093.stdout:3263: finishing write tid 3 to smithi093153278-176 2023-08-27T20:54:51.118 INFO:tasks.rados.rados.0.smithi093.stdout:3263: finishing write tid 4 to smithi093153278-176 2023-08-27T20:54:51.119 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 176 v 539 (ObjNum 2050 snap 0 seq_num 2050) dirty exists 2023-08-27T20:54:51.119 INFO:tasks.rados.rados.0.smithi093.stdout:3263: left oid 176 (ObjNum 2050 snap 0 seq_num 2050) 2023-08-27T20:54:51.119 INFO:tasks.rados.rados.0.smithi093.stdout:3264: expect (ObjNum 1773 snap 0 seq_num 1773) 2023-08-27T20:54:51.238 INFO:tasks.rados.rados.0.smithi093.stdout:3265: seq_num 2051 ranges {471157=434656,1636333=414378} 2023-08-27T20:54:51.242 INFO:tasks.rados.rados.0.smithi093.stdout:3265: writing smithi093153278-418 from 471157 to 905813 tid 1 2023-08-27T20:54:51.249 INFO:tasks.rados.rados.0.smithi093.stdout:3265: writing smithi093153278-418 from 1636333 to 2050711 tid 2 2023-08-27T20:54:51.249 INFO:tasks.rados.rados.0.smithi093.stdout:3259: done (6 left) 2023-08-27T20:54:51.249 INFO:tasks.rados.rados.0.smithi093.stdout:3260: done (5 left) 2023-08-27T20:54:51.249 INFO:tasks.rados.rados.0.smithi093.stdout:3261: done (4 left) 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3262: done (3 left) 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3263: done (2 left) 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3264: done (1 left) 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3266: read oid 361 snap -1 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3266: expect (ObjNum 360 snap 0 seq_num 360) 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3267: write oid 38 current snap is 0 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3265: finishing write tid 1 to smithi093153278-418 2023-08-27T20:54:51.250 INFO:tasks.rados.rados.0.smithi093.stdout:3267: seq_num 2052 ranges {487964=509905,1694815=674389} 2023-08-27T20:54:51.254 INFO:tasks.rados.rados.0.smithi093.stdout:3267: writing smithi093153278-38 from 487964 to 997869 tid 1 2023-08-27T20:54:51.262 INFO:tasks.rados.rados.0.smithi093.stdout:3267: writing smithi093153278-38 from 1694815 to 2369204 tid 2 2023-08-27T20:54:51.262 INFO:tasks.rados.rados.0.smithi093.stdout:3268: write oid 25 current snap is 0 2023-08-27T20:54:51.262 INFO:tasks.rados.rados.0.smithi093.stdout:3266: expect (ObjNum 360 snap 0 seq_num 360) 2023-08-27T20:54:51.324 INFO:tasks.rados.rados.0.smithi093.stdout:3265: finishing write tid 2 to smithi093153278-418 2023-08-27T20:54:51.324 INFO:tasks.rados.rados.0.smithi093.stdout:3265: finishing write tid 3 to smithi093153278-418 2023-08-27T20:54:51.324 INFO:tasks.rados.rados.0.smithi093.stdout:3268: seq_num 2053 ranges {504771=701507,1869650=418047} 2023-08-27T20:54:51.330 INFO:tasks.rados.rados.0.smithi093.stdout:3268: writing smithi093153278-25 from 504771 to 1206278 tid 1 2023-08-27T20:54:51.337 INFO:tasks.rados.rados.0.smithi093.stdout:3268: writing smithi093153278-25 from 1869650 to 2287697 tid 2 2023-08-27T20:54:51.337 INFO:tasks.rados.rados.0.smithi093.stdout:3266: done (3 left) 2023-08-27T20:54:51.337 INFO:tasks.rados.rados.0.smithi093.stdout:3269: read oid 273 snap -1 2023-08-27T20:54:51.337 INFO:tasks.rados.rados.0.smithi093.stdout:3269: expect deleted 2023-08-27T20:54:51.337 INFO:tasks.rados.rados.0.smithi093.stdout:3270: read oid 52 snap -1 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3270: expect deleted 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3271: delete oid 123 current snap is 0 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3265: finishing write tid 4 to smithi093153278-418 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 418 v 455 (ObjNum 2051 snap 0 seq_num 2051) dirty exists 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3265: left oid 418 (ObjNum 2051 snap 0 seq_num 2051) 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3267: finishing write tid 1 to smithi093153278-38 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3267: finishing write tid 2 to smithi093153278-38 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3267: finishing write tid 3 to smithi093153278-38 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3267: finishing write tid 4 to smithi093153278-38 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 38 v 372 (ObjNum 2052 snap 0 seq_num 2052) dirty exists 2023-08-27T20:54:51.338 INFO:tasks.rados.rados.0.smithi093.stdout:3267: left oid 38 (ObjNum 2052 snap 0 seq_num 2052) 2023-08-27T20:54:51.339 INFO:tasks.rados.rados.0.smithi093.stdout:3268: finishing write tid 1 to smithi093153278-25 2023-08-27T20:54:51.339 INFO:tasks.rados.rados.0.smithi093.stdout:3265: done (5 left) 2023-08-27T20:54:51.339 INFO:tasks.rados.rados.0.smithi093.stdout:3267: done (4 left) 2023-08-27T20:54:51.339 INFO:tasks.rados.rados.0.smithi093.stdout:3269: done (3 left) 2023-08-27T20:54:51.340 INFO:tasks.rados.rados.0.smithi093.stdout:3271: done (2 left) 2023-08-27T20:54:51.340 INFO:tasks.rados.rados.0.smithi093.stdout:3272: read oid 55 snap -1 2023-08-27T20:54:51.340 INFO:tasks.rados.rados.0.smithi093.stdout:3272: expect (ObjNum 1970 snap 0 seq_num 1970) 2023-08-27T20:54:51.340 INFO:tasks.rados.rados.0.smithi093.stdout:3273: read oid 138 snap -1 2023-08-27T20:54:51.340 INFO:tasks.rados.rados.0.smithi093.stdout:3273: expect (ObjNum 1886 snap 0 seq_num 1886) 2023-08-27T20:54:51.340 INFO:tasks.rados.rados.0.smithi093.stdout:3274: write oid 413 current snap is 0 2023-08-27T20:54:51.340 INFO:tasks.rados.rados.0.smithi093.stdout:3274: seq_num 2055 ranges {538385=452005,1470261=421716} 2023-08-27T20:54:51.344 INFO:tasks.rados.rados.0.smithi093.stdout:3274: writing smithi093153278-413 from 538385 to 990390 tid 1 2023-08-27T20:54:51.350 INFO:tasks.rados.rados.0.smithi093.stdout:3274: writing smithi093153278-413 from 1470261 to 1891977 tid 2 2023-08-27T20:54:51.350 INFO:tasks.rados.rados.0.smithi093.stdout:3275: read oid 294 snap -1 2023-08-27T20:54:51.350 INFO:tasks.rados.rados.0.smithi093.stdout:3275: expect (ObjNum 1410 snap 0 seq_num 1410) 2023-08-27T20:54:51.350 INFO:tasks.rados.rados.0.smithi093.stdout:3276: read oid 347 snap -1 2023-08-27T20:54:51.350 INFO:tasks.rados.rados.0.smithi093.stdout:3276: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:54:51.350 INFO:tasks.rados.rados.0.smithi093.stdout:3277: read oid 47 snap -1 2023-08-27T20:54:51.350 INFO:tasks.rados.rados.0.smithi093.stdout:3277: expect (ObjNum 1826 snap 0 seq_num 1826) 2023-08-27T20:54:51.351 INFO:tasks.rados.rados.0.smithi093.stdout:3278: write oid 73 current snap is 0 2023-08-27T20:54:51.351 INFO:tasks.rados.rados.0.smithi093.stdout:3278: seq_num 2056 ranges {555192=527254,1528743=565374} 2023-08-27T20:54:51.356 INFO:tasks.rados.rados.0.smithi093.stdout:3278: writing smithi093153278-73 from 555192 to 1082446 tid 1 2023-08-27T20:54:51.362 INFO:tasks.rados.rados.0.smithi093.stdout:3278: writing smithi093153278-73 from 1528743 to 2094117 tid 2 2023-08-27T20:54:51.362 INFO:tasks.rados.rados.0.smithi093.stdout:3279: delete oid 229 current snap is 0 2023-08-27T20:54:51.362 INFO:tasks.rados.rados.0.smithi093.stdout:3268: finishing write tid 2 to smithi093153278-25 2023-08-27T20:54:51.362 INFO:tasks.rados.rados.0.smithi093.stdout:3268: finishing write tid 3 to smithi093153278-25 2023-08-27T20:54:51.363 INFO:tasks.rados.rados.0.smithi093.stdout:3268: finishing write tid 4 to smithi093153278-25 2023-08-27T20:54:51.363 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 25 v 412 (ObjNum 2053 snap 0 seq_num 2053) dirty exists 2023-08-27T20:54:51.363 INFO:tasks.rados.rados.0.smithi093.stdout:3268: left oid 25 (ObjNum 2053 snap 0 seq_num 2053) 2023-08-27T20:54:51.363 INFO:tasks.rados.rados.0.smithi093.stdout:3273: expect (ObjNum 1886 snap 0 seq_num 1886) 2023-08-27T20:54:51.544 INFO:tasks.rados.rados.0.smithi093.stdout:3272: expect (ObjNum 1970 snap 0 seq_num 1970) 2023-08-27T20:54:51.713 INFO:tasks.rados.rados.0.smithi093.stdout:3274: finishing write tid 1 to smithi093153278-413 2023-08-27T20:54:51.713 INFO:tasks.rados.rados.0.smithi093.stdout:3274: finishing write tid 2 to smithi093153278-413 2023-08-27T20:54:51.713 INFO:tasks.rados.rados.0.smithi093.stdout:3274: finishing write tid 3 to smithi093153278-413 2023-08-27T20:54:51.713 INFO:tasks.rados.rados.0.smithi093.stdout:3274: finishing write tid 4 to smithi093153278-413 2023-08-27T20:54:51.713 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 413 v 383 (ObjNum 2055 snap 0 seq_num 2055) dirty exists 2023-08-27T20:54:51.713 INFO:tasks.rados.rados.0.smithi093.stdout:3274: left oid 413 (ObjNum 2055 snap 0 seq_num 2055) 2023-08-27T20:54:51.713 INFO:tasks.rados.rados.0.smithi093.stdout:3276: expect (ObjNum 346 snap 0 seq_num 346) 2023-08-27T20:54:51.831 INFO:tasks.rados.rados.0.smithi093.stdout:3277: expect (ObjNum 1826 snap 0 seq_num 1826) 2023-08-27T20:54:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:51 smithi136 ceph-mon[113531]: pgmap v39: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 84 MiB/s rd, 12 MiB/s wr, 117 op/s 2023-08-27T20:54:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2048233395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:51.953 INFO:tasks.rados.rados.0.smithi093.stdout:3275: expect (ObjNum 1410 snap 0 seq_num 1410) 2023-08-27T20:54:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:51 smithi093 ceph-mon[111335]: pgmap v39: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 84 MiB/s rd, 12 MiB/s wr, 117 op/s 2023-08-27T20:54:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2048233395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:51 smithi093 ceph-mon[118902]: pgmap v39: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 84 MiB/s rd, 12 MiB/s wr, 117 op/s 2023-08-27T20:54:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2048233395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:51.627Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:51.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3278: finishing write tid 1 to smithi093153278-73 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3278: finishing write tid 2 to smithi093153278-73 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3278: finishing write tid 3 to smithi093153278-73 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3278: finishing write tid 4 to smithi093153278-73 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 73 v 458 (ObjNum 2056 snap 0 seq_num 2056) dirty exists 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3278: left oid 73 (ObjNum 2056 snap 0 seq_num 2056) 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3268: done (9 left) 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3270: done (8 left) 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3272: done (7 left) 2023-08-27T20:54:52.115 INFO:tasks.rados.rados.0.smithi093.stdout:3273: done (6 left) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3274: done (5 left) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3275: done (4 left) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3276: done (3 left) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3277: done (2 left) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3278: done (1 left) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3279: done (0 left) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3280: read oid 149 snap -1 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3280: expect (ObjNum 1890 snap 0 seq_num 1890) 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3281: write oid 326 current snap is 0 2023-08-27T20:54:52.116 INFO:tasks.rados.rados.0.smithi093.stdout:3281: seq_num 2058 ranges {588806=677752,2045707=543099} 2023-08-27T20:54:52.121 INFO:tasks.rados.rados.0.smithi093.stdout:3281: writing smithi093153278-326 from 588806 to 1266558 tid 1 2023-08-27T20:54:52.129 INFO:tasks.rados.rados.0.smithi093.stdout:3281: writing smithi093153278-326 from 2045707 to 2588806 tid 2 2023-08-27T20:54:52.129 INFO:tasks.rados.rados.0.smithi093.stdout:3282: read oid 380 snap -1 2023-08-27T20:54:52.129 INFO:tasks.rados.rados.0.smithi093.stdout:3282: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:54:52.129 INFO:tasks.rados.rados.0.smithi093.stdout:3283: write oid 91 current snap is 0 2023-08-27T20:54:52.129 INFO:tasks.rados.rados.0.smithi093.stdout:3283: seq_num 2059 ranges {605613=753001,1987836=429054} 2023-08-27T20:54:52.136 INFO:tasks.rados.rados.0.smithi093.stdout:3283: writing smithi093153278-91 from 605613 to 1358614 tid 1 2023-08-27T20:54:52.142 INFO:tasks.rados.rados.0.smithi093.stdout:3283: writing smithi093153278-91 from 1987836 to 2416890 tid 2 2023-08-27T20:54:52.142 INFO:tasks.rados.rados.0.smithi093.stdout:3284: read oid 379 snap -1 2023-08-27T20:54:52.142 INFO:tasks.rados.rados.0.smithi093.stdout:3284: expect (ObjNum 2029 snap 0 seq_num 2029) 2023-08-27T20:54:52.143 INFO:tasks.rados.rados.0.smithi093.stdout:3285: delete oid 216 current snap is 0 2023-08-27T20:54:52.143 INFO:tasks.rados.rados.0.smithi093.stdout:3280: expect (ObjNum 1890 snap 0 seq_num 1890) 2023-08-27T20:54:52.257 INFO:tasks.rados.rados.0.smithi093.stdout:3281: finishing write tid 1 to smithi093153278-326 2023-08-27T20:54:52.257 INFO:tasks.rados.rados.0.smithi093.stdout:3281: finishing write tid 2 to smithi093153278-326 2023-08-27T20:54:52.257 INFO:tasks.rados.rados.0.smithi093.stdout:3281: finishing write tid 3 to smithi093153278-326 2023-08-27T20:54:52.257 INFO:tasks.rados.rados.0.smithi093.stdout:3281: finishing write tid 4 to smithi093153278-326 2023-08-27T20:54:52.257 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 340 (ObjNum 2058 snap 0 seq_num 2058) dirty exists 2023-08-27T20:54:52.257 INFO:tasks.rados.rados.0.smithi093.stdout:3281: left oid 326 (ObjNum 2058 snap 0 seq_num 2058) 2023-08-27T20:54:52.257 INFO:tasks.rados.rados.0.smithi093.stdout:3282: expect (ObjNum 910 snap 0 seq_num 910) 2023-08-27T20:54:52.421 INFO:tasks.rados.rados.0.smithi093.stdout:3283: finishing write tid 1 to smithi093153278-91 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3283: finishing write tid 2 to smithi093153278-91 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3280: done (5 left) 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3281: done (4 left) 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3282: done (3 left) 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3285: done (2 left) 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3286: read oid 385 snap -1 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3286: expect deleted 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3287: write oid 253 current snap is 0 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3283: finishing write tid 3 to smithi093153278-91 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3283: finishing write tid 4 to smithi093153278-91 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 91 v 479 (ObjNum 2059 snap 0 seq_num 2059) dirty exists 2023-08-27T20:54:52.422 INFO:tasks.rados.rados.0.smithi093.stdout:3283: left oid 91 (ObjNum 2059 snap 0 seq_num 2059) 2023-08-27T20:54:52.423 INFO:tasks.rados.rados.0.smithi093.stdout:3284: expect (ObjNum 2029 snap 0 seq_num 2029) 2023-08-27T20:54:52.513 INFO:tasks.rados.rados.0.smithi093.stdout:3287: seq_num 2061 ranges {639227=619852,1821153=432723} 2023-08-27T20:54:52.519 INFO:tasks.rados.rados.0.smithi093.stdout:3287: writing smithi093153278-253 from 639227 to 1259079 tid 1 2023-08-27T20:54:52.525 INFO:tasks.rados.rados.0.smithi093.stdout:3287: writing smithi093153278-253 from 1821153 to 2253876 tid 2 2023-08-27T20:54:52.525 INFO:tasks.rados.rados.0.smithi093.stdout:3283: done (3 left) 2023-08-27T20:54:52.525 INFO:tasks.rados.rados.0.smithi093.stdout:3284: done (2 left) 2023-08-27T20:54:52.525 INFO:tasks.rados.rados.0.smithi093.stdout:3286: done (1 left) 2023-08-27T20:54:52.526 INFO:tasks.rados.rados.0.smithi093.stdout:3288: read oid 198 snap -1 2023-08-27T20:54:52.526 INFO:tasks.rados.rados.0.smithi093.stdout:3288: expect deleted 2023-08-27T20:54:52.526 INFO:tasks.rados.rados.0.smithi093.stdout:3289: write oid 308 current snap is 0 2023-08-27T20:54:52.526 INFO:tasks.rados.rados.0.smithi093.stdout:3289: seq_num 2062 ranges {656034=695101,1879635=576381} 2023-08-27T20:54:52.532 INFO:tasks.rados.rados.0.smithi093.stdout:3289: writing smithi093153278-308 from 656034 to 1351135 tid 1 2023-08-27T20:54:52.539 INFO:tasks.rados.rados.0.smithi093.stdout:3289: writing smithi093153278-308 from 1879635 to 2456016 tid 2 2023-08-27T20:54:52.539 INFO:tasks.rados.rados.0.smithi093.stdout:3290: read oid 475 snap -1 2023-08-27T20:54:52.539 INFO:tasks.rados.rados.0.smithi093.stdout:3290: expect deleted 2023-08-27T20:54:52.539 INFO:tasks.rados.rados.0.smithi093.stdout:3291: write oid 150 current snap is 0 2023-08-27T20:54:52.539 INFO:tasks.rados.rados.0.smithi093.stdout:3287: finishing write tid 1 to smithi093153278-253 2023-08-27T20:54:52.539 INFO:tasks.rados.rados.0.smithi093.stdout:3291: seq_num 2063 ranges {672841=770350,2221764=436392} 2023-08-27T20:54:52.546 INFO:tasks.rados.rados.0.smithi093.stdout:3291: writing smithi093153278-150 from 672841 to 1443191 tid 1 2023-08-27T20:54:52.553 INFO:tasks.rados.rados.0.smithi093.stdout:3291: writing smithi093153278-150 from 2221764 to 2658156 tid 2 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:3292: write oid 386 current snap is 0 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:3287: finishing write tid 2 to smithi093153278-253 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:3287: finishing write tid 3 to smithi093153278-253 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:3287: finishing write tid 4 to smithi093153278-253 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 253 v 441 (ObjNum 2061 snap 0 seq_num 2061) dirty exists 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:3287: left oid 253 (ObjNum 2061 snap 0 seq_num 2061) 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:3289: finishing write tid 1 to smithi093153278-308 2023-08-27T20:54:52.554 INFO:tasks.rados.rados.0.smithi093.stdout:3292: seq_num 2064 ranges {689648=445599,1880246=580050} 2023-08-27T20:54:52.559 INFO:tasks.rados.rados.0.smithi093.stdout:3292: writing smithi093153278-386 from 689648 to 1135247 tid 1 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3292: writing smithi093153278-386 from 1880246 to 2460296 tid 2 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3287: done (5 left) 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3288: done (4 left) 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3293: read oid 183 snap -1 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3293: expect (ObjNum 1569 snap 0 seq_num 1569) 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3294: read oid 237 snap -1 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3294: expect (ObjNum 1882 snap 0 seq_num 1882) 2023-08-27T20:54:52.568 INFO:tasks.rados.rados.0.smithi093.stdout:3295: read oid 267 snap -1 2023-08-27T20:54:52.569 INFO:tasks.rados.rados.0.smithi093.stdout:3295: expect (ObjNum 1771 snap 0 seq_num 1771) 2023-08-27T20:54:52.569 INFO:tasks.rados.rados.0.smithi093.stdout:3296: read oid 201 snap -1 2023-08-27T20:54:52.569 INFO:tasks.rados.rados.0.smithi093.stdout:3296: expect (ObjNum 1769 snap 0 seq_num 1769) 2023-08-27T20:54:52.569 INFO:tasks.rados.rados.0.smithi093.stdout:3297: write oid 9 current snap is 0 2023-08-27T20:54:52.569 INFO:tasks.rados.rados.0.smithi093.stdout:3289: finishing write tid 2 to smithi093153278-308 2023-08-27T20:54:52.569 INFO:tasks.rados.rados.0.smithi093.stdout:3289: finishing write tid 3 to smithi093153278-308 2023-08-27T20:54:52.569 INFO:tasks.rados.rados.0.smithi093.stdout:3297: seq_num 2065 ranges {706455=520848,1938728=440061} 2023-08-27T20:54:52.574 INFO:tasks.rados.rados.0.smithi093.stdout:3297: writing smithi093153278-9 from 706455 to 1227303 tid 1 2023-08-27T20:54:52.581 INFO:tasks.rados.rados.0.smithi093.stdout:3297: writing smithi093153278-9 from 1938728 to 2378789 tid 2 2023-08-27T20:54:52.581 INFO:tasks.rados.rados.0.smithi093.stdout:3290: done (8 left) 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3298: read oid 144 snap -1 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3298: expect (ObjNum 1672 snap 0 seq_num 1672) 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3299: read oid 445 snap -1 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3299: expect (ObjNum 1316 snap 0 seq_num 1316) 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3300: delete oid 272 current snap is 0 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3289: finishing write tid 4 to smithi093153278-308 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 308 v 388 (ObjNum 2062 snap 0 seq_num 2062) dirty exists 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3289: left oid 308 (ObjNum 2062 snap 0 seq_num 2062) 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3291: finishing write tid 1 to smithi093153278-150 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3291: finishing write tid 2 to smithi093153278-150 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3291: finishing write tid 3 to smithi093153278-150 2023-08-27T20:54:52.582 INFO:tasks.rados.rados.0.smithi093.stdout:3291: finishing write tid 4 to smithi093153278-150 2023-08-27T20:54:52.583 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 150 v 482 (ObjNum 2063 snap 0 seq_num 2063) dirty exists 2023-08-27T20:54:52.583 INFO:tasks.rados.rados.0.smithi093.stdout:3291: left oid 150 (ObjNum 2063 snap 0 seq_num 2063) 2023-08-27T20:54:52.583 INFO:tasks.rados.rados.0.smithi093.stdout:3292: finishing write tid 1 to smithi093153278-386 2023-08-27T20:54:52.583 INFO:tasks.rados.rados.0.smithi093.stdout:3292: finishing write tid 2 to smithi093153278-386 2023-08-27T20:54:52.583 INFO:tasks.rados.rados.0.smithi093.stdout:3295: expect (ObjNum 1771 snap 0 seq_num 1771) 2023-08-27T20:54:52.846 INFO:tasks.rados.rados.0.smithi093.stdout:3292: finishing write tid 3 to smithi093153278-386 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3292: finishing write tid 4 to smithi093153278-386 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 386 v 375 (ObjNum 2064 snap 0 seq_num 2064) dirty exists 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3292: left oid 386 (ObjNum 2064 snap 0 seq_num 2064) 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3289: done (10 left) 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3291: done (9 left) 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3292: done (8 left) 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3295: done (7 left) 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3300: done (6 left) 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3301: write oid 149 current snap is 0 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3297: finishing write tid 1 to smithi093153278-9 2023-08-27T20:54:52.847 INFO:tasks.rados.rados.0.smithi093.stdout:3293: expect (ObjNum 1569 snap 0 seq_num 1569) 2023-08-27T20:54:52.966 INFO:tasks.rados.rados.0.smithi093.stdout:3298: expect (ObjNum 1672 snap 0 seq_num 1672) 2023-08-27T20:54:53.034 INFO:tasks.rados.rados.0.smithi093.stdout:3299: expect (ObjNum 1316 snap 0 seq_num 1316) 2023-08-27T20:54:53.105 INFO:tasks.rados.rados.0.smithi093.stdout:3297: finishing write tid 2 to smithi093153278-9 2023-08-27T20:54:53.115 INFO:tasks.rados.rados.0.smithi093.stdout:3297: finishing write tid 3 to smithi093153278-9 2023-08-27T20:54:53.116 INFO:tasks.rados.rados.0.smithi093.stdout:3297: finishing write tid 4 to smithi093153278-9 2023-08-27T20:54:53.116 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 9 v 436 (ObjNum 2065 snap 0 seq_num 2065) dirty exists 2023-08-27T20:54:53.116 INFO:tasks.rados.rados.0.smithi093.stdout:3297: left oid 9 (ObjNum 2065 snap 0 seq_num 2065) 2023-08-27T20:54:53.116 INFO:tasks.rados.rados.0.smithi093.stdout:3294: expect (ObjNum 1882 snap 0 seq_num 1882) 2023-08-27T20:54:53.269 INFO:tasks.rados.rados.0.smithi093.stdout:3296: expect (ObjNum 1769 snap 0 seq_num 1769) 2023-08-27T20:54:53.430 INFO:tasks.rados.rados.0.smithi093.stdout:3301: seq_num 2067 ranges {740069=671346,1939339=443730} 2023-08-27T20:54:53.437 INFO:tasks.rados.rados.0.smithi093.stdout:3301: writing smithi093153278-149 from 740069 to 1411415 tid 1 2023-08-27T20:54:53.443 INFO:tasks.rados.rados.0.smithi093.stdout:3301: writing smithi093153278-149 from 1939339 to 2383069 tid 2 2023-08-27T20:54:53.443 INFO:tasks.rados.rados.0.smithi093.stdout:3293: done (6 left) 2023-08-27T20:54:53.444 INFO:tasks.rados.rados.0.smithi093.stdout:3294: done (5 left) 2023-08-27T20:54:53.444 INFO:tasks.rados.rados.0.smithi093.stdout:3296: done (4 left) 2023-08-27T20:54:53.444 INFO:tasks.rados.rados.0.smithi093.stdout:3297: done (3 left) 2023-08-27T20:54:53.444 INFO:tasks.rados.rados.0.smithi093.stdout:3298: done (2 left) 2023-08-27T20:54:53.444 INFO:tasks.rados.rados.0.smithi093.stdout:3299: done (1 left) 2023-08-27T20:54:53.444 INFO:tasks.rados.rados.0.smithi093.stdout:3302: write oid 222 current snap is 0 2023-08-27T20:54:53.444 INFO:tasks.rados.rados.0.smithi093.stdout:3302: seq_num 2068 ranges {756876=462948,1714174=587388} 2023-08-27T20:54:53.450 INFO:tasks.rados.rados.0.smithi093.stdout:3302: writing smithi093153278-222 from 756876 to 1219824 tid 1 2023-08-27T20:54:53.456 INFO:tasks.rados.rados.0.smithi093.stdout:3302: writing smithi093153278-222 from 1714174 to 2301562 tid 2 2023-08-27T20:54:53.456 INFO:tasks.rados.rados.0.smithi093.stdout:3301: finishing write tid 1 to smithi093153278-149 2023-08-27T20:54:53.457 INFO:tasks.rados.rados.0.smithi093.stdout:3303: read oid 33 snap -1 2023-08-27T20:54:53.457 INFO:tasks.rados.rados.0.smithi093.stdout:3303: expect (ObjNum 1732 snap 0 seq_num 1732) 2023-08-27T20:54:53.457 INFO:tasks.rados.rados.0.smithi093.stdout:3304: write oid 425 current snap is 0 2023-08-27T20:54:53.457 INFO:tasks.rados.rados.0.smithi093.stdout:3301: finishing write tid 2 to smithi093153278-149 2023-08-27T20:54:53.457 INFO:tasks.rados.rados.0.smithi093.stdout:3304: seq_num 2069 ranges {773683=538197,1772656=731046} 2023-08-27T20:54:53.464 INFO:tasks.rados.rados.0.smithi093.stdout:3304: writing smithi093153278-425 from 773683 to 1311880 tid 1 2023-08-27T20:54:53.471 INFO:tasks.rados.rados.0.smithi093.stdout:3304: writing smithi093153278-425 from 1772656 to 2503702 tid 2 2023-08-27T20:54:53.471 INFO:tasks.rados.rados.0.smithi093.stdout:3305: write oid 233 current snap is 0 2023-08-27T20:54:53.471 INFO:tasks.rados.rados.0.smithi093.stdout:3301: finishing write tid 3 to smithi093153278-149 2023-08-27T20:54:53.471 INFO:tasks.rados.rados.0.smithi093.stdout:3305: seq_num 2070 ranges {790490=613446,1831138=591057} 2023-08-27T20:54:53.479 INFO:tasks.rados.rados.0.smithi093.stdout:3305: writing smithi093153278-233 from 790490 to 1403936 tid 1 2023-08-27T20:54:53.485 INFO:tasks.rados.rados.0.smithi093.stdout:3305: writing smithi093153278-233 from 1831138 to 2422195 tid 2 2023-08-27T20:54:53.485 INFO:tasks.rados.rados.0.smithi093.stdout:3306: write oid 343 current snap is 0 2023-08-27T20:54:53.485 INFO:tasks.rados.rados.0.smithi093.stdout:3301: finishing write tid 4 to smithi093153278-149 2023-08-27T20:54:53.485 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 149 v 416 (ObjNum 2067 snap 0 seq_num 2067) dirty exists 2023-08-27T20:54:53.485 INFO:tasks.rados.rados.0.smithi093.stdout:3301: left oid 149 (ObjNum 2067 snap 0 seq_num 2067) 2023-08-27T20:54:53.485 INFO:tasks.rados.rados.0.smithi093.stdout:3302: finishing write tid 1 to smithi093153278-222 2023-08-27T20:54:53.485 INFO:tasks.rados.rados.0.smithi093.stdout:3306: seq_num 2071 ranges {407297=688695,1773267=734715} 2023-08-27T20:54:53.490 INFO:tasks.rados.rados.0.smithi093.stdout:3306: writing smithi093153278-343 from 407297 to 1095992 tid 1 2023-08-27T20:54:53.499 INFO:tasks.rados.rados.0.smithi093.stdout:3306: writing smithi093153278-343 from 1773267 to 2507982 tid 2 2023-08-27T20:54:53.499 INFO:tasks.rados.rados.0.smithi093.stdout:3301: done (5 left) 2023-08-27T20:54:53.499 INFO:tasks.rados.rados.0.smithi093.stdout:3307: read oid 430 snap -1 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3307: expect (ObjNum 1861 snap 0 seq_num 1861) 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3308: read oid 407 snap -1 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3308: expect (ObjNum 1136 snap 0 seq_num 1136) 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3309: read oid 455 snap -1 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3309: expect (ObjNum 1670 snap 0 seq_num 1670) 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3310: read oid 359 snap -1 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3310: expect deleted 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3311: delete oid 107 current snap is 0 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3302: finishing write tid 2 to smithi093153278-222 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3302: finishing write tid 3 to smithi093153278-222 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:3302: finishing write tid 4 to smithi093153278-222 2023-08-27T20:54:53.500 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 222 v 440 (ObjNum 2068 snap 0 seq_num 2068) dirty exists 2023-08-27T20:54:53.501 INFO:tasks.rados.rados.0.smithi093.stdout:3302: left oid 222 (ObjNum 2068 snap 0 seq_num 2068) 2023-08-27T20:54:53.501 INFO:tasks.rados.rados.0.smithi093.stdout:3303: expect (ObjNum 1732 snap 0 seq_num 1732) 2023-08-27T20:54:53.631 INFO:tasks.rados.rados.0.smithi093.stdout:3304: finishing write tid 1 to smithi093153278-425 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3304: finishing write tid 2 to smithi093153278-425 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3304: finishing write tid 3 to smithi093153278-425 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3304: finishing write tid 4 to smithi093153278-425 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 425 v 419 (ObjNum 2069 snap 0 seq_num 2069) dirty exists 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3304: left oid 425 (ObjNum 2069 snap 0 seq_num 2069) 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3305: finishing write tid 1 to smithi093153278-233 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3302: done (9 left) 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3303: done (8 left) 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3304: done (7 left) 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3311: done (6 left) 2023-08-27T20:54:53.632 INFO:tasks.rados.rados.0.smithi093.stdout:3312: write oid 443 current snap is 0 2023-08-27T20:54:53.633 INFO:tasks.rados.rados.0.smithi093.stdout:3305: finishing write tid 2 to smithi093153278-233 2023-08-27T20:54:53.633 INFO:tasks.rados.rados.0.smithi093.stdout:3305: finishing write tid 3 to smithi093153278-233 2023-08-27T20:54:53.633 INFO:tasks.rados.rados.0.smithi093.stdout:3305: finishing write tid 4 to smithi093153278-233 2023-08-27T20:54:53.634 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 233 v 461 (ObjNum 2070 snap 0 seq_num 2070) dirty exists 2023-08-27T20:54:53.634 INFO:tasks.rados.rados.0.smithi093.stdout:3305: left oid 233 (ObjNum 2070 snap 0 seq_num 2070) 2023-08-27T20:54:53.634 INFO:tasks.rados.rados.0.smithi093.stdout:3306: finishing write tid 1 to smithi093153278-343 2023-08-27T20:54:53.634 INFO:tasks.rados.rados.0.smithi093.stdout:3312: seq_num 2073 ranges {440911=439193,1490231=738384} 2023-08-27T20:54:53.636 INFO:tasks.rados.rados.0.smithi093.stdout:3312: writing smithi093153278-443 from 440911 to 880104 tid 1 2023-08-27T20:54:53.643 INFO:tasks.rados.rados.0.smithi093.stdout:3312: writing smithi093153278-443 from 1490231 to 2228615 tid 2 2023-08-27T20:54:53.643 INFO:tasks.rados.rados.0.smithi093.stdout:3305: done (6 left) 2023-08-27T20:54:53.643 INFO:tasks.rados.rados.0.smithi093.stdout:3313: write oid 416 current snap is 0 2023-08-27T20:54:53.644 INFO:tasks.rados.rados.0.smithi093.stdout:3309: expect (ObjNum 1670 snap 0 seq_num 1670) 2023-08-27T20:54:53.769 INFO:tasks.rados.rados.0.smithi093.stdout:3306: finishing write tid 2 to smithi093153278-343 2023-08-27T20:54:53.769 INFO:tasks.rados.rados.0.smithi093.stdout:3306: finishing write tid 3 to smithi093153278-343 2023-08-27T20:54:53.769 INFO:tasks.rados.rados.0.smithi093.stdout:3306: finishing write tid 4 to smithi093153278-343 2023-08-27T20:54:53.769 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 343 v 389 (ObjNum 2071 snap 0 seq_num 2071) dirty exists 2023-08-27T20:54:53.769 INFO:tasks.rados.rados.0.smithi093.stdout:3306: left oid 343 (ObjNum 2071 snap 0 seq_num 2071) 2023-08-27T20:54:53.769 INFO:tasks.rados.rados.0.smithi093.stdout:3307: expect (ObjNum 1861 snap 0 seq_num 1861) 2023-08-27T20:54:53.870 INFO:tasks.rados.rados.0.smithi093.stdout:3308: expect (ObjNum 1136 snap 0 seq_num 1136) 2023-08-27T20:54:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:53 smithi136 ceph-mon[113531]: pgmap v40: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:54:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/887528007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:53 smithi093 ceph-mon[111335]: pgmap v40: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:54:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/887528007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:53 smithi093 ceph-mon[118902]: pgmap v40: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:54:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/887528007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:53.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:54:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:54:53.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:54:54.044 INFO:tasks.rados.rados.0.smithi093.stdout:3312: finishing write tid 1 to smithi093153278-443 2023-08-27T20:54:54.044 INFO:tasks.rados.rados.0.smithi093.stdout:3312: finishing write tid 2 to smithi093153278-443 2023-08-27T20:54:54.045 INFO:tasks.rados.rados.0.smithi093.stdout:3313: seq_num 2074 ranges {457718=514442,1548713=598395} 2023-08-27T20:54:54.049 INFO:tasks.rados.rados.0.smithi093.stdout:3313: writing smithi093153278-416 from 457718 to 972160 tid 1 2023-08-27T20:54:54.057 INFO:tasks.rados.rados.0.smithi093.stdout:3313: writing smithi093153278-416 from 1548713 to 2147108 tid 2 2023-08-27T20:54:54.057 INFO:tasks.rados.rados.0.smithi093.stdout:3306: done (6 left) 2023-08-27T20:54:54.057 INFO:tasks.rados.rados.0.smithi093.stdout:3307: done (5 left) 2023-08-27T20:54:54.057 INFO:tasks.rados.rados.0.smithi093.stdout:3308: done (4 left) 2023-08-27T20:54:54.057 INFO:tasks.rados.rados.0.smithi093.stdout:3309: done (3 left) 2023-08-27T20:54:54.057 INFO:tasks.rados.rados.0.smithi093.stdout:3310: done (2 left) 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:3314: read oid 252 snap -1 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:3314: expect (ObjNum 1725 snap 0 seq_num 1725) 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:3315: write oid 178 current snap is 0 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:3312: finishing write tid 3 to smithi093153278-443 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:3312: finishing write tid 4 to smithi093153278-443 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 443 (ObjNum 2073 snap 0 seq_num 2073) dirty exists 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:3312: left oid 443 (ObjNum 2073 snap 0 seq_num 2073) 2023-08-27T20:54:54.058 INFO:tasks.rados.rados.0.smithi093.stdout:3315: seq_num 2075 ranges {474525=589691,1490842=742053,2832710=41815} 2023-08-27T20:54:54.062 INFO:tasks.rados.rados.0.smithi093.stdout:3315: writing smithi093153278-178 from 474525 to 1064216 tid 1 2023-08-27T20:54:54.069 INFO:tasks.rados.rados.0.smithi093.stdout:3315: writing smithi093153278-178 from 1490842 to 2232895 tid 2 2023-08-27T20:54:54.073 INFO:tasks.rados.rados.0.smithi093.stdout:3315: writing smithi093153278-178 from 2832710 to 2874525 tid 3 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3312: done (3 left) 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3316: read oid 20 snap -1 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3316: expect (ObjNum 1572 snap 0 seq_num 1572) 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3317: read oid 164 snap -1 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3317: expect (ObjNum 1818 snap 0 seq_num 1818) 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3318: write oid 443 current snap is 0 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3313: finishing write tid 1 to smithi093153278-416 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3313: finishing write tid 2 to smithi093153278-416 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3313: finishing write tid 3 to smithi093153278-416 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:3313: finishing write tid 4 to smithi093153278-416 2023-08-27T20:54:54.074 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 416 v 386 (ObjNum 2074 snap 0 seq_num 2074) dirty exists 2023-08-27T20:54:54.075 INFO:tasks.rados.rados.0.smithi093.stdout:3313: left oid 416 (ObjNum 2074 snap 0 seq_num 2074) 2023-08-27T20:54:54.075 INFO:tasks.rados.rados.0.smithi093.stdout:3314: expect (ObjNum 1725 snap 0 seq_num 1725) 2023-08-27T20:54:54.243 INFO:tasks.rados.rados.0.smithi093.stdout:3315: finishing write tid 1 to smithi093153278-178 2023-08-27T20:54:54.938 INFO:tasks.rados.rados.0.smithi093.stdout:3315: finishing write tid 2 to smithi093153278-178 2023-08-27T20:54:54.938 INFO:tasks.rados.rados.0.smithi093.stdout:3316: expect (ObjNum 1572 snap 0 seq_num 1572) 2023-08-27T20:54:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:54.948 INFO:tasks.rados.rados.0.smithi093.stdout:3315: finishing write tid 3 to smithi093153278-178 2023-08-27T20:54:54.948 INFO:tasks.rados.rados.0.smithi093.stdout:3315: finishing write tid 4 to smithi093153278-178 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3315: finishing write tid 5 to smithi093153278-178 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 178 v 262 (ObjNum 2075 snap 0 seq_num 2075) dirty exists 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3315: left oid 178 (ObjNum 2075 snap 0 seq_num 2075) 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3317: expect (ObjNum 1818 snap 0 seq_num 1818) 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3318: seq_num 2076 ranges {491332=781293,2065677=602064} 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3318: writing smithi093153278-443 from 491332 to 1272625 tid 1 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3318: writing smithi093153278-443 from 2065677 to 2667741 tid 2 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3313: done (5 left) 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3314: done (4 left) 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3315: done (3 left) 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3316: done (2 left) 2023-08-27T20:54:54.949 INFO:tasks.rados.rados.0.smithi093.stdout:3317: done (1 left) 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3319: delete oid 257 current snap is 0 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3318: finishing write tid 1 to smithi093153278-443 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3319: done (1 left) 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3320: delete oid 386 current snap is 0 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3318: finishing write tid 2 to smithi093153278-443 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3318: finishing write tid 3 to smithi093153278-443 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3318: finishing write tid 4 to smithi093153278-443 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 446 (ObjNum 2076 snap 0 seq_num 2076) dirty exists 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3318: left oid 443 (ObjNum 2076 snap 0 seq_num 2076) 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3318: done (1 left) 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3320: done (0 left) 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3321: read oid 155 snap -1 2023-08-27T20:54:54.950 INFO:tasks.rados.rados.0.smithi093.stdout:3321: expect deleted 2023-08-27T20:54:54.951 INFO:tasks.rados.rados.0.smithi093.stdout:3322: write oid 294 current snap is 0 2023-08-27T20:54:54.951 INFO:tasks.rados.rados.0.smithi093.stdout:3322: seq_num 2079 ranges {541753=607040,1724770=749391} 2023-08-27T20:54:54.951 INFO:tasks.rados.rados.0.smithi093.stdout:3322: writing smithi093153278-294 from 541753 to 1148793 tid 1 2023-08-27T20:54:54.951 INFO:tasks.rados.rados.0.smithi093.stdout:3322: writing smithi093153278-294 from 1724770 to 2474161 tid 2 2023-08-27T20:54:55.236 INFO:tasks.rados.rados.0.smithi093.stdout:3323: read oid 1 snap -1 2023-08-27T20:54:55.241 INFO:tasks.rados.rados.0.smithi093.stdout:3323: expect (ObjNum 0 snap 0 seq_num 0) 2023-08-27T20:54:55.241 INFO:tasks.rados.rados.0.smithi093.stdout:3324: write oid 460 current snap is 0 2023-08-27T20:54:55.241 INFO:tasks.rados.rados.0.smithi093.stdout:3322: finishing write tid 1 to smithi093153278-294 2023-08-27T20:54:55.242 INFO:tasks.rados.rados.0.smithi093.stdout:3324: seq_num 2080 ranges {558560=682289,1783252=493049} 2023-08-27T20:54:55.242 INFO:tasks.rados.rados.0.smithi093.stdout:3324: writing smithi093153278-460 from 558560 to 1240849 tid 1 2023-08-27T20:54:55.242 INFO:tasks.rados.rados.0.smithi093.stdout:3324: writing smithi093153278-460 from 1783252 to 2276301 tid 2 2023-08-27T20:54:55.243 INFO:tasks.rados.rados.0.smithi093.stdout:3321: done (3 left) 2023-08-27T20:54:55.243 INFO:tasks.rados.rados.0.smithi093.stdout:3325: delete oid 167 current snap is 0 2023-08-27T20:54:55.243 INFO:tasks.rados.rados.0.smithi093.stdout:3322: finishing write tid 2 to smithi093153278-294 2023-08-27T20:54:55.243 INFO:tasks.rados.rados.0.smithi093.stdout:3322: finishing write tid 3 to smithi093153278-294 2023-08-27T20:54:55.244 INFO:tasks.rados.rados.0.smithi093.stdout:3322: finishing write tid 4 to smithi093153278-294 2023-08-27T20:54:55.244 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 294 v 422 (ObjNum 2079 snap 0 seq_num 2079) dirty exists 2023-08-27T20:54:55.244 INFO:tasks.rados.rados.0.smithi093.stdout:3322: left oid 294 (ObjNum 2079 snap 0 seq_num 2079) 2023-08-27T20:54:55.244 INFO:tasks.rados.rados.0.smithi093.stdout:3323: expect (ObjNum 0 snap 0 seq_num 0) 2023-08-27T20:54:55.245 INFO:tasks.rados.rados.0.smithi093.stdout:3324: finishing write tid 1 to smithi093153278-460 2023-08-27T20:54:55.245 INFO:tasks.rados.rados.0.smithi093.stdout:3324: finishing write tid 2 to smithi093153278-460 2023-08-27T20:54:55.246 INFO:tasks.rados.rados.0.smithi093.stdout:3324: finishing write tid 3 to smithi093153278-460 2023-08-27T20:54:55.246 INFO:tasks.rados.rados.0.smithi093.stdout:3324: finishing write tid 4 to smithi093153278-460 2023-08-27T20:54:55.246 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 460 v 343 (ObjNum 2080 snap 0 seq_num 2080) dirty exists 2023-08-27T20:54:55.246 INFO:tasks.rados.rados.0.smithi093.stdout:3324: left oid 460 (ObjNum 2080 snap 0 seq_num 2080) 2023-08-27T20:54:55.246 INFO:tasks.rados.rados.0.smithi093.stdout:3322: done (3 left) 2023-08-27T20:54:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:3323: done (2 left) 2023-08-27T20:54:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:3324: done (1 left) 2023-08-27T20:54:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:3325: done (0 left) 2023-08-27T20:54:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:3326: read oid 413 snap -1 2023-08-27T20:54:55.247 INFO:tasks.rados.rados.0.smithi093.stdout:3326: expect (ObjNum 2055 snap 0 seq_num 2055) 2023-08-27T20:54:55.248 INFO:tasks.rados.rados.0.smithi093.stdout:3327: read oid 283 snap -1 2023-08-27T20:54:55.248 INFO:tasks.rados.rados.0.smithi093.stdout:3327: expect (ObjNum 1896 snap 0 seq_num 1896) 2023-08-27T20:54:55.248 INFO:tasks.rados.rados.0.smithi093.stdout:3328: write oid 109 current snap is 0 2023-08-27T20:54:55.248 INFO:tasks.rados.rados.0.smithi093.stdout:3328: seq_num 2082 ranges {592174=432787,1500216=496718,2624671=367503} 2023-08-27T20:54:55.248 INFO:tasks.rados.rados.0.smithi093.stdout:3328: writing smithi093153278-109 from 592174 to 1024961 tid 1 2023-08-27T20:54:55.248 INFO:tasks.rados.rados.0.smithi093.stdout:3328: writing smithi093153278-109 from 1500216 to 1996934 tid 2 2023-08-27T20:54:55.249 INFO:tasks.rados.rados.0.smithi093.stdout:3328: writing smithi093153278-109 from 2624671 to 2992174 tid 3 2023-08-27T20:54:55.249 INFO:tasks.rados.rados.0.smithi093.stdout:3329: delete oid 380 current snap is 0 2023-08-27T20:54:55.619 INFO:tasks.rados.rados.0.smithi093.stdout:3328: finishing write tid 1 to smithi093153278-109 2023-08-27T20:54:55.619 INFO:tasks.rados.rados.0.smithi093.stdout:3326: expect (ObjNum 2055 snap 0 seq_num 2055) 2023-08-27T20:54:55.619 INFO:tasks.rados.rados.0.smithi093.stdout:3327: expect (ObjNum 1896 snap 0 seq_num 1896) 2023-08-27T20:54:55.620 INFO:tasks.rados.rados.0.smithi093.stdout:3328: finishing write tid 2 to smithi093153278-109 2023-08-27T20:54:55.620 INFO:tasks.rados.rados.0.smithi093.stdout:3328: finishing write tid 3 to smithi093153278-109 2023-08-27T20:54:55.620 INFO:tasks.rados.rados.0.smithi093.stdout:3328: finishing write tid 4 to smithi093153278-109 2023-08-27T20:54:55.620 INFO:tasks.rados.rados.0.smithi093.stdout:3328: finishing write tid 5 to smithi093153278-109 2023-08-27T20:54:55.620 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 109 v 450 (ObjNum 2082 snap 0 seq_num 2082) dirty exists 2023-08-27T20:54:55.621 INFO:tasks.rados.rados.0.smithi093.stdout:3328: left oid 109 (ObjNum 2082 snap 0 seq_num 2082) 2023-08-27T20:54:55.621 INFO:tasks.rados.rados.0.smithi093.stdout:3326: done (3 left) 2023-08-27T20:54:55.621 INFO:tasks.rados.rados.0.smithi093.stdout:3327: done (2 left) 2023-08-27T20:54:55.621 INFO:tasks.rados.rados.0.smithi093.stdout:3328: done (1 left) 2023-08-27T20:54:55.621 INFO:tasks.rados.rados.0.smithi093.stdout:3329: done (0 left) 2023-08-27T20:54:55.622 INFO:tasks.rados.rados.0.smithi093.stdout:3330: read oid 28 snap -1 2023-08-27T20:54:55.622 INFO:tasks.rados.rados.0.smithi093.stdout:3330: expect (ObjNum 1962 snap 0 seq_num 1962) 2023-08-27T20:54:55.622 INFO:tasks.rados.rados.0.smithi093.stdout:3331: delete oid 291 current snap is 0 2023-08-27T20:54:55.622 INFO:tasks.rados.rados.0.smithi093.stdout:3331: done (1 left) 2023-08-27T20:54:55.622 INFO:tasks.rados.rados.0.smithi093.stdout:3332: read oid 218 snap -1 2023-08-27T20:54:55.623 INFO:tasks.rados.rados.0.smithi093.stdout:3332: expect (ObjNum 1674 snap 0 seq_num 1674) 2023-08-27T20:54:55.629 INFO:tasks.rados.rados.0.smithi093.stdout:3333: read oid 210 snap -1 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3333: expect (ObjNum 1647 snap 0 seq_num 1647) 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3334: delete oid 478 current snap is 0 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3334: done (3 left) 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3335: read oid 362 snap -1 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3335: expect (ObjNum 1964 snap 0 seq_num 1964) 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3336: read oid 239 snap -1 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3336: expect (ObjNum 1991 snap 0 seq_num 1991) 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3337: read oid 124 snap -1 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3337: expect (ObjNum 1747 snap 0 seq_num 1747) 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3338: write oid 328 current snap is 0 2023-08-27T20:54:55.630 INFO:tasks.rados.rados.0.smithi093.stdout:3338: seq_num 2086 ranges {659402=450136,1734144=504056,2734363=325039} 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3338: writing smithi093153278-328 from 659402 to 1109538 tid 1 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3338: writing smithi093153278-328 from 1734144 to 2238200 tid 2 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3338: writing smithi093153278-328 from 2734363 to 3059402 tid 3 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3339: write oid 45 current snap is 0 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3339: seq_num 2087 ranges {676209=525385,1676273=764067,3045433=30776} 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3339: writing smithi093153278-45 from 676209 to 1201594 tid 1 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3339: writing smithi093153278-45 from 1676273 to 2440340 tid 2 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3339: writing smithi093153278-45 from 3045433 to 3076209 tid 3 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3340: write oid 20 current snap is 0 2023-08-27T20:54:55.631 INFO:tasks.rados.rados.0.smithi093.stdout:3332: expect (ObjNum 1674 snap 0 seq_num 1674) 2023-08-27T20:54:55.632 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:55.632 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3333: expect (ObjNum 1647 snap 0 seq_num 1647) 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3330: expect (ObjNum 1962 snap 0 seq_num 1962) 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3338: finishing write tid 1 to smithi093153278-328 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3338: finishing write tid 2 to smithi093153278-328 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3335: expect (ObjNum 1964 snap 0 seq_num 1964) 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3338: finishing write tid 3 to smithi093153278-328 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3338: finishing write tid 4 to smithi093153278-328 2023-08-27T20:54:55.634 INFO:tasks.rados.rados.0.smithi093.stdout:3340: seq_num 2088 ranges {693016=600634,1734755=507725,2672856=420160} 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3340: writing smithi093153278-20 from 693016 to 1293650 tid 1 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3340: writing smithi093153278-20 from 1734755 to 2242480 tid 2 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3340: writing smithi093153278-20 from 2672856 to 3093016 tid 3 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3330: done (8 left) 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3332: done (7 left) 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3333: done (6 left) 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3335: done (5 left) 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3341: read oid 494 snap -1 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3341: expect (ObjNum 1026 snap 0 seq_num 1026) 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3342: read oid 324 snap -1 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3342: expect (ObjNum 2016 snap 0 seq_num 2016) 2023-08-27T20:54:55.635 INFO:tasks.rados.rados.0.smithi093.stdout:3343: write oid 126 current snap is 0 2023-08-27T20:54:55.636 INFO:tasks.rados.rados.0.smithi093.stdout:3338: finishing write tid 5 to smithi093153278-328 2023-08-27T20:54:55.636 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 467 (ObjNum 2086 snap 0 seq_num 2086) dirty exists 2023-08-27T20:54:55.636 INFO:tasks.rados.rados.0.smithi093.stdout:3338: left oid 328 (ObjNum 2086 snap 0 seq_num 2086) 2023-08-27T20:54:55.636 INFO:tasks.rados.rados.0.smithi093.stdout:3336: expect (ObjNum 1991 snap 0 seq_num 1991) 2023-08-27T20:54:55.710 INFO:tasks.rados.rados.0.smithi093.stdout:3339: finishing write tid 1 to smithi093153278-45 2023-08-27T20:54:56.026 INFO:tasks.rados.rados.0.smithi093.stdout:3337: expect (ObjNum 1747 snap 0 seq_num 1747) 2023-08-27T20:54:56.027 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:55 smithi136 ceph-mon[113531]: pgmap v41: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:54:56.028 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:55 smithi093 ceph-mon[111335]: pgmap v41: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:54:56.029 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:54:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:54:55] "GET /metrics HTTP/1.1" 200 242572 "" "Prometheus/2.43.0" 2023-08-27T20:54:56.029 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:55 smithi093 ceph-mon[118902]: pgmap v41: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 60 MiB/s rd, 9.1 MiB/s wr, 84 op/s 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3339: finishing write tid 2 to smithi093153278-45 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3339: finishing write tid 3 to smithi093153278-45 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3339: finishing write tid 4 to smithi093153278-45 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3339: finishing write tid 5 to smithi093153278-45 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 45 v 393 (ObjNum 2087 snap 0 seq_num 2087) dirty exists 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3339: left oid 45 (ObjNum 2087 snap 0 seq_num 2087) 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3343: seq_num 2089 ranges {709823=675883,1793237=767736,3100279=9544} 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3343: writing smithi093153278-126 from 709823 to 1385706 tid 1 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3343: writing smithi093153278-126 from 1793237 to 2560973 tid 2 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3343: writing smithi093153278-126 from 3100279 to 3109823 tid 3 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3336: done (7 left) 2023-08-27T20:54:56.032 INFO:tasks.rados.rados.0.smithi093.stdout:3337: done (6 left) 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3338: done (5 left) 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3339: done (4 left) 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3344: read oid 69 snap -1 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3344: expect (ObjNum 1710 snap 0 seq_num 1710) 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3345: write oid 4 current snap is 0 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3340: finishing write tid 1 to smithi093153278-20 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3340: finishing write tid 2 to smithi093153278-20 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3345: seq_num 2090 ranges {726630=751132,2251719=511394} 2023-08-27T20:54:56.033 INFO:tasks.rados.rados.0.smithi093.stdout:3345: writing smithi093153278-4 from 726630 to 1477762 tid 1 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3345: writing smithi093153278-4 from 2251719 to 2763113 tid 2 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3346: read oid 229 snap -1 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3346: expect deleted 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3347: write oid 486 current snap is 0 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3347: seq_num 2091 ranges {743437=542734,1910201=771405} 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3347: writing smithi093153278-486 from 743437 to 1286171 tid 1 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3347: writing smithi093153278-486 from 1910201 to 2681606 tid 2 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3348: delete oid 328 current snap is 0 2023-08-27T20:54:56.034 INFO:tasks.rados.rados.0.smithi093.stdout:3341: expect (ObjNum 1026 snap 0 seq_num 1026) 2023-08-27T20:54:56.042 INFO:tasks.rados.rados.0.smithi093.stdout:3340: finishing write tid 3 to smithi093153278-20 2023-08-27T20:54:56.042 INFO:tasks.rados.rados.0.smithi093.stdout:3340: finishing write tid 4 to smithi093153278-20 2023-08-27T20:54:56.042 INFO:tasks.rados.rados.0.smithi093.stdout:3340: finishing write tid 5 to smithi093153278-20 2023-08-27T20:54:56.042 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 426 (ObjNum 2088 snap 0 seq_num 2088) dirty exists 2023-08-27T20:54:56.042 INFO:tasks.rados.rados.0.smithi093.stdout:3340: left oid 20 (ObjNum 2088 snap 0 seq_num 2088) 2023-08-27T20:54:56.042 INFO:tasks.rados.rados.0.smithi093.stdout:3342: expect (ObjNum 2016 snap 0 seq_num 2016) 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:3343: finishing write tid 1 to smithi093153278-126 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:3343: finishing write tid 2 to smithi093153278-126 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:3343: finishing write tid 3 to smithi093153278-126 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:3343: finishing write tid 4 to smithi093153278-126 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:3343: finishing write tid 5 to smithi093153278-126 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 392 (ObjNum 2089 snap 0 seq_num 2089) dirty exists 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:3343: left oid 126 (ObjNum 2089 snap 0 seq_num 2089) 2023-08-27T20:54:56.252 INFO:tasks.rados.rados.0.smithi093.stdout:3344: expect (ObjNum 1710 snap 0 seq_num 1710) 2023-08-27T20:54:56.398 INFO:tasks.rados.rados.0.smithi093.stdout:3345: finishing write tid 1 to smithi093153278-4 2023-08-27T20:54:56.570 INFO:tasks.rados.rados.0.smithi093.stdout:3340: done (8 left) 2023-08-27T20:54:56.570 INFO:tasks.rados.rados.0.smithi093.stdout:3341: done (7 left) 2023-08-27T20:54:56.570 INFO:tasks.rados.rados.0.smithi093.stdout:3342: done (6 left) 2023-08-27T20:54:56.571 INFO:tasks.rados.rados.0.smithi093.stdout:3343: done (5 left) 2023-08-27T20:54:56.571 INFO:tasks.rados.rados.0.smithi093.stdout:3344: done (4 left) 2023-08-27T20:54:56.571 INFO:tasks.rados.rados.0.smithi093.stdout:3348: done (3 left) 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3349: write oid 443 current snap is 0 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3345: finishing write tid 2 to smithi093153278-4 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3345: finishing write tid 3 to smithi093153278-4 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3345: finishing write tid 4 to smithi093153278-4 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 4 v 495 (ObjNum 2090 snap 0 seq_num 2090) dirty exists 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3345: left oid 4 (ObjNum 2090 snap 0 seq_num 2090) 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3347: finishing write tid 1 to smithi093153278-486 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3347: finishing write tid 2 to smithi093153278-486 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3347: finishing write tid 3 to smithi093153278-486 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3347: finishing write tid 4 to smithi093153278-486 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 470 (ObjNum 2091 snap 0 seq_num 2091) dirty exists 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3347: left oid 486 (ObjNum 2091 snap 0 seq_num 2091) 2023-08-27T20:54:56.572 INFO:tasks.rados.rados.0.smithi093.stdout:3349: seq_num 2093 ranges {777051=693232,2027165=658721} 2023-08-27T20:54:56.578 INFO:tasks.rados.rados.0.smithi093.stdout:3349: writing smithi093153278-443 from 777051 to 1470283 tid 1 2023-08-27T20:54:56.581 INFO:tasks.rados.rados.0.smithi093.stdout:3349: writing smithi093153278-443 from 2027165 to 2685886 tid 2 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3345: done (3 left) 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3346: done (2 left) 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3347: done (1 left) 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3350: write oid 20 current snap is 0 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3349: finishing write tid 1 to smithi093153278-443 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3350: seq_num 2094 ranges {793858=768481,2085647=518732} 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3350: writing smithi093153278-20 from 793858 to 1562339 tid 1 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3350: writing smithi093153278-20 from 2085647 to 2604379 tid 2 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3351: write oid 214 current snap is 0 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3349: finishing write tid 2 to smithi093153278-443 2023-08-27T20:54:56.582 INFO:tasks.rados.rados.0.smithi093.stdout:3349: finishing write tid 3 to smithi093153278-443 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3349: finishing write tid 4 to smithi093153278-443 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 443 v 453 (ObjNum 2093 snap 0 seq_num 2093) dirty exists 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3349: left oid 443 (ObjNum 2093 snap 0 seq_num 2093) 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3350: finishing write tid 1 to smithi093153278-20 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3351: seq_num 2095 ranges {410665=443730,1627776=662390,2748464=462201} 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3351: writing smithi093153278-214 from 410665 to 854395 tid 1 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3351: writing smithi093153278-214 from 1627776 to 2290166 tid 2 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3351: writing smithi093153278-214 from 2748464 to 3210665 tid 3 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3349: done (2 left) 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3352: write oid 244 current snap is 0 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3350: finishing write tid 2 to smithi093153278-20 2023-08-27T20:54:56.583 INFO:tasks.rados.rados.0.smithi093.stdout:3350: finishing write tid 3 to smithi093153278-20 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3352: seq_num 2096 ranges {427472=518979,1686258=522401,2775887=451585} 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3352: writing smithi093153278-244 from 427472 to 946451 tid 1 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3352: writing smithi093153278-244 from 1686258 to 2208659 tid 2 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3352: writing smithi093153278-244 from 2775887 to 3227472 tid 3 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3353: write oid 19 current snap is 0 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3350: finishing write tid 4 to smithi093153278-20 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 429 (ObjNum 2094 snap 0 seq_num 2094) dirty exists 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3350: left oid 20 (ObjNum 2094 snap 0 seq_num 2094) 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3351: finishing write tid 1 to smithi093153278-214 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3353: seq_num 2097 ranges {444279=594228,1744740=666059,3203310=40969} 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3353: writing smithi093153278-19 from 444279 to 1038507 tid 1 2023-08-27T20:54:56.584 INFO:tasks.rados.rados.0.smithi093.stdout:3353: writing smithi093153278-19 from 1744740 to 2410799 tid 2 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3353: writing smithi093153278-19 from 3203310 to 3244279 tid 3 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3350: done (3 left) 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3354: read oid 398 snap -1 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3354: expect deleted 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3355: write oid 221 current snap is 0 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3351: finishing write tid 2 to smithi093153278-214 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3355: seq_num 2098 ranges {461086=669477,1803222=526070,2830733=430353} 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3355: writing smithi093153278-221 from 461086 to 1130563 tid 1 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3355: writing smithi093153278-221 from 1803222 to 2329292 tid 2 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3355: writing smithi093153278-221 from 2830733 to 3261086 tid 3 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3356: write oid 139 current snap is 0 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3351: finishing write tid 3 to smithi093153278-214 2023-08-27T20:54:56.585 INFO:tasks.rados.rados.0.smithi093.stdout:3351: finishing write tid 4 to smithi093153278-214 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3356: seq_num 2099 ranges {477893=461079,1578057=669728,2974509=303384} 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3356: writing smithi093153278-139 from 477893 to 938972 tid 1 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3356: writing smithi093153278-139 from 1578057 to 2247785 tid 2 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3356: writing smithi093153278-139 from 2974509 to 3277893 tid 3 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3357: delete oid 243 current snap is 0 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3351: finishing write tid 5 to smithi093153278-214 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 214 v 266 (ObjNum 2095 snap 0 seq_num 2095) dirty exists 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3351: left oid 214 (ObjNum 2095 snap 0 seq_num 2095) 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3352: finishing write tid 1 to smithi093153278-244 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3352: finishing write tid 2 to smithi093153278-244 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3352: finishing write tid 3 to smithi093153278-244 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3352: finishing write tid 4 to smithi093153278-244 2023-08-27T20:54:56.586 INFO:tasks.rados.rados.0.smithi093.stdout:3352: finishing write tid 5 to smithi093153278-244 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 244 v 270 (ObjNum 2096 snap 0 seq_num 2096) dirty exists 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3352: left oid 244 (ObjNum 2096 snap 0 seq_num 2096) 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3353: finishing write tid 1 to smithi093153278-19 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3353: finishing write tid 2 to smithi093153278-19 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3353: finishing write tid 3 to smithi093153278-19 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3353: finishing write tid 4 to smithi093153278-19 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3353: finishing write tid 5 to smithi093153278-19 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 19 v 446 (ObjNum 2097 snap 0 seq_num 2097) dirty exists 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3353: left oid 19 (ObjNum 2097 snap 0 seq_num 2097) 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3355: finishing write tid 1 to smithi093153278-221 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3355: finishing write tid 2 to smithi093153278-221 2023-08-27T20:54:56.587 INFO:tasks.rados.rados.0.smithi093.stdout:3355: finishing write tid 3 to smithi093153278-221 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3355: finishing write tid 4 to smithi093153278-221 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3355: finishing write tid 5 to smithi093153278-221 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 221 v 544 (ObjNum 2098 snap 0 seq_num 2098) dirty exists 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3355: left oid 221 (ObjNum 2098 snap 0 seq_num 2098) 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3356: finishing write tid 1 to smithi093153278-139 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3356: finishing write tid 2 to smithi093153278-139 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3351: done (6 left) 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3352: done (5 left) 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3353: done (4 left) 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3354: done (3 left) 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3355: done (2 left) 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3357: done (1 left) 2023-08-27T20:54:56.588 INFO:tasks.rados.rados.0.smithi093.stdout:3358: read oid 273 snap -1 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3358: expect deleted 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3359: delete oid 223 current snap is 0 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3356: finishing write tid 3 to smithi093153278-139 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3356: finishing write tid 4 to smithi093153278-139 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3356: finishing write tid 5 to smithi093153278-139 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 139 v 409 (ObjNum 2099 snap 0 seq_num 2099) dirty exists 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3356: left oid 139 (ObjNum 2099 snap 0 seq_num 2099) 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3356: done (2 left) 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3358: done (1 left) 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3359: done (0 left) 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3360: read oid 316 snap -1 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3360: expect (ObjNum 1772 snap 0 seq_num 1772) 2023-08-27T20:54:56.589 INFO:tasks.rados.rados.0.smithi093.stdout:3361: read oid 152 snap -1 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3361: expect (ObjNum 1687 snap 0 seq_num 1687) 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3362: write oid 126 current snap is 0 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3362: seq_num 2102 ranges {528314=686826,1637150=533408,2940425=387889} 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3362: writing smithi093153278-126 from 528314 to 1215140 tid 1 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3362: writing smithi093153278-126 from 1637150 to 2170558 tid 2 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3362: writing smithi093153278-126 from 2940425 to 3328314 tid 3 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3363: read oid 381 snap -1 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3363: expect deleted 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3364: delete oid 197 current snap is 0 2023-08-27T20:54:56.590 INFO:tasks.rados.rados.0.smithi093.stdout:3360: expect (ObjNum 1772 snap 0 seq_num 1772) 2023-08-27T20:54:56.716 INFO:tasks.rados.rados.0.smithi093.stdout:3361: expect (ObjNum 1687 snap 0 seq_num 1687) 2023-08-27T20:54:56.848 INFO:tasks.rados.rados.0.smithi093.stdout:3362: finishing write tid 1 to smithi093153278-126 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:3362: finishing write tid 2 to smithi093153278-126 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:3362: finishing write tid 3 to smithi093153278-126 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:3362: finishing write tid 4 to smithi093153278-126 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:3362: finishing write tid 5 to smithi093153278-126 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 396 (ObjNum 2102 snap 0 seq_num 2102) dirty exists 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:3362: left oid 126 (ObjNum 2102 snap 0 seq_num 2102) 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:3360: done (4 left) 2023-08-27T20:54:56.849 INFO:tasks.rados.rados.0.smithi093.stdout:3361: done (3 left) 2023-08-27T20:54:56.850 INFO:tasks.rados.rados.0.smithi093.stdout:3362: done (2 left) 2023-08-27T20:54:56.850 INFO:tasks.rados.rados.0.smithi093.stdout:3363: done (1 left) 2023-08-27T20:54:56.850 INFO:tasks.rados.rados.0.smithi093.stdout:3364: done (0 left) 2023-08-27T20:54:56.850 INFO:tasks.rados.rados.0.smithi093.stdout:3365: read oid 90 snap -1 2023-08-27T20:54:56.850 INFO:tasks.rados.rados.0.smithi093.stdout:3365: expect (ObjNum 1851 snap 0 seq_num 1851) 2023-08-27T20:54:56.850 INFO:tasks.rados.rados.0.smithi093.stdout:3366: write oid 97 current snap is 0 2023-08-27T20:54:56.850 INFO:tasks.rados.rados.0.smithi093.stdout:3366: seq_num 2104 ranges {561928=437324,1637761=420724,2762565=438668} 2023-08-27T20:54:56.854 INFO:tasks.rados.rados.0.smithi093.stdout:3366: writing smithi093153278-97 from 561928 to 999252 tid 1 2023-08-27T20:54:56.859 INFO:tasks.rados.rados.0.smithi093.stdout:3366: writing smithi093153278-97 from 1637761 to 2058485 tid 2 2023-08-27T20:54:56.866 INFO:tasks.rados.rados.0.smithi093.stdout:3366: writing smithi093153278-97 from 2762565 to 3201233 tid 3 2023-08-27T20:54:56.866 INFO:tasks.rados.rados.0.smithi093.stdout:3367: read oid 305 snap -1 2023-08-27T20:54:56.867 INFO:tasks.rados.rados.0.smithi093.stdout:3367: expect (ObjNum 1720 snap 0 seq_num 1720) 2023-08-27T20:54:56.867 INFO:tasks.rados.rados.0.smithi093.stdout:3368: read oid 471 snap -1 2023-08-27T20:54:56.867 INFO:tasks.rados.rados.0.smithi093.stdout:3368: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:54:56.867 INFO:tasks.rados.rados.0.smithi093.stdout:3369: read oid 273 snap -1 2023-08-27T20:54:56.867 INFO:tasks.rados.rados.0.smithi093.stdout:3369: expect deleted 2023-08-27T20:54:56.867 INFO:tasks.rados.rados.0.smithi093.stdout:3370: write oid 500 current snap is 0 2023-08-27T20:54:56.867 INFO:tasks.rados.rados.0.smithi093.stdout:3370: seq_num 2105 ranges {578735=512573,1696243=680735,2906341=472394} 2023-08-27T20:54:56.872 INFO:tasks.rados.rados.0.smithi093.stdout:3370: writing smithi093153278-500 from 578735 to 1091308 tid 1 2023-08-27T20:54:56.880 INFO:tasks.rados.rados.0.smithi093.stdout:3370: writing smithi093153278-500 from 1696243 to 2376978 tid 2 2023-08-27T20:54:56.886 INFO:tasks.rados.rados.0.smithi093.stdout:3370: writing smithi093153278-500 from 2906341 to 3378735 tid 3 2023-08-27T20:54:56.886 INFO:tasks.rados.rados.0.smithi093.stdout:3371: read oid 58 snap -1 2023-08-27T20:54:56.887 INFO:tasks.rados.rados.0.smithi093.stdout:3371: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:54:56.887 INFO:tasks.rados.rados.0.smithi093.stdout:3372: read oid 178 snap -1 2023-08-27T20:54:56.887 INFO:tasks.rados.rados.0.smithi093.stdout:3372: expect (ObjNum 2075 snap 0 seq_num 2075) 2023-08-27T20:54:56.887 INFO:tasks.rados.rados.0.smithi093.stdout:3373: read oid 20 snap -1 2023-08-27T20:54:56.887 INFO:tasks.rados.rados.0.smithi093.stdout:3373: expect (ObjNum 2094 snap 0 seq_num 2094) 2023-08-27T20:54:56.887 INFO:tasks.rados.rados.0.smithi093.stdout:3374: write oid 211 current snap is 0 2023-08-27T20:54:56.887 INFO:tasks.rados.rados.0.smithi093.stdout:3374: seq_num 2106 ranges {595542=704175,1871078=424393,3050117=345425} 2023-08-27T20:54:56.894 INFO:tasks.rados.rados.0.smithi093.stdout:3374: writing smithi093153278-211 from 595542 to 1299717 tid 1 2023-08-27T20:54:56.899 INFO:tasks.rados.rados.0.smithi093.stdout:3374: writing smithi093153278-211 from 1871078 to 2295471 tid 2 2023-08-27T20:54:56.906 INFO:tasks.rados.rados.0.smithi093.stdout:3374: writing smithi093153278-211 from 3050117 to 3395542 tid 3 2023-08-27T20:54:56.906 INFO:tasks.rados.rados.0.smithi093.stdout:3375: write oid 392 current snap is 0 2023-08-27T20:54:56.906 INFO:tasks.rados.rados.0.smithi093.stdout:3365: expect (ObjNum 1851 snap 0 seq_num 1851) 2023-08-27T20:54:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3535210109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3535210109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:57.355 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3535210109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:57.356 INFO:tasks.rados.rados.0.smithi093.stdout:3366: finishing write tid 1 to smithi093153278-97 2023-08-27T20:54:57.356 INFO:tasks.rados.rados.0.smithi093.stdout:3366: finishing write tid 2 to smithi093153278-97 2023-08-27T20:54:57.357 INFO:tasks.rados.rados.0.smithi093.stdout:3366: finishing write tid 3 to smithi093153278-97 2023-08-27T20:54:57.357 INFO:tasks.rados.rados.0.smithi093.stdout:3366: finishing write tid 4 to smithi093153278-97 2023-08-27T20:54:57.357 INFO:tasks.rados.rados.0.smithi093.stdout:3366: finishing write tid 5 to smithi093153278-97 2023-08-27T20:54:57.357 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 97 v 275 (ObjNum 2104 snap 0 seq_num 2104) dirty exists 2023-08-27T20:54:57.357 INFO:tasks.rados.rados.0.smithi093.stdout:3366: left oid 97 (ObjNum 2104 snap 0 seq_num 2104) 2023-08-27T20:54:57.358 INFO:tasks.rados.rados.0.smithi093.stdout:3367: expect (ObjNum 1720 snap 0 seq_num 1720) 2023-08-27T20:54:57.358 INFO:tasks.rados.rados.0.smithi093.stdout:3370: finishing write tid 1 to smithi093153278-500 2023-08-27T20:54:57.358 INFO:tasks.rados.rados.0.smithi093.stdout:3368: expect (ObjNum 470 snap 0 seq_num 470) 2023-08-27T20:54:57.358 INFO:tasks.rados.rados.0.smithi093.stdout:3370: finishing write tid 2 to smithi093153278-500 2023-08-27T20:54:57.358 INFO:tasks.rados.rados.0.smithi093.stdout:3370: finishing write tid 3 to smithi093153278-500 2023-08-27T20:54:57.358 INFO:tasks.rados.rados.0.smithi093.stdout:3370: finishing write tid 4 to smithi093153278-500 2023-08-27T20:54:57.359 INFO:tasks.rados.rados.0.smithi093.stdout:3370: finishing write tid 5 to smithi093153278-500 2023-08-27T20:54:57.359 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 500 v 487 (ObjNum 2105 snap 0 seq_num 2105) dirty exists 2023-08-27T20:54:57.359 INFO:tasks.rados.rados.0.smithi093.stdout:3370: left oid 500 (ObjNum 2105 snap 0 seq_num 2105) 2023-08-27T20:54:57.359 INFO:tasks.rados.rados.0.smithi093.stdout:3371: expect (ObjNum 57 snap 0 seq_num 57) 2023-08-27T20:54:57.414 INFO:tasks.rados.rados.0.smithi093.stdout:3373: expect (ObjNum 2094 snap 0 seq_num 2094) 2023-08-27T20:54:57.587 INFO:tasks.rados.rados.0.smithi093.stdout:3372: expect (ObjNum 2075 snap 0 seq_num 2075) 2023-08-27T20:54:57.714 INFO:tasks.rados.rados.0.smithi093.stdout:3374: finishing write tid 1 to smithi093153278-211 2023-08-27T20:54:57.714 INFO:tasks.rados.rados.0.smithi093.stdout:3374: finishing write tid 2 to smithi093153278-211 2023-08-27T20:54:57.714 INFO:tasks.rados.rados.0.smithi093.stdout:3374: finishing write tid 3 to smithi093153278-211 2023-08-27T20:54:57.715 INFO:tasks.rados.rados.0.smithi093.stdout:3374: finishing write tid 4 to smithi093153278-211 2023-08-27T20:54:57.715 INFO:tasks.rados.rados.0.smithi093.stdout:3374: finishing write tid 5 to smithi093153278-211 2023-08-27T20:54:57.715 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 211 v 548 (ObjNum 2106 snap 0 seq_num 2106) dirty exists 2023-08-27T20:54:57.715 INFO:tasks.rados.rados.0.smithi093.stdout:3374: left oid 211 (ObjNum 2106 snap 0 seq_num 2106) 2023-08-27T20:54:57.715 INFO:tasks.rados.rados.0.smithi093.stdout:3375: seq_num 2107 ranges {612349=779424,1929560=684404,3077540=334809} 2023-08-27T20:54:57.721 INFO:tasks.rados.rados.0.smithi093.stdout:3375: writing smithi093153278-392 from 612349 to 1391773 tid 1 2023-08-27T20:54:57.728 INFO:tasks.rados.rados.0.smithi093.stdout:3375: writing smithi093153278-392 from 1929560 to 2613964 tid 2 2023-08-27T20:54:57.734 INFO:tasks.rados.rados.0.smithi093.stdout:3375: writing smithi093153278-392 from 3077540 to 3412349 tid 3 2023-08-27T20:54:57.734 INFO:tasks.rados.rados.0.smithi093.stdout:3365: done (10 left) 2023-08-27T20:54:57.734 INFO:tasks.rados.rados.0.smithi093.stdout:3366: done (9 left) 2023-08-27T20:54:57.734 INFO:tasks.rados.rados.0.smithi093.stdout:3367: done (8 left) 2023-08-27T20:54:57.734 INFO:tasks.rados.rados.0.smithi093.stdout:3368: done (7 left) 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3369: done (6 left) 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3370: done (5 left) 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3371: done (4 left) 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3372: done (3 left) 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3373: done (2 left) 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3374: done (1 left) 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3376: write oid 70 current snap is 0 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3375: finishing write tid 1 to smithi093153278-392 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3375: finishing write tid 2 to smithi093153278-392 2023-08-27T20:54:57.735 INFO:tasks.rados.rados.0.smithi093.stdout:3376: seq_num 2108 ranges {629156=454673,1871689=428062,2988610=440546} 2023-08-27T20:54:57.739 INFO:tasks.rados.rados.0.smithi093.stdout:3376: writing smithi093153278-70 from 629156 to 1083829 tid 1 2023-08-27T20:54:57.746 INFO:tasks.rados.rados.0.smithi093.stdout:3376: writing smithi093153278-70 from 1871689 to 2299751 tid 2 2023-08-27T20:54:57.753 INFO:tasks.rados.rados.0.smithi093.stdout:3376: writing smithi093153278-70 from 2988610 to 3429156 tid 3 2023-08-27T20:54:57.753 INFO:tasks.rados.rados.0.smithi093.stdout:3377: write oid 42 current snap is 0 2023-08-27T20:54:57.753 INFO:tasks.rados.rados.0.smithi093.stdout:3375: finishing write tid 3 to smithi093153278-392 2023-08-27T20:54:57.753 INFO:tasks.rados.rados.0.smithi093.stdout:3375: finishing write tid 4 to smithi093153278-392 2023-08-27T20:54:57.754 INFO:tasks.rados.rados.0.smithi093.stdout:3375: finishing write tid 5 to smithi093153278-392 2023-08-27T20:54:57.754 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 392 v 450 (ObjNum 2107 snap 0 seq_num 2107) dirty exists 2023-08-27T20:54:57.754 INFO:tasks.rados.rados.0.smithi093.stdout:3375: left oid 392 (ObjNum 2107 snap 0 seq_num 2107) 2023-08-27T20:54:57.754 INFO:tasks.rados.rados.0.smithi093.stdout:3376: finishing write tid 1 to smithi093153278-70 2023-08-27T20:54:57.754 INFO:tasks.rados.rados.0.smithi093.stdout:3376: finishing write tid 2 to smithi093153278-70 2023-08-27T20:54:57.754 INFO:tasks.rados.rados.0.smithi093.stdout:3377: seq_num 2109 ranges {645963=529922,1930171=688073,3416033=29930} 2023-08-27T20:54:57.759 INFO:tasks.rados.rados.0.smithi093.stdout:3377: writing smithi093153278-42 from 645963 to 1175885 tid 1 2023-08-27T20:54:57.767 INFO:tasks.rados.rados.0.smithi093.stdout:3377: writing smithi093153278-42 from 1930171 to 2618244 tid 2 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3377: writing smithi093153278-42 from 3416033 to 3445963 tid 3 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3375: done (2 left) 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3378: read oid 291 snap -1 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3378: expect deleted 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3379: read oid 171 snap -1 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3379: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3380: write oid 360 current snap is 0 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3376: finishing write tid 3 to smithi093153278-70 2023-08-27T20:54:57.773 INFO:tasks.rados.rados.0.smithi093.stdout:3380: seq_num 2110 ranges {662770=605171,1988653=431731,3043456=419314} 2023-08-27T20:54:57.779 INFO:tasks.rados.rados.0.smithi093.stdout:3380: writing smithi093153278-360 from 662770 to 1267941 tid 1 2023-08-27T20:54:57.786 INFO:tasks.rados.rados.0.smithi093.stdout:3380: writing smithi093153278-360 from 1988653 to 2420384 tid 2 2023-08-27T20:54:57.792 INFO:tasks.rados.rados.0.smithi093.stdout:3380: writing smithi093153278-360 from 3043456 to 3462770 tid 3 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3381: read oid 403 snap -1 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3381: expect (ObjNum 1285 snap 0 seq_num 1285) 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3382: read oid 207 snap -1 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3382: expect deleted 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3383: delete oid 215 current snap is 0 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3376: finishing write tid 4 to smithi093153278-70 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3376: finishing write tid 5 to smithi093153278-70 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 70 v 390 (ObjNum 2108 snap 0 seq_num 2108) dirty exists 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3376: left oid 70 (ObjNum 2108 snap 0 seq_num 2108) 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3377: finishing write tid 1 to smithi093153278-42 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3377: finishing write tid 2 to smithi093153278-42 2023-08-27T20:54:57.793 INFO:tasks.rados.rados.0.smithi093.stdout:3377: finishing write tid 3 to smithi093153278-42 2023-08-27T20:54:57.794 INFO:tasks.rados.rados.0.smithi093.stdout:3377: finishing write tid 4 to smithi093153278-42 2023-08-27T20:54:57.794 INFO:tasks.rados.rados.0.smithi093.stdout:3377: finishing write tid 5 to smithi093153278-42 2023-08-27T20:54:57.794 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 42 v 394 (ObjNum 2109 snap 0 seq_num 2109) dirty exists 2023-08-27T20:54:57.794 INFO:tasks.rados.rados.0.smithi093.stdout:3377: left oid 42 (ObjNum 2109 snap 0 seq_num 2109) 2023-08-27T20:54:57.794 INFO:tasks.rados.rados.0.smithi093.stdout:3379: expect (ObjNum 170 snap 0 seq_num 170) 2023-08-27T20:54:57.888 INFO:tasks.rados.rados.0.smithi093.stdout:3380: finishing write tid 1 to smithi093153278-360 2023-08-27T20:54:57.888 INFO:tasks.rados.rados.0.smithi093.stdout:3380: finishing write tid 2 to smithi093153278-360 2023-08-27T20:54:57.888 INFO:tasks.rados.rados.0.smithi093.stdout:3376: done (7 left) 2023-08-27T20:54:57.888 INFO:tasks.rados.rados.0.smithi093.stdout:3377: done (6 left) 2023-08-27T20:54:57.888 INFO:tasks.rados.rados.0.smithi093.stdout:3378: done (5 left) 2023-08-27T20:54:57.888 INFO:tasks.rados.rados.0.smithi093.stdout:3379: done (4 left) 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3383: done (3 left) 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3384: read oid 304 snap -1 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3384: expect (ObjNum 1814 snap 0 seq_num 1814) 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3385: read oid 499 snap -1 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3385: expect (ObjNum 953 snap 0 seq_num 953) 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3386: read oid 19 snap -1 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3386: expect (ObjNum 2097 snap 0 seq_num 2097) 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3387: write oid 449 current snap is 0 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3380: finishing write tid 3 to smithi093153278-360 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3380: finishing write tid 4 to smithi093153278-360 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:3380: finishing write tid 5 to smithi093153278-360 2023-08-27T20:54:57.889 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 360 v 367 (ObjNum 2110 snap 0 seq_num 2110) dirty exists 2023-08-27T20:54:58.169 INFO:tasks.rados.rados.0.smithi093.stdout:3380: left oid 360 (ObjNum 2110 snap 0 seq_num 2110) 2023-08-27T20:54:58.170 INFO:tasks.rados.rados.0.smithi093.stdout:3381: expect (ObjNum 1285 snap 0 seq_num 1285) 2023-08-27T20:54:58.170 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:57 smithi136 ceph-mon[113531]: pgmap v42: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 MiB/s rd, 11 MiB/s wr, 119 op/s 2023-08-27T20:54:58.171 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:57 smithi093 ceph-mon[111335]: pgmap v42: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 MiB/s rd, 11 MiB/s wr, 119 op/s 2023-08-27T20:54:58.171 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:57 smithi093 ceph-mon[118902]: pgmap v42: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 93 MiB/s rd, 11 MiB/s wr, 119 op/s 2023-08-27T20:54:58.171 INFO:tasks.rados.rados.0.smithi093.stdout:3386: expect (ObjNum 2097 snap 0 seq_num 2097) 2023-08-27T20:54:58.231 INFO:tasks.rados.rados.0.smithi093.stdout:3384: expect (ObjNum 1814 snap 0 seq_num 1814) 2023-08-27T20:54:58.325 INFO:tasks.rados.rados.0.smithi093.stdout:3385: expect (ObjNum 953 snap 0 seq_num 953) 2023-08-27T20:54:58.402 INFO:tasks.rados.rados.0.smithi093.stdout:3387: seq_num 2112 ranges {696384=755669,1989264=435400,2981949=514435} 2023-08-27T20:54:58.410 INFO:tasks.rados.rados.0.smithi093.stdout:3387: writing smithi093153278-449 from 696384 to 1452053 tid 1 2023-08-27T20:54:58.415 INFO:tasks.rados.rados.0.smithi093.stdout:3387: writing smithi093153278-449 from 1989264 to 2424664 tid 2 2023-08-27T20:54:58.422 INFO:tasks.rados.rados.0.smithi093.stdout:3387: writing smithi093153278-449 from 2981949 to 3496384 tid 3 2023-08-27T20:54:58.422 INFO:tasks.rados.rados.0.smithi093.stdout:3380: done (6 left) 2023-08-27T20:54:58.422 INFO:tasks.rados.rados.0.smithi093.stdout:3381: done (5 left) 2023-08-27T20:54:58.422 INFO:tasks.rados.rados.0.smithi093.stdout:3382: done (4 left) 2023-08-27T20:54:58.423 INFO:tasks.rados.rados.0.smithi093.stdout:3384: done (3 left) 2023-08-27T20:54:58.423 INFO:tasks.rados.rados.0.smithi093.stdout:3385: done (2 left) 2023-08-27T20:54:58.423 INFO:tasks.rados.rados.0.smithi093.stdout:3386: done (1 left) 2023-08-27T20:54:58.423 INFO:tasks.rados.rados.0.smithi093.stdout:3388: write oid 344 current snap is 0 2023-08-27T20:54:58.423 INFO:tasks.rados.rados.0.smithi093.stdout:3387: finishing write tid 1 to smithi093153278-449 2023-08-27T20:54:58.423 INFO:tasks.rados.rados.0.smithi093.stdout:3387: finishing write tid 2 to smithi093153278-449 2023-08-27T20:54:58.423 INFO:tasks.rados.rados.0.smithi093.stdout:3388: seq_num 2113 ranges {713191=430918,1647746=695411,3009372=503819} 2023-08-27T20:54:58.428 INFO:tasks.rados.rados.0.smithi093.stdout:3388: writing smithi093153278-344 from 713191 to 1144109 tid 1 2023-08-27T20:54:58.435 INFO:tasks.rados.rados.0.smithi093.stdout:3388: writing smithi093153278-344 from 1647746 to 2343157 tid 2 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3388: writing smithi093153278-344 from 3009372 to 3513191 tid 3 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3389: read oid 337 snap -1 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3389: expect (ObjNum 787 snap 0 seq_num 787) 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3390: read oid 190 snap -1 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3390: expect deleted 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3391: delete oid 126 current snap is 0 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3387: finishing write tid 3 to smithi093153278-449 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3387: finishing write tid 4 to smithi093153278-449 2023-08-27T20:54:58.442 INFO:tasks.rados.rados.0.smithi093.stdout:3387: finishing write tid 5 to smithi093153278-449 2023-08-27T20:54:58.443 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 491 (ObjNum 2112 snap 0 seq_num 2112) dirty exists 2023-08-27T20:54:58.443 INFO:tasks.rados.rados.0.smithi093.stdout:3387: left oid 449 (ObjNum 2112 snap 0 seq_num 2112) 2023-08-27T20:54:58.443 INFO:tasks.rados.rados.0.smithi093.stdout:3388: finishing write tid 1 to smithi093153278-344 2023-08-27T20:54:58.443 INFO:tasks.rados.rados.0.smithi093.stdout:3388: finishing write tid 2 to smithi093153278-344 2023-08-27T20:54:58.444 INFO:tasks.rados.rados.0.smithi093.stdout:3387: done (4 left) 2023-08-27T20:54:58.444 INFO:tasks.rados.rados.0.smithi093.stdout:3390: done (3 left) 2023-08-27T20:54:58.444 INFO:tasks.rados.rados.0.smithi093.stdout:3391: done (2 left) 2023-08-27T20:54:58.444 INFO:tasks.rados.rados.0.smithi093.stdout:3392: write oid 472 current snap is 0 2023-08-27T20:54:58.444 INFO:tasks.rados.rados.0.smithi093.stdout:3392: seq_num 2115 ranges {746805=697769,1881063=699080,3180571=366234} 2023-08-27T20:54:58.451 INFO:tasks.rados.rados.0.smithi093.stdout:3392: writing smithi093153278-472 from 746805 to 1444574 tid 1 2023-08-27T20:54:58.458 INFO:tasks.rados.rados.0.smithi093.stdout:3392: writing smithi093153278-472 from 1881063 to 2580143 tid 2 2023-08-27T20:54:58.464 INFO:tasks.rados.rados.0.smithi093.stdout:3392: writing smithi093153278-472 from 3180571 to 3546805 tid 3 2023-08-27T20:54:58.464 INFO:tasks.rados.rados.0.smithi093.stdout:3393: write oid 245 current snap is 0 2023-08-27T20:54:58.464 INFO:tasks.rados.rados.0.smithi093.stdout:3388: finishing write tid 3 to smithi093153278-344 2023-08-27T20:54:58.464 INFO:tasks.rados.rados.0.smithi093.stdout:3393: seq_num 2116 ranges {763612=773018,2223192=442738,3091641=471971} 2023-08-27T20:54:58.472 INFO:tasks.rados.rados.0.smithi093.stdout:3393: writing smithi093153278-245 from 763612 to 1536630 tid 1 2023-08-27T20:54:58.478 INFO:tasks.rados.rados.0.smithi093.stdout:3393: writing smithi093153278-245 from 2223192 to 2665930 tid 2 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3393: writing smithi093153278-245 from 3091641 to 3563612 tid 3 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3394: read oid 71 snap -1 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3394: expect (ObjNum 1099 snap 0 seq_num 1099) 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3395: read oid 260 snap -1 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3395: expect (ObjNum 1032 snap 0 seq_num 1032) 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3396: delete oid 311 current snap is 0 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3388: finishing write tid 4 to smithi093153278-344 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:3388: finishing write tid 5 to smithi093153278-344 2023-08-27T20:54:58.484 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 344 v 475 (ObjNum 2113 snap 0 seq_num 2113) dirty exists 2023-08-27T20:54:58.485 INFO:tasks.rados.rados.0.smithi093.stdout:3388: left oid 344 (ObjNum 2113 snap 0 seq_num 2113) 2023-08-27T20:54:58.485 INFO:tasks.rados.rados.0.smithi093.stdout:3389: expect (ObjNum 787 snap 0 seq_num 787) 2023-08-27T20:54:58.644 INFO:tasks.rados.rados.0.smithi093.stdout:3392: finishing write tid 1 to smithi093153278-472 2023-08-27T20:54:58.644 INFO:tasks.rados.rados.0.smithi093.stdout:3392: finishing write tid 2 to smithi093153278-472 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3392: finishing write tid 3 to smithi093153278-472 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3392: finishing write tid 4 to smithi093153278-472 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3392: finishing write tid 5 to smithi093153278-472 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 472 v 454 (ObjNum 2115 snap 0 seq_num 2115) dirty exists 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3392: left oid 472 (ObjNum 2115 snap 0 seq_num 2115) 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3393: finishing write tid 1 to smithi093153278-245 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3388: done (6 left) 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3389: done (5 left) 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3392: done (4 left) 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3396: done (3 left) 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3397: write oid 33 current snap is 0 2023-08-27T20:54:58.645 INFO:tasks.rados.rados.0.smithi093.stdout:3393: finishing write tid 2 to smithi093153278-245 2023-08-27T20:54:58.646 INFO:tasks.rados.rados.0.smithi093.stdout:3397: seq_num 2118 ranges {797226=523516,1940156=446407,3146487=450739} 2023-08-27T20:54:58.651 INFO:tasks.rados.rados.0.smithi093.stdout:3397: writing smithi093153278-33 from 797226 to 1320742 tid 1 2023-08-27T20:54:58.657 INFO:tasks.rados.rados.0.smithi093.stdout:3397: writing smithi093153278-33 from 1940156 to 2386563 tid 2 2023-08-27T20:54:58.664 INFO:tasks.rados.rados.0.smithi093.stdout:3397: writing smithi093153278-33 from 3146487 to 3597226 tid 3 2023-08-27T20:54:58.664 INFO:tasks.rados.rados.0.smithi093.stdout:3398: read oid 315 snap -1 2023-08-27T20:54:58.664 INFO:tasks.rados.rados.0.smithi093.stdout:3398: expect (ObjNum 2033 snap 0 seq_num 2033) 2023-08-27T20:54:58.665 INFO:tasks.rados.rados.0.smithi093.stdout:3399: write oid 86 current snap is 0 2023-08-27T20:54:58.665 INFO:tasks.rados.rados.0.smithi093.stdout:3393: finishing write tid 3 to smithi093153278-245 2023-08-27T20:54:58.665 INFO:tasks.rados.rados.0.smithi093.stdout:3393: finishing write tid 4 to smithi093153278-245 2023-08-27T20:54:58.665 INFO:tasks.rados.rados.0.smithi093.stdout:3399: seq_num 2119 ranges {414033=598765,1598638=590065,2657557=756807} 2023-08-27T20:54:58.669 INFO:tasks.rados.rados.0.smithi093.stdout:3399: writing smithi093153278-86 from 414033 to 1012798 tid 1 2023-08-27T20:54:58.676 INFO:tasks.rados.rados.0.smithi093.stdout:3399: writing smithi093153278-86 from 1598638 to 2188703 tid 2 2023-08-27T20:54:58.683 INFO:tasks.rados.rados.0.smithi093.stdout:3399: writing smithi093153278-86 from 2657557 to 3414364 tid 3 2023-08-27T20:54:58.683 INFO:tasks.rados.rados.0.smithi093.stdout:3400: write oid 216 current snap is 0 2023-08-27T20:54:58.683 INFO:tasks.rados.rados.0.smithi093.stdout:3400: seq_num 2120 ranges {430840=674014,1540767=450076,2684980=684432} 2023-08-27T20:54:58.689 INFO:tasks.rados.rados.0.smithi093.stdout:3400: writing smithi093153278-216 from 430840 to 1104854 tid 1 2023-08-27T20:54:58.694 INFO:tasks.rados.rados.0.smithi093.stdout:3400: writing smithi093153278-216 from 1540767 to 1990843 tid 2 2023-08-27T20:54:58.701 INFO:tasks.rados.rados.0.smithi093.stdout:3400: writing smithi093153278-216 from 2684980 to 3369412 tid 3 2023-08-27T20:54:58.702 INFO:tasks.rados.rados.0.smithi093.stdout:3401: write oid 188 current snap is 0 2023-08-27T20:54:58.702 INFO:tasks.rados.rados.0.smithi093.stdout:3393: finishing write tid 5 to smithi093153278-245 2023-08-27T20:54:58.702 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 245 v 552 (ObjNum 2116 snap 0 seq_num 2116) dirty exists 2023-08-27T20:54:58.702 INFO:tasks.rados.rados.0.smithi093.stdout:3393: left oid 245 (ObjNum 2116 snap 0 seq_num 2116) 2023-08-27T20:54:58.702 INFO:tasks.rados.rados.0.smithi093.stdout:3395: expect (ObjNum 1032 snap 0 seq_num 1032) 2023-08-27T20:54:58.838 INFO:tasks.rados.rados.0.smithi093.stdout:3394: expect (ObjNum 1099 snap 0 seq_num 1099) 2023-08-27T20:54:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1496670645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:58.981 INFO:tasks.rados.rados.0.smithi093.stdout:3397: finishing write tid 1 to smithi093153278-33 2023-08-27T20:54:58.981 INFO:tasks.rados.rados.0.smithi093.stdout:3397: finishing write tid 2 to smithi093153278-33 2023-08-27T20:54:58.981 INFO:tasks.rados.rados.0.smithi093.stdout:3397: finishing write tid 3 to smithi093153278-33 2023-08-27T20:54:58.981 INFO:tasks.rados.rados.0.smithi093.stdout:3397: finishing write tid 4 to smithi093153278-33 2023-08-27T20:54:58.981 INFO:tasks.rados.rados.0.smithi093.stdout:3397: finishing write tid 5 to smithi093153278-33 2023-08-27T20:54:58.981 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 33 v 479 (ObjNum 2118 snap 0 seq_num 2118) dirty exists 2023-08-27T20:54:58.982 INFO:tasks.rados.rados.0.smithi093.stdout:3397: left oid 33 (ObjNum 2118 snap 0 seq_num 2118) 2023-08-27T20:54:58.982 INFO:tasks.rados.rados.0.smithi093.stdout:3399: finishing write tid 1 to smithi093153278-86 2023-08-27T20:54:58.982 INFO:tasks.rados.rados.0.smithi093.stdout:3401: seq_num 2121 ranges {447647=749263,1599249=593734,2596050=495704,3631525=16122} 2023-08-27T20:54:58.987 INFO:tasks.rados.rados.0.smithi093.stdout:3401: writing smithi093153278-188 from 447647 to 1196910 tid 1 2023-08-27T20:54:58.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1496670645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1496670645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:54:58.993 INFO:tasks.rados.rados.0.smithi093.stdout:3401: writing smithi093153278-188 from 1599249 to 2192983 tid 2 2023-08-27T20:54:58.998 INFO:tasks.rados.rados.0.smithi093.stdout:3401: writing smithi093153278-188 from 2596050 to 3091754 tid 3 2023-08-27T20:54:59.002 INFO:tasks.rados.rados.0.smithi093.stdout:3401: writing smithi093153278-188 from 3631525 to 3647647 tid 4 2023-08-27T20:54:59.002 INFO:tasks.rados.rados.0.smithi093.stdout:3393: done (7 left) 2023-08-27T20:54:59.002 INFO:tasks.rados.rados.0.smithi093.stdout:3394: done (6 left) 2023-08-27T20:54:59.002 INFO:tasks.rados.rados.0.smithi093.stdout:3395: done (5 left) 2023-08-27T20:54:59.002 INFO:tasks.rados.rados.0.smithi093.stdout:3397: done (4 left) 2023-08-27T20:54:59.002 INFO:tasks.rados.rados.0.smithi093.stdout:3402: write oid 320 current snap is 0 2023-08-27T20:54:59.002 INFO:tasks.rados.rados.0.smithi093.stdout:3402: seq_num 2122 ranges {464454=540865,1774084=453745,2856179=706976} 2023-08-27T20:54:59.007 INFO:tasks.rados.rados.0.smithi093.stdout:3402: writing smithi093153278-320 from 464454 to 1005319 tid 1 2023-08-27T20:54:59.013 INFO:tasks.rados.rados.0.smithi093.stdout:3402: writing smithi093153278-320 from 1774084 to 2227829 tid 2 2023-08-27T20:54:59.021 INFO:tasks.rados.rados.0.smithi093.stdout:3402: writing smithi093153278-320 from 2856179 to 3563155 tid 3 2023-08-27T20:54:59.021 INFO:tasks.rados.rados.0.smithi093.stdout:3403: write oid 388 current snap is 0 2023-08-27T20:54:59.021 INFO:tasks.rados.rados.0.smithi093.stdout:3403: seq_num 2123 ranges {481261=616114,1832566=597403,2883602=518248} 2023-08-27T20:54:59.027 INFO:tasks.rados.rados.0.smithi093.stdout:3403: writing smithi093153278-388 from 481261 to 1097375 tid 1 2023-08-27T20:54:59.034 INFO:tasks.rados.rados.0.smithi093.stdout:3403: writing smithi093153278-388 from 1832566 to 2429969 tid 2 2023-08-27T20:54:59.041 INFO:tasks.rados.rados.0.smithi093.stdout:3403: writing smithi093153278-388 from 2883602 to 3401850 tid 3 2023-08-27T20:54:59.041 INFO:tasks.rados.rados.0.smithi093.stdout:3404: read oid 297 snap -1 2023-08-27T20:54:59.042 INFO:tasks.rados.rados.0.smithi093.stdout:3404: expect (ObjNum 1929 snap 0 seq_num 1929) 2023-08-27T20:54:59.042 INFO:tasks.rados.rados.0.smithi093.stdout:3405: delete oid 416 current snap is 0 2023-08-27T20:54:59.042 INFO:tasks.rados.rados.0.smithi093.stdout:3398: expect (ObjNum 2033 snap 0 seq_num 2033) 2023-08-27T20:54:59.185 INFO:tasks.rados.rados.0.smithi093.stdout:3399: finishing write tid 2 to smithi093153278-86 2023-08-27T20:54:59.185 INFO:tasks.rados.rados.0.smithi093.stdout:3399: finishing write tid 3 to smithi093153278-86 2023-08-27T20:54:59.186 INFO:tasks.rados.rados.0.smithi093.stdout:3399: finishing write tid 4 to smithi093153278-86 2023-08-27T20:54:59.186 INFO:tasks.rados.rados.0.smithi093.stdout:3399: finishing write tid 5 to smithi093153278-86 2023-08-27T20:54:59.186 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 86 v 401 (ObjNum 2119 snap 0 seq_num 2119) dirty exists 2023-08-27T20:54:59.186 INFO:tasks.rados.rados.0.smithi093.stdout:3399: left oid 86 (ObjNum 2119 snap 0 seq_num 2119) 2023-08-27T20:54:59.186 INFO:tasks.rados.rados.0.smithi093.stdout:3400: finishing write tid 1 to smithi093153278-216 2023-08-27T20:54:59.186 INFO:tasks.rados.rados.0.smithi093.stdout:3398: done (7 left) 2023-08-27T20:54:59.187 INFO:tasks.rados.rados.0.smithi093.stdout:3399: done (6 left) 2023-08-27T20:54:59.187 INFO:tasks.rados.rados.0.smithi093.stdout:3405: done (5 left) 2023-08-27T20:54:59.187 INFO:tasks.rados.rados.0.smithi093.stdout:3406: read oid 393 snap -1 2023-08-27T20:54:59.187 INFO:tasks.rados.rados.0.smithi093.stdout:3406: expect (ObjNum 1186 snap 0 seq_num 1186) 2023-08-27T20:54:59.588 INFO:tasks.rados.rados.0.smithi093.stdout:3400: finishing write tid 2 to smithi093153278-216 2023-08-27T20:54:59.588 INFO:tasks.rados.rados.0.smithi093.stdout:3400: finishing write tid 3 to smithi093153278-216 2023-08-27T20:54:59.588 INFO:tasks.rados.rados.0.smithi093.stdout:3400: finishing write tid 4 to smithi093153278-216 2023-08-27T20:54:59.589 INFO:tasks.rados.rados.0.smithi093.stdout:3400: finishing write tid 5 to smithi093153278-216 2023-08-27T20:54:59.589 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 216 v 458 (ObjNum 2120 snap 0 seq_num 2120) dirty exists 2023-08-27T20:54:59.589 INFO:tasks.rados.rados.0.smithi093.stdout:3400: left oid 216 (ObjNum 2120 snap 0 seq_num 2120) 2023-08-27T20:54:59.589 INFO:tasks.rados.rados.0.smithi093.stdout:3401: finishing write tid 1 to smithi093153278-188 2023-08-27T20:54:59.589 INFO:tasks.rados.rados.0.smithi093.stdout:3401: finishing write tid 2 to smithi093153278-188 2023-08-27T20:54:59.743 INFO:tasks.rados.rados.0.smithi093.stdout:3407: read oid 12 snap -1 2023-08-27T20:54:59.743 INFO:tasks.rados.rados.0.smithi093.stdout:3407: expect deleted 2023-08-27T20:54:59.744 INFO:tasks.rados.rados.0.smithi093.stdout:3400: done (6 left) 2023-08-27T20:54:59.744 INFO:tasks.rados.rados.0.smithi093.stdout:3408: read oid 274 snap -1 2023-08-27T20:54:59.744 INFO:tasks.rados.rados.0.smithi093.stdout:3408: expect (ObjNum 1849 snap 0 seq_num 1849) 2023-08-27T20:54:59.744 INFO:tasks.rados.rados.0.smithi093.stdout:3409: read oid 277 snap -1 2023-08-27T20:54:59.745 INFO:tasks.rados.rados.0.smithi093.stdout:3409: expect (ObjNum 1884 snap 0 seq_num 1884) 2023-08-27T20:54:59.745 INFO:tasks.rados.rados.0.smithi093.stdout:3410: write oid 279 current snap is 0 2023-08-27T20:54:59.746 INFO:tasks.rados.rados.0.smithi093.stdout:3401: finishing write tid 3 to smithi093153278-188 2023-08-27T20:54:59.746 INFO:tasks.rados.rados.0.smithi093.stdout:3401: finishing write tid 4 to smithi093153278-188 2023-08-27T20:54:59.746 INFO:tasks.rados.rados.0.smithi093.stdout:3410: seq_num 2125 ranges {514875=766612,1833177=601072,3222095=492780} 2023-08-27T20:54:59.749 INFO:tasks.rados.rados.0.smithi093.stdout:3410: writing smithi093153278-279 from 514875 to 1281487 tid 1 2023-08-27T20:54:59.749 INFO:tasks.rados.rados.0.smithi093.stdout:3410: writing smithi093153278-279 from 1833177 to 2434249 tid 2 2023-08-27T20:54:59.749 INFO:tasks.rados.rados.0.smithi093.stdout:3410: writing smithi093153278-279 from 3222095 to 3714875 tid 3 2023-08-27T20:54:59.749 INFO:tasks.rados.rados.0.smithi093.stdout:3411: delete oid 194 current snap is 0 2023-08-27T20:54:59.749 INFO:tasks.rados.rados.0.smithi093.stdout:3401: finishing write tid 5 to smithi093153278-188 2023-08-27T20:54:59.750 INFO:tasks.rados.rados.0.smithi093.stdout:3401: finishing write tid 6 to smithi093153278-188 2023-08-27T20:54:59.750 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 188 v 501 (ObjNum 2121 snap 0 seq_num 2121) dirty exists 2023-08-27T20:54:59.750 INFO:tasks.rados.rados.0.smithi093.stdout:3401: left oid 188 (ObjNum 2121 snap 0 seq_num 2121) 2023-08-27T20:54:59.750 INFO:tasks.rados.rados.0.smithi093.stdout:3402: finishing write tid 1 to smithi093153278-320 2023-08-27T20:54:59.750 INFO:tasks.rados.rados.0.smithi093.stdout:3402: finishing write tid 2 to smithi093153278-320 2023-08-27T20:54:59.751 INFO:tasks.rados.rados.0.smithi093.stdout:3402: finishing write tid 3 to smithi093153278-320 2023-08-27T20:54:59.751 INFO:tasks.rados.rados.0.smithi093.stdout:3402: finishing write tid 4 to smithi093153278-320 2023-08-27T20:54:59.751 INFO:tasks.rados.rados.0.smithi093.stdout:3402: finishing write tid 5 to smithi093153278-320 2023-08-27T20:54:59.751 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 320 v 556 (ObjNum 2122 snap 0 seq_num 2122) dirty exists 2023-08-27T20:54:59.751 INFO:tasks.rados.rados.0.smithi093.stdout:3402: left oid 320 (ObjNum 2122 snap 0 seq_num 2122) 2023-08-27T20:54:59.752 INFO:tasks.rados.rados.0.smithi093.stdout:3403: finishing write tid 1 to smithi093153278-388 2023-08-27T20:54:59.752 INFO:tasks.rados.rados.0.smithi093.stdout:3403: finishing write tid 2 to smithi093153278-388 2023-08-27T20:54:59.752 INFO:tasks.rados.rados.0.smithi093.stdout:3403: finishing write tid 3 to smithi093153278-388 2023-08-27T20:54:59.752 INFO:tasks.rados.rados.0.smithi093.stdout:3403: finishing write tid 4 to smithi093153278-388 2023-08-27T20:54:59.752 INFO:tasks.rados.rados.0.smithi093.stdout:3403: finishing write tid 5 to smithi093153278-388 2023-08-27T20:54:59.752 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 388 v 483 (ObjNum 2123 snap 0 seq_num 2123) dirty exists 2023-08-27T20:54:59.753 INFO:tasks.rados.rados.0.smithi093.stdout:3403: left oid 388 (ObjNum 2123 snap 0 seq_num 2123) 2023-08-27T20:54:59.753 INFO:tasks.rados.rados.0.smithi093.stdout:3404: expect (ObjNum 1929 snap 0 seq_num 1929) 2023-08-27T20:54:59.753 INFO:tasks.rados.rados.0.smithi093.stdout:3408: expect (ObjNum 1849 snap 0 seq_num 1849) 2023-08-27T20:54:59.753 INFO:tasks.rados.rados.0.smithi093.stdout:3409: expect (ObjNum 1884 snap 0 seq_num 1884) 2023-08-27T20:54:59.753 INFO:tasks.rados.rados.0.smithi093.stdout:3410: finishing write tid 1 to smithi093153278-279 2023-08-27T20:54:59.754 INFO:tasks.rados.rados.0.smithi093.stdout:3406: expect (ObjNum 1186 snap 0 seq_num 1186) 2023-08-27T20:54:59.816 INFO:tasks.rados.rados.0.smithi093.stdout:3410: finishing write tid 2 to smithi093153278-279 2023-08-27T20:55:00.015 INFO:tasks.rados.rados.0.smithi093.stdout:3410: finishing write tid 3 to smithi093153278-279 2023-08-27T20:55:00.015 INFO:tasks.rados.rados.0.smithi093.stdout:3410: finishing write tid 4 to smithi093153278-279 2023-08-27T20:55:00.015 INFO:tasks.rados.rados.0.smithi093.stdout:3410: finishing write tid 5 to smithi093153278-279 2023-08-27T20:55:00.016 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 279 v 505 (ObjNum 2125 snap 0 seq_num 2125) dirty exists 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3410: left oid 279 (ObjNum 2125 snap 0 seq_num 2125) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3401: done (9 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3402: done (8 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3403: done (7 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3404: done (6 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3406: done (5 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3407: done (4 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3408: done (3 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3409: done (2 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3410: done (1 left) 2023-08-27T20:55:00.345 INFO:tasks.rados.rados.0.smithi093.stdout:3411: done (0 left) 2023-08-27T20:55:00.346 INFO:tasks.rados.rados.0.smithi093.stdout:3412: read oid 20 snap -1 2023-08-27T20:55:00.346 INFO:tasks.rados.rados.0.smithi093.stdout:3412: expect (ObjNum 2094 snap 0 seq_num 2094) 2023-08-27T20:55:00.346 INFO:tasks.rados.rados.0.smithi093.stdout:3413: write oid 351 current snap is 0 2023-08-27T20:55:00.346 INFO:tasks.rados.rados.0.smithi093.stdout:3413: seq_num 2127 ranges {548489=517110,1550141=604741,2876941=679689} 2023-08-27T20:55:00.347 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:54:59 smithi136 ceph-mon[113531]: pgmap v43: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 7.3 MiB/s wr, 76 op/s 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3413: writing smithi093153278-351 from 548489 to 1065599 tid 1 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3413: writing smithi093153278-351 from 1550141 to 2154882 tid 2 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3413: writing smithi093153278-351 from 2876941 to 3556630 tid 3 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3414: read oid 324 snap -1 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3414: expect (ObjNum 2016 snap 0 seq_num 2016) 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3415: read oid 285 snap -1 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3415: expect (ObjNum 2015 snap 0 seq_num 2015) 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3416: read oid 419 snap -1 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3416: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:55:00.348 INFO:tasks.rados.rados.0.smithi093.stdout:3417: write oid 4 current snap is 0 2023-08-27T20:55:00.349 INFO:tasks.rados.rados.0.smithi093.stdout:3412: expect (ObjNum 2094 snap 0 seq_num 2094) 2023-08-27T20:55:00.349 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:54:59 smithi093 ceph-mon[111335]: pgmap v43: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 7.3 MiB/s wr, 76 op/s 2023-08-27T20:55:00.349 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:54:59 smithi093 ceph-mon[118902]: pgmap v43: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 7.3 MiB/s wr, 76 op/s 2023-08-27T20:55:00.350 INFO:tasks.rados.rados.0.smithi093.stdout:3413: finishing write tid 1 to smithi093153278-351 2023-08-27T20:55:00.350 INFO:tasks.rados.rados.0.smithi093.stdout:3413: finishing write tid 2 to smithi093153278-351 2023-08-27T20:55:00.350 INFO:tasks.rados.rados.0.smithi093.stdout:3417: seq_num 2128 ranges {565296=592359,1892270=464752,2788011=490961} 2023-08-27T20:55:00.350 INFO:tasks.rados.rados.0.smithi093.stdout:3417: writing smithi093153278-4 from 565296 to 1157655 tid 1 2023-08-27T20:55:00.350 INFO:tasks.rados.rados.0.smithi093.stdout:3417: writing smithi093153278-4 from 1892270 to 2357022 tid 2 2023-08-27T20:55:00.351 INFO:tasks.rados.rados.0.smithi093.stdout:3417: writing smithi093153278-4 from 2788011 to 3278972 tid 3 2023-08-27T20:55:00.351 INFO:tasks.rados.rados.0.smithi093.stdout:3412: done (5 left) 2023-08-27T20:55:00.351 INFO:tasks.rados.rados.0.smithi093.stdout:3418: write oid 486 current snap is 0 2023-08-27T20:55:00.820 INFO:tasks.rados.rados.0.smithi093.stdout:3414: expect (ObjNum 2016 snap 0 seq_num 2016) 2023-08-27T20:55:00.821 INFO:tasks.rados.rados.0.smithi093.stdout:3416: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:55:00.821 INFO:tasks.rados.rados.0.smithi093.stdout:3413: finishing write tid 3 to smithi093153278-351 2023-08-27T20:55:00.821 INFO:tasks.rados.rados.0.smithi093.stdout:3413: finishing write tid 4 to smithi093153278-351 2023-08-27T20:55:00.821 INFO:tasks.rados.rados.0.smithi093.stdout:3413: finishing write tid 5 to smithi093153278-351 2023-08-27T20:55:00.821 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 351 v 377 (ObjNum 2127 snap 0 seq_num 2127) dirty exists 2023-08-27T20:55:00.821 INFO:tasks.rados.rados.0.smithi093.stdout:3413: left oid 351 (ObjNum 2127 snap 0 seq_num 2127) 2023-08-27T20:55:00.821 INFO:tasks.rados.rados.0.smithi093.stdout:3415: expect (ObjNum 2015 snap 0 seq_num 2015) 2023-08-27T20:55:00.830 INFO:tasks.rados.rados.0.smithi093.stdout:3417: finishing write tid 1 to smithi093153278-4 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3417: finishing write tid 2 to smithi093153278-4 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3417: finishing write tid 3 to smithi093153278-4 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3417: finishing write tid 4 to smithi093153278-4 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3417: finishing write tid 5 to smithi093153278-4 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 4 v 509 (ObjNum 2128 snap 0 seq_num 2128) dirty exists 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3417: left oid 4 (ObjNum 2128 snap 0 seq_num 2128) 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3418: seq_num 2129 ranges {582103=783961,2067105=608410,3331787=418586} 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3418: writing smithi093153278-486 from 582103 to 1366064 tid 1 2023-08-27T20:55:00.847 INFO:tasks.rados.rados.0.smithi093.stdout:3418: writing smithi093153278-486 from 2067105 to 2675515 tid 2 2023-08-27T20:55:00.848 INFO:tasks.rados.rados.0.smithi093.stdout:3418: writing smithi093153278-486 from 3331787 to 3750373 tid 3 2023-08-27T20:55:00.848 INFO:tasks.rados.rados.0.smithi093.stdout:3413: done (5 left) 2023-08-27T20:55:00.848 INFO:tasks.rados.rados.0.smithi093.stdout:3414: done (4 left) 2023-08-27T20:55:00.848 INFO:tasks.rados.rados.0.smithi093.stdout:3415: done (3 left) 2023-08-27T20:55:00.848 INFO:tasks.rados.rados.0.smithi093.stdout:3416: done (2 left) 2023-08-27T20:55:00.848 INFO:tasks.rados.rados.0.smithi093.stdout:3417: done (1 left) 2023-08-27T20:55:00.849 INFO:tasks.rados.rados.0.smithi093.stdout:3419: write oid 32 current snap is 0 2023-08-27T20:55:00.849 INFO:tasks.rados.rados.0.smithi093.stdout:3418: finishing write tid 1 to smithi093153278-486 2023-08-27T20:55:00.849 INFO:tasks.rados.rados.0.smithi093.stdout:3418: finishing write tid 2 to smithi093153278-486 2023-08-27T20:55:00.849 INFO:tasks.rados.rados.0.smithi093.stdout:3419: seq_num 2130 ranges {598910=459210,1725587=752068,3242857=556053} 2023-08-27T20:55:00.849 INFO:tasks.rados.rados.0.smithi093.stdout:3419: writing smithi093153278-32 from 598910 to 1058120 tid 1 2023-08-27T20:55:00.849 INFO:tasks.rados.rados.0.smithi093.stdout:3419: writing smithi093153278-32 from 1725587 to 2477655 tid 2 2023-08-27T20:55:00.850 INFO:tasks.rados.rados.0.smithi093.stdout:3419: writing smithi093153278-32 from 3242857 to 3798910 tid 3 2023-08-27T20:55:00.850 INFO:tasks.rados.rados.0.smithi093.stdout:3420: write oid 290 current snap is 0 2023-08-27T20:55:00.850 INFO:tasks.rados.rados.0.smithi093.stdout:3418: finishing write tid 3 to smithi093153278-486 2023-08-27T20:55:00.850 INFO:tasks.rados.rados.0.smithi093.stdout:3420: seq_num 2131 ranges {615717=534459,1784069=612079,2986633=441130} 2023-08-27T20:55:00.850 INFO:tasks.rados.rados.0.smithi093.stdout:3420: writing smithi093153278-290 from 615717 to 1150176 tid 1 2023-08-27T20:55:00.850 INFO:tasks.rados.rados.0.smithi093.stdout:3420: writing smithi093153278-290 from 1784069 to 2396148 tid 2 2023-08-27T20:55:00.851 INFO:tasks.rados.rados.0.smithi093.stdout:3420: writing smithi093153278-290 from 2986633 to 3427763 tid 3 2023-08-27T20:55:00.851 INFO:tasks.rados.rados.0.smithi093.stdout:3421: write oid 126 current snap is 0 2023-08-27T20:55:00.851 INFO:tasks.rados.rados.0.smithi093.stdout:3418: finishing write tid 4 to smithi093153278-486 2023-08-27T20:55:00.851 INFO:tasks.rados.rados.0.smithi093.stdout:3418: finishing write tid 5 to smithi093153278-486 2023-08-27T20:55:00.851 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 487 (ObjNum 2129 snap 0 seq_num 2129) dirty exists 2023-08-27T20:55:00.852 INFO:tasks.rados.rados.0.smithi093.stdout:3418: left oid 486 (ObjNum 2129 snap 0 seq_num 2129) 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3419: finishing write tid 1 to smithi093153278-32 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3419: finishing write tid 2 to smithi093153278-32 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3421: seq_num 2132 ranges {632524=609708,1726198=755737,3181350=651174} 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3421: writing smithi093153278-126 from 632524 to 1242232 tid 1 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3421: writing smithi093153278-126 from 1726198 to 2481935 tid 2 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3421: writing smithi093153278-126 from 3181350 to 3832524 tid 3 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3418: done (3 left) 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3422: read oid 19 snap -1 2023-08-27T20:55:01.142 INFO:tasks.rados.rados.0.smithi093.stdout:3422: expect (ObjNum 2097 snap 0 seq_num 2097) 2023-08-27T20:55:01.143 INFO:tasks.rados.rados.0.smithi093.stdout:3423: write oid 33 current snap is 0 2023-08-27T20:55:01.147 INFO:tasks.rados.rados.0.smithi093.stdout:3419: finishing write tid 3 to smithi093153278-32 2023-08-27T20:55:01.147 INFO:tasks.rados.rados.0.smithi093.stdout:3419: finishing write tid 4 to smithi093153278-32 2023-08-27T20:55:01.147 INFO:tasks.rados.rados.0.smithi093.stdout:3419: finishing write tid 5 to smithi093153278-32 2023-08-27T20:55:01.147 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 32 v 513 (ObjNum 2130 snap 0 seq_num 2130) dirty exists 2023-08-27T20:55:01.147 INFO:tasks.rados.rados.0.smithi093.stdout:3419: left oid 32 (ObjNum 2130 snap 0 seq_num 2130) 2023-08-27T20:55:01.148 INFO:tasks.rados.rados.0.smithi093.stdout:3420: finishing write tid 1 to smithi093153278-290 2023-08-27T20:55:01.148 INFO:tasks.rados.rados.0.smithi093.stdout:3420: finishing write tid 2 to smithi093153278-290 2023-08-27T20:55:01.148 INFO:tasks.rados.rados.0.smithi093.stdout:3420: finishing write tid 3 to smithi093153278-290 2023-08-27T20:55:01.148 INFO:tasks.rados.rados.0.smithi093.stdout:3420: finishing write tid 4 to smithi093153278-290 2023-08-27T20:55:01.148 INFO:tasks.rados.rados.0.smithi093.stdout:3423: seq_num 2133 ranges {649331=684957,1784680=615748,2925126=580027} 2023-08-27T20:55:01.150 INFO:tasks.rados.rados.0.smithi093.stdout:3423: writing smithi093153278-33 from 649331 to 1334288 tid 1 2023-08-27T20:55:01.150 INFO:tasks.rados.rados.0.smithi093.stdout:3423: writing smithi093153278-33 from 1784680 to 2400428 tid 2 2023-08-27T20:55:01.150 INFO:tasks.rados.rados.0.smithi093.stdout:3423: writing smithi093153278-33 from 2925126 to 3505153 tid 3 2023-08-27T20:55:01.150 INFO:tasks.rados.rados.0.smithi093.stdout:3419: done (4 left) 2023-08-27T20:55:01.150 INFO:tasks.rados.rados.0.smithi093.stdout:3424: read oid 263 snap -1 2023-08-27T20:55:01.150 INFO:tasks.rados.rados.0.smithi093.stdout:3424: expect (ObjNum 1900 snap 0 seq_num 1900) 2023-08-27T20:55:01.150 INFO:tasks.rados.rados.0.smithi093.stdout:3425: read oid 79 snap -1 2023-08-27T20:55:01.151 INFO:tasks.rados.rados.0.smithi093.stdout:3425: expect (ObjNum 1804 snap 0 seq_num 1804) 2023-08-27T20:55:01.151 INFO:tasks.rados.rados.0.smithi093.stdout:3426: delete oid 129 current snap is 0 2023-08-27T20:55:01.151 INFO:tasks.rados.rados.0.smithi093.stdout:3420: finishing write tid 5 to smithi093153278-290 2023-08-27T20:55:01.152 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 290 v 397 (ObjNum 2131 snap 0 seq_num 2131) dirty exists 2023-08-27T20:55:01.152 INFO:tasks.rados.rados.0.smithi093.stdout:3420: left oid 290 (ObjNum 2131 snap 0 seq_num 2131) 2023-08-27T20:55:01.152 INFO:tasks.rados.rados.0.smithi093.stdout:3421: finishing write tid 1 to smithi093153278-126 2023-08-27T20:55:01.152 INFO:tasks.rados.rados.0.smithi093.stdout:3421: finishing write tid 2 to smithi093153278-126 2023-08-27T20:55:01.152 INFO:tasks.rados.rados.0.smithi093.stdout:3421: finishing write tid 3 to smithi093153278-126 2023-08-27T20:55:01.152 INFO:tasks.rados.rados.0.smithi093.stdout:3421: finishing write tid 4 to smithi093153278-126 2023-08-27T20:55:01.152 INFO:tasks.rados.rados.0.smithi093.stdout:3421: finishing write tid 5 to smithi093153278-126 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 405 (ObjNum 2132 snap 0 seq_num 2132) dirty exists 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3421: left oid 126 (ObjNum 2132 snap 0 seq_num 2132) 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3422: expect (ObjNum 2097 snap 0 seq_num 2097) 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3423: finishing write tid 1 to smithi093153278-33 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3423: finishing write tid 2 to smithi093153278-33 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3420: done (6 left) 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3421: done (5 left) 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3422: done (4 left) 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3426: done (3 left) 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3427: read oid 443 snap -1 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3427: expect (ObjNum 2093 snap 0 seq_num 2093) 2023-08-27T20:55:01.153 INFO:tasks.rados.rados.0.smithi093.stdout:3428: read oid 174 snap -1 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3428: expect (ObjNum 1988 snap 0 seq_num 1988) 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3429: read oid 14 snap -1 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3429: expect (ObjNum 1577 snap 0 seq_num 1577) 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3430: write oid 366 current snap is 0 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3423: finishing write tid 3 to smithi093153278-33 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3423: finishing write tid 4 to smithi093153278-33 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3430: seq_num 2135 ranges {682945=435455,1901644=619417,2979972=602571} 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3430: writing smithi093153278-366 from 682945 to 1118400 tid 1 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3430: writing smithi093153278-366 from 1901644 to 2521061 tid 2 2023-08-27T20:55:01.154 INFO:tasks.rados.rados.0.smithi093.stdout:3430: writing smithi093153278-366 from 2979972 to 3582543 tid 3 2023-08-27T20:55:01.155 INFO:tasks.rados.rados.0.smithi093.stdout:3431: delete oid 24 current snap is 0 2023-08-27T20:55:01.155 INFO:tasks.rados.rados.0.smithi093.stdout:3423: finishing write tid 5 to smithi093153278-33 2023-08-27T20:55:01.155 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 33 v 491 (ObjNum 2133 snap 0 seq_num 2133) dirty exists 2023-08-27T20:55:01.155 INFO:tasks.rados.rados.0.smithi093.stdout:3423: left oid 33 (ObjNum 2133 snap 0 seq_num 2133) 2023-08-27T20:55:01.155 INFO:tasks.rados.rados.0.smithi093.stdout:3425: expect (ObjNum 1804 snap 0 seq_num 1804) 2023-08-27T20:55:01.155 INFO:tasks.rados.rados.0.smithi093.stdout:3424: expect (ObjNum 1900 snap 0 seq_num 1900) 2023-08-27T20:55:01.156 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3855648911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:01.156 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3855648911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:01.156 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3855648911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:01.156 INFO:tasks.rados.rados.0.smithi093.stdout:3427: expect (ObjNum 2093 snap 0 seq_num 2093) 2023-08-27T20:55:01.314 INFO:tasks.rados.rados.0.smithi093.stdout:3429: expect (ObjNum 1577 snap 0 seq_num 1577) 2023-08-27T20:55:01.467 INFO:tasks.rados.rados.0.smithi093.stdout:3430: finishing write tid 1 to smithi093153278-366 2023-08-27T20:55:01.534 INFO:tasks.rados.rados.0.smithi093.stdout:3428: expect (ObjNum 1988 snap 0 seq_num 1988) 2023-08-27T20:55:01.611 INFO:tasks.rados.rados.0.smithi093.stdout:3430: finishing write tid 2 to smithi093153278-366 2023-08-27T20:55:01.706 INFO:tasks.rados.rados.0.smithi093.stdout:3430: finishing write tid 3 to smithi093153278-366 2023-08-27T20:55:01.706 INFO:tasks.rados.rados.0.smithi093.stdout:3430: finishing write tid 4 to smithi093153278-366 2023-08-27T20:55:01.707 INFO:tasks.rados.rados.0.smithi093.stdout:3430: finishing write tid 5 to smithi093153278-366 2023-08-27T20:55:01.707 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 366 v 347 (ObjNum 2135 snap 0 seq_num 2135) dirty exists 2023-08-27T20:55:01.707 INFO:tasks.rados.rados.0.smithi093.stdout:3430: left oid 366 (ObjNum 2135 snap 0 seq_num 2135) 2023-08-27T20:55:01.707 INFO:tasks.rados.rados.0.smithi093.stdout:3423: done (7 left) 2023-08-27T20:55:01.707 INFO:tasks.rados.rados.0.smithi093.stdout:3424: done (6 left) 2023-08-27T20:55:01.707 INFO:tasks.rados.rados.0.smithi093.stdout:3425: done (5 left) 2023-08-27T20:55:01.708 INFO:tasks.rados.rados.0.smithi093.stdout:3427: done (4 left) 2023-08-27T20:55:01.708 INFO:tasks.rados.rados.0.smithi093.stdout:3428: done (3 left) 2023-08-27T20:55:01.708 INFO:tasks.rados.rados.0.smithi093.stdout:3429: done (2 left) 2023-08-27T20:55:02.047 INFO:tasks.rados.rados.0.smithi093.stdout:3430: done (1 left) 2023-08-27T20:55:02.048 INFO:tasks.rados.rados.0.smithi093.stdout:3431: done (0 left) 2023-08-27T20:55:02.048 INFO:tasks.rados.rados.0.smithi093.stdout:3432: write oid 417 current snap is 0 2023-08-27T20:55:02.048 INFO:tasks.rados.rados.0.smithi093.stdout:3432: seq_num 2137 ranges {716559=702306,2018608=623086,3434818=481741} 2023-08-27T20:55:02.049 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:01 smithi136 ceph-mon[113531]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 12 MiB/s wr, 118 op/s 2023-08-27T20:55:02.054 INFO:tasks.rados.rados.0.smithi093.stdout:3432: writing smithi093153278-417 from 716559 to 1418865 tid 1 2023-08-27T20:55:02.269 INFO:tasks.rados.rados.0.smithi093.stdout:3432: writing smithi093153278-417 from 2018608 to 2641694 tid 2 2023-08-27T20:55:02.269 INFO:tasks.rados.rados.0.smithi093.stdout:3432: writing smithi093153278-417 from 3434818 to 3916559 tid 3 2023-08-27T20:55:02.275 INFO:tasks.rados.rados.0.smithi093.stdout:3433: read oid 435 snap -1 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3433: expect (ObjNum 1981 snap 0 seq_num 1981) 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3434: read oid 281 snap -1 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3434: expect (ObjNum 1963 snap 0 seq_num 1963) 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3435: write oid 322 current snap is 0 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3432: finishing write tid 1 to smithi093153278-417 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3432: finishing write tid 2 to smithi093153278-417 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3435: seq_num 2138 ranges {733366=777555,2077090=766744,3462241=471125} 2023-08-27T20:55:02.276 INFO:tasks.rados.rados.0.smithi093.stdout:3435: writing smithi093153278-322 from 733366 to 1510921 tid 1 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3435: writing smithi093153278-322 from 2077090 to 2843834 tid 2 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3435: writing smithi093153278-322 from 3462241 to 3933366 tid 3 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3436: read oid 377 snap -1 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3436: expect deleted 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3437: write oid 478 current snap is 0 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3432: finishing write tid 3 to smithi093153278-417 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3437: seq_num 2139 ranges {750173=452804,1735572=626755,3089664=764012} 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3437: writing smithi093153278-478 from 750173 to 1202977 tid 1 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3437: writing smithi093153278-478 from 1735572 to 2362327 tid 2 2023-08-27T20:55:02.277 INFO:tasks.rados.rados.0.smithi093.stdout:3437: writing smithi093153278-478 from 3089664 to 3853676 tid 3 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3438: read oid 36 snap -1 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3438: expect (ObjNum 2022 snap 0 seq_num 2022) 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3439: write oid 101 current snap is 0 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3432: finishing write tid 4 to smithi093153278-417 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3432: finishing write tid 5 to smithi093153278-417 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 417 v 457 (ObjNum 2137 snap 0 seq_num 2137) dirty exists 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3432: left oid 417 (ObjNum 2137 snap 0 seq_num 2137) 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3439: seq_num 2140 ranges {766980=528053,2077701=770413,3400734=566246} 2023-08-27T20:55:02.278 INFO:tasks.rados.rados.0.smithi093.stdout:3439: writing smithi093153278-101 from 766980 to 1295033 tid 1 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3439: writing smithi093153278-101 from 2077701 to 2848114 tid 2 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3439: writing smithi093153278-101 from 3400734 to 3966980 tid 3 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3432: done (7 left) 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3440: read oid 419 snap -1 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3440: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3441: delete oid 192 current snap is 0 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3434: expect (ObjNum 1963 snap 0 seq_num 1963) 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3433: expect (ObjNum 1981 snap 0 seq_num 1981) 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3435: finishing write tid 1 to smithi093153278-322 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3435: finishing write tid 2 to smithi093153278-322 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3435: finishing write tid 3 to smithi093153278-322 2023-08-27T20:55:02.280 INFO:tasks.rados.rados.0.smithi093.stdout:3435: finishing write tid 4 to smithi093153278-322 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3435: finishing write tid 5 to smithi093153278-322 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 322 v 462 (ObjNum 2138 snap 0 seq_num 2138) dirty exists 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3435: left oid 322 (ObjNum 2138 snap 0 seq_num 2138) 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3437: finishing write tid 1 to smithi093153278-478 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3433: done (8 left) 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3434: done (7 left) 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3435: done (6 left) 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3436: done (5 left) 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3441: done (4 left) 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3442: read oid 486 snap -1 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3442: expect (ObjNum 2129 snap 0 seq_num 2129) 2023-08-27T20:55:02.281 INFO:tasks.rados.rados.0.smithi093.stdout:3443: read oid 16 snap -1 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3443: expect (ObjNum 2036 snap 0 seq_num 2036) 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3444: read oid 26 snap -1 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3444: expect deleted 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3445: read oid 445 snap -1 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3445: expect (ObjNum 1316 snap 0 seq_num 1316) 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3446: write oid 173 current snap is 0 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3437: finishing write tid 2 to smithi093153278-478 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3437: finishing write tid 3 to smithi093153278-478 2023-08-27T20:55:02.282 INFO:tasks.rados.rados.0.smithi093.stdout:3437: finishing write tid 4 to smithi093153278-478 2023-08-27T20:55:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:3437: finishing write tid 5 to smithi093153278-478 2023-08-27T20:55:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 478 v 495 (ObjNum 2139 snap 0 seq_num 2139) dirty exists 2023-08-27T20:55:02.427 INFO:tasks.rados.rados.0.smithi093.stdout:3437: left oid 478 (ObjNum 2139 snap 0 seq_num 2139) 2023-08-27T20:55:02.430 INFO:tasks.rados.rados.0.smithi093.stdout:3438: expect (ObjNum 2022 snap 0 seq_num 2022) 2023-08-27T20:55:02.430 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:01 smithi093 ceph-mon[111335]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 12 MiB/s wr, 118 op/s 2023-08-27T20:55:02.431 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:01 smithi093 ceph-mon[118902]: pgmap v44: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 12 MiB/s wr, 118 op/s 2023-08-27T20:55:02.431 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:01.627Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:02.431 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:01.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:02.433 INFO:tasks.rados.rados.0.smithi093.stdout:3439: finishing write tid 1 to smithi093153278-101 2023-08-27T20:55:02.433 INFO:tasks.rados.rados.0.smithi093.stdout:3439: finishing write tid 2 to smithi093153278-101 2023-08-27T20:55:02.433 INFO:tasks.rados.rados.0.smithi093.stdout:3446: seq_num 2142 ranges {400594=678551,1794665=205929} 2023-08-27T20:55:02.433 INFO:tasks.rados.rados.0.smithi093.stdout:3446: writing smithi093153278-173 from 400594 to 1079145 tid 1 2023-08-27T20:55:02.433 INFO:tasks.rados.rados.0.smithi093.stdout:3446: writing smithi093153278-173 from 1794665 to 2000594 tid 2 2023-08-27T20:55:02.433 INFO:tasks.rados.rados.0.smithi093.stdout:3437: done (8 left) 2023-08-27T20:55:02.433 INFO:tasks.rados.rados.0.smithi093.stdout:3438: done (7 left) 2023-08-27T20:55:02.434 INFO:tasks.rados.rados.0.smithi093.stdout:3447: write oid 366 current snap is 0 2023-08-27T20:55:02.434 INFO:tasks.rados.rados.0.smithi093.stdout:3447: seq_num 2143 ranges {417401=753800,1853147=164254} 2023-08-27T20:55:02.434 INFO:tasks.rados.rados.0.smithi093.stdout:3447: writing smithi093153278-366 from 417401 to 1171201 tid 1 2023-08-27T20:55:02.434 INFO:tasks.rados.rados.0.smithi093.stdout:3447: writing smithi093153278-366 from 1853147 to 2017401 tid 2 2023-08-27T20:55:02.434 INFO:tasks.rados.rados.0.smithi093.stdout:3448: write oid 479 current snap is 0 2023-08-27T20:55:02.435 INFO:tasks.rados.rados.0.smithi093.stdout:3439: finishing write tid 3 to smithi093153278-101 2023-08-27T20:55:02.435 INFO:tasks.rados.rados.0.smithi093.stdout:3448: seq_num 2144 ranges {434208=545402,1627982=406226} 2023-08-27T20:55:02.435 INFO:tasks.rados.rados.0.smithi093.stdout:3448: writing smithi093153278-479 from 434208 to 979610 tid 1 2023-08-27T20:55:02.435 INFO:tasks.rados.rados.0.smithi093.stdout:3448: writing smithi093153278-479 from 1627982 to 2034208 tid 2 2023-08-27T20:55:02.435 INFO:tasks.rados.rados.0.smithi093.stdout:3449: write oid 418 current snap is 0 2023-08-27T20:55:02.435 INFO:tasks.rados.rados.0.smithi093.stdout:3440: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:55:02.435 INFO:tasks.rados.rados.0.smithi093.stdout:3439: finishing write tid 4 to smithi093153278-101 2023-08-27T20:55:02.436 INFO:tasks.rados.rados.0.smithi093.stdout:3439: finishing write tid 5 to smithi093153278-101 2023-08-27T20:55:02.436 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 101 v 382 (ObjNum 2140 snap 0 seq_num 2140) dirty exists 2023-08-27T20:55:02.436 INFO:tasks.rados.rados.0.smithi093.stdout:3439: left oid 101 (ObjNum 2140 snap 0 seq_num 2140) 2023-08-27T20:55:02.436 INFO:tasks.rados.rados.0.smithi093.stdout:3445: expect (ObjNum 1316 snap 0 seq_num 1316) 2023-08-27T20:55:02.436 INFO:tasks.rados.rados.0.smithi093.stdout:3442: expect (ObjNum 2129 snap 0 seq_num 2129) 2023-08-27T20:55:02.729 INFO:tasks.rados.rados.0.smithi093.stdout:3443: expect (ObjNum 2036 snap 0 seq_num 2036) 2023-08-27T20:55:02.824 INFO:tasks.rados.rados.0.smithi093.stdout:3446: finishing write tid 1 to smithi093153278-173 2023-08-27T20:55:02.915 INFO:tasks.rados.rados.0.smithi093.stdout:3446: finishing write tid 2 to smithi093153278-173 2023-08-27T20:55:02.915 INFO:tasks.rados.rados.0.smithi093.stdout:3449: seq_num 2145 ranges {451015=620651,1570111=480904} 2023-08-27T20:55:02.923 INFO:tasks.rados.rados.0.smithi093.stdout:3449: writing smithi093153278-418 from 451015 to 1071666 tid 1 2023-08-27T20:55:02.931 INFO:tasks.rados.rados.0.smithi093.stdout:3449: writing smithi093153278-418 from 1570111 to 2051015 tid 2 2023-08-27T20:55:02.932 INFO:tasks.rados.rados.0.smithi093.stdout:3439: done (9 left) 2023-08-27T20:55:02.932 INFO:tasks.rados.rados.0.smithi093.stdout:3440: done (8 left) 2023-08-27T20:55:02.932 INFO:tasks.rados.rados.0.smithi093.stdout:3442: done (7 left) 2023-08-27T20:55:02.932 INFO:tasks.rados.rados.0.smithi093.stdout:3443: done (6 left) 2023-08-27T20:55:02.932 INFO:tasks.rados.rados.0.smithi093.stdout:3444: done (5 left) 2023-08-27T20:55:02.933 INFO:tasks.rados.rados.0.smithi093.stdout:3445: done (4 left) 2023-08-27T20:55:02.933 INFO:tasks.rados.rados.0.smithi093.stdout:3450: write oid 273 current snap is 0 2023-08-27T20:55:02.933 INFO:tasks.rados.rados.0.smithi093.stdout:3446: finishing write tid 3 to smithi093153278-173 2023-08-27T20:55:02.933 INFO:tasks.rados.rados.0.smithi093.stdout:3446: finishing write tid 4 to smithi093153278-173 2023-08-27T20:55:02.933 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 173 v 412 (ObjNum 2142 snap 0 seq_num 2142) dirty exists 2023-08-27T20:55:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:3446: left oid 173 (ObjNum 2142 snap 0 seq_num 2142) 2023-08-27T20:55:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:3447: finishing write tid 1 to smithi093153278-366 2023-08-27T20:55:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:3447: finishing write tid 2 to smithi093153278-366 2023-08-27T20:55:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:3447: finishing write tid 3 to smithi093153278-366 2023-08-27T20:55:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:3447: finishing write tid 4 to smithi093153278-366 2023-08-27T20:55:02.934 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 366 v 350 (ObjNum 2143 snap 0 seq_num 2143) dirty exists 2023-08-27T20:55:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:3447: left oid 366 (ObjNum 2143 snap 0 seq_num 2143) 2023-08-27T20:55:02.935 INFO:tasks.rados.rados.0.smithi093.stdout:3448: finishing write tid 1 to smithi093153278-479 2023-08-27T20:55:02.936 INFO:tasks.rados.rados.0.smithi093.stdout:3448: finishing write tid 2 to smithi093153278-479 2023-08-27T20:55:02.936 INFO:tasks.rados.rados.0.smithi093.stdout:3448: finishing write tid 3 to smithi093153278-479 2023-08-27T20:55:02.936 INFO:tasks.rados.rados.0.smithi093.stdout:3450: seq_num 2146 ranges {467822=695900,1628593=439229} 2023-08-27T20:55:02.936 INFO:tasks.rados.rados.0.smithi093.stdout:3450: writing smithi093153278-273 from 467822 to 1163722 tid 1 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3450: writing smithi093153278-273 from 1628593 to 2067822 tid 2 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3446: done (4 left) 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3447: done (3 left) 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3451: write oid 38 current snap is 0 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3448: finishing write tid 4 to smithi093153278-479 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 479 v 380 (ObjNum 2144 snap 0 seq_num 2144) dirty exists 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3448: left oid 479 (ObjNum 2144 snap 0 seq_num 2144) 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3451: seq_num 2147 ranges {484629=771149,1687075=397554} 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3451: writing smithi093153278-38 from 484629 to 1255778 tid 1 2023-08-27T20:55:03.066 INFO:tasks.rados.rados.0.smithi093.stdout:3451: writing smithi093153278-38 from 1687075 to 2084629 tid 2 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3448: done (3 left) 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3452: write oid 266 current snap is 0 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3449: finishing write tid 1 to smithi093153278-418 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3449: finishing write tid 2 to smithi093153278-418 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3449: finishing write tid 3 to smithi093153278-418 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3449: finishing write tid 4 to smithi093153278-418 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 418 v 498 (ObjNum 2145 snap 0 seq_num 2145) dirty exists 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3449: left oid 418 (ObjNum 2145 snap 0 seq_num 2145) 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3450: finishing write tid 1 to smithi093153278-273 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3452: seq_num 2148 ranges {501436=446398,1745557=355879} 2023-08-27T20:55:03.067 INFO:tasks.rados.rados.0.smithi093.stdout:3452: writing smithi093153278-266 from 501436 to 947834 tid 1 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3452: writing smithi093153278-266 from 1745557 to 2101436 tid 2 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3449: done (3 left) 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3453: write oid 193 current snap is 0 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3450: finishing write tid 2 to smithi093153278-273 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3450: finishing write tid 3 to smithi093153278-273 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3450: finishing write tid 4 to smithi093153278-273 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 273 v 559 (ObjNum 2146 snap 0 seq_num 2146) dirty exists 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3450: left oid 273 (ObjNum 2146 snap 0 seq_num 2146) 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3451: finishing write tid 1 to smithi093153278-38 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3453: seq_num 2149 ranges {518243=521647,1687686=430557} 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3453: writing smithi093153278-193 from 518243 to 1039890 tid 1 2023-08-27T20:55:03.068 INFO:tasks.rados.rados.0.smithi093.stdout:3453: writing smithi093153278-193 from 1687686 to 2118243 tid 2 2023-08-27T20:55:03.069 INFO:tasks.rados.rados.0.smithi093.stdout:3450: done (3 left) 2023-08-27T20:55:03.069 INFO:tasks.rados.rados.0.smithi093.stdout:3454: read oid 5 snap -1 2023-08-27T20:55:03.069 INFO:tasks.rados.rados.0.smithi093.stdout:3454: expect (ObjNum 1932 snap 0 seq_num 1932) 2023-08-27T20:55:03.069 INFO:tasks.rados.rados.0.smithi093.stdout:3455: read oid 294 snap -1 2023-08-27T20:55:03.069 INFO:tasks.rados.rados.0.smithi093.stdout:3455: expect (ObjNum 2079 snap 0 seq_num 2079) 2023-08-27T20:55:03.077 INFO:tasks.rados.rados.0.smithi093.stdout:3456: read oid 125 snap -1 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3456: expect (ObjNum 1734 snap 0 seq_num 1734) 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3457: write oid 451 current snap is 0 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3451: finishing write tid 2 to smithi093153278-38 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3451: finishing write tid 3 to smithi093153278-38 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3451: finishing write tid 4 to smithi093153278-38 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 38 v 383 (ObjNum 2147 snap 0 seq_num 2147) dirty exists 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3451: left oid 38 (ObjNum 2147 snap 0 seq_num 2147) 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3452: finishing write tid 1 to smithi093153278-266 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3457: seq_num 2150 ranges {535050=596896,1746168=388882} 2023-08-27T20:55:03.079 INFO:tasks.rados.rados.0.smithi093.stdout:3457: writing smithi093153278-451 from 535050 to 1131946 tid 1 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3457: writing smithi093153278-451 from 1746168 to 2135050 tid 2 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3451: done (6 left) 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3458: read oid 351 snap -1 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3458: expect (ObjNum 2127 snap 0 seq_num 2127) 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3459: read oid 428 snap -1 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3459: expect deleted 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3460: write oid 110 current snap is 0 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3452: finishing write tid 2 to smithi093153278-266 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3460: seq_num 2151 ranges {551857=672145,1804650=347207} 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3460: writing smithi093153278-110 from 551857 to 1224002 tid 1 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3460: writing smithi093153278-110 from 1804650 to 2151857 tid 2 2023-08-27T20:55:03.080 INFO:tasks.rados.rados.0.smithi093.stdout:3461: write oid 12 current snap is 0 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3461: seq_num 2152 ranges {568664=463747,1579485=589179} 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3461: writing smithi093153278-12 from 568664 to 1032411 tid 1 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3461: writing smithi093153278-12 from 1579485 to 2168664 tid 2 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3462: write oid 33 current snap is 0 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3452: finishing write tid 3 to smithi093153278-266 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3452: finishing write tid 4 to smithi093153278-266 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 266 v 386 (ObjNum 2148 snap 0 seq_num 2148) dirty exists 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3452: left oid 266 (ObjNum 2148 snap 0 seq_num 2148) 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3453: finishing write tid 1 to smithi093153278-193 2023-08-27T20:55:03.081 INFO:tasks.rados.rados.0.smithi093.stdout:3462: seq_num 2153 ranges {585471=538996,1921614=263857} 2023-08-27T20:55:03.082 INFO:tasks.rados.rados.0.smithi093.stdout:3462: writing smithi093153278-33 from 585471 to 1124467 tid 1 2023-08-27T20:55:03.083 INFO:tasks.rados.rados.0.smithi093.stdout:3462: writing smithi093153278-33 from 1921614 to 2185471 tid 2 2023-08-27T20:55:03.095 INFO:tasks.rados.rados.0.smithi093.stdout:3452: done (10 left) 2023-08-27T20:55:03.095 INFO:tasks.rados.rados.0.smithi093.stdout:3463: delete oid 395 current snap is 0 2023-08-27T20:55:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:3453: finishing write tid 2 to smithi093153278-193 2023-08-27T20:55:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:3453: finishing write tid 3 to smithi093153278-193 2023-08-27T20:55:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:3453: finishing write tid 4 to smithi093153278-193 2023-08-27T20:55:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 193 v 279 (ObjNum 2149 snap 0 seq_num 2149) dirty exists 2023-08-27T20:55:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:3453: left oid 193 (ObjNum 2149 snap 0 seq_num 2149) 2023-08-27T20:55:03.096 INFO:tasks.rados.rados.0.smithi093.stdout:3455: expect (ObjNum 2079 snap 0 seq_num 2079) 2023-08-27T20:55:03.101 INFO:tasks.rados.rados.0.smithi093.stdout:3457: finishing write tid 1 to smithi093153278-451 2023-08-27T20:55:03.295 INFO:tasks.rados.rados.0.smithi093.stdout:3457: finishing write tid 2 to smithi093153278-451 2023-08-27T20:55:03.296 INFO:tasks.rados.rados.0.smithi093.stdout:3457: finishing write tid 3 to smithi093153278-451 2023-08-27T20:55:03.296 INFO:tasks.rados.rados.0.smithi093.stdout:3457: finishing write tid 4 to smithi093153278-451 2023-08-27T20:55:03.296 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 451 v 370 (ObjNum 2150 snap 0 seq_num 2150) dirty exists 2023-08-27T20:55:03.296 INFO:tasks.rados.rados.0.smithi093.stdout:3457: left oid 451 (ObjNum 2150 snap 0 seq_num 2150) 2023-08-27T20:55:03.296 INFO:tasks.rados.rados.0.smithi093.stdout:3456: expect (ObjNum 1734 snap 0 seq_num 1734) 2023-08-27T20:55:03.296 INFO:tasks.rados.rados.0.smithi093.stdout:3454: expect (ObjNum 1932 snap 0 seq_num 1932) 2023-08-27T20:55:03.382 INFO:tasks.rados.rados.0.smithi093.stdout:3458: expect (ObjNum 2127 snap 0 seq_num 2127) 2023-08-27T20:55:03.706 INFO:tasks.rados.rados.0.smithi093.stdout:3460: finishing write tid 1 to smithi093153278-110 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:3460: finishing write tid 2 to smithi093153278-110 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:3460: finishing write tid 3 to smithi093153278-110 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:3460: finishing write tid 4 to smithi093153278-110 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 110 v 460 (ObjNum 2151 snap 0 seq_num 2151) dirty exists 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:3460: left oid 110 (ObjNum 2151 snap 0 seq_num 2151) 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:3461: finishing write tid 1 to smithi093153278-12 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:3453: done (10 left) 2023-08-27T20:55:03.711 INFO:tasks.rados.rados.0.smithi093.stdout:3454: done (9 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3455: done (8 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3456: done (7 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3457: done (6 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3458: done (5 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3459: done (4 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3460: done (3 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3463: done (2 left) 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3464: write oid 151 current snap is 0 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3461: finishing write tid 2 to smithi093153278-12 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3461: finishing write tid 3 to smithi093153278-12 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:3461: finishing write tid 4 to smithi093153278-12 2023-08-27T20:55:03.712 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 12 v 385 (ObjNum 2152 snap 0 seq_num 2152) dirty exists 2023-08-27T20:55:03.713 INFO:tasks.rados.rados.0.smithi093.stdout:3461: left oid 12 (ObjNum 2152 snap 0 seq_num 2152) 2023-08-27T20:55:03.713 INFO:tasks.rados.rados.0.smithi093.stdout:3462: finishing write tid 1 to smithi093153278-33 2023-08-27T20:55:03.713 INFO:tasks.rados.rados.0.smithi093.stdout:3462: finishing write tid 2 to smithi093153278-33 2023-08-27T20:55:03.865 INFO:tasks.rados.rados.0.smithi093.stdout:3462: finishing write tid 3 to smithi093153278-33 2023-08-27T20:55:03.865 INFO:tasks.rados.rados.0.smithi093.stdout:3462: finishing write tid 4 to smithi093153278-33 2023-08-27T20:55:03.865 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 33 v 501 (ObjNum 2153 snap 0 seq_num 2153) dirty exists 2023-08-27T20:55:03.865 INFO:tasks.rados.rados.0.smithi093.stdout:3462: left oid 33 (ObjNum 2153 snap 0 seq_num 2153) 2023-08-27T20:55:03.865 INFO:tasks.rados.rados.0.smithi093.stdout:3464: seq_num 2155 ranges {619085=689494,2038578=180507} 2023-08-27T20:55:03.865 INFO:tasks.rados.rados.0.smithi093.stdout:3464: writing smithi093153278-151 from 619085 to 1308579 tid 1 2023-08-27T20:55:03.865 INFO:tasks.rados.rados.0.smithi093.stdout:3464: writing smithi093153278-151 from 2038578 to 2219085 tid 2 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3461: done (2 left) 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3462: done (1 left) 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3465: read oid 353 snap -1 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3465: expect deleted 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3466: read oid 44 snap -1 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3466: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3467: read oid 274 snap -1 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3467: expect (ObjNum 1849 snap 0 seq_num 1849) 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3468: read oid 213 snap -1 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3468: expect (ObjNum 981 snap 0 seq_num 981) 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3469: read oid 91 snap -1 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3469: expect (ObjNum 2059 snap 0 seq_num 2059) 2023-08-27T20:55:03.866 INFO:tasks.rados.rados.0.smithi093.stdout:3470: read oid 160 snap -1 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3470: expect (ObjNum 1941 snap 0 seq_num 1941) 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3471: read oid 447 snap -1 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3471: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3472: read oid 319 snap -1 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3472: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3473: read oid 157 snap -1 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3473: expect (ObjNum 156 snap 0 seq_num 156) 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3474: write oid 67 current snap is 0 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3474: seq_num 2156 ranges {635892=764743,2097060=138832} 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3474: writing smithi093153278-67 from 635892 to 1400635 tid 1 2023-08-27T20:55:03.867 INFO:tasks.rados.rados.0.smithi093.stdout:3474: writing smithi093153278-67 from 2097060 to 2235892 tid 2 2023-08-27T20:55:03.868 INFO:tasks.rados.rados.0.smithi093.stdout:3475: delete oid 325 current snap is 0 2023-08-27T20:55:03.868 INFO:tasks.rados.rados.0.smithi093.stdout:3464: finishing write tid 1 to smithi093153278-151 2023-08-27T20:55:03.868 INFO:tasks.rados.rados.0.smithi093.stdout:3467: expect (ObjNum 1849 snap 0 seq_num 1849) 2023-08-27T20:55:03.868 INFO:tasks.rados.rados.0.smithi093.stdout:3473: expect (ObjNum 156 snap 0 seq_num 156) 2023-08-27T20:55:03.868 INFO:tasks.rados.rados.0.smithi093.stdout:3469: expect (ObjNum 2059 snap 0 seq_num 2059) 2023-08-27T20:55:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:03 smithi136 ceph-mon[113531]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 6.6 MiB/s wr, 77 op/s 2023-08-27T20:55:04.481 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/417431537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:04.483 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:03 smithi093 ceph-mon[111335]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 6.6 MiB/s wr, 77 op/s 2023-08-27T20:55:04.995 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/417431537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:04.995 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:03 smithi093 ceph-mon[118902]: pgmap v45: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 6.6 MiB/s wr, 77 op/s 2023-08-27T20:55:04.996 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/417431537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:04.996 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:03.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:04.996 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:03.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:05.006 INFO:tasks.rados.rados.0.smithi093.stdout:3468: expect (ObjNum 981 snap 0 seq_num 981) 2023-08-27T20:55:05.006 INFO:tasks.rados.rados.0.smithi093.stdout:3471: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:05.007 INFO:tasks.rados.rados.0.smithi093.stdout:3466: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:55:05.007 INFO:tasks.rados.rados.0.smithi093.stdout:3470: expect (ObjNum 1941 snap 0 seq_num 1941) 2023-08-27T20:55:05.007 INFO:tasks.rados.rados.0.smithi093.stdout:3464: finishing write tid 2 to smithi093153278-151 2023-08-27T20:55:05.007 INFO:tasks.rados.rados.0.smithi093.stdout:3464: finishing write tid 3 to smithi093153278-151 2023-08-27T20:55:05.007 INFO:tasks.rados.rados.0.smithi093.stdout:3464: finishing write tid 4 to smithi093153278-151 2023-08-27T20:55:05.008 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 151 v 399 (ObjNum 2155 snap 0 seq_num 2155) dirty exists 2023-08-27T20:55:05.008 INFO:tasks.rados.rados.0.smithi093.stdout:3464: left oid 151 (ObjNum 2155 snap 0 seq_num 2155) 2023-08-27T20:55:05.008 INFO:tasks.rados.rados.0.smithi093.stdout:3474: finishing write tid 1 to smithi093153278-67 2023-08-27T20:55:05.008 INFO:tasks.rados.rados.0.smithi093.stdout:3472: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:55:05.008 INFO:tasks.rados.rados.0.smithi093.stdout:3474: finishing write tid 2 to smithi093153278-67 2023-08-27T20:55:05.008 INFO:tasks.rados.rados.0.smithi093.stdout:3474: finishing write tid 3 to smithi093153278-67 2023-08-27T20:55:05.009 INFO:tasks.rados.rados.0.smithi093.stdout:3474: finishing write tid 4 to smithi093153278-67 2023-08-27T20:55:05.155 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 67 v 388 (ObjNum 2156 snap 0 seq_num 2156) dirty exists 2023-08-27T20:55:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:3474: left oid 67 (ObjNum 2156 snap 0 seq_num 2156) 2023-08-27T20:55:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:3464: done (11 left) 2023-08-27T20:55:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:3465: done (10 left) 2023-08-27T20:55:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:3466: done (9 left) 2023-08-27T20:55:05.156 INFO:tasks.rados.rados.0.smithi093.stdout:3467: done (8 left) 2023-08-27T20:55:05.157 INFO:tasks.rados.rados.0.smithi093.stdout:3468: done (7 left) 2023-08-27T20:55:05.157 INFO:tasks.rados.rados.0.smithi093.stdout:3469: done (6 left) 2023-08-27T20:55:05.157 INFO:tasks.rados.rados.0.smithi093.stdout:3470: done (5 left) 2023-08-27T20:55:05.157 INFO:tasks.rados.rados.0.smithi093.stdout:3471: done (4 left) 2023-08-27T20:55:05.157 INFO:tasks.rados.rados.0.smithi093.stdout:3472: done (3 left) 2023-08-27T20:55:05.157 INFO:tasks.rados.rados.0.smithi093.stdout:3473: done (2 left) 2023-08-27T20:55:05.158 INFO:tasks.rados.rados.0.smithi093.stdout:3474: done (1 left) 2023-08-27T20:55:05.158 INFO:tasks.rados.rados.0.smithi093.stdout:3475: done (0 left) 2023-08-27T20:55:05.158 INFO:tasks.rados.rados.0.smithi093.stdout:3476: read oid 65 snap -1 2023-08-27T20:55:05.158 INFO:tasks.rados.rados.0.smithi093.stdout:3476: expect deleted 2023-08-27T20:55:05.158 INFO:tasks.rados.rados.0.smithi093.stdout:3477: write oid 252 current snap is 0 2023-08-27T20:55:05.159 INFO:tasks.rados.rados.0.smithi093.stdout:3477: seq_num 2158 ranges {669506=515241,1697671=571835} 2023-08-27T20:55:05.159 INFO:tasks.rados.rados.0.smithi093.stdout:3477: writing smithi093153278-252 from 669506 to 1184747 tid 1 2023-08-27T20:55:05.159 INFO:tasks.rados.rados.0.smithi093.stdout:3477: writing smithi093153278-252 from 1697671 to 2269506 tid 2 2023-08-27T20:55:05.160 INFO:tasks.rados.rados.0.smithi093.stdout:3478: write oid 103 current snap is 0 2023-08-27T20:55:05.160 INFO:tasks.rados.rados.0.smithi093.stdout:3478: seq_num 2159 ranges {686313=590490,1756153=530160} 2023-08-27T20:55:05.160 INFO:tasks.rados.rados.0.smithi093.stdout:3478: writing smithi093153278-103 from 686313 to 1276803 tid 1 2023-08-27T20:55:05.160 INFO:tasks.rados.rados.0.smithi093.stdout:3478: writing smithi093153278-103 from 1756153 to 2286313 tid 2 2023-08-27T20:55:05.160 INFO:tasks.rados.rados.0.smithi093.stdout:3479: read oid 351 snap -1 2023-08-27T20:55:05.160 INFO:tasks.rados.rados.0.smithi093.stdout:3479: expect (ObjNum 2127 snap 0 seq_num 2127) 2023-08-27T20:55:05.161 INFO:tasks.rados.rados.0.smithi093.stdout:3480: write oid 118 current snap is 0 2023-08-27T20:55:05.161 INFO:tasks.rados.rados.0.smithi093.stdout:3477: finishing write tid 1 to smithi093153278-252 2023-08-27T20:55:05.161 INFO:tasks.rados.rados.0.smithi093.stdout:3480: seq_num 2160 ranges {703120=782092,1930988=372132} 2023-08-27T20:55:05.161 INFO:tasks.rados.rados.0.smithi093.stdout:3480: writing smithi093153278-118 from 703120 to 1485212 tid 1 2023-08-27T20:55:05.166 INFO:tasks.rados.rados.0.smithi093.stdout:3480: writing smithi093153278-118 from 1930988 to 2303120 tid 2 2023-08-27T20:55:05.166 INFO:tasks.rados.rados.0.smithi093.stdout:3476: done (4 left) 2023-08-27T20:55:05.166 INFO:tasks.rados.rados.0.smithi093.stdout:3481: write oid 60 current snap is 0 2023-08-27T20:55:05.166 INFO:tasks.rados.rados.0.smithi093.stdout:3477: finishing write tid 2 to smithi093153278-252 2023-08-27T20:55:05.166 INFO:tasks.rados.rados.0.smithi093.stdout:3477: finishing write tid 3 to smithi093153278-252 2023-08-27T20:55:05.166 INFO:tasks.rados.rados.0.smithi093.stdout:3477: finishing write tid 4 to smithi093153278-252 2023-08-27T20:55:05.166 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 252 v 401 (ObjNum 2158 snap 0 seq_num 2158) dirty exists 2023-08-27T20:55:05.167 INFO:tasks.rados.rados.0.smithi093.stdout:3477: left oid 252 (ObjNum 2158 snap 0 seq_num 2158) 2023-08-27T20:55:05.167 INFO:tasks.rados.rados.0.smithi093.stdout:3478: finishing write tid 1 to smithi093153278-103 2023-08-27T20:55:05.167 INFO:tasks.rados.rados.0.smithi093.stdout:3481: seq_num 2161 ranges {719927=457341,1873117=446810} 2023-08-27T20:55:05.167 INFO:tasks.rados.rados.0.smithi093.stdout:3481: writing smithi093153278-60 from 719927 to 1177268 tid 1 2023-08-27T20:55:05.167 INFO:tasks.rados.rados.0.smithi093.stdout:3481: writing smithi093153278-60 from 1873117 to 2319927 tid 2 2023-08-27T20:55:05.167 INFO:tasks.rados.rados.0.smithi093.stdout:3477: done (4 left) 2023-08-27T20:55:05.167 INFO:tasks.rados.rados.0.smithi093.stdout:3482: delete oid 179 current snap is 0 2023-08-27T20:55:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:3478: finishing write tid 2 to smithi093153278-103 2023-08-27T20:55:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:3478: finishing write tid 3 to smithi093153278-103 2023-08-27T20:55:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:3478: finishing write tid 4 to smithi093153278-103 2023-08-27T20:55:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 103 v 353 (ObjNum 2159 snap 0 seq_num 2159) dirty exists 2023-08-27T20:55:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:3478: left oid 103 (ObjNum 2159 snap 0 seq_num 2159) 2023-08-27T20:55:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:3479: expect (ObjNum 2127 snap 0 seq_num 2127) 2023-08-27T20:55:05.168 INFO:tasks.rados.rados.0.smithi093.stdout:3480: finishing write tid 1 to smithi093153278-118 2023-08-27T20:55:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:3480: finishing write tid 2 to smithi093153278-118 2023-08-27T20:55:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:3480: finishing write tid 3 to smithi093153278-118 2023-08-27T20:55:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:3480: finishing write tid 4 to smithi093153278-118 2023-08-27T20:55:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 118 v 415 (ObjNum 2160 snap 0 seq_num 2160) dirty exists 2023-08-27T20:55:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:3480: left oid 118 (ObjNum 2160 snap 0 seq_num 2160) 2023-08-27T20:55:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:3481: finishing write tid 1 to smithi093153278-60 2023-08-27T20:55:05.169 INFO:tasks.rados.rados.0.smithi093.stdout:3478: done (4 left) 2023-08-27T20:55:05.170 INFO:tasks.rados.rados.0.smithi093.stdout:3479: done (3 left) 2023-08-27T20:55:05.170 INFO:tasks.rados.rados.0.smithi093.stdout:3480: done (2 left) 2023-08-27T20:55:05.170 INFO:tasks.rados.rados.0.smithi093.stdout:3482: done (1 left) 2023-08-27T20:55:05.317 INFO:tasks.rados.rados.0.smithi093.stdout:3483: read oid 5 snap -1 2023-08-27T20:55:05.317 INFO:tasks.rados.rados.0.smithi093.stdout:3483: expect (ObjNum 1932 snap 0 seq_num 1932) 2023-08-27T20:55:05.317 INFO:tasks.rados.rados.0.smithi093.stdout:3484: read oid 303 snap -1 2023-08-27T20:55:05.317 INFO:tasks.rados.rados.0.smithi093.stdout:3484: expect deleted 2023-08-27T20:55:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:3485: read oid 250 snap -1 2023-08-27T20:55:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:3485: expect (ObjNum 1579 snap 0 seq_num 1579) 2023-08-27T20:55:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:3486: read oid 197 snap -1 2023-08-27T20:55:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:3486: expect deleted 2023-08-27T20:55:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:3487: delete oid 97 current snap is 0 2023-08-27T20:55:05.318 INFO:tasks.rados.rados.0.smithi093.stdout:3481: finishing write tid 2 to smithi093153278-60 2023-08-27T20:55:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:3481: finishing write tid 3 to smithi093153278-60 2023-08-27T20:55:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:3481: finishing write tid 4 to smithi093153278-60 2023-08-27T20:55:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 60 v 404 (ObjNum 2161 snap 0 seq_num 2161) dirty exists 2023-08-27T20:55:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:3481: left oid 60 (ObjNum 2161 snap 0 seq_num 2161) 2023-08-27T20:55:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:3481: done (5 left) 2023-08-27T20:55:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:3484: done (4 left) 2023-08-27T20:55:05.319 INFO:tasks.rados.rados.0.smithi093.stdout:3486: done (3 left) 2023-08-27T20:55:05.320 INFO:tasks.rados.rados.0.smithi093.stdout:3487: done (2 left) 2023-08-27T20:55:05.320 INFO:tasks.rados.rados.0.smithi093.stdout:3488: delete oid 175 current snap is 0 2023-08-27T20:55:05.320 INFO:tasks.rados.rados.0.smithi093.stdout:3485: expect (ObjNum 1579 snap 0 seq_num 1579) 2023-08-27T20:55:05.320 INFO:tasks.rados.rados.0.smithi093.stdout:3483: expect (ObjNum 1932 snap 0 seq_num 1932) 2023-08-27T20:55:05.324 INFO:tasks.rados.rados.0.smithi093.stdout:3483: done (2 left) 2023-08-27T20:55:05.324 INFO:tasks.rados.rados.0.smithi093.stdout:3485: done (1 left) 2023-08-27T20:55:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:3488: done (0 left) 2023-08-27T20:55:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:3489: read oid 163 snap -1 2023-08-27T20:55:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:3489: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:55:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:3490: write oid 173 current snap is 0 2023-08-27T20:55:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:3490: seq_num 2165 ranges {787155=758337,1990692=396463} 2023-08-27T20:55:05.325 INFO:tasks.rados.rados.0.smithi093.stdout:3490: writing smithi093153278-173 from 787155 to 1545492 tid 1 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3490: writing smithi093153278-173 from 1990692 to 2387155 tid 2 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3491: write oid 107 current snap is 0 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3491: seq_num 2166 ranges {403962=433586,1249174=701757} 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3491: writing smithi093153278-107 from 403962 to 837548 tid 1 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3491: writing smithi093153278-107 from 1249174 to 1950931 tid 2 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3492: write oid 383 current snap is 0 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3489: expect (ObjNum 1493 snap 0 seq_num 1493) 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3490: finishing write tid 1 to smithi093153278-173 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3490: finishing write tid 2 to smithi093153278-173 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3490: finishing write tid 3 to smithi093153278-173 2023-08-27T20:55:05.326 INFO:tasks.rados.rados.0.smithi093.stdout:3492: seq_num 2167 ranges {420769=625188,1824009=445415} 2023-08-27T20:55:05.327 INFO:tasks.rados.rados.0.smithi093.stdout:3492: writing smithi093153278-383 from 420769 to 1045957 tid 1 2023-08-27T20:55:05.327 INFO:tasks.rados.rados.0.smithi093.stdout:3492: writing smithi093153278-383 from 1824009 to 2269424 tid 2 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3489: done (3 left) 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3493: write oid 304 current snap is 0 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3490: finishing write tid 4 to smithi093153278-173 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 173 v 418 (ObjNum 2165 snap 0 seq_num 2165) dirty exists 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3490: left oid 173 (ObjNum 2165 snap 0 seq_num 2165) 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3491: finishing write tid 1 to smithi093153278-107 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3491: finishing write tid 2 to smithi093153278-107 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3493: seq_num 2168 ranges {437576=700437,1882491=555085} 2023-08-27T20:55:05.652 INFO:tasks.rados.rados.0.smithi093.stdout:3493: writing smithi093153278-304 from 437576 to 1138013 tid 1 2023-08-27T20:55:05.656 INFO:tasks.rados.rados.0.smithi093.stdout:3493: writing smithi093153278-304 from 1882491 to 2437576 tid 2 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3490: done (3 left) 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3494: write oid 132 current snap is 0 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3494: seq_num 2169 ranges {454383=775686,1824620=449084} 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3494: writing smithi093153278-132 from 454383 to 1230069 tid 1 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3494: writing smithi093153278-132 from 1824620 to 2273704 tid 2 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3495: write oid 277 current snap is 0 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3491: finishing write tid 3 to smithi093153278-107 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3491: finishing write tid 4 to smithi093153278-107 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 107 v 373 (ObjNum 2166 snap 0 seq_num 2166) dirty exists 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3491: left oid 107 (ObjNum 2166 snap 0 seq_num 2166) 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3495: seq_num 2170 ranges {471190=450935,1483102=709095} 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3495: writing smithi093153278-277 from 471190 to 922125 tid 1 2023-08-27T20:55:05.806 INFO:tasks.rados.rados.0.smithi093.stdout:3495: writing smithi093153278-277 from 1483102 to 2192197 tid 2 2023-08-27T20:55:05.807 INFO:tasks.rados.rados.0.smithi093.stdout:3491: done (4 left) 2023-08-27T20:55:05.810 INFO:tasks.rados.rados.0.smithi093.stdout:3496: delete oid 373 current snap is 0 2023-08-27T20:55:05.810 INFO:tasks.rados.rados.0.smithi093.stdout:3492: finishing write tid 1 to smithi093153278-383 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3492: finishing write tid 2 to smithi093153278-383 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3492: finishing write tid 3 to smithi093153278-383 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3492: finishing write tid 4 to smithi093153278-383 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 383 v 494 (ObjNum 2167 snap 0 seq_num 2167) dirty exists 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3492: left oid 383 (ObjNum 2167 snap 0 seq_num 2167) 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3493: finishing write tid 1 to smithi093153278-304 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3493: finishing write tid 2 to smithi093153278-304 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3493: finishing write tid 3 to smithi093153278-304 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3493: finishing write tid 4 to smithi093153278-304 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 304 v 516 (ObjNum 2168 snap 0 seq_num 2168) dirty exists 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3493: left oid 304 (ObjNum 2168 snap 0 seq_num 2168) 2023-08-27T20:55:05.811 INFO:tasks.rados.rados.0.smithi093.stdout:3494: finishing write tid 1 to smithi093153278-132 2023-08-27T20:55:05.812 INFO:tasks.rados.rados.0.smithi093.stdout:3494: finishing write tid 2 to smithi093153278-132 2023-08-27T20:55:05.815 INFO:tasks.rados.rados.0.smithi093.stdout:3494: finishing write tid 3 to smithi093153278-132 2023-08-27T20:55:05.815 INFO:tasks.rados.rados.0.smithi093.stdout:3494: finishing write tid 4 to smithi093153278-132 2023-08-27T20:55:05.816 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 132 v 421 (ObjNum 2169 snap 0 seq_num 2169) dirty exists 2023-08-27T20:55:05.816 INFO:tasks.rados.rados.0.smithi093.stdout:3494: left oid 132 (ObjNum 2169 snap 0 seq_num 2169) 2023-08-27T20:55:05.816 INFO:tasks.rados.rados.0.smithi093.stdout:3495: finishing write tid 1 to smithi093153278-277 2023-08-27T20:55:05.832 INFO:tasks.rados.rados.0.smithi093.stdout:3492: done (4 left) 2023-08-27T20:55:05.832 INFO:tasks.rados.rados.0.smithi093.stdout:3493: done (3 left) 2023-08-27T20:55:05.832 INFO:tasks.rados.rados.0.smithi093.stdout:3494: done (2 left) 2023-08-27T20:55:05.832 INFO:tasks.rados.rados.0.smithi093.stdout:3496: done (1 left) 2023-08-27T20:55:05.832 INFO:tasks.rados.rados.0.smithi093.stdout:3497: read oid 419 snap -1 2023-08-27T20:55:05.833 INFO:tasks.rados.rados.0.smithi093.stdout:3497: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:55:05.833 INFO:tasks.rados.rados.0.smithi093.stdout:3498: write oid 468 current snap is 0 2023-08-27T20:55:05.833 INFO:tasks.rados.rados.0.smithi093.stdout:3498: seq_num 2172 ranges {504804=601433,1600066=712764} 2023-08-27T20:55:05.833 INFO:tasks.rados.rados.0.smithi093.stdout:3498: writing smithi093153278-468 from 504804 to 1106237 tid 1 2023-08-27T20:55:05.833 INFO:tasks.rados.rados.0.smithi093.stdout:3498: writing smithi093153278-468 from 1600066 to 2312830 tid 2 2023-08-27T20:55:05.833 INFO:tasks.rados.rados.0.smithi093.stdout:3499: read oid 160 snap -1 2023-08-27T20:55:05.834 INFO:tasks.rados.rados.0.smithi093.stdout:3499: expect (ObjNum 1941 snap 0 seq_num 1941) 2023-08-27T20:55:05.834 INFO:tasks.rados.rados.0.smithi093.stdout:3500: read oid 137 snap -1 2023-08-27T20:55:05.834 INFO:tasks.rados.rados.0.smithi093.stdout:3500: expect deleted 2023-08-27T20:55:05.834 INFO:tasks.rados.rados.0.smithi093.stdout:3501: read oid 231 snap -1 2023-08-27T20:55:05.834 INFO:tasks.rados.rados.0.smithi093.stdout:3501: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:55:05.834 INFO:tasks.rados.rados.0.smithi093.stdout:3502: write oid 142 current snap is 0 2023-08-27T20:55:05.835 INFO:tasks.rados.rados.0.smithi093.stdout:3502: seq_num 2173 ranges {521611=676682,1942195=456422} 2023-08-27T20:55:05.835 INFO:tasks.rados.rados.0.smithi093.stdout:3502: writing smithi093153278-142 from 521611 to 1198293 tid 1 2023-08-27T20:55:05.836 INFO:tasks.rados.rados.0.smithi093.stdout:3502: writing smithi093153278-142 from 1942195 to 2398617 tid 2 2023-08-27T20:55:05.836 INFO:tasks.rados.rados.0.smithi093.stdout:3503: read oid 380 snap -1 2023-08-27T20:55:05.837 INFO:tasks.rados.rados.0.smithi093.stdout:3503: expect deleted 2023-08-27T20:55:05.837 INFO:tasks.rados.rados.0.smithi093.stdout:3504: delete oid 442 current snap is 0 2023-08-27T20:55:05.837 INFO:tasks.rados.rados.0.smithi093.stdout:3495: finishing write tid 2 to smithi093153278-277 2023-08-27T20:55:05.837 INFO:tasks.rados.rados.0.smithi093.stdout:3495: finishing write tid 3 to smithi093153278-277 2023-08-27T20:55:05.837 INFO:tasks.rados.rados.0.smithi093.stdout:3495: finishing write tid 4 to smithi093153278-277 2023-08-27T20:55:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 277 v 464 (ObjNum 2170 snap 0 seq_num 2170) dirty exists 2023-08-27T20:55:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:3495: left oid 277 (ObjNum 2170 snap 0 seq_num 2170) 2023-08-27T20:55:05.947 INFO:tasks.rados.rados.0.smithi093.stdout:3497: expect (ObjNum 1676 snap 0 seq_num 1676) 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:3498: finishing write tid 1 to smithi093153278-468 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:3498: finishing write tid 2 to smithi093153278-468 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:3498: finishing write tid 3 to smithi093153278-468 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:3499: expect (ObjNum 1941 snap 0 seq_num 1941) 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:3498: finishing write tid 4 to smithi093153278-468 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 468 v 424 (ObjNum 2172 snap 0 seq_num 2172) dirty exists 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:3498: left oid 468 (ObjNum 2172 snap 0 seq_num 2172) 2023-08-27T20:55:05.951 INFO:tasks.rados.rados.0.smithi093.stdout:3501: expect (ObjNum 230 snap 0 seq_num 230) 2023-08-27T20:55:05.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:05 smithi136 ceph-mon[113531]: pgmap v46: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 6.6 MiB/s wr, 77 op/s 2023-08-27T20:55:05.952 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/773357112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:05.955 INFO:tasks.rados.rados.0.smithi093.stdout:3502: finishing write tid 1 to smithi093153278-142 2023-08-27T20:55:05.968 INFO:tasks.rados.rados.0.smithi093.stdout:3502: finishing write tid 2 to smithi093153278-142 2023-08-27T20:55:05.968 INFO:tasks.rados.rados.0.smithi093.stdout:3502: finishing write tid 3 to smithi093153278-142 2023-08-27T20:55:05.968 INFO:tasks.rados.rados.0.smithi093.stdout:3502: finishing write tid 4 to smithi093153278-142 2023-08-27T20:55:05.968 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 142 v 427 (ObjNum 2173 snap 0 seq_num 2173) dirty exists 2023-08-27T20:55:05.968 INFO:tasks.rados.rados.0.smithi093.stdout:3502: left oid 142 (ObjNum 2173 snap 0 seq_num 2173) 2023-08-27T20:55:05.968 INFO:tasks.rados.rados.0.smithi093.stdout:3495: done (8 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3497: done (7 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3498: done (6 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3499: done (5 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3500: done (4 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3501: done (3 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3502: done (2 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3503: done (1 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3504: done (0 left) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3505: read oid 376 snap -1 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3505: expect (ObjNum 1904 snap 0 seq_num 1904) 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3506: read oid 404 snap -1 2023-08-27T20:55:05.969 INFO:tasks.rados.rados.0.smithi093.stdout:3506: expect (ObjNum 1799 snap 0 seq_num 1799) 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3507: write oid 255 current snap is 0 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3507: seq_num 2175 ranges {555225=543533,1775512=460091} 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3507: writing smithi093153278-255 from 555225 to 1098758 tid 1 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3507: writing smithi093153278-255 from 1775512 to 2235603 tid 2 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3508: read oid 127 snap -1 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3508: expect deleted 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3509: read oid 250 snap -1 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3509: expect (ObjNum 1579 snap 0 seq_num 1579) 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3510: write oid 42 current snap is 0 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3510: seq_num 2176 ranges {572032=618782,1833994=720102} 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3510: writing smithi093153278-42 from 572032 to 1190814 tid 1 2023-08-27T20:55:05.970 INFO:tasks.rados.rados.0.smithi093.stdout:3510: writing smithi093153278-42 from 1833994 to 2554096 tid 2 2023-08-27T20:55:05.972 INFO:tasks.rados.rados.0.smithi093.stdout:3511: write oid 374 current snap is 0 2023-08-27T20:55:05.972 INFO:tasks.rados.rados.0.smithi093.stdout:3511: seq_num 2177 ranges {588839=694031,1776123=463760} 2023-08-27T20:55:05.972 INFO:tasks.rados.rados.0.smithi093.stdout:3511: writing smithi093153278-374 from 588839 to 1282870 tid 1 2023-08-27T20:55:05.972 INFO:tasks.rados.rados.0.smithi093.stdout:3511: writing smithi093153278-374 from 1776123 to 2239883 tid 2 2023-08-27T20:55:05.973 INFO:tasks.rados.rados.0.smithi093.stdout:3512: write oid 119 current snap is 0 2023-08-27T20:55:05.973 INFO:tasks.rados.rados.0.smithi093.stdout:3512: seq_num 2178 ranges {605646=769280,1834605=607418} 2023-08-27T20:55:05.973 INFO:tasks.rados.rados.0.smithi093.stdout:3512: writing smithi093153278-119 from 605646 to 1374926 tid 1 2023-08-27T20:55:05.973 INFO:tasks.rados.rados.0.smithi093.stdout:3512: writing smithi093153278-119 from 1834605 to 2442023 tid 2 2023-08-27T20:55:05.973 INFO:tasks.rados.rados.0.smithi093.stdout:3513: read oid 345 snap -1 2023-08-27T20:55:05.973 INFO:tasks.rados.rados.0.smithi093.stdout:3513: expect (ObjNum 1529 snap 0 seq_num 1529) 2023-08-27T20:55:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:3514: read oid 227 snap -1 2023-08-27T20:55:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:3514: expect (ObjNum 226 snap 0 seq_num 226) 2023-08-27T20:55:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:3515: write oid 61 current snap is 0 2023-08-27T20:55:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:3515: seq_num 2179 ranges {622453=444529,1493087=467429} 2023-08-27T20:55:05.974 INFO:tasks.rados.rados.0.smithi093.stdout:3515: writing smithi093153278-61 from 622453 to 1066982 tid 1 2023-08-27T20:55:05.975 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:55:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:55:05] "GET /metrics HTTP/1.1" 200 242572 "" "Prometheus/2.43.0" 2023-08-27T20:55:05.975 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:05 smithi093 ceph-mon[111335]: pgmap v46: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 6.6 MiB/s wr, 77 op/s 2023-08-27T20:55:05.976 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/773357112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:05.976 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:05 smithi093 ceph-mon[118902]: pgmap v46: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 63 MiB/s rd, 6.6 MiB/s wr, 77 op/s 2023-08-27T20:55:05.976 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/773357112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:05.978 INFO:tasks.rados.rados.0.smithi093.stdout:3515: writing smithi093153278-61 from 1493087 to 1960516 tid 2 2023-08-27T20:55:05.979 INFO:tasks.rados.rados.0.smithi093.stdout:3516: read oid 177 snap -1 2023-08-27T20:55:05.979 INFO:tasks.rados.rados.0.smithi093.stdout:3516: expect (ObjNum 1974 snap 0 seq_num 1974) 2023-08-27T20:55:05.979 INFO:tasks.rados.rados.0.smithi093.stdout:3517: read oid 302 snap -1 2023-08-27T20:55:05.979 INFO:tasks.rados.rados.0.smithi093.stdout:3517: expect (ObjNum 1947 snap 0 seq_num 1947) 2023-08-27T20:55:05.979 INFO:tasks.rados.rados.0.smithi093.stdout:3518: read oid 427 snap -1 2023-08-27T20:55:05.980 INFO:tasks.rados.rados.0.smithi093.stdout:3518: expect (ObjNum 2046 snap 0 seq_num 2046) 2023-08-27T20:55:05.980 INFO:tasks.rados.rados.0.smithi093.stdout:3519: read oid 55 snap -1 2023-08-27T20:55:05.980 INFO:tasks.rados.rados.0.smithi093.stdout:3519: expect (ObjNum 1970 snap 0 seq_num 1970) 2023-08-27T20:55:05.980 INFO:tasks.rados.rados.0.smithi093.stdout:3520: delete oid 408 current snap is 0 2023-08-27T20:55:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:3506: expect (ObjNum 1799 snap 0 seq_num 1799) 2023-08-27T20:55:05.995 INFO:tasks.rados.rados.0.smithi093.stdout:3505: expect (ObjNum 1904 snap 0 seq_num 1904) 2023-08-27T20:55:06.036 INFO:tasks.rados.rados.0.smithi093.stdout:3507: finishing write tid 1 to smithi093153278-255 2023-08-27T20:55:06.134 INFO:tasks.rados.rados.0.smithi093.stdout:3507: finishing write tid 2 to smithi093153278-255 2023-08-27T20:55:06.134 INFO:tasks.rados.rados.0.smithi093.stdout:3507: finishing write tid 3 to smithi093153278-255 2023-08-27T20:55:06.134 INFO:tasks.rados.rados.0.smithi093.stdout:3507: finishing write tid 4 to smithi093153278-255 2023-08-27T20:55:06.135 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 255 v 563 (ObjNum 2175 snap 0 seq_num 2175) dirty exists 2023-08-27T20:55:06.449 INFO:tasks.rados.rados.0.smithi093.stdout:3507: left oid 255 (ObjNum 2175 snap 0 seq_num 2175) 2023-08-27T20:55:06.449 INFO:tasks.rados.rados.0.smithi093.stdout:3509: expect (ObjNum 1579 snap 0 seq_num 1579) 2023-08-27T20:55:06.458 INFO:tasks.rados.rados.0.smithi093.stdout:3510: finishing write tid 1 to smithi093153278-42 2023-08-27T20:55:06.458 INFO:tasks.rados.rados.0.smithi093.stdout:3510: finishing write tid 2 to smithi093153278-42 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3510: finishing write tid 3 to smithi093153278-42 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3510: finishing write tid 4 to smithi093153278-42 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 42 v 402 (ObjNum 2176 snap 0 seq_num 2176) dirty exists 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3510: left oid 42 (ObjNum 2176 snap 0 seq_num 2176) 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3511: finishing write tid 1 to smithi093153278-374 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3505: done (15 left) 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3506: done (14 left) 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3507: done (13 left) 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3508: done (12 left) 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3509: done (11 left) 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3510: done (10 left) 2023-08-27T20:55:06.459 INFO:tasks.rados.rados.0.smithi093.stdout:3520: done (9 left) 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3521: write oid 365 current snap is 0 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3511: finishing write tid 2 to smithi093153278-374 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3511: finishing write tid 3 to smithi093153278-374 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3511: finishing write tid 4 to smithi093153278-374 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 374 v 376 (ObjNum 2177 snap 0 seq_num 2177) dirty exists 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3511: left oid 374 (ObjNum 2177 snap 0 seq_num 2177) 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3512: finishing write tid 1 to smithi093153278-119 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3521: seq_num 2181 ranges {656067=595027,1893698=471098} 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3521: writing smithi093153278-365 from 656067 to 1251094 tid 1 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3521: writing smithi093153278-365 from 1893698 to 2364796 tid 2 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3511: done (9 left) 2023-08-27T20:55:06.460 INFO:tasks.rados.rados.0.smithi093.stdout:3522: write oid 14 current snap is 0 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3512: finishing write tid 2 to smithi093153278-119 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3512: finishing write tid 3 to smithi093153278-119 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3512: finishing write tid 4 to smithi093153278-119 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 119 v 391 (ObjNum 2178 snap 0 seq_num 2178) dirty exists 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3512: left oid 119 (ObjNum 2178 snap 0 seq_num 2178) 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3522: seq_num 2182 ranges {672874=786629,2068533=604341} 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3522: writing smithi093153278-14 from 672874 to 1459503 tid 1 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3522: writing smithi093153278-14 from 2068533 to 2672874 tid 2 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3512: done (9 left) 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3523: read oid 141 snap -1 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3523: expect (ObjNum 1625 snap 0 seq_num 1625) 2023-08-27T20:55:06.461 INFO:tasks.rados.rados.0.smithi093.stdout:3524: write oid 425 current snap is 0 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3524: seq_num 2183 ranges {689681=461878,1727015=474767} 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3524: writing smithi093153278-425 from 689681 to 1151559 tid 1 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3524: writing smithi093153278-425 from 1727015 to 2201782 tid 2 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3525: read oid 498 snap -1 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3525: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3526: write oid 74 current snap is 0 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3515: finishing write tid 1 to smithi093153278-61 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3514: expect (ObjNum 226 snap 0 seq_num 226) 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3515: finishing write tid 2 to smithi093153278-61 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3515: finishing write tid 3 to smithi093153278-61 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:3515: finishing write tid 4 to smithi093153278-61 2023-08-27T20:55:06.462 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 61 v 504 (ObjNum 2179 snap 0 seq_num 2179) dirty exists 2023-08-27T20:55:06.463 INFO:tasks.rados.rados.0.smithi093.stdout:3515: left oid 61 (ObjNum 2179 snap 0 seq_num 2179) 2023-08-27T20:55:06.463 INFO:tasks.rados.rados.0.smithi093.stdout:3517: expect (ObjNum 1947 snap 0 seq_num 1947) 2023-08-27T20:55:06.463 INFO:tasks.rados.rados.0.smithi093.stdout:3513: expect (ObjNum 1529 snap 0 seq_num 1529) 2023-08-27T20:55:06.585 INFO:tasks.rados.rados.0.smithi093.stdout:3516: expect (ObjNum 1974 snap 0 seq_num 1974) 2023-08-27T20:55:06.726 INFO:tasks.rados.rados.0.smithi093.stdout:3519: expect (ObjNum 1970 snap 0 seq_num 1970) 2023-08-27T20:55:06.893 INFO:tasks.rados.rados.0.smithi093.stdout:3518: expect (ObjNum 2046 snap 0 seq_num 2046) 2023-08-27T20:55:06.967 INFO:tasks.rados.rados.0.smithi093.stdout:3521: finishing write tid 1 to smithi093153278-365 2023-08-27T20:55:06.967 INFO:tasks.rados.rados.0.smithi093.stdout:3521: finishing write tid 2 to smithi093153278-365 2023-08-27T20:55:06.967 INFO:tasks.rados.rados.0.smithi093.stdout:3521: finishing write tid 3 to smithi093153278-365 2023-08-27T20:55:06.967 INFO:tasks.rados.rados.0.smithi093.stdout:3521: finishing write tid 4 to smithi093153278-365 2023-08-27T20:55:06.967 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 365 v 519 (ObjNum 2181 snap 0 seq_num 2181) dirty exists 2023-08-27T20:55:06.968 INFO:tasks.rados.rados.0.smithi093.stdout:3521: left oid 365 (ObjNum 2181 snap 0 seq_num 2181) 2023-08-27T20:55:06.968 INFO:tasks.rados.rados.0.smithi093.stdout:3522: finishing write tid 1 to smithi093153278-14 2023-08-27T20:55:06.968 INFO:tasks.rados.rados.0.smithi093.stdout:3522: finishing write tid 2 to smithi093153278-14 2023-08-27T20:55:06.968 INFO:tasks.rados.rados.0.smithi093.stdout:3526: seq_num 2184 ranges {706488=537127,1785497=618425} 2023-08-27T20:55:06.973 INFO:tasks.rados.rados.0.smithi093.stdout:3526: writing smithi093153278-74 from 706488 to 1243615 tid 1 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3526: writing smithi093153278-74 from 1785497 to 2403922 tid 2 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3513: done (12 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3514: done (11 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3515: done (10 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3516: done (9 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3517: done (8 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3518: done (7 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3519: done (6 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3521: done (5 left) 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3527: read oid 461 snap -1 2023-08-27T20:55:06.981 INFO:tasks.rados.rados.0.smithi093.stdout:3527: expect (ObjNum 704 snap 0 seq_num 704) 2023-08-27T20:55:06.982 INFO:tasks.rados.rados.0.smithi093.stdout:3528: write oid 4 current snap is 0 2023-08-27T20:55:06.982 INFO:tasks.rados.rados.0.smithi093.stdout:3522: finishing write tid 3 to smithi093153278-14 2023-08-27T20:55:06.982 INFO:tasks.rados.rados.0.smithi093.stdout:3522: finishing write tid 4 to smithi093153278-14 2023-08-27T20:55:06.982 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 14 v 430 (ObjNum 2182 snap 0 seq_num 2182) dirty exists 2023-08-27T20:55:06.982 INFO:tasks.rados.rados.0.smithi093.stdout:3522: left oid 14 (ObjNum 2182 snap 0 seq_num 2182) 2023-08-27T20:55:06.982 INFO:tasks.rados.rados.0.smithi093.stdout:3523: expect (ObjNum 1625 snap 0 seq_num 1625) 2023-08-27T20:55:07.155 INFO:tasks.rados.rados.0.smithi093.stdout:3524: finishing write tid 1 to smithi093153278-425 2023-08-27T20:55:07.155 INFO:tasks.rados.rados.0.smithi093.stdout:3524: finishing write tid 2 to smithi093153278-425 2023-08-27T20:55:07.155 INFO:tasks.rados.rados.0.smithi093.stdout:3524: finishing write tid 3 to smithi093153278-425 2023-08-27T20:55:07.155 INFO:tasks.rados.rados.0.smithi093.stdout:3524: finishing write tid 4 to smithi093153278-425 2023-08-27T20:55:07.155 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 425 v 434 (ObjNum 2183 snap 0 seq_num 2183) dirty exists 2023-08-27T20:55:07.155 INFO:tasks.rados.rados.0.smithi093.stdout:3524: left oid 425 (ObjNum 2183 snap 0 seq_num 2183) 2023-08-27T20:55:07.156 INFO:tasks.rados.rados.0.smithi093.stdout:3525: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:55:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:3526: finishing write tid 1 to smithi093153278-74 2023-08-27T20:55:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:3526: finishing write tid 2 to smithi093153278-74 2023-08-27T20:55:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:3526: finishing write tid 3 to smithi093153278-74 2023-08-27T20:55:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:3526: finishing write tid 4 to smithi093153278-74 2023-08-27T20:55:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 74 v 566 (ObjNum 2184 snap 0 seq_num 2184) dirty exists 2023-08-27T20:55:07.317 INFO:tasks.rados.rados.0.smithi093.stdout:3526: left oid 74 (ObjNum 2184 snap 0 seq_num 2184) 2023-08-27T20:55:07.318 INFO:tasks.rados.rados.0.smithi093.stdout:3527: expect (ObjNum 704 snap 0 seq_num 704) 2023-08-27T20:55:07.530 INFO:tasks.rados.rados.0.smithi093.stdout:3528: seq_num 2185 ranges {723295=612376,2127626=478436} 2023-08-27T20:55:07.537 INFO:tasks.rados.rados.0.smithi093.stdout:3528: writing smithi093153278-4 from 723295 to 1335671 tid 1 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3528: writing smithi093153278-4 from 2127626 to 2606062 tid 2 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3522: done (6 left) 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3523: done (5 left) 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3524: done (4 left) 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3525: done (3 left) 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3526: done (2 left) 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3527: done (1 left) 2023-08-27T20:55:07.545 INFO:tasks.rados.rados.0.smithi093.stdout:3529: write oid 71 current snap is 0 2023-08-27T20:55:07.546 INFO:tasks.rados.rados.0.smithi093.stdout:3528: finishing write tid 1 to smithi093153278-4 2023-08-27T20:55:07.546 INFO:tasks.rados.rados.0.smithi093.stdout:3529: seq_num 2186 ranges {740102=687625,2186108=553994} 2023-08-27T20:55:07.552 INFO:tasks.rados.rados.0.smithi093.stdout:3529: writing smithi093153278-71 from 740102 to 1427727 tid 1 2023-08-27T20:55:07.560 INFO:tasks.rados.rados.0.smithi093.stdout:3529: writing smithi093153278-71 from 2186108 to 2740102 tid 2 2023-08-27T20:55:07.560 INFO:tasks.rados.rados.0.smithi093.stdout:3530: read oid 402 snap -1 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3530: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3531: write oid 431 current snap is 0 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3528: finishing write tid 2 to smithi093153278-4 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3528: finishing write tid 3 to smithi093153278-4 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3528: finishing write tid 4 to smithi093153278-4 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 4 v 522 (ObjNum 2185 snap 0 seq_num 2185) dirty exists 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3528: left oid 4 (ObjNum 2185 snap 0 seq_num 2185) 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3529: finishing write tid 1 to smithi093153278-71 2023-08-27T20:55:07.561 INFO:tasks.rados.rados.0.smithi093.stdout:3531: seq_num 2187 ranges {756909=762874,2244590=482105} 2023-08-27T20:55:07.568 INFO:tasks.rados.rados.0.smithi093.stdout:3531: writing smithi093153278-431 from 756909 to 1519783 tid 1 2023-08-27T20:55:07.575 INFO:tasks.rados.rados.0.smithi093.stdout:3531: writing smithi093153278-431 from 2244590 to 2726695 tid 2 2023-08-27T20:55:07.575 INFO:tasks.rados.rados.0.smithi093.stdout:3528: done (3 left) 2023-08-27T20:55:07.575 INFO:tasks.rados.rados.0.smithi093.stdout:3532: read oid 176 snap -1 2023-08-27T20:55:07.575 INFO:tasks.rados.rados.0.smithi093.stdout:3532: expect (ObjNum 2050 snap 0 seq_num 2050) 2023-08-27T20:55:07.575 INFO:tasks.rados.rados.0.smithi093.stdout:3533: write oid 48 current snap is 0 2023-08-27T20:55:07.575 INFO:tasks.rados.rados.0.smithi093.stdout:3533: seq_num 2188 ranges {773716=438123,1903072=625763} 2023-08-27T20:55:07.582 INFO:tasks.rados.rados.0.smithi093.stdout:3533: writing smithi093153278-48 from 773716 to 1211839 tid 1 2023-08-27T20:55:07.589 INFO:tasks.rados.rados.0.smithi093.stdout:3533: writing smithi093153278-48 from 1903072 to 2528835 tid 2 2023-08-27T20:55:07.589 INFO:tasks.rados.rados.0.smithi093.stdout:3534: write oid 229 current snap is 0 2023-08-27T20:55:07.590 INFO:tasks.rados.rados.0.smithi093.stdout:3531: finishing write tid 1 to smithi093153278-431 2023-08-27T20:55:07.590 INFO:tasks.rados.rados.0.smithi093.stdout:3534: seq_num 2189 ranges {790523=513372,1961554=769421} 2023-08-27T20:55:07.596 INFO:tasks.rados.rados.0.smithi093.stdout:3534: writing smithi093153278-229 from 790523 to 1303895 tid 1 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:3534: writing smithi093153278-229 from 1961554 to 2730975 tid 2 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:3535: write oid 453 current snap is 0 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:3529: finishing write tid 2 to smithi093153278-71 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:3529: finishing write tid 3 to smithi093153278-71 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:3529: finishing write tid 4 to smithi093153278-71 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 71 v 408 (ObjNum 2186 snap 0 seq_num 2186) dirty exists 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:3529: left oid 71 (ObjNum 2186 snap 0 seq_num 2186) 2023-08-27T20:55:07.605 INFO:tasks.rados.rados.0.smithi093.stdout:3530: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:55:07.785 INFO:tasks.rados.rados.0.smithi093.stdout:3531: finishing write tid 2 to smithi093153278-431 2023-08-27T20:55:07.785 INFO:tasks.rados.rados.0.smithi093.stdout:3531: finishing write tid 3 to smithi093153278-431 2023-08-27T20:55:07.785 INFO:tasks.rados.rados.0.smithi093.stdout:3535: seq_num 2190 ranges {407330=704974,1620036=629432} 2023-08-27T20:55:07.790 INFO:tasks.rados.rados.0.smithi093.stdout:3535: writing smithi093153278-453 from 407330 to 1112304 tid 1 2023-08-27T20:55:07.797 INFO:tasks.rados.rados.0.smithi093.stdout:3535: writing smithi093153278-453 from 1620036 to 2249468 tid 2 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3529: done (6 left) 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3530: done (5 left) 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3536: read oid 482 snap -1 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3536: expect (ObjNum 1935 snap 0 seq_num 1935) 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3537: read oid 391 snap -1 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3537: expect (ObjNum 1624 snap 0 seq_num 1624) 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3538: read oid 105 snap -1 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3538: expect (ObjNum 1850 snap 0 seq_num 1850) 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3539: read oid 50 snap -1 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3539: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:55:07.798 INFO:tasks.rados.rados.0.smithi093.stdout:3540: write oid 395 current snap is 0 2023-08-27T20:55:07.799 INFO:tasks.rados.rados.0.smithi093.stdout:3531: finishing write tid 4 to smithi093153278-431 2023-08-27T20:55:07.799 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 431 v 433 (ObjNum 2187 snap 0 seq_num 2187) dirty exists 2023-08-27T20:55:07.799 INFO:tasks.rados.rados.0.smithi093.stdout:3531: left oid 431 (ObjNum 2187 snap 0 seq_num 2187) 2023-08-27T20:55:07.799 INFO:tasks.rados.rados.0.smithi093.stdout:3532: expect (ObjNum 2050 snap 0 seq_num 2050) 2023-08-27T20:55:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:3533: finishing write tid 1 to smithi093153278-48 2023-08-27T20:55:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:3533: finishing write tid 2 to smithi093153278-48 2023-08-27T20:55:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:3533: finishing write tid 3 to smithi093153278-48 2023-08-27T20:55:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:3533: finishing write tid 4 to smithi093153278-48 2023-08-27T20:55:07.875 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 48 v 436 (ObjNum 2188 snap 0 seq_num 2188) dirty exists 2023-08-27T20:55:07.876 INFO:tasks.rados.rados.0.smithi093.stdout:3533: left oid 48 (ObjNum 2188 snap 0 seq_num 2188) 2023-08-27T20:55:07.876 INFO:tasks.rados.rados.0.smithi093.stdout:3534: finishing write tid 1 to smithi093153278-229 2023-08-27T20:55:07.876 INFO:tasks.rados.rados.0.smithi093.stdout:3540: seq_num 2191 ranges {424137=780223,1678518=773090} 2023-08-27T20:55:07.881 INFO:tasks.rados.rados.0.smithi093.stdout:3540: writing smithi093153278-395 from 424137 to 1204360 tid 1 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3540: writing smithi093153278-395 from 1678518 to 2451608 tid 2 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3531: done (9 left) 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3532: done (8 left) 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3533: done (7 left) 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3541: read oid 333 snap -1 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3541: expect (ObjNum 2049 snap 0 seq_num 2049) 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3542: write oid 238 current snap is 0 2023-08-27T20:55:07.889 INFO:tasks.rados.rados.0.smithi093.stdout:3534: finishing write tid 2 to smithi093153278-229 2023-08-27T20:55:07.890 INFO:tasks.rados.rados.0.smithi093.stdout:3534: finishing write tid 3 to smithi093153278-229 2023-08-27T20:55:07.890 INFO:tasks.rados.rados.0.smithi093.stdout:3542: seq_num 2192 ranges {440944=455472,1337000=633101,2528612=312332} 2023-08-27T20:55:07.894 INFO:tasks.rados.rados.0.smithi093.stdout:3542: writing smithi093153278-238 from 440944 to 896416 tid 1 2023-08-27T20:55:07.900 INFO:tasks.rados.rados.0.smithi093.stdout:3542: writing smithi093153278-238 from 1337000 to 1970101 tid 2 2023-08-27T20:55:07.905 INFO:tasks.rados.rados.0.smithi093.stdout:3542: writing smithi093153278-238 from 2528612 to 2840944 tid 3 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3543: read oid 102 snap -1 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3543: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3544: read oid 498 snap -1 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3544: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3545: read oid 126 snap -1 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3545: expect (ObjNum 2132 snap 0 seq_num 2132) 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3546: write oid 199 current snap is 0 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3534: finishing write tid 4 to smithi093153278-229 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 229 v 437 (ObjNum 2189 snap 0 seq_num 2189) dirty exists 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3534: left oid 229 (ObjNum 2189 snap 0 seq_num 2189) 2023-08-27T20:55:07.906 INFO:tasks.rados.rados.0.smithi093.stdout:3535: finishing write tid 1 to smithi093153278-453 2023-08-27T20:55:07.907 INFO:tasks.rados.rados.0.smithi093.stdout:3537: expect (ObjNum 1624 snap 0 seq_num 1624) 2023-08-27T20:55:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:07 smithi136 ceph-mon[113531]: pgmap v47: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s rd, 9.4 MiB/s wr, 110 op/s 2023-08-27T20:55:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:07 smithi093 ceph-mon[111335]: pgmap v47: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s rd, 9.4 MiB/s wr, 110 op/s 2023-08-27T20:55:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:07 smithi093 ceph-mon[118902]: pgmap v47: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 90 MiB/s rd, 9.4 MiB/s wr, 110 op/s 2023-08-27T20:55:08.046 INFO:tasks.rados.rados.0.smithi093.stdout:3538: expect (ObjNum 1850 snap 0 seq_num 1850) 2023-08-27T20:55:08.221 INFO:tasks.rados.rados.0.smithi093.stdout:3539: expect (ObjNum 1406 snap 0 seq_num 1406) 2023-08-27T20:55:08.377 INFO:tasks.rados.rados.0.smithi093.stdout:3535: finishing write tid 2 to smithi093153278-453 2023-08-27T20:55:08.377 INFO:tasks.rados.rados.0.smithi093.stdout:3535: finishing write tid 3 to smithi093153278-453 2023-08-27T20:55:08.378 INFO:tasks.rados.rados.0.smithi093.stdout:3536: expect (ObjNum 1935 snap 0 seq_num 1935) 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3535: finishing write tid 4 to smithi093153278-453 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 453 v 405 (ObjNum 2190 snap 0 seq_num 2190) dirty exists 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3535: left oid 453 (ObjNum 2190 snap 0 seq_num 2190) 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3540: finishing write tid 1 to smithi093153278-395 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3540: finishing write tid 2 to smithi093153278-395 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3540: finishing write tid 3 to smithi093153278-395 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3540: finishing write tid 4 to smithi093153278-395 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 395 v 569 (ObjNum 2191 snap 0 seq_num 2191) dirty exists 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3540: left oid 395 (ObjNum 2191 snap 0 seq_num 2191) 2023-08-27T20:55:08.486 INFO:tasks.rados.rados.0.smithi093.stdout:3542: finishing write tid 1 to smithi093153278-238 2023-08-27T20:55:08.487 INFO:tasks.rados.rados.0.smithi093.stdout:3541: expect (ObjNum 2049 snap 0 seq_num 2049) 2023-08-27T20:55:08.631 INFO:tasks.rados.rados.0.smithi093.stdout:3542: finishing write tid 2 to smithi093153278-238 2023-08-27T20:55:08.631 INFO:tasks.rados.rados.0.smithi093.stdout:3542: finishing write tid 3 to smithi093153278-238 2023-08-27T20:55:08.631 INFO:tasks.rados.rados.0.smithi093.stdout:3542: finishing write tid 4 to smithi093153278-238 2023-08-27T20:55:08.631 INFO:tasks.rados.rados.0.smithi093.stdout:3542: finishing write tid 5 to smithi093153278-238 2023-08-27T20:55:08.631 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 238 v 573 (ObjNum 2192 snap 0 seq_num 2192) dirty exists 2023-08-27T20:55:08.631 INFO:tasks.rados.rados.0.smithi093.stdout:3542: left oid 238 (ObjNum 2192 snap 0 seq_num 2192) 2023-08-27T20:55:08.632 INFO:tasks.rados.rados.0.smithi093.stdout:3544: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:55:08.792 INFO:tasks.rados.rados.0.smithi093.stdout:3545: expect (ObjNum 2132 snap 0 seq_num 2132) 2023-08-27T20:55:08.906 INFO:tasks.rados.rados.0.smithi093.stdout:3543: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:55:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3352744353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3352744353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3352744353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:09.113 INFO:tasks.rados.rados.0.smithi093.stdout:3546: seq_num 2193 ranges {457751=530721,1395482=776759} 2023-08-27T20:55:09.118 INFO:tasks.rados.rados.0.smithi093.stdout:3546: writing smithi093153278-199 from 457751 to 988472 tid 1 2023-08-27T20:55:09.126 INFO:tasks.rados.rados.0.smithi093.stdout:3546: writing smithi093153278-199 from 1395482 to 2172241 tid 2 2023-08-27T20:55:09.126 INFO:tasks.rados.rados.0.smithi093.stdout:3534: done (12 left) 2023-08-27T20:55:09.126 INFO:tasks.rados.rados.0.smithi093.stdout:3535: done (11 left) 2023-08-27T20:55:09.126 INFO:tasks.rados.rados.0.smithi093.stdout:3536: done (10 left) 2023-08-27T20:55:09.126 INFO:tasks.rados.rados.0.smithi093.stdout:3537: done (9 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3538: done (8 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3539: done (7 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3540: done (6 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3541: done (5 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3542: done (4 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3543: done (3 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3544: done (2 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3545: done (1 left) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3547: read oid 190 snap -1 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3547: expect deleted 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3548: read oid 500 snap -1 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3548: expect (ObjNum 2105 snap 0 seq_num 2105) 2023-08-27T20:55:09.127 INFO:tasks.rados.rados.0.smithi093.stdout:3549: read oid 17 snap -1 2023-08-27T20:55:09.128 INFO:tasks.rados.rados.0.smithi093.stdout:3549: expect (ObjNum 913 snap 0 seq_num 913) 2023-08-27T20:55:09.128 INFO:tasks.rados.rados.0.smithi093.stdout:3550: write oid 352 current snap is 0 2023-08-27T20:55:09.128 INFO:tasks.rados.rados.0.smithi093.stdout:3546: finishing write tid 1 to smithi093153278-199 2023-08-27T20:55:09.128 INFO:tasks.rados.rados.0.smithi093.stdout:3550: seq_num 2194 ranges {474558=605970,1737611=636770,2867105=7453} 2023-08-27T20:55:09.134 INFO:tasks.rados.rados.0.smithi093.stdout:3550: writing smithi093153278-352 from 474558 to 1080528 tid 1 2023-08-27T20:55:09.139 INFO:tasks.rados.rados.0.smithi093.stdout:3550: writing smithi093153278-352 from 1737611 to 2374381 tid 2 2023-08-27T20:55:09.142 INFO:tasks.rados.rados.0.smithi093.stdout:3550: writing smithi093153278-352 from 2867105 to 2874558 tid 3 2023-08-27T20:55:09.142 INFO:tasks.rados.rados.0.smithi093.stdout:3551: write oid 429 current snap is 0 2023-08-27T20:55:09.142 INFO:tasks.rados.rados.0.smithi093.stdout:3546: finishing write tid 2 to smithi093153278-199 2023-08-27T20:55:09.142 INFO:tasks.rados.rados.0.smithi093.stdout:3546: finishing write tid 3 to smithi093153278-199 2023-08-27T20:55:09.143 INFO:tasks.rados.rados.0.smithi093.stdout:3551: seq_num 2195 ranges {491365=681219,1796093=780428} 2023-08-27T20:55:09.148 INFO:tasks.rados.rados.0.smithi093.stdout:3551: writing smithi093153278-429 from 491365 to 1172584 tid 1 2023-08-27T20:55:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:3551: writing smithi093153278-429 from 1796093 to 2576521 tid 2 2023-08-27T20:55:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:3547: done (5 left) 2023-08-27T20:55:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:3552: write oid 386 current snap is 0 2023-08-27T20:55:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:3546: finishing write tid 4 to smithi093153278-199 2023-08-27T20:55:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 199 v 576 (ObjNum 2193 snap 0 seq_num 2193) dirty exists 2023-08-27T20:55:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:3546: left oid 199 (ObjNum 2193 snap 0 seq_num 2193) 2023-08-27T20:55:09.157 INFO:tasks.rados.rados.0.smithi093.stdout:3548: expect (ObjNum 2105 snap 0 seq_num 2105) 2023-08-27T20:55:09.375 INFO:tasks.rados.rados.0.smithi093.stdout:3549: expect (ObjNum 913 snap 0 seq_num 913) 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:3550: finishing write tid 1 to smithi093153278-352 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:3550: finishing write tid 2 to smithi093153278-352 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:3550: finishing write tid 3 to smithi093153278-352 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:3550: finishing write tid 4 to smithi093153278-352 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:3550: finishing write tid 5 to smithi093153278-352 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 352 v 469 (ObjNum 2194 snap 0 seq_num 2194) dirty exists 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:3550: left oid 352 (ObjNum 2194 snap 0 seq_num 2194) 2023-08-27T20:55:09.524 INFO:tasks.rados.rados.0.smithi093.stdout:3551: finishing write tid 1 to smithi093153278-429 2023-08-27T20:55:09.525 INFO:tasks.rados.rados.0.smithi093.stdout:3552: seq_num 2196 ranges {508172=756468,1854575=640439} 2023-08-27T20:55:09.530 INFO:tasks.rados.rados.0.smithi093.stdout:3552: writing smithi093153278-386 from 508172 to 1264640 tid 1 2023-08-27T20:55:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:3552: writing smithi093153278-386 from 1854575 to 2495014 tid 2 2023-08-27T20:55:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:3546: done (5 left) 2023-08-27T20:55:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:3548: done (4 left) 2023-08-27T20:55:09.537 INFO:tasks.rados.rados.0.smithi093.stdout:3549: done (3 left) 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3550: done (2 left) 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3553: write oid 434 current snap is 0 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3551: finishing write tid 2 to smithi093153278-429 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3551: finishing write tid 3 to smithi093153278-429 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3551: finishing write tid 4 to smithi093153278-429 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 429 v 472 (ObjNum 2195 snap 0 seq_num 2195) dirty exists 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3551: left oid 429 (ObjNum 2195 snap 0 seq_num 2195) 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3552: finishing write tid 1 to smithi093153278-386 2023-08-27T20:55:09.538 INFO:tasks.rados.rados.0.smithi093.stdout:3553: seq_num 2197 ranges {524979=431717,1513057=784097} 2023-08-27T20:55:09.542 INFO:tasks.rados.rados.0.smithi093.stdout:3553: writing smithi093153278-434 from 524979 to 956696 tid 1 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3553: writing smithi093153278-434 from 1513057 to 2297154 tid 2 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3551: done (2 left) 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3554: write oid 435 current snap is 0 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3552: finishing write tid 2 to smithi093153278-386 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3552: finishing write tid 3 to smithi093153278-386 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3552: finishing write tid 4 to smithi093153278-386 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 386 v 394 (ObjNum 2196 snap 0 seq_num 2196) dirty exists 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3552: left oid 386 (ObjNum 2196 snap 0 seq_num 2196) 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3553: finishing write tid 1 to smithi093153278-434 2023-08-27T20:55:09.550 INFO:tasks.rados.rados.0.smithi093.stdout:3554: seq_num 2198 ranges {541786=623319,1571539=644108,2693150=248636} 2023-08-27T20:55:09.556 INFO:tasks.rados.rados.0.smithi093.stdout:3554: writing smithi093153278-435 from 541786 to 1165105 tid 1 2023-08-27T20:55:09.561 INFO:tasks.rados.rados.0.smithi093.stdout:3554: writing smithi093153278-435 from 1571539 to 2215647 tid 2 2023-08-27T20:55:09.566 INFO:tasks.rados.rados.0.smithi093.stdout:3554: writing smithi093153278-435 from 2693150 to 2941786 tid 3 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3552: done (2 left) 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3555: read oid 254 snap -1 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3555: expect (ObjNum 1971 snap 0 seq_num 1971) 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3556: read oid 283 snap -1 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3556: expect (ObjNum 1896 snap 0 seq_num 1896) 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3557: write oid 361 current snap is 0 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3553: finishing write tid 2 to smithi093153278-434 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3553: finishing write tid 3 to smithi093153278-434 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3553: finishing write tid 4 to smithi093153278-434 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 407 (ObjNum 2197 snap 0 seq_num 2197) dirty exists 2023-08-27T20:55:09.567 INFO:tasks.rados.rados.0.smithi093.stdout:3553: left oid 434 (ObjNum 2197 snap 0 seq_num 2197) 2023-08-27T20:55:09.568 INFO:tasks.rados.rados.0.smithi093.stdout:3554: finishing write tid 1 to smithi093153278-435 2023-08-27T20:55:09.568 INFO:tasks.rados.rados.0.smithi093.stdout:3554: finishing write tid 2 to smithi093153278-435 2023-08-27T20:55:09.568 INFO:tasks.rados.rados.0.smithi093.stdout:3557: seq_num 2199 ranges {558593=698568,2030021=787766} 2023-08-27T20:55:09.573 INFO:tasks.rados.rados.0.smithi093.stdout:3557: writing smithi093153278-361 from 558593 to 1257161 tid 1 2023-08-27T20:55:09.582 INFO:tasks.rados.rados.0.smithi093.stdout:3557: writing smithi093153278-361 from 2030021 to 2817787 tid 2 2023-08-27T20:55:09.582 INFO:tasks.rados.rados.0.smithi093.stdout:3553: done (4 left) 2023-08-27T20:55:09.583 INFO:tasks.rados.rados.0.smithi093.stdout:3558: read oid 294 snap -1 2023-08-27T20:55:09.583 INFO:tasks.rados.rados.0.smithi093.stdout:3558: expect (ObjNum 2079 snap 0 seq_num 2079) 2023-08-27T20:55:09.583 INFO:tasks.rados.rados.0.smithi093.stdout:3559: write oid 131 current snap is 0 2023-08-27T20:55:09.583 INFO:tasks.rados.rados.0.smithi093.stdout:3559: seq_num 2200 ranges {575400=773817,2088503=647777} 2023-08-27T20:55:09.590 INFO:tasks.rados.rados.0.smithi093.stdout:3559: writing smithi093153278-131 from 575400 to 1349217 tid 1 2023-08-27T20:55:09.598 INFO:tasks.rados.rados.0.smithi093.stdout:3559: writing smithi093153278-131 from 2088503 to 2736280 tid 2 2023-08-27T20:55:09.598 INFO:tasks.rados.rados.0.smithi093.stdout:3560: read oid 452 snap -1 2023-08-27T20:55:09.598 INFO:tasks.rados.rados.0.smithi093.stdout:3560: expect (ObjNum 920 snap 0 seq_num 920) 2023-08-27T20:55:09.598 INFO:tasks.rados.rados.0.smithi093.stdout:3561: write oid 298 current snap is 0 2023-08-27T20:55:09.598 INFO:tasks.rados.rados.0.smithi093.stdout:3561: seq_num 2201 ranges {592207=449066,1746985=791435} 2023-08-27T20:55:09.604 INFO:tasks.rados.rados.0.smithi093.stdout:3561: writing smithi093153278-298 from 592207 to 1041273 tid 1 2023-08-27T20:55:09.612 INFO:tasks.rados.rados.0.smithi093.stdout:3561: writing smithi093153278-298 from 1746985 to 2538420 tid 2 2023-08-27T20:55:09.612 INFO:tasks.rados.rados.0.smithi093.stdout:3562: write oid 189 current snap is 0 2023-08-27T20:55:09.613 INFO:tasks.rados.rados.0.smithi093.stdout:3562: seq_num 2202 ranges {609014=524315,1689114=535093,2970136=38878} 2023-08-27T20:55:09.619 INFO:tasks.rados.rados.0.smithi093.stdout:3562: writing smithi093153278-189 from 609014 to 1133329 tid 1 2023-08-27T20:55:09.625 INFO:tasks.rados.rados.0.smithi093.stdout:3562: writing smithi093153278-189 from 1689114 to 2224207 tid 2 2023-08-27T20:55:09.629 INFO:tasks.rados.rados.0.smithi093.stdout:3562: writing smithi093153278-189 from 2970136 to 3009014 tid 3 2023-08-27T20:55:09.630 INFO:tasks.rados.rados.0.smithi093.stdout:3563: read oid 479 snap -1 2023-08-27T20:55:09.630 INFO:tasks.rados.rados.0.smithi093.stdout:3563: expect (ObjNum 2144 snap 0 seq_num 2144) 2023-08-27T20:55:09.630 INFO:tasks.rados.rados.0.smithi093.stdout:3564: read oid 498 snap -1 2023-08-27T20:55:09.630 INFO:tasks.rados.rados.0.smithi093.stdout:3564: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:55:09.630 INFO:tasks.rados.rados.0.smithi093.stdout:3565: write oid 410 current snap is 0 2023-08-27T20:55:09.630 INFO:tasks.rados.rados.0.smithi093.stdout:3565: seq_num 2203 ranges {625821=599564,1747596=795104,2997559=28262} 2023-08-27T20:55:09.636 INFO:tasks.rados.rados.0.smithi093.stdout:3565: writing smithi093153278-410 from 625821 to 1225385 tid 1 2023-08-27T20:55:09.644 INFO:tasks.rados.rados.0.smithi093.stdout:3565: writing smithi093153278-410 from 1747596 to 2542700 tid 2 2023-08-27T20:55:09.646 INFO:tasks.rados.rados.0.smithi093.stdout:3565: writing smithi093153278-410 from 2997559 to 3025821 tid 3 2023-08-27T20:55:09.646 INFO:tasks.rados.rados.0.smithi093.stdout:3566: write oid 51 current snap is 0 2023-08-27T20:55:09.646 INFO:tasks.rados.rados.0.smithi093.stdout:3555: expect (ObjNum 1971 snap 0 seq_num 1971) 2023-08-27T20:55:09.820 INFO:tasks.rados.rados.0.smithi093.stdout:3554: finishing write tid 3 to smithi093153278-435 2023-08-27T20:55:09.820 INFO:tasks.rados.rados.0.smithi093.stdout:3566: seq_num 2204 ranges {642628=674813,1806078=538762,3024982=17646} 2023-08-27T20:55:09.827 INFO:tasks.rados.rados.0.smithi093.stdout:3566: writing smithi093153278-51 from 642628 to 1317441 tid 1 2023-08-27T20:55:09.833 INFO:tasks.rados.rados.0.smithi093.stdout:3566: writing smithi093153278-51 from 1806078 to 2344840 tid 2 2023-08-27T20:55:09.837 INFO:tasks.rados.rados.0.smithi093.stdout:3566: writing smithi093153278-51 from 3024982 to 3042628 tid 3 2023-08-27T20:55:09.837 INFO:tasks.rados.rados.0.smithi093.stdout:3555: done (12 left) 2023-08-27T20:55:09.838 INFO:tasks.rados.rados.0.smithi093.stdout:3567: delete oid 439 current snap is 0 2023-08-27T20:55:09.838 INFO:tasks.rados.rados.0.smithi093.stdout:3554: finishing write tid 4 to smithi093153278-435 2023-08-27T20:55:09.838 INFO:tasks.rados.rados.0.smithi093.stdout:3554: finishing write tid 5 to smithi093153278-435 2023-08-27T20:55:09.838 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 435 v 508 (ObjNum 2198 snap 0 seq_num 2198) dirty exists 2023-08-27T20:55:09.838 INFO:tasks.rados.rados.0.smithi093.stdout:3554: left oid 435 (ObjNum 2198 snap 0 seq_num 2198) 2023-08-27T20:55:09.838 INFO:tasks.rados.rados.0.smithi093.stdout:3557: finishing write tid 1 to smithi093153278-361 2023-08-27T20:55:09.838 INFO:tasks.rados.rados.0.smithi093.stdout:3556: expect (ObjNum 1896 snap 0 seq_num 1896) 2023-08-27T20:55:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:09 smithi136 ceph-mon[113531]: pgmap v48: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 7.1 MiB/s wr, 75 op/s 2023-08-27T20:55:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:09.964 INFO:tasks.rados.rados.0.smithi093.stdout:3557: finishing write tid 2 to smithi093153278-361 2023-08-27T20:55:09.965 INFO:tasks.rados.rados.0.smithi093.stdout:3558: expect (ObjNum 2079 snap 0 seq_num 2079) 2023-08-27T20:55:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:09 smithi093 ceph-mon[118902]: pgmap v48: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 7.1 MiB/s wr, 75 op/s 2023-08-27T20:55:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:09 smithi093 ceph-mon[111335]: pgmap v48: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 7.1 MiB/s wr, 75 op/s 2023-08-27T20:55:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:10.138 INFO:tasks.rados.rados.0.smithi093.stdout:3557: finishing write tid 3 to smithi093153278-361 2023-08-27T20:55:10.138 INFO:tasks.rados.rados.0.smithi093.stdout:3557: finishing write tid 4 to smithi093153278-361 2023-08-27T20:55:10.138 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 361 v 389 (ObjNum 2199 snap 0 seq_num 2199) dirty exists 2023-08-27T20:55:10.138 INFO:tasks.rados.rados.0.smithi093.stdout:3557: left oid 361 (ObjNum 2199 snap 0 seq_num 2199) 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3559: finishing write tid 1 to smithi093153278-131 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3559: finishing write tid 2 to smithi093153278-131 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3554: done (12 left) 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3556: done (11 left) 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3557: done (10 left) 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3558: done (9 left) 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3567: done (8 left) 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3568: write oid 448 current snap is 0 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3559: finishing write tid 3 to smithi093153278-131 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3559: finishing write tid 4 to smithi093153278-131 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 131 v 392 (ObjNum 2200 snap 0 seq_num 2200) dirty exists 2023-08-27T20:55:10.139 INFO:tasks.rados.rados.0.smithi093.stdout:3559: left oid 131 (ObjNum 2200 snap 0 seq_num 2200) 2023-08-27T20:55:10.140 INFO:tasks.rados.rados.0.smithi093.stdout:3560: expect (ObjNum 920 snap 0 seq_num 920) 2023-08-27T20:55:10.293 INFO:tasks.rados.rados.0.smithi093.stdout:3561: finishing write tid 1 to smithi093153278-298 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:3561: finishing write tid 2 to smithi093153278-298 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:3561: finishing write tid 3 to smithi093153278-298 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:3561: finishing write tid 4 to smithi093153278-298 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 298 v 408 (ObjNum 2201 snap 0 seq_num 2201) dirty exists 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:3561: left oid 298 (ObjNum 2201 snap 0 seq_num 2201) 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:3562: finishing write tid 1 to smithi093153278-189 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:3562: finishing write tid 2 to smithi093153278-189 2023-08-27T20:55:10.294 INFO:tasks.rados.rados.0.smithi093.stdout:3568: seq_num 2206 ranges {676242=541664,1923042=542431} 2023-08-27T20:55:10.300 INFO:tasks.rados.rados.0.smithi093.stdout:3568: writing smithi093153278-448 from 676242 to 1217906 tid 1 2023-08-27T20:55:10.307 INFO:tasks.rados.rados.0.smithi093.stdout:3568: writing smithi093153278-448 from 1923042 to 2465473 tid 2 2023-08-27T20:55:10.307 INFO:tasks.rados.rados.0.smithi093.stdout:3559: done (8 left) 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3560: done (7 left) 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3561: done (6 left) 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3569: delete oid 324 current snap is 0 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3562: finishing write tid 3 to smithi093153278-189 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3562: finishing write tid 4 to smithi093153278-189 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3562: finishing write tid 5 to smithi093153278-189 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 189 v 380 (ObjNum 2202 snap 0 seq_num 2202) dirty exists 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3562: left oid 189 (ObjNum 2202 snap 0 seq_num 2202) 2023-08-27T20:55:10.308 INFO:tasks.rados.rados.0.smithi093.stdout:3563: expect (ObjNum 2144 snap 0 seq_num 2144) 2023-08-27T20:55:10.405 INFO:tasks.rados.rados.0.smithi093.stdout:3564: expect (ObjNum 1264 snap 0 seq_num 1264) 2023-08-27T20:55:10.568 INFO:tasks.rados.rados.0.smithi093.stdout:3565: finishing write tid 1 to smithi093153278-410 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3565: finishing write tid 2 to smithi093153278-410 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3565: finishing write tid 3 to smithi093153278-410 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3565: finishing write tid 4 to smithi093153278-410 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3565: finishing write tid 5 to smithi093153278-410 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 410 v 412 (ObjNum 2203 snap 0 seq_num 2203) dirty exists 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3565: left oid 410 (ObjNum 2203 snap 0 seq_num 2203) 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3566: finishing write tid 1 to smithi093153278-51 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3566: finishing write tid 2 to smithi093153278-51 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3562: done (6 left) 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3563: done (5 left) 2023-08-27T20:55:10.569 INFO:tasks.rados.rados.0.smithi093.stdout:3564: done (4 left) 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3565: done (3 left) 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3569: done (2 left) 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3570: read oid 71 snap -1 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3570: expect (ObjNum 2186 snap 0 seq_num 2186) 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3571: read oid 120 snap -1 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3571: expect (ObjNum 1393 snap 0 seq_num 1393) 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3572: write oid 365 current snap is 0 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3566: finishing write tid 3 to smithi093153278-51 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3566: finishing write tid 4 to smithi093153278-51 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3566: finishing write tid 5 to smithi093153278-51 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 51 v 411 (ObjNum 2204 snap 0 seq_num 2204) dirty exists 2023-08-27T20:55:10.570 INFO:tasks.rados.rados.0.smithi093.stdout:3566: left oid 51 (ObjNum 2204 snap 0 seq_num 2204) 2023-08-27T20:55:10.571 INFO:tasks.rados.rados.0.smithi093.stdout:3568: finishing write tid 1 to smithi093153278-448 2023-08-27T20:55:10.571 INFO:tasks.rados.rados.0.smithi093.stdout:3568: finishing write tid 2 to smithi093153278-448 2023-08-27T20:55:10.571 INFO:tasks.rados.rados.0.smithi093.stdout:3572: seq_num 2208 ranges {709856=692162,2040006=546100} 2023-08-27T20:55:10.576 INFO:tasks.rados.rados.0.smithi093.stdout:3572: writing smithi093153278-365 from 709856 to 1402018 tid 1 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:3572: writing smithi093153278-365 from 2040006 to 2586106 tid 2 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:3566: done (4 left) 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:3573: write oid 477 current snap is 0 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:3568: finishing write tid 3 to smithi093153278-448 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:3568: finishing write tid 4 to smithi093153278-448 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 448 v 439 (ObjNum 2206 snap 0 seq_num 2206) dirty exists 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:3568: left oid 448 (ObjNum 2206 snap 0 seq_num 2206) 2023-08-27T20:55:10.583 INFO:tasks.rados.rados.0.smithi093.stdout:3573: seq_num 2209 ranges {726663=767411,2098488=406111} 2023-08-27T20:55:10.591 INFO:tasks.rados.rados.0.smithi093.stdout:3573: writing smithi093153278-477 from 726663 to 1494074 tid 1 2023-08-27T20:55:10.596 INFO:tasks.rados.rados.0.smithi093.stdout:3573: writing smithi093153278-477 from 2098488 to 2504599 tid 2 2023-08-27T20:55:10.597 INFO:tasks.rados.rados.0.smithi093.stdout:3568: done (4 left) 2023-08-27T20:55:10.597 INFO:tasks.rados.rados.0.smithi093.stdout:3574: write oid 197 current snap is 0 2023-08-27T20:55:10.597 INFO:tasks.rados.rados.0.smithi093.stdout:3571: expect (ObjNum 1393 snap 0 seq_num 1393) 2023-08-27T20:55:10.787 INFO:tasks.rados.rados.0.smithi093.stdout:3570: expect (ObjNum 2186 snap 0 seq_num 2186) 2023-08-27T20:55:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:10 smithi136 ceph-mon[113531]: mgrmap e26: y(active, since 91s) 2023-08-27T20:55:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4056936527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:10.958 INFO:tasks.rados.rados.0.smithi093.stdout:3572: finishing write tid 1 to smithi093153278-365 2023-08-27T20:55:10.959 INFO:tasks.rados.rados.0.smithi093.stdout:3572: finishing write tid 2 to smithi093153278-365 2023-08-27T20:55:10.959 INFO:tasks.rados.rados.0.smithi093.stdout:3572: finishing write tid 3 to smithi093153278-365 2023-08-27T20:55:10.959 INFO:tasks.rados.rados.0.smithi093.stdout:3572: finishing write tid 4 to smithi093153278-365 2023-08-27T20:55:10.959 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 365 v 525 (ObjNum 2208 snap 0 seq_num 2208) dirty exists 2023-08-27T20:55:10.959 INFO:tasks.rados.rados.0.smithi093.stdout:3572: left oid 365 (ObjNum 2208 snap 0 seq_num 2208) 2023-08-27T20:55:10.959 INFO:tasks.rados.rados.0.smithi093.stdout:3573: finishing write tid 1 to smithi093153278-477 2023-08-27T20:55:10.959 INFO:tasks.rados.rados.0.smithi093.stdout:3574: seq_num 2210 ranges {743470=442660,1640617=549769,2673167=470303} 2023-08-27T20:55:10.965 INFO:tasks.rados.rados.0.smithi093.stdout:3574: writing smithi093153278-197 from 743470 to 1186130 tid 1 2023-08-27T20:55:10.971 INFO:tasks.rados.rados.0.smithi093.stdout:3574: writing smithi093153278-197 from 1640617 to 2190386 tid 2 2023-08-27T20:55:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:3574: writing smithi093153278-197 from 2673167 to 3143470 tid 3 2023-08-27T20:55:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:3570: done (4 left) 2023-08-27T20:55:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:3571: done (3 left) 2023-08-27T20:55:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:3572: done (2 left) 2023-08-27T20:55:10.977 INFO:tasks.rados.rados.0.smithi093.stdout:3575: read oid 286 snap -1 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3575: expect (ObjNum 1933 snap 0 seq_num 1933) 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3576: read oid 372 snap -1 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3576: expect (ObjNum 1197 snap 0 seq_num 1197) 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3577: read oid 204 snap -1 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3577: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3578: delete oid 408 current snap is 0 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3573: finishing write tid 2 to smithi093153278-477 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3573: finishing write tid 3 to smithi093153278-477 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3573: finishing write tid 4 to smithi093153278-477 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 477 v 498 (ObjNum 2209 snap 0 seq_num 2209) dirty exists 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3573: left oid 477 (ObjNum 2209 snap 0 seq_num 2209) 2023-08-27T20:55:10.978 INFO:tasks.rados.rados.0.smithi093.stdout:3574: finishing write tid 1 to smithi093153278-197 2023-08-27T20:55:10.979 INFO:tasks.rados.rados.0.smithi093.stdout:3574: finishing write tid 2 to smithi093153278-197 2023-08-27T20:55:10.979 INFO:tasks.rados.rados.0.smithi093.stdout:3573: done (5 left) 2023-08-27T20:55:10.980 INFO:tasks.rados.rados.0.smithi093.stdout:3578: done (4 left) 2023-08-27T20:55:10.980 INFO:tasks.rados.rados.0.smithi093.stdout:3579: read oid 180 snap -1 2023-08-27T20:55:10.980 INFO:tasks.rados.rados.0.smithi093.stdout:3579: expect (ObjNum 2034 snap 0 seq_num 2034) 2023-08-27T20:55:10.980 INFO:tasks.rados.rados.0.smithi093.stdout:3580: write oid 454 current snap is 0 2023-08-27T20:55:10.980 INFO:tasks.rados.rados.0.smithi093.stdout:3580: seq_num 2212 ranges {777084=593158,2157581=553438,3128013=49071} 2023-08-27T20:55:10.987 INFO:tasks.rados.rados.0.smithi093.stdout:3580: writing smithi093153278-454 from 777084 to 1370242 tid 1 2023-08-27T20:55:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:10 smithi093 ceph-mon[111335]: mgrmap e26: y(active, since 91s) 2023-08-27T20:55:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4056936527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:10 smithi093 ceph-mon[118902]: mgrmap e26: y(active, since 91s) 2023-08-27T20:55:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4056936527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:10.994 INFO:tasks.rados.rados.0.smithi093.stdout:3580: writing smithi093153278-454 from 2157581 to 2711019 tid 2 2023-08-27T20:55:10.997 INFO:tasks.rados.rados.0.smithi093.stdout:3580: writing smithi093153278-454 from 3128013 to 3177084 tid 3 2023-08-27T20:55:10.997 INFO:tasks.rados.rados.0.smithi093.stdout:3581: write oid 483 current snap is 0 2023-08-27T20:55:10.998 INFO:tasks.rados.rados.0.smithi093.stdout:3577: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:55:11.111 INFO:tasks.rados.rados.0.smithi093.stdout:3576: expect (ObjNum 1197 snap 0 seq_num 1197) 2023-08-27T20:55:11.222 INFO:tasks.rados.rados.0.smithi093.stdout:3575: expect (ObjNum 1933 snap 0 seq_num 1933) 2023-08-27T20:55:11.331 INFO:tasks.rados.rados.0.smithi093.stdout:3574: finishing write tid 3 to smithi093153278-197 2023-08-27T20:55:11.331 INFO:tasks.rados.rados.0.smithi093.stdout:3574: finishing write tid 4 to smithi093153278-197 2023-08-27T20:55:11.331 INFO:tasks.rados.rados.0.smithi093.stdout:3574: finishing write tid 5 to smithi093153278-197 2023-08-27T20:55:11.332 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 197 v 384 (ObjNum 2210 snap 0 seq_num 2210) dirty exists 2023-08-27T20:55:11.332 INFO:tasks.rados.rados.0.smithi093.stdout:3574: left oid 197 (ObjNum 2210 snap 0 seq_num 2210) 2023-08-27T20:55:11.332 INFO:tasks.rados.rados.0.smithi093.stdout:3579: expect (ObjNum 2034 snap 0 seq_num 2034) 2023-08-27T20:55:11.502 INFO:tasks.rados.rados.0.smithi093.stdout:3580: finishing write tid 1 to smithi093153278-454 2023-08-27T20:55:11.502 INFO:tasks.rados.rados.0.smithi093.stdout:3580: finishing write tid 2 to smithi093153278-454 2023-08-27T20:55:11.502 INFO:tasks.rados.rados.0.smithi093.stdout:3580: finishing write tid 3 to smithi093153278-454 2023-08-27T20:55:11.503 INFO:tasks.rados.rados.0.smithi093.stdout:3580: finishing write tid 4 to smithi093153278-454 2023-08-27T20:55:11.503 INFO:tasks.rados.rados.0.smithi093.stdout:3580: finishing write tid 5 to smithi093153278-454 2023-08-27T20:55:11.503 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 454 v 476 (ObjNum 2212 snap 0 seq_num 2212) dirty exists 2023-08-27T20:55:11.503 INFO:tasks.rados.rados.0.smithi093.stdout:3580: left oid 454 (ObjNum 2212 snap 0 seq_num 2212) 2023-08-27T20:55:11.503 INFO:tasks.rados.rados.0.smithi093.stdout:3581: seq_num 2213 ranges {793891=784760,2332416=697096} 2023-08-27T20:55:11.511 INFO:tasks.rados.rados.0.smithi093.stdout:3581: writing smithi093153278-483 from 793891 to 1578651 tid 1 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3581: writing smithi093153278-483 from 2332416 to 3029512 tid 2 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3574: done (6 left) 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3575: done (5 left) 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3576: done (4 left) 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3577: done (3 left) 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3579: done (2 left) 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3580: done (1 left) 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3582: write oid 75 current snap is 0 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3581: finishing write tid 1 to smithi093153278-483 2023-08-27T20:55:11.520 INFO:tasks.rados.rados.0.smithi093.stdout:3582: seq_num 2214 ranges {410698=460009,1474545=557107,2782859=427839} 2023-08-27T20:55:11.524 INFO:tasks.rados.rados.0.smithi093.stdout:3582: writing smithi093153278-75 from 410698 to 870707 tid 1 2023-08-27T20:55:11.531 INFO:tasks.rados.rados.0.smithi093.stdout:3582: writing smithi093153278-75 from 1474545 to 2031652 tid 2 2023-08-27T20:55:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:3582: writing smithi093153278-75 from 2782859 to 3210698 tid 3 2023-08-27T20:55:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:3583: write oid 377 current snap is 0 2023-08-27T20:55:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:3581: finishing write tid 2 to smithi093153278-483 2023-08-27T20:55:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:3581: finishing write tid 3 to smithi093153278-483 2023-08-27T20:55:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:3581: finishing write tid 4 to smithi093153278-483 2023-08-27T20:55:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 483 v 397 (ObjNum 2213 snap 0 seq_num 2213) dirty exists 2023-08-27T20:55:11.538 INFO:tasks.rados.rados.0.smithi093.stdout:3581: left oid 483 (ObjNum 2213 snap 0 seq_num 2213) 2023-08-27T20:55:11.539 INFO:tasks.rados.rados.0.smithi093.stdout:3582: finishing write tid 1 to smithi093153278-75 2023-08-27T20:55:11.539 INFO:tasks.rados.rados.0.smithi093.stdout:3582: finishing write tid 2 to smithi093153278-75 2023-08-27T20:55:11.539 INFO:tasks.rados.rados.0.smithi093.stdout:3583: seq_num 2215 ranges {427505=535258,1533027=700765,2810282=417223} 2023-08-27T20:55:11.543 INFO:tasks.rados.rados.0.smithi093.stdout:3583: writing smithi093153278-377 from 427505 to 962763 tid 1 2023-08-27T20:55:11.550 INFO:tasks.rados.rados.0.smithi093.stdout:3583: writing smithi093153278-377 from 1533027 to 2233792 tid 2 2023-08-27T20:55:11.556 INFO:tasks.rados.rados.0.smithi093.stdout:3583: writing smithi093153278-377 from 2810282 to 3227505 tid 3 2023-08-27T20:55:11.556 INFO:tasks.rados.rados.0.smithi093.stdout:3581: done (2 left) 2023-08-27T20:55:11.556 INFO:tasks.rados.rados.0.smithi093.stdout:3584: write oid 157 current snap is 0 2023-08-27T20:55:11.556 INFO:tasks.rados.rados.0.smithi093.stdout:3582: finishing write tid 3 to smithi093153278-75 2023-08-27T20:55:11.556 INFO:tasks.rados.rados.0.smithi093.stdout:3584: seq_num 2216 ranges {444312=610507,1591509=560776,2837705=406607} 2023-08-27T20:55:11.562 INFO:tasks.rados.rados.0.smithi093.stdout:3584: writing smithi093153278-157 from 444312 to 1054819 tid 1 2023-08-27T20:55:11.568 INFO:tasks.rados.rados.0.smithi093.stdout:3584: writing smithi093153278-157 from 1591509 to 2152285 tid 2 2023-08-27T20:55:11.575 INFO:tasks.rados.rados.0.smithi093.stdout:3584: writing smithi093153278-157 from 2837705 to 3244312 tid 3 2023-08-27T20:55:11.575 INFO:tasks.rados.rados.0.smithi093.stdout:3585: write oid 313 current snap is 0 2023-08-27T20:55:11.575 INFO:tasks.rados.rados.0.smithi093.stdout:3582: finishing write tid 4 to smithi093153278-75 2023-08-27T20:55:11.575 INFO:tasks.rados.rados.0.smithi093.stdout:3582: finishing write tid 5 to smithi093153278-75 2023-08-27T20:55:11.576 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 75 v 580 (ObjNum 2214 snap 0 seq_num 2214) dirty exists 2023-08-27T20:55:11.576 INFO:tasks.rados.rados.0.smithi093.stdout:3582: left oid 75 (ObjNum 2214 snap 0 seq_num 2214) 2023-08-27T20:55:11.576 INFO:tasks.rados.rados.0.smithi093.stdout:3583: finishing write tid 1 to smithi093153278-377 2023-08-27T20:55:11.576 INFO:tasks.rados.rados.0.smithi093.stdout:3583: finishing write tid 2 to smithi093153278-377 2023-08-27T20:55:11.576 INFO:tasks.rados.rados.0.smithi093.stdout:3585: seq_num 2217 ranges {461119=685756,1649991=704434,2865128=395991} 2023-08-27T20:55:11.581 INFO:tasks.rados.rados.0.smithi093.stdout:3585: writing smithi093153278-313 from 461119 to 1146875 tid 1 2023-08-27T20:55:11.588 INFO:tasks.rados.rados.0.smithi093.stdout:3585: writing smithi093153278-313 from 1649991 to 2354425 tid 2 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3585: writing smithi093153278-313 from 2865128 to 3261119 tid 3 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3582: done (3 left) 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3586: read oid 468 snap -1 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3586: expect (ObjNum 2172 snap 0 seq_num 2172) 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3587: read oid 391 snap -1 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3587: expect (ObjNum 1624 snap 0 seq_num 1624) 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3588: read oid 447 snap -1 2023-08-27T20:55:11.594 INFO:tasks.rados.rados.0.smithi093.stdout:3588: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3589: write oid 374 current snap is 0 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3583: finishing write tid 3 to smithi093153278-377 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3583: finishing write tid 4 to smithi093153278-377 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3583: finishing write tid 5 to smithi093153278-377 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 377 v 466 (ObjNum 2215 snap 0 seq_num 2215) dirty exists 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3583: left oid 377 (ObjNum 2215 snap 0 seq_num 2215) 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3584: finishing write tid 1 to smithi093153278-157 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3584: finishing write tid 2 to smithi093153278-157 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3584: finishing write tid 3 to smithi093153278-157 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3584: finishing write tid 4 to smithi093153278-157 2023-08-27T20:55:11.595 INFO:tasks.rados.rados.0.smithi093.stdout:3584: finishing write tid 5 to smithi093153278-157 2023-08-27T20:55:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 157 v 480 (ObjNum 2216 snap 0 seq_num 2216) dirty exists 2023-08-27T20:55:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:3584: left oid 157 (ObjNum 2216 snap 0 seq_num 2216) 2023-08-27T20:55:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:3585: finishing write tid 1 to smithi093153278-313 2023-08-27T20:55:11.596 INFO:tasks.rados.rados.0.smithi093.stdout:3589: seq_num 2218 ranges {477926=761005,1992120=564445,3176198=101728} 2023-08-27T20:55:11.600 INFO:tasks.rados.rados.0.smithi093.stdout:3589: writing smithi093153278-374 from 477926 to 1238931 tid 1 2023-08-27T20:55:11.608 INFO:tasks.rados.rados.0.smithi093.stdout:3589: writing smithi093153278-374 from 1992120 to 2556565 tid 2 2023-08-27T20:55:11.612 INFO:tasks.rados.rados.0.smithi093.stdout:3589: writing smithi093153278-374 from 3176198 to 3277926 tid 3 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3583: done (6 left) 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3584: done (5 left) 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3590: read oid 28 snap -1 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3590: expect (ObjNum 1962 snap 0 seq_num 1962) 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3591: read oid 409 snap -1 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3591: expect (ObjNum 2030 snap 0 seq_num 2030) 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3592: write oid 263 current snap is 0 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3585: finishing write tid 2 to smithi093153278-313 2023-08-27T20:55:11.613 INFO:tasks.rados.rados.0.smithi093.stdout:3592: seq_num 2219 ranges {494733=436254,1650602=708103,2803621=443773} 2023-08-27T20:55:11.617 INFO:tasks.rados.rados.0.smithi093.stdout:3592: writing smithi093153278-263 from 494733 to 930987 tid 1 2023-08-27T20:55:11.626 INFO:tasks.rados.rados.0.smithi093.stdout:3592: writing smithi093153278-263 from 1650602 to 2358705 tid 2 2023-08-27T20:55:11.631 INFO:tasks.rados.rados.0.smithi093.stdout:3592: writing smithi093153278-263 from 2803621 to 3247394 tid 3 2023-08-27T20:55:11.631 INFO:tasks.rados.rados.0.smithi093.stdout:3593: write oid 451 current snap is 0 2023-08-27T20:55:11.631 INFO:tasks.rados.rados.0.smithi093.stdout:3593: seq_num 2220 ranges {511540=627856,1825437=568114,2947397=364143} 2023-08-27T20:55:11.637 INFO:tasks.rados.rados.0.smithi093.stdout:3593: writing smithi093153278-451 from 511540 to 1139396 tid 1 2023-08-27T20:55:11.645 INFO:tasks.rados.rados.0.smithi093.stdout:3593: writing smithi093153278-451 from 1825437 to 2393551 tid 2 2023-08-27T20:55:11.651 INFO:tasks.rados.rados.0.smithi093.stdout:3593: writing smithi093153278-451 from 2947397 to 3311540 tid 3 2023-08-27T20:55:11.651 INFO:tasks.rados.rados.0.smithi093.stdout:3594: read oid 41 snap -1 2023-08-27T20:55:11.651 INFO:tasks.rados.rados.0.smithi093.stdout:3594: expect (ObjNum 1803 snap 0 seq_num 1803) 2023-08-27T20:55:11.651 INFO:tasks.rados.rados.0.smithi093.stdout:3595: read oid 130 snap -1 2023-08-27T20:55:11.651 INFO:tasks.rados.rados.0.smithi093.stdout:3595: expect deleted 2023-08-27T20:55:11.651 INFO:tasks.rados.rados.0.smithi093.stdout:3596: read oid 49 snap -1 2023-08-27T20:55:11.652 INFO:tasks.rados.rados.0.smithi093.stdout:3596: expect (ObjNum 1189 snap 0 seq_num 1189) 2023-08-27T20:55:11.652 INFO:tasks.rados.rados.0.smithi093.stdout:3597: write oid 267 current snap is 0 2023-08-27T20:55:11.652 INFO:tasks.rados.rados.0.smithi093.stdout:3597: seq_num 2221 ranges {528347=703105,1883919=711772} 2023-08-27T20:55:11.659 INFO:tasks.rados.rados.0.smithi093.stdout:3597: writing smithi093153278-267 from 528347 to 1231452 tid 1 2023-08-27T20:55:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:3597: writing smithi093153278-267 from 1883919 to 2595691 tid 2 2023-08-27T20:55:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:3598: write oid 340 current snap is 0 2023-08-27T20:55:11.667 INFO:tasks.rados.rados.0.smithi093.stdout:3598: seq_num 2222 ranges {545154=778354,1826048=571783,2885890=459264} 2023-08-27T20:55:11.674 INFO:tasks.rados.rados.0.smithi093.stdout:3598: writing smithi093153278-340 from 545154 to 1323508 tid 1 2023-08-27T20:55:11.680 INFO:tasks.rados.rados.0.smithi093.stdout:3598: writing smithi093153278-340 from 1826048 to 2397831 tid 2 2023-08-27T20:55:11.686 INFO:tasks.rados.rados.0.smithi093.stdout:3598: writing smithi093153278-340 from 2885890 to 3345154 tid 3 2023-08-27T20:55:11.686 INFO:tasks.rados.rados.0.smithi093.stdout:3599: write oid 167 current snap is 0 2023-08-27T20:55:11.686 INFO:tasks.rados.rados.0.smithi093.stdout:3585: finishing write tid 3 to smithi093153278-313 2023-08-27T20:55:11.687 INFO:tasks.rados.rados.0.smithi093.stdout:3599: seq_num 2223 ranges {561961=453603,1484530=715441,2913313=448648} 2023-08-27T20:55:11.692 INFO:tasks.rados.rados.0.smithi093.stdout:3599: writing smithi093153278-167 from 561961 to 1015564 tid 1 2023-08-27T20:55:11.698 INFO:tasks.rados.rados.0.smithi093.stdout:3599: writing smithi093153278-167 from 1484530 to 2199971 tid 2 2023-08-27T20:55:11.705 INFO:tasks.rados.rados.0.smithi093.stdout:3599: writing smithi093153278-167 from 2913313 to 3361961 tid 3 2023-08-27T20:55:11.705 INFO:tasks.rados.rados.0.smithi093.stdout:3600: write oid 335 current snap is 0 2023-08-27T20:55:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:3585: finishing write tid 4 to smithi093153278-313 2023-08-27T20:55:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:3585: finishing write tid 5 to smithi093153278-313 2023-08-27T20:55:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 313 v 284 (ObjNum 2217 snap 0 seq_num 2217) dirty exists 2023-08-27T20:55:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:3585: left oid 313 (ObjNum 2217 snap 0 seq_num 2217) 2023-08-27T20:55:11.706 INFO:tasks.rados.rados.0.smithi093.stdout:3600: seq_num 2224 ranges {578768=528852,1543012=575452,2540736=416486} 2023-08-27T20:55:11.712 INFO:tasks.rados.rados.0.smithi093.stdout:3600: writing smithi093153278-335 from 578768 to 1107620 tid 1 2023-08-27T20:55:11.717 INFO:tasks.rados.rados.0.smithi093.stdout:3600: writing smithi093153278-335 from 1543012 to 2118464 tid 2 2023-08-27T20:55:11.723 INFO:tasks.rados.rados.0.smithi093.stdout:3600: writing smithi093153278-335 from 2540736 to 2957222 tid 3 2023-08-27T20:55:11.723 INFO:tasks.rados.rados.0.smithi093.stdout:3585: done (15 left) 2023-08-27T20:55:11.723 INFO:tasks.rados.rados.0.smithi093.stdout:3601: read oid 406 snap -1 2023-08-27T20:55:11.723 INFO:tasks.rados.rados.0.smithi093.stdout:3601: expect (ObjNum 1661 snap 0 seq_num 1661) 2023-08-27T20:55:11.723 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:55:11.723 INFO:tasks.rados.rados.0.smithi093.stdout:3588: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:11.848 INFO:tasks.rados.rados.0.smithi093.stdout:3587: expect (ObjNum 1624 snap 0 seq_num 1624) 2023-08-27T20:55:11.915 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:11 smithi093 ceph-mon[111335]: pgmap v49: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 10 MiB/s wr, 112 op/s 2023-08-27T20:55:11.916 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:55:11.916 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:11 smithi093 ceph-mon[118902]: pgmap v49: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 10 MiB/s wr, 112 op/s 2023-08-27T20:55:11.916 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:55:11.916 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:11.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:11.916 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:11.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:11 smithi136 ceph-mon[113531]: pgmap v49: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 10 MiB/s wr, 112 op/s 2023-08-27T20:55:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:55:11.990 INFO:tasks.rados.rados.0.smithi093.stdout:3586: expect (ObjNum 2172 snap 0 seq_num 2172) 2023-08-27T20:55:12.068 INFO:tasks.rados.rados.0.smithi093.stdout:3589: finishing write tid 1 to smithi093153278-374 2023-08-27T20:55:12.068 INFO:tasks.rados.rados.0.smithi093.stdout:3589: finishing write tid 2 to smithi093153278-374 2023-08-27T20:55:12.068 INFO:tasks.rados.rados.0.smithi093.stdout:3586: done (15 left) 2023-08-27T20:55:12.068 INFO:tasks.rados.rados.0.smithi093.stdout:3587: done (14 left) 2023-08-27T20:55:12.068 INFO:tasks.rados.rados.0.smithi093.stdout:3588: done (13 left) 2023-08-27T20:55:12.068 INFO:tasks.rados.rados.0.smithi093.stdout:3602: read oid 164 snap -1 2023-08-27T20:55:12.069 INFO:tasks.rados.rados.0.smithi093.stdout:3602: expect (ObjNum 1818 snap 0 seq_num 1818) 2023-08-27T20:55:12.069 INFO:tasks.rados.rados.0.smithi093.stdout:3603: write oid 455 current snap is 0 2023-08-27T20:55:12.069 INFO:tasks.rados.rados.0.smithi093.stdout:3589: finishing write tid 3 to smithi093153278-374 2023-08-27T20:55:12.069 INFO:tasks.rados.rados.0.smithi093.stdout:3589: finishing write tid 4 to smithi093153278-374 2023-08-27T20:55:12.069 INFO:tasks.rados.rados.0.smithi093.stdout:3589: finishing write tid 5 to smithi093153278-374 2023-08-27T20:55:12.069 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 374 v 388 (ObjNum 2218 snap 0 seq_num 2218) dirty exists 2023-08-27T20:55:12.069 INFO:tasks.rados.rados.0.smithi093.stdout:3589: left oid 374 (ObjNum 2218 snap 0 seq_num 2218) 2023-08-27T20:55:12.070 INFO:tasks.rados.rados.0.smithi093.stdout:3591: expect (ObjNum 2030 snap 0 seq_num 2030) 2023-08-27T20:55:12.208 INFO:tasks.rados.rados.0.smithi093.stdout:3592: finishing write tid 1 to smithi093153278-263 2023-08-27T20:55:12.208 INFO:tasks.rados.rados.0.smithi093.stdout:3590: expect (ObjNum 1962 snap 0 seq_num 1962) 2023-08-27T20:55:12.376 INFO:tasks.rados.rados.0.smithi093.stdout:3592: finishing write tid 2 to smithi093153278-263 2023-08-27T20:55:12.376 INFO:tasks.rados.rados.0.smithi093.stdout:3592: finishing write tid 3 to smithi093153278-263 2023-08-27T20:55:12.376 INFO:tasks.rados.rados.0.smithi093.stdout:3592: finishing write tid 4 to smithi093153278-263 2023-08-27T20:55:12.376 INFO:tasks.rados.rados.0.smithi093.stdout:3592: finishing write tid 5 to smithi093153278-263 2023-08-27T20:55:12.377 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 263 v 503 (ObjNum 2219 snap 0 seq_num 2219) dirty exists 2023-08-27T20:55:12.377 INFO:tasks.rados.rados.0.smithi093.stdout:3592: left oid 263 (ObjNum 2219 snap 0 seq_num 2219) 2023-08-27T20:55:12.377 INFO:tasks.rados.rados.0.smithi093.stdout:3593: finishing write tid 1 to smithi093153278-451 2023-08-27T20:55:12.377 INFO:tasks.rados.rados.0.smithi093.stdout:3593: finishing write tid 2 to smithi093153278-451 2023-08-27T20:55:12.377 INFO:tasks.rados.rados.0.smithi093.stdout:3603: seq_num 2225 ranges {595575=604101,1601494=719110,2968159=427416} 2023-08-27T20:55:12.383 INFO:tasks.rados.rados.0.smithi093.stdout:3603: writing smithi093153278-455 from 595575 to 1199676 tid 1 2023-08-27T20:55:12.389 INFO:tasks.rados.rados.0.smithi093.stdout:3603: writing smithi093153278-455 from 1601494 to 2320604 tid 2 2023-08-27T20:55:12.396 INFO:tasks.rados.rados.0.smithi093.stdout:3603: writing smithi093153278-455 from 2968159 to 3395575 tid 3 2023-08-27T20:55:12.396 INFO:tasks.rados.rados.0.smithi093.stdout:3589: done (14 left) 2023-08-27T20:55:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:3590: done (13 left) 2023-08-27T20:55:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:3591: done (12 left) 2023-08-27T20:55:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:3592: done (11 left) 2023-08-27T20:55:12.397 INFO:tasks.rados.rados.0.smithi093.stdout:3604: write oid 283 current snap is 0 2023-08-27T20:55:12.398 INFO:tasks.rados.rados.0.smithi093.stdout:3604: seq_num 2226 ranges {612382=679350,1943623=462768,3162876=249506} 2023-08-27T20:55:12.402 INFO:tasks.rados.rados.0.smithi093.stdout:3604: writing smithi093153278-283 from 612382 to 1291732 tid 1 2023-08-27T20:55:12.408 INFO:tasks.rados.rados.0.smithi093.stdout:3604: writing smithi093153278-283 from 1943623 to 2406391 tid 2 2023-08-27T20:55:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:3604: writing smithi093153278-283 from 3162876 to 3412382 tid 3 2023-08-27T20:55:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:3605: write oid 88 current snap is 0 2023-08-27T20:55:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:3593: finishing write tid 3 to smithi093153278-451 2023-08-27T20:55:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:3593: finishing write tid 4 to smithi093153278-451 2023-08-27T20:55:12.414 INFO:tasks.rados.rados.0.smithi093.stdout:3605: seq_num 2227 ranges {629189=754599,2002105=722779,3306652=122537} 2023-08-27T20:55:12.421 INFO:tasks.rados.rados.0.smithi093.stdout:3605: writing smithi093153278-88 from 629189 to 1383788 tid 1 2023-08-27T20:55:12.429 INFO:tasks.rados.rados.0.smithi093.stdout:3605: writing smithi093153278-88 from 2002105 to 2724884 tid 2 2023-08-27T20:55:12.433 INFO:tasks.rados.rados.0.smithi093.stdout:3605: writing smithi093153278-88 from 3306652 to 3429189 tid 3 2023-08-27T20:55:12.433 INFO:tasks.rados.rados.0.smithi093.stdout:3595: done (12 left) 2023-08-27T20:55:12.433 INFO:tasks.rados.rados.0.smithi093.stdout:3606: read oid 204 snap -1 2023-08-27T20:55:12.434 INFO:tasks.rados.rados.0.smithi093.stdout:3606: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:55:12.434 INFO:tasks.rados.rados.0.smithi093.stdout:3607: write oid 31 current snap is 0 2023-08-27T20:55:12.434 INFO:tasks.rados.rados.0.smithi093.stdout:3593: finishing write tid 5 to smithi093153278-451 2023-08-27T20:55:12.434 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 451 v 392 (ObjNum 2220 snap 0 seq_num 2220) dirty exists 2023-08-27T20:55:12.434 INFO:tasks.rados.rados.0.smithi093.stdout:3593: left oid 451 (ObjNum 2220 snap 0 seq_num 2220) 2023-08-27T20:55:12.434 INFO:tasks.rados.rados.0.smithi093.stdout:3594: expect (ObjNum 1803 snap 0 seq_num 1803) 2023-08-27T20:55:12.557 INFO:tasks.rados.rados.0.smithi093.stdout:3597: finishing write tid 1 to smithi093153278-267 2023-08-27T20:55:12.557 INFO:tasks.rados.rados.0.smithi093.stdout:3596: expect (ObjNum 1189 snap 0 seq_num 1189) 2023-08-27T20:55:12.830 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1436419324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:12.875 INFO:tasks.rados.rados.0.smithi093.stdout:3597: finishing write tid 2 to smithi093153278-267 2023-08-27T20:55:12.876 INFO:tasks.rados.rados.0.smithi093.stdout:3597: finishing write tid 3 to smithi093153278-267 2023-08-27T20:55:12.876 INFO:tasks.rados.rados.0.smithi093.stdout:3597: finishing write tid 4 to smithi093153278-267 2023-08-27T20:55:12.876 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 267 v 442 (ObjNum 2221 snap 0 seq_num 2221) dirty exists 2023-08-27T20:55:12.876 INFO:tasks.rados.rados.0.smithi093.stdout:3597: left oid 267 (ObjNum 2221 snap 0 seq_num 2221) 2023-08-27T20:55:12.876 INFO:tasks.rados.rados.0.smithi093.stdout:3598: finishing write tid 1 to smithi093153278-340 2023-08-27T20:55:12.876 INFO:tasks.rados.rados.0.smithi093.stdout:3598: finishing write tid 2 to smithi093153278-340 2023-08-27T20:55:12.876 INFO:tasks.rados.rados.0.smithi093.stdout:3607: seq_num 2228 ranges {645996=546201,1776940=466437,2650428=577927} 2023-08-27T20:55:12.882 INFO:tasks.rados.rados.0.smithi093.stdout:3607: writing smithi093153278-31 from 645996 to 1192197 tid 1 2023-08-27T20:55:12.888 INFO:tasks.rados.rados.0.smithi093.stdout:3607: writing smithi093153278-31 from 1776940 to 2243377 tid 2 2023-08-27T20:55:12.894 INFO:tasks.rados.rados.0.smithi093.stdout:3607: writing smithi093153278-31 from 2650428 to 3228355 tid 3 2023-08-27T20:55:12.894 INFO:tasks.rados.rados.0.smithi093.stdout:3593: done (13 left) 2023-08-27T20:55:12.894 INFO:tasks.rados.rados.0.smithi093.stdout:3594: done (12 left) 2023-08-27T20:55:12.894 INFO:tasks.rados.rados.0.smithi093.stdout:3596: done (11 left) 2023-08-27T20:55:12.894 INFO:tasks.rados.rados.0.smithi093.stdout:3597: done (10 left) 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3608: read oid 118 snap -1 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3608: expect (ObjNum 2160 snap 0 seq_num 2160) 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3609: read oid 336 snap -1 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3609: expect (ObjNum 1946 snap 0 seq_num 1946) 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3610: write oid 239 current snap is 0 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3598: finishing write tid 3 to smithi093153278-340 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3598: finishing write tid 4 to smithi093153278-340 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3598: finishing write tid 5 to smithi093153278-340 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 340 v 415 (ObjNum 2222 snap 0 seq_num 2222) dirty exists 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3598: left oid 340 (ObjNum 2222 snap 0 seq_num 2222) 2023-08-27T20:55:12.895 INFO:tasks.rados.rados.0.smithi093.stdout:3599: finishing write tid 1 to smithi093153278-167 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3599: finishing write tid 2 to smithi093153278-167 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3599: finishing write tid 3 to smithi093153278-167 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3599: finishing write tid 4 to smithi093153278-167 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3599: finishing write tid 5 to smithi093153278-167 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 167 v 507 (ObjNum 2223 snap 0 seq_num 2223) dirty exists 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3599: left oid 167 (ObjNum 2223 snap 0 seq_num 2223) 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3600: finishing write tid 1 to smithi093153278-335 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3600: finishing write tid 2 to smithi093153278-335 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3600: finishing write tid 3 to smithi093153278-335 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3600: finishing write tid 4 to smithi093153278-335 2023-08-27T20:55:12.896 INFO:tasks.rados.rados.0.smithi093.stdout:3600: finishing write tid 5 to smithi093153278-335 2023-08-27T20:55:12.897 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 335 v 358 (ObjNum 2224 snap 0 seq_num 2224) dirty exists 2023-08-27T20:55:12.897 INFO:tasks.rados.rados.0.smithi093.stdout:3600: left oid 335 (ObjNum 2224 snap 0 seq_num 2224) 2023-08-27T20:55:12.897 INFO:tasks.rados.rados.0.smithi093.stdout:3601: expect (ObjNum 1661 snap 0 seq_num 1661) 2023-08-27T20:55:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1436419324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1436419324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:13.020 INFO:tasks.rados.rados.0.smithi093.stdout:3602: expect (ObjNum 1818 snap 0 seq_num 1818) 2023-08-27T20:55:13.215 INFO:tasks.rados.rados.0.smithi093.stdout:3603: finishing write tid 1 to smithi093153278-455 2023-08-27T20:55:13.215 INFO:tasks.rados.rados.0.smithi093.stdout:3603: finishing write tid 2 to smithi093153278-455 2023-08-27T20:55:13.216 INFO:tasks.rados.rados.0.smithi093.stdout:3603: finishing write tid 3 to smithi093153278-455 2023-08-27T20:55:13.216 INFO:tasks.rados.rados.0.smithi093.stdout:3603: finishing write tid 4 to smithi093153278-455 2023-08-27T20:55:13.216 INFO:tasks.rados.rados.0.smithi093.stdout:3603: finishing write tid 5 to smithi093153278-455 2023-08-27T20:55:13.216 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 455 v 511 (ObjNum 2225 snap 0 seq_num 2225) dirty exists 2023-08-27T20:55:13.216 INFO:tasks.rados.rados.0.smithi093.stdout:3603: left oid 455 (ObjNum 2225 snap 0 seq_num 2225) 2023-08-27T20:55:13.216 INFO:tasks.rados.rados.0.smithi093.stdout:3604: finishing write tid 1 to smithi093153278-283 2023-08-27T20:55:13.216 INFO:tasks.rados.rados.0.smithi093.stdout:3610: seq_num 2229 ranges {662803=621450,1835422=726448,3077851=384952} 2023-08-27T20:55:13.222 INFO:tasks.rados.rados.0.smithi093.stdout:3610: writing smithi093153278-239 from 662803 to 1284253 tid 1 2023-08-27T20:55:13.229 INFO:tasks.rados.rados.0.smithi093.stdout:3610: writing smithi093153278-239 from 1835422 to 2561870 tid 2 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3610: writing smithi093153278-239 from 3077851 to 3462803 tid 3 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3598: done (12 left) 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3599: done (11 left) 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3600: done (10 left) 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3601: done (9 left) 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3602: done (8 left) 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3603: done (7 left) 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3611: read oid 447 snap -1 2023-08-27T20:55:13.235 INFO:tasks.rados.rados.0.smithi093.stdout:3611: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3612: read oid 158 snap -1 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3612: expect deleted 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3613: read oid 230 snap -1 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3613: expect (ObjNum 2020 snap 0 seq_num 2020) 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3614: write oid 36 current snap is 0 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3604: finishing write tid 2 to smithi093153278-283 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3604: finishing write tid 3 to smithi093153278-283 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3604: finishing write tid 4 to smithi093153278-283 2023-08-27T20:55:13.236 INFO:tasks.rados.rados.0.smithi093.stdout:3614: seq_num 2230 ranges {679610=696699,1777551=470106,2988921=490689} 2023-08-27T20:55:13.242 INFO:tasks.rados.rados.0.smithi093.stdout:3614: writing smithi093153278-36 from 679610 to 1376309 tid 1 2023-08-27T20:55:13.247 INFO:tasks.rados.rados.0.smithi093.stdout:3614: writing smithi093153278-36 from 1777551 to 2247657 tid 2 2023-08-27T20:55:13.254 INFO:tasks.rados.rados.0.smithi093.stdout:3614: writing smithi093153278-36 from 2988921 to 3479610 tid 3 2023-08-27T20:55:13.254 INFO:tasks.rados.rados.0.smithi093.stdout:3615: write oid 6 current snap is 0 2023-08-27T20:55:13.254 INFO:tasks.rados.rados.0.smithi093.stdout:3604: finishing write tid 5 to smithi093153278-283 2023-08-27T20:55:13.254 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 283 v 512 (ObjNum 2226 snap 0 seq_num 2226) dirty exists 2023-08-27T20:55:13.255 INFO:tasks.rados.rados.0.smithi093.stdout:3604: left oid 283 (ObjNum 2226 snap 0 seq_num 2226) 2023-08-27T20:55:13.255 INFO:tasks.rados.rados.0.smithi093.stdout:3605: finishing write tid 1 to smithi093153278-88 2023-08-27T20:55:13.255 INFO:tasks.rados.rados.0.smithi093.stdout:3605: finishing write tid 2 to smithi093153278-88 2023-08-27T20:55:13.255 INFO:tasks.rados.rados.0.smithi093.stdout:3615: seq_num 2231 ranges {696417=771948,2236033=730117,3416344=80073} 2023-08-27T20:55:13.262 INFO:tasks.rados.rados.0.smithi093.stdout:3615: writing smithi093153278-6 from 696417 to 1468365 tid 1 2023-08-27T20:55:13.270 INFO:tasks.rados.rados.0.smithi093.stdout:3615: writing smithi093153278-6 from 2236033 to 2966150 tid 2 2023-08-27T20:55:13.273 INFO:tasks.rados.rados.0.smithi093.stdout:3615: writing smithi093153278-6 from 3416344 to 3496417 tid 3 2023-08-27T20:55:13.273 INFO:tasks.rados.rados.0.smithi093.stdout:3604: done (11 left) 2023-08-27T20:55:13.273 INFO:tasks.rados.rados.0.smithi093.stdout:3616: read oid 420 snap -1 2023-08-27T20:55:13.273 INFO:tasks.rados.rados.0.smithi093.stdout:3616: expect (ObjNum 881 snap 0 seq_num 881) 2023-08-27T20:55:13.273 INFO:tasks.rados.rados.0.smithi093.stdout:3617: read oid 484 snap -1 2023-08-27T20:55:13.273 INFO:tasks.rados.rados.0.smithi093.stdout:3617: expect (ObjNum 1695 snap 0 seq_num 1695) 2023-08-27T20:55:13.273 INFO:tasks.rados.rados.0.smithi093.stdout:3618: write oid 23 current snap is 0 2023-08-27T20:55:13.274 INFO:tasks.rados.rados.0.smithi093.stdout:3605: finishing write tid 3 to smithi093153278-88 2023-08-27T20:55:13.274 INFO:tasks.rados.rados.0.smithi093.stdout:3618: seq_num 2232 ranges {713224=447197,1894515=473775,3043767=469457} 2023-08-27T20:55:13.279 INFO:tasks.rados.rados.0.smithi093.stdout:3618: writing smithi093153278-23 from 713224 to 1160421 tid 1 2023-08-27T20:55:13.286 INFO:tasks.rados.rados.0.smithi093.stdout:3618: writing smithi093153278-23 from 1894515 to 2368290 tid 2 2023-08-27T20:55:13.292 INFO:tasks.rados.rados.0.smithi093.stdout:3618: writing smithi093153278-23 from 3043767 to 3513224 tid 3 2023-08-27T20:55:13.292 INFO:tasks.rados.rados.0.smithi093.stdout:3619: write oid 418 current snap is 0 2023-08-27T20:55:13.292 INFO:tasks.rados.rados.0.smithi093.stdout:3606: expect (ObjNum 1094 snap 0 seq_num 1094) 2023-08-27T20:55:13.399 INFO:tasks.rados.rados.0.smithi093.stdout:3605: finishing write tid 4 to smithi093153278-88 2023-08-27T20:55:13.399 INFO:tasks.rados.rados.0.smithi093.stdout:3605: finishing write tid 5 to smithi093153278-88 2023-08-27T20:55:13.399 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 88 v 529 (ObjNum 2227 snap 0 seq_num 2227) dirty exists 2023-08-27T20:55:13.399 INFO:tasks.rados.rados.0.smithi093.stdout:3605: left oid 88 (ObjNum 2227 snap 0 seq_num 2227) 2023-08-27T20:55:13.399 INFO:tasks.rados.rados.0.smithi093.stdout:3607: finishing write tid 1 to smithi093153278-31 2023-08-27T20:55:13.399 INFO:tasks.rados.rados.0.smithi093.stdout:3607: finishing write tid 2 to smithi093153278-31 2023-08-27T20:55:13.400 INFO:tasks.rados.rados.0.smithi093.stdout:3619: seq_num 2233 ranges {730031=522446,1952997=733786,3471190=58841} 2023-08-27T20:55:13.405 INFO:tasks.rados.rados.0.smithi093.stdout:3619: writing smithi093153278-418 from 730031 to 1252477 tid 1 2023-08-27T20:55:13.414 INFO:tasks.rados.rados.0.smithi093.stdout:3619: writing smithi093153278-418 from 1952997 to 2686783 tid 2 2023-08-27T20:55:13.418 INFO:tasks.rados.rados.0.smithi093.stdout:3619: writing smithi093153278-418 from 3471190 to 3530031 tid 3 2023-08-27T20:55:13.418 INFO:tasks.rados.rados.0.smithi093.stdout:3605: done (14 left) 2023-08-27T20:55:13.418 INFO:tasks.rados.rados.0.smithi093.stdout:3606: done (13 left) 2023-08-27T20:55:13.418 INFO:tasks.rados.rados.0.smithi093.stdout:3620: read oid 175 snap -1 2023-08-27T20:55:13.418 INFO:tasks.rados.rados.0.smithi093.stdout:3620: expect deleted 2023-08-27T20:55:13.418 INFO:tasks.rados.rados.0.smithi093.stdout:3621: write oid 341 current snap is 0 2023-08-27T20:55:13.418 INFO:tasks.rados.rados.0.smithi093.stdout:3608: expect (ObjNum 2160 snap 0 seq_num 2160) 2023-08-27T20:55:13.533 INFO:tasks.rados.rados.0.smithi093.stdout:3607: finishing write tid 3 to smithi093153278-31 2023-08-27T20:55:13.533 INFO:tasks.rados.rados.0.smithi093.stdout:3607: finishing write tid 4 to smithi093153278-31 2023-08-27T20:55:13.533 INFO:tasks.rados.rados.0.smithi093.stdout:3621: seq_num 2234 ranges {746838=597695,2011479=477444,3098613=448225} 2023-08-27T20:55:13.540 INFO:tasks.rados.rados.0.smithi093.stdout:3621: writing smithi093153278-341 from 746838 to 1344533 tid 1 2023-08-27T20:55:13.546 INFO:tasks.rados.rados.0.smithi093.stdout:3621: writing smithi093153278-341 from 2011479 to 2488923 tid 2 2023-08-27T20:55:13.552 INFO:tasks.rados.rados.0.smithi093.stdout:3621: writing smithi093153278-341 from 3098613 to 3546838 tid 3 2023-08-27T20:55:13.552 INFO:tasks.rados.rados.0.smithi093.stdout:3608: done (14 left) 2023-08-27T20:55:13.552 INFO:tasks.rados.rados.0.smithi093.stdout:3622: write oid 101 current snap is 0 2023-08-27T20:55:13.552 INFO:tasks.rados.rados.0.smithi093.stdout:3607: finishing write tid 5 to smithi093153278-31 2023-08-27T20:55:13.553 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 31 v 362 (ObjNum 2228 snap 0 seq_num 2228) dirty exists 2023-08-27T20:55:13.553 INFO:tasks.rados.rados.0.smithi093.stdout:3607: left oid 31 (ObjNum 2228 snap 0 seq_num 2228) 2023-08-27T20:55:13.553 INFO:tasks.rados.rados.0.smithi093.stdout:3609: expect (ObjNum 1946 snap 0 seq_num 1946) 2023-08-27T20:55:13.643 INFO:tasks.rados.rados.0.smithi093.stdout:3610: finishing write tid 1 to smithi093153278-239 2023-08-27T20:55:13.643 INFO:tasks.rados.rados.0.smithi093.stdout:3610: finishing write tid 2 to smithi093153278-239 2023-08-27T20:55:13.643 INFO:tasks.rados.rados.0.smithi093.stdout:3610: finishing write tid 3 to smithi093153278-239 2023-08-27T20:55:13.643 INFO:tasks.rados.rados.0.smithi093.stdout:3610: finishing write tid 4 to smithi093153278-239 2023-08-27T20:55:13.643 INFO:tasks.rados.rados.0.smithi093.stdout:3622: seq_num 2235 ranges {763645=672944,1953608=737455,3409683=153962} 2023-08-27T20:55:13.651 INFO:tasks.rados.rados.0.smithi093.stdout:3622: writing smithi093153278-101 from 763645 to 1436589 tid 1 2023-08-27T20:55:13.657 INFO:tasks.rados.rados.0.smithi093.stdout:3622: writing smithi093153278-101 from 1953608 to 2691063 tid 2 2023-08-27T20:55:13.662 INFO:tasks.rados.rados.0.smithi093.stdout:3622: writing smithi093153278-101 from 3409683 to 3563645 tid 3 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3607: done (14 left) 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3609: done (13 left) 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3612: done (12 left) 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3623: read oid 220 snap -1 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3623: expect (ObjNum 1854 snap 0 seq_num 1854) 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3624: write oid 338 current snap is 0 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3610: finishing write tid 5 to smithi093153278-239 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 239 v 417 (ObjNum 2229 snap 0 seq_num 2229) dirty exists 2023-08-27T20:55:13.663 INFO:tasks.rados.rados.0.smithi093.stdout:3610: left oid 239 (ObjNum 2229 snap 0 seq_num 2229) 2023-08-27T20:55:13.664 INFO:tasks.rados.rados.0.smithi093.stdout:3611: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:13.778 INFO:tasks.rados.rados.0.smithi093.stdout:3613: expect (ObjNum 2020 snap 0 seq_num 2020) 2023-08-27T20:55:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: pgmap v50: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 5.8 MiB/s wr, 69 op/s 2023-08-27T20:55:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:55:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:55:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:13.518Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:13.519Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: pgmap v50: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 5.8 MiB/s wr, 69 op/s 2023-08-27T20:55:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:55:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:55:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: pgmap v50: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 5.8 MiB/s wr, 69 op/s 2023-08-27T20:55:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:55:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:55:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:3614: finishing write tid 1 to smithi093153278-36 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:3614: finishing write tid 2 to smithi093153278-36 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:3614: finishing write tid 3 to smithi093153278-36 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:3614: finishing write tid 4 to smithi093153278-36 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:3614: finishing write tid 5 to smithi093153278-36 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 442 (ObjNum 2230 snap 0 seq_num 2230) dirty exists 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:3614: left oid 36 (ObjNum 2230 snap 0 seq_num 2230) 2023-08-27T20:55:14.072 INFO:tasks.rados.rados.0.smithi093.stdout:3615: finishing write tid 1 to smithi093153278-6 2023-08-27T20:55:14.073 INFO:tasks.rados.rados.0.smithi093.stdout:3615: finishing write tid 2 to smithi093153278-6 2023-08-27T20:55:14.073 INFO:tasks.rados.rados.0.smithi093.stdout:3624: seq_num 2236 ranges {780452=464546,1728443=481113,2753459=500809} 2023-08-27T20:55:14.078 INFO:tasks.rados.rados.0.smithi093.stdout:3624: writing smithi093153278-338 from 780452 to 1244998 tid 1 2023-08-27T20:55:14.084 INFO:tasks.rados.rados.0.smithi093.stdout:3624: writing smithi093153278-338 from 1728443 to 2209556 tid 2 2023-08-27T20:55:14.090 INFO:tasks.rados.rados.0.smithi093.stdout:3624: writing smithi093153278-338 from 2753459 to 3254268 tid 3 2023-08-27T20:55:14.090 INFO:tasks.rados.rados.0.smithi093.stdout:3610: done (13 left) 2023-08-27T20:55:14.090 INFO:tasks.rados.rados.0.smithi093.stdout:3611: done (12 left) 2023-08-27T20:55:14.090 INFO:tasks.rados.rados.0.smithi093.stdout:3613: done (11 left) 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:3614: done (10 left) 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:3625: write oid 100 current snap is 0 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:3615: finishing write tid 3 to smithi093153278-6 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:3615: finishing write tid 4 to smithi093153278-6 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:3615: finishing write tid 5 to smithi093153278-6 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 6 v 412 (ObjNum 2231 snap 0 seq_num 2231) dirty exists 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:3615: left oid 6 (ObjNum 2231 snap 0 seq_num 2231) 2023-08-27T20:55:14.091 INFO:tasks.rados.rados.0.smithi093.stdout:3617: expect (ObjNum 1695 snap 0 seq_num 1695) 2023-08-27T20:55:14.165 INFO:tasks.rados.rados.0.smithi093.stdout:3616: expect (ObjNum 881 snap 0 seq_num 881) 2023-08-27T20:55:14.278 INFO:tasks.rados.rados.0.smithi093.stdout:3618: finishing write tid 1 to smithi093153278-23 2023-08-27T20:55:14.278 INFO:tasks.rados.rados.0.smithi093.stdout:3618: finishing write tid 2 to smithi093153278-23 2023-08-27T20:55:14.279 INFO:tasks.rados.rados.0.smithi093.stdout:3618: finishing write tid 3 to smithi093153278-23 2023-08-27T20:55:14.279 INFO:tasks.rados.rados.0.smithi093.stdout:3618: finishing write tid 4 to smithi093153278-23 2023-08-27T20:55:14.279 INFO:tasks.rados.rados.0.smithi093.stdout:3618: finishing write tid 5 to smithi093153278-23 2023-08-27T20:55:14.279 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 23 v 515 (ObjNum 2232 snap 0 seq_num 2232) dirty exists 2023-08-27T20:55:14.279 INFO:tasks.rados.rados.0.smithi093.stdout:3618: left oid 23 (ObjNum 2232 snap 0 seq_num 2232) 2023-08-27T20:55:14.279 INFO:tasks.rados.rados.0.smithi093.stdout:3619: finishing write tid 1 to smithi093153278-418 2023-08-27T20:55:14.279 INFO:tasks.rados.rados.0.smithi093.stdout:3619: finishing write tid 2 to smithi093153278-418 2023-08-27T20:55:14.280 INFO:tasks.rados.rados.0.smithi093.stdout:3625: seq_num 2237 ranges {797259=539795,1786925=741124,3180882=416377} 2023-08-27T20:55:14.285 INFO:tasks.rados.rados.0.smithi093.stdout:3625: writing smithi093153278-100 from 797259 to 1337054 tid 1 2023-08-27T20:55:14.292 INFO:tasks.rados.rados.0.smithi093.stdout:3625: writing smithi093153278-100 from 1786925 to 2528049 tid 2 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3625: writing smithi093153278-100 from 3180882 to 3597259 tid 3 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3615: done (10 left) 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3616: done (9 left) 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3617: done (8 left) 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3618: done (7 left) 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3626: read oid 490 snap -1 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3626: expect (ObjNum 1979 snap 0 seq_num 1979) 2023-08-27T20:55:14.299 INFO:tasks.rados.rados.0.smithi093.stdout:3627: read oid 476 snap -1 2023-08-27T20:55:14.300 INFO:tasks.rados.rados.0.smithi093.stdout:3627: expect (ObjNum 1644 snap 0 seq_num 1644) 2023-08-27T20:55:14.300 INFO:tasks.rados.rados.0.smithi093.stdout:3628: read oid 3 snap -1 2023-08-27T20:55:14.300 INFO:tasks.rados.rados.0.smithi093.stdout:3628: expect (ObjNum 696 snap 0 seq_num 696) 2023-08-27T20:55:14.300 INFO:tasks.rados.rados.0.smithi093.stdout:3629: write oid 328 current snap is 0 2023-08-27T20:55:14.300 INFO:tasks.rados.rados.0.smithi093.stdout:3629: seq_num 2238 ranges {414066=615044,1445407=484782,2408305=639706} 2023-08-27T20:55:14.304 INFO:tasks.rados.rados.0.smithi093.stdout:3629: writing smithi093153278-328 from 414066 to 1029110 tid 1 2023-08-27T20:55:14.309 INFO:tasks.rados.rados.0.smithi093.stdout:3629: writing smithi093153278-328 from 1445407 to 1930189 tid 2 2023-08-27T20:55:14.316 INFO:tasks.rados.rados.0.smithi093.stdout:3629: writing smithi093153278-328 from 2408305 to 3048011 tid 3 2023-08-27T20:55:14.316 INFO:tasks.rados.rados.0.smithi093.stdout:3630: write oid 398 current snap is 0 2023-08-27T20:55:14.316 INFO:tasks.rados.rados.0.smithi093.stdout:3619: finishing write tid 3 to smithi093153278-418 2023-08-27T20:55:14.316 INFO:tasks.rados.rados.0.smithi093.stdout:3619: finishing write tid 4 to smithi093153278-418 2023-08-27T20:55:14.316 INFO:tasks.rados.rados.0.smithi093.stdout:3619: finishing write tid 5 to smithi093153278-418 2023-08-27T20:55:14.316 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 418 v 516 (ObjNum 2233 snap 0 seq_num 2233) dirty exists 2023-08-27T20:55:14.316 INFO:tasks.rados.rados.0.smithi093.stdout:3619: left oid 418 (ObjNum 2233 snap 0 seq_num 2233) 2023-08-27T20:55:14.317 INFO:tasks.rados.rados.0.smithi093.stdout:3621: finishing write tid 1 to smithi093153278-341 2023-08-27T20:55:14.317 INFO:tasks.rados.rados.0.smithi093.stdout:3630: seq_num 2239 ranges {430873=690293,1787536=628440,3003022=450978} 2023-08-27T20:55:14.322 INFO:tasks.rados.rados.0.smithi093.stdout:3630: writing smithi093153278-398 from 430873 to 1121166 tid 1 2023-08-27T20:55:14.329 INFO:tasks.rados.rados.0.smithi093.stdout:3630: writing smithi093153278-398 from 1787536 to 2415976 tid 2 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3630: writing smithi093153278-398 from 3003022 to 3454000 tid 3 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3619: done (11 left) 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3620: done (10 left) 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3631: read oid 442 snap -1 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3631: expect deleted 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3632: read oid 319 snap -1 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3632: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3633: write oid 449 current snap is 0 2023-08-27T20:55:14.336 INFO:tasks.rados.rados.0.smithi093.stdout:3621: finishing write tid 2 to smithi093153278-341 2023-08-27T20:55:14.337 INFO:tasks.rados.rados.0.smithi093.stdout:3621: finishing write tid 3 to smithi093153278-341 2023-08-27T20:55:14.337 INFO:tasks.rados.rados.0.smithi093.stdout:3621: finishing write tid 4 to smithi093153278-341 2023-08-27T20:55:14.337 INFO:tasks.rados.rados.0.smithi093.stdout:3621: finishing write tid 5 to smithi093153278-341 2023-08-27T20:55:14.337 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 341 v 446 (ObjNum 2234 snap 0 seq_num 2234) dirty exists 2023-08-27T20:55:14.337 INFO:tasks.rados.rados.0.smithi093.stdout:3621: left oid 341 (ObjNum 2234 snap 0 seq_num 2234) 2023-08-27T20:55:14.337 INFO:tasks.rados.rados.0.smithi093.stdout:3633: seq_num 2240 ranges {447680=765542,1846018=488451,2746798=662250} 2023-08-27T20:55:14.342 INFO:tasks.rados.rados.0.smithi093.stdout:3633: writing smithi093153278-449 from 447680 to 1213222 tid 1 2023-08-27T20:55:14.348 INFO:tasks.rados.rados.0.smithi093.stdout:3633: writing smithi093153278-449 from 1846018 to 2334469 tid 2 2023-08-27T20:55:14.355 INFO:tasks.rados.rados.0.smithi093.stdout:3633: writing smithi093153278-449 from 2746798 to 3409048 tid 3 2023-08-27T20:55:14.355 INFO:tasks.rados.rados.0.smithi093.stdout:3621: done (12 left) 2023-08-27T20:55:14.356 INFO:tasks.rados.rados.0.smithi093.stdout:3634: read oid 348 snap -1 2023-08-27T20:55:14.356 INFO:tasks.rados.rados.0.smithi093.stdout:3634: expect (ObjNum 1457 snap 0 seq_num 1457) 2023-08-27T20:55:14.356 INFO:tasks.rados.rados.0.smithi093.stdout:3635: read oid 48 snap -1 2023-08-27T20:55:14.356 INFO:tasks.rados.rados.0.smithi093.stdout:3635: expect (ObjNum 2188 snap 0 seq_num 2188) 2023-08-27T20:55:14.356 INFO:tasks.rados.rados.0.smithi093.stdout:3636: read oid 464 snap -1 2023-08-27T20:55:14.356 INFO:tasks.rados.rados.0.smithi093.stdout:3636: expect deleted 2023-08-27T20:55:14.357 INFO:tasks.rados.rados.0.smithi093.stdout:3637: write oid 195 current snap is 0 2023-08-27T20:55:14.357 INFO:tasks.rados.rados.0.smithi093.stdout:3622: finishing write tid 1 to smithi093153278-101 2023-08-27T20:55:14.357 INFO:tasks.rados.rados.0.smithi093.stdout:3622: finishing write tid 2 to smithi093153278-101 2023-08-27T20:55:14.357 INFO:tasks.rados.rados.0.smithi093.stdout:3622: finishing write tid 3 to smithi093153278-101 2023-08-27T20:55:14.357 INFO:tasks.rados.rados.0.smithi093.stdout:3637: seq_num 2241 ranges {464487=440791,1504500=632109,2657868=473522} 2023-08-27T20:55:14.360 INFO:tasks.rados.rados.0.smithi093.stdout:3637: writing smithi093153278-195 from 464487 to 905278 tid 1 2023-08-27T20:55:14.367 INFO:tasks.rados.rados.0.smithi093.stdout:3637: writing smithi093153278-195 from 1504500 to 2136609 tid 2 2023-08-27T20:55:14.372 INFO:tasks.rados.rados.0.smithi093.stdout:3637: writing smithi093153278-195 from 2657868 to 3131390 tid 3 2023-08-27T20:55:14.373 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 16 2023-08-27T20:55:14.373 INFO:tasks.rados.rados.0.smithi093.stdout:3622: finishing write tid 4 to smithi093153278-101 2023-08-27T20:55:14.373 INFO:tasks.rados.rados.0.smithi093.stdout:3622: finishing write tid 5 to smithi093153278-101 2023-08-27T20:55:14.373 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 101 v 401 (ObjNum 2235 snap 0 seq_num 2235) dirty exists 2023-08-27T20:55:14.373 INFO:tasks.rados.rados.0.smithi093.stdout:3622: left oid 101 (ObjNum 2235 snap 0 seq_num 2235) 2023-08-27T20:55:14.373 INFO:tasks.rados.rados.0.smithi093.stdout:3623: expect (ObjNum 1854 snap 0 seq_num 1854) 2023-08-27T20:55:14.589 INFO:tasks.rados.rados.0.smithi093.stdout:3624: finishing write tid 1 to smithi093153278-338 2023-08-27T20:55:14.589 INFO:tasks.rados.rados.0.smithi093.stdout:3624: finishing write tid 2 to smithi093153278-338 2023-08-27T20:55:14.589 INFO:tasks.rados.rados.0.smithi093.stdout:3624: finishing write tid 3 to smithi093153278-338 2023-08-27T20:55:14.589 INFO:tasks.rados.rados.0.smithi093.stdout:3622: done (15 left) 2023-08-27T20:55:14.589 INFO:tasks.rados.rados.0.smithi093.stdout:3623: done (14 left) 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3638: write oid 250 current snap is 0 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3624: finishing write tid 4 to smithi093153278-338 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3624: finishing write tid 5 to smithi093153278-338 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 338 v 450 (ObjNum 2236 snap 0 seq_num 2236) dirty exists 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3624: left oid 338 (ObjNum 2236 snap 0 seq_num 2236) 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3625: finishing write tid 1 to smithi093153278-100 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3625: finishing write tid 2 to smithi093153278-100 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3625: finishing write tid 3 to smithi093153278-100 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3625: finishing write tid 4 to smithi093153278-100 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3625: finishing write tid 5 to smithi093153278-100 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 100 v 421 (ObjNum 2237 snap 0 seq_num 2237) dirty exists 2023-08-27T20:55:14.590 INFO:tasks.rados.rados.0.smithi093.stdout:3625: left oid 100 (ObjNum 2237 snap 0 seq_num 2237) 2023-08-27T20:55:14.591 INFO:tasks.rados.rados.0.smithi093.stdout:3628: expect (ObjNum 696 snap 0 seq_num 696) 2023-08-27T20:55:14.758 INFO:tasks.rados.rados.0.smithi093.stdout:3627: expect (ObjNum 1644 snap 0 seq_num 1644) 2023-08-27T20:55:14.946 INFO:tasks.rados.rados.0.smithi093.stdout:3629: finishing write tid 1 to smithi093153278-328 2023-08-27T20:55:14.947 INFO:tasks.rados.rados.0.smithi093.stdout:3626: expect (ObjNum 1979 snap 0 seq_num 1979) 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:3629: finishing write tid 2 to smithi093153278-328 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:3629: finishing write tid 3 to smithi093153278-328 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:3629: finishing write tid 4 to smithi093153278-328 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:3629: finishing write tid 5 to smithi093153278-328 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 328 v 520 (ObjNum 2238 snap 0 seq_num 2238) dirty exists 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:3629: left oid 328 (ObjNum 2238 snap 0 seq_num 2238) 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:3630: finishing write tid 1 to smithi093153278-398 2023-08-27T20:55:15.043 INFO:tasks.rados.rados.0.smithi093.stdout:3630: finishing write tid 2 to smithi093153278-398 2023-08-27T20:55:15.044 INFO:tasks.rados.rados.0.smithi093.stdout:3638: seq_num 2242 ranges {481294=516040,1562982=492120,2801644=684794} 2023-08-27T20:55:15.048 INFO:tasks.rados.rados.0.smithi093.stdout:3638: writing smithi093153278-250 from 481294 to 997334 tid 1 2023-08-27T20:55:15.054 INFO:tasks.rados.rados.0.smithi093.stdout:3638: writing smithi093153278-250 from 1562982 to 2055102 tid 2 2023-08-27T20:55:15.064 INFO:tasks.rados.rados.0.smithi093.stdout:3638: writing smithi093153278-250 from 2801644 to 3486438 tid 3 2023-08-27T20:55:15.064 INFO:tasks.rados.rados.0.smithi093.stdout:3624: done (14 left) 2023-08-27T20:55:15.065 INFO:tasks.rados.rados.0.smithi093.stdout:3625: done (13 left) 2023-08-27T20:55:15.065 INFO:tasks.rados.rados.0.smithi093.stdout:3626: done (12 left) 2023-08-27T20:55:15.065 INFO:tasks.rados.rados.0.smithi093.stdout:3627: done (11 left) 2023-08-27T20:55:15.065 INFO:tasks.rados.rados.0.smithi093.stdout:3628: done (10 left) 2023-08-27T20:55:15.065 INFO:tasks.rados.rados.0.smithi093.stdout:3629: done (9 left) 2023-08-27T20:55:15.065 INFO:tasks.rados.rados.0.smithi093.stdout:3639: read oid 312 snap -1 2023-08-27T20:55:15.066 INFO:tasks.rados.rados.0.smithi093.stdout:3639: expect (ObjNum 1817 snap 0 seq_num 1817) 2023-08-27T20:55:15.066 INFO:tasks.rados.rados.0.smithi093.stdout:3640: write oid 60 current snap is 0 2023-08-27T20:55:15.066 INFO:tasks.rados.rados.0.smithi093.stdout:3630: finishing write tid 3 to smithi093153278-398 2023-08-27T20:55:15.066 INFO:tasks.rados.rados.0.smithi093.stdout:3630: finishing write tid 4 to smithi093153278-398 2023-08-27T20:55:15.066 INFO:tasks.rados.rados.0.smithi093.stdout:3640: seq_num 2243 ranges {498101=707642,1621464=635778,2712714=612419} 2023-08-27T20:55:15.069 INFO:tasks.rados.rados.0.smithi093.stdout:3640: writing smithi093153278-60 from 498101 to 1205743 tid 1 2023-08-27T20:55:15.075 INFO:tasks.rados.rados.0.smithi093.stdout:3640: writing smithi093153278-60 from 1621464 to 2257242 tid 2 2023-08-27T20:55:15.082 INFO:tasks.rados.rados.0.smithi093.stdout:3640: writing smithi093153278-60 from 2712714 to 3325133 tid 3 2023-08-27T20:55:15.082 INFO:tasks.rados.rados.0.smithi093.stdout:3631: done (10 left) 2023-08-27T20:55:15.082 INFO:tasks.rados.rados.0.smithi093.stdout:3641: write oid 20 current snap is 0 2023-08-27T20:55:15.082 INFO:tasks.rados.rados.0.smithi093.stdout:3630: finishing write tid 5 to smithi093153278-398 2023-08-27T20:55:15.083 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 398 v 524 (ObjNum 2239 snap 0 seq_num 2239) dirty exists 2023-08-27T20:55:15.083 INFO:tasks.rados.rados.0.smithi093.stdout:3630: left oid 398 (ObjNum 2239 snap 0 seq_num 2239) 2023-08-27T20:55:15.083 INFO:tasks.rados.rados.0.smithi093.stdout:3632: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:55:15.232 INFO:tasks.rados.rados.0.smithi093.stdout:3633: finishing write tid 1 to smithi093153278-449 2023-08-27T20:55:15.233 INFO:tasks.rados.rados.0.smithi093.stdout:3633: finishing write tid 2 to smithi093153278-449 2023-08-27T20:55:15.233 INFO:tasks.rados.rados.0.smithi093.stdout:3641: seq_num 2244 ranges {514908=782891,2079946=495789,3256490=423691} 2023-08-27T20:55:15.239 INFO:tasks.rados.rados.0.smithi093.stdout:3641: writing smithi093153278-20 from 514908 to 1297799 tid 1 2023-08-27T20:55:15.246 INFO:tasks.rados.rados.0.smithi093.stdout:3641: writing smithi093153278-20 from 2079946 to 2575735 tid 2 2023-08-27T20:55:15.252 INFO:tasks.rados.rados.0.smithi093.stdout:3641: writing smithi093153278-20 from 3256490 to 3680181 tid 3 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3630: done (10 left) 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3632: done (9 left) 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3642: write oid 1 current snap is 0 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3633: finishing write tid 3 to smithi093153278-449 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3633: finishing write tid 4 to smithi093153278-449 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3633: finishing write tid 5 to smithi093153278-449 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 519 (ObjNum 2240 snap 0 seq_num 2240) dirty exists 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3633: left oid 449 (ObjNum 2240 snap 0 seq_num 2240) 2023-08-27T20:55:15.253 INFO:tasks.rados.rados.0.smithi093.stdout:3635: expect (ObjNum 2188 snap 0 seq_num 2188) 2023-08-27T20:55:15.454 INFO:tasks.rados.rados.0.smithi093.stdout:3634: expect (ObjNum 1457 snap 0 seq_num 1457) 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3637: finishing write tid 1 to smithi093153278-195 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3637: finishing write tid 2 to smithi093153278-195 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3637: finishing write tid 3 to smithi093153278-195 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3637: finishing write tid 4 to smithi093153278-195 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3637: finishing write tid 5 to smithi093153278-195 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 195 v 405 (ObjNum 2241 snap 0 seq_num 2241) dirty exists 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3637: left oid 195 (ObjNum 2241 snap 0 seq_num 2241) 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3638: finishing write tid 1 to smithi093153278-250 2023-08-27T20:55:15.575 INFO:tasks.rados.rados.0.smithi093.stdout:3638: finishing write tid 2 to smithi093153278-250 2023-08-27T20:55:15.576 INFO:tasks.rados.rados.0.smithi093.stdout:3638: finishing write tid 3 to smithi093153278-250 2023-08-27T20:55:15.576 INFO:tasks.rados.rados.0.smithi093.stdout:3638: finishing write tid 4 to smithi093153278-250 2023-08-27T20:55:15.576 INFO:tasks.rados.rados.0.smithi093.stdout:3638: finishing write tid 5 to smithi093153278-250 2023-08-27T20:55:15.576 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 250 v 584 (ObjNum 2242 snap 0 seq_num 2242) dirty exists 2023-08-27T20:55:15.576 INFO:tasks.rados.rados.0.smithi093.stdout:3638: left oid 250 (ObjNum 2242 snap 0 seq_num 2242) 2023-08-27T20:55:15.576 INFO:tasks.rados.rados.0.smithi093.stdout:3639: expect (ObjNum 1817 snap 0 seq_num 1817) 2023-08-27T20:55:15.680 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:55:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:55:15] "GET /metrics HTTP/1.1" 200 242563 "" "Prometheus/2.43.0" 2023-08-27T20:55:15.707 INFO:tasks.rados.rados.0.smithi093.stdout:3640: finishing write tid 1 to smithi093153278-60 2023-08-27T20:55:15.707 INFO:tasks.rados.rados.0.smithi093.stdout:3640: finishing write tid 2 to smithi093153278-60 2023-08-27T20:55:15.707 INFO:tasks.rados.rados.0.smithi093.stdout:3640: finishing write tid 3 to smithi093153278-60 2023-08-27T20:55:15.707 INFO:tasks.rados.rados.0.smithi093.stdout:3640: finishing write tid 4 to smithi093153278-60 2023-08-27T20:55:15.708 INFO:tasks.rados.rados.0.smithi093.stdout:3640: finishing write tid 5 to smithi093153278-60 2023-08-27T20:55:15.708 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 60 v 419 (ObjNum 2243 snap 0 seq_num 2243) dirty exists 2023-08-27T20:55:15.708 INFO:tasks.rados.rados.0.smithi093.stdout:3640: left oid 60 (ObjNum 2243 snap 0 seq_num 2243) 2023-08-27T20:55:15.708 INFO:tasks.rados.rados.0.smithi093.stdout:3641: finishing write tid 1 to smithi093153278-20 2023-08-27T20:55:15.708 INFO:tasks.rados.rados.0.smithi093.stdout:3642: seq_num 2245 ranges {531715=458140,1738428=639447,2883913=634963} 2023-08-27T20:55:15.712 INFO:tasks.rados.rados.0.smithi093.stdout:3642: writing smithi093153278-1 from 531715 to 989855 tid 1 2023-08-27T20:55:15.719 INFO:tasks.rados.rados.0.smithi093.stdout:3642: writing smithi093153278-1 from 1738428 to 2377875 tid 2 2023-08-27T20:55:15.727 INFO:tasks.rados.rados.0.smithi093.stdout:3642: writing smithi093153278-1 from 2883913 to 3518876 tid 3 2023-08-27T20:55:15.727 INFO:tasks.rados.rados.0.smithi093.stdout:3633: done (9 left) 2023-08-27T20:55:15.727 INFO:tasks.rados.rados.0.smithi093.stdout:3634: done (8 left) 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3635: done (7 left) 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3636: done (6 left) 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3637: done (5 left) 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3638: done (4 left) 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3639: done (3 left) 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3640: done (2 left) 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3643: write oid 377 current snap is 0 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3641: finishing write tid 2 to smithi093153278-20 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3641: finishing write tid 3 to smithi093153278-20 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3641: finishing write tid 4 to smithi093153278-20 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:3641: finishing write tid 5 to smithi093153278-20 2023-08-27T20:55:15.728 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 20 v 454 (ObjNum 2244 snap 0 seq_num 2244) dirty exists 2023-08-27T20:55:15.729 INFO:tasks.rados.rados.0.smithi093.stdout:3641: left oid 20 (ObjNum 2244 snap 0 seq_num 2244) 2023-08-27T20:55:15.729 INFO:tasks.rados.rados.0.smithi093.stdout:3642: finishing write tid 1 to smithi093153278-1 2023-08-27T20:55:15.729 INFO:tasks.rados.rados.0.smithi093.stdout:3642: finishing write tid 2 to smithi093153278-1 2023-08-27T20:55:15.729 INFO:tasks.rados.rados.0.smithi093.stdout:3643: seq_num 2246 ranges {548522=533389,1796910=499458,2911336=446235} 2023-08-27T20:55:15.732 INFO:tasks.rados.rados.0.smithi093.stdout:3643: writing smithi093153278-377 from 548522 to 1081911 tid 1 2023-08-27T20:55:15.739 INFO:tasks.rados.rados.0.smithi093.stdout:3643: writing smithi093153278-377 from 1796910 to 2296368 tid 2 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3643: writing smithi093153278-377 from 2911336 to 3357571 tid 3 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3641: done (2 left) 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3644: read oid 166 snap -1 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3644: expect deleted 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3645: read oid 386 snap -1 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3645: expect (ObjNum 2196 snap 0 seq_num 2196) 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3646: read oid 333 snap -1 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3646: expect (ObjNum 2049 snap 0 seq_num 2049) 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3647: write oid 113 current snap is 0 2023-08-27T20:55:15.746 INFO:tasks.rados.rados.0.smithi093.stdout:3642: finishing write tid 3 to smithi093153278-1 2023-08-27T20:55:15.747 INFO:tasks.rados.rados.0.smithi093.stdout:3642: finishing write tid 4 to smithi093153278-1 2023-08-27T20:55:15.747 INFO:tasks.rados.rados.0.smithi093.stdout:3642: finishing write tid 5 to smithi093153278-1 2023-08-27T20:55:15.747 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 1 v 423 (ObjNum 2245 snap 0 seq_num 2245) dirty exists 2023-08-27T20:55:15.747 INFO:tasks.rados.rados.0.smithi093.stdout:3642: left oid 1 (ObjNum 2245 snap 0 seq_num 2245) 2023-08-27T20:55:15.747 INFO:tasks.rados.rados.0.smithi093.stdout:3643: finishing write tid 1 to smithi093153278-377 2023-08-27T20:55:15.747 INFO:tasks.rados.rados.0.smithi093.stdout:3643: finishing write tid 2 to smithi093153278-377 2023-08-27T20:55:15.747 INFO:tasks.rados.rados.0.smithi093.stdout:3647: seq_num 2247 ranges {565329=608638,1739039=643116,2822406=773860} 2023-08-27T20:55:15.751 INFO:tasks.rados.rados.0.smithi093.stdout:3647: writing smithi093153278-113 from 565329 to 1173967 tid 1 2023-08-27T20:55:15.758 INFO:tasks.rados.rados.0.smithi093.stdout:3647: writing smithi093153278-113 from 1739039 to 2382155 tid 2 2023-08-27T20:55:15.766 INFO:tasks.rados.rados.0.smithi093.stdout:3647: writing smithi093153278-113 from 2822406 to 3596266 tid 3 2023-08-27T20:55:15.766 INFO:tasks.rados.rados.0.smithi093.stdout:3642: done (5 left) 2023-08-27T20:55:15.766 INFO:tasks.rados.rados.0.smithi093.stdout:3648: delete oid 429 current snap is 0 2023-08-27T20:55:15.766 INFO:tasks.rados.rados.0.smithi093.stdout:3643: finishing write tid 3 to smithi093153278-377 2023-08-27T20:55:15.766 INFO:tasks.rados.rados.0.smithi093.stdout:3643: finishing write tid 4 to smithi093153278-377 2023-08-27T20:55:15.767 INFO:tasks.rados.rados.0.smithi093.stdout:3643: finishing write tid 5 to smithi093153278-377 2023-08-27T20:55:15.767 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 377 v 470 (ObjNum 2246 snap 0 seq_num 2246) dirty exists 2023-08-27T20:55:15.767 INFO:tasks.rados.rados.0.smithi093.stdout:3643: left oid 377 (ObjNum 2246 snap 0 seq_num 2246) 2023-08-27T20:55:15.767 INFO:tasks.rados.rados.0.smithi093.stdout:3646: expect (ObjNum 2049 snap 0 seq_num 2049) 2023-08-27T20:55:15.909 INFO:tasks.rados.rados.0.smithi093.stdout:3647: finishing write tid 1 to smithi093153278-113 2023-08-27T20:55:15.910 INFO:tasks.rados.rados.0.smithi093.stdout:3645: expect (ObjNum 2196 snap 0 seq_num 2196) 2023-08-27T20:55:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:15 smithi136 ceph-mon[113531]: pgmap v51: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 5.8 MiB/s wr, 69 op/s 2023-08-27T20:55:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1754081132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:15 smithi093 ceph-mon[111335]: pgmap v51: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 5.8 MiB/s wr, 69 op/s 2023-08-27T20:55:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1754081132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:15 smithi093 ceph-mon[118902]: pgmap v51: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 5.8 MiB/s wr, 69 op/s 2023-08-27T20:55:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1754081132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:15.999 INFO:tasks.rados.rados.0.smithi093.stdout:3647: finishing write tid 2 to smithi093153278-113 2023-08-27T20:55:15.999 INFO:tasks.rados.rados.0.smithi093.stdout:3647: finishing write tid 3 to smithi093153278-113 2023-08-27T20:55:15.999 INFO:tasks.rados.rados.0.smithi093.stdout:3647: finishing write tid 4 to smithi093153278-113 2023-08-27T20:55:16.000 INFO:tasks.rados.rados.0.smithi093.stdout:3647: finishing write tid 5 to smithi093153278-113 2023-08-27T20:55:16.000 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 113 v 366 (ObjNum 2247 snap 0 seq_num 2247) dirty exists 2023-08-27T20:55:16.000 INFO:tasks.rados.rados.0.smithi093.stdout:3647: left oid 113 (ObjNum 2247 snap 0 seq_num 2247) 2023-08-27T20:55:16.000 INFO:tasks.rados.rados.0.smithi093.stdout:3643: done (5 left) 2023-08-27T20:55:16.000 INFO:tasks.rados.rados.0.smithi093.stdout:3644: done (4 left) 2023-08-27T20:55:16.000 INFO:tasks.rados.rados.0.smithi093.stdout:3645: done (3 left) 2023-08-27T20:55:16.000 INFO:tasks.rados.rados.0.smithi093.stdout:3646: done (2 left) 2023-08-27T20:55:16.001 INFO:tasks.rados.rados.0.smithi093.stdout:3647: done (1 left) 2023-08-27T20:55:16.001 INFO:tasks.rados.rados.0.smithi093.stdout:3648: done (0 left) 2023-08-27T20:55:16.001 INFO:tasks.rados.rados.0.smithi093.stdout:3649: delete oid 195 current snap is 0 2023-08-27T20:55:16.002 INFO:tasks.rados.rados.0.smithi093.stdout:3649: done (0 left) 2023-08-27T20:55:16.002 INFO:tasks.rados.rados.0.smithi093.stdout:3650: read oid 196 snap -1 2023-08-27T20:55:16.002 INFO:tasks.rados.rados.0.smithi093.stdout:3650: expect deleted 2023-08-27T20:55:16.002 INFO:tasks.rados.rados.0.smithi093.stdout:3651: delete oid 84 current snap is 0 2023-08-27T20:55:16.003 INFO:tasks.rados.rados.0.smithi093.stdout:3650: done (1 left) 2023-08-27T20:55:16.003 INFO:tasks.rados.rados.0.smithi093.stdout:3651: done (0 left) 2023-08-27T20:55:16.003 INFO:tasks.rados.rados.0.smithi093.stdout:3652: write oid 370 current snap is 0 2023-08-27T20:55:16.003 INFO:tasks.rados.rados.0.smithi093.stdout:3652: seq_num 2251 ranges {632557=625987,1972967=650454,3332098=418948} 2023-08-27T20:55:16.009 INFO:tasks.rados.rados.0.smithi093.stdout:3652: writing smithi093153278-370 from 632557 to 1258544 tid 1 2023-08-27T20:55:16.016 INFO:tasks.rados.rados.0.smithi093.stdout:3652: writing smithi093153278-370 from 1972967 to 2623421 tid 2 2023-08-27T20:55:16.023 INFO:tasks.rados.rados.0.smithi093.stdout:3652: writing smithi093153278-370 from 3332098 to 3751046 tid 3 2023-08-27T20:55:16.024 INFO:tasks.rados.rados.0.smithi093.stdout:3653: read oid 490 snap -1 2023-08-27T20:55:16.024 INFO:tasks.rados.rados.0.smithi093.stdout:3653: expect (ObjNum 1979 snap 0 seq_num 1979) 2023-08-27T20:55:16.024 INFO:tasks.rados.rados.0.smithi093.stdout:3654: write oid 329 current snap is 0 2023-08-27T20:55:16.024 INFO:tasks.rados.rados.0.smithi093.stdout:3652: finishing write tid 1 to smithi093153278-370 2023-08-27T20:55:16.024 INFO:tasks.rados.rados.0.smithi093.stdout:3654: seq_num 2252 ranges {649364=701236,2031449=794112,3243168=606196} 2023-08-27T20:55:16.031 INFO:tasks.rados.rados.0.smithi093.stdout:3654: writing smithi093153278-329 from 649364 to 1350600 tid 1 2023-08-27T20:55:16.038 INFO:tasks.rados.rados.0.smithi093.stdout:3654: writing smithi093153278-329 from 2031449 to 2825561 tid 2 2023-08-27T20:55:16.044 INFO:tasks.rados.rados.0.smithi093.stdout:3654: writing smithi093153278-329 from 3243168 to 3849364 tid 3 2023-08-27T20:55:16.044 INFO:tasks.rados.rados.0.smithi093.stdout:3655: delete oid 484 current snap is 0 2023-08-27T20:55:16.045 INFO:tasks.rados.rados.0.smithi093.stdout:3652: finishing write tid 2 to smithi093153278-370 2023-08-27T20:55:16.045 INFO:tasks.rados.rados.0.smithi093.stdout:3652: finishing write tid 3 to smithi093153278-370 2023-08-27T20:55:16.045 INFO:tasks.rados.rados.0.smithi093.stdout:3652: finishing write tid 4 to smithi093153278-370 2023-08-27T20:55:16.045 INFO:tasks.rados.rados.0.smithi093.stdout:3652: finishing write tid 5 to smithi093153278-370 2023-08-27T20:55:16.045 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 370 v 474 (ObjNum 2251 snap 0 seq_num 2251) dirty exists 2023-08-27T20:55:16.045 INFO:tasks.rados.rados.0.smithi093.stdout:3652: left oid 370 (ObjNum 2251 snap 0 seq_num 2251) 2023-08-27T20:55:16.045 INFO:tasks.rados.rados.0.smithi093.stdout:3653: expect (ObjNum 1979 snap 0 seq_num 1979) 2023-08-27T20:55:16.142 INFO:tasks.rados.rados.0.smithi093.stdout:3654: finishing write tid 1 to smithi093153278-329 2023-08-27T20:55:16.142 INFO:tasks.rados.rados.0.smithi093.stdout:3654: finishing write tid 2 to smithi093153278-329 2023-08-27T20:55:16.142 INFO:tasks.rados.rados.0.smithi093.stdout:3654: finishing write tid 3 to smithi093153278-329 2023-08-27T20:55:16.142 INFO:tasks.rados.rados.0.smithi093.stdout:3654: finishing write tid 4 to smithi093153278-329 2023-08-27T20:55:16.142 INFO:tasks.rados.rados.0.smithi093.stdout:3654: finishing write tid 5 to smithi093153278-329 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 329 v 416 (ObjNum 2252 snap 0 seq_num 2252) dirty exists 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:3654: left oid 329 (ObjNum 2252 snap 0 seq_num 2252) 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:3652: done (3 left) 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:3653: done (2 left) 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:3654: done (1 left) 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:3655: done (0 left) 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:3656: write oid 139 current snap is 0 2023-08-27T20:55:16.143 INFO:tasks.rados.rados.0.smithi093.stdout:3656: seq_num 2254 ranges {682978=451734,1748413=797781,3298014=584964} 2023-08-27T20:55:16.147 INFO:tasks.rados.rados.0.smithi093.stdout:3656: writing smithi093153278-139 from 682978 to 1134712 tid 1 2023-08-27T20:55:16.155 INFO:tasks.rados.rados.0.smithi093.stdout:3656: writing smithi093153278-139 from 1748413 to 2546194 tid 2 2023-08-27T20:55:16.162 INFO:tasks.rados.rados.0.smithi093.stdout:3656: writing smithi093153278-139 from 3298014 to 3882978 tid 3 2023-08-27T20:55:16.162 INFO:tasks.rados.rados.0.smithi093.stdout:3657: delete oid 273 current snap is 0 2023-08-27T20:55:16.163 INFO:tasks.rados.rados.0.smithi093.stdout:3656: finishing write tid 1 to smithi093153278-139 2023-08-27T20:55:16.163 INFO:tasks.rados.rados.0.smithi093.stdout:3656: finishing write tid 2 to smithi093153278-139 2023-08-27T20:55:16.164 INFO:tasks.rados.rados.0.smithi093.stdout:3657: done (1 left) 2023-08-27T20:55:16.164 INFO:tasks.rados.rados.0.smithi093.stdout:3658: read oid 402 snap -1 2023-08-27T20:55:16.164 INFO:tasks.rados.rados.0.smithi093.stdout:3658: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:55:16.165 INFO:tasks.rados.rados.0.smithi093.stdout:3659: write oid 234 current snap is 0 2023-08-27T20:55:16.165 INFO:tasks.rados.rados.0.smithi093.stdout:3659: seq_num 2256 ranges {716592=602232,1749024=401450,2836507=791661} 2023-08-27T20:55:16.172 INFO:tasks.rados.rados.0.smithi093.stdout:3659: writing smithi093153278-234 from 716592 to 1318824 tid 1 2023-08-27T20:55:16.177 INFO:tasks.rados.rados.0.smithi093.stdout:3659: writing smithi093153278-234 from 1749024 to 2150474 tid 2 2023-08-27T20:55:16.185 INFO:tasks.rados.rados.0.smithi093.stdout:3659: writing smithi093153278-234 from 2836507 to 3628168 tid 3 2023-08-27T20:55:16.185 INFO:tasks.rados.rados.0.smithi093.stdout:3660: write oid 34 current snap is 0 2023-08-27T20:55:16.185 INFO:tasks.rados.rados.0.smithi093.stdout:3656: finishing write tid 3 to smithi093153278-139 2023-08-27T20:55:16.185 INFO:tasks.rados.rados.0.smithi093.stdout:3656: finishing write tid 4 to smithi093153278-139 2023-08-27T20:55:16.185 INFO:tasks.rados.rados.0.smithi093.stdout:3656: finishing write tid 5 to smithi093153278-139 2023-08-27T20:55:16.186 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 139 v 446 (ObjNum 2254 snap 0 seq_num 2254) dirty exists 2023-08-27T20:55:16.186 INFO:tasks.rados.rados.0.smithi093.stdout:3656: left oid 139 (ObjNum 2254 snap 0 seq_num 2254) 2023-08-27T20:55:16.186 INFO:tasks.rados.rados.0.smithi093.stdout:3658: expect (ObjNum 1037 snap 0 seq_num 1037) 2023-08-27T20:55:16.349 INFO:tasks.rados.rados.0.smithi093.stdout:3659: finishing write tid 1 to smithi093153278-234 2023-08-27T20:55:16.349 INFO:tasks.rados.rados.0.smithi093.stdout:3659: finishing write tid 2 to smithi093153278-234 2023-08-27T20:55:16.349 INFO:tasks.rados.rados.0.smithi093.stdout:3659: finishing write tid 3 to smithi093153278-234 2023-08-27T20:55:16.349 INFO:tasks.rados.rados.0.smithi093.stdout:3659: finishing write tid 4 to smithi093153278-234 2023-08-27T20:55:16.349 INFO:tasks.rados.rados.0.smithi093.stdout:3659: finishing write tid 5 to smithi093153278-234 2023-08-27T20:55:16.349 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 234 v 420 (ObjNum 2256 snap 0 seq_num 2256) dirty exists 2023-08-27T20:55:16.350 INFO:tasks.rados.rados.0.smithi093.stdout:3659: left oid 234 (ObjNum 2256 snap 0 seq_num 2256) 2023-08-27T20:55:16.350 INFO:tasks.rados.rados.0.smithi093.stdout:3660: seq_num 2257 ranges {733399=677481,2207506=661461,3380283=553116} 2023-08-27T20:55:16.356 INFO:tasks.rados.rados.0.smithi093.stdout:3660: writing smithi093153278-34 from 733399 to 1410880 tid 1 2023-08-27T20:55:16.364 INFO:tasks.rados.rados.0.smithi093.stdout:3660: writing smithi093153278-34 from 2207506 to 2868967 tid 2 2023-08-27T20:55:16.371 INFO:tasks.rados.rados.0.smithi093.stdout:3660: writing smithi093153278-34 from 3380283 to 3933399 tid 3 2023-08-27T20:55:16.371 INFO:tasks.rados.rados.0.smithi093.stdout:3656: done (3 left) 2023-08-27T20:55:16.371 INFO:tasks.rados.rados.0.smithi093.stdout:3658: done (2 left) 2023-08-27T20:55:16.371 INFO:tasks.rados.rados.0.smithi093.stdout:3659: done (1 left) 2023-08-27T20:55:16.371 INFO:tasks.rados.rados.0.smithi093.stdout:3661: write oid 417 current snap is 0 2023-08-27T20:55:16.371 INFO:tasks.rados.rados.0.smithi093.stdout:3660: finishing write tid 1 to smithi093153278-34 2023-08-27T20:55:16.371 INFO:tasks.rados.rados.0.smithi093.stdout:3660: finishing write tid 2 to smithi093153278-34 2023-08-27T20:55:16.372 INFO:tasks.rados.rados.0.smithi093.stdout:3661: seq_num 2258 ranges {750206=469083,1982341=405119,3007706=530558} 2023-08-27T20:55:16.377 INFO:tasks.rados.rados.0.smithi093.stdout:3661: writing smithi093153278-417 from 750206 to 1219289 tid 1 2023-08-27T20:55:16.384 INFO:tasks.rados.rados.0.smithi093.stdout:3661: writing smithi093153278-417 from 1982341 to 2387460 tid 2 2023-08-27T20:55:16.390 INFO:tasks.rados.rados.0.smithi093.stdout:3661: writing smithi093153278-417 from 3007706 to 3538264 tid 3 2023-08-27T20:55:16.391 INFO:tasks.rados.rados.0.smithi093.stdout:3662: read oid 240 snap -1 2023-08-27T20:55:16.391 INFO:tasks.rados.rados.0.smithi093.stdout:3662: expect deleted 2023-08-27T20:55:16.391 INFO:tasks.rados.rados.0.smithi093.stdout:3663: write oid 10 current snap is 0 2023-08-27T20:55:16.391 INFO:tasks.rados.rados.0.smithi093.stdout:3660: finishing write tid 3 to smithi093153278-34 2023-08-27T20:55:16.391 INFO:tasks.rados.rados.0.smithi093.stdout:3663: seq_num 2259 ranges {767013=544332,1924470=665130,3035129=741830} 2023-08-27T20:55:16.397 INFO:tasks.rados.rados.0.smithi093.stdout:3663: writing smithi093153278-10 from 767013 to 1311345 tid 1 2023-08-27T20:55:16.405 INFO:tasks.rados.rados.0.smithi093.stdout:3663: writing smithi093153278-10 from 1924470 to 2589600 tid 2 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3663: writing smithi093153278-10 from 3035129 to 3776959 tid 3 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3664: read oid 135 snap -1 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3664: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3665: read oid 98 snap -1 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3665: expect deleted 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3666: delete oid 466 current snap is 0 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3660: finishing write tid 4 to smithi093153278-34 2023-08-27T20:55:16.412 INFO:tasks.rados.rados.0.smithi093.stdout:3660: finishing write tid 5 to smithi093153278-34 2023-08-27T20:55:16.413 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 34 v 425 (ObjNum 2257 snap 0 seq_num 2257) dirty exists 2023-08-27T20:55:16.413 INFO:tasks.rados.rados.0.smithi093.stdout:3660: left oid 34 (ObjNum 2257 snap 0 seq_num 2257) 2023-08-27T20:55:16.413 INFO:tasks.rados.rados.0.smithi093.stdout:3661: finishing write tid 1 to smithi093153278-417 2023-08-27T20:55:16.413 INFO:tasks.rados.rados.0.smithi093.stdout:3661: finishing write tid 2 to smithi093153278-417 2023-08-27T20:55:16.413 INFO:tasks.rados.rados.0.smithi093.stdout:3661: finishing write tid 3 to smithi093153278-417 2023-08-27T20:55:16.413 INFO:tasks.rados.rados.0.smithi093.stdout:3661: finishing write tid 4 to smithi093153278-417 2023-08-27T20:55:16.413 INFO:tasks.rados.rados.0.smithi093.stdout:3661: finishing write tid 5 to smithi093153278-417 2023-08-27T20:55:16.414 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 417 v 485 (ObjNum 2258 snap 0 seq_num 2258) dirty exists 2023-08-27T20:55:16.414 INFO:tasks.rados.rados.0.smithi093.stdout:3661: left oid 417 (ObjNum 2258 snap 0 seq_num 2258) 2023-08-27T20:55:16.414 INFO:tasks.rados.rados.0.smithi093.stdout:3663: finishing write tid 1 to smithi093153278-10 2023-08-27T20:55:16.414 INFO:tasks.rados.rados.0.smithi093.stdout:3663: finishing write tid 2 to smithi093153278-10 2023-08-27T20:55:16.415 INFO:tasks.rados.rados.0.smithi093.stdout:3660: done (6 left) 2023-08-27T20:55:16.415 INFO:tasks.rados.rados.0.smithi093.stdout:3661: done (5 left) 2023-08-27T20:55:16.415 INFO:tasks.rados.rados.0.smithi093.stdout:3662: done (4 left) 2023-08-27T20:55:16.415 INFO:tasks.rados.rados.0.smithi093.stdout:3665: done (3 left) 2023-08-27T20:55:16.416 INFO:tasks.rados.rados.0.smithi093.stdout:3666: done (2 left) 2023-08-27T20:55:16.416 INFO:tasks.rados.rados.0.smithi093.stdout:3667: write oid 87 current snap is 0 2023-08-27T20:55:16.416 INFO:tasks.rados.rados.0.smithi093.stdout:3667: seq_num 2261 ranges {400627=694830,1641434=359193} 2023-08-27T20:55:16.419 INFO:tasks.rados.rados.0.smithi093.stdout:3667: writing smithi093153278-87 from 400627 to 1095457 tid 1 2023-08-27T20:55:16.424 INFO:tasks.rados.rados.0.smithi093.stdout:3667: writing smithi093153278-87 from 1641434 to 2000627 tid 2 2023-08-27T20:55:16.424 INFO:tasks.rados.rados.0.smithi093.stdout:3668: write oid 162 current snap is 0 2023-08-27T20:55:16.424 INFO:tasks.rados.rados.0.smithi093.stdout:3663: finishing write tid 3 to smithi093153278-10 2023-08-27T20:55:16.424 INFO:tasks.rados.rados.0.smithi093.stdout:3668: seq_num 2262 ranges {417434=770079,1699916=317518} 2023-08-27T20:55:16.430 INFO:tasks.rados.rados.0.smithi093.stdout:3668: writing smithi093153278-162 from 417434 to 1187513 tid 1 2023-08-27T20:55:16.436 INFO:tasks.rados.rados.0.smithi093.stdout:3668: writing smithi093153278-162 from 1699916 to 2017434 tid 2 2023-08-27T20:55:16.436 INFO:tasks.rados.rados.0.smithi093.stdout:3669: write oid 495 current snap is 0 2023-08-27T20:55:16.436 INFO:tasks.rados.rados.0.smithi093.stdout:3663: finishing write tid 4 to smithi093153278-10 2023-08-27T20:55:16.436 INFO:tasks.rados.rados.0.smithi093.stdout:3663: finishing write tid 5 to smithi093153278-10 2023-08-27T20:55:16.436 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 10 v 370 (ObjNum 2259 snap 0 seq_num 2259) dirty exists 2023-08-27T20:55:16.436 INFO:tasks.rados.rados.0.smithi093.stdout:3663: left oid 10 (ObjNum 2259 snap 0 seq_num 2259) 2023-08-27T20:55:16.437 INFO:tasks.rados.rados.0.smithi093.stdout:3664: expect (ObjNum 1526 snap 0 seq_num 1526) 2023-08-27T20:55:16.587 INFO:tasks.rados.rados.0.smithi093.stdout:3667: finishing write tid 1 to smithi093153278-87 2023-08-27T20:55:16.587 INFO:tasks.rados.rados.0.smithi093.stdout:3667: finishing write tid 2 to smithi093153278-87 2023-08-27T20:55:16.587 INFO:tasks.rados.rados.0.smithi093.stdout:3667: finishing write tid 3 to smithi093153278-87 2023-08-27T20:55:16.587 INFO:tasks.rados.rados.0.smithi093.stdout:3667: finishing write tid 4 to smithi093153278-87 2023-08-27T20:55:16.587 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 426 (ObjNum 2261 snap 0 seq_num 2261) dirty exists 2023-08-27T20:55:16.587 INFO:tasks.rados.rados.0.smithi093.stdout:3667: left oid 87 (ObjNum 2261 snap 0 seq_num 2261) 2023-08-27T20:55:16.587 INFO:tasks.rados.rados.0.smithi093.stdout:3668: finishing write tid 1 to smithi093153278-162 2023-08-27T20:55:16.588 INFO:tasks.rados.rados.0.smithi093.stdout:3669: seq_num 2263 ranges {434241=445328,1642045=392196} 2023-08-27T20:55:16.591 INFO:tasks.rados.rados.0.smithi093.stdout:3669: writing smithi093153278-495 from 434241 to 879569 tid 1 2023-08-27T20:55:16.598 INFO:tasks.rados.rados.0.smithi093.stdout:3669: writing smithi093153278-495 from 1642045 to 2034241 tid 2 2023-08-27T20:55:16.598 INFO:tasks.rados.rados.0.smithi093.stdout:3663: done (4 left) 2023-08-27T20:55:16.598 INFO:tasks.rados.rados.0.smithi093.stdout:3664: done (3 left) 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3667: done (2 left) 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3670: read oid 487 snap -1 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3670: expect (ObjNum 1594 snap 0 seq_num 1594) 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3671: read oid 480 snap -1 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3671: expect deleted 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3672: write oid 444 current snap is 0 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3668: finishing write tid 2 to smithi093153278-162 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3668: finishing write tid 3 to smithi093153278-162 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:3668: finishing write tid 4 to smithi093153278-162 2023-08-27T20:55:16.599 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 162 v 409 (ObjNum 2262 snap 0 seq_num 2262) dirty exists 2023-08-27T20:55:16.600 INFO:tasks.rados.rados.0.smithi093.stdout:3668: left oid 162 (ObjNum 2262 snap 0 seq_num 2262) 2023-08-27T20:55:16.600 INFO:tasks.rados.rados.0.smithi093.stdout:3669: finishing write tid 1 to smithi093153278-495 2023-08-27T20:55:16.600 INFO:tasks.rados.rados.0.smithi093.stdout:3672: seq_num 2264 ranges {451048=520577,1700527=350521} 2023-08-27T20:55:16.604 INFO:tasks.rados.rados.0.smithi093.stdout:3672: writing smithi093153278-444 from 451048 to 971625 tid 1 2023-08-27T20:55:16.610 INFO:tasks.rados.rados.0.smithi093.stdout:3672: writing smithi093153278-444 from 1700527 to 2051048 tid 2 2023-08-27T20:55:16.610 INFO:tasks.rados.rados.0.smithi093.stdout:3668: done (4 left) 2023-08-27T20:55:16.610 INFO:tasks.rados.rados.0.smithi093.stdout:3673: write oid 113 current snap is 0 2023-08-27T20:55:16.610 INFO:tasks.rados.rados.0.smithi093.stdout:3673: seq_num 2265 ranges {467855=595826,1759009=308846} 2023-08-27T20:55:16.615 INFO:tasks.rados.rados.0.smithi093.stdout:3673: writing smithi093153278-113 from 467855 to 1063681 tid 1 2023-08-27T20:55:16.621 INFO:tasks.rados.rados.0.smithi093.stdout:3673: writing smithi093153278-113 from 1759009 to 2067855 tid 2 2023-08-27T20:55:16.622 INFO:tasks.rados.rados.0.smithi093.stdout:3674: delete oid 283 current snap is 0 2023-08-27T20:55:16.622 INFO:tasks.rados.rados.0.smithi093.stdout:3669: finishing write tid 2 to smithi093153278-495 2023-08-27T20:55:16.622 INFO:tasks.rados.rados.0.smithi093.stdout:3669: finishing write tid 3 to smithi093153278-495 2023-08-27T20:55:16.622 INFO:tasks.rados.rados.0.smithi093.stdout:3669: finishing write tid 4 to smithi093153278-495 2023-08-27T20:55:16.622 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 495 v 532 (ObjNum 2263 snap 0 seq_num 2263) dirty exists 2023-08-27T20:55:16.622 INFO:tasks.rados.rados.0.smithi093.stdout:3669: left oid 495 (ObjNum 2263 snap 0 seq_num 2263) 2023-08-27T20:55:16.622 INFO:tasks.rados.rados.0.smithi093.stdout:3670: expect (ObjNum 1594 snap 0 seq_num 1594) 2023-08-27T20:55:16.712 INFO:tasks.rados.rados.0.smithi093.stdout:3672: finishing write tid 1 to smithi093153278-444 2023-08-27T20:55:16.712 INFO:tasks.rados.rados.0.smithi093.stdout:3672: finishing write tid 2 to smithi093153278-444 2023-08-27T20:55:16.712 INFO:tasks.rados.rados.0.smithi093.stdout:3672: finishing write tid 3 to smithi093153278-444 2023-08-27T20:55:16.712 INFO:tasks.rados.rados.0.smithi093.stdout:3672: finishing write tid 4 to smithi093153278-444 2023-08-27T20:55:16.712 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 444 v 477 (ObjNum 2264 snap 0 seq_num 2264) dirty exists 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3672: left oid 444 (ObjNum 2264 snap 0 seq_num 2264) 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3673: finishing write tid 1 to smithi093153278-113 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3673: finishing write tid 2 to smithi093153278-113 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3673: finishing write tid 3 to smithi093153278-113 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3673: finishing write tid 4 to smithi093153278-113 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 113 v 373 (ObjNum 2265 snap 0 seq_num 2265) dirty exists 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3673: left oid 113 (ObjNum 2265 snap 0 seq_num 2265) 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3669: done (5 left) 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3670: done (4 left) 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3671: done (3 left) 2023-08-27T20:55:16.713 INFO:tasks.rados.rados.0.smithi093.stdout:3672: done (2 left) 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3673: done (1 left) 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3674: done (0 left) 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3675: read oid 104 snap -1 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3675: expect (ObjNum 1450 snap 0 seq_num 1450) 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3676: read oid 407 snap -1 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3676: expect (ObjNum 1136 snap 0 seq_num 1136) 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3677: delete oid 334 current snap is 0 2023-08-27T20:55:16.714 INFO:tasks.rados.rados.0.smithi093.stdout:3675: expect (ObjNum 1450 snap 0 seq_num 1450) 2023-08-27T20:55:16.821 INFO:tasks.rados.rados.0.smithi093.stdout:3676: expect (ObjNum 1136 snap 0 seq_num 1136) 2023-08-27T20:55:17.000 INFO:tasks.rados.rados.0.smithi093.stdout:3675: done (2 left) 2023-08-27T20:55:17.000 INFO:tasks.rados.rados.0.smithi093.stdout:3676: done (1 left) 2023-08-27T20:55:17.000 INFO:tasks.rados.rados.0.smithi093.stdout:3677: done (0 left) 2023-08-27T20:55:17.000 INFO:tasks.rados.rados.0.smithi093.stdout:3678: read oid 97 snap -1 2023-08-27T20:55:17.001 INFO:tasks.rados.rados.0.smithi093.stdout:3678: expect deleted 2023-08-27T20:55:17.001 INFO:tasks.rados.rados.0.smithi093.stdout:3679: read oid 33 snap -1 2023-08-27T20:55:17.001 INFO:tasks.rados.rados.0.smithi093.stdout:3679: expect (ObjNum 2153 snap 0 seq_num 2153) 2023-08-27T20:55:17.001 INFO:tasks.rados.rados.0.smithi093.stdout:3680: write oid 364 current snap is 0 2023-08-27T20:55:17.001 INFO:tasks.rados.rados.0.smithi093.stdout:3680: seq_num 2268 ranges {518276=537926,1534455=423464} 2023-08-27T20:55:17.005 INFO:tasks.rados.rados.0.smithi093.stdout:3680: writing smithi093153278-364 from 518276 to 1056202 tid 1 2023-08-27T20:55:17.011 INFO:tasks.rados.rados.0.smithi093.stdout:3680: writing smithi093153278-364 from 1534455 to 1957919 tid 2 2023-08-27T20:55:17.011 INFO:tasks.rados.rados.0.smithi093.stdout:3681: read oid 312 snap -1 2023-08-27T20:55:17.011 INFO:tasks.rados.rados.0.smithi093.stdout:3681: expect (ObjNum 1817 snap 0 seq_num 1817) 2023-08-27T20:55:17.011 INFO:tasks.rados.rados.0.smithi093.stdout:3682: write oid 53 current snap is 0 2023-08-27T20:55:17.011 INFO:tasks.rados.rados.0.smithi093.stdout:3682: seq_num 2269 ranges {535083=613175,1592937=542146} 2023-08-27T20:55:17.017 INFO:tasks.rados.rados.0.smithi093.stdout:3682: writing smithi093153278-53 from 535083 to 1148258 tid 1 2023-08-27T20:55:17.023 INFO:tasks.rados.rados.0.smithi093.stdout:3682: writing smithi093153278-53 from 1592937 to 2135083 tid 2 2023-08-27T20:55:17.023 INFO:tasks.rados.rados.0.smithi093.stdout:3683: delete oid 465 current snap is 0 2023-08-27T20:55:17.023 INFO:tasks.rados.rados.0.smithi093.stdout:3679: expect (ObjNum 2153 snap 0 seq_num 2153) 2023-08-27T20:55:17.168 INFO:tasks.rados.rados.0.smithi093.stdout:3680: finishing write tid 1 to smithi093153278-364 2023-08-27T20:55:17.168 INFO:tasks.rados.rados.0.smithi093.stdout:3678: done (5 left) 2023-08-27T20:55:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:3679: done (4 left) 2023-08-27T20:55:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:3683: done (3 left) 2023-08-27T20:55:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:3684: delete oid 461 current snap is 0 2023-08-27T20:55:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:3680: finishing write tid 2 to smithi093153278-364 2023-08-27T20:55:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:3680: finishing write tid 3 to smithi093153278-364 2023-08-27T20:55:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:3680: finishing write tid 4 to smithi093153278-364 2023-08-27T20:55:17.169 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 364 v 376 (ObjNum 2268 snap 0 seq_num 2268) dirty exists 2023-08-27T20:55:17.170 INFO:tasks.rados.rados.0.smithi093.stdout:3680: left oid 364 (ObjNum 2268 snap 0 seq_num 2268) 2023-08-27T20:55:17.170 INFO:tasks.rados.rados.0.smithi093.stdout:3681: expect (ObjNum 1817 snap 0 seq_num 1817) 2023-08-27T20:55:17.307 INFO:tasks.rados.rados.0.smithi093.stdout:3682: finishing write tid 1 to smithi093153278-53 2023-08-27T20:55:17.307 INFO:tasks.rados.rados.0.smithi093.stdout:3682: finishing write tid 2 to smithi093153278-53 2023-08-27T20:55:17.307 INFO:tasks.rados.rados.0.smithi093.stdout:3682: finishing write tid 3 to smithi093153278-53 2023-08-27T20:55:17.308 INFO:tasks.rados.rados.0.smithi093.stdout:3682: finishing write tid 4 to smithi093153278-53 2023-08-27T20:55:17.308 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 53 v 523 (ObjNum 2269 snap 0 seq_num 2269) dirty exists 2023-08-27T20:55:17.308 INFO:tasks.rados.rados.0.smithi093.stdout:3682: left oid 53 (ObjNum 2269 snap 0 seq_num 2269) 2023-08-27T20:55:17.308 INFO:tasks.rados.rados.0.smithi093.stdout:3680: done (3 left) 2023-08-27T20:55:17.308 INFO:tasks.rados.rados.0.smithi093.stdout:3681: done (2 left) 2023-08-27T20:55:17.308 INFO:tasks.rados.rados.0.smithi093.stdout:3682: done (1 left) 2023-08-27T20:55:17.308 INFO:tasks.rados.rados.0.smithi093.stdout:3684: done (0 left) 2023-08-27T20:55:17.309 INFO:tasks.rados.rados.0.smithi093.stdout:3685: delete oid 458 current snap is 0 2023-08-27T20:55:17.309 INFO:tasks.rados.rados.0.smithi093.stdout:3685: done (0 left) 2023-08-27T20:55:17.309 INFO:tasks.rados.rados.0.smithi093.stdout:3686: write oid 455 current snap is 0 2023-08-27T20:55:17.309 INFO:tasks.rados.rados.0.smithi093.stdout:3686: seq_num 2273 ranges {602311=514171,1710512=491799} 2023-08-27T20:55:17.315 INFO:tasks.rados.rados.0.smithi093.stdout:3686: writing smithi093153278-455 from 602311 to 1116482 tid 1 2023-08-27T20:55:17.321 INFO:tasks.rados.rados.0.smithi093.stdout:3686: writing smithi093153278-455 from 1710512 to 2202311 tid 2 2023-08-27T20:55:17.321 INFO:tasks.rados.rados.0.smithi093.stdout:3687: read oid 325 snap -1 2023-08-27T20:55:17.321 INFO:tasks.rados.rados.0.smithi093.stdout:3687: expect deleted 2023-08-27T20:55:17.322 INFO:tasks.rados.rados.0.smithi093.stdout:3688: write oid 169 current snap is 0 2023-08-27T20:55:17.322 INFO:tasks.rados.rados.0.smithi093.stdout:3686: finishing write tid 1 to smithi093153278-455 2023-08-27T20:55:17.322 INFO:tasks.rados.rados.0.smithi093.stdout:3688: seq_num 2274 ranges {619118=705773,1885347=333771} 2023-08-27T20:55:17.328 INFO:tasks.rados.rados.0.smithi093.stdout:3688: writing smithi093153278-169 from 619118 to 1324891 tid 1 2023-08-27T20:55:17.334 INFO:tasks.rados.rados.0.smithi093.stdout:3688: writing smithi093153278-169 from 1885347 to 2219118 tid 2 2023-08-27T20:55:17.334 INFO:tasks.rados.rados.0.smithi093.stdout:3689: read oid 27 snap -1 2023-08-27T20:55:17.334 INFO:tasks.rados.rados.0.smithi093.stdout:3689: expect (ObjNum 1920 snap 0 seq_num 1920) 2023-08-27T20:55:17.334 INFO:tasks.rados.rados.0.smithi093.stdout:3690: read oid 447 snap -1 2023-08-27T20:55:17.334 INFO:tasks.rados.rados.0.smithi093.stdout:3690: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:17.334 INFO:tasks.rados.rados.0.smithi093.stdout:3691: delete oid 249 current snap is 0 2023-08-27T20:55:17.334 INFO:tasks.rados.rados.0.smithi093.stdout:3686: finishing write tid 2 to smithi093153278-455 2023-08-27T20:55:17.335 INFO:tasks.rados.rados.0.smithi093.stdout:3686: finishing write tid 3 to smithi093153278-455 2023-08-27T20:55:17.335 INFO:tasks.rados.rados.0.smithi093.stdout:3686: finishing write tid 4 to smithi093153278-455 2023-08-27T20:55:17.335 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 455 v 526 (ObjNum 2273 snap 0 seq_num 2273) dirty exists 2023-08-27T20:55:17.335 INFO:tasks.rados.rados.0.smithi093.stdout:3686: left oid 455 (ObjNum 2273 snap 0 seq_num 2273) 2023-08-27T20:55:17.335 INFO:tasks.rados.rados.0.smithi093.stdout:3688: finishing write tid 1 to smithi093153278-169 2023-08-27T20:55:17.337 INFO:tasks.rados.rados.0.smithi093.stdout:3686: done (5 left) 2023-08-27T20:55:17.337 INFO:tasks.rados.rados.0.smithi093.stdout:3687: done (4 left) 2023-08-27T20:55:17.337 INFO:tasks.rados.rados.0.smithi093.stdout:3691: done (3 left) 2023-08-27T20:55:17.337 INFO:tasks.rados.rados.0.smithi093.stdout:3692: read oid 233 snap -1 2023-08-27T20:55:17.337 INFO:tasks.rados.rados.0.smithi093.stdout:3692: expect (ObjNum 2070 snap 0 seq_num 2070) 2023-08-27T20:55:17.337 INFO:tasks.rados.rados.0.smithi093.stdout:3693: write oid 444 current snap is 0 2023-08-27T20:55:17.338 INFO:tasks.rados.rados.0.smithi093.stdout:3693: seq_num 2276 ranges {652732=456271,1885958=366774} 2023-08-27T20:55:17.342 INFO:tasks.rados.rados.0.smithi093.stdout:3693: writing smithi093153278-444 from 652732 to 1109003 tid 1 2023-08-27T20:55:17.349 INFO:tasks.rados.rados.0.smithi093.stdout:3693: writing smithi093153278-444 from 1885958 to 2252732 tid 2 2023-08-27T20:55:17.349 INFO:tasks.rados.rados.0.smithi093.stdout:3694: write oid 181 current snap is 0 2023-08-27T20:55:17.349 INFO:tasks.rados.rados.0.smithi093.stdout:3694: seq_num 2277 ranges {669539=531520,1944440=325099} 2023-08-27T20:55:17.355 INFO:tasks.rados.rados.0.smithi093.stdout:3694: writing smithi093153278-181 from 669539 to 1201059 tid 1 2023-08-27T20:55:17.361 INFO:tasks.rados.rados.0.smithi093.stdout:3694: writing smithi093153278-181 from 1944440 to 2269539 tid 2 2023-08-27T20:55:17.361 INFO:tasks.rados.rados.0.smithi093.stdout:3695: write oid 474 current snap is 0 2023-08-27T20:55:17.361 INFO:tasks.rados.rados.0.smithi093.stdout:3695: seq_num 2278 ranges {686346=606769,2002922=283424} 2023-08-27T20:55:17.368 INFO:tasks.rados.rados.0.smithi093.stdout:3695: writing smithi093153278-474 from 686346 to 1293115 tid 1 2023-08-27T20:55:17.374 INFO:tasks.rados.rados.0.smithi093.stdout:3695: writing smithi093153278-474 from 2002922 to 2286346 tid 2 2023-08-27T20:55:17.374 INFO:tasks.rados.rados.0.smithi093.stdout:3696: read oid 361 snap -1 2023-08-27T20:55:17.374 INFO:tasks.rados.rados.0.smithi093.stdout:3696: expect (ObjNum 2199 snap 0 seq_num 2199) 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3697: read oid 14 snap -1 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3697: expect (ObjNum 2182 snap 0 seq_num 2182) 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3698: read oid 150 snap -1 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3698: expect (ObjNum 2063 snap 0 seq_num 2063) 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3699: read oid 290 snap -1 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3699: expect (ObjNum 2131 snap 0 seq_num 2131) 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3700: write oid 464 current snap is 0 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3688: finishing write tid 2 to smithi093153278-169 2023-08-27T20:55:17.375 INFO:tasks.rados.rados.0.smithi093.stdout:3689: expect (ObjNum 1920 snap 0 seq_num 1920) 2023-08-27T20:55:17.444 INFO:tasks.rados.rados.0.smithi093.stdout:3690: expect (ObjNum 1684 snap 0 seq_num 1684) 2023-08-27T20:55:17.560 INFO:tasks.rados.rados.0.smithi093.stdout:3688: finishing write tid 3 to smithi093153278-169 2023-08-27T20:55:17.560 INFO:tasks.rados.rados.0.smithi093.stdout:3688: finishing write tid 4 to smithi093153278-169 2023-08-27T20:55:17.560 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 169 v 457 (ObjNum 2274 snap 0 seq_num 2274) dirty exists 2023-08-27T20:55:17.560 INFO:tasks.rados.rados.0.smithi093.stdout:3688: left oid 169 (ObjNum 2274 snap 0 seq_num 2274) 2023-08-27T20:55:17.560 INFO:tasks.rados.rados.0.smithi093.stdout:3692: expect (ObjNum 2070 snap 0 seq_num 2070) 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:3693: finishing write tid 1 to smithi093153278-444 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:3693: finishing write tid 2 to smithi093153278-444 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:3693: finishing write tid 3 to smithi093153278-444 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:3693: finishing write tid 4 to smithi093153278-444 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 444 v 480 (ObjNum 2276 snap 0 seq_num 2276) dirty exists 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:3693: left oid 444 (ObjNum 2276 snap 0 seq_num 2276) 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:3694: finishing write tid 1 to smithi093153278-181 2023-08-27T20:55:17.676 INFO:tasks.rados.rados.0.smithi093.stdout:3700: seq_num 2279 ranges {703153=682018,2061404=241749} 2023-08-27T20:55:17.683 INFO:tasks.rados.rados.0.smithi093.stdout:3700: writing smithi093153278-464 from 703153 to 1385171 tid 1 2023-08-27T20:55:17.689 INFO:tasks.rados.rados.0.smithi093.stdout:3700: writing smithi093153278-464 from 2061404 to 2303153 tid 2 2023-08-27T20:55:17.689 INFO:tasks.rados.rados.0.smithi093.stdout:3688: done (11 left) 2023-08-27T20:55:17.689 INFO:tasks.rados.rados.0.smithi093.stdout:3689: done (10 left) 2023-08-27T20:55:17.689 INFO:tasks.rados.rados.0.smithi093.stdout:3690: done (9 left) 2023-08-27T20:55:17.689 INFO:tasks.rados.rados.0.smithi093.stdout:3692: done (8 left) 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3693: done (7 left) 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3701: write oid 499 current snap is 0 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3694: finishing write tid 2 to smithi093153278-181 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3694: finishing write tid 3 to smithi093153278-181 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3694: finishing write tid 4 to smithi093153278-181 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 181 v 423 (ObjNum 2277 snap 0 seq_num 2277) dirty exists 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3694: left oid 181 (ObjNum 2277 snap 0 seq_num 2277) 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3695: finishing write tid 1 to smithi093153278-474 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3695: finishing write tid 2 to smithi093153278-474 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3695: finishing write tid 3 to smithi093153278-474 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:3695: finishing write tid 4 to smithi093153278-474 2023-08-27T20:55:17.690 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 474 v 426 (ObjNum 2278 snap 0 seq_num 2278) dirty exists 2023-08-27T20:55:17.691 INFO:tasks.rados.rados.0.smithi093.stdout:3695: left oid 474 (ObjNum 2278 snap 0 seq_num 2278) 2023-08-27T20:55:17.691 INFO:tasks.rados.rados.0.smithi093.stdout:3698: expect (ObjNum 2063 snap 0 seq_num 2063) 2023-08-27T20:55:17.771 INFO:tasks.rados.rados.0.smithi093.stdout:3696: expect (ObjNum 2199 snap 0 seq_num 2199) 2023-08-27T20:55:17.897 INFO:tasks.rados.rados.0.smithi093.stdout:3697: expect (ObjNum 2182 snap 0 seq_num 2182) 2023-08-27T20:55:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:17 smithi136 ceph-mon[113531]: pgmap v52: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s rd, 11 MiB/s wr, 115 op/s 2023-08-27T20:55:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1242592158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:17 smithi093 ceph-mon[118902]: pgmap v52: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s rd, 11 MiB/s wr, 115 op/s 2023-08-27T20:55:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1242592158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:17 smithi093 ceph-mon[111335]: pgmap v52: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 88 MiB/s rd, 11 MiB/s wr, 115 op/s 2023-08-27T20:55:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1242592158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:18.054 INFO:tasks.rados.rados.0.smithi093.stdout:3699: expect (ObjNum 2131 snap 0 seq_num 2131) 2023-08-27T20:55:18.263 INFO:tasks.rados.rados.0.smithi093.stdout:3700: finishing write tid 1 to smithi093153278-464 2023-08-27T20:55:18.264 INFO:tasks.rados.rados.0.smithi093.stdout:3700: finishing write tid 2 to smithi093153278-464 2023-08-27T20:55:18.264 INFO:tasks.rados.rados.0.smithi093.stdout:3700: finishing write tid 3 to smithi093153278-464 2023-08-27T20:55:18.264 INFO:tasks.rados.rados.0.smithi093.stdout:3700: finishing write tid 4 to smithi093153278-464 2023-08-27T20:55:18.264 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 464 v 429 (ObjNum 2279 snap 0 seq_num 2279) dirty exists 2023-08-27T20:55:18.264 INFO:tasks.rados.rados.0.smithi093.stdout:3700: left oid 464 (ObjNum 2279 snap 0 seq_num 2279) 2023-08-27T20:55:18.264 INFO:tasks.rados.rados.0.smithi093.stdout:3701: seq_num 2280 ranges {719960=757267,2003533=316427} 2023-08-27T20:55:18.272 INFO:tasks.rados.rados.0.smithi093.stdout:3701: writing smithi093153278-499 from 719960 to 1477227 tid 1 2023-08-27T20:55:18.277 INFO:tasks.rados.rados.0.smithi093.stdout:3701: writing smithi093153278-499 from 2003533 to 2319960 tid 2 2023-08-27T20:55:18.277 INFO:tasks.rados.rados.0.smithi093.stdout:3694: done (7 left) 2023-08-27T20:55:18.277 INFO:tasks.rados.rados.0.smithi093.stdout:3695: done (6 left) 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3696: done (5 left) 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3697: done (4 left) 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3698: done (3 left) 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3699: done (2 left) 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3700: done (1 left) 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3702: read oid 461 snap -1 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3702: expect deleted 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3703: read oid 141 snap -1 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3703: expect (ObjNum 1625 snap 0 seq_num 1625) 2023-08-27T20:55:18.278 INFO:tasks.rados.rados.0.smithi093.stdout:3704: write oid 77 current snap is 0 2023-08-27T20:55:18.279 INFO:tasks.rados.rados.0.smithi093.stdout:3701: finishing write tid 1 to smithi093153278-499 2023-08-27T20:55:18.279 INFO:tasks.rados.rados.0.smithi093.stdout:3704: seq_num 2281 ranges {736767=548869,1778368=558399} 2023-08-27T20:55:18.284 INFO:tasks.rados.rados.0.smithi093.stdout:3704: writing smithi093153278-77 from 736767 to 1285636 tid 1 2023-08-27T20:55:18.290 INFO:tasks.rados.rados.0.smithi093.stdout:3704: writing smithi093153278-77 from 1778368 to 2336767 tid 2 2023-08-27T20:55:18.290 INFO:tasks.rados.rados.0.smithi093.stdout:3705: write oid 488 current snap is 0 2023-08-27T20:55:18.290 INFO:tasks.rados.rados.0.smithi093.stdout:3705: seq_num 2282 ranges {753574=624118,1836850=516724} 2023-08-27T20:55:18.297 INFO:tasks.rados.rados.0.smithi093.stdout:3705: writing smithi093153278-488 from 753574 to 1377692 tid 1 2023-08-27T20:55:18.302 INFO:tasks.rados.rados.0.smithi093.stdout:3705: writing smithi093153278-488 from 1836850 to 2353574 tid 2 2023-08-27T20:55:18.303 INFO:tasks.rados.rados.0.smithi093.stdout:3706: write oid 500 current snap is 0 2023-08-27T20:55:18.303 INFO:tasks.rados.rados.0.smithi093.stdout:3706: seq_num 2283 ranges {770381=699367,1895332=475049} 2023-08-27T20:55:18.310 INFO:tasks.rados.rados.0.smithi093.stdout:3706: writing smithi093153278-500 from 770381 to 1469748 tid 1 2023-08-27T20:55:18.316 INFO:tasks.rados.rados.0.smithi093.stdout:3706: writing smithi093153278-500 from 1895332 to 2370381 tid 2 2023-08-27T20:55:18.316 INFO:tasks.rados.rados.0.smithi093.stdout:3707: read oid 297 snap -1 2023-08-27T20:55:18.316 INFO:tasks.rados.rados.0.smithi093.stdout:3707: expect (ObjNum 1929 snap 0 seq_num 1929) 2023-08-27T20:55:18.316 INFO:tasks.rados.rados.0.smithi093.stdout:3708: delete oid 253 current snap is 0 2023-08-27T20:55:18.316 INFO:tasks.rados.rados.0.smithi093.stdout:3701: finishing write tid 2 to smithi093153278-499 2023-08-27T20:55:18.317 INFO:tasks.rados.rados.0.smithi093.stdout:3701: finishing write tid 3 to smithi093153278-499 2023-08-27T20:55:18.317 INFO:tasks.rados.rados.0.smithi093.stdout:3701: finishing write tid 4 to smithi093153278-499 2023-08-27T20:55:18.317 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 499 v 412 (ObjNum 2280 snap 0 seq_num 2280) dirty exists 2023-08-27T20:55:18.317 INFO:tasks.rados.rados.0.smithi093.stdout:3701: left oid 499 (ObjNum 2280 snap 0 seq_num 2280) 2023-08-27T20:55:18.317 INFO:tasks.rados.rados.0.smithi093.stdout:3703: expect (ObjNum 1625 snap 0 seq_num 1625) 2023-08-27T20:55:18.490 INFO:tasks.rados.rados.0.smithi093.stdout:3704: finishing write tid 1 to smithi093153278-77 2023-08-27T20:55:18.490 INFO:tasks.rados.rados.0.smithi093.stdout:3704: finishing write tid 2 to smithi093153278-77 2023-08-27T20:55:18.490 INFO:tasks.rados.rados.0.smithi093.stdout:3704: finishing write tid 3 to smithi093153278-77 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3704: finishing write tid 4 to smithi093153278-77 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 77 v 395 (ObjNum 2281 snap 0 seq_num 2281) dirty exists 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3704: left oid 77 (ObjNum 2281 snap 0 seq_num 2281) 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3701: done (7 left) 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3702: done (6 left) 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3703: done (5 left) 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3704: done (4 left) 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3708: done (3 left) 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3709: write oid 312 current snap is 0 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3705: finishing write tid 1 to smithi093153278-488 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3705: finishing write tid 2 to smithi093153278-488 2023-08-27T20:55:18.491 INFO:tasks.rados.rados.0.smithi093.stdout:3705: finishing write tid 3 to smithi093153278-488 2023-08-27T20:55:18.492 INFO:tasks.rados.rados.0.smithi093.stdout:3709: seq_num 2285 ranges {403995=449865,1495943=596474} 2023-08-27T20:55:18.494 INFO:tasks.rados.rados.0.smithi093.stdout:3709: writing smithi093153278-312 from 403995 to 853860 tid 1 2023-08-27T20:55:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:3709: writing smithi093153278-312 from 1495943 to 2092417 tid 2 2023-08-27T20:55:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:3710: write oid 131 current snap is 0 2023-08-27T20:55:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:3705: finishing write tid 4 to smithi093153278-488 2023-08-27T20:55:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 488 v 535 (ObjNum 2282 snap 0 seq_num 2282) dirty exists 2023-08-27T20:55:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:3705: left oid 488 (ObjNum 2282 snap 0 seq_num 2282) 2023-08-27T20:55:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:3706: finishing write tid 1 to smithi093153278-500 2023-08-27T20:55:18.502 INFO:tasks.rados.rados.0.smithi093.stdout:3710: seq_num 2286 ranges {420802=525114,1554425=740132} 2023-08-27T20:55:18.506 INFO:tasks.rados.rados.0.smithi093.stdout:3710: writing smithi093153278-131 from 420802 to 945916 tid 1 2023-08-27T20:55:18.514 INFO:tasks.rados.rados.0.smithi093.stdout:3710: writing smithi093153278-131 from 1554425 to 2294557 tid 2 2023-08-27T20:55:18.514 INFO:tasks.rados.rados.0.smithi093.stdout:3705: done (4 left) 2023-08-27T20:55:18.514 INFO:tasks.rados.rados.0.smithi093.stdout:3711: read oid 377 snap -1 2023-08-27T20:55:18.514 INFO:tasks.rados.rados.0.smithi093.stdout:3711: expect (ObjNum 2246 snap 0 seq_num 2246) 2023-08-27T20:55:18.514 INFO:tasks.rados.rados.0.smithi093.stdout:3712: write oid 441 current snap is 0 2023-08-27T20:55:18.514 INFO:tasks.rados.rados.0.smithi093.stdout:3707: expect (ObjNum 1929 snap 0 seq_num 1929) 2023-08-27T20:55:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:3706: finishing write tid 2 to smithi093153278-500 2023-08-27T20:55:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:3706: finishing write tid 3 to smithi093153278-500 2023-08-27T20:55:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:3706: finishing write tid 4 to smithi093153278-500 2023-08-27T20:55:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 500 v 529 (ObjNum 2283 snap 0 seq_num 2283) dirty exists 2023-08-27T20:55:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:3706: left oid 500 (ObjNum 2283 snap 0 seq_num 2283) 2023-08-27T20:55:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:3709: finishing write tid 1 to smithi093153278-312 2023-08-27T20:55:18.588 INFO:tasks.rados.rados.0.smithi093.stdout:3709: finishing write tid 2 to smithi093153278-312 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3709: finishing write tid 3 to smithi093153278-312 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3709: finishing write tid 4 to smithi093153278-312 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 312 v 429 (ObjNum 2285 snap 0 seq_num 2285) dirty exists 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3709: left oid 312 (ObjNum 2285 snap 0 seq_num 2285) 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3710: finishing write tid 1 to smithi093153278-131 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3710: finishing write tid 2 to smithi093153278-131 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3710: finishing write tid 3 to smithi093153278-131 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3710: finishing write tid 4 to smithi093153278-131 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 131 v 396 (ObjNum 2286 snap 0 seq_num 2286) dirty exists 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3710: left oid 131 (ObjNum 2286 snap 0 seq_num 2286) 2023-08-27T20:55:18.589 INFO:tasks.rados.rados.0.smithi093.stdout:3711: expect (ObjNum 2246 snap 0 seq_num 2246) 2023-08-27T20:55:18.753 INFO:tasks.rados.rados.0.smithi093.stdout:3712: seq_num 2287 ranges {437609=600363,1612907=483790} 2023-08-27T20:55:18.759 INFO:tasks.rados.rados.0.smithi093.stdout:3712: writing smithi093153278-441 from 437609 to 1037972 tid 1 2023-08-27T20:55:18.765 INFO:tasks.rados.rados.0.smithi093.stdout:3712: writing smithi093153278-441 from 1612907 to 2096697 tid 2 2023-08-27T20:55:18.765 INFO:tasks.rados.rados.0.smithi093.stdout:3706: done (5 left) 2023-08-27T20:55:18.765 INFO:tasks.rados.rados.0.smithi093.stdout:3707: done (4 left) 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3709: done (3 left) 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3710: done (2 left) 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3711: done (1 left) 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3713: read oid 191 snap -1 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3713: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3714: write oid 188 current snap is 0 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3712: finishing write tid 1 to smithi093153278-441 2023-08-27T20:55:18.766 INFO:tasks.rados.rados.0.smithi093.stdout:3714: seq_num 2288 ranges {454416=675612,1555036=743801} 2023-08-27T20:55:18.771 INFO:tasks.rados.rados.0.smithi093.stdout:3714: writing smithi093153278-188 from 454416 to 1130028 tid 1 2023-08-27T20:55:18.778 INFO:tasks.rados.rados.0.smithi093.stdout:3714: writing smithi093153278-188 from 1555036 to 2298837 tid 2 2023-08-27T20:55:18.778 INFO:tasks.rados.rados.0.smithi093.stdout:3715: read oid 315 snap -1 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3715: expect (ObjNum 2033 snap 0 seq_num 2033) 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3716: read oid 421 snap -1 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3716: expect (ObjNum 1387 snap 0 seq_num 1387) 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3717: read oid 328 snap -1 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3717: expect (ObjNum 2238 snap 0 seq_num 2238) 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3718: write oid 211 current snap is 0 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3712: finishing write tid 2 to smithi093153278-441 2023-08-27T20:55:18.779 INFO:tasks.rados.rados.0.smithi093.stdout:3718: seq_num 2289 ranges {471223=467214,1729871=487459} 2023-08-27T20:55:18.783 INFO:tasks.rados.rados.0.smithi093.stdout:3718: writing smithi093153278-211 from 471223 to 938437 tid 1 2023-08-27T20:55:18.791 INFO:tasks.rados.rados.0.smithi093.stdout:3718: writing smithi093153278-211 from 1729871 to 2217330 tid 2 2023-08-27T20:55:18.791 INFO:tasks.rados.rados.0.smithi093.stdout:3719: read oid 478 snap -1 2023-08-27T20:55:18.791 INFO:tasks.rados.rados.0.smithi093.stdout:3719: expect (ObjNum 2139 snap 0 seq_num 2139) 2023-08-27T20:55:18.791 INFO:tasks.rados.rados.0.smithi093.stdout:3720: delete oid 138 current snap is 0 2023-08-27T20:55:18.792 INFO:tasks.rados.rados.0.smithi093.stdout:3712: finishing write tid 3 to smithi093153278-441 2023-08-27T20:55:18.792 INFO:tasks.rados.rados.0.smithi093.stdout:3712: finishing write tid 4 to smithi093153278-441 2023-08-27T20:55:18.792 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 441 v 398 (ObjNum 2287 snap 0 seq_num 2287) dirty exists 2023-08-27T20:55:18.792 INFO:tasks.rados.rados.0.smithi093.stdout:3712: left oid 441 (ObjNum 2287 snap 0 seq_num 2287) 2023-08-27T20:55:18.792 INFO:tasks.rados.rados.0.smithi093.stdout:3713: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:55:18.907 INFO:tasks.rados.rados.0.smithi093.stdout:3714: finishing write tid 1 to smithi093153278-188 2023-08-27T20:55:18.908 INFO:tasks.rados.rados.0.smithi093.stdout:3714: finishing write tid 2 to smithi093153278-188 2023-08-27T20:55:18.908 INFO:tasks.rados.rados.0.smithi093.stdout:3714: finishing write tid 3 to smithi093153278-188 2023-08-27T20:55:18.908 INFO:tasks.rados.rados.0.smithi093.stdout:3714: finishing write tid 4 to smithi093153278-188 2023-08-27T20:55:18.908 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 188 v 538 (ObjNum 2288 snap 0 seq_num 2288) dirty exists 2023-08-27T20:55:18.908 INFO:tasks.rados.rados.0.smithi093.stdout:3714: left oid 188 (ObjNum 2288 snap 0 seq_num 2288) 2023-08-27T20:55:18.908 INFO:tasks.rados.rados.0.smithi093.stdout:3716: expect (ObjNum 1387 snap 0 seq_num 1387) 2023-08-27T20:55:19.143 INFO:tasks.rados.rados.0.smithi093.stdout:3715: expect (ObjNum 2033 snap 0 seq_num 2033) 2023-08-27T20:55:19.284 INFO:tasks.rados.rados.0.smithi093.stdout:3718: finishing write tid 1 to smithi093153278-211 2023-08-27T20:55:19.284 INFO:tasks.rados.rados.0.smithi093.stdout:3718: finishing write tid 2 to smithi093153278-211 2023-08-27T20:55:19.284 INFO:tasks.rados.rados.0.smithi093.stdout:3718: finishing write tid 3 to smithi093153278-211 2023-08-27T20:55:19.284 INFO:tasks.rados.rados.0.smithi093.stdout:3717: expect (ObjNum 2238 snap 0 seq_num 2238) 2023-08-27T20:55:19.400 INFO:tasks.rados.rados.0.smithi093.stdout:3718: finishing write tid 4 to smithi093153278-211 2023-08-27T20:55:19.400 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 211 v 590 (ObjNum 2289 snap 0 seq_num 2289) dirty exists 2023-08-27T20:55:19.400 INFO:tasks.rados.rados.0.smithi093.stdout:3718: left oid 211 (ObjNum 2289 snap 0 seq_num 2289) 2023-08-27T20:55:19.400 INFO:tasks.rados.rados.0.smithi093.stdout:3719: expect (ObjNum 2139 snap 0 seq_num 2139) 2023-08-27T20:55:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:3712: done (8 left) 2023-08-27T20:55:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:3713: done (7 left) 2023-08-27T20:55:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:3714: done (6 left) 2023-08-27T20:55:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:3715: done (5 left) 2023-08-27T20:55:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:3716: done (4 left) 2023-08-27T20:55:19.527 INFO:tasks.rados.rados.0.smithi093.stdout:3717: done (3 left) 2023-08-27T20:55:19.528 INFO:tasks.rados.rados.0.smithi093.stdout:3718: done (2 left) 2023-08-27T20:55:19.528 INFO:tasks.rados.rados.0.smithi093.stdout:3719: done (1 left) 2023-08-27T20:55:19.528 INFO:tasks.rados.rados.0.smithi093.stdout:3720: done (0 left) 2023-08-27T20:55:19.528 INFO:tasks.rados.rados.0.smithi093.stdout:3721: write oid 244 current snap is 0 2023-08-27T20:55:19.528 INFO:tasks.rados.rados.0.smithi093.stdout:3721: seq_num 2291 ranges {504837=617712,1846835=491128} 2023-08-27T20:55:19.533 INFO:tasks.rados.rados.0.smithi093.stdout:3721: writing smithi093153278-244 from 504837 to 1122549 tid 1 2023-08-27T20:55:19.540 INFO:tasks.rados.rados.0.smithi093.stdout:3721: writing smithi093153278-244 from 1846835 to 2337963 tid 2 2023-08-27T20:55:19.540 INFO:tasks.rados.rados.0.smithi093.stdout:3722: read oid 53 snap -1 2023-08-27T20:55:19.540 INFO:tasks.rados.rados.0.smithi093.stdout:3722: expect (ObjNum 2269 snap 0 seq_num 2269) 2023-08-27T20:55:19.540 INFO:tasks.rados.rados.0.smithi093.stdout:3723: write oid 46 current snap is 0 2023-08-27T20:55:19.541 INFO:tasks.rados.rados.0.smithi093.stdout:3721: finishing write tid 1 to smithi093153278-244 2023-08-27T20:55:19.541 INFO:tasks.rados.rados.0.smithi093.stdout:3723: seq_num 2292 ranges {521644=692961,1788964=732680} 2023-08-27T20:55:19.546 INFO:tasks.rados.rados.0.smithi093.stdout:3723: writing smithi093153278-46 from 521644 to 1214605 tid 1 2023-08-27T20:55:19.554 INFO:tasks.rados.rados.0.smithi093.stdout:3723: writing smithi093153278-46 from 1788964 to 2521644 tid 2 2023-08-27T20:55:19.554 INFO:tasks.rados.rados.0.smithi093.stdout:3724: read oid 196 snap -1 2023-08-27T20:55:19.554 INFO:tasks.rados.rados.0.smithi093.stdout:3724: expect deleted 2023-08-27T20:55:19.554 INFO:tasks.rados.rados.0.smithi093.stdout:3725: read oid 429 snap -1 2023-08-27T20:55:19.554 INFO:tasks.rados.rados.0.smithi093.stdout:3725: expect deleted 2023-08-27T20:55:19.554 INFO:tasks.rados.rados.0.smithi093.stdout:3726: write oid 307 current snap is 0 2023-08-27T20:55:19.555 INFO:tasks.rados.rados.0.smithi093.stdout:3726: seq_num 2293 ranges {538451=768210,1847446=494797} 2023-08-27T20:55:19.561 INFO:tasks.rados.rados.0.smithi093.stdout:3726: writing smithi093153278-307 from 538451 to 1306661 tid 1 2023-08-27T20:55:19.567 INFO:tasks.rados.rados.0.smithi093.stdout:3726: writing smithi093153278-307 from 1847446 to 2342243 tid 2 2023-08-27T20:55:19.567 INFO:tasks.rados.rados.0.smithi093.stdout:3727: read oid 484 snap -1 2023-08-27T20:55:19.567 INFO:tasks.rados.rados.0.smithi093.stdout:3727: expect deleted 2023-08-27T20:55:19.567 INFO:tasks.rados.rados.0.smithi093.stdout:3728: write oid 36 current snap is 0 2023-08-27T20:55:19.568 INFO:tasks.rados.rados.0.smithi093.stdout:3728: seq_num 2294 ranges {555258=443459,1505928=754808} 2023-08-27T20:55:19.572 INFO:tasks.rados.rados.0.smithi093.stdout:3728: writing smithi093153278-36 from 555258 to 998717 tid 1 2023-08-27T20:55:19.580 INFO:tasks.rados.rados.0.smithi093.stdout:3728: writing smithi093153278-36 from 1505928 to 2260736 tid 2 2023-08-27T20:55:19.580 INFO:tasks.rados.rados.0.smithi093.stdout:3729: read oid 324 snap -1 2023-08-27T20:55:19.580 INFO:tasks.rados.rados.0.smithi093.stdout:3729: expect deleted 2023-08-27T20:55:19.580 INFO:tasks.rados.rados.0.smithi093.stdout:3730: read oid 395 snap -1 2023-08-27T20:55:19.580 INFO:tasks.rados.rados.0.smithi093.stdout:3730: expect (ObjNum 2191 snap 0 seq_num 2191) 2023-08-27T20:55:19.580 INFO:tasks.rados.rados.0.smithi093.stdout:3731: write oid 311 current snap is 0 2023-08-27T20:55:19.581 INFO:tasks.rados.rados.0.smithi093.stdout:3722: expect (ObjNum 2269 snap 0 seq_num 2269) 2023-08-27T20:55:19.680 INFO:tasks.rados.rados.0.smithi093.stdout:3721: finishing write tid 2 to smithi093153278-244 2023-08-27T20:55:19.680 INFO:tasks.rados.rados.0.smithi093.stdout:3721: finishing write tid 3 to smithi093153278-244 2023-08-27T20:55:19.680 INFO:tasks.rados.rados.0.smithi093.stdout:3721: finishing write tid 4 to smithi093153278-244 2023-08-27T20:55:19.680 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 244 v 287 (ObjNum 2291 snap 0 seq_num 2291) dirty exists 2023-08-27T20:55:19.680 INFO:tasks.rados.rados.0.smithi093.stdout:3721: left oid 244 (ObjNum 2291 snap 0 seq_num 2291) 2023-08-27T20:55:19.680 INFO:tasks.rados.rados.0.smithi093.stdout:3723: finishing write tid 1 to smithi093153278-46 2023-08-27T20:55:19.681 INFO:tasks.rados.rados.0.smithi093.stdout:3731: seq_num 2295 ranges {572065=518708,1564410=498466} 2023-08-27T20:55:19.685 INFO:tasks.rados.rados.0.smithi093.stdout:3731: writing smithi093153278-311 from 572065 to 1090773 tid 1 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3731: writing smithi093153278-311 from 1564410 to 2062876 tid 2 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3721: done (10 left) 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3722: done (9 left) 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3732: read oid 69 snap -1 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3732: expect (ObjNum 1710 snap 0 seq_num 1710) 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3733: write oid 326 current snap is 0 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3723: finishing write tid 2 to smithi093153278-46 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3723: finishing write tid 3 to smithi093153278-46 2023-08-27T20:55:19.692 INFO:tasks.rados.rados.0.smithi093.stdout:3733: seq_num 2296 ranges {588872=710310,2022892=565980} 2023-08-27T20:55:19.698 INFO:tasks.rados.rados.0.smithi093.stdout:3733: writing smithi093153278-326 from 588872 to 1299182 tid 1 2023-08-27T20:55:19.706 INFO:tasks.rados.rados.0.smithi093.stdout:3733: writing smithi093153278-326 from 2022892 to 2588872 tid 2 2023-08-27T20:55:19.706 INFO:tasks.rados.rados.0.smithi093.stdout:3724: done (10 left) 2023-08-27T20:55:19.706 INFO:tasks.rados.rados.0.smithi093.stdout:3725: done (9 left) 2023-08-27T20:55:19.706 INFO:tasks.rados.rados.0.smithi093.stdout:3734: read oid 115 snap -1 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:3734: expect deleted 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:3735: write oid 383 current snap is 0 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:3723: finishing write tid 4 to smithi093153278-46 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 46 v 593 (ObjNum 2292 snap 0 seq_num 2292) dirty exists 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:3723: left oid 46 (ObjNum 2292 snap 0 seq_num 2292) 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:3726: finishing write tid 1 to smithi093153278-307 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:3726: finishing write tid 2 to smithi093153278-307 2023-08-27T20:55:19.707 INFO:tasks.rados.rados.0.smithi093.stdout:3735: seq_num 2297 ranges {605679=785559,2081374=502135} 2023-08-27T20:55:19.714 INFO:tasks.rados.rados.0.smithi093.stdout:3735: writing smithi093153278-383 from 605679 to 1391238 tid 1 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3735: writing smithi093153278-383 from 2081374 to 2583509 tid 2 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3723: done (10 left) 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3727: done (9 left) 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3736: read oid 240 snap -1 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3736: expect deleted 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3737: write oid 377 current snap is 0 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3726: finishing write tid 3 to smithi093153278-307 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:3726: finishing write tid 4 to smithi093153278-307 2023-08-27T20:55:19.721 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 307 v 449 (ObjNum 2293 snap 0 seq_num 2293) dirty exists 2023-08-27T20:55:19.722 INFO:tasks.rados.rados.0.smithi093.stdout:3726: left oid 307 (ObjNum 2293 snap 0 seq_num 2293) 2023-08-27T20:55:19.722 INFO:tasks.rados.rados.0.smithi093.stdout:3728: finishing write tid 1 to smithi093153278-36 2023-08-27T20:55:19.722 INFO:tasks.rados.rados.0.smithi093.stdout:3737: seq_num 2298 ranges {622486=460808,1739856=645793} 2023-08-27T20:55:19.726 INFO:tasks.rados.rados.0.smithi093.stdout:3737: writing smithi093153278-377 from 622486 to 1083294 tid 1 2023-08-27T20:55:19.734 INFO:tasks.rados.rados.0.smithi093.stdout:3737: writing smithi093153278-377 from 1739856 to 2385649 tid 2 2023-08-27T20:55:19.734 INFO:tasks.rados.rados.0.smithi093.stdout:3726: done (10 left) 2023-08-27T20:55:19.734 INFO:tasks.rados.rados.0.smithi093.stdout:3738: write oid 166 current snap is 0 2023-08-27T20:55:19.734 INFO:tasks.rados.rados.0.smithi093.stdout:3738: seq_num 2299 ranges {639293=536057,1798338=505804} 2023-08-27T20:55:19.740 INFO:tasks.rados.rados.0.smithi093.stdout:3738: writing smithi093153278-166 from 639293 to 1175350 tid 1 2023-08-27T20:55:19.746 INFO:tasks.rados.rados.0.smithi093.stdout:3738: writing smithi093153278-166 from 1798338 to 2304142 tid 2 2023-08-27T20:55:19.747 INFO:tasks.rados.rados.0.smithi093.stdout:3739: write oid 27 current snap is 0 2023-08-27T20:55:19.747 INFO:tasks.rados.rados.0.smithi093.stdout:3728: finishing write tid 2 to smithi093153278-36 2023-08-27T20:55:19.747 INFO:tasks.rados.rados.0.smithi093.stdout:3728: finishing write tid 3 to smithi093153278-36 2023-08-27T20:55:19.747 INFO:tasks.rados.rados.0.smithi093.stdout:3728: finishing write tid 4 to smithi093153278-36 2023-08-27T20:55:19.747 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 36 v 460 (ObjNum 2294 snap 0 seq_num 2294) dirty exists 2023-08-27T20:55:19.747 INFO:tasks.rados.rados.0.smithi093.stdout:3728: left oid 36 (ObjNum 2294 snap 0 seq_num 2294) 2023-08-27T20:55:19.747 INFO:tasks.rados.rados.0.smithi093.stdout:3730: expect (ObjNum 2191 snap 0 seq_num 2191) 2023-08-27T20:55:19.830 INFO:tasks.rados.rados.0.smithi093.stdout:3731: finishing write tid 1 to smithi093153278-311 2023-08-27T20:55:19.831 INFO:tasks.rados.rados.0.smithi093.stdout:3731: finishing write tid 2 to smithi093153278-311 2023-08-27T20:55:19.831 INFO:tasks.rados.rados.0.smithi093.stdout:3731: finishing write tid 3 to smithi093153278-311 2023-08-27T20:55:19.831 INFO:tasks.rados.rados.0.smithi093.stdout:3731: finishing write tid 4 to smithi093153278-311 2023-08-27T20:55:19.831 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 311 v 399 (ObjNum 2295 snap 0 seq_num 2295) dirty exists 2023-08-27T20:55:19.831 INFO:tasks.rados.rados.0.smithi093.stdout:3731: left oid 311 (ObjNum 2295 snap 0 seq_num 2295) 2023-08-27T20:55:19.831 INFO:tasks.rados.rados.0.smithi093.stdout:3732: expect (ObjNum 1710 snap 0 seq_num 1710) 2023-08-27T20:55:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:19 smithi136 ceph-mon[113531]: pgmap v53: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 8.2 MiB/s wr, 81 op/s 2023-08-27T20:55:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3733983757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:3733: finishing write tid 1 to smithi093153278-326 2023-08-27T20:55:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:3733: finishing write tid 2 to smithi093153278-326 2023-08-27T20:55:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:3733: finishing write tid 3 to smithi093153278-326 2023-08-27T20:55:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:3733: finishing write tid 4 to smithi093153278-326 2023-08-27T20:55:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 326 v 379 (ObjNum 2296 snap 0 seq_num 2296) dirty exists 2023-08-27T20:55:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:3733: left oid 326 (ObjNum 2296 snap 0 seq_num 2296) 2023-08-27T20:55:19.975 INFO:tasks.rados.rados.0.smithi093.stdout:3735: finishing write tid 1 to smithi093153278-383 2023-08-27T20:55:19.976 INFO:tasks.rados.rados.0.smithi093.stdout:3739: seq_num 2300 ranges {656100=611306,1740467=649462} 2023-08-27T20:55:19.981 INFO:tasks.rados.rados.0.smithi093.stdout:3739: writing smithi093153278-27 from 656100 to 1267406 tid 1 2023-08-27T20:55:19.988 INFO:tasks.rados.rados.0.smithi093.stdout:3739: writing smithi093153278-27 from 1740467 to 2389929 tid 2 2023-08-27T20:55:19.988 INFO:tasks.rados.rados.0.smithi093.stdout:3728: done (11 left) 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3729: done (10 left) 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3730: done (9 left) 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3731: done (8 left) 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3732: done (7 left) 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3733: done (6 left) 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3734: done (5 left) 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3740: write oid 261 current snap is 0 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3735: finishing write tid 2 to smithi093153278-383 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3735: finishing write tid 3 to smithi093153278-383 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3735: finishing write tid 4 to smithi093153278-383 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 383 v 532 (ObjNum 2297 snap 0 seq_num 2297) dirty exists 2023-08-27T20:55:19.989 INFO:tasks.rados.rados.0.smithi093.stdout:3735: left oid 383 (ObjNum 2297 snap 0 seq_num 2297) 2023-08-27T20:55:19.990 INFO:tasks.rados.rados.0.smithi093.stdout:3737: finishing write tid 1 to smithi093153278-377 2023-08-27T20:55:19.990 INFO:tasks.rados.rados.0.smithi093.stdout:3737: finishing write tid 2 to smithi093153278-377 2023-08-27T20:55:19.990 INFO:tasks.rados.rados.0.smithi093.stdout:3740: seq_num 2301 ranges {672907=686555,1798949=509473} 2023-08-27T20:55:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:19 smithi093 ceph-mon[118902]: pgmap v53: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 8.2 MiB/s wr, 81 op/s 2023-08-27T20:55:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3733983757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:19 smithi093 ceph-mon[111335]: pgmap v53: 89 pgs: 89 active+clean; 1.2 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 61 MiB/s rd, 8.2 MiB/s wr, 81 op/s 2023-08-27T20:55:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3733983757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:19.995 INFO:tasks.rados.rados.0.smithi093.stdout:3740: writing smithi093153278-261 from 672907 to 1359462 tid 1 2023-08-27T20:55:20.001 INFO:tasks.rados.rados.0.smithi093.stdout:3740: writing smithi093153278-261 from 1798949 to 2308422 tid 2 2023-08-27T20:55:20.001 INFO:tasks.rados.rados.0.smithi093.stdout:3735: done (5 left) 2023-08-27T20:55:20.001 INFO:tasks.rados.rados.0.smithi093.stdout:3736: done (4 left) 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3741: write oid 447 current snap is 0 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3737: finishing write tid 3 to smithi093153278-377 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3737: finishing write tid 4 to smithi093153278-377 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 377 v 484 (ObjNum 2298 snap 0 seq_num 2298) dirty exists 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3737: left oid 377 (ObjNum 2298 snap 0 seq_num 2298) 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3738: finishing write tid 1 to smithi093153278-166 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3738: finishing write tid 2 to smithi093153278-166 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3738: finishing write tid 3 to smithi093153278-166 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3738: finishing write tid 4 to smithi093153278-166 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 166 v 402 (ObjNum 2299 snap 0 seq_num 2299) dirty exists 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3738: left oid 166 (ObjNum 2299 snap 0 seq_num 2299) 2023-08-27T20:55:20.002 INFO:tasks.rados.rados.0.smithi093.stdout:3741: seq_num 2302 ranges {689714=761804,1857431=653131} 2023-08-27T20:55:20.008 INFO:tasks.rados.rados.0.smithi093.stdout:3741: writing smithi093153278-447 from 689714 to 1451518 tid 1 2023-08-27T20:55:20.015 INFO:tasks.rados.rados.0.smithi093.stdout:3741: writing smithi093153278-447 from 1857431 to 2510562 tid 2 2023-08-27T20:55:20.015 INFO:tasks.rados.rados.0.smithi093.stdout:3737: done (4 left) 2023-08-27T20:55:20.015 INFO:tasks.rados.rados.0.smithi093.stdout:3738: done (3 left) 2023-08-27T20:55:20.016 INFO:tasks.rados.rados.0.smithi093.stdout:3742: read oid 5 snap -1 2023-08-27T20:55:20.016 INFO:tasks.rados.rados.0.smithi093.stdout:3742: expect (ObjNum 1932 snap 0 seq_num 1932) 2023-08-27T20:55:20.016 INFO:tasks.rados.rados.0.smithi093.stdout:3743: write oid 258 current snap is 0 2023-08-27T20:55:20.016 INFO:tasks.rados.rados.0.smithi093.stdout:3739: finishing write tid 1 to smithi093153278-27 2023-08-27T20:55:20.016 INFO:tasks.rados.rados.0.smithi093.stdout:3739: finishing write tid 2 to smithi093153278-27 2023-08-27T20:55:20.016 INFO:tasks.rados.rados.0.smithi093.stdout:3743: seq_num 2303 ranges {706521=437053,1915913=513142} 2023-08-27T20:55:20.021 INFO:tasks.rados.rados.0.smithi093.stdout:3743: writing smithi093153278-258 from 706521 to 1143574 tid 1 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3743: writing smithi093153278-258 from 1915913 to 2429055 tid 2 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3744: read oid 279 snap -1 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3744: expect (ObjNum 2125 snap 0 seq_num 2125) 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3745: write oid 318 current snap is 0 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3739: finishing write tid 3 to smithi093153278-27 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3739: finishing write tid 4 to smithi093153278-27 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 27 v 405 (ObjNum 2300 snap 0 seq_num 2300) dirty exists 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3739: left oid 27 (ObjNum 2300 snap 0 seq_num 2300) 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3740: finishing write tid 1 to smithi093153278-261 2023-08-27T20:55:20.029 INFO:tasks.rados.rados.0.smithi093.stdout:3745: seq_num 2304 ranges {723328=628655,1974395=656800} 2023-08-27T20:55:20.036 INFO:tasks.rados.rados.0.smithi093.stdout:3745: writing smithi093153278-318 from 723328 to 1351983 tid 1 2023-08-27T20:55:20.043 INFO:tasks.rados.rados.0.smithi093.stdout:3745: writing smithi093153278-318 from 1974395 to 2631195 tid 2 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3739: done (6 left) 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3746: read oid 191 snap -1 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3746: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3747: read oid 233 snap -1 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3747: expect (ObjNum 2070 snap 0 seq_num 2070) 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3748: write oid 477 current snap is 0 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3740: finishing write tid 2 to smithi093153278-261 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3740: finishing write tid 3 to smithi093153278-261 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3740: finishing write tid 4 to smithi093153278-261 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 261 v 487 (ObjNum 2301 snap 0 seq_num 2301) dirty exists 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3740: left oid 261 (ObjNum 2301 snap 0 seq_num 2301) 2023-08-27T20:55:20.044 INFO:tasks.rados.rados.0.smithi093.stdout:3748: seq_num 2305 ranges {740135=703904,2032877=516811} 2023-08-27T20:55:20.051 INFO:tasks.rados.rados.0.smithi093.stdout:3748: writing smithi093153278-477 from 740135 to 1444039 tid 1 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3748: writing smithi093153278-477 from 2032877 to 2549688 tid 2 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3740: done (8 left) 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3749: read oid 376 snap -1 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3749: expect (ObjNum 1904 snap 0 seq_num 1904) 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3750: read oid 219 snap -1 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3750: expect (ObjNum 1708 snap 0 seq_num 1708) 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3751: read oid 473 snap -1 2023-08-27T20:55:20.058 INFO:tasks.rados.rados.0.smithi093.stdout:3751: expect (ObjNum 1233 snap 0 seq_num 1233) 2023-08-27T20:55:20.059 INFO:tasks.rados.rados.0.smithi093.stdout:3752: write oid 363 current snap is 0 2023-08-27T20:55:20.059 INFO:tasks.rados.rados.0.smithi093.stdout:3752: seq_num 2306 ranges {756942=779153,2091359=660469} 2023-08-27T20:55:20.066 INFO:tasks.rados.rados.0.smithi093.stdout:3752: writing smithi093153278-363 from 756942 to 1536095 tid 1 2023-08-27T20:55:20.073 INFO:tasks.rados.rados.0.smithi093.stdout:3752: writing smithi093153278-363 from 2091359 to 2751828 tid 2 2023-08-27T20:55:20.073 INFO:tasks.rados.rados.0.smithi093.stdout:3753: write oid 384 current snap is 0 2023-08-27T20:55:20.073 INFO:tasks.rados.rados.0.smithi093.stdout:3741: finishing write tid 1 to smithi093153278-447 2023-08-27T20:55:20.073 INFO:tasks.rados.rados.0.smithi093.stdout:3741: finishing write tid 2 to smithi093153278-447 2023-08-27T20:55:20.073 INFO:tasks.rados.rados.0.smithi093.stdout:3753: seq_num 2307 ranges {773749=454402,1749841=520480} 2023-08-27T20:55:20.080 INFO:tasks.rados.rados.0.smithi093.stdout:3753: writing smithi093153278-384 from 773749 to 1228151 tid 1 2023-08-27T20:55:20.086 INFO:tasks.rados.rados.0.smithi093.stdout:3753: writing smithi093153278-384 from 1749841 to 2270321 tid 2 2023-08-27T20:55:20.086 INFO:tasks.rados.rados.0.smithi093.stdout:3754: read oid 87 snap -1 2023-08-27T20:55:20.086 INFO:tasks.rados.rados.0.smithi093.stdout:3754: expect (ObjNum 2261 snap 0 seq_num 2261) 2023-08-27T20:55:20.086 INFO:tasks.rados.rados.0.smithi093.stdout:3755: write oid 405 current snap is 0 2023-08-27T20:55:20.086 INFO:tasks.rados.rados.0.smithi093.stdout:3741: finishing write tid 3 to smithi093153278-447 2023-08-27T20:55:20.086 INFO:tasks.rados.rados.0.smithi093.stdout:3741: finishing write tid 4 to smithi093153278-447 2023-08-27T20:55:20.086 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 447 v 535 (ObjNum 2302 snap 0 seq_num 2302) dirty exists 2023-08-27T20:55:20.087 INFO:tasks.rados.rados.0.smithi093.stdout:3741: left oid 447 (ObjNum 2302 snap 0 seq_num 2302) 2023-08-27T20:55:20.087 INFO:tasks.rados.rados.0.smithi093.stdout:3742: expect (ObjNum 1932 snap 0 seq_num 1932) 2023-08-27T20:55:20.191 INFO:tasks.rados.rados.0.smithi093.stdout:3743: finishing write tid 1 to smithi093153278-258 2023-08-27T20:55:20.191 INFO:tasks.rados.rados.0.smithi093.stdout:3755: seq_num 2308 ranges {790556=529651,2091970=664138} 2023-08-27T20:55:20.198 INFO:tasks.rados.rados.0.smithi093.stdout:3755: writing smithi093153278-405 from 790556 to 1320207 tid 1 2023-08-27T20:55:20.206 INFO:tasks.rados.rados.0.smithi093.stdout:3755: writing smithi093153278-405 from 2091970 to 2756108 tid 2 2023-08-27T20:55:20.206 INFO:tasks.rados.rados.0.smithi093.stdout:3741: done (14 left) 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3742: done (13 left) 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3756: read oid 399 snap -1 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3756: expect (ObjNum 1934 snap 0 seq_num 1934) 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3757: delete oid 434 current snap is 0 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3743: finishing write tid 2 to smithi093153278-258 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3743: finishing write tid 3 to smithi093153278-258 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3743: finishing write tid 4 to smithi093153278-258 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 258 v 401 (ObjNum 2303 snap 0 seq_num 2303) dirty exists 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3743: left oid 258 (ObjNum 2303 snap 0 seq_num 2303) 2023-08-27T20:55:20.207 INFO:tasks.rados.rados.0.smithi093.stdout:3744: expect (ObjNum 2125 snap 0 seq_num 2125) 2023-08-27T20:55:20.375 INFO:tasks.rados.rados.0.smithi093.stdout:3745: finishing write tid 1 to smithi093153278-318 2023-08-27T20:55:20.375 INFO:tasks.rados.rados.0.smithi093.stdout:3746: expect (ObjNum 511 snap 0 seq_num 511) 2023-08-27T20:55:20.492 INFO:tasks.rados.rados.0.smithi093.stdout:3745: finishing write tid 2 to smithi093153278-318 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:3745: finishing write tid 3 to smithi093153278-318 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:3745: finishing write tid 4 to smithi093153278-318 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 318 v 452 (ObjNum 2304 snap 0 seq_num 2304) dirty exists 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:3745: left oid 318 (ObjNum 2304 snap 0 seq_num 2304) 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:3743: done (14 left) 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:3744: done (13 left) 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:3745: done (12 left) 2023-08-27T20:55:20.493 INFO:tasks.rados.rados.0.smithi093.stdout:3746: done (11 left) 2023-08-27T20:55:20.494 INFO:tasks.rados.rados.0.smithi093.stdout:3757: done (10 left) 2023-08-27T20:55:20.494 INFO:tasks.rados.rados.0.smithi093.stdout:3758: delete oid 498 current snap is 0 2023-08-27T20:55:20.494 INFO:tasks.rados.rados.0.smithi093.stdout:3747: expect (ObjNum 2070 snap 0 seq_num 2070) 2023-08-27T20:55:20.609 INFO:tasks.rados.rados.0.smithi093.stdout:3748: finishing write tid 1 to smithi093153278-477 2023-08-27T20:55:20.609 INFO:tasks.rados.rados.0.smithi093.stdout:3748: finishing write tid 2 to smithi093153278-477 2023-08-27T20:55:20.609 INFO:tasks.rados.rados.0.smithi093.stdout:3749: expect (ObjNum 1904 snap 0 seq_num 1904) 2023-08-27T20:55:20.739 INFO:tasks.rados.rados.0.smithi093.stdout:3751: expect (ObjNum 1233 snap 0 seq_num 1233) 2023-08-27T20:55:20.928 INFO:tasks.rados.rados.0.smithi093.stdout:3748: finishing write tid 3 to smithi093153278-477 2023-08-27T20:55:20.929 INFO:tasks.rados.rados.0.smithi093.stdout:3748: finishing write tid 4 to smithi093153278-477 2023-08-27T20:55:20.929 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 477 v 538 (ObjNum 2305 snap 0 seq_num 2305) dirty exists 2023-08-27T20:55:20.929 INFO:tasks.rados.rados.0.smithi093.stdout:3748: left oid 477 (ObjNum 2305 snap 0 seq_num 2305) 2023-08-27T20:55:20.929 INFO:tasks.rados.rados.0.smithi093.stdout:3750: expect (ObjNum 1708 snap 0 seq_num 1708) 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:3752: finishing write tid 1 to smithi093153278-363 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:3752: finishing write tid 2 to smithi093153278-363 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:3752: finishing write tid 3 to smithi093153278-363 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:3752: finishing write tid 4 to smithi093153278-363 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 596 (ObjNum 2306 snap 0 seq_num 2306) dirty exists 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:3752: left oid 363 (ObjNum 2306 snap 0 seq_num 2306) 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:3753: finishing write tid 1 to smithi093153278-384 2023-08-27T20:55:21.121 INFO:tasks.rados.rados.0.smithi093.stdout:3753: finishing write tid 2 to smithi093153278-384 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3747: done (10 left) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3748: done (9 left) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3749: done (8 left) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3750: done (7 left) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3751: done (6 left) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3752: done (5 left) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3758: done (4 left) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3759: read oid 18 snap -1 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3759: expect (ObjNum 1545 snap 0 seq_num 1545) 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3760: delete oid 207 current snap is 0 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3753: finishing write tid 3 to smithi093153278-384 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:3753: finishing write tid 4 to smithi093153278-384 2023-08-27T20:55:21.122 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 384 v 408 (ObjNum 2307 snap 0 seq_num 2307) dirty exists 2023-08-27T20:55:21.123 INFO:tasks.rados.rados.0.smithi093.stdout:3753: left oid 384 (ObjNum 2307 snap 0 seq_num 2307) 2023-08-27T20:55:21.123 INFO:tasks.rados.rados.0.smithi093.stdout:3754: expect (ObjNum 2261 snap 0 seq_num 2261) 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3755: finishing write tid 1 to smithi093153278-405 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3755: finishing write tid 2 to smithi093153278-405 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3755: finishing write tid 3 to smithi093153278-405 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3753: done (5 left) 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3754: done (4 left) 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3760: done (3 left) 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3761: write oid 118 current snap is 0 2023-08-27T20:55:21.223 INFO:tasks.rados.rados.0.smithi093.stdout:3756: expect (ObjNum 1934 snap 0 seq_num 1934) 2023-08-27T20:55:21.352 INFO:tasks.rados.rados.0.smithi093.stdout:3755: finishing write tid 4 to smithi093153278-405 2023-08-27T20:55:21.352 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 405 v 490 (ObjNum 2308 snap 0 seq_num 2308) dirty exists 2023-08-27T20:55:21.352 INFO:tasks.rados.rados.0.smithi093.stdout:3755: left oid 405 (ObjNum 2308 snap 0 seq_num 2308) 2023-08-27T20:55:21.352 INFO:tasks.rados.rados.0.smithi093.stdout:3759: expect (ObjNum 1545 snap 0 seq_num 1545) 2023-08-27T20:55:21.526 INFO:tasks.rados.rados.0.smithi093.stdout:3761: seq_num 2312 ranges {457784=547000,1525898=671476} 2023-08-27T20:55:21.531 INFO:tasks.rados.rados.0.smithi093.stdout:3761: writing smithi093153278-118 from 457784 to 1004784 tid 1 2023-08-27T20:55:21.538 INFO:tasks.rados.rados.0.smithi093.stdout:3761: writing smithi093153278-118 from 1525898 to 2197374 tid 2 2023-08-27T20:55:21.538 INFO:tasks.rados.rados.0.smithi093.stdout:3755: done (3 left) 2023-08-27T20:55:21.538 INFO:tasks.rados.rados.0.smithi093.stdout:3756: done (2 left) 2023-08-27T20:55:21.538 INFO:tasks.rados.rados.0.smithi093.stdout:3759: done (1 left) 2023-08-27T20:55:21.538 INFO:tasks.rados.rados.0.smithi093.stdout:3762: write oid 82 current snap is 0 2023-08-27T20:55:21.538 INFO:tasks.rados.rados.0.smithi093.stdout:3761: finishing write tid 1 to smithi093153278-118 2023-08-27T20:55:21.539 INFO:tasks.rados.rados.0.smithi093.stdout:3762: seq_num 2313 ranges {474591=622249,1584380=415134,2501500=373091} 2023-08-27T20:55:21.544 INFO:tasks.rados.rados.0.smithi093.stdout:3762: writing smithi093153278-82 from 474591 to 1096840 tid 1 2023-08-27T20:55:21.549 INFO:tasks.rados.rados.0.smithi093.stdout:3762: writing smithi093153278-82 from 1584380 to 1999514 tid 2 2023-08-27T20:55:21.554 INFO:tasks.rados.rados.0.smithi093.stdout:3762: writing smithi093153278-82 from 2501500 to 2874591 tid 3 2023-08-27T20:55:21.554 INFO:tasks.rados.rados.0.smithi093.stdout:3763: read oid 200 snap -1 2023-08-27T20:55:21.554 INFO:tasks.rados.rados.0.smithi093.stdout:3763: expect deleted 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:3764: delete oid 108 current snap is 0 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:3761: finishing write tid 2 to smithi093153278-118 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:3761: finishing write tid 3 to smithi093153278-118 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:3761: finishing write tid 4 to smithi093153278-118 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 118 v 455 (ObjNum 2312 snap 0 seq_num 2312) dirty exists 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:3761: left oid 118 (ObjNum 2312 snap 0 seq_num 2312) 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:3762: finishing write tid 1 to smithi093153278-82 2023-08-27T20:55:21.555 INFO:tasks.rados.rados.0.smithi093.stdout:3762: finishing write tid 2 to smithi093153278-82 2023-08-27T20:55:21.556 INFO:tasks.rados.rados.0.smithi093.stdout:3761: done (3 left) 2023-08-27T20:55:21.556 INFO:tasks.rados.rados.0.smithi093.stdout:3763: done (2 left) 2023-08-27T20:55:21.556 INFO:tasks.rados.rados.0.smithi093.stdout:3764: done (1 left) 2023-08-27T20:55:21.556 INFO:tasks.rados.rados.0.smithi093.stdout:3765: read oid 235 snap -1 2023-08-27T20:55:21.556 INFO:tasks.rados.rados.0.smithi093.stdout:3765: expect (ObjNum 1835 snap 0 seq_num 1835) 2023-08-27T20:55:21.556 INFO:tasks.rados.rados.0.smithi093.stdout:3766: write oid 429 current snap is 0 2023-08-27T20:55:21.556 INFO:tasks.rados.rados.0.smithi093.stdout:3766: seq_num 2315 ranges {508205=772747,1701344=418803,2556346=351859} 2023-08-27T20:55:21.563 INFO:tasks.rados.rados.0.smithi093.stdout:3766: writing smithi093153278-429 from 508205 to 1280952 tid 1 2023-08-27T20:55:21.567 INFO:tasks.rados.rados.0.smithi093.stdout:3766: writing smithi093153278-429 from 1701344 to 2120147 tid 2 2023-08-27T20:55:21.572 INFO:tasks.rados.rados.0.smithi093.stdout:3766: writing smithi093153278-429 from 2556346 to 2908205 tid 3 2023-08-27T20:55:21.572 INFO:tasks.rados.rados.0.smithi093.stdout:3767: write oid 435 current snap is 0 2023-08-27T20:55:21.572 INFO:tasks.rados.rados.0.smithi093.stdout:3762: finishing write tid 3 to smithi093153278-82 2023-08-27T20:55:21.572 INFO:tasks.rados.rados.0.smithi093.stdout:3762: finishing write tid 4 to smithi093153278-82 2023-08-27T20:55:21.573 INFO:tasks.rados.rados.0.smithi093.stdout:3762: finishing write tid 5 to smithi093153278-82 2023-08-27T20:55:21.573 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 82 v 464 (ObjNum 2313 snap 0 seq_num 2313) dirty exists 2023-08-27T20:55:21.573 INFO:tasks.rados.rados.0.smithi093.stdout:3762: left oid 82 (ObjNum 2313 snap 0 seq_num 2313) 2023-08-27T20:55:21.573 INFO:tasks.rados.rados.0.smithi093.stdout:3765: expect (ObjNum 1835 snap 0 seq_num 1835) 2023-08-27T20:55:21.694 INFO:tasks.rados.rados.0.smithi093.stdout:3766: finishing write tid 1 to smithi093153278-429 2023-08-27T20:55:21.694 INFO:tasks.rados.rados.0.smithi093.stdout:3766: finishing write tid 2 to smithi093153278-429 2023-08-27T20:55:21.694 INFO:tasks.rados.rados.0.smithi093.stdout:3767: seq_num 2316 ranges {525012=447996,1643473=678814,2867416=57596} 2023-08-27T20:55:21.699 INFO:tasks.rados.rados.0.smithi093.stdout:3767: writing smithi093153278-435 from 525012 to 973008 tid 1 2023-08-27T20:55:21.707 INFO:tasks.rados.rados.0.smithi093.stdout:3767: writing smithi093153278-435 from 1643473 to 2322287 tid 2 2023-08-27T20:55:21.711 INFO:tasks.rados.rados.0.smithi093.stdout:3767: writing smithi093153278-435 from 2867416 to 2925012 tid 3 2023-08-27T20:55:21.711 INFO:tasks.rados.rados.0.smithi093.stdout:3762: done (3 left) 2023-08-27T20:55:21.711 INFO:tasks.rados.rados.0.smithi093.stdout:3765: done (2 left) 2023-08-27T20:55:21.711 INFO:tasks.rados.rados.0.smithi093.stdout:3768: write oid 329 current snap is 0 2023-08-27T20:55:21.711 INFO:tasks.rados.rados.0.smithi093.stdout:3766: finishing write tid 3 to smithi093153278-429 2023-08-27T20:55:21.711 INFO:tasks.rados.rados.0.smithi093.stdout:3768: seq_num 2317 ranges {541819=523245,1701955=422472,2894839=46980} 2023-08-27T20:55:21.716 INFO:tasks.rados.rados.0.smithi093.stdout:3768: writing smithi093153278-329 from 541819 to 1065064 tid 1 2023-08-27T20:55:21.722 INFO:tasks.rados.rados.0.smithi093.stdout:3768: writing smithi093153278-329 from 1701955 to 2124427 tid 2 2023-08-27T20:55:21.727 INFO:tasks.rados.rados.0.smithi093.stdout:3768: writing smithi093153278-329 from 2894839 to 2941819 tid 3 2023-08-27T20:55:21.727 INFO:tasks.rados.rados.0.smithi093.stdout:3769: write oid 158 current snap is 0 2023-08-27T20:55:21.727 INFO:tasks.rados.rados.0.smithi093.stdout:3766: finishing write tid 4 to smithi093153278-429 2023-08-27T20:55:21.727 INFO:tasks.rados.rados.0.smithi093.stdout:3766: finishing write tid 5 to smithi093153278-429 2023-08-27T20:55:21.727 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 429 v 489 (ObjNum 2315 snap 0 seq_num 2315) dirty exists 2023-08-27T20:55:21.727 INFO:tasks.rados.rados.0.smithi093.stdout:3766: left oid 429 (ObjNum 2315 snap 0 seq_num 2315) 2023-08-27T20:55:21.727 INFO:tasks.rados.rados.0.smithi093.stdout:3769: seq_num 2318 ranges {558626=598494,1760437=682483,2922262=36364} 2023-08-27T20:55:21.733 INFO:tasks.rados.rados.0.smithi093.stdout:3769: writing smithi093153278-158 from 558626 to 1157120 tid 1 2023-08-27T20:55:21.740 INFO:tasks.rados.rados.0.smithi093.stdout:3769: writing smithi093153278-158 from 1760437 to 2442920 tid 2 2023-08-27T20:55:21.743 INFO:tasks.rados.rados.0.smithi093.stdout:3769: writing smithi093153278-158 from 2922262 to 2958626 tid 3 2023-08-27T20:55:21.743 INFO:tasks.rados.rados.0.smithi093.stdout:3766: done (3 left) 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3770: read oid 235 snap -1 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3770: expect (ObjNum 1835 snap 0 seq_num 1835) 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3771: read oid 306 snap -1 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3771: expect (ObjNum 1899 snap 0 seq_num 1899) 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3772: write oid 452 current snap is 0 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3767: finishing write tid 1 to smithi093153278-435 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3767: finishing write tid 2 to smithi093153278-435 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3767: finishing write tid 3 to smithi093153278-435 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3767: finishing write tid 4 to smithi093153278-435 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3767: finishing write tid 5 to smithi093153278-435 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 435 v 532 (ObjNum 2316 snap 0 seq_num 2316) dirty exists 2023-08-27T20:55:21.744 INFO:tasks.rados.rados.0.smithi093.stdout:3767: left oid 435 (ObjNum 2316 snap 0 seq_num 2316) 2023-08-27T20:55:21.745 INFO:tasks.rados.rados.0.smithi093.stdout:3768: finishing write tid 1 to smithi093153278-329 2023-08-27T20:55:21.745 INFO:tasks.rados.rados.0.smithi093.stdout:3768: finishing write tid 2 to smithi093153278-329 2023-08-27T20:55:21.745 INFO:tasks.rados.rados.0.smithi093.stdout:3772: seq_num 2319 ranges {575433=790096,1935272=426141} 2023-08-27T20:55:21.750 INFO:tasks.rados.rados.0.smithi093.stdout:3772: writing smithi093153278-452 from 575433 to 1365529 tid 1 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3772: writing smithi093153278-452 from 1935272 to 2361413 tid 2 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3767: done (5 left) 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3773: write oid 42 current snap is 0 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3768: finishing write tid 3 to smithi093153278-329 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3768: finishing write tid 4 to smithi093153278-329 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3768: finishing write tid 5 to smithi093153278-329 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 329 v 433 (ObjNum 2317 snap 0 seq_num 2317) dirty exists 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3768: left oid 329 (ObjNum 2317 snap 0 seq_num 2317) 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3769: finishing write tid 1 to smithi093153278-158 2023-08-27T20:55:21.757 INFO:tasks.rados.rados.0.smithi093.stdout:3773: seq_num 2320 ranges {592240=465345,1593754=686152,2809814=182426} 2023-08-27T20:55:21.762 INFO:tasks.rados.rados.0.smithi093.stdout:3773: writing smithi093153278-42 from 592240 to 1057585 tid 1 2023-08-27T20:55:21.769 INFO:tasks.rados.rados.0.smithi093.stdout:3773: writing smithi093153278-42 from 1593754 to 2279906 tid 2 2023-08-27T20:55:21.773 INFO:tasks.rados.rados.0.smithi093.stdout:3773: writing smithi093153278-42 from 2809814 to 2992240 tid 3 2023-08-27T20:55:21.773 INFO:tasks.rados.rados.0.smithi093.stdout:3768: done (5 left) 2023-08-27T20:55:21.773 INFO:tasks.rados.rados.0.smithi093.stdout:3774: write oid 449 current snap is 0 2023-08-27T20:55:21.773 INFO:tasks.rados.rados.0.smithi093.stdout:3769: finishing write tid 2 to smithi093153278-158 2023-08-27T20:55:21.773 INFO:tasks.rados.rados.0.smithi093.stdout:3774: seq_num 2321 ranges {609047=540594,1935883=429810,3004531=4516} 2023-08-27T20:55:21.779 INFO:tasks.rados.rados.0.smithi093.stdout:3774: writing smithi093153278-449 from 609047 to 1149641 tid 1 2023-08-27T20:55:21.786 INFO:tasks.rados.rados.0.smithi093.stdout:3774: writing smithi093153278-449 from 1935883 to 2365693 tid 2 2023-08-27T20:55:21.790 INFO:tasks.rados.rados.0.smithi093.stdout:3774: writing smithi093153278-449 from 3004531 to 3009047 tid 3 2023-08-27T20:55:21.790 INFO:tasks.rados.rados.0.smithi093.stdout:3775: write oid 378 current snap is 0 2023-08-27T20:55:21.790 INFO:tasks.rados.rados.0.smithi093.stdout:3769: finishing write tid 3 to smithi093153278-158 2023-08-27T20:55:21.790 INFO:tasks.rados.rados.0.smithi093.stdout:3775: seq_num 2322 ranges {625854=615843,1994365=573468} 2023-08-27T20:55:21.796 INFO:tasks.rados.rados.0.smithi093.stdout:3775: writing smithi093153278-378 from 625854 to 1241697 tid 1 2023-08-27T20:55:21.804 INFO:tasks.rados.rados.0.smithi093.stdout:3775: writing smithi093153278-378 from 1994365 to 2567833 tid 2 2023-08-27T20:55:21.804 INFO:tasks.rados.rados.0.smithi093.stdout:3776: write oid 263 current snap is 0 2023-08-27T20:55:21.804 INFO:tasks.rados.rados.0.smithi093.stdout:3769: finishing write tid 4 to smithi093153278-158 2023-08-27T20:55:21.804 INFO:tasks.rados.rados.0.smithi093.stdout:3769: finishing write tid 5 to smithi093153278-158 2023-08-27T20:55:21.805 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 158 v 405 (ObjNum 2318 snap 0 seq_num 2318) dirty exists 2023-08-27T20:55:21.805 INFO:tasks.rados.rados.0.smithi093.stdout:3769: left oid 158 (ObjNum 2318 snap 0 seq_num 2318) 2023-08-27T20:55:21.805 INFO:tasks.rados.rados.0.smithi093.stdout:3770: expect (ObjNum 1835 snap 0 seq_num 1835) 2023-08-27T20:55:21.930 INFO:tasks.rados.rados.0.smithi093.stdout:3772: finishing write tid 1 to smithi093153278-452 2023-08-27T20:55:21.930 INFO:tasks.rados.rados.0.smithi093.stdout:3776: seq_num 2323 ranges {642661=691092,2052847=433479} 2023-08-27T20:55:21.937 INFO:tasks.rados.rados.0.smithi093.stdout:3776: writing smithi093153278-263 from 642661 to 1333753 tid 1 2023-08-27T20:55:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:21 smithi136 ceph-mon[113531]: pgmap v54: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 12 MiB/s wr, 117 op/s 2023-08-27T20:55:21.944 INFO:tasks.rados.rados.0.smithi093.stdout:3776: writing smithi093153278-263 from 2052847 to 2486326 tid 2 2023-08-27T20:55:21.944 INFO:tasks.rados.rados.0.smithi093.stdout:3769: done (7 left) 2023-08-27T20:55:21.944 INFO:tasks.rados.rados.0.smithi093.stdout:3770: done (6 left) 2023-08-27T20:55:21.944 INFO:tasks.rados.rados.0.smithi093.stdout:3777: write oid 275 current snap is 0 2023-08-27T20:55:21.944 INFO:tasks.rados.rados.0.smithi093.stdout:3772: finishing write tid 2 to smithi093153278-452 2023-08-27T20:55:21.944 INFO:tasks.rados.rados.0.smithi093.stdout:3772: finishing write tid 3 to smithi093153278-452 2023-08-27T20:55:21.945 INFO:tasks.rados.rados.0.smithi093.stdout:3772: finishing write tid 4 to smithi093153278-452 2023-08-27T20:55:21.945 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 452 v 535 (ObjNum 2319 snap 0 seq_num 2319) dirty exists 2023-08-27T20:55:21.945 INFO:tasks.rados.rados.0.smithi093.stdout:3772: left oid 452 (ObjNum 2319 snap 0 seq_num 2319) 2023-08-27T20:55:21.945 INFO:tasks.rados.rados.0.smithi093.stdout:3771: expect (ObjNum 1899 snap 0 seq_num 1899) 2023-08-27T20:55:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:21 smithi093 ceph-mon[111335]: pgmap v54: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 12 MiB/s wr, 117 op/s 2023-08-27T20:55:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:21 smithi093 ceph-mon[118902]: pgmap v54: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 12 MiB/s wr, 117 op/s 2023-08-27T20:55:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:21.629Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:21.630Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:22.123 INFO:tasks.rados.rados.0.smithi093.stdout:3773: finishing write tid 1 to smithi093153278-42 2023-08-27T20:55:22.123 INFO:tasks.rados.rados.0.smithi093.stdout:3773: finishing write tid 2 to smithi093153278-42 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:3773: finishing write tid 3 to smithi093153278-42 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:3773: finishing write tid 4 to smithi093153278-42 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:3773: finishing write tid 5 to smithi093153278-42 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 42 v 433 (ObjNum 2320 snap 0 seq_num 2320) dirty exists 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:3773: left oid 42 (ObjNum 2320 snap 0 seq_num 2320) 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:3774: finishing write tid 1 to smithi093153278-449 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:3774: finishing write tid 2 to smithi093153278-449 2023-08-27T20:55:22.124 INFO:tasks.rados.rados.0.smithi093.stdout:3777: seq_num 2324 ranges {659468=766341,2111329=577137} 2023-08-27T20:55:22.130 INFO:tasks.rados.rados.0.smithi093.stdout:3777: writing smithi093153278-275 from 659468 to 1425809 tid 1 2023-08-27T20:55:22.138 INFO:tasks.rados.rados.0.smithi093.stdout:3777: writing smithi093153278-275 from 2111329 to 2688466 tid 2 2023-08-27T20:55:22.138 INFO:tasks.rados.rados.0.smithi093.stdout:3771: done (6 left) 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3772: done (5 left) 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3773: done (4 left) 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3778: read oid 98 snap -1 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3778: expect deleted 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3779: write oid 297 current snap is 0 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3774: finishing write tid 3 to smithi093153278-449 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3774: finishing write tid 4 to smithi093153278-449 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3774: finishing write tid 5 to smithi093153278-449 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 543 (ObjNum 2321 snap 0 seq_num 2321) dirty exists 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3774: left oid 449 (ObjNum 2321 snap 0 seq_num 2321) 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3775: finishing write tid 1 to smithi093153278-378 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3775: finishing write tid 2 to smithi093153278-378 2023-08-27T20:55:22.139 INFO:tasks.rados.rados.0.smithi093.stdout:3775: finishing write tid 3 to smithi093153278-378 2023-08-27T20:55:22.140 INFO:tasks.rados.rados.0.smithi093.stdout:3775: finishing write tid 4 to smithi093153278-378 2023-08-27T20:55:22.140 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 378 v 494 (ObjNum 2322 snap 0 seq_num 2322) dirty exists 2023-08-27T20:55:22.140 INFO:tasks.rados.rados.0.smithi093.stdout:3775: left oid 378 (ObjNum 2322 snap 0 seq_num 2322) 2023-08-27T20:55:22.140 INFO:tasks.rados.rados.0.smithi093.stdout:3776: finishing write tid 1 to smithi093153278-263 2023-08-27T20:55:22.140 INFO:tasks.rados.rados.0.smithi093.stdout:3776: finishing write tid 2 to smithi093153278-263 2023-08-27T20:55:22.140 INFO:tasks.rados.rados.0.smithi093.stdout:3779: seq_num 2325 ranges {676275=441590,1653458=437148,2597870=431077} 2023-08-27T20:55:22.144 INFO:tasks.rados.rados.0.smithi093.stdout:3779: writing smithi093153278-297 from 676275 to 1117865 tid 1 2023-08-27T20:55:22.149 INFO:tasks.rados.rados.0.smithi093.stdout:3779: writing smithi093153278-297 from 1653458 to 2090606 tid 2 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3779: writing smithi093153278-297 from 2597870 to 3028947 tid 3 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3774: done (5 left) 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3775: done (4 left) 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3780: read oid 27 snap -1 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3780: expect (ObjNum 2300 snap 0 seq_num 2300) 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3781: write oid 363 current snap is 0 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3776: finishing write tid 3 to smithi093153278-263 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:3776: finishing write tid 4 to smithi093153278-263 2023-08-27T20:55:22.155 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 263 v 546 (ObjNum 2323 snap 0 seq_num 2323) dirty exists 2023-08-27T20:55:22.156 INFO:tasks.rados.rados.0.smithi093.stdout:3776: left oid 263 (ObjNum 2323 snap 0 seq_num 2323) 2023-08-27T20:55:22.156 INFO:tasks.rados.rados.0.smithi093.stdout:3777: finishing write tid 1 to smithi093153278-275 2023-08-27T20:55:22.156 INFO:tasks.rados.rados.0.smithi093.stdout:3781: seq_num 2326 ranges {693082=516839,1711940=580806,3025293=67789} 2023-08-27T20:55:22.161 INFO:tasks.rados.rados.0.smithi093.stdout:3781: writing smithi093153278-363 from 693082 to 1209921 tid 1 2023-08-27T20:55:22.168 INFO:tasks.rados.rados.0.smithi093.stdout:3781: writing smithi093153278-363 from 1711940 to 2292746 tid 2 2023-08-27T20:55:22.172 INFO:tasks.rados.rados.0.smithi093.stdout:3781: writing smithi093153278-363 from 3025293 to 3093082 tid 3 2023-08-27T20:55:22.172 INFO:tasks.rados.rados.0.smithi093.stdout:3776: done (5 left) 2023-08-27T20:55:22.173 INFO:tasks.rados.rados.0.smithi093.stdout:3782: read oid 458 snap -1 2023-08-27T20:55:22.173 INFO:tasks.rados.rados.0.smithi093.stdout:3782: expect deleted 2023-08-27T20:55:22.173 INFO:tasks.rados.rados.0.smithi093.stdout:3783: write oid 81 current snap is 0 2023-08-27T20:55:22.173 INFO:tasks.rados.rados.0.smithi093.stdout:3783: seq_num 2327 ranges {709889=708441,1886775=440817,2769069=340820} 2023-08-27T20:55:22.180 INFO:tasks.rados.rados.0.smithi093.stdout:3783: writing smithi093153278-81 from 709889 to 1418330 tid 1 2023-08-27T20:55:22.185 INFO:tasks.rados.rados.0.smithi093.stdout:3783: writing smithi093153278-81 from 1886775 to 2327592 tid 2 2023-08-27T20:55:22.190 INFO:tasks.rados.rados.0.smithi093.stdout:3783: writing smithi093153278-81 from 2769069 to 3109889 tid 3 2023-08-27T20:55:22.190 INFO:tasks.rados.rados.0.smithi093.stdout:3784: read oid 226 snap -1 2023-08-27T20:55:22.190 INFO:tasks.rados.rados.0.smithi093.stdout:3784: expect (ObjNum 1823 snap 0 seq_num 1823) 2023-08-27T20:55:22.190 INFO:tasks.rados.rados.0.smithi093.stdout:3785: write oid 51 current snap is 0 2023-08-27T20:55:22.190 INFO:tasks.rados.rados.0.smithi093.stdout:3777: finishing write tid 2 to smithi093153278-275 2023-08-27T20:55:22.190 INFO:tasks.rados.rados.0.smithi093.stdout:3777: finishing write tid 3 to smithi093153278-275 2023-08-27T20:55:22.190 INFO:tasks.rados.rados.0.smithi093.stdout:3777: finishing write tid 4 to smithi093153278-275 2023-08-27T20:55:22.191 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 275 v 458 (ObjNum 2324 snap 0 seq_num 2324) dirty exists 2023-08-27T20:55:22.191 INFO:tasks.rados.rados.0.smithi093.stdout:3777: left oid 275 (ObjNum 2324 snap 0 seq_num 2324) 2023-08-27T20:55:22.191 INFO:tasks.rados.rados.0.smithi093.stdout:3779: finishing write tid 1 to smithi093153278-297 2023-08-27T20:55:22.191 INFO:tasks.rados.rados.0.smithi093.stdout:3779: finishing write tid 2 to smithi093153278-297 2023-08-27T20:55:22.191 INFO:tasks.rados.rados.0.smithi093.stdout:3785: seq_num 2328 ranges {726696=783690,1945257=584475} 2023-08-27T20:55:22.198 INFO:tasks.rados.rados.0.smithi093.stdout:3785: writing smithi093153278-51 from 726696 to 1510386 tid 1 2023-08-27T20:55:22.204 INFO:tasks.rados.rados.0.smithi093.stdout:3785: writing smithi093153278-51 from 1945257 to 2529732 tid 2 2023-08-27T20:55:22.204 INFO:tasks.rados.rados.0.smithi093.stdout:3777: done (8 left) 2023-08-27T20:55:22.204 INFO:tasks.rados.rados.0.smithi093.stdout:3778: done (7 left) 2023-08-27T20:55:22.204 INFO:tasks.rados.rados.0.smithi093.stdout:3786: write oid 235 current snap is 0 2023-08-27T20:55:22.204 INFO:tasks.rados.rados.0.smithi093.stdout:3779: finishing write tid 3 to smithi093153278-297 2023-08-27T20:55:22.205 INFO:tasks.rados.rados.0.smithi093.stdout:3779: finishing write tid 4 to smithi093153278-297 2023-08-27T20:55:22.205 INFO:tasks.rados.rados.0.smithi093.stdout:3779: finishing write tid 5 to smithi093153278-297 2023-08-27T20:55:22.205 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 297 v 412 (ObjNum 2325 snap 0 seq_num 2325) dirty exists 2023-08-27T20:55:22.205 INFO:tasks.rados.rados.0.smithi093.stdout:3779: left oid 297 (ObjNum 2325 snap 0 seq_num 2325) 2023-08-27T20:55:22.205 INFO:tasks.rados.rados.0.smithi093.stdout:3780: expect (ObjNum 2300 snap 0 seq_num 2300) 2023-08-27T20:55:22.323 INFO:tasks.rados.rados.0.smithi093.stdout:3781: finishing write tid 1 to smithi093153278-363 2023-08-27T20:55:22.323 INFO:tasks.rados.rados.0.smithi093.stdout:3786: seq_num 2329 ranges {743503=458939,1887386=444486,3107562=35941} 2023-08-27T20:55:22.329 INFO:tasks.rados.rados.0.smithi093.stdout:3786: writing smithi093153278-235 from 743503 to 1202442 tid 1 2023-08-27T20:55:22.335 INFO:tasks.rados.rados.0.smithi093.stdout:3786: writing smithi093153278-235 from 1887386 to 2331872 tid 2 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3786: writing smithi093153278-235 from 3107562 to 3143503 tid 3 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3779: done (7 left) 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3780: done (6 left) 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3787: delete oid 437 current snap is 0 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3781: finishing write tid 2 to smithi093153278-363 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3781: finishing write tid 3 to smithi093153278-363 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3781: finishing write tid 4 to smithi093153278-363 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:3781: finishing write tid 5 to smithi093153278-363 2023-08-27T20:55:22.341 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 363 v 600 (ObjNum 2326 snap 0 seq_num 2326) dirty exists 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3781: left oid 363 (ObjNum 2326 snap 0 seq_num 2326) 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3783: finishing write tid 1 to smithi093153278-81 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3783: finishing write tid 2 to smithi093153278-81 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3783: finishing write tid 3 to smithi093153278-81 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3783: finishing write tid 4 to smithi093153278-81 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3783: finishing write tid 5 to smithi093153278-81 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 81 v 604 (ObjNum 2327 snap 0 seq_num 2327) dirty exists 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3783: left oid 81 (ObjNum 2327 snap 0 seq_num 2327) 2023-08-27T20:55:22.342 INFO:tasks.rados.rados.0.smithi093.stdout:3784: expect (ObjNum 1823 snap 0 seq_num 1823) 2023-08-27T20:55:22.475 INFO:tasks.rados.rados.0.smithi093.stdout:3785: finishing write tid 1 to smithi093153278-51 2023-08-27T20:55:22.475 INFO:tasks.rados.rados.0.smithi093.stdout:3785: finishing write tid 2 to smithi093153278-51 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:3785: finishing write tid 3 to smithi093153278-51 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:3785: finishing write tid 4 to smithi093153278-51 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 51 v 430 (ObjNum 2328 snap 0 seq_num 2328) dirty exists 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:3785: left oid 51 (ObjNum 2328 snap 0 seq_num 2328) 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:3786: finishing write tid 1 to smithi093153278-235 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:3781: done (6 left) 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:3782: done (5 left) 2023-08-27T20:55:22.476 INFO:tasks.rados.rados.0.smithi093.stdout:3783: done (4 left) 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3784: done (3 left) 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3785: done (2 left) 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3787: done (1 left) 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3788: delete oid 108 current snap is 0 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3786: finishing write tid 2 to smithi093153278-235 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3786: finishing write tid 3 to smithi093153278-235 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3786: finishing write tid 4 to smithi093153278-235 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:3786: finishing write tid 5 to smithi093153278-235 2023-08-27T20:55:22.477 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 235 v 437 (ObjNum 2329 snap 0 seq_num 2329) dirty exists 2023-08-27T20:55:22.478 INFO:tasks.rados.rados.0.smithi093.stdout:3786: left oid 235 (ObjNum 2329 snap 0 seq_num 2329) 2023-08-27T20:55:22.478 INFO:tasks.rados.rados.0.smithi093.stdout:3786: done (1 left) 2023-08-27T20:55:22.478 INFO:tasks.rados.rados.0.smithi093.stdout:3788: done (0 left) 2023-08-27T20:55:22.479 INFO:tasks.rados.rados.0.smithi093.stdout:3789: read oid 55 snap -1 2023-08-27T20:55:22.479 INFO:tasks.rados.rados.0.smithi093.stdout:3789: expect (ObjNum 1970 snap 0 seq_num 1970) 2023-08-27T20:55:22.479 INFO:tasks.rados.rados.0.smithi093.stdout:3790: write oid 221 current snap is 0 2023-08-27T20:55:22.479 INFO:tasks.rados.rados.0.smithi093.stdout:3790: seq_num 2332 ranges {793924=684686,2062832=591813,3189831=4093} 2023-08-27T20:55:22.485 INFO:tasks.rados.rados.0.smithi093.stdout:3790: writing smithi093153278-221 from 793924 to 1478610 tid 1 2023-08-27T20:55:22.491 INFO:tasks.rados.rados.0.smithi093.stdout:3790: writing smithi093153278-221 from 2062832 to 2654645 tid 2 2023-08-27T20:55:22.495 INFO:tasks.rados.rados.0.smithi093.stdout:3790: writing smithi093153278-221 from 3189831 to 3193924 tid 3 2023-08-27T20:55:22.495 INFO:tasks.rados.rados.0.smithi093.stdout:3791: read oid 428 snap -1 2023-08-27T20:55:22.495 INFO:tasks.rados.rados.0.smithi093.stdout:3791: expect deleted 2023-08-27T20:55:22.495 INFO:tasks.rados.rados.0.smithi093.stdout:3792: write oid 265 current snap is 0 2023-08-27T20:55:22.495 INFO:tasks.rados.rados.0.smithi093.stdout:3792: seq_num 2333 ranges {410731=759935,1604961=451824,2700901=509830} 2023-08-27T20:55:22.501 INFO:tasks.rados.rados.0.smithi093.stdout:3792: writing smithi093153278-265 from 410731 to 1170666 tid 1 2023-08-27T20:55:22.506 INFO:tasks.rados.rados.0.smithi093.stdout:3792: writing smithi093153278-265 from 1604961 to 2056785 tid 2 2023-08-27T20:55:22.512 INFO:tasks.rados.rados.0.smithi093.stdout:3792: writing smithi093153278-265 from 2700901 to 3210731 tid 3 2023-08-27T20:55:22.512 INFO:tasks.rados.rados.0.smithi093.stdout:3793: write oid 256 current snap is 0 2023-08-27T20:55:22.513 INFO:tasks.rados.rados.0.smithi093.stdout:3789: expect (ObjNum 1970 snap 0 seq_num 1970) 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:3790: finishing write tid 1 to smithi093153278-221 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:3790: finishing write tid 2 to smithi093153278-221 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:3790: finishing write tid 3 to smithi093153278-221 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:3790: finishing write tid 4 to smithi093153278-221 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:3790: finishing write tid 5 to smithi093153278-221 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 221 v 608 (ObjNum 2332 snap 0 seq_num 2332) dirty exists 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:3790: left oid 221 (ObjNum 2332 snap 0 seq_num 2332) 2023-08-27T20:55:22.680 INFO:tasks.rados.rados.0.smithi093.stdout:3792: finishing write tid 1 to smithi093153278-265 2023-08-27T20:55:22.681 INFO:tasks.rados.rados.0.smithi093.stdout:3793: seq_num 2334 ranges {427538=551537,1379796=595482,2444677=565231} 2023-08-27T20:55:22.685 INFO:tasks.rados.rados.0.smithi093.stdout:3793: writing smithi093153278-256 from 427538 to 979075 tid 1 2023-08-27T20:55:22.690 INFO:tasks.rados.rados.0.smithi093.stdout:3793: writing smithi093153278-256 from 1379796 to 1975278 tid 2 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3793: writing smithi093153278-256 from 2444677 to 3009908 tid 3 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3789: done (4 left) 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3790: done (3 left) 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3791: done (2 left) 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3794: write oid 361 current snap is 0 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3792: finishing write tid 2 to smithi093153278-265 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3792: finishing write tid 3 to smithi093153278-265 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3792: finishing write tid 4 to smithi093153278-265 2023-08-27T20:55:22.697 INFO:tasks.rados.rados.0.smithi093.stdout:3794: seq_num 2335 ranges {444345=626786,1838278=739140} 2023-08-27T20:55:22.702 INFO:tasks.rados.rados.0.smithi093.stdout:3794: writing smithi093153278-361 from 444345 to 1071131 tid 1 2023-08-27T20:55:22.711 INFO:tasks.rados.rados.0.smithi093.stdout:3794: writing smithi093153278-361 from 1838278 to 2577418 tid 2 2023-08-27T20:55:22.711 INFO:tasks.rados.rados.0.smithi093.stdout:3795: read oid 462 snap -1 2023-08-27T20:55:22.711 INFO:tasks.rados.rados.0.smithi093.stdout:3795: expect (ObjNum 1412 snap 0 seq_num 1412) 2023-08-27T20:55:22.711 INFO:tasks.rados.rados.0.smithi093.stdout:3796: write oid 313 current snap is 0 2023-08-27T20:55:22.712 INFO:tasks.rados.rados.0.smithi093.stdout:3792: finishing write tid 5 to smithi093153278-265 2023-08-27T20:55:22.712 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 265 v 493 (ObjNum 2333 snap 0 seq_num 2333) dirty exists 2023-08-27T20:55:22.712 INFO:tasks.rados.rados.0.smithi093.stdout:3792: left oid 265 (ObjNum 2333 snap 0 seq_num 2333) 2023-08-27T20:55:22.712 INFO:tasks.rados.rados.0.smithi093.stdout:3796: seq_num 2336 ranges {461152=702035,1896760=599151,2899523=361629} 2023-08-27T20:55:22.718 INFO:tasks.rados.rados.0.smithi093.stdout:3796: writing smithi093153278-313 from 461152 to 1163187 tid 1 2023-08-27T20:55:22.725 INFO:tasks.rados.rados.0.smithi093.stdout:3796: writing smithi093153278-313 from 1896760 to 2495911 tid 2 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3796: writing smithi093153278-313 from 2899523 to 3261152 tid 3 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3792: done (4 left) 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3797: read oid 185 snap -1 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3797: expect deleted 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3798: read oid 205 snap -1 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3798: expect (ObjNum 1554 snap 0 seq_num 1554) 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3799: read oid 51 snap -1 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3799: expect (ObjNum 2328 snap 0 seq_num 2328) 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3800: write oid 318 current snap is 0 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3793: finishing write tid 1 to smithi093153278-256 2023-08-27T20:55:22.730 INFO:tasks.rados.rados.0.smithi093.stdout:3793: finishing write tid 2 to smithi093153278-256 2023-08-27T20:55:22.731 INFO:tasks.rados.rados.0.smithi093.stdout:3793: finishing write tid 3 to smithi093153278-256 2023-08-27T20:55:22.731 INFO:tasks.rados.rados.0.smithi093.stdout:3793: finishing write tid 4 to smithi093153278-256 2023-08-27T20:55:22.731 INFO:tasks.rados.rados.0.smithi093.stdout:3800: seq_num 2337 ranges {477959=777284,1838889=742809,3210593=67366} 2023-08-27T20:55:22.736 INFO:tasks.rados.rados.0.smithi093.stdout:3800: writing smithi093153278-318 from 477959 to 1255243 tid 1 2023-08-27T20:55:22.744 INFO:tasks.rados.rados.0.smithi093.stdout:3800: writing smithi093153278-318 from 1838889 to 2581698 tid 2 2023-08-27T20:55:22.748 INFO:tasks.rados.rados.0.smithi093.stdout:3800: writing smithi093153278-318 from 3210593 to 3277959 tid 3 2023-08-27T20:55:22.748 INFO:tasks.rados.rados.0.smithi093.stdout:3801: write oid 21 current snap is 0 2023-08-27T20:55:22.748 INFO:tasks.rados.rados.0.smithi093.stdout:3793: finishing write tid 5 to smithi093153278-256 2023-08-27T20:55:22.749 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 256 v 462 (ObjNum 2334 snap 0 seq_num 2334) dirty exists 2023-08-27T20:55:22.749 INFO:tasks.rados.rados.0.smithi093.stdout:3793: left oid 256 (ObjNum 2334 snap 0 seq_num 2334) 2023-08-27T20:55:22.749 INFO:tasks.rados.rados.0.smithi093.stdout:3794: finishing write tid 1 to smithi093153278-361 2023-08-27T20:55:22.749 INFO:tasks.rados.rados.0.smithi093.stdout:3801: seq_num 2338 ranges {494766=452533,1497371=602820,2838016=456750} 2023-08-27T20:55:22.753 INFO:tasks.rados.rados.0.smithi093.stdout:3801: writing smithi093153278-21 from 494766 to 947299 tid 1 2023-08-27T20:55:22.759 INFO:tasks.rados.rados.0.smithi093.stdout:3801: writing smithi093153278-21 from 1497371 to 2100191 tid 2 2023-08-27T20:55:22.766 INFO:tasks.rados.rados.0.smithi093.stdout:3801: writing smithi093153278-21 from 2838016 to 3294766 tid 3 2023-08-27T20:55:22.766 INFO:tasks.rados.rados.0.smithi093.stdout:3793: done (8 left) 2023-08-27T20:55:22.766 INFO:tasks.rados.rados.0.smithi093.stdout:3802: write oid 95 current snap is 0 2023-08-27T20:55:22.766 INFO:tasks.rados.rados.0.smithi093.stdout:3802: seq_num 2339 ranges {511573=527782,1555853=746478,2865439=446134} 2023-08-27T20:55:22.772 INFO:tasks.rados.rados.0.smithi093.stdout:3802: writing smithi093153278-95 from 511573 to 1039355 tid 1 2023-08-27T20:55:22.779 INFO:tasks.rados.rados.0.smithi093.stdout:3802: writing smithi093153278-95 from 1555853 to 2302331 tid 2 2023-08-27T20:55:22.785 INFO:tasks.rados.rados.0.smithi093.stdout:3802: writing smithi093153278-95 from 2865439 to 3311573 tid 3 2023-08-27T20:55:22.785 INFO:tasks.rados.rados.0.smithi093.stdout:3803: read oid 255 snap -1 2023-08-27T20:55:22.785 INFO:tasks.rados.rados.0.smithi093.stdout:3803: expect (ObjNum 2175 snap 0 seq_num 2175) 2023-08-27T20:55:22.785 INFO:tasks.rados.rados.0.smithi093.stdout:3804: read oid 492 snap -1 2023-08-27T20:55:22.785 INFO:tasks.rados.rados.0.smithi093.stdout:3804: expect (ObjNum 1968 snap 0 seq_num 1968) 2023-08-27T20:55:22.785 INFO:tasks.rados.rados.0.smithi093.stdout:3805: delete oid 466 current snap is 0 2023-08-27T20:55:22.785 INFO:tasks.rados.rados.0.smithi093.stdout:3795: expect (ObjNum 1412 snap 0 seq_num 1412) 2023-08-27T20:55:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2712415640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2712415640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:22.997 INFO:tasks.rados.rados.0.smithi093.stdout:3794: finishing write tid 2 to smithi093153278-361 2023-08-27T20:55:22.997 INFO:tasks.rados.rados.0.smithi093.stdout:3794: finishing write tid 3 to smithi093153278-361 2023-08-27T20:55:22.997 INFO:tasks.rados.rados.0.smithi093.stdout:3794: finishing write tid 4 to smithi093153278-361 2023-08-27T20:55:22.997 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 361 v 415 (ObjNum 2335 snap 0 seq_num 2335) dirty exists 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3794: left oid 361 (ObjNum 2335 snap 0 seq_num 2335) 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3796: finishing write tid 1 to smithi093153278-313 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3794: done (11 left) 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3795: done (10 left) 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3805: done (9 left) 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3806: delete oid 465 current snap is 0 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3796: finishing write tid 2 to smithi093153278-313 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3796: finishing write tid 3 to smithi093153278-313 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3796: finishing write tid 4 to smithi093153278-313 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3796: finishing write tid 5 to smithi093153278-313 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 313 v 292 (ObjNum 2336 snap 0 seq_num 2336) dirty exists 2023-08-27T20:55:22.998 INFO:tasks.rados.rados.0.smithi093.stdout:3796: left oid 313 (ObjNum 2336 snap 0 seq_num 2336) 2023-08-27T20:55:22.999 INFO:tasks.rados.rados.0.smithi093.stdout:3800: finishing write tid 1 to smithi093153278-318 2023-08-27T20:55:22.999 INFO:tasks.rados.rados.0.smithi093.stdout:3798: expect (ObjNum 1554 snap 0 seq_num 1554) 2023-08-27T20:55:23.134 INFO:tasks.rados.rados.0.smithi093.stdout:3799: expect (ObjNum 2328 snap 0 seq_num 2328) 2023-08-27T20:55:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2712415640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:23.362 INFO:tasks.rados.rados.0.smithi093.stdout:3800: finishing write tid 2 to smithi093153278-318 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3800: finishing write tid 3 to smithi093153278-318 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3800: finishing write tid 4 to smithi093153278-318 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3800: finishing write tid 5 to smithi093153278-318 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 318 v 466 (ObjNum 2337 snap 0 seq_num 2337) dirty exists 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3800: left oid 318 (ObjNum 2337 snap 0 seq_num 2337) 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3801: finishing write tid 1 to smithi093153278-21 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3796: done (9 left) 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3797: done (8 left) 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3798: done (7 left) 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3799: done (6 left) 2023-08-27T20:55:23.363 INFO:tasks.rados.rados.0.smithi093.stdout:3800: done (5 left) 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3806: done (4 left) 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3807: read oid 280 snap -1 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3807: expect (ObjNum 1801 snap 0 seq_num 1801) 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3808: read oid 76 snap -1 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3808: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3809: read oid 35 snap -1 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3809: expect (ObjNum 1952 snap 0 seq_num 1952) 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3810: write oid 389 current snap is 0 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3801: finishing write tid 2 to smithi093153278-21 2023-08-27T20:55:23.364 INFO:tasks.rados.rados.0.smithi093.stdout:3810: seq_num 2342 ranges {561994=469882,1731299=610158,2947708=414286} 2023-08-27T20:55:23.368 INFO:tasks.rados.rados.0.smithi093.stdout:3810: writing smithi093153278-389 from 561994 to 1031876 tid 1 2023-08-27T20:55:23.375 INFO:tasks.rados.rados.0.smithi093.stdout:3810: writing smithi093153278-389 from 1731299 to 2341457 tid 2 2023-08-27T20:55:23.381 INFO:tasks.rados.rados.0.smithi093.stdout:3810: writing smithi093153278-389 from 2947708 to 3361994 tid 3 2023-08-27T20:55:23.381 INFO:tasks.rados.rados.0.smithi093.stdout:3811: write oid 235 current snap is 0 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3801: finishing write tid 3 to smithi093153278-21 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3801: finishing write tid 4 to smithi093153278-21 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3801: finishing write tid 5 to smithi093153278-21 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 21 v 544 (ObjNum 2338 snap 0 seq_num 2338) dirty exists 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3801: left oid 21 (ObjNum 2338 snap 0 seq_num 2338) 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3802: finishing write tid 1 to smithi093153278-95 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3802: finishing write tid 2 to smithi093153278-95 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3802: finishing write tid 3 to smithi093153278-95 2023-08-27T20:55:23.382 INFO:tasks.rados.rados.0.smithi093.stdout:3811: seq_num 2343 ranges {578801=545131,1789781=753816,2975131=403670} 2023-08-27T20:55:23.387 INFO:tasks.rados.rados.0.smithi093.stdout:3811: writing smithi093153278-235 from 578801 to 1123932 tid 1 2023-08-27T20:55:23.395 INFO:tasks.rados.rados.0.smithi093.stdout:3811: writing smithi093153278-235 from 1789781 to 2543597 tid 2 2023-08-27T20:55:23.400 INFO:tasks.rados.rados.0.smithi093.stdout:3811: writing smithi093153278-235 from 2975131 to 3378801 tid 3 2023-08-27T20:55:23.400 INFO:tasks.rados.rados.0.smithi093.stdout:3801: done (8 left) 2023-08-27T20:55:23.400 INFO:tasks.rados.rados.0.smithi093.stdout:3812: read oid 320 snap -1 2023-08-27T20:55:23.400 INFO:tasks.rados.rados.0.smithi093.stdout:3812: expect (ObjNum 2122 snap 0 seq_num 2122) 2023-08-27T20:55:23.400 INFO:tasks.rados.rados.0.smithi093.stdout:3813: write oid 109 current snap is 0 2023-08-27T20:55:23.400 INFO:tasks.rados.rados.0.smithi093.stdout:3802: finishing write tid 4 to smithi093153278-95 2023-08-27T20:55:23.400 INFO:tasks.rados.rados.0.smithi093.stdout:3813: seq_num 2344 ranges {595608=620380,1848263=613827,3002554=393054} 2023-08-27T20:55:23.406 INFO:tasks.rados.rados.0.smithi093.stdout:3813: writing smithi093153278-109 from 595608 to 1215988 tid 1 2023-08-27T20:55:23.413 INFO:tasks.rados.rados.0.smithi093.stdout:3813: writing smithi093153278-109 from 1848263 to 2462090 tid 2 2023-08-27T20:55:23.419 INFO:tasks.rados.rados.0.smithi093.stdout:3813: writing smithi093153278-109 from 3002554 to 3395608 tid 3 2023-08-27T20:55:23.419 INFO:tasks.rados.rados.0.smithi093.stdout:3814: write oid 3 current snap is 0 2023-08-27T20:55:23.419 INFO:tasks.rados.rados.0.smithi093.stdout:3802: finishing write tid 5 to smithi093153278-95 2023-08-27T20:55:23.419 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 95 v 539 (ObjNum 2339 snap 0 seq_num 2339) dirty exists 2023-08-27T20:55:23.419 INFO:tasks.rados.rados.0.smithi093.stdout:3802: left oid 95 (ObjNum 2339 snap 0 seq_num 2339) 2023-08-27T20:55:23.419 INFO:tasks.rados.rados.0.smithi093.stdout:3803: expect (ObjNum 2175 snap 0 seq_num 2175) 2023-08-27T20:55:23.599 INFO:tasks.rados.rados.0.smithi093.stdout:3804: expect (ObjNum 1968 snap 0 seq_num 1968) 2023-08-27T20:55:23.848 INFO:tasks.rados.rados.0.smithi093.stdout:3809: expect (ObjNum 1952 snap 0 seq_num 1952) 2023-08-27T20:55:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:23 smithi093 ceph-mon[111335]: pgmap v55: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:23 smithi093 ceph-mon[118902]: pgmap v55: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:23.519Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:23.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:24.007 INFO:tasks.rados.rados.0.smithi093.stdout:3810: finishing write tid 1 to smithi093153278-389 2023-08-27T20:55:24.007 INFO:tasks.rados.rados.0.smithi093.stdout:3807: expect (ObjNum 1801 snap 0 seq_num 1801) 2023-08-27T20:55:24.125 INFO:tasks.rados.rados.0.smithi093.stdout:3808: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:55:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:23 smithi136 ceph-mon[113531]: pgmap v55: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:24.229 INFO:tasks.rados.rados.0.smithi093.stdout:3810: finishing write tid 2 to smithi093153278-389 2023-08-27T20:55:24.230 INFO:tasks.rados.rados.0.smithi093.stdout:3810: finishing write tid 3 to smithi093153278-389 2023-08-27T20:55:24.230 INFO:tasks.rados.rados.0.smithi093.stdout:3810: finishing write tid 4 to smithi093153278-389 2023-08-27T20:55:24.230 INFO:tasks.rados.rados.0.smithi093.stdout:3810: finishing write tid 5 to smithi093153278-389 2023-08-27T20:55:24.230 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 389 v 548 (ObjNum 2342 snap 0 seq_num 2342) dirty exists 2023-08-27T20:55:24.230 INFO:tasks.rados.rados.0.smithi093.stdout:3810: left oid 389 (ObjNum 2342 snap 0 seq_num 2342) 2023-08-27T20:55:24.230 INFO:tasks.rados.rados.0.smithi093.stdout:3811: finishing write tid 1 to smithi093153278-235 2023-08-27T20:55:24.230 INFO:tasks.rados.rados.0.smithi093.stdout:3814: seq_num 2345 ranges {612415=695629,1790392=757485,3313624=98791} 2023-08-27T20:55:24.236 INFO:tasks.rados.rados.0.smithi093.stdout:3814: writing smithi093153278-3 from 612415 to 1308044 tid 1 2023-08-27T20:55:24.243 INFO:tasks.rados.rados.0.smithi093.stdout:3814: writing smithi093153278-3 from 1790392 to 2547877 tid 2 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3814: writing smithi093153278-3 from 3313624 to 3412415 tid 3 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3802: done (10 left) 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3803: done (9 left) 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3804: done (8 left) 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3807: done (7 left) 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3808: done (6 left) 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3809: done (5 left) 2023-08-27T20:55:24.249 INFO:tasks.rados.rados.0.smithi093.stdout:3810: done (4 left) 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:3815: write oid 448 current snap is 0 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:3811: finishing write tid 2 to smithi093153278-235 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:3811: finishing write tid 3 to smithi093153278-235 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:3811: finishing write tid 4 to smithi093153278-235 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:3811: finishing write tid 5 to smithi093153278-235 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 235 v 441 (ObjNum 2343 snap 0 seq_num 2343) dirty exists 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:3811: left oid 235 (ObjNum 2343 snap 0 seq_num 2343) 2023-08-27T20:55:24.250 INFO:tasks.rados.rados.0.smithi093.stdout:3812: expect (ObjNum 2122 snap 0 seq_num 2122) 2023-08-27T20:55:24.367 INFO:tasks.rados.rados.0.smithi093.stdout:3813: finishing write tid 1 to smithi093153278-109 2023-08-27T20:55:24.367 INFO:tasks.rados.rados.0.smithi093.stdout:3813: finishing write tid 2 to smithi093153278-109 2023-08-27T20:55:24.367 INFO:tasks.rados.rados.0.smithi093.stdout:3815: seq_num 2346 ranges {629222=770878,1848874=501143,2824694=533201} 2023-08-27T20:55:24.374 INFO:tasks.rados.rados.0.smithi093.stdout:3815: writing smithi093153278-448 from 629222 to 1400100 tid 1 2023-08-27T20:55:24.379 INFO:tasks.rados.rados.0.smithi093.stdout:3815: writing smithi093153278-448 from 1848874 to 2350017 tid 2 2023-08-27T20:55:24.386 INFO:tasks.rados.rados.0.smithi093.stdout:3815: writing smithi093153278-448 from 2824694 to 3357895 tid 3 2023-08-27T20:55:24.386 INFO:tasks.rados.rados.0.smithi093.stdout:3811: done (4 left) 2023-08-27T20:55:24.386 INFO:tasks.rados.rados.0.smithi093.stdout:3812: done (3 left) 2023-08-27T20:55:24.386 INFO:tasks.rados.rados.0.smithi093.stdout:3813: finishing write tid 3 to smithi093153278-109 2023-08-27T20:55:24.387 INFO:tasks.rados.rados.0.smithi093.stdout:3813: finishing write tid 4 to smithi093153278-109 2023-08-27T20:55:24.387 INFO:tasks.rados.rados.0.smithi093.stdout:3816: read oid 174 snap -1 2023-08-27T20:55:24.387 INFO:tasks.rados.rados.0.smithi093.stdout:3816: expect (ObjNum 1988 snap 0 seq_num 1988) 2023-08-27T20:55:24.387 INFO:tasks.rados.rados.0.smithi093.stdout:3817: write oid 452 current snap is 0 2023-08-27T20:55:24.387 INFO:tasks.rados.rados.0.smithi093.stdout:3813: finishing write tid 5 to smithi093153278-109 2023-08-27T20:55:24.387 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 109 v 497 (ObjNum 2344 snap 0 seq_num 2344) dirty exists 2023-08-27T20:55:24.387 INFO:tasks.rados.rados.0.smithi093.stdout:3813: left oid 109 (ObjNum 2344 snap 0 seq_num 2344) 2023-08-27T20:55:24.388 INFO:tasks.rados.rados.0.smithi093.stdout:3814: finishing write tid 1 to smithi093153278-3 2023-08-27T20:55:24.388 INFO:tasks.rados.rados.0.smithi093.stdout:3814: finishing write tid 2 to smithi093153278-3 2023-08-27T20:55:24.388 INFO:tasks.rados.rados.0.smithi093.stdout:3817: seq_num 2347 ranges {646029=446127,1507356=761154,2968470=477559} 2023-08-27T20:55:24.391 INFO:tasks.rados.rados.0.smithi093.stdout:3817: writing smithi093153278-452 from 646029 to 1092156 tid 1 2023-08-27T20:55:24.398 INFO:tasks.rados.rados.0.smithi093.stdout:3817: writing smithi093153278-452 from 1507356 to 2268510 tid 2 2023-08-27T20:55:24.405 INFO:tasks.rados.rados.0.smithi093.stdout:3817: writing smithi093153278-452 from 2968470 to 3446029 tid 3 2023-08-27T20:55:24.405 INFO:tasks.rados.rados.0.smithi093.stdout:3813: done (4 left) 2023-08-27T20:55:24.405 INFO:tasks.rados.rados.0.smithi093.stdout:3818: read oid 81 snap -1 2023-08-27T20:55:24.405 INFO:tasks.rados.rados.0.smithi093.stdout:3818: expect (ObjNum 2327 snap 0 seq_num 2327) 2023-08-27T20:55:24.406 INFO:tasks.rados.rados.0.smithi093.stdout:3819: write oid 385 current snap is 0 2023-08-27T20:55:24.406 INFO:tasks.rados.rados.0.smithi093.stdout:3814: finishing write tid 3 to smithi093153278-3 2023-08-27T20:55:24.406 INFO:tasks.rados.rados.0.smithi093.stdout:3814: finishing write tid 4 to smithi093153278-3 2023-08-27T20:55:24.406 INFO:tasks.rados.rados.0.smithi093.stdout:3814: finishing write tid 5 to smithi093153278-3 2023-08-27T20:55:24.406 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 3 v 552 (ObjNum 2345 snap 0 seq_num 2345) dirty exists 2023-08-27T20:55:24.406 INFO:tasks.rados.rados.0.smithi093.stdout:3814: left oid 3 (ObjNum 2345 snap 0 seq_num 2345) 2023-08-27T20:55:24.407 INFO:tasks.rados.rados.0.smithi093.stdout:3815: finishing write tid 1 to smithi093153278-448 2023-08-27T20:55:24.407 INFO:tasks.rados.rados.0.smithi093.stdout:3815: finishing write tid 2 to smithi093153278-448 2023-08-27T20:55:24.407 INFO:tasks.rados.rados.0.smithi093.stdout:3815: finishing write tid 3 to smithi093153278-448 2023-08-27T20:55:24.407 INFO:tasks.rados.rados.0.smithi093.stdout:3815: finishing write tid 4 to smithi093153278-448 2023-08-27T20:55:24.407 INFO:tasks.rados.rados.0.smithi093.stdout:3815: finishing write tid 5 to smithi093153278-448 2023-08-27T20:55:24.407 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 448 v 470 (ObjNum 2346 snap 0 seq_num 2346) dirty exists 2023-08-27T20:55:24.407 INFO:tasks.rados.rados.0.smithi093.stdout:3815: left oid 448 (ObjNum 2346 snap 0 seq_num 2346) 2023-08-27T20:55:24.408 INFO:tasks.rados.rados.0.smithi093.stdout:3817: finishing write tid 1 to smithi093153278-452 2023-08-27T20:55:24.408 INFO:tasks.rados.rados.0.smithi093.stdout:3816: expect (ObjNum 1988 snap 0 seq_num 1988) 2023-08-27T20:55:24.554 INFO:tasks.rados.rados.0.smithi093.stdout:3817: finishing write tid 2 to smithi093153278-452 2023-08-27T20:55:24.554 INFO:tasks.rados.rados.0.smithi093.stdout:3817: finishing write tid 3 to smithi093153278-452 2023-08-27T20:55:24.554 INFO:tasks.rados.rados.0.smithi093.stdout:3817: finishing write tid 4 to smithi093153278-452 2023-08-27T20:55:24.555 INFO:tasks.rados.rados.0.smithi093.stdout:3817: finishing write tid 5 to smithi093153278-452 2023-08-27T20:55:24.555 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 452 v 543 (ObjNum 2347 snap 0 seq_num 2347) dirty exists 2023-08-27T20:55:24.555 INFO:tasks.rados.rados.0.smithi093.stdout:3817: left oid 452 (ObjNum 2347 snap 0 seq_num 2347) 2023-08-27T20:55:24.555 INFO:tasks.rados.rados.0.smithi093.stdout:3818: expect (ObjNum 2327 snap 0 seq_num 2327) 2023-08-27T20:55:24.773 INFO:tasks.rados.rados.0.smithi093.stdout:3819: seq_num 2348 ranges {662836=521376,1965838=504812,2879540=583296} 2023-08-27T20:55:24.779 INFO:tasks.rados.rados.0.smithi093.stdout:3819: writing smithi093153278-385 from 662836 to 1184212 tid 1 2023-08-27T20:55:24.786 INFO:tasks.rados.rados.0.smithi093.stdout:3819: writing smithi093153278-385 from 1965838 to 2470650 tid 2 2023-08-27T20:55:24.792 INFO:tasks.rados.rados.0.smithi093.stdout:3819: writing smithi093153278-385 from 2879540 to 3462836 tid 3 2023-08-27T20:55:24.792 INFO:tasks.rados.rados.0.smithi093.stdout:3814: done (5 left) 2023-08-27T20:55:24.792 INFO:tasks.rados.rados.0.smithi093.stdout:3815: done (4 left) 2023-08-27T20:55:24.792 INFO:tasks.rados.rados.0.smithi093.stdout:3816: done (3 left) 2023-08-27T20:55:24.792 INFO:tasks.rados.rados.0.smithi093.stdout:3817: done (2 left) 2023-08-27T20:55:24.792 INFO:tasks.rados.rados.0.smithi093.stdout:3818: done (1 left) 2023-08-27T20:55:24.792 INFO:tasks.rados.rados.0.smithi093.stdout:3820: write oid 429 current snap is 0 2023-08-27T20:55:24.793 INFO:tasks.rados.rados.0.smithi093.stdout:3819: finishing write tid 1 to smithi093153278-385 2023-08-27T20:55:24.793 INFO:tasks.rados.rados.0.smithi093.stdout:3819: finishing write tid 2 to smithi093153278-385 2023-08-27T20:55:24.793 INFO:tasks.rados.rados.0.smithi093.stdout:3820: seq_num 2349 ranges {679643=596625,1907967=764823,3306963=172680} 2023-08-27T20:55:24.798 INFO:tasks.rados.rados.0.smithi093.stdout:3820: writing smithi093153278-429 from 679643 to 1276268 tid 1 2023-08-27T20:55:24.806 INFO:tasks.rados.rados.0.smithi093.stdout:3820: writing smithi093153278-429 from 1907967 to 2672790 tid 2 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:3820: writing smithi093153278-429 from 3306963 to 3479643 tid 3 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:3821: write oid 238 current snap is 0 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:3819: finishing write tid 3 to smithi093153278-385 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:3819: finishing write tid 4 to smithi093153278-385 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:3819: finishing write tid 5 to smithi093153278-385 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 385 v 547 (ObjNum 2348 snap 0 seq_num 2348) dirty exists 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:3819: left oid 385 (ObjNum 2348 snap 0 seq_num 2348) 2023-08-27T20:55:24.812 INFO:tasks.rados.rados.0.smithi093.stdout:3820: finishing write tid 1 to smithi093153278-429 2023-08-27T20:55:24.813 INFO:tasks.rados.rados.0.smithi093.stdout:3820: finishing write tid 2 to smithi093153278-429 2023-08-27T20:55:24.813 INFO:tasks.rados.rados.0.smithi093.stdout:3821: seq_num 2350 ranges {696450=788227,2082802=508481,3050739=445711} 2023-08-27T20:55:24.819 INFO:tasks.rados.rados.0.smithi093.stdout:3821: writing smithi093153278-238 from 696450 to 1484677 tid 1 2023-08-27T20:55:24.825 INFO:tasks.rados.rados.0.smithi093.stdout:3821: writing smithi093153278-238 from 2082802 to 2591283 tid 2 2023-08-27T20:55:24.831 INFO:tasks.rados.rados.0.smithi093.stdout:3821: writing smithi093153278-238 from 3050739 to 3496450 tid 3 2023-08-27T20:55:24.831 INFO:tasks.rados.rados.0.smithi093.stdout:3819: done (2 left) 2023-08-27T20:55:24.831 INFO:tasks.rados.rados.0.smithi093.stdout:3822: read oid 76 snap -1 2023-08-27T20:55:24.831 INFO:tasks.rados.rados.0.smithi093.stdout:3822: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3823: read oid 162 snap -1 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3823: expect (ObjNum 2262 snap 0 seq_num 2262) 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3824: read oid 62 snap -1 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3824: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3825: write oid 349 current snap is 0 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3820: finishing write tid 3 to smithi093153278-429 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3820: finishing write tid 4 to smithi093153278-429 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3820: finishing write tid 5 to smithi093153278-429 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 429 v 501 (ObjNum 2349 snap 0 seq_num 2349) dirty exists 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3820: left oid 429 (ObjNum 2349 snap 0 seq_num 2349) 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3821: finishing write tid 1 to smithi093153278-238 2023-08-27T20:55:24.832 INFO:tasks.rados.rados.0.smithi093.stdout:3821: finishing write tid 2 to smithi093153278-238 2023-08-27T20:55:24.833 INFO:tasks.rados.rados.0.smithi093.stdout:3825: seq_num 2351 ranges {713257=463476,1741284=768492,3078162=435095} 2023-08-27T20:55:24.837 INFO:tasks.rados.rados.0.smithi093.stdout:3825: writing smithi093153278-349 from 713257 to 1176733 tid 1 2023-08-27T20:55:24.845 INFO:tasks.rados.rados.0.smithi093.stdout:3825: writing smithi093153278-349 from 1741284 to 2509776 tid 2 2023-08-27T20:55:24.851 INFO:tasks.rados.rados.0.smithi093.stdout:3825: writing smithi093153278-349 from 3078162 to 3513257 tid 3 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3820: done (5 left) 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3826: read oid 251 snap -1 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3826: expect (ObjNum 1546 snap 0 seq_num 1546) 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3827: write oid 320 current snap is 0 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3821: finishing write tid 3 to smithi093153278-238 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3821: finishing write tid 4 to smithi093153278-238 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3821: finishing write tid 5 to smithi093153278-238 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 238 v 613 (ObjNum 2350 snap 0 seq_num 2350) dirty exists 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3821: left oid 238 (ObjNum 2350 snap 0 seq_num 2350) 2023-08-27T20:55:24.852 INFO:tasks.rados.rados.0.smithi093.stdout:3824: expect (ObjNum 888 snap 0 seq_num 888) 2023-08-27T20:55:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4047380635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4047380635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:25.053 INFO:tasks.rados.rados.0.smithi093.stdout:3825: finishing write tid 1 to smithi093153278-349 2023-08-27T20:55:25.054 INFO:tasks.rados.rados.0.smithi093.stdout:3823: expect (ObjNum 2262 snap 0 seq_num 2262) 2023-08-27T20:55:25.181 INFO:tasks.rados.rados.0.smithi093.stdout:3822: expect (ObjNum 75 snap 0 seq_num 75) 2023-08-27T20:55:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4047380635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:25.280 INFO:tasks.rados.rados.0.smithi093.stdout:3825: finishing write tid 2 to smithi093153278-349 2023-08-27T20:55:25.280 INFO:tasks.rados.rados.0.smithi093.stdout:3825: finishing write tid 3 to smithi093153278-349 2023-08-27T20:55:25.280 INFO:tasks.rados.rados.0.smithi093.stdout:3825: finishing write tid 4 to smithi093153278-349 2023-08-27T20:55:25.280 INFO:tasks.rados.rados.0.smithi093.stdout:3825: finishing write tid 5 to smithi093153278-349 2023-08-27T20:55:25.280 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 349 v 556 (ObjNum 2351 snap 0 seq_num 2351) dirty exists 2023-08-27T20:55:25.280 INFO:tasks.rados.rados.0.smithi093.stdout:3825: left oid 349 (ObjNum 2351 snap 0 seq_num 2351) 2023-08-27T20:55:25.280 INFO:tasks.rados.rados.0.smithi093.stdout:3827: seq_num 2352 ranges {730064=538725,1799766=512150,3105585=424479} 2023-08-27T20:55:25.286 INFO:tasks.rados.rados.0.smithi093.stdout:3827: writing smithi093153278-320 from 730064 to 1268789 tid 1 2023-08-27T20:55:25.292 INFO:tasks.rados.rados.0.smithi093.stdout:3827: writing smithi093153278-320 from 1799766 to 2311916 tid 2 2023-08-27T20:55:25.299 INFO:tasks.rados.rados.0.smithi093.stdout:3827: writing smithi093153278-320 from 3105585 to 3530064 tid 3 2023-08-27T20:55:25.299 INFO:tasks.rados.rados.0.smithi093.stdout:3821: done (6 left) 2023-08-27T20:55:25.299 INFO:tasks.rados.rados.0.smithi093.stdout:3822: done (5 left) 2023-08-27T20:55:25.299 INFO:tasks.rados.rados.0.smithi093.stdout:3823: done (4 left) 2023-08-27T20:55:25.300 INFO:tasks.rados.rados.0.smithi093.stdout:3824: done (3 left) 2023-08-27T20:55:25.300 INFO:tasks.rados.rados.0.smithi093.stdout:3825: done (2 left) 2023-08-27T20:55:25.300 INFO:tasks.rados.rados.0.smithi093.stdout:3828: delete oid 117 current snap is 0 2023-08-27T20:55:25.300 INFO:tasks.rados.rados.0.smithi093.stdout:3826: expect (ObjNum 1546 snap 0 seq_num 1546) 2023-08-27T20:55:25.486 INFO:tasks.rados.rados.0.smithi093.stdout:3827: finishing write tid 1 to smithi093153278-320 2023-08-27T20:55:25.486 INFO:tasks.rados.rados.0.smithi093.stdout:3827: finishing write tid 2 to smithi093153278-320 2023-08-27T20:55:25.486 INFO:tasks.rados.rados.0.smithi093.stdout:3827: finishing write tid 3 to smithi093153278-320 2023-08-27T20:55:25.486 INFO:tasks.rados.rados.0.smithi093.stdout:3827: finishing write tid 4 to smithi093153278-320 2023-08-27T20:55:25.486 INFO:tasks.rados.rados.0.smithi093.stdout:3827: finishing write tid 5 to smithi093153278-320 2023-08-27T20:55:25.486 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 320 v 617 (ObjNum 2352 snap 0 seq_num 2352) dirty exists 2023-08-27T20:55:25.486 INFO:tasks.rados.rados.0.smithi093.stdout:3827: left oid 320 (ObjNum 2352 snap 0 seq_num 2352) 2023-08-27T20:55:25.487 INFO:tasks.rados.rados.0.smithi093.stdout:3826: done (2 left) 2023-08-27T20:55:25.487 INFO:tasks.rados.rados.0.smithi093.stdout:3827: done (1 left) 2023-08-27T20:55:25.487 INFO:tasks.rados.rados.0.smithi093.stdout:3828: done (0 left) 2023-08-27T20:55:25.487 INFO:tasks.rados.rados.0.smithi093.stdout:3829: write oid 87 current snap is 0 2023-08-27T20:55:25.487 INFO:tasks.rados.rados.0.smithi093.stdout:3829: seq_num 2354 ranges {763678=689223,2200377=515819,3444078=119600} 2023-08-27T20:55:25.493 INFO:tasks.rados.rados.0.smithi093.stdout:3829: writing smithi093153278-87 from 763678 to 1452901 tid 1 2023-08-27T20:55:25.500 INFO:tasks.rados.rados.0.smithi093.stdout:3829: writing smithi093153278-87 from 2200377 to 2716196 tid 2 2023-08-27T20:55:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:3829: writing smithi093153278-87 from 3444078 to 3563678 tid 3 2023-08-27T20:55:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:3830: write oid 93 current snap is 0 2023-08-27T20:55:25.505 INFO:tasks.rados.rados.0.smithi093.stdout:3829: finishing write tid 1 to smithi093153278-87 2023-08-27T20:55:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:3829: finishing write tid 2 to smithi093153278-87 2023-08-27T20:55:25.506 INFO:tasks.rados.rados.0.smithi093.stdout:3830: seq_num 2355 ranges {780485=764472,2258859=775830,3471501=108984} 2023-08-27T20:55:25.513 INFO:tasks.rados.rados.0.smithi093.stdout:3830: writing smithi093153278-93 from 780485 to 1544957 tid 1 2023-08-27T20:55:25.521 INFO:tasks.rados.rados.0.smithi093.stdout:3830: writing smithi093153278-93 from 2258859 to 3034689 tid 2 2023-08-27T20:55:25.525 INFO:tasks.rados.rados.0.smithi093.stdout:3830: writing smithi093153278-93 from 3471501 to 3580485 tid 3 2023-08-27T20:55:25.525 INFO:tasks.rados.rados.0.smithi093.stdout:3831: write oid 309 current snap is 0 2023-08-27T20:55:25.525 INFO:tasks.rados.rados.0.smithi093.stdout:3829: finishing write tid 3 to smithi093153278-87 2023-08-27T20:55:25.525 INFO:tasks.rados.rados.0.smithi093.stdout:3829: finishing write tid 4 to smithi093153278-87 2023-08-27T20:55:25.525 INFO:tasks.rados.rados.0.smithi093.stdout:3829: finishing write tid 5 to smithi093153278-87 2023-08-27T20:55:25.526 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 434 (ObjNum 2354 snap 0 seq_num 2354) dirty exists 2023-08-27T20:55:25.526 INFO:tasks.rados.rados.0.smithi093.stdout:3829: left oid 87 (ObjNum 2354 snap 0 seq_num 2354) 2023-08-27T20:55:25.526 INFO:tasks.rados.rados.0.smithi093.stdout:3830: finishing write tid 1 to smithi093153278-93 2023-08-27T20:55:25.526 INFO:tasks.rados.rados.0.smithi093.stdout:3830: finishing write tid 2 to smithi093153278-93 2023-08-27T20:55:25.526 INFO:tasks.rados.rados.0.smithi093.stdout:3831: seq_num 2356 ranges {797292=439721,1917341=519488,3098924=478627} 2023-08-27T20:55:25.531 INFO:tasks.rados.rados.0.smithi093.stdout:3831: writing smithi093153278-309 from 797292 to 1237013 tid 1 2023-08-27T20:55:25.538 INFO:tasks.rados.rados.0.smithi093.stdout:3831: writing smithi093153278-309 from 1917341 to 2436829 tid 2 2023-08-27T20:55:25.544 INFO:tasks.rados.rados.0.smithi093.stdout:3831: writing smithi093153278-309 from 3098924 to 3577551 tid 3 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3829: done (2 left) 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3832: write oid 492 current snap is 0 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3830: finishing write tid 3 to smithi093153278-93 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3830: finishing write tid 4 to smithi093153278-93 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3830: finishing write tid 5 to smithi093153278-93 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 93 v 419 (ObjNum 2355 snap 0 seq_num 2355) dirty exists 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3830: left oid 93 (ObjNum 2355 snap 0 seq_num 2355) 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3831: finishing write tid 1 to smithi093153278-309 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3831: finishing write tid 2 to smithi093153278-309 2023-08-27T20:55:25.545 INFO:tasks.rados.rados.0.smithi093.stdout:3832: seq_num 2357 ranges {414099=631323,1575823=779499,3126347=406252} 2023-08-27T20:55:25.550 INFO:tasks.rados.rados.0.smithi093.stdout:3832: writing smithi093153278-492 from 414099 to 1045422 tid 1 2023-08-27T20:55:25.557 INFO:tasks.rados.rados.0.smithi093.stdout:3832: writing smithi093153278-492 from 1575823 to 2355322 tid 2 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3832: writing smithi093153278-492 from 3126347 to 3532599 tid 3 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3830: done (2 left) 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3833: delete oid 131 current snap is 0 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3831: finishing write tid 3 to smithi093153278-309 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3831: finishing write tid 4 to smithi093153278-309 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3831: finishing write tid 5 to smithi093153278-309 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 309 v 416 (ObjNum 2356 snap 0 seq_num 2356) dirty exists 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3831: left oid 309 (ObjNum 2356 snap 0 seq_num 2356) 2023-08-27T20:55:25.564 INFO:tasks.rados.rados.0.smithi093.stdout:3832: finishing write tid 1 to smithi093153278-492 2023-08-27T20:55:25.565 INFO:tasks.rados.rados.0.smithi093.stdout:3832: finishing write tid 2 to smithi093153278-492 2023-08-27T20:55:25.565 INFO:tasks.rados.rados.0.smithi093.stdout:3831: done (2 left) 2023-08-27T20:55:25.565 INFO:tasks.rados.rados.0.smithi093.stdout:3833: done (1 left) 2023-08-27T20:55:25.565 INFO:tasks.rados.rados.0.smithi093.stdout:3834: read oid 334 snap -1 2023-08-27T20:55:25.566 INFO:tasks.rados.rados.0.smithi093.stdout:3834: expect deleted 2023-08-27T20:55:25.566 INFO:tasks.rados.rados.0.smithi093.stdout:3835: read oid 470 snap -1 2023-08-27T20:55:25.566 INFO:tasks.rados.rados.0.smithi093.stdout:3835: expect (ObjNum 1796 snap 0 seq_num 1796) 2023-08-27T20:55:25.566 INFO:tasks.rados.rados.0.smithi093.stdout:3836: write oid 149 current snap is 0 2023-08-27T20:55:25.566 INFO:tasks.rados.rados.0.smithi093.stdout:3836: seq_num 2359 ranges {447713=781821,1692787=666815,3064840=428796} 2023-08-27T20:55:25.571 INFO:tasks.rados.rados.0.smithi093.stdout:3836: writing smithi093153278-149 from 447713 to 1229534 tid 1 2023-08-27T20:55:25.578 INFO:tasks.rados.rados.0.smithi093.stdout:3836: writing smithi093153278-149 from 1692787 to 2359602 tid 2 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3836: writing smithi093153278-149 from 3064840 to 3493636 tid 3 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3837: read oid 2 snap -1 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3837: expect (ObjNum 1149 snap 0 seq_num 1149) 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3838: read oid 247 snap -1 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3838: expect deleted 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3839: read oid 352 snap -1 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3839: expect (ObjNum 2194 snap 0 seq_num 2194) 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3840: write oid 486 current snap is 0 2023-08-27T20:55:25.585 INFO:tasks.rados.rados.0.smithi093.stdout:3840: seq_num 2360 ranges {464520=457070,1351269=526826,2408616=640068} 2023-08-27T20:55:25.589 INFO:tasks.rados.rados.0.smithi093.stdout:3840: writing smithi093153278-486 from 464520 to 921590 tid 1 2023-08-27T20:55:25.595 INFO:tasks.rados.rados.0.smithi093.stdout:3840: writing smithi093153278-486 from 1351269 to 1878095 tid 2 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3840: writing smithi093153278-486 from 2408616 to 3048684 tid 3 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3841: read oid 175 snap -1 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3841: expect deleted 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3842: read oid 39 snap -1 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3842: expect (ObjNum 1745 snap 0 seq_num 1745) 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3843: read oid 203 snap -1 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3843: expect (ObjNum 1883 snap 0 seq_num 1883) 2023-08-27T20:55:25.602 INFO:tasks.rados.rados.0.smithi093.stdout:3844: delete oid 81 current snap is 0 2023-08-27T20:55:25.603 INFO:tasks.rados.rados.0.smithi093.stdout:3832: finishing write tid 3 to smithi093153278-492 2023-08-27T20:55:25.603 INFO:tasks.rados.rados.0.smithi093.stdout:3832: finishing write tid 4 to smithi093153278-492 2023-08-27T20:55:25.603 INFO:tasks.rados.rados.0.smithi093.stdout:3832: finishing write tid 5 to smithi093153278-492 2023-08-27T20:55:25.603 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 492 v 409 (ObjNum 2357 snap 0 seq_num 2357) dirty exists 2023-08-27T20:55:25.603 INFO:tasks.rados.rados.0.smithi093.stdout:3832: left oid 492 (ObjNum 2357 snap 0 seq_num 2357) 2023-08-27T20:55:25.603 INFO:tasks.rados.rados.0.smithi093.stdout:3835: expect (ObjNum 1796 snap 0 seq_num 1796) 2023-08-27T20:55:25.701 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:55:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:55:25] "GET /metrics HTTP/1.1" 200 242575 "" "Prometheus/2.43.0" 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:3836: finishing write tid 1 to smithi093153278-149 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:3836: finishing write tid 2 to smithi093153278-149 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:3836: finishing write tid 3 to smithi093153278-149 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:3836: finishing write tid 4 to smithi093153278-149 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:3836: finishing write tid 5 to smithi093153278-149 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 149 v 468 (ObjNum 2359 snap 0 seq_num 2359) dirty exists 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:3836: left oid 149 (ObjNum 2359 snap 0 seq_num 2359) 2023-08-27T20:55:25.739 INFO:tasks.rados.rados.0.smithi093.stdout:3837: expect (ObjNum 1149 snap 0 seq_num 1149) 2023-08-27T20:55:25.932 INFO:tasks.rados.rados.0.smithi093.stdout:3839: expect (ObjNum 2194 snap 0 seq_num 2194) 2023-08-27T20:55:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:25 smithi093 ceph-mon[111335]: pgmap v56: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:25 smithi093 ceph-mon[118902]: pgmap v56: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:26.098 INFO:tasks.rados.rados.0.smithi093.stdout:3840: finishing write tid 1 to smithi093153278-486 2023-08-27T20:55:26.098 INFO:tasks.rados.rados.0.smithi093.stdout:3840: finishing write tid 2 to smithi093153278-486 2023-08-27T20:55:26.098 INFO:tasks.rados.rados.0.smithi093.stdout:3840: finishing write tid 3 to smithi093153278-486 2023-08-27T20:55:26.098 INFO:tasks.rados.rados.0.smithi093.stdout:3840: finishing write tid 4 to smithi093153278-486 2023-08-27T20:55:26.098 INFO:tasks.rados.rados.0.smithi093.stdout:3842: expect (ObjNum 1745 snap 0 seq_num 1745) 2023-08-27T20:55:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:25 smithi136 ceph-mon[113531]: pgmap v56: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:26.240 INFO:tasks.rados.rados.0.smithi093.stdout:3840: finishing write tid 5 to smithi093153278-486 2023-08-27T20:55:26.240 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 486 v 552 (ObjNum 2360 snap 0 seq_num 2360) dirty exists 2023-08-27T20:55:26.240 INFO:tasks.rados.rados.0.smithi093.stdout:3840: left oid 486 (ObjNum 2360 snap 0 seq_num 2360) 2023-08-27T20:55:26.241 INFO:tasks.rados.rados.0.smithi093.stdout:3843: expect (ObjNum 1883 snap 0 seq_num 1883) 2023-08-27T20:55:26.434 INFO:tasks.rados.rados.0.smithi093.stdout:3832: done (11 left) 2023-08-27T20:55:26.434 INFO:tasks.rados.rados.0.smithi093.stdout:3834: done (10 left) 2023-08-27T20:55:26.434 INFO:tasks.rados.rados.0.smithi093.stdout:3835: done (9 left) 2023-08-27T20:55:26.434 INFO:tasks.rados.rados.0.smithi093.stdout:3836: done (8 left) 2023-08-27T20:55:26.434 INFO:tasks.rados.rados.0.smithi093.stdout:3837: done (7 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3838: done (6 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3839: done (5 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3840: done (4 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3841: done (3 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3842: done (2 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3843: done (1 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3844: done (0 left) 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3845: write oid 313 current snap is 0 2023-08-27T20:55:26.435 INFO:tasks.rados.rados.0.smithi093.stdout:3845: seq_num 2362 ranges {498134=607568,1751880=530495,2747109=778965} 2023-08-27T20:55:26.439 INFO:tasks.rados.rados.0.smithi093.stdout:3845: writing smithi093153278-313 from 498134 to 1105702 tid 1 2023-08-27T20:55:26.446 INFO:tasks.rados.rados.0.smithi093.stdout:3845: writing smithi093153278-313 from 1751880 to 2282375 tid 2 2023-08-27T20:55:26.453 INFO:tasks.rados.rados.0.smithi093.stdout:3845: writing smithi093153278-313 from 2747109 to 3526074 tid 3 2023-08-27T20:55:26.453 INFO:tasks.rados.rados.0.smithi093.stdout:3846: write oid 55 current snap is 0 2023-08-27T20:55:26.453 INFO:tasks.rados.rados.0.smithi093.stdout:3845: finishing write tid 1 to smithi093153278-313 2023-08-27T20:55:26.453 INFO:tasks.rados.rados.0.smithi093.stdout:3845: finishing write tid 2 to smithi093153278-313 2023-08-27T20:55:26.453 INFO:tasks.rados.rados.0.smithi093.stdout:3846: seq_num 2363 ranges {514941=682817,1810362=674153,3058179=590237} 2023-08-27T20:55:26.459 INFO:tasks.rados.rados.0.smithi093.stdout:3846: writing smithi093153278-55 from 514941 to 1197758 tid 1 2023-08-27T20:55:26.466 INFO:tasks.rados.rados.0.smithi093.stdout:3846: writing smithi093153278-55 from 1810362 to 2484515 tid 2 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3846: writing smithi093153278-55 from 3058179 to 3648416 tid 3 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3847: read oid 443 snap -1 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3847: expect (ObjNum 2093 snap 0 seq_num 2093) 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3848: read oid 100 snap -1 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3848: expect (ObjNum 2237 snap 0 seq_num 2237) 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3849: read oid 193 snap -1 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3849: expect (ObjNum 2149 snap 0 seq_num 2149) 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3850: read oid 11 snap -1 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3850: expect (ObjNum 2047 snap 0 seq_num 2047) 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3851: write oid 415 current snap is 0 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3845: finishing write tid 3 to smithi093153278-313 2023-08-27T20:55:26.474 INFO:tasks.rados.rados.0.smithi093.stdout:3851: seq_num 2364 ranges {531748=758066,1868844=534164,3201955=401509} 2023-08-27T20:55:26.481 INFO:tasks.rados.rados.0.smithi093.stdout:3851: writing smithi093153278-415 from 531748 to 1289814 tid 1 2023-08-27T20:55:26.487 INFO:tasks.rados.rados.0.smithi093.stdout:3851: writing smithi093153278-415 from 1868844 to 2403008 tid 2 2023-08-27T20:55:26.495 INFO:tasks.rados.rados.0.smithi093.stdout:3851: writing smithi093153278-415 from 3201955 to 3603464 tid 3 2023-08-27T20:55:26.495 INFO:tasks.rados.rados.0.smithi093.stdout:3852: write oid 423 current snap is 0 2023-08-27T20:55:26.495 INFO:tasks.rados.rados.0.smithi093.stdout:3845: finishing write tid 4 to smithi093153278-313 2023-08-27T20:55:26.495 INFO:tasks.rados.rados.0.smithi093.stdout:3845: finishing write tid 5 to smithi093153278-313 2023-08-27T20:55:26.495 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 313 v 296 (ObjNum 2362 snap 0 seq_num 2362) dirty exists 2023-08-27T20:55:26.495 INFO:tasks.rados.rados.0.smithi093.stdout:3845: left oid 313 (ObjNum 2362 snap 0 seq_num 2362) 2023-08-27T20:55:26.496 INFO:tasks.rados.rados.0.smithi093.stdout:3846: finishing write tid 1 to smithi093153278-55 2023-08-27T20:55:26.496 INFO:tasks.rados.rados.0.smithi093.stdout:3852: seq_num 2365 ranges {548555=549668,1643679=677822,2829378=612781} 2023-08-27T20:55:26.500 INFO:tasks.rados.rados.0.smithi093.stdout:3852: writing smithi093153278-423 from 548555 to 1098223 tid 1 2023-08-27T20:55:26.507 INFO:tasks.rados.rados.0.smithi093.stdout:3852: writing smithi093153278-423 from 1643679 to 2321501 tid 2 2023-08-27T20:55:26.513 INFO:tasks.rados.rados.0.smithi093.stdout:3852: writing smithi093153278-423 from 2829378 to 3442159 tid 3 2023-08-27T20:55:26.513 INFO:tasks.rados.rados.0.smithi093.stdout:3845: done (7 left) 2023-08-27T20:55:26.513 INFO:tasks.rados.rados.0.smithi093.stdout:3853: write oid 401 current snap is 0 2023-08-27T20:55:26.514 INFO:tasks.rados.rados.0.smithi093.stdout:3846: finishing write tid 2 to smithi093153278-55 2023-08-27T20:55:26.514 INFO:tasks.rados.rados.0.smithi093.stdout:3853: seq_num 2366 ranges {565362=624917,1985808=537833,3256801=508561} 2023-08-27T20:55:26.519 INFO:tasks.rados.rados.0.smithi093.stdout:3853: writing smithi093153278-401 from 565362 to 1190279 tid 1 2023-08-27T20:55:26.527 INFO:tasks.rados.rados.0.smithi093.stdout:3853: writing smithi093153278-401 from 1985808 to 2523641 tid 2 2023-08-27T20:55:26.534 INFO:tasks.rados.rados.0.smithi093.stdout:3853: writing smithi093153278-401 from 3256801 to 3765362 tid 3 2023-08-27T20:55:26.534 INFO:tasks.rados.rados.0.smithi093.stdout:3854: read oid 438 snap -1 2023-08-27T20:55:26.534 INFO:tasks.rados.rados.0.smithi093.stdout:3854: expect (ObjNum 1939 snap 0 seq_num 1939) 2023-08-27T20:55:26.535 INFO:tasks.rados.rados.0.smithi093.stdout:3855: write oid 205 current snap is 0 2023-08-27T20:55:26.535 INFO:tasks.rados.rados.0.smithi093.stdout:3850: expect (ObjNum 2047 snap 0 seq_num 2047) 2023-08-27T20:55:26.673 INFO:tasks.rados.rados.0.smithi093.stdout:3849: expect (ObjNum 2149 snap 0 seq_num 2149) 2023-08-27T20:55:26.792 INFO:tasks.rados.rados.0.smithi093.stdout:3847: expect (ObjNum 2093 snap 0 seq_num 2093) 2023-08-27T20:55:27.038 INFO:tasks.rados.rados.0.smithi093.stdout:3851: finishing write tid 1 to smithi093153278-415 2023-08-27T20:55:27.038 INFO:tasks.rados.rados.0.smithi093.stdout:3846: finishing write tid 3 to smithi093153278-55 2023-08-27T20:55:27.038 INFO:tasks.rados.rados.0.smithi093.stdout:3846: finishing write tid 4 to smithi093153278-55 2023-08-27T20:55:27.038 INFO:tasks.rados.rados.0.smithi093.stdout:3846: finishing write tid 5 to smithi093153278-55 2023-08-27T20:55:27.038 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 55 v 445 (ObjNum 2363 snap 0 seq_num 2363) dirty exists 2023-08-27T20:55:27.039 INFO:tasks.rados.rados.0.smithi093.stdout:3846: left oid 55 (ObjNum 2363 snap 0 seq_num 2363) 2023-08-27T20:55:27.039 INFO:tasks.rados.rados.0.smithi093.stdout:3855: seq_num 2367 ranges {582169=700166,2044290=681491,3284224=497945} 2023-08-27T20:55:27.044 INFO:tasks.rados.rados.0.smithi093.stdout:3855: writing smithi093153278-205 from 582169 to 1282335 tid 1 2023-08-27T20:55:27.052 INFO:tasks.rados.rados.0.smithi093.stdout:3855: writing smithi093153278-205 from 2044290 to 2725781 tid 2 2023-08-27T20:55:27.059 INFO:tasks.rados.rados.0.smithi093.stdout:3855: writing smithi093153278-205 from 3284224 to 3782169 tid 3 2023-08-27T20:55:27.059 INFO:tasks.rados.rados.0.smithi093.stdout:3846: done (9 left) 2023-08-27T20:55:27.059 INFO:tasks.rados.rados.0.smithi093.stdout:3847: done (8 left) 2023-08-27T20:55:27.059 INFO:tasks.rados.rados.0.smithi093.stdout:3849: done (7 left) 2023-08-27T20:55:27.060 INFO:tasks.rados.rados.0.smithi093.stdout:3850: done (6 left) 2023-08-27T20:55:27.060 INFO:tasks.rados.rados.0.smithi093.stdout:3856: read oid 399 snap -1 2023-08-27T20:55:27.060 INFO:tasks.rados.rados.0.smithi093.stdout:3856: expect (ObjNum 1934 snap 0 seq_num 1934) 2023-08-27T20:55:27.060 INFO:tasks.rados.rados.0.smithi093.stdout:3851: finishing write tid 2 to smithi093153278-415 2023-08-27T20:55:27.060 INFO:tasks.rados.rados.0.smithi093.stdout:3848: expect (ObjNum 2237 snap 0 seq_num 2237) 2023-08-27T20:55:27.205 INFO:tasks.rados.rados.0.smithi093.stdout:3851: finishing write tid 3 to smithi093153278-415 2023-08-27T20:55:27.205 INFO:tasks.rados.rados.0.smithi093.stdout:3851: finishing write tid 4 to smithi093153278-415 2023-08-27T20:55:27.205 INFO:tasks.rados.rados.0.smithi093.stdout:3851: finishing write tid 5 to smithi093153278-415 2023-08-27T20:55:27.205 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 415 v 505 (ObjNum 2364 snap 0 seq_num 2364) dirty exists 2023-08-27T20:55:27.205 INFO:tasks.rados.rados.0.smithi093.stdout:3851: left oid 415 (ObjNum 2364 snap 0 seq_num 2364) 2023-08-27T20:55:27.206 INFO:tasks.rados.rados.0.smithi093.stdout:3848: done (6 left) 2023-08-27T20:55:27.206 INFO:tasks.rados.rados.0.smithi093.stdout:3851: done (5 left) 2023-08-27T20:55:27.206 INFO:tasks.rados.rados.0.smithi093.stdout:3857: write oid 459 current snap is 0 2023-08-27T20:55:27.206 INFO:tasks.rados.rados.0.smithi093.stdout:3852: finishing write tid 1 to smithi093153278-423 2023-08-27T20:55:27.206 INFO:tasks.rados.rados.0.smithi093.stdout:3852: finishing write tid 2 to smithi093153278-423 2023-08-27T20:55:27.206 INFO:tasks.rados.rados.0.smithi093.stdout:3857: seq_num 2368 ranges {598976=775415,2102772=541502,3311647=487329} 2023-08-27T20:55:27.212 INFO:tasks.rados.rados.0.smithi093.stdout:3857: writing smithi093153278-459 from 598976 to 1374391 tid 1 2023-08-27T20:55:27.219 INFO:tasks.rados.rados.0.smithi093.stdout:3857: writing smithi093153278-459 from 2102772 to 2644274 tid 2 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3857: writing smithi093153278-459 from 3311647 to 3798976 tid 3 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3858: write oid 387 current snap is 0 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3852: finishing write tid 3 to smithi093153278-423 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3852: finishing write tid 4 to smithi093153278-423 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3852: finishing write tid 5 to smithi093153278-423 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 423 v 300 (ObjNum 2365 snap 0 seq_num 2365) dirty exists 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3852: left oid 423 (ObjNum 2365 snap 0 seq_num 2365) 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3853: finishing write tid 1 to smithi093153278-401 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3853: finishing write tid 2 to smithi093153278-401 2023-08-27T20:55:27.226 INFO:tasks.rados.rados.0.smithi093.stdout:3853: finishing write tid 3 to smithi093153278-401 2023-08-27T20:55:27.227 INFO:tasks.rados.rados.0.smithi093.stdout:3853: finishing write tid 4 to smithi093153278-401 2023-08-27T20:55:27.227 INFO:tasks.rados.rados.0.smithi093.stdout:3853: finishing write tid 5 to smithi093153278-401 2023-08-27T20:55:27.227 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 401 v 304 (ObjNum 2366 snap 0 seq_num 2366) dirty exists 2023-08-27T20:55:27.227 INFO:tasks.rados.rados.0.smithi093.stdout:3853: left oid 401 (ObjNum 2366 snap 0 seq_num 2366) 2023-08-27T20:55:27.227 INFO:tasks.rados.rados.0.smithi093.stdout:3854: expect (ObjNum 1939 snap 0 seq_num 1939) 2023-08-27T20:55:27.376 INFO:tasks.rados.rados.0.smithi093.stdout:3855: finishing write tid 1 to smithi093153278-205 2023-08-27T20:55:27.376 INFO:tasks.rados.rados.0.smithi093.stdout:3855: finishing write tid 2 to smithi093153278-205 2023-08-27T20:55:27.376 INFO:tasks.rados.rados.0.smithi093.stdout:3855: finishing write tid 3 to smithi093153278-205 2023-08-27T20:55:27.376 INFO:tasks.rados.rados.0.smithi093.stdout:3855: finishing write tid 4 to smithi093153278-205 2023-08-27T20:55:27.376 INFO:tasks.rados.rados.0.smithi093.stdout:3855: finishing write tid 5 to smithi093153278-205 2023-08-27T20:55:27.377 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 205 v 308 (ObjNum 2367 snap 0 seq_num 2367) dirty exists 2023-08-27T20:55:27.377 INFO:tasks.rados.rados.0.smithi093.stdout:3855: left oid 205 (ObjNum 2367 snap 0 seq_num 2367) 2023-08-27T20:55:27.377 INFO:tasks.rados.rados.0.smithi093.stdout:3856: expect (ObjNum 1934 snap 0 seq_num 1934) 2023-08-27T20:55:27.505 INFO:tasks.rados.rados.0.smithi093.stdout:3857: finishing write tid 1 to smithi093153278-459 2023-08-27T20:55:27.506 INFO:tasks.rados.rados.0.smithi093.stdout:3857: finishing write tid 2 to smithi093153278-459 2023-08-27T20:55:27.506 INFO:tasks.rados.rados.0.smithi093.stdout:3857: finishing write tid 3 to smithi093153278-459 2023-08-27T20:55:27.506 INFO:tasks.rados.rados.0.smithi093.stdout:3857: finishing write tid 4 to smithi093153278-459 2023-08-27T20:55:27.506 INFO:tasks.rados.rados.0.smithi093.stdout:3857: finishing write tid 5 to smithi093153278-459 2023-08-27T20:55:27.506 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 459 v 413 (ObjNum 2368 snap 0 seq_num 2368) dirty exists 2023-08-27T20:55:27.506 INFO:tasks.rados.rados.0.smithi093.stdout:3857: left oid 459 (ObjNum 2368 snap 0 seq_num 2368) 2023-08-27T20:55:27.507 INFO:tasks.rados.rados.0.smithi093.stdout:3858: seq_num 2369 ranges {615783=450664,1761254=685160,2939070=774222} 2023-08-27T20:55:27.511 INFO:tasks.rados.rados.0.smithi093.stdout:3858: writing smithi093153278-387 from 615783 to 1066447 tid 1 2023-08-27T20:55:27.518 INFO:tasks.rados.rados.0.smithi093.stdout:3858: writing smithi093153278-387 from 1761254 to 2446414 tid 2 2023-08-27T20:55:27.526 INFO:tasks.rados.rados.0.smithi093.stdout:3858: writing smithi093153278-387 from 2939070 to 3713292 tid 3 2023-08-27T20:55:27.526 INFO:tasks.rados.rados.0.smithi093.stdout:3852: done (6 left) 2023-08-27T20:55:27.526 INFO:tasks.rados.rados.0.smithi093.stdout:3853: done (5 left) 2023-08-27T20:55:27.526 INFO:tasks.rados.rados.0.smithi093.stdout:3854: done (4 left) 2023-08-27T20:55:27.527 INFO:tasks.rados.rados.0.smithi093.stdout:3855: done (3 left) 2023-08-27T20:55:27.527 INFO:tasks.rados.rados.0.smithi093.stdout:3856: done (2 left) 2023-08-27T20:55:27.527 INFO:tasks.rados.rados.0.smithi093.stdout:3857: done (1 left) 2023-08-27T20:55:27.527 INFO:tasks.rados.rados.0.smithi093.stdout:3859: read oid 66 snap -1 2023-08-27T20:55:27.527 INFO:tasks.rados.rados.0.smithi093.stdout:3859: expect (ObjNum 1704 snap 0 seq_num 1704) 2023-08-27T20:55:27.527 INFO:tasks.rados.rados.0.smithi093.stdout:3860: write oid 50 current snap is 0 2023-08-27T20:55:27.528 INFO:tasks.rados.rados.0.smithi093.stdout:3858: finishing write tid 1 to smithi093153278-387 2023-08-27T20:55:27.528 INFO:tasks.rados.rados.0.smithi093.stdout:3858: finishing write tid 2 to smithi093153278-387 2023-08-27T20:55:27.528 INFO:tasks.rados.rados.0.smithi093.stdout:3860: seq_num 2370 ranges {632590=525913,1703383=545171,2850140=585494} 2023-08-27T20:55:27.532 INFO:tasks.rados.rados.0.smithi093.stdout:3860: writing smithi093153278-50 from 632590 to 1158503 tid 1 2023-08-27T20:55:27.538 INFO:tasks.rados.rados.0.smithi093.stdout:3860: writing smithi093153278-50 from 1703383 to 2248554 tid 2 2023-08-27T20:55:27.544 INFO:tasks.rados.rados.0.smithi093.stdout:3860: writing smithi093153278-50 from 2850140 to 3435634 tid 3 2023-08-27T20:55:27.545 INFO:tasks.rados.rados.0.smithi093.stdout:3861: read oid 156 snap -1 2023-08-27T20:55:27.545 INFO:tasks.rados.rados.0.smithi093.stdout:3861: expect (ObjNum 1737 snap 0 seq_num 1737) 2023-08-27T20:55:27.545 INFO:tasks.rados.rados.0.smithi093.stdout:3862: write oid 222 current snap is 0 2023-08-27T20:55:27.545 INFO:tasks.rados.rados.0.smithi093.stdout:3858: finishing write tid 3 to smithi093153278-387 2023-08-27T20:55:27.545 INFO:tasks.rados.rados.0.smithi093.stdout:3858: finishing write tid 4 to smithi093153278-387 2023-08-27T20:55:27.545 INFO:tasks.rados.rados.0.smithi093.stdout:3858: finishing write tid 5 to smithi093153278-387 2023-08-27T20:55:27.546 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 387 v 449 (ObjNum 2369 snap 0 seq_num 2369) dirty exists 2023-08-27T20:55:27.546 INFO:tasks.rados.rados.0.smithi093.stdout:3858: left oid 387 (ObjNum 2369 snap 0 seq_num 2369) 2023-08-27T20:55:27.546 INFO:tasks.rados.rados.0.smithi093.stdout:3859: expect (ObjNum 1704 snap 0 seq_num 1704) 2023-08-27T20:55:27.627 INFO:tasks.rados.rados.0.smithi093.stdout:3860: finishing write tid 1 to smithi093153278-50 2023-08-27T20:55:27.627 INFO:tasks.rados.rados.0.smithi093.stdout:3860: finishing write tid 2 to smithi093153278-50 2023-08-27T20:55:27.627 INFO:tasks.rados.rados.0.smithi093.stdout:3860: finishing write tid 3 to smithi093153278-50 2023-08-27T20:55:27.627 INFO:tasks.rados.rados.0.smithi093.stdout:3860: finishing write tid 4 to smithi093153278-50 2023-08-27T20:55:27.627 INFO:tasks.rados.rados.0.smithi093.stdout:3860: finishing write tid 5 to smithi093153278-50 2023-08-27T20:55:27.628 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 50 v 437 (ObjNum 2370 snap 0 seq_num 2370) dirty exists 2023-08-27T20:55:27.628 INFO:tasks.rados.rados.0.smithi093.stdout:3860: left oid 50 (ObjNum 2370 snap 0 seq_num 2370) 2023-08-27T20:55:27.628 INFO:tasks.rados.rados.0.smithi093.stdout:3861: expect (ObjNum 1737 snap 0 seq_num 1737) 2023-08-27T20:55:27.847 INFO:tasks.rados.rados.0.smithi093.stdout:3862: seq_num 2371 ranges {649397=601162,1761865=688829,2877563=513119,3812689=36708} 2023-08-27T20:55:27.853 INFO:tasks.rados.rados.0.smithi093.stdout:3862: writing smithi093153278-222 from 649397 to 1250559 tid 1 2023-08-27T20:55:27.860 INFO:tasks.rados.rados.0.smithi093.stdout:3862: writing smithi093153278-222 from 1761865 to 2450694 tid 2 2023-08-27T20:55:27.866 INFO:tasks.rados.rados.0.smithi093.stdout:3862: writing smithi093153278-222 from 2877563 to 3390682 tid 3 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3862: writing smithi093153278-222 from 3812689 to 3849397 tid 4 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3858: done (4 left) 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3859: done (3 left) 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3860: done (2 left) 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3861: done (1 left) 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3863: read oid 424 snap -1 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3863: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3864: delete oid 477 current snap is 0 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3862: finishing write tid 1 to smithi093153278-222 2023-08-27T20:55:27.869 INFO:tasks.rados.rados.0.smithi093.stdout:3862: finishing write tid 2 to smithi093153278-222 2023-08-27T20:55:27.870 INFO:tasks.rados.rados.0.smithi093.stdout:3862: finishing write tid 3 to smithi093153278-222 2023-08-27T20:55:27.870 INFO:tasks.rados.rados.0.smithi093.stdout:3862: finishing write tid 4 to smithi093153278-222 2023-08-27T20:55:27.870 INFO:tasks.rados.rados.0.smithi093.stdout:3862: finishing write tid 5 to smithi093153278-222 2023-08-27T20:55:27.870 INFO:tasks.rados.rados.0.smithi093.stdout:3862: finishing write tid 6 to smithi093153278-222 2023-08-27T20:55:27.871 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 222 v 510 (ObjNum 2371 snap 0 seq_num 2371) dirty exists 2023-08-27T20:55:27.871 INFO:tasks.rados.rados.0.smithi093.stdout:3862: left oid 222 (ObjNum 2371 snap 0 seq_num 2371) 2023-08-27T20:55:27.875 INFO:tasks.rados.rados.0.smithi093.stdout:3863: expect (ObjNum 1376 snap 0 seq_num 1376) 2023-08-27T20:55:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:27 smithi093 ceph-mon[111335]: pgmap v57: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 14 MiB/s wr, 126 op/s 2023-08-27T20:55:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3858824966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:27 smithi093 ceph-mon[118902]: pgmap v57: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 14 MiB/s wr, 126 op/s 2023-08-27T20:55:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3858824966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:28.049 INFO:tasks.rados.rados.0.smithi093.stdout:3862: done (2 left) 2023-08-27T20:55:28.049 INFO:tasks.rados.rados.0.smithi093.stdout:3863: done (1 left) 2023-08-27T20:55:28.050 INFO:tasks.rados.rados.0.smithi093.stdout:3864: done (0 left) 2023-08-27T20:55:28.050 INFO:tasks.rados.rados.0.smithi093.stdout:3865: read oid 140 snap -1 2023-08-27T20:55:28.050 INFO:tasks.rados.rados.0.smithi093.stdout:3865: expect (ObjNum 1751 snap 0 seq_num 1751) 2023-08-27T20:55:28.050 INFO:tasks.rados.rados.0.smithi093.stdout:3866: write oid 291 current snap is 0 2023-08-27T20:55:28.050 INFO:tasks.rados.rados.0.smithi093.stdout:3866: seq_num 2373 ranges {683011=468013,1595182=692498,3048762=535663} 2023-08-27T20:55:28.056 INFO:tasks.rados.rados.0.smithi093.stdout:3866: writing smithi093153278-291 from 683011 to 1151024 tid 1 2023-08-27T20:55:28.062 INFO:tasks.rados.rados.0.smithi093.stdout:3866: writing smithi093153278-291 from 1595182 to 2287680 tid 2 2023-08-27T20:55:28.069 INFO:tasks.rados.rados.0.smithi093.stdout:3866: writing smithi093153278-291 from 3048762 to 3584425 tid 3 2023-08-27T20:55:28.069 INFO:tasks.rados.rados.0.smithi093.stdout:3867: write oid 315 current snap is 0 2023-08-27T20:55:28.069 INFO:tasks.rados.rados.0.smithi093.stdout:3867: seq_num 2374 ranges {699818=543262,1937311=436156,2843479=746935} 2023-08-27T20:55:28.076 INFO:tasks.rados.rados.0.smithi093.stdout:3867: writing smithi093153278-315 from 699818 to 1243080 tid 1 2023-08-27T20:55:28.082 INFO:tasks.rados.rados.0.smithi093.stdout:3867: writing smithi093153278-315 from 1937311 to 2373467 tid 2 2023-08-27T20:55:28.089 INFO:tasks.rados.rados.0.smithi093.stdout:3867: writing smithi093153278-315 from 2843479 to 3590414 tid 3 2023-08-27T20:55:28.089 INFO:tasks.rados.rados.0.smithi093.stdout:3868: write oid 497 current snap is 0 2023-08-27T20:55:28.089 INFO:tasks.rados.rados.0.smithi093.stdout:3868: seq_num 2375 ranges {716625=618511,1995793=696167,3387255=529370} 2023-08-27T20:55:28.096 INFO:tasks.rados.rados.0.smithi093.stdout:3868: writing smithi093153278-497 from 716625 to 1335136 tid 1 2023-08-27T20:55:28.103 INFO:tasks.rados.rados.0.smithi093.stdout:3868: writing smithi093153278-497 from 1995793 to 2691960 tid 2 2023-08-27T20:55:28.110 INFO:tasks.rados.rados.0.smithi093.stdout:3868: writing smithi093153278-497 from 3387255 to 3916625 tid 3 2023-08-27T20:55:28.110 INFO:tasks.rados.rados.0.smithi093.stdout:3869: read oid 431 snap -1 2023-08-27T20:55:28.110 INFO:tasks.rados.rados.0.smithi093.stdout:3869: expect (ObjNum 2187 snap 0 seq_num 2187) 2023-08-27T20:55:28.110 INFO:tasks.rados.rados.0.smithi093.stdout:3870: read oid 371 snap -1 2023-08-27T20:55:28.111 INFO:tasks.rados.rados.0.smithi093.stdout:3870: expect (ObjNum 1396 snap 0 seq_num 1396) 2023-08-27T20:55:28.111 INFO:tasks.rados.rados.0.smithi093.stdout:3871: write oid 87 current snap is 0 2023-08-27T20:55:28.111 INFO:tasks.rados.rados.0.smithi093.stdout:3865: expect (ObjNum 1751 snap 0 seq_num 1751) 2023-08-27T20:55:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:27 smithi136 ceph-mon[113531]: pgmap v57: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 87 MiB/s rd, 14 MiB/s wr, 126 op/s 2023-08-27T20:55:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3858824966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:28.254 INFO:tasks.rados.rados.0.smithi093.stdout:3866: finishing write tid 1 to smithi093153278-291 2023-08-27T20:55:28.254 INFO:tasks.rados.rados.0.smithi093.stdout:3866: finishing write tid 2 to smithi093153278-291 2023-08-27T20:55:28.254 INFO:tasks.rados.rados.0.smithi093.stdout:3866: finishing write tid 3 to smithi093153278-291 2023-08-27T20:55:28.254 INFO:tasks.rados.rados.0.smithi093.stdout:3866: finishing write tid 4 to smithi093153278-291 2023-08-27T20:55:28.254 INFO:tasks.rados.rados.0.smithi093.stdout:3866: finishing write tid 5 to smithi093153278-291 2023-08-27T20:55:28.254 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 291 v 498 (ObjNum 2373 snap 0 seq_num 2373) dirty exists 2023-08-27T20:55:28.255 INFO:tasks.rados.rados.0.smithi093.stdout:3866: left oid 291 (ObjNum 2373 snap 0 seq_num 2373) 2023-08-27T20:55:28.255 INFO:tasks.rados.rados.0.smithi093.stdout:3867: finishing write tid 1 to smithi093153278-315 2023-08-27T20:55:28.255 INFO:tasks.rados.rados.0.smithi093.stdout:3871: seq_num 2376 ranges {733432=693760,2054275=439825,2898325=485832} 2023-08-27T20:55:28.261 INFO:tasks.rados.rados.0.smithi093.stdout:3871: writing smithi093153278-87 from 733432 to 1427192 tid 1 2023-08-27T20:55:28.267 INFO:tasks.rados.rados.0.smithi093.stdout:3871: writing smithi093153278-87 from 2054275 to 2494100 tid 2 2023-08-27T20:55:28.272 INFO:tasks.rados.rados.0.smithi093.stdout:3871: writing smithi093153278-87 from 2898325 to 3384157 tid 3 2023-08-27T20:55:28.273 INFO:tasks.rados.rados.0.smithi093.stdout:3865: done (6 left) 2023-08-27T20:55:28.273 INFO:tasks.rados.rados.0.smithi093.stdout:3866: done (5 left) 2023-08-27T20:55:28.273 INFO:tasks.rados.rados.0.smithi093.stdout:3872: write oid 438 current snap is 0 2023-08-27T20:55:28.273 INFO:tasks.rados.rados.0.smithi093.stdout:3867: finishing write tid 2 to smithi093153278-315 2023-08-27T20:55:28.273 INFO:tasks.rados.rados.0.smithi093.stdout:3867: finishing write tid 3 to smithi093153278-315 2023-08-27T20:55:28.273 INFO:tasks.rados.rados.0.smithi093.stdout:3872: seq_num 2377 ranges {750239=769009,2112757=699836,3442101=508138} 2023-08-27T20:55:28.280 INFO:tasks.rados.rados.0.smithi093.stdout:3872: writing smithi093153278-438 from 750239 to 1519248 tid 1 2023-08-27T20:55:28.288 INFO:tasks.rados.rados.0.smithi093.stdout:3872: writing smithi093153278-438 from 2112757 to 2812593 tid 2 2023-08-27T20:55:28.295 INFO:tasks.rados.rados.0.smithi093.stdout:3872: writing smithi093153278-438 from 3442101 to 3950239 tid 3 2023-08-27T20:55:28.295 INFO:tasks.rados.rados.0.smithi093.stdout:3873: read oid 40 snap -1 2023-08-27T20:55:28.295 INFO:tasks.rados.rados.0.smithi093.stdout:3873: expect deleted 2023-08-27T20:55:28.295 INFO:tasks.rados.rados.0.smithi093.stdout:3874: read oid 486 snap -1 2023-08-27T20:55:28.295 INFO:tasks.rados.rados.0.smithi093.stdout:3874: expect (ObjNum 2360 snap 0 seq_num 2360) 2023-08-27T20:55:28.295 INFO:tasks.rados.rados.0.smithi093.stdout:3875: read oid 38 snap -1 2023-08-27T20:55:28.295 INFO:tasks.rados.rados.0.smithi093.stdout:3875: expect (ObjNum 2147 snap 0 seq_num 2147) 2023-08-27T20:55:28.296 INFO:tasks.rados.rados.0.smithi093.stdout:3876: write oid 134 current snap is 0 2023-08-27T20:55:28.296 INFO:tasks.rados.rados.0.smithi093.stdout:3867: finishing write tid 4 to smithi093153278-315 2023-08-27T20:55:28.296 INFO:tasks.rados.rados.0.smithi093.stdout:3867: finishing write tid 5 to smithi093153278-315 2023-08-27T20:55:28.296 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 556 (ObjNum 2374 snap 0 seq_num 2374) dirty exists 2023-08-27T20:55:28.296 INFO:tasks.rados.rados.0.smithi093.stdout:3867: left oid 315 (ObjNum 2374 snap 0 seq_num 2374) 2023-08-27T20:55:28.296 INFO:tasks.rados.rados.0.smithi093.stdout:3868: finishing write tid 1 to smithi093153278-497 2023-08-27T20:55:28.296 INFO:tasks.rados.rados.0.smithi093.stdout:3876: seq_num 2378 ranges {767046=444258,1654886=443494,2836818=508376,3760009=207037} 2023-08-27T20:55:28.302 INFO:tasks.rados.rados.0.smithi093.stdout:3876: writing smithi093153278-134 from 767046 to 1211304 tid 1 2023-08-27T20:55:28.307 INFO:tasks.rados.rados.0.smithi093.stdout:3876: writing smithi093153278-134 from 1654886 to 2098380 tid 2 2023-08-27T20:55:28.314 INFO:tasks.rados.rados.0.smithi093.stdout:3876: writing smithi093153278-134 from 2836818 to 3345194 tid 3 2023-08-27T20:55:28.318 INFO:tasks.rados.rados.0.smithi093.stdout:3876: writing smithi093153278-134 from 3760009 to 3967046 tid 4 2023-08-27T20:55:28.318 INFO:tasks.rados.rados.0.smithi093.stdout:3867: done (9 left) 2023-08-27T20:55:28.318 INFO:tasks.rados.rados.0.smithi093.stdout:3877: read oid 31 snap -1 2023-08-27T20:55:28.318 INFO:tasks.rados.rados.0.smithi093.stdout:3877: expect (ObjNum 2228 snap 0 seq_num 2228) 2023-08-27T20:55:28.319 INFO:tasks.rados.rados.0.smithi093.stdout:3878: delete oid 266 current snap is 0 2023-08-27T20:55:28.319 INFO:tasks.rados.rados.0.smithi093.stdout:3868: finishing write tid 2 to smithi093153278-497 2023-08-27T20:55:28.319 INFO:tasks.rados.rados.0.smithi093.stdout:3869: expect (ObjNum 2187 snap 0 seq_num 2187) 2023-08-27T20:55:28.427 INFO:tasks.rados.rados.0.smithi093.stdout:3868: finishing write tid 3 to smithi093153278-497 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3868: finishing write tid 4 to smithi093153278-497 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3868: finishing write tid 5 to smithi093153278-497 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 497 v 441 (ObjNum 2375 snap 0 seq_num 2375) dirty exists 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3868: left oid 497 (ObjNum 2375 snap 0 seq_num 2375) 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3868: done (10 left) 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3869: done (9 left) 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3878: done (8 left) 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3879: read oid 131 snap -1 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3879: expect deleted 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3880: read oid 300 snap -1 2023-08-27T20:55:28.428 INFO:tasks.rados.rados.0.smithi093.stdout:3880: expect (ObjNum 1958 snap 0 seq_num 1958) 2023-08-27T20:55:28.429 INFO:tasks.rados.rados.0.smithi093.stdout:3881: read oid 207 snap -1 2023-08-27T20:55:28.429 INFO:tasks.rados.rados.0.smithi093.stdout:3881: expect deleted 2023-08-27T20:55:28.429 INFO:tasks.rados.rados.0.smithi093.stdout:3882: write oid 85 current snap is 0 2023-08-27T20:55:28.429 INFO:tasks.rados.rados.0.smithi093.stdout:3870: expect (ObjNum 1396 snap 0 seq_num 1396) 2023-08-27T20:55:28.569 INFO:tasks.rados.rados.0.smithi093.stdout:3871: finishing write tid 1 to smithi093153278-87 2023-08-27T20:55:28.569 INFO:tasks.rados.rados.0.smithi093.stdout:3871: finishing write tid 2 to smithi093153278-87 2023-08-27T20:55:28.569 INFO:tasks.rados.rados.0.smithi093.stdout:3882: seq_num 2380 ranges {400660=711109,1888203=112457} 2023-08-27T20:55:28.575 INFO:tasks.rados.rados.0.smithi093.stdout:3882: writing smithi093153278-85 from 400660 to 1111769 tid 1 2023-08-27T20:55:28.580 INFO:tasks.rados.rados.0.smithi093.stdout:3882: writing smithi093153278-85 from 1888203 to 2000660 tid 2 2023-08-27T20:55:28.580 INFO:tasks.rados.rados.0.smithi093.stdout:3870: done (11 left) 2023-08-27T20:55:28.580 INFO:tasks.rados.rados.0.smithi093.stdout:3883: delete oid 450 current snap is 0 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3871: finishing write tid 3 to smithi093153278-87 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3871: finishing write tid 4 to smithi093153278-87 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3871: finishing write tid 5 to smithi093153278-87 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 87 v 438 (ObjNum 2376 snap 0 seq_num 2376) dirty exists 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3871: left oid 87 (ObjNum 2376 snap 0 seq_num 2376) 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3872: finishing write tid 1 to smithi093153278-438 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3872: finishing write tid 2 to smithi093153278-438 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3872: finishing write tid 3 to smithi093153278-438 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3872: finishing write tid 4 to smithi093153278-438 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:3872: finishing write tid 5 to smithi093153278-438 2023-08-27T20:55:28.581 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 438 v 417 (ObjNum 2377 snap 0 seq_num 2377) dirty exists 2023-08-27T20:55:28.582 INFO:tasks.rados.rados.0.smithi093.stdout:3872: left oid 438 (ObjNum 2377 snap 0 seq_num 2377) 2023-08-27T20:55:28.582 INFO:tasks.rados.rados.0.smithi093.stdout:3875: expect (ObjNum 2147 snap 0 seq_num 2147) 2023-08-27T20:55:28.642 INFO:tasks.rados.rados.0.smithi093.stdout:3874: expect (ObjNum 2360 snap 0 seq_num 2360) 2023-08-27T20:55:28.895 INFO:tasks.rados.rados.0.smithi093.stdout:3876: finishing write tid 1 to smithi093153278-134 2023-08-27T20:55:28.895 INFO:tasks.rados.rados.0.smithi093.stdout:3876: finishing write tid 2 to smithi093153278-134 2023-08-27T20:55:28.895 INFO:tasks.rados.rados.0.smithi093.stdout:3876: finishing write tid 3 to smithi093153278-134 2023-08-27T20:55:28.895 INFO:tasks.rados.rados.0.smithi093.stdout:3876: finishing write tid 4 to smithi093153278-134 2023-08-27T20:55:28.895 INFO:tasks.rados.rados.0.smithi093.stdout:3876: finishing write tid 5 to smithi093153278-134 2023-08-27T20:55:28.895 INFO:tasks.rados.rados.0.smithi093.stdout:3876: finishing write tid 6 to smithi093153278-134 2023-08-27T20:55:28.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 134 v 454 (ObjNum 2378 snap 0 seq_num 2378) dirty exists 2023-08-27T20:55:28.896 INFO:tasks.rados.rados.0.smithi093.stdout:3876: left oid 134 (ObjNum 2378 snap 0 seq_num 2378) 2023-08-27T20:55:28.896 INFO:tasks.rados.rados.0.smithi093.stdout:3877: expect (ObjNum 2228 snap 0 seq_num 2228) 2023-08-27T20:55:29.120 INFO:tasks.rados.rados.0.smithi093.stdout:3880: expect (ObjNum 1958 snap 0 seq_num 1958) 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3882: finishing write tid 1 to smithi093153278-85 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3882: finishing write tid 2 to smithi093153278-85 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3882: finishing write tid 3 to smithi093153278-85 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3871: done (11 left) 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3872: done (10 left) 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3873: done (9 left) 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3874: done (8 left) 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3875: done (7 left) 2023-08-27T20:55:29.244 INFO:tasks.rados.rados.0.smithi093.stdout:3876: done (6 left) 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3877: done (5 left) 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3879: done (4 left) 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3880: done (3 left) 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3881: done (2 left) 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3883: done (1 left) 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3884: delete oid 451 current snap is 0 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3882: finishing write tid 4 to smithi093153278-85 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 85 v 551 (ObjNum 2380 snap 0 seq_num 2380) dirty exists 2023-08-27T20:55:29.245 INFO:tasks.rados.rados.0.smithi093.stdout:3882: left oid 85 (ObjNum 2380 snap 0 seq_num 2380) 2023-08-27T20:55:29.246 INFO:tasks.rados.rados.0.smithi093.stdout:3882: done (1 left) 2023-08-27T20:55:29.246 INFO:tasks.rados.rados.0.smithi093.stdout:3884: done (0 left) 2023-08-27T20:55:29.246 INFO:tasks.rados.rados.0.smithi093.stdout:3885: read oid 44 snap -1 2023-08-27T20:55:29.247 INFO:tasks.rados.rados.0.smithi093.stdout:3885: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:55:29.247 INFO:tasks.rados.rados.0.smithi093.stdout:3886: read oid 266 snap -1 2023-08-27T20:55:29.247 INFO:tasks.rados.rados.0.smithi093.stdout:3886: expect deleted 2023-08-27T20:55:29.247 INFO:tasks.rados.rados.0.smithi093.stdout:3887: write oid 101 current snap is 0 2023-08-27T20:55:29.247 INFO:tasks.rados.rados.0.smithi093.stdout:3887: seq_num 2383 ranges {451081=536856,1547296=503785} 2023-08-27T20:55:29.251 INFO:tasks.rados.rados.0.smithi093.stdout:3887: writing smithi093153278-101 from 451081 to 987937 tid 1 2023-08-27T20:55:29.257 INFO:tasks.rados.rados.0.smithi093.stdout:3887: writing smithi093153278-101 from 1547296 to 2051081 tid 2 2023-08-27T20:55:29.257 INFO:tasks.rados.rados.0.smithi093.stdout:3888: read oid 74 snap -1 2023-08-27T20:55:29.257 INFO:tasks.rados.rados.0.smithi093.stdout:3888: expect (ObjNum 2184 snap 0 seq_num 2184) 2023-08-27T20:55:29.257 INFO:tasks.rados.rados.0.smithi093.stdout:3889: write oid 450 current snap is 0 2023-08-27T20:55:29.257 INFO:tasks.rados.rados.0.smithi093.stdout:3889: seq_num 2384 ranges {467888=612105,1605778=454501} 2023-08-27T20:55:29.263 INFO:tasks.rados.rados.0.smithi093.stdout:3889: writing smithi093153278-450 from 467888 to 1079993 tid 1 2023-08-27T20:55:29.268 INFO:tasks.rados.rados.0.smithi093.stdout:3889: writing smithi093153278-450 from 1605778 to 2060279 tid 2 2023-08-27T20:55:29.269 INFO:tasks.rados.rados.0.smithi093.stdout:3890: read oid 57 snap -1 2023-08-27T20:55:29.269 INFO:tasks.rados.rados.0.smithi093.stdout:3890: expect (ObjNum 1539 snap 0 seq_num 1539) 2023-08-27T20:55:29.269 INFO:tasks.rados.rados.0.smithi093.stdout:3891: read oid 148 snap -1 2023-08-27T20:55:29.269 INFO:tasks.rados.rados.0.smithi093.stdout:3891: expect (ObjNum 1827 snap 0 seq_num 1827) 2023-08-27T20:55:29.269 INFO:tasks.rados.rados.0.smithi093.stdout:3892: write oid 475 current snap is 0 2023-08-27T20:55:29.269 INFO:tasks.rados.rados.0.smithi093.stdout:3885: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:55:29.378 INFO:tasks.rados.rados.0.smithi093.stdout:3887: finishing write tid 1 to smithi093153278-101 2023-08-27T20:55:29.378 INFO:tasks.rados.rados.0.smithi093.stdout:3888: expect (ObjNum 2184 snap 0 seq_num 2184) 2023-08-27T20:55:29.578 INFO:tasks.rados.rados.0.smithi093.stdout:3887: finishing write tid 2 to smithi093153278-101 2023-08-27T20:55:29.578 INFO:tasks.rados.rados.0.smithi093.stdout:3887: finishing write tid 3 to smithi093153278-101 2023-08-27T20:55:29.578 INFO:tasks.rados.rados.0.smithi093.stdout:3887: finishing write tid 4 to smithi093153278-101 2023-08-27T20:55:29.578 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 101 v 419 (ObjNum 2383 snap 0 seq_num 2383) dirty exists 2023-08-27T20:55:29.578 INFO:tasks.rados.rados.0.smithi093.stdout:3887: left oid 101 (ObjNum 2383 snap 0 seq_num 2383) 2023-08-27T20:55:29.578 INFO:tasks.rados.rados.0.smithi093.stdout:3889: finishing write tid 1 to smithi093153278-450 2023-08-27T20:55:29.579 INFO:tasks.rados.rados.0.smithi093.stdout:3889: finishing write tid 2 to smithi093153278-450 2023-08-27T20:55:29.579 INFO:tasks.rados.rados.0.smithi093.stdout:3889: finishing write tid 3 to smithi093153278-450 2023-08-27T20:55:29.579 INFO:tasks.rados.rados.0.smithi093.stdout:3892: seq_num 2385 ranges {484695=687354,1664260=420435} 2023-08-27T20:55:29.584 INFO:tasks.rados.rados.0.smithi093.stdout:3892: writing smithi093153278-475 from 484695 to 1172049 tid 1 2023-08-27T20:55:29.589 INFO:tasks.rados.rados.0.smithi093.stdout:3892: writing smithi093153278-475 from 1664260 to 2084695 tid 2 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3885: done (7 left) 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3886: done (6 left) 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3887: done (5 left) 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3888: done (4 left) 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3893: write oid 141 current snap is 0 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3889: finishing write tid 4 to smithi093153278-450 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 450 v 383 (ObjNum 2384 snap 0 seq_num 2384) dirty exists 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3889: left oid 450 (ObjNum 2384 snap 0 seq_num 2384) 2023-08-27T20:55:29.590 INFO:tasks.rados.rados.0.smithi093.stdout:3893: seq_num 2386 ranges {501502=762603,2006389=95113} 2023-08-27T20:55:29.596 INFO:tasks.rados.rados.0.smithi093.stdout:3893: writing smithi093153278-141 from 501502 to 1264105 tid 1 2023-08-27T20:55:29.601 INFO:tasks.rados.rados.0.smithi093.stdout:3893: writing smithi093153278-141 from 2006389 to 2101502 tid 2 2023-08-27T20:55:29.601 INFO:tasks.rados.rados.0.smithi093.stdout:3889: done (4 left) 2023-08-27T20:55:29.601 INFO:tasks.rados.rados.0.smithi093.stdout:3894: read oid 482 snap -1 2023-08-27T20:55:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:3894: expect (ObjNum 1935 snap 0 seq_num 1935) 2023-08-27T20:55:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:3895: read oid 306 snap -1 2023-08-27T20:55:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:3895: expect (ObjNum 1899 snap 0 seq_num 1899) 2023-08-27T20:55:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:3896: write oid 289 current snap is 0 2023-08-27T20:55:29.602 INFO:tasks.rados.rados.0.smithi093.stdout:3891: expect (ObjNum 1827 snap 0 seq_num 1827) 2023-08-27T20:55:29.728 INFO:tasks.rados.rados.0.smithi093.stdout:3890: expect (ObjNum 1539 snap 0 seq_num 1539) 2023-08-27T20:55:29.845 INFO:tasks.rados.rados.0.smithi093.stdout:3892: finishing write tid 1 to smithi093153278-475 2023-08-27T20:55:29.845 INFO:tasks.rados.rados.0.smithi093.stdout:3892: finishing write tid 2 to smithi093153278-475 2023-08-27T20:55:29.845 INFO:tasks.rados.rados.0.smithi093.stdout:3892: finishing write tid 3 to smithi093153278-475 2023-08-27T20:55:29.845 INFO:tasks.rados.rados.0.smithi093.stdout:3892: finishing write tid 4 to smithi093153278-475 2023-08-27T20:55:29.846 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 475 v 473 (ObjNum 2385 snap 0 seq_num 2385) dirty exists 2023-08-27T20:55:29.846 INFO:tasks.rados.rados.0.smithi093.stdout:3892: left oid 475 (ObjNum 2385 snap 0 seq_num 2385) 2023-08-27T20:55:29.846 INFO:tasks.rados.rados.0.smithi093.stdout:3893: finishing write tid 1 to smithi093153278-141 2023-08-27T20:55:29.846 INFO:tasks.rados.rados.0.smithi093.stdout:3893: finishing write tid 2 to smithi093153278-141 2023-08-27T20:55:29.846 INFO:tasks.rados.rados.0.smithi093.stdout:3893: finishing write tid 3 to smithi093153278-141 2023-08-27T20:55:29.846 INFO:tasks.rados.rados.0.smithi093.stdout:3893: finishing write tid 4 to smithi093153278-141 2023-08-27T20:55:29.846 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 141 v 476 (ObjNum 2386 snap 0 seq_num 2386) dirty exists 2023-08-27T20:55:29.847 INFO:tasks.rados.rados.0.smithi093.stdout:3893: left oid 141 (ObjNum 2386 snap 0 seq_num 2386) 2023-08-27T20:55:29.847 INFO:tasks.rados.rados.0.smithi093.stdout:3896: seq_num 2387 ranges {518309=437852,1664871=453438} 2023-08-27T20:55:29.850 INFO:tasks.rados.rados.0.smithi093.stdout:3896: writing smithi093153278-289 from 518309 to 956161 tid 1 2023-08-27T20:55:29.856 INFO:tasks.rados.rados.0.smithi093.stdout:3896: writing smithi093153278-289 from 1664871 to 2118309 tid 2 2023-08-27T20:55:29.857 INFO:tasks.rados.rados.0.smithi093.stdout:3890: done (6 left) 2023-08-27T20:55:29.857 INFO:tasks.rados.rados.0.smithi093.stdout:3891: done (5 left) 2023-08-27T20:55:29.857 INFO:tasks.rados.rados.0.smithi093.stdout:3892: done (4 left) 2023-08-27T20:55:29.857 INFO:tasks.rados.rados.0.smithi093.stdout:3893: done (3 left) 2023-08-27T20:55:29.857 INFO:tasks.rados.rados.0.smithi093.stdout:3897: write oid 125 current snap is 0 2023-08-27T20:55:29.857 INFO:tasks.rados.rados.0.smithi093.stdout:3897: seq_num 2388 ranges {535116=629454,1839706=295410} 2023-08-27T20:55:29.862 INFO:tasks.rados.rados.0.smithi093.stdout:3897: writing smithi093153278-125 from 535116 to 1164570 tid 1 2023-08-27T20:55:29.868 INFO:tasks.rados.rados.0.smithi093.stdout:3897: writing smithi093153278-125 from 1839706 to 2135116 tid 2 2023-08-27T20:55:29.868 INFO:tasks.rados.rados.0.smithi093.stdout:3898: delete oid 268 current snap is 0 2023-08-27T20:55:29.868 INFO:tasks.rados.rados.0.smithi093.stdout:3894: expect (ObjNum 1935 snap 0 seq_num 1935) 2023-08-27T20:55:29.978 INFO:tasks.rados.rados.0.smithi093.stdout:3895: expect (ObjNum 1899 snap 0 seq_num 1899) 2023-08-27T20:55:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:29 smithi093 ceph-mon[111335]: pgmap v58: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1282246371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:29 smithi093 ceph-mon[118902]: pgmap v58: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1282246371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:30.150 INFO:tasks.rados.rados.0.smithi093.stdout:3896: finishing write tid 1 to smithi093153278-289 2023-08-27T20:55:30.150 INFO:tasks.rados.rados.0.smithi093.stdout:3896: finishing write tid 2 to smithi093153278-289 2023-08-27T20:55:30.150 INFO:tasks.rados.rados.0.smithi093.stdout:3896: finishing write tid 3 to smithi093153278-289 2023-08-27T20:55:30.150 INFO:tasks.rados.rados.0.smithi093.stdout:3896: finishing write tid 4 to smithi093153278-289 2023-08-27T20:55:30.150 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 289 v 501 (ObjNum 2387 snap 0 seq_num 2387) dirty exists 2023-08-27T20:55:30.150 INFO:tasks.rados.rados.0.smithi093.stdout:3896: left oid 289 (ObjNum 2387 snap 0 seq_num 2387) 2023-08-27T20:55:30.150 INFO:tasks.rados.rados.0.smithi093.stdout:3897: finishing write tid 1 to smithi093153278-125 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3894: done (4 left) 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3895: done (3 left) 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3896: done (2 left) 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3898: done (1 left) 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3899: read oid 411 snap -1 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3899: expect (ObjNum 1627 snap 0 seq_num 1627) 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3900: write oid 359 current snap is 0 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3897: finishing write tid 2 to smithi093153278-125 2023-08-27T20:55:30.151 INFO:tasks.rados.rados.0.smithi093.stdout:3900: seq_num 2390 ranges {568730=779952,1840317=328413} 2023-08-27T20:55:30.157 INFO:tasks.rados.rados.0.smithi093.stdout:3900: writing smithi093153278-359 from 568730 to 1348682 tid 1 2023-08-27T20:55:30.162 INFO:tasks.rados.rados.0.smithi093.stdout:3900: writing smithi093153278-359 from 1840317 to 2168730 tid 2 2023-08-27T20:55:30.162 INFO:tasks.rados.rados.0.smithi093.stdout:3901: write oid 365 current snap is 0 2023-08-27T20:55:30.162 INFO:tasks.rados.rados.0.smithi093.stdout:3897: finishing write tid 3 to smithi093153278-125 2023-08-27T20:55:30.162 INFO:tasks.rados.rados.0.smithi093.stdout:3897: finishing write tid 4 to smithi093153278-125 2023-08-27T20:55:30.162 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 125 v 559 (ObjNum 2388 snap 0 seq_num 2388) dirty exists 2023-08-27T20:55:30.162 INFO:tasks.rados.rados.0.smithi093.stdout:3897: left oid 125 (ObjNum 2388 snap 0 seq_num 2388) 2023-08-27T20:55:30.162 INFO:tasks.rados.rados.0.smithi093.stdout:3901: seq_num 2391 ranges {585537=455201,1498799=609166} 2023-08-27T20:55:30.167 INFO:tasks.rados.rados.0.smithi093.stdout:3901: writing smithi093153278-365 from 585537 to 1040738 tid 1 2023-08-27T20:55:30.173 INFO:tasks.rados.rados.0.smithi093.stdout:3901: writing smithi093153278-365 from 1498799 to 2107965 tid 2 2023-08-27T20:55:30.173 INFO:tasks.rados.rados.0.smithi093.stdout:3897: done (3 left) 2023-08-27T20:55:30.173 INFO:tasks.rados.rados.0.smithi093.stdout:3902: write oid 124 current snap is 0 2023-08-27T20:55:30.174 INFO:tasks.rados.rados.0.smithi093.stdout:3902: seq_num 2392 ranges {602344=530450,1557281=469177} 2023-08-27T20:55:30.179 INFO:tasks.rados.rados.0.smithi093.stdout:3902: writing smithi093153278-124 from 602344 to 1132794 tid 1 2023-08-27T20:55:30.185 INFO:tasks.rados.rados.0.smithi093.stdout:3902: writing smithi093153278-124 from 1557281 to 2026458 tid 2 2023-08-27T20:55:30.185 INFO:tasks.rados.rados.0.smithi093.stdout:3903: read oid 267 snap -1 2023-08-27T20:55:30.185 INFO:tasks.rados.rados.0.smithi093.stdout:3903: expect (ObjNum 2221 snap 0 seq_num 2221) 2023-08-27T20:55:30.185 INFO:tasks.rados.rados.0.smithi093.stdout:3904: write oid 434 current snap is 0 2023-08-27T20:55:30.185 INFO:tasks.rados.rados.0.smithi093.stdout:3904: seq_num 2393 ranges {619151=605699,2015763=203388} 2023-08-27T20:55:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:29 smithi136 ceph-mon[113531]: pgmap v58: 89 pgs: 89 active+clean; 1.1 GiB data, 2.3 GiB used, 713 GiB / 715 GiB avail; 56 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1282246371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:30.191 INFO:tasks.rados.rados.0.smithi093.stdout:3904: writing smithi093153278-434 from 619151 to 1224850 tid 1 2023-08-27T20:55:30.197 INFO:tasks.rados.rados.0.smithi093.stdout:3904: writing smithi093153278-434 from 2015763 to 2219151 tid 2 2023-08-27T20:55:30.197 INFO:tasks.rados.rados.0.smithi093.stdout:3905: write oid 433 current snap is 0 2023-08-27T20:55:30.197 INFO:tasks.rados.rados.0.smithi093.stdout:3899: expect (ObjNum 1627 snap 0 seq_num 1627) 2023-08-27T20:55:30.335 INFO:tasks.rados.rados.0.smithi093.stdout:3900: finishing write tid 1 to smithi093153278-359 2023-08-27T20:55:30.335 INFO:tasks.rados.rados.0.smithi093.stdout:3900: finishing write tid 2 to smithi093153278-359 2023-08-27T20:55:30.335 INFO:tasks.rados.rados.0.smithi093.stdout:3900: finishing write tid 3 to smithi093153278-359 2023-08-27T20:55:30.335 INFO:tasks.rados.rados.0.smithi093.stdout:3905: seq_num 2394 ranges {635958=680948,1957892=278066} 2023-08-27T20:55:30.342 INFO:tasks.rados.rados.0.smithi093.stdout:3905: writing smithi093153278-433 from 635958 to 1316906 tid 1 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3905: writing smithi093153278-433 from 1957892 to 2235958 tid 2 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3899: done (6 left) 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3906: write oid 123 current snap is 0 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3900: finishing write tid 4 to smithi093153278-359 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 359 v 621 (ObjNum 2390 snap 0 seq_num 2390) dirty exists 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3900: left oid 359 (ObjNum 2390 snap 0 seq_num 2390) 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3901: finishing write tid 1 to smithi093153278-365 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3901: finishing write tid 2 to smithi093153278-365 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3901: finishing write tid 3 to smithi093153278-365 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:3901: finishing write tid 4 to smithi093153278-365 2023-08-27T20:55:30.348 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 365 v 554 (ObjNum 2391 snap 0 seq_num 2391) dirty exists 2023-08-27T20:55:30.349 INFO:tasks.rados.rados.0.smithi093.stdout:3901: left oid 365 (ObjNum 2391 snap 0 seq_num 2391) 2023-08-27T20:55:30.349 INFO:tasks.rados.rados.0.smithi093.stdout:3902: finishing write tid 1 to smithi093153278-124 2023-08-27T20:55:30.349 INFO:tasks.rados.rados.0.smithi093.stdout:3902: finishing write tid 2 to smithi093153278-124 2023-08-27T20:55:30.349 INFO:tasks.rados.rados.0.smithi093.stdout:3902: finishing write tid 3 to smithi093153278-124 2023-08-27T20:55:30.349 INFO:tasks.rados.rados.0.smithi093.stdout:3906: seq_num 2395 ranges {652765=472550,1732727=520038} 2023-08-27T20:55:30.354 INFO:tasks.rados.rados.0.smithi093.stdout:3906: writing smithi093153278-123 from 652765 to 1125315 tid 1 2023-08-27T20:55:30.360 INFO:tasks.rados.rados.0.smithi093.stdout:3906: writing smithi093153278-123 from 1732727 to 2252765 tid 2 2023-08-27T20:55:30.360 INFO:tasks.rados.rados.0.smithi093.stdout:3900: done (6 left) 2023-08-27T20:55:30.360 INFO:tasks.rados.rados.0.smithi093.stdout:3901: done (5 left) 2023-08-27T20:55:30.360 INFO:tasks.rados.rados.0.smithi093.stdout:3907: write oid 344 current snap is 0 2023-08-27T20:55:30.361 INFO:tasks.rados.rados.0.smithi093.stdout:3902: finishing write tid 4 to smithi093153278-124 2023-08-27T20:55:30.361 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 124 v 457 (ObjNum 2392 snap 0 seq_num 2392) dirty exists 2023-08-27T20:55:30.361 INFO:tasks.rados.rados.0.smithi093.stdout:3902: left oid 124 (ObjNum 2392 snap 0 seq_num 2392) 2023-08-27T20:55:30.361 INFO:tasks.rados.rados.0.smithi093.stdout:3903: expect (ObjNum 2221 snap 0 seq_num 2221) 2023-08-27T20:55:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:3904: finishing write tid 1 to smithi093153278-434 2023-08-27T20:55:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:3904: finishing write tid 2 to smithi093153278-434 2023-08-27T20:55:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:3904: finishing write tid 3 to smithi093153278-434 2023-08-27T20:55:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:3904: finishing write tid 4 to smithi093153278-434 2023-08-27T20:55:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 434 v 441 (ObjNum 2393 snap 0 seq_num 2393) dirty exists 2023-08-27T20:55:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:3904: left oid 434 (ObjNum 2393 snap 0 seq_num 2393) 2023-08-27T20:55:30.615 INFO:tasks.rados.rados.0.smithi093.stdout:3907: seq_num 2396 ranges {669572=547799,1791209=478363} 2023-08-27T20:55:30.621 INFO:tasks.rados.rados.0.smithi093.stdout:3907: writing smithi093153278-344 from 669572 to 1217371 tid 1 2023-08-27T20:55:30.627 INFO:tasks.rados.rados.0.smithi093.stdout:3907: writing smithi093153278-344 from 1791209 to 2269572 tid 2 2023-08-27T20:55:30.627 INFO:tasks.rados.rados.0.smithi093.stdout:3902: done (5 left) 2023-08-27T20:55:30.627 INFO:tasks.rados.rados.0.smithi093.stdout:3903: done (4 left) 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3904: done (3 left) 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3908: write oid 38 current snap is 0 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3905: finishing write tid 1 to smithi093153278-433 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3905: finishing write tid 2 to smithi093153278-433 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3905: finishing write tid 3 to smithi093153278-433 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3905: finishing write tid 4 to smithi093153278-433 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 433 v 513 (ObjNum 2394 snap 0 seq_num 2394) dirty exists 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3905: left oid 433 (ObjNum 2394 snap 0 seq_num 2394) 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3906: finishing write tid 1 to smithi093153278-123 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3906: finishing write tid 2 to smithi093153278-123 2023-08-27T20:55:30.628 INFO:tasks.rados.rados.0.smithi093.stdout:3906: finishing write tid 3 to smithi093153278-123 2023-08-27T20:55:30.629 INFO:tasks.rados.rados.0.smithi093.stdout:3906: finishing write tid 4 to smithi093153278-123 2023-08-27T20:55:30.629 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 123 v 479 (ObjNum 2395 snap 0 seq_num 2395) dirty exists 2023-08-27T20:55:30.629 INFO:tasks.rados.rados.0.smithi093.stdout:3906: left oid 123 (ObjNum 2395 snap 0 seq_num 2395) 2023-08-27T20:55:30.629 INFO:tasks.rados.rados.0.smithi093.stdout:3908: seq_num 2397 ranges {686379=623048,1849691=436688} 2023-08-27T20:55:30.634 INFO:tasks.rados.rados.0.smithi093.stdout:3908: writing smithi093153278-38 from 686379 to 1309427 tid 1 2023-08-27T20:55:30.640 INFO:tasks.rados.rados.0.smithi093.stdout:3908: writing smithi093153278-38 from 1849691 to 2286379 tid 2 2023-08-27T20:55:30.640 INFO:tasks.rados.rados.0.smithi093.stdout:3905: done (3 left) 2023-08-27T20:55:30.640 INFO:tasks.rados.rados.0.smithi093.stdout:3906: done (2 left) 2023-08-27T20:55:30.640 INFO:tasks.rados.rados.0.smithi093.stdout:3909: write oid 27 current snap is 0 2023-08-27T20:55:30.640 INFO:tasks.rados.rados.0.smithi093.stdout:3907: finishing write tid 1 to smithi093153278-344 2023-08-27T20:55:30.640 INFO:tasks.rados.rados.0.smithi093.stdout:3909: seq_num 2398 ranges {703186=698297,2191820=111366} 2023-08-27T20:55:30.647 INFO:tasks.rados.rados.0.smithi093.stdout:3909: writing smithi093153278-27 from 703186 to 1401483 tid 1 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:3909: writing smithi093153278-27 from 2191820 to 2303186 tid 2 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:3910: write oid 141 current snap is 0 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:3907: finishing write tid 2 to smithi093153278-344 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:3907: finishing write tid 3 to smithi093153278-344 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:3907: finishing write tid 4 to smithi093153278-344 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 344 v 562 (ObjNum 2396 snap 0 seq_num 2396) dirty exists 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:3907: left oid 344 (ObjNum 2396 snap 0 seq_num 2396) 2023-08-27T20:55:30.653 INFO:tasks.rados.rados.0.smithi093.stdout:3910: seq_num 2399 ranges {719993=773546,2250302=69691} 2023-08-27T20:55:30.661 INFO:tasks.rados.rados.0.smithi093.stdout:3910: writing smithi093153278-141 from 719993 to 1493539 tid 1 2023-08-27T20:55:30.666 INFO:tasks.rados.rados.0.smithi093.stdout:3910: writing smithi093153278-141 from 2250302 to 2319993 tid 2 2023-08-27T20:55:30.666 INFO:tasks.rados.rados.0.smithi093.stdout:3907: done (3 left) 2023-08-27T20:55:30.666 INFO:tasks.rados.rados.0.smithi093.stdout:3911: write oid 155 current snap is 0 2023-08-27T20:55:30.666 INFO:tasks.rados.rados.0.smithi093.stdout:3908: finishing write tid 1 to smithi093153278-38 2023-08-27T20:55:30.666 INFO:tasks.rados.rados.0.smithi093.stdout:3911: seq_num 2400 ranges {736800=448795,1908784=428016} 2023-08-27T20:55:30.672 INFO:tasks.rados.rados.0.smithi093.stdout:3911: writing smithi093153278-155 from 736800 to 1185595 tid 1 2023-08-27T20:55:30.678 INFO:tasks.rados.rados.0.smithi093.stdout:3911: writing smithi093153278-155 from 1908784 to 2336800 tid 2 2023-08-27T20:55:30.679 INFO:tasks.rados.rados.0.smithi093.stdout:3912: write oid 73 current snap is 0 2023-08-27T20:55:30.679 INFO:tasks.rados.rados.0.smithi093.stdout:3908: finishing write tid 2 to smithi093153278-38 2023-08-27T20:55:30.679 INFO:tasks.rados.rados.0.smithi093.stdout:3908: finishing write tid 3 to smithi093153278-38 2023-08-27T20:55:30.679 INFO:tasks.rados.rados.0.smithi093.stdout:3912: seq_num 2401 ranges {753607=524044,1967266=386341} 2023-08-27T20:55:30.685 INFO:tasks.rados.rados.0.smithi093.stdout:3912: writing smithi093153278-73 from 753607 to 1277651 tid 1 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:3912: writing smithi093153278-73 from 1967266 to 2353607 tid 2 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:3908: finishing write tid 4 to smithi093153278-38 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 38 v 425 (ObjNum 2397 snap 0 seq_num 2397) dirty exists 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:3908: left oid 38 (ObjNum 2397 snap 0 seq_num 2397) 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:3909: finishing write tid 1 to smithi093153278-27 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:3909: finishing write tid 2 to smithi093153278-27 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:3909: finishing write tid 3 to smithi093153278-27 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:3909: finishing write tid 4 to smithi093153278-27 2023-08-27T20:55:30.692 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 27 v 428 (ObjNum 2398 snap 0 seq_num 2398) dirty exists 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3909: left oid 27 (ObjNum 2398 snap 0 seq_num 2398) 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3910: finishing write tid 1 to smithi093153278-141 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3908: done (4 left) 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3909: done (3 left) 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3913: read oid 377 snap -1 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3913: expect (ObjNum 2298 snap 0 seq_num 2298) 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3914: write oid 62 current snap is 0 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3910: finishing write tid 2 to smithi093153278-141 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3910: finishing write tid 3 to smithi093153278-141 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3910: finishing write tid 4 to smithi093153278-141 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 141 v 482 (ObjNum 2399 snap 0 seq_num 2399) dirty exists 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3910: left oid 141 (ObjNum 2399 snap 0 seq_num 2399) 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3911: finishing write tid 1 to smithi093153278-155 2023-08-27T20:55:30.693 INFO:tasks.rados.rados.0.smithi093.stdout:3911: finishing write tid 2 to smithi093153278-155 2023-08-27T20:55:30.694 INFO:tasks.rados.rados.0.smithi093.stdout:3911: finishing write tid 3 to smithi093153278-155 2023-08-27T20:55:30.694 INFO:tasks.rados.rados.0.smithi093.stdout:3914: seq_num 2402 ranges {770414=599293,1909395=461019} 2023-08-27T20:55:30.699 INFO:tasks.rados.rados.0.smithi093.stdout:3914: writing smithi093153278-62 from 770414 to 1369707 tid 1 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:3914: writing smithi093153278-62 from 1909395 to 2370414 tid 2 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:3910: done (4 left) 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:3915: write oid 449 current snap is 0 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:3911: finishing write tid 4 to smithi093153278-155 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 155 v 504 (ObjNum 2400 snap 0 seq_num 2400) dirty exists 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:3911: left oid 155 (ObjNum 2400 snap 0 seq_num 2400) 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:3912: finishing write tid 1 to smithi093153278-73 2023-08-27T20:55:30.705 INFO:tasks.rados.rados.0.smithi093.stdout:3915: seq_num 2403 ranges {787221=790895,2084230=302991} 2023-08-27T20:55:30.713 INFO:tasks.rados.rados.0.smithi093.stdout:3915: writing smithi093153278-449 from 787221 to 1578116 tid 1 2023-08-27T20:55:30.718 INFO:tasks.rados.rados.0.smithi093.stdout:3915: writing smithi093153278-449 from 2084230 to 2387221 tid 2 2023-08-27T20:55:30.718 INFO:tasks.rados.rados.0.smithi093.stdout:3911: done (4 left) 2023-08-27T20:55:30.718 INFO:tasks.rados.rados.0.smithi093.stdout:3916: write oid 394 current snap is 0 2023-08-27T20:55:30.718 INFO:tasks.rados.rados.0.smithi093.stdout:3912: finishing write tid 2 to smithi093153278-73 2023-08-27T20:55:30.718 INFO:tasks.rados.rados.0.smithi093.stdout:3912: finishing write tid 3 to smithi093153278-73 2023-08-27T20:55:30.719 INFO:tasks.rados.rados.0.smithi093.stdout:3912: finishing write tid 4 to smithi093153278-73 2023-08-27T20:55:30.719 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 73 v 565 (ObjNum 2401 snap 0 seq_num 2401) dirty exists 2023-08-27T20:55:30.719 INFO:tasks.rados.rados.0.smithi093.stdout:3912: left oid 73 (ObjNum 2401 snap 0 seq_num 2401) 2023-08-27T20:55:30.719 INFO:tasks.rados.rados.0.smithi093.stdout:3913: expect (ObjNum 2298 snap 0 seq_num 2298) 2023-08-27T20:55:30.832 INFO:tasks.rados.rados.0.smithi093.stdout:3914: finishing write tid 1 to smithi093153278-62 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3914: finishing write tid 2 to smithi093153278-62 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3914: finishing write tid 3 to smithi093153278-62 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3914: finishing write tid 4 to smithi093153278-62 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 62 v 471 (ObjNum 2402 snap 0 seq_num 2402) dirty exists 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3914: left oid 62 (ObjNum 2402 snap 0 seq_num 2402) 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3915: finishing write tid 1 to smithi093153278-449 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3915: finishing write tid 2 to smithi093153278-449 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3915: finishing write tid 3 to smithi093153278-449 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3915: finishing write tid 4 to smithi093153278-449 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 449 v 560 (ObjNum 2403 snap 0 seq_num 2403) dirty exists 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3915: left oid 449 (ObjNum 2403 snap 0 seq_num 2403) 2023-08-27T20:55:30.833 INFO:tasks.rados.rados.0.smithi093.stdout:3916: seq_num 2404 ranges {404028=466144,1342712=774838} 2023-08-27T20:55:30.837 INFO:tasks.rados.rados.0.smithi093.stdout:3916: writing smithi093153278-394 from 404028 to 870172 tid 1 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3916: writing smithi093153278-394 from 1342712 to 2117550 tid 2 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3912: done (4 left) 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3913: done (3 left) 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3914: done (2 left) 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3915: done (1 left) 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3917: write oid 126 current snap is 0 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3916: finishing write tid 1 to smithi093153278-394 2023-08-27T20:55:30.844 INFO:tasks.rados.rados.0.smithi093.stdout:3917: seq_num 2405 ranges {420835=541393,1401194=634849} 2023-08-27T20:55:30.849 INFO:tasks.rados.rados.0.smithi093.stdout:3917: writing smithi093153278-126 from 420835 to 962228 tid 1 2023-08-27T20:55:30.855 INFO:tasks.rados.rados.0.smithi093.stdout:3917: writing smithi093153278-126 from 1401194 to 2036043 tid 2 2023-08-27T20:55:30.855 INFO:tasks.rados.rados.0.smithi093.stdout:3918: write oid 406 current snap is 0 2023-08-27T20:55:30.855 INFO:tasks.rados.rados.0.smithi093.stdout:3918: seq_num 2406 ranges {437642=616642,1743323=694319} 2023-08-27T20:55:30.860 INFO:tasks.rados.rados.0.smithi093.stdout:3918: writing smithi093153278-406 from 437642 to 1054284 tid 1 2023-08-27T20:55:30.868 INFO:tasks.rados.rados.0.smithi093.stdout:3918: writing smithi093153278-406 from 1743323 to 2437642 tid 2 2023-08-27T20:55:30.869 INFO:tasks.rados.rados.0.smithi093.stdout:3919: write oid 410 current snap is 0 2023-08-27T20:55:30.869 INFO:tasks.rados.rados.0.smithi093.stdout:3916: finishing write tid 2 to smithi093153278-394 2023-08-27T20:55:30.869 INFO:tasks.rados.rados.0.smithi093.stdout:3919: seq_num 2407 ranges {454449=691891,1801805=522165} 2023-08-27T20:55:30.875 INFO:tasks.rados.rados.0.smithi093.stdout:3919: writing smithi093153278-410 from 454449 to 1146340 tid 1 2023-08-27T20:55:30.882 INFO:tasks.rados.rados.0.smithi093.stdout:3919: writing smithi093153278-410 from 1801805 to 2323970 tid 2 2023-08-27T20:55:30.882 INFO:tasks.rados.rados.0.smithi093.stdout:3920: read oid 247 snap -1 2023-08-27T20:55:30.882 INFO:tasks.rados.rados.0.smithi093.stdout:3920: expect deleted 2023-08-27T20:55:30.882 INFO:tasks.rados.rados.0.smithi093.stdout:3921: read oid 169 snap -1 2023-08-27T20:55:30.882 INFO:tasks.rados.rados.0.smithi093.stdout:3921: expect (ObjNum 2274 snap 0 seq_num 2274) 2023-08-27T20:55:30.882 INFO:tasks.rados.rados.0.smithi093.stdout:3922: write oid 456 current snap is 0 2023-08-27T20:55:30.883 INFO:tasks.rados.rados.0.smithi093.stdout:3916: finishing write tid 3 to smithi093153278-394 2023-08-27T20:55:30.883 INFO:tasks.rados.rados.0.smithi093.stdout:3916: finishing write tid 4 to smithi093153278-394 2023-08-27T20:55:30.883 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 394 v 568 (ObjNum 2404 snap 0 seq_num 2404) dirty exists 2023-08-27T20:55:30.883 INFO:tasks.rados.rados.0.smithi093.stdout:3916: left oid 394 (ObjNum 2404 snap 0 seq_num 2404) 2023-08-27T20:55:30.883 INFO:tasks.rados.rados.0.smithi093.stdout:3917: finishing write tid 1 to smithi093153278-126 2023-08-27T20:55:30.883 INFO:tasks.rados.rados.0.smithi093.stdout:3922: seq_num 2408 ranges {471256=767140,1860287=610969} 2023-08-27T20:55:30.888 INFO:tasks.rados.rados.0.smithi093.stdout:3922: writing smithi093153278-456 from 471256 to 1238396 tid 1 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3922: writing smithi093153278-456 from 1860287 to 2471256 tid 2 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3916: done (6 left) 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3923: write oid 122 current snap is 0 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3917: finishing write tid 2 to smithi093153278-126 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3917: finishing write tid 3 to smithi093153278-126 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3917: finishing write tid 4 to smithi093153278-126 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 126 v 444 (ObjNum 2405 snap 0 seq_num 2405) dirty exists 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3917: left oid 126 (ObjNum 2405 snap 0 seq_num 2405) 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3918: finishing write tid 1 to smithi093153278-406 2023-08-27T20:55:30.896 INFO:tasks.rados.rados.0.smithi093.stdout:3923: seq_num 2409 ranges {488063=442389,1518769=525834} 2023-08-27T20:55:30.900 INFO:tasks.rados.rados.0.smithi093.stdout:3923: writing smithi093153278-122 from 488063 to 930452 tid 1 2023-08-27T20:55:30.907 INFO:tasks.rados.rados.0.smithi093.stdout:3923: writing smithi093153278-122 from 1518769 to 2044603 tid 2 2023-08-27T20:55:30.907 INFO:tasks.rados.rados.0.smithi093.stdout:3917: done (6 left) 2023-08-27T20:55:30.907 INFO:tasks.rados.rados.0.smithi093.stdout:3924: write oid 292 current snap is 0 2023-08-27T20:55:30.907 INFO:tasks.rados.rados.0.smithi093.stdout:3918: finishing write tid 2 to smithi093153278-406 2023-08-27T20:55:30.907 INFO:tasks.rados.rados.0.smithi093.stdout:3918: finishing write tid 3 to smithi093153278-406 2023-08-27T20:55:30.908 INFO:tasks.rados.rados.0.smithi093.stdout:3918: finishing write tid 4 to smithi093153278-406 2023-08-27T20:55:30.908 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 406 v 431 (ObjNum 2406 snap 0 seq_num 2406) dirty exists 2023-08-27T20:55:30.908 INFO:tasks.rados.rados.0.smithi093.stdout:3918: left oid 406 (ObjNum 2406 snap 0 seq_num 2406) 2023-08-27T20:55:30.908 INFO:tasks.rados.rados.0.smithi093.stdout:3919: finishing write tid 1 to smithi093153278-410 2023-08-27T20:55:30.908 INFO:tasks.rados.rados.0.smithi093.stdout:3924: seq_num 2410 ranges {504870=633991,1693604=785845} 2023-08-27T20:55:30.913 INFO:tasks.rados.rados.0.smithi093.stdout:3924: writing smithi093153278-292 from 504870 to 1138861 tid 1 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3924: writing smithi093153278-292 from 1693604 to 2479449 tid 2 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3918: done (6 left) 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3925: delete oid 289 current snap is 0 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3919: finishing write tid 2 to smithi093153278-410 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3919: finishing write tid 3 to smithi093153278-410 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3919: finishing write tid 4 to smithi093153278-410 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 410 v 460 (ObjNum 2407 snap 0 seq_num 2407) dirty exists 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3919: left oid 410 (ObjNum 2407 snap 0 seq_num 2407) 2023-08-27T20:55:30.921 INFO:tasks.rados.rados.0.smithi093.stdout:3921: expect (ObjNum 2274 snap 0 seq_num 2274) 2023-08-27T20:55:30.989 INFO:tasks.rados.rados.0.smithi093.stdout:3922: finishing write tid 1 to smithi093153278-456 2023-08-27T20:55:30.989 INFO:tasks.rados.rados.0.smithi093.stdout:3922: finishing write tid 2 to smithi093153278-456 2023-08-27T20:55:30.989 INFO:tasks.rados.rados.0.smithi093.stdout:3922: finishing write tid 3 to smithi093153278-456 2023-08-27T20:55:30.989 INFO:tasks.rados.rados.0.smithi093.stdout:3922: finishing write tid 4 to smithi093153278-456 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 456 v 563 (ObjNum 2408 snap 0 seq_num 2408) dirty exists 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3922: left oid 456 (ObjNum 2408 snap 0 seq_num 2408) 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3923: finishing write tid 1 to smithi093153278-122 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3923: finishing write tid 2 to smithi093153278-122 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3919: done (6 left) 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3920: done (5 left) 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3921: done (4 left) 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3922: done (3 left) 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3925: done (2 left) 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3926: read oid 130 snap -1 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3926: expect deleted 2023-08-27T20:55:30.990 INFO:tasks.rados.rados.0.smithi093.stdout:3927: write oid 325 current snap is 0 2023-08-27T20:55:30.991 INFO:tasks.rados.rados.0.smithi093.stdout:3923: finishing write tid 3 to smithi093153278-122 2023-08-27T20:55:30.991 INFO:tasks.rados.rados.0.smithi093.stdout:3927: seq_num 2412 ranges {538484=784489,2094215=444269} 2023-08-27T20:55:30.996 INFO:tasks.rados.rados.0.smithi093.stdout:3927: writing smithi093153278-325 from 538484 to 1322973 tid 1 2023-08-27T20:55:31.003 INFO:tasks.rados.rados.0.smithi093.stdout:3927: writing smithi093153278-325 from 2094215 to 2538484 tid 2 2023-08-27T20:55:31.003 INFO:tasks.rados.rados.0.smithi093.stdout:3928: write oid 309 current snap is 0 2023-08-27T20:55:31.003 INFO:tasks.rados.rados.0.smithi093.stdout:3923: finishing write tid 4 to smithi093153278-122 2023-08-27T20:55:31.003 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 122 v 311 (ObjNum 2409 snap 0 seq_num 2409) dirty exists 2023-08-27T20:55:31.003 INFO:tasks.rados.rados.0.smithi093.stdout:3923: left oid 122 (ObjNum 2409 snap 0 seq_num 2409) 2023-08-27T20:55:31.003 INFO:tasks.rados.rados.0.smithi093.stdout:3924: finishing write tid 1 to smithi093153278-292 2023-08-27T20:55:31.004 INFO:tasks.rados.rados.0.smithi093.stdout:3924: finishing write tid 2 to smithi093153278-292 2023-08-27T20:55:31.004 INFO:tasks.rados.rados.0.smithi093.stdout:3924: finishing write tid 3 to smithi093153278-292 2023-08-27T20:55:31.004 INFO:tasks.rados.rados.0.smithi093.stdout:3928: seq_num 2413 ranges {555291=459738,1752697=533172} 2023-08-27T20:55:31.008 INFO:tasks.rados.rados.0.smithi093.stdout:3928: writing smithi093153278-309 from 555291 to 1015029 tid 1 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3928: writing smithi093153278-309 from 1752697 to 2285869 tid 2 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3923: done (4 left) 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3929: read oid 244 snap -1 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3929: expect (ObjNum 2291 snap 0 seq_num 2291) 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3930: read oid 77 snap -1 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3930: expect (ObjNum 2281 snap 0 seq_num 2281) 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3931: read oid 222 snap -1 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3931: expect (ObjNum 2371 snap 0 seq_num 2371) 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3932: write oid 299 current snap is 0 2023-08-27T20:55:31.016 INFO:tasks.rados.rados.0.smithi093.stdout:3924: finishing write tid 4 to smithi093153278-292 2023-08-27T20:55:31.017 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 292 v 463 (ObjNum 2410 snap 0 seq_num 2410) dirty exists 2023-08-27T20:55:31.017 INFO:tasks.rados.rados.0.smithi093.stdout:3924: left oid 292 (ObjNum 2410 snap 0 seq_num 2410) 2023-08-27T20:55:31.017 INFO:tasks.rados.rados.0.smithi093.stdout:3932: seq_num 2414 ranges {572098=534987,1811179=760919} 2023-08-27T20:55:31.021 INFO:tasks.rados.rados.0.smithi093.stdout:3932: writing smithi093153278-299 from 572098 to 1107085 tid 1 2023-08-27T20:55:31.030 INFO:tasks.rados.rados.0.smithi093.stdout:3932: writing smithi093153278-299 from 1811179 to 2572098 tid 2 2023-08-27T20:55:31.030 INFO:tasks.rados.rados.0.smithi093.stdout:3924: done (7 left) 2023-08-27T20:55:31.030 INFO:tasks.rados.rados.0.smithi093.stdout:3933: write oid 357 current snap is 0 2023-08-27T20:55:31.030 INFO:tasks.rados.rados.0.smithi093.stdout:3933: seq_num 2415 ranges {588905=610236,1753308=536841} 2023-08-27T20:55:31.036 INFO:tasks.rados.rados.0.smithi093.stdout:3933: writing smithi093153278-357 from 588905 to 1199141 tid 1 2023-08-27T20:55:31.042 INFO:tasks.rados.rados.0.smithi093.stdout:3933: writing smithi093153278-357 from 1753308 to 2290149 tid 2 2023-08-27T20:55:31.043 INFO:tasks.rados.rados.0.smithi093.stdout:3934: write oid 99 current snap is 0 2023-08-27T20:55:31.043 INFO:tasks.rados.rados.0.smithi093.stdout:3934: seq_num 2416 ranges {605712=685485,1811790=793922} 2023-08-27T20:55:31.049 INFO:tasks.rados.rados.0.smithi093.stdout:3934: writing smithi093153278-99 from 605712 to 1291197 tid 1 2023-08-27T20:55:31.057 INFO:tasks.rados.rados.0.smithi093.stdout:3934: writing smithi093153278-99 from 1811790 to 2605712 tid 2 2023-08-27T20:55:31.057 INFO:tasks.rados.rados.0.smithi093.stdout:3935: read oid 359 snap -1 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3935: expect (ObjNum 2390 snap 0 seq_num 2390) 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3936: delete oid 124 current snap is 0 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3927: finishing write tid 1 to smithi093153278-325 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3927: finishing write tid 2 to smithi093153278-325 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3927: finishing write tid 3 to smithi093153278-325 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3927: finishing write tid 4 to smithi093153278-325 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 325 v 516 (ObjNum 2412 snap 0 seq_num 2412) dirty exists 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3927: left oid 325 (ObjNum 2412 snap 0 seq_num 2412) 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3928: finishing write tid 1 to smithi093153278-309 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3928: finishing write tid 2 to smithi093153278-309 2023-08-27T20:55:31.058 INFO:tasks.rados.rados.0.smithi093.stdout:3931: expect (ObjNum 2371 snap 0 seq_num 2371) 2023-08-27T20:55:31.229 INFO:tasks.rados.rados.0.smithi093.stdout:3928: finishing write tid 3 to smithi093153278-309 2023-08-27T20:55:31.229 INFO:tasks.rados.rados.0.smithi093.stdout:3928: finishing write tid 4 to smithi093153278-309 2023-08-27T20:55:31.229 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 309 v 422 (ObjNum 2413 snap 0 seq_num 2413) dirty exists 2023-08-27T20:55:31.229 INFO:tasks.rados.rados.0.smithi093.stdout:3928: left oid 309 (ObjNum 2413 snap 0 seq_num 2413) 2023-08-27T20:55:31.229 INFO:tasks.rados.rados.0.smithi093.stdout:3932: finishing write tid 1 to smithi093153278-299 2023-08-27T20:55:31.229 INFO:tasks.rados.rados.0.smithi093.stdout:3930: expect (ObjNum 2281 snap 0 seq_num 2281) 2023-08-27T20:55:31.373 INFO:tasks.rados.rados.0.smithi093.stdout:3929: expect (ObjNum 2291 snap 0 seq_num 2291) 2023-08-27T20:55:31.585 INFO:tasks.rados.rados.0.smithi093.stdout:3932: finishing write tid 2 to smithi093153278-299 2023-08-27T20:55:31.585 INFO:tasks.rados.rados.0.smithi093.stdout:3932: finishing write tid 3 to smithi093153278-299 2023-08-27T20:55:31.585 INFO:tasks.rados.rados.0.smithi093.stdout:3926: done (10 left) 2023-08-27T20:55:31.585 INFO:tasks.rados.rados.0.smithi093.stdout:3927: done (9 left) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3928: done (8 left) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3929: done (7 left) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3930: done (6 left) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3931: done (5 left) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3936: done (4 left) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3937: read oid 290 snap -1 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3937: expect (ObjNum 2131 snap 0 seq_num 2131) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3938: read oid 332 snap -1 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3938: expect (ObjNum 2021 snap 0 seq_num 2021) 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3939: write oid 461 current snap is 0 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:3932: finishing write tid 4 to smithi093153278-299 2023-08-27T20:55:31.586 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 299 v 386 (ObjNum 2414 snap 0 seq_num 2414) dirty exists 2023-08-27T20:55:31.587 INFO:tasks.rados.rados.0.smithi093.stdout:3932: left oid 299 (ObjNum 2414 snap 0 seq_num 2414) 2023-08-27T20:55:31.587 INFO:tasks.rados.rados.0.smithi093.stdout:3933: finishing write tid 1 to smithi093153278-357 2023-08-27T20:55:31.587 INFO:tasks.rados.rados.0.smithi093.stdout:3939: seq_num 2418 ranges {639326=552336,1645107=400521,2501032=138294} 2023-08-27T20:55:31.591 INFO:tasks.rados.rados.0.smithi093.stdout:3939: writing smithi093153278-461 from 639326 to 1191662 tid 1 2023-08-27T20:55:31.596 INFO:tasks.rados.rados.0.smithi093.stdout:3939: writing smithi093153278-461 from 1645107 to 2045628 tid 2 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:3939: writing smithi093153278-461 from 2501032 to 2639326 tid 3 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:3932: done (6 left) 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:3940: write oid 103 current snap is 0 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:3933: finishing write tid 2 to smithi093153278-357 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:3933: finishing write tid 3 to smithi093153278-357 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:3933: finishing write tid 4 to smithi093153278-357 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 357 v 389 (ObjNum 2415 snap 0 seq_num 2415) dirty exists 2023-08-27T20:55:31.600 INFO:tasks.rados.rados.0.smithi093.stdout:3933: left oid 357 (ObjNum 2415 snap 0 seq_num 2415) 2023-08-27T20:55:31.601 INFO:tasks.rados.rados.0.smithi093.stdout:3934: finishing write tid 1 to smithi093153278-99 2023-08-27T20:55:31.601 INFO:tasks.rados.rados.0.smithi093.stdout:3935: expect (ObjNum 2390 snap 0 seq_num 2390) 2023-08-27T20:55:31.668 INFO:tasks.rados.rados.0.smithi093.stdout:3934: finishing write tid 2 to smithi093153278-99 2023-08-27T20:55:31.668 INFO:tasks.rados.rados.0.smithi093.stdout:3934: finishing write tid 3 to smithi093153278-99 2023-08-27T20:55:31.668 INFO:tasks.rados.rados.0.smithi093.stdout:3940: seq_num 2419 ranges {656133=627585,1987236=544179} 2023-08-27T20:55:31.675 INFO:tasks.rados.rados.0.smithi093.stdout:3940: writing smithi093153278-103 from 656133 to 1283718 tid 1 2023-08-27T20:55:31.682 INFO:tasks.rados.rados.0.smithi093.stdout:3940: writing smithi093153278-103 from 1987236 to 2531415 tid 2 2023-08-27T20:55:31.682 INFO:tasks.rados.rados.0.smithi093.stdout:3933: done (6 left) 2023-08-27T20:55:31.682 INFO:tasks.rados.rados.0.smithi093.stdout:3935: done (5 left) 2023-08-27T20:55:31.682 INFO:tasks.rados.rados.0.smithi093.stdout:3941: write oid 25 current snap is 0 2023-08-27T20:55:31.682 INFO:tasks.rados.rados.0.smithi093.stdout:3934: finishing write tid 4 to smithi093153278-99 2023-08-27T20:55:31.683 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 99 v 508 (ObjNum 2416 snap 0 seq_num 2416) dirty exists 2023-08-27T20:55:31.683 INFO:tasks.rados.rados.0.smithi093.stdout:3934: left oid 99 (ObjNum 2416 snap 0 seq_num 2416) 2023-08-27T20:55:31.683 INFO:tasks.rados.rados.0.smithi093.stdout:3938: expect (ObjNum 2021 snap 0 seq_num 2021) 2023-08-27T20:55:31.938 INFO:tasks.rados.rados.0.smithi093.stdout:3937: expect (ObjNum 2131 snap 0 seq_num 2131) 2023-08-27T20:55:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:31 smithi093 ceph-mon[118902]: pgmap v59: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 MiB/s rd, 13 MiB/s wr, 117 op/s 2023-08-27T20:55:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/517562209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:31 smithi093 ceph-mon[111335]: pgmap v59: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 MiB/s rd, 13 MiB/s wr, 117 op/s 2023-08-27T20:55:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/517562209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:31.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:31.630Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:32.144 INFO:tasks.rados.rados.0.smithi093.stdout:3939: finishing write tid 1 to smithi093153278-461 2023-08-27T20:55:32.144 INFO:tasks.rados.rados.0.smithi093.stdout:3939: finishing write tid 2 to smithi093153278-461 2023-08-27T20:55:32.144 INFO:tasks.rados.rados.0.smithi093.stdout:3941: seq_num 2420 ranges {672940=702834,2045718=627222} 2023-08-27T20:55:32.151 INFO:tasks.rados.rados.0.smithi093.stdout:3941: writing smithi093153278-25 from 672940 to 1375774 tid 1 2023-08-27T20:55:32.159 INFO:tasks.rados.rados.0.smithi093.stdout:3941: writing smithi093153278-25 from 2045718 to 2672940 tid 2 2023-08-27T20:55:32.159 INFO:tasks.rados.rados.0.smithi093.stdout:3934: done (5 left) 2023-08-27T20:55:32.159 INFO:tasks.rados.rados.0.smithi093.stdout:3937: done (4 left) 2023-08-27T20:55:32.159 INFO:tasks.rados.rados.0.smithi093.stdout:3938: done (3 left) 2023-08-27T20:55:32.159 INFO:tasks.rados.rados.0.smithi093.stdout:3942: write oid 160 current snap is 0 2023-08-27T20:55:32.159 INFO:tasks.rados.rados.0.smithi093.stdout:3939: finishing write tid 3 to smithi093153278-461 2023-08-27T20:55:32.159 INFO:tasks.rados.rados.0.smithi093.stdout:3942: seq_num 2421 ranges {689747=778083,2104200=547848} 2023-08-27T20:55:32.166 INFO:tasks.rados.rados.0.smithi093.stdout:3942: writing smithi093153278-160 from 689747 to 1467830 tid 1 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:3942: writing smithi093153278-160 from 2104200 to 2652048 tid 2 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:3943: write oid 346 current snap is 0 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:3939: finishing write tid 4 to smithi093153278-461 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:3939: finishing write tid 5 to smithi093153278-461 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 461 v 468 (ObjNum 2418 snap 0 seq_num 2418) dirty exists 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:3939: left oid 461 (ObjNum 2418 snap 0 seq_num 2418) 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:3940: finishing write tid 1 to smithi093153278-103 2023-08-27T20:55:32.174 INFO:tasks.rados.rados.0.smithi093.stdout:3943: seq_num 2422 ranges {706554=453332,1762682=691506} 2023-08-27T20:55:32.180 INFO:tasks.rados.rados.0.smithi093.stdout:3943: writing smithi093153278-346 from 706554 to 1159886 tid 1 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3943: writing smithi093153278-346 from 1762682 to 2454188 tid 2 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3939: done (4 left) 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3944: read oid 102 snap -1 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3944: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3945: read oid 296 snap -1 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3945: expect (ObjNum 1654 snap 0 seq_num 1654) 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3946: read oid 372 snap -1 2023-08-27T20:55:32.188 INFO:tasks.rados.rados.0.smithi093.stdout:3946: expect (ObjNum 1197 snap 0 seq_num 1197) 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3947: read oid 223 snap -1 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3947: expect deleted 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3948: delete oid 227 current snap is 0 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3940: finishing write tid 2 to smithi093153278-103 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3940: finishing write tid 3 to smithi093153278-103 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3940: finishing write tid 4 to smithi093153278-103 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 103 v 392 (ObjNum 2419 snap 0 seq_num 2419) dirty exists 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3940: left oid 103 (ObjNum 2419 snap 0 seq_num 2419) 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3941: finishing write tid 1 to smithi093153278-25 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3941: finishing write tid 2 to smithi093153278-25 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3941: finishing write tid 3 to smithi093153278-25 2023-08-27T20:55:32.189 INFO:tasks.rados.rados.0.smithi093.stdout:3941: finishing write tid 4 to smithi093153278-25 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 25 v 474 (ObjNum 2420 snap 0 seq_num 2420) dirty exists 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:3941: left oid 25 (ObjNum 2420 snap 0 seq_num 2420) 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:3942: finishing write tid 1 to smithi093153278-160 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:3942: finishing write tid 2 to smithi093153278-160 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:3942: finishing write tid 3 to smithi093153278-160 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:3942: finishing write tid 4 to smithi093153278-160 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 160 v 557 (ObjNum 2421 snap 0 seq_num 2421) dirty exists 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:3942: left oid 160 (ObjNum 2421 snap 0 seq_num 2421) 2023-08-27T20:55:32.190 INFO:tasks.rados.rados.0.smithi093.stdout:3943: finishing write tid 1 to smithi093153278-346 2023-08-27T20:55:32.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:31 smithi136 ceph-mon[113531]: pgmap v59: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 83 MiB/s rd, 13 MiB/s wr, 117 op/s 2023-08-27T20:55:32.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/517562209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:32.191 INFO:tasks.rados.rados.0.smithi093.stdout:3940: done (8 left) 2023-08-27T20:55:32.191 INFO:tasks.rados.rados.0.smithi093.stdout:3941: done (7 left) 2023-08-27T20:55:32.192 INFO:tasks.rados.rados.0.smithi093.stdout:3942: done (6 left) 2023-08-27T20:55:32.192 INFO:tasks.rados.rados.0.smithi093.stdout:3947: done (5 left) 2023-08-27T20:55:32.192 INFO:tasks.rados.rados.0.smithi093.stdout:3948: done (4 left) 2023-08-27T20:55:32.192 INFO:tasks.rados.rados.0.smithi093.stdout:3949: write oid 315 current snap is 0 2023-08-27T20:55:32.192 INFO:tasks.rados.rados.0.smithi093.stdout:3949: seq_num 2424 ranges {740168=603830,1763293=695175} 2023-08-27T20:55:32.197 INFO:tasks.rados.rados.0.smithi093.stdout:3949: writing smithi093153278-315 from 740168 to 1343998 tid 1 2023-08-27T20:55:32.203 INFO:tasks.rados.rados.0.smithi093.stdout:3949: writing smithi093153278-315 from 1763293 to 2458468 tid 2 2023-08-27T20:55:32.203 INFO:tasks.rados.rados.0.smithi093.stdout:3950: delete oid 253 current snap is 0 2023-08-27T20:55:32.203 INFO:tasks.rados.rados.0.smithi093.stdout:3943: finishing write tid 2 to smithi093153278-346 2023-08-27T20:55:32.204 INFO:tasks.rados.rados.0.smithi093.stdout:3943: finishing write tid 3 to smithi093153278-346 2023-08-27T20:55:32.204 INFO:tasks.rados.rados.0.smithi093.stdout:3943: finishing write tid 4 to smithi093153278-346 2023-08-27T20:55:32.204 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 346 v 566 (ObjNum 2422 snap 0 seq_num 2422) dirty exists 2023-08-27T20:55:32.204 INFO:tasks.rados.rados.0.smithi093.stdout:3943: left oid 346 (ObjNum 2422 snap 0 seq_num 2422) 2023-08-27T20:55:32.204 INFO:tasks.rados.rados.0.smithi093.stdout:3946: expect (ObjNum 1197 snap 0 seq_num 1197) 2023-08-27T20:55:32.308 INFO:tasks.rados.rados.0.smithi093.stdout:3945: expect (ObjNum 1654 snap 0 seq_num 1654) 2023-08-27T20:55:32.616 INFO:tasks.rados.rados.0.smithi093.stdout:3944: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:55:32.827 INFO:tasks.rados.rados.0.smithi093.stdout:3949: finishing write tid 1 to smithi093153278-315 2023-08-27T20:55:32.827 INFO:tasks.rados.rados.0.smithi093.stdout:3949: finishing write tid 2 to smithi093153278-315 2023-08-27T20:55:32.827 INFO:tasks.rados.rados.0.smithi093.stdout:3949: finishing write tid 3 to smithi093153278-315 2023-08-27T20:55:32.827 INFO:tasks.rados.rados.0.smithi093.stdout:3949: finishing write tid 4 to smithi093153278-315 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 315 v 571 (ObjNum 2424 snap 0 seq_num 2424) dirty exists 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:3949: left oid 315 (ObjNum 2424 snap 0 seq_num 2424) 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:3943: done (5 left) 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:3944: done (4 left) 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:3945: done (3 left) 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:3946: done (2 left) 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:3949: done (1 left) 2023-08-27T20:55:32.828 INFO:tasks.rados.rados.0.smithi093.stdout:3950: done (0 left) 2023-08-27T20:55:32.829 INFO:tasks.rados.rados.0.smithi093.stdout:3951: read oid 15 snap -1 2023-08-27T20:55:32.829 INFO:tasks.rados.rados.0.smithi093.stdout:3951: expect (ObjNum 1576 snap 0 seq_num 1576) 2023-08-27T20:55:32.829 INFO:tasks.rados.rados.0.smithi093.stdout:3952: write oid 64 current snap is 0 2023-08-27T20:55:32.829 INFO:tasks.rados.rados.0.smithi093.stdout:3952: seq_num 2426 ranges {773782=470681,1996610=698844} 2023-08-27T20:55:32.832 INFO:tasks.rados.rados.0.smithi093.stdout:3952: writing smithi093153278-64 from 773782 to 1244463 tid 1 2023-08-27T20:55:32.840 INFO:tasks.rados.rados.0.smithi093.stdout:3952: writing smithi093153278-64 from 1996610 to 2695454 tid 2 2023-08-27T20:55:32.840 INFO:tasks.rados.rados.0.smithi093.stdout:3953: write oid 466 current snap is 0 2023-08-27T20:55:32.841 INFO:tasks.rados.rados.0.smithi093.stdout:3951: expect (ObjNum 1576 snap 0 seq_num 1576) 2023-08-27T20:55:33.017 INFO:tasks.rados.rados.0.smithi093.stdout:3952: finishing write tid 1 to smithi093153278-64 2023-08-27T20:55:33.017 INFO:tasks.rados.rados.0.smithi093.stdout:3952: finishing write tid 2 to smithi093153278-64 2023-08-27T20:55:33.017 INFO:tasks.rados.rados.0.smithi093.stdout:3952: finishing write tid 3 to smithi093153278-64 2023-08-27T20:55:33.017 INFO:tasks.rados.rados.0.smithi093.stdout:3952: finishing write tid 4 to smithi093153278-64 2023-08-27T20:55:33.017 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 64 v 314 (ObjNum 2426 snap 0 seq_num 2426) dirty exists 2023-08-27T20:55:33.017 INFO:tasks.rados.rados.0.smithi093.stdout:3952: left oid 64 (ObjNum 2426 snap 0 seq_num 2426) 2023-08-27T20:55:33.017 INFO:tasks.rados.rados.0.smithi093.stdout:3953: seq_num 2427 ranges {790589=545930,1938739=558855} 2023-08-27T20:55:33.023 INFO:tasks.rados.rados.0.smithi093.stdout:3953: writing smithi093153278-466 from 790589 to 1336519 tid 1 2023-08-27T20:55:33.031 INFO:tasks.rados.rados.0.smithi093.stdout:3953: writing smithi093153278-466 from 1938739 to 2497594 tid 2 2023-08-27T20:55:33.031 INFO:tasks.rados.rados.0.smithi093.stdout:3951: done (2 left) 2023-08-27T20:55:33.032 INFO:tasks.rados.rados.0.smithi093.stdout:3952: done (1 left) 2023-08-27T20:55:33.032 INFO:tasks.rados.rados.0.smithi093.stdout:3954: write oid 459 current snap is 0 2023-08-27T20:55:33.032 INFO:tasks.rados.rados.0.smithi093.stdout:3953: finishing write tid 1 to smithi093153278-466 2023-08-27T20:55:33.032 INFO:tasks.rados.rados.0.smithi093.stdout:3954: seq_num 2428 ranges {407396=621179,1597221=702513} 2023-08-27T20:55:33.036 INFO:tasks.rados.rados.0.smithi093.stdout:3954: writing smithi093153278-459 from 407396 to 1028575 tid 1 2023-08-27T20:55:33.043 INFO:tasks.rados.rados.0.smithi093.stdout:3954: writing smithi093153278-459 from 1597221 to 2299734 tid 2 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:3955: write oid 352 current snap is 0 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:3953: finishing write tid 2 to smithi093153278-466 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:3953: finishing write tid 3 to smithi093153278-466 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:3953: finishing write tid 4 to smithi093153278-466 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 466 v 569 (ObjNum 2427 snap 0 seq_num 2427) dirty exists 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:3953: left oid 466 (ObjNum 2427 snap 0 seq_num 2427) 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:3954: finishing write tid 1 to smithi093153278-459 2023-08-27T20:55:33.044 INFO:tasks.rados.rados.0.smithi093.stdout:3955: seq_num 2429 ranges {424203=696428,1655703=562524} 2023-08-27T20:55:33.049 INFO:tasks.rados.rados.0.smithi093.stdout:3955: writing smithi093153278-352 from 424203 to 1120631 tid 1 2023-08-27T20:55:33.056 INFO:tasks.rados.rados.0.smithi093.stdout:3955: writing smithi093153278-352 from 1655703 to 2218227 tid 2 2023-08-27T20:55:33.056 INFO:tasks.rados.rados.0.smithi093.stdout:3953: done (2 left) 2023-08-27T20:55:33.056 INFO:tasks.rados.rados.0.smithi093.stdout:3956: write oid 163 current snap is 0 2023-08-27T20:55:33.056 INFO:tasks.rados.rados.0.smithi093.stdout:3954: finishing write tid 2 to smithi093153278-459 2023-08-27T20:55:33.056 INFO:tasks.rados.rados.0.smithi093.stdout:3956: seq_num 2430 ranges {441010=771677,1714185=706182} 2023-08-27T20:55:33.062 INFO:tasks.rados.rados.0.smithi093.stdout:3956: writing smithi093153278-163 from 441010 to 1212687 tid 1 2023-08-27T20:55:33.070 INFO:tasks.rados.rados.0.smithi093.stdout:3956: writing smithi093153278-163 from 1714185 to 2420367 tid 2 2023-08-27T20:55:33.070 INFO:tasks.rados.rados.0.smithi093.stdout:3957: delete oid 295 current snap is 0 2023-08-27T20:55:33.071 INFO:tasks.rados.rados.0.smithi093.stdout:3954: finishing write tid 3 to smithi093153278-459 2023-08-27T20:55:33.071 INFO:tasks.rados.rados.0.smithi093.stdout:3954: finishing write tid 4 to smithi093153278-459 2023-08-27T20:55:33.071 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 459 v 421 (ObjNum 2428 snap 0 seq_num 2428) dirty exists 2023-08-27T20:55:33.071 INFO:tasks.rados.rados.0.smithi093.stdout:3954: left oid 459 (ObjNum 2428 snap 0 seq_num 2428) 2023-08-27T20:55:33.071 INFO:tasks.rados.rados.0.smithi093.stdout:3955: finishing write tid 1 to smithi093153278-352 2023-08-27T20:55:33.072 INFO:tasks.rados.rados.0.smithi093.stdout:3955: finishing write tid 2 to smithi093153278-352 2023-08-27T20:55:33.072 INFO:tasks.rados.rados.0.smithi093.stdout:3955: finishing write tid 3 to smithi093153278-352 2023-08-27T20:55:33.072 INFO:tasks.rados.rados.0.smithi093.stdout:3955: finishing write tid 4 to smithi093153278-352 2023-08-27T20:55:33.072 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 352 v 519 (ObjNum 2429 snap 0 seq_num 2429) dirty exists 2023-08-27T20:55:33.072 INFO:tasks.rados.rados.0.smithi093.stdout:3955: left oid 352 (ObjNum 2429 snap 0 seq_num 2429) 2023-08-27T20:55:33.073 INFO:tasks.rados.rados.0.smithi093.stdout:3956: finishing write tid 1 to smithi093153278-163 2023-08-27T20:55:33.073 INFO:tasks.rados.rados.0.smithi093.stdout:3954: done (3 left) 2023-08-27T20:55:33.074 INFO:tasks.rados.rados.0.smithi093.stdout:3955: done (2 left) 2023-08-27T20:55:33.074 INFO:tasks.rados.rados.0.smithi093.stdout:3957: done (1 left) 2023-08-27T20:55:33.074 INFO:tasks.rados.rados.0.smithi093.stdout:3958: write oid 476 current snap is 0 2023-08-27T20:55:33.074 INFO:tasks.rados.rados.0.smithi093.stdout:3958: seq_num 2432 ranges {474624=522175,1714796=709851} 2023-08-27T20:55:33.077 INFO:tasks.rados.rados.0.smithi093.stdout:3958: writing smithi093153278-476 from 474624 to 996799 tid 1 2023-08-27T20:55:33.086 INFO:tasks.rados.rados.0.smithi093.stdout:3958: writing smithi093153278-476 from 1714796 to 2424647 tid 2 2023-08-27T20:55:33.086 INFO:tasks.rados.rados.0.smithi093.stdout:3959: read oid 93 snap -1 2023-08-27T20:55:33.086 INFO:tasks.rados.rados.0.smithi093.stdout:3959: expect (ObjNum 2355 snap 0 seq_num 2355) 2023-08-27T20:55:33.086 INFO:tasks.rados.rados.0.smithi093.stdout:3960: read oid 274 snap -1 2023-08-27T20:55:33.086 INFO:tasks.rados.rados.0.smithi093.stdout:3960: expect (ObjNum 1849 snap 0 seq_num 1849) 2023-08-27T20:55:33.087 INFO:tasks.rados.rados.0.smithi093.stdout:3961: read oid 172 snap -1 2023-08-27T20:55:33.087 INFO:tasks.rados.rados.0.smithi093.stdout:3961: expect deleted 2023-08-27T20:55:33.087 INFO:tasks.rados.rados.0.smithi093.stdout:3962: read oid 62 snap -1 2023-08-27T20:55:33.087 INFO:tasks.rados.rados.0.smithi093.stdout:3962: expect (ObjNum 2402 snap 0 seq_num 2402) 2023-08-27T20:55:33.087 INFO:tasks.rados.rados.0.smithi093.stdout:3963: delete oid 68 current snap is 0 2023-08-27T20:55:33.088 INFO:tasks.rados.rados.0.smithi093.stdout:3956: finishing write tid 2 to smithi093153278-163 2023-08-27T20:55:33.088 INFO:tasks.rados.rados.0.smithi093.stdout:3956: finishing write tid 3 to smithi093153278-163 2023-08-27T20:55:33.088 INFO:tasks.rados.rados.0.smithi093.stdout:3956: finishing write tid 4 to smithi093153278-163 2023-08-27T20:55:33.088 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 163 v 485 (ObjNum 2430 snap 0 seq_num 2430) dirty exists 2023-08-27T20:55:33.088 INFO:tasks.rados.rados.0.smithi093.stdout:3956: left oid 163 (ObjNum 2430 snap 0 seq_num 2430) 2023-08-27T20:55:33.088 INFO:tasks.rados.rados.0.smithi093.stdout:3958: finishing write tid 1 to smithi093153278-476 2023-08-27T20:55:33.089 INFO:tasks.rados.rados.0.smithi093.stdout:3962: expect (ObjNum 2402 snap 0 seq_num 2402) 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3958: finishing write tid 2 to smithi093153278-476 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3958: finishing write tid 3 to smithi093153278-476 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3956: done (6 left) 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3961: done (5 left) 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3962: done (4 left) 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3963: done (3 left) 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3964: write oid 275 current snap is 0 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:3958: finishing write tid 4 to smithi093153278-476 2023-08-27T20:55:33.208 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 476 v 560 (ObjNum 2432 snap 0 seq_num 2432) dirty exists 2023-08-27T20:55:33.209 INFO:tasks.rados.rados.0.smithi093.stdout:3958: left oid 476 (ObjNum 2432 snap 0 seq_num 2432) 2023-08-27T20:55:33.209 INFO:tasks.rados.rados.0.smithi093.stdout:3960: expect (ObjNum 1849 snap 0 seq_num 1849) 2023-08-27T20:55:33.370 INFO:tasks.rados.rados.0.smithi093.stdout:3959: expect (ObjNum 2355 snap 0 seq_num 2355) 2023-08-27T20:55:33.579 INFO:tasks.rados.rados.0.smithi093.stdout:3964: seq_num 2434 ranges {508238=789026,1948113=713520} 2023-08-27T20:55:33.586 INFO:tasks.rados.rados.0.smithi093.stdout:3964: writing smithi093153278-275 from 508238 to 1297264 tid 1 2023-08-27T20:55:33.594 INFO:tasks.rados.rados.0.smithi093.stdout:3964: writing smithi093153278-275 from 1948113 to 2661633 tid 2 2023-08-27T20:55:33.594 INFO:tasks.rados.rados.0.smithi093.stdout:3958: done (3 left) 2023-08-27T20:55:33.594 INFO:tasks.rados.rados.0.smithi093.stdout:3959: done (2 left) 2023-08-27T20:55:33.594 INFO:tasks.rados.rados.0.smithi093.stdout:3960: done (1 left) 2023-08-27T20:55:33.594 INFO:tasks.rados.rados.0.smithi093.stdout:3965: read oid 10 snap -1 2023-08-27T20:55:33.594 INFO:tasks.rados.rados.0.smithi093.stdout:3965: expect (ObjNum 2259 snap 0 seq_num 2259) 2023-08-27T20:55:33.595 INFO:tasks.rados.rados.0.smithi093.stdout:3966: write oid 494 current snap is 0 2023-08-27T20:55:33.595 INFO:tasks.rados.rados.0.smithi093.stdout:3964: finishing write tid 1 to smithi093153278-275 2023-08-27T20:55:33.595 INFO:tasks.rados.rados.0.smithi093.stdout:3966: seq_num 2435 ranges {525045=464275,1490242=457178,2501811=423234} 2023-08-27T20:55:33.599 INFO:tasks.rados.rados.0.smithi093.stdout:3966: writing smithi093153278-494 from 525045 to 989320 tid 1 2023-08-27T20:55:33.605 INFO:tasks.rados.rados.0.smithi093.stdout:3966: writing smithi093153278-494 from 1490242 to 1947420 tid 2 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:3966: writing smithi093153278-494 from 2501811 to 2925045 tid 3 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:3964: finishing write tid 2 to smithi093153278-275 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:3967: read oid 162 snap -1 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:3967: expect (ObjNum 2262 snap 0 seq_num 2262) 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:3968: write oid 254 current snap is 0 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:3964: finishing write tid 3 to smithi093153278-275 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:3964: finishing write tid 4 to smithi093153278-275 2023-08-27T20:55:33.611 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 275 v 488 (ObjNum 2434 snap 0 seq_num 2434) dirty exists 2023-08-27T20:55:33.612 INFO:tasks.rados.rados.0.smithi093.stdout:3964: left oid 275 (ObjNum 2434 snap 0 seq_num 2434) 2023-08-27T20:55:33.612 INFO:tasks.rados.rados.0.smithi093.stdout:3965: expect (ObjNum 2259 snap 0 seq_num 2259) 2023-08-27T20:55:33.798 INFO:tasks.rados.rados.0.smithi093.stdout:3966: finishing write tid 1 to smithi093153278-494 2023-08-27T20:55:33.799 INFO:tasks.rados.rados.0.smithi093.stdout:3966: finishing write tid 2 to smithi093153278-494 2023-08-27T20:55:33.799 INFO:tasks.rados.rados.0.smithi093.stdout:3968: seq_num 2436 ranges {541852=539524,1548724=717189,2929234=12618} 2023-08-27T20:55:33.804 INFO:tasks.rados.rados.0.smithi093.stdout:3968: writing smithi093153278-254 from 541852 to 1081376 tid 1 2023-08-27T20:55:33.811 INFO:tasks.rados.rados.0.smithi093.stdout:3968: writing smithi093153278-254 from 1548724 to 2265913 tid 2 2023-08-27T20:55:33.815 INFO:tasks.rados.rados.0.smithi093.stdout:3968: writing smithi093153278-254 from 2929234 to 2941852 tid 3 2023-08-27T20:55:33.815 INFO:tasks.rados.rados.0.smithi093.stdout:3964: done (4 left) 2023-08-27T20:55:33.815 INFO:tasks.rados.rados.0.smithi093.stdout:3965: done (3 left) 2023-08-27T20:55:33.815 INFO:tasks.rados.rados.0.smithi093.stdout:3969: write oid 451 current snap is 0 2023-08-27T20:55:33.815 INFO:tasks.rados.rados.0.smithi093.stdout:3966: finishing write tid 3 to smithi093153278-494 2023-08-27T20:55:33.815 INFO:tasks.rados.rados.0.smithi093.stdout:3966: finishing write tid 4 to smithi093153278-494 2023-08-27T20:55:33.816 INFO:tasks.rados.rados.0.smithi093.stdout:3966: finishing write tid 5 to smithi093153278-494 2023-08-27T20:55:33.816 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 494 v 396 (ObjNum 2435 snap 0 seq_num 2435) dirty exists 2023-08-27T20:55:33.816 INFO:tasks.rados.rados.0.smithi093.stdout:3966: left oid 494 (ObjNum 2435 snap 0 seq_num 2435) 2023-08-27T20:55:33.816 INFO:tasks.rados.rados.0.smithi093.stdout:3967: expect (ObjNum 2262 snap 0 seq_num 2262) 2023-08-27T20:55:33.950 INFO:tasks.rados.rados.0.smithi093.stdout:3968: finishing write tid 1 to smithi093153278-254 2023-08-27T20:55:33.950 INFO:tasks.rados.rados.0.smithi093.stdout:3968: finishing write tid 2 to smithi093153278-254 2023-08-27T20:55:33.950 INFO:tasks.rados.rados.0.smithi093.stdout:3968: finishing write tid 3 to smithi093153278-254 2023-08-27T20:55:33.950 INFO:tasks.rados.rados.0.smithi093.stdout:3968: finishing write tid 4 to smithi093153278-254 2023-08-27T20:55:33.950 INFO:tasks.rados.rados.0.smithi093.stdout:3968: finishing write tid 5 to smithi093153278-254 2023-08-27T20:55:33.951 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 254 v 492 (ObjNum 2436 snap 0 seq_num 2436) dirty exists 2023-08-27T20:55:33.951 INFO:tasks.rados.rados.0.smithi093.stdout:3968: left oid 254 (ObjNum 2436 snap 0 seq_num 2436) 2023-08-27T20:55:33.951 INFO:tasks.rados.rados.0.smithi093.stdout:3969: seq_num 2437 ranges {558659=614773,1607206=460847,2556657=402002} 2023-08-27T20:55:33.956 INFO:tasks.rados.rados.0.smithi093.stdout:3969: writing smithi093153278-451 from 558659 to 1173432 tid 1 2023-08-27T20:55:33.961 INFO:tasks.rados.rados.0.smithi093.stdout:3969: writing smithi093153278-451 from 1607206 to 2068053 tid 2 2023-08-27T20:55:33.967 INFO:tasks.rados.rados.0.smithi093.stdout:3969: writing smithi093153278-451 from 2556657 to 2958659 tid 3 2023-08-27T20:55:33.968 INFO:tasks.rados.rados.0.smithi093.stdout:3966: done (3 left) 2023-08-27T20:55:33.968 INFO:tasks.rados.rados.0.smithi093.stdout:3967: done (2 left) 2023-08-27T20:55:33.968 INFO:tasks.rados.rados.0.smithi093.stdout:3968: done (1 left) 2023-08-27T20:55:33.968 INFO:tasks.rados.rados.0.smithi093.stdout:3970: write oid 153 current snap is 0 2023-08-27T20:55:33.968 INFO:tasks.rados.rados.0.smithi093.stdout:3969: finishing write tid 1 to smithi093153278-451 2023-08-27T20:55:33.968 INFO:tasks.rados.rados.0.smithi093.stdout:3969: finishing write tid 2 to smithi093153278-451 2023-08-27T20:55:33.968 INFO:tasks.rados.rados.0.smithi093.stdout:3970: seq_num 2438 ranges {575466=690022,1665688=720858} 2023-08-27T20:55:33.973 INFO:tasks.rados.rados.0.smithi093.stdout:3970: writing smithi093153278-153 from 575466 to 1265488 tid 1 2023-08-27T20:55:33.980 INFO:tasks.rados.rados.0.smithi093.stdout:3970: writing smithi093153278-153 from 1665688 to 2386546 tid 2 2023-08-27T20:55:33.980 INFO:tasks.rados.rados.0.smithi093.stdout:3971: read oid 114 snap -1 2023-08-27T20:55:33.980 INFO:tasks.rados.rados.0.smithi093.stdout:3971: expect (ObjNum 644 snap 0 seq_num 644) 2023-08-27T20:55:33.980 INFO:tasks.rados.rados.0.smithi093.stdout:3972: read oid 247 snap -1 2023-08-27T20:55:33.980 INFO:tasks.rados.rados.0.smithi093.stdout:3972: expect deleted 2023-08-27T20:55:33.980 INFO:tasks.rados.rados.0.smithi093.stdout:3973: write oid 105 current snap is 0 2023-08-27T20:55:33.981 INFO:tasks.rados.rados.0.smithi093.stdout:3969: finishing write tid 3 to smithi093153278-451 2023-08-27T20:55:33.981 INFO:tasks.rados.rados.0.smithi093.stdout:3973: seq_num 2439 ranges {592273=765271,2007817=464516,2895150=97123} 2023-08-27T20:55:33.987 INFO:tasks.rados.rados.0.smithi093.stdout:3973: writing smithi093153278-105 from 592273 to 1357544 tid 1 2023-08-27T20:55:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:33 smithi093 ceph-mon[111335]: pgmap v60: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 81 op/s 2023-08-27T20:55:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:33 smithi093 ceph-mon[118902]: pgmap v60: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 81 op/s 2023-08-27T20:55:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:33.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:33.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:33.993 INFO:tasks.rados.rados.0.smithi093.stdout:3973: writing smithi093153278-105 from 2007817 to 2472333 tid 2 2023-08-27T20:55:33.996 INFO:tasks.rados.rados.0.smithi093.stdout:3973: writing smithi093153278-105 from 2895150 to 2992273 tid 3 2023-08-27T20:55:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:3974: write oid 439 current snap is 0 2023-08-27T20:55:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:3969: finishing write tid 4 to smithi093153278-451 2023-08-27T20:55:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:3969: finishing write tid 5 to smithi093153278-451 2023-08-27T20:55:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 451 v 425 (ObjNum 2437 snap 0 seq_num 2437) dirty exists 2023-08-27T20:55:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:3969: left oid 451 (ObjNum 2437 snap 0 seq_num 2437) 2023-08-27T20:55:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:3970: finishing write tid 1 to smithi093153278-153 2023-08-27T20:55:33.997 INFO:tasks.rados.rados.0.smithi093.stdout:3974: seq_num 2440 ranges {609080=440520,1666299=724527,2922573=86507} 2023-08-27T20:55:34.002 INFO:tasks.rados.rados.0.smithi093.stdout:3974: writing smithi093153278-439 from 609080 to 1049600 tid 1 2023-08-27T20:55:34.010 INFO:tasks.rados.rados.0.smithi093.stdout:3974: writing smithi093153278-439 from 1666299 to 2390826 tid 2 2023-08-27T20:55:34.013 INFO:tasks.rados.rados.0.smithi093.stdout:3974: writing smithi093153278-439 from 2922573 to 3009080 tid 3 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3969: done (5 left) 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3975: read oid 102 snap -1 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3975: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3976: delete oid 227 current snap is 0 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3970: finishing write tid 2 to smithi093153278-153 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3970: finishing write tid 3 to smithi093153278-153 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3970: finishing write tid 4 to smithi093153278-153 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 153 v 574 (ObjNum 2438 snap 0 seq_num 2438) dirty exists 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3970: left oid 153 (ObjNum 2438 snap 0 seq_num 2438) 2023-08-27T20:55:34.014 INFO:tasks.rados.rados.0.smithi093.stdout:3971: expect (ObjNum 644 snap 0 seq_num 644) 2023-08-27T20:55:34.143 INFO:tasks.rados.rados.0.smithi093.stdout:3973: finishing write tid 1 to smithi093153278-105 2023-08-27T20:55:34.143 INFO:tasks.rados.rados.0.smithi093.stdout:3970: done (6 left) 2023-08-27T20:55:34.143 INFO:tasks.rados.rados.0.smithi093.stdout:3971: done (5 left) 2023-08-27T20:55:34.143 INFO:tasks.rados.rados.0.smithi093.stdout:3972: done (4 left) 2023-08-27T20:55:34.143 INFO:tasks.rados.rados.0.smithi093.stdout:3976: done (3 left) 2023-08-27T20:55:34.143 INFO:tasks.rados.rados.0.smithi093.stdout:3977: read oid 44 snap -1 2023-08-27T20:55:34.143 INFO:tasks.rados.rados.0.smithi093.stdout:3977: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3978: read oid 319 snap -1 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3978: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3979: read oid 28 snap -1 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3979: expect (ObjNum 1962 snap 0 seq_num 1962) 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3980: read oid 450 snap -1 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3980: expect (ObjNum 2384 snap 0 seq_num 2384) 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3981: read oid 345 snap -1 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3981: expect (ObjNum 1529 snap 0 seq_num 1529) 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3982: write oid 432 current snap is 0 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3973: finishing write tid 2 to smithi093153278-105 2023-08-27T20:55:34.144 INFO:tasks.rados.rados.0.smithi093.stdout:3982: seq_num 2442 ranges {642694=707371,1899616=728196} 2023-08-27T20:55:34.150 INFO:tasks.rados.rados.0.smithi093.stdout:3982: writing smithi093153278-432 from 642694 to 1350065 tid 1 2023-08-27T20:55:34.157 INFO:tasks.rados.rados.0.smithi093.stdout:3982: writing smithi093153278-432 from 1899616 to 2627812 tid 2 2023-08-27T20:55:34.158 INFO:tasks.rados.rados.0.smithi093.stdout:3983: write oid 221 current snap is 0 2023-08-27T20:55:34.158 INFO:tasks.rados.rados.0.smithi093.stdout:3973: finishing write tid 3 to smithi093153278-105 2023-08-27T20:55:34.158 INFO:tasks.rados.rados.0.smithi093.stdout:3983: seq_num 2443 ranges {659501=782620,2241745=471854} 2023-08-27T20:55:34.165 INFO:tasks.rados.rados.0.smithi093.stdout:3983: writing smithi093153278-221 from 659501 to 1442121 tid 1 2023-08-27T20:55:34.173 INFO:tasks.rados.rados.0.smithi093.stdout:3983: writing smithi093153278-221 from 2241745 to 2713599 tid 2 2023-08-27T20:55:34.173 INFO:tasks.rados.rados.0.smithi093.stdout:3984: read oid 418 snap -1 2023-08-27T20:55:34.173 INFO:tasks.rados.rados.0.smithi093.stdout:3984: expect (ObjNum 2233 snap 0 seq_num 2233) 2023-08-27T20:55:34.173 INFO:tasks.rados.rados.0.smithi093.stdout:3985: write oid 220 current snap is 0 2023-08-27T20:55:34.174 INFO:tasks.rados.rados.0.smithi093.stdout:3973: finishing write tid 4 to smithi093153278-105 2023-08-27T20:55:34.174 INFO:tasks.rados.rados.0.smithi093.stdout:3985: seq_num 2444 ranges {676308=457869,1900227=615512,3032265=44043} 2023-08-27T20:55:34.179 INFO:tasks.rados.rados.0.smithi093.stdout:3985: writing smithi093153278-220 from 676308 to 1134177 tid 1 2023-08-27T20:55:34.187 INFO:tasks.rados.rados.0.smithi093.stdout:3985: writing smithi093153278-220 from 1900227 to 2515739 tid 2 2023-08-27T20:55:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:33 smithi136 ceph-mon[113531]: pgmap v60: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 81 op/s 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:3985: writing smithi093153278-220 from 3032265 to 3076308 tid 3 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:3986: write oid 330 current snap is 0 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:3973: finishing write tid 5 to smithi093153278-105 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 105 v 496 (ObjNum 2439 snap 0 seq_num 2439) dirty exists 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:3973: left oid 105 (ObjNum 2439 snap 0 seq_num 2439) 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:3974: finishing write tid 1 to smithi093153278-439 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:3974: finishing write tid 2 to smithi093153278-439 2023-08-27T20:55:34.191 INFO:tasks.rados.rados.0.smithi093.stdout:3974: finishing write tid 3 to smithi093153278-439 2023-08-27T20:55:34.192 INFO:tasks.rados.rados.0.smithi093.stdout:3986: seq_num 2445 ranges {693115=533118,1958709=475523,3059688=33427} 2023-08-27T20:55:34.197 INFO:tasks.rados.rados.0.smithi093.stdout:3986: writing smithi093153278-330 from 693115 to 1226233 tid 1 2023-08-27T20:55:34.203 INFO:tasks.rados.rados.0.smithi093.stdout:3986: writing smithi093153278-330 from 1958709 to 2434232 tid 2 2023-08-27T20:55:34.207 INFO:tasks.rados.rados.0.smithi093.stdout:3986: writing smithi093153278-330 from 3059688 to 3093115 tid 3 2023-08-27T20:55:34.207 INFO:tasks.rados.rados.0.smithi093.stdout:3973: done (12 left) 2023-08-27T20:55:34.208 INFO:tasks.rados.rados.0.smithi093.stdout:3987: delete oid 489 current snap is 0 2023-08-27T20:55:34.208 INFO:tasks.rados.rados.0.smithi093.stdout:3974: finishing write tid 4 to smithi093153278-439 2023-08-27T20:55:34.208 INFO:tasks.rados.rados.0.smithi093.stdout:3974: finishing write tid 5 to smithi093153278-439 2023-08-27T20:55:34.208 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 439 v 472 (ObjNum 2440 snap 0 seq_num 2440) dirty exists 2023-08-27T20:55:34.208 INFO:tasks.rados.rados.0.smithi093.stdout:3974: left oid 439 (ObjNum 2440 snap 0 seq_num 2440) 2023-08-27T20:55:34.208 INFO:tasks.rados.rados.0.smithi093.stdout:3975: expect (ObjNum 1306 snap 0 seq_num 1306) 2023-08-27T20:55:34.415 INFO:tasks.rados.rados.0.smithi093.stdout:3974: done (12 left) 2023-08-27T20:55:34.416 INFO:tasks.rados.rados.0.smithi093.stdout:3975: done (11 left) 2023-08-27T20:55:34.416 INFO:tasks.rados.rados.0.smithi093.stdout:3987: done (10 left) 2023-08-27T20:55:34.416 INFO:tasks.rados.rados.0.smithi093.stdout:3988: read oid 392 snap -1 2023-08-27T20:55:34.416 INFO:tasks.rados.rados.0.smithi093.stdout:3988: expect (ObjNum 2107 snap 0 seq_num 2107) 2023-08-27T20:55:34.416 INFO:tasks.rados.rados.0.smithi093.stdout:3989: delete oid 15 current snap is 0 2023-08-27T20:55:34.416 INFO:tasks.rados.rados.0.smithi093.stdout:3977: expect (ObjNum 623 snap 0 seq_num 623) 2023-08-27T20:55:34.524 INFO:tasks.rados.rados.0.smithi093.stdout:3980: expect (ObjNum 2384 snap 0 seq_num 2384) 2023-08-27T20:55:34.586 INFO:tasks.rados.rados.0.smithi093.stdout:3982: finishing write tid 1 to smithi093153278-432 2023-08-27T20:55:34.586 INFO:tasks.rados.rados.0.smithi093.stdout:3982: finishing write tid 2 to smithi093153278-432 2023-08-27T20:55:34.586 INFO:tasks.rados.rados.0.smithi093.stdout:3982: finishing write tid 3 to smithi093153278-432 2023-08-27T20:55:34.586 INFO:tasks.rados.rados.0.smithi093.stdout:3982: finishing write tid 4 to smithi093153278-432 2023-08-27T20:55:34.586 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 432 v 399 (ObjNum 2442 snap 0 seq_num 2442) dirty exists 2023-08-27T20:55:34.586 INFO:tasks.rados.rados.0.smithi093.stdout:3982: left oid 432 (ObjNum 2442 snap 0 seq_num 2442) 2023-08-27T20:55:34.587 INFO:tasks.rados.rados.0.smithi093.stdout:3979: expect (ObjNum 1962 snap 0 seq_num 1962) 2023-08-27T20:55:34.746 INFO:tasks.rados.rados.0.smithi093.stdout:3978: expect (ObjNum 799 snap 0 seq_num 799) 2023-08-27T20:55:34.897 INFO:tasks.rados.rados.0.smithi093.stdout:3981: expect (ObjNum 1529 snap 0 seq_num 1529) 2023-08-27T20:55:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/413587065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/413587065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:35.075 INFO:tasks.rados.rados.0.smithi093.stdout:3983: finishing write tid 1 to smithi093153278-221 2023-08-27T20:55:35.075 INFO:tasks.rados.rados.0.smithi093.stdout:3983: finishing write tid 2 to smithi093153278-221 2023-08-27T20:55:35.076 INFO:tasks.rados.rados.0.smithi093.stdout:3983: finishing write tid 3 to smithi093153278-221 2023-08-27T20:55:35.076 INFO:tasks.rados.rados.0.smithi093.stdout:3983: finishing write tid 4 to smithi093153278-221 2023-08-27T20:55:35.076 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 221 v 624 (ObjNum 2443 snap 0 seq_num 2443) dirty exists 2023-08-27T20:55:35.076 INFO:tasks.rados.rados.0.smithi093.stdout:3983: left oid 221 (ObjNum 2443 snap 0 seq_num 2443) 2023-08-27T20:55:35.076 INFO:tasks.rados.rados.0.smithi093.stdout:3984: expect (ObjNum 2233 snap 0 seq_num 2233) 2023-08-27T20:55:35.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/413587065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:35.222 INFO:tasks.rados.rados.0.smithi093.stdout:3985: finishing write tid 1 to smithi093153278-220 2023-08-27T20:55:35.222 INFO:tasks.rados.rados.0.smithi093.stdout:3985: finishing write tid 2 to smithi093153278-220 2023-08-27T20:55:35.222 INFO:tasks.rados.rados.0.smithi093.stdout:3985: finishing write tid 3 to smithi093153278-220 2023-08-27T20:55:35.222 INFO:tasks.rados.rados.0.smithi093.stdout:3977: done (11 left) 2023-08-27T20:55:35.222 INFO:tasks.rados.rados.0.smithi093.stdout:3978: done (10 left) 2023-08-27T20:55:35.222 INFO:tasks.rados.rados.0.smithi093.stdout:3979: done (9 left) 2023-08-27T20:55:35.222 INFO:tasks.rados.rados.0.smithi093.stdout:3980: done (8 left) 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3981: done (7 left) 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3982: done (6 left) 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3983: done (5 left) 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3984: done (4 left) 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3989: done (3 left) 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3990: write oid 423 current snap is 0 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3985: finishing write tid 4 to smithi093153278-220 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3985: finishing write tid 5 to smithi093153278-220 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 220 v 573 (ObjNum 2444 snap 0 seq_num 2444) dirty exists 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3985: left oid 220 (ObjNum 2444 snap 0 seq_num 2444) 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3986: finishing write tid 1 to smithi093153278-330 2023-08-27T20:55:35.223 INFO:tasks.rados.rados.0.smithi093.stdout:3986: finishing write tid 2 to smithi093153278-330 2023-08-27T20:55:35.224 INFO:tasks.rados.rados.0.smithi093.stdout:3986: finishing write tid 3 to smithi093153278-330 2023-08-27T20:55:35.224 INFO:tasks.rados.rados.0.smithi093.stdout:3986: finishing write tid 4 to smithi093153278-330 2023-08-27T20:55:35.224 INFO:tasks.rados.rados.0.smithi093.stdout:3986: finishing write tid 5 to smithi093153278-330 2023-08-27T20:55:35.224 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 330 v 435 (ObjNum 2445 snap 0 seq_num 2445) dirty exists 2023-08-27T20:55:35.224 INFO:tasks.rados.rados.0.smithi093.stdout:3986: left oid 330 (ObjNum 2445 snap 0 seq_num 2445) 2023-08-27T20:55:35.224 INFO:tasks.rados.rados.0.smithi093.stdout:3990: seq_num 2448 ranges {743536=475218,1734155=622850,3141957=1579} 2023-08-27T20:55:35.228 INFO:tasks.rados.rados.0.smithi093.stdout:3990: writing smithi093153278-423 from 743536 to 1218754 tid 1 2023-08-27T20:55:35.234 INFO:tasks.rados.rados.0.smithi093.stdout:3990: writing smithi093153278-423 from 1734155 to 2357005 tid 2 2023-08-27T20:55:35.239 INFO:tasks.rados.rados.0.smithi093.stdout:3990: writing smithi093153278-423 from 3141957 to 3143536 tid 3 2023-08-27T20:55:35.239 INFO:tasks.rados.rados.0.smithi093.stdout:3985: done (3 left) 2023-08-27T20:55:35.239 INFO:tasks.rados.rados.0.smithi093.stdout:3986: done (2 left) 2023-08-27T20:55:35.239 INFO:tasks.rados.rados.0.smithi093.stdout:3991: write oid 264 current snap is 0 2023-08-27T20:55:35.239 INFO:tasks.rados.rados.0.smithi093.stdout:3988: expect (ObjNum 2107 snap 0 seq_num 2107) 2023-08-27T20:55:35.416 INFO:tasks.rados.rados.0.smithi093.stdout:3990: finishing write tid 1 to smithi093153278-423 2023-08-27T20:55:35.416 INFO:tasks.rados.rados.0.smithi093.stdout:3990: finishing write tid 2 to smithi093153278-423 2023-08-27T20:55:35.416 INFO:tasks.rados.rados.0.smithi093.stdout:3990: finishing write tid 3 to smithi093153278-423 2023-08-27T20:55:35.416 INFO:tasks.rados.rados.0.smithi093.stdout:3990: finishing write tid 4 to smithi093153278-423 2023-08-27T20:55:35.417 INFO:tasks.rados.rados.0.smithi093.stdout:3990: finishing write tid 5 to smithi093153278-423 2023-08-27T20:55:35.417 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 423 v 318 (ObjNum 2448 snap 0 seq_num 2448) dirty exists 2023-08-27T20:55:35.417 INFO:tasks.rados.rados.0.smithi093.stdout:3990: left oid 423 (ObjNum 2448 snap 0 seq_num 2448) 2023-08-27T20:55:35.417 INFO:tasks.rados.rados.0.smithi093.stdout:3991: seq_num 2449 ranges {760343=550467,1792637=482861,2769380=390963} 2023-08-27T20:55:35.423 INFO:tasks.rados.rados.0.smithi093.stdout:3991: writing smithi093153278-264 from 760343 to 1310810 tid 1 2023-08-27T20:55:35.428 INFO:tasks.rados.rados.0.smithi093.stdout:3991: writing smithi093153278-264 from 1792637 to 2275498 tid 2 2023-08-27T20:55:35.433 INFO:tasks.rados.rados.0.smithi093.stdout:3991: writing smithi093153278-264 from 2769380 to 3160343 tid 3 2023-08-27T20:55:35.433 INFO:tasks.rados.rados.0.smithi093.stdout:3988: done (2 left) 2023-08-27T20:55:35.433 INFO:tasks.rados.rados.0.smithi093.stdout:3990: done (1 left) 2023-08-27T20:55:35.433 INFO:tasks.rados.rados.0.smithi093.stdout:3992: write oid 485 current snap is 0 2023-08-27T20:55:35.433 INFO:tasks.rados.rados.0.smithi093.stdout:3991: finishing write tid 1 to smithi093153278-264 2023-08-27T20:55:35.434 INFO:tasks.rados.rados.0.smithi093.stdout:3991: finishing write tid 2 to smithi093153278-264 2023-08-27T20:55:35.434 INFO:tasks.rados.rados.0.smithi093.stdout:3992: seq_num 2450 ranges {777150=625716,1851119=626519} 2023-08-27T20:55:35.440 INFO:tasks.rados.rados.0.smithi093.stdout:3992: writing smithi093153278-485 from 777150 to 1402866 tid 1 2023-08-27T20:55:35.447 INFO:tasks.rados.rados.0.smithi093.stdout:3992: writing smithi093153278-485 from 1851119 to 2477638 tid 2 2023-08-27T20:55:35.447 INFO:tasks.rados.rados.0.smithi093.stdout:3993: read oid 196 snap -1 2023-08-27T20:55:35.447 INFO:tasks.rados.rados.0.smithi093.stdout:3993: expect deleted 2023-08-27T20:55:35.447 INFO:tasks.rados.rados.0.smithi093.stdout:3994: read oid 84 snap -1 2023-08-27T20:55:35.448 INFO:tasks.rados.rados.0.smithi093.stdout:3994: expect deleted 2023-08-27T20:55:35.448 INFO:tasks.rados.rados.0.smithi093.stdout:3995: write oid 442 current snap is 0 2023-08-27T20:55:35.448 INFO:tasks.rados.rados.0.smithi093.stdout:3991: finishing write tid 3 to smithi093153278-264 2023-08-27T20:55:35.448 INFO:tasks.rados.rados.0.smithi093.stdout:3991: finishing write tid 4 to smithi093153278-264 2023-08-27T20:55:35.448 INFO:tasks.rados.rados.0.smithi093.stdout:3995: seq_num 2451 ranges {793957=700965,2193248=486530,3107873=86084} 2023-08-27T20:55:35.455 INFO:tasks.rados.rados.0.smithi093.stdout:3995: writing smithi093153278-442 from 793957 to 1494922 tid 1 2023-08-27T20:55:35.461 INFO:tasks.rados.rados.0.smithi093.stdout:3995: writing smithi093153278-442 from 2193248 to 2679778 tid 2 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:3995: writing smithi093153278-442 from 3107873 to 3193957 tid 3 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:3996: read oid 336 snap -1 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:3996: expect (ObjNum 1946 snap 0 seq_num 1946) 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:3997: write oid 3 current snap is 0 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:3991: finishing write tid 5 to smithi093153278-264 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 264 v 448 (ObjNum 2449 snap 0 seq_num 2449) dirty exists 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:3991: left oid 264 (ObjNum 2449 snap 0 seq_num 2449) 2023-08-27T20:55:35.465 INFO:tasks.rados.rados.0.smithi093.stdout:3992: finishing write tid 1 to smithi093153278-485 2023-08-27T20:55:35.466 INFO:tasks.rados.rados.0.smithi093.stdout:3997: seq_num 2452 ranges {410764=776214,1851730=630188,3135296=75468} 2023-08-27T20:55:35.471 INFO:tasks.rados.rados.0.smithi093.stdout:3997: writing smithi093153278-3 from 410764 to 1186978 tid 1 2023-08-27T20:55:35.479 INFO:tasks.rados.rados.0.smithi093.stdout:3997: writing smithi093153278-3 from 1851730 to 2481918 tid 2 2023-08-27T20:55:35.482 INFO:tasks.rados.rados.0.smithi093.stdout:3997: writing smithi093153278-3 from 3135296 to 3210764 tid 3 2023-08-27T20:55:35.483 INFO:tasks.rados.rados.0.smithi093.stdout:3991: done (6 left) 2023-08-27T20:55:35.483 INFO:tasks.rados.rados.0.smithi093.stdout:3998: write oid 492 current snap is 0 2023-08-27T20:55:35.483 INFO:tasks.rados.rados.0.smithi093.stdout:3998: seq_num 2453 ranges {427571=451463,1510212=490199,2762719=464852} 2023-08-27T20:55:35.487 INFO:tasks.rados.rados.0.smithi093.stdout:3998: writing smithi093153278-492 from 427571 to 879034 tid 1 2023-08-27T20:55:35.493 INFO:tasks.rados.rados.0.smithi093.stdout:3998: writing smithi093153278-492 from 1510212 to 2000411 tid 2 2023-08-27T20:55:35.501 INFO:tasks.rados.rados.0.smithi093.stdout:3998: writing smithi093153278-492 from 2762719 to 3227571 tid 3 2023-08-27T20:55:35.501 INFO:tasks.rados.rados.0.smithi093.stdout:3999: read oid 182 snap -1 2023-08-27T20:55:35.501 INFO:tasks.rados.rados.0.smithi093.stdout:3999: expect (ObjNum 2014 snap 0 seq_num 2014) 2023-08-27T20:55:35.501 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 8 2023-08-27T20:55:35.501 INFO:tasks.rados.rados.0.smithi093.stdout:3993: done (7 left) 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3994: done (6 left) 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 6 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3992: finishing write tid 2 to smithi093153278-485 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3992: finishing write tid 3 to smithi093153278-485 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3992: finishing write tid 4 to smithi093153278-485 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 485 v 475 (ObjNum 2450 snap 0 seq_num 2450) dirty exists 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3992: left oid 485 (ObjNum 2450 snap 0 seq_num 2450) 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3995: finishing write tid 1 to smithi093153278-442 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3995: finishing write tid 2 to smithi093153278-442 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3992: done (5 left) 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 5 2023-08-27T20:55:35.502 INFO:tasks.rados.rados.0.smithi093.stdout:3995: finishing write tid 3 to smithi093153278-442 2023-08-27T20:55:35.503 INFO:tasks.rados.rados.0.smithi093.stdout:3995: finishing write tid 4 to smithi093153278-442 2023-08-27T20:55:35.503 INFO:tasks.rados.rados.0.smithi093.stdout:3995: finishing write tid 5 to smithi093153278-442 2023-08-27T20:55:35.503 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 442 v 523 (ObjNum 2451 snap 0 seq_num 2451) dirty exists 2023-08-27T20:55:35.503 INFO:tasks.rados.rados.0.smithi093.stdout:3995: left oid 442 (ObjNum 2451 snap 0 seq_num 2451) 2023-08-27T20:55:35.503 INFO:tasks.rados.rados.0.smithi093.stdout:3996: expect (ObjNum 1946 snap 0 seq_num 1946) 2023-08-27T20:55:35.588 INFO:tasks.rados.rados.0.smithi093.stdout:3997: finishing write tid 1 to smithi093153278-3 2023-08-27T20:55:35.588 INFO:tasks.rados.rados.0.smithi093.stdout:3997: finishing write tid 2 to smithi093153278-3 2023-08-27T20:55:35.588 INFO:tasks.rados.rados.0.smithi093.stdout:3997: finishing write tid 3 to smithi093153278-3 2023-08-27T20:55:35.588 INFO:tasks.rados.rados.0.smithi093.stdout:3997: finishing write tid 4 to smithi093153278-3 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3997: finishing write tid 5 to smithi093153278-3 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 3 v 577 (ObjNum 2452 snap 0 seq_num 2452) dirty exists 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3997: left oid 3 (ObjNum 2452 snap 0 seq_num 2452) 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3995: done (4 left) 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3996: done (3 left) 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3997: done (2 left) 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout: waiting on 2 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3998: finishing write tid 1 to smithi093153278-492 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3998: finishing write tid 2 to smithi093153278-492 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3998: finishing write tid 3 to smithi093153278-492 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3998: finishing write tid 4 to smithi093153278-492 2023-08-27T20:55:35.589 INFO:tasks.rados.rados.0.smithi093.stdout:3998: finishing write tid 5 to smithi093153278-492 2023-08-27T20:55:35.590 INFO:tasks.rados.rados.0.smithi093.stdout:update_object_version oid 492 v 429 (ObjNum 2453 snap 0 seq_num 2453) dirty exists 2023-08-27T20:55:35.590 INFO:tasks.rados.rados.0.smithi093.stdout:3998: left oid 492 (ObjNum 2453 snap 0 seq_num 2453) 2023-08-27T20:55:35.590 INFO:tasks.rados.rados.0.smithi093.stdout:3999: expect (ObjNum 2014 snap 0 seq_num 2014) 2023-08-27T20:55:35.716 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:55:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:55:35] "GET /metrics HTTP/1.1" 200 242575 "" "Prometheus/2.43.0" 2023-08-27T20:55:35.793 INFO:tasks.rados.rados.0.smithi093.stdout:3998: done (1 left) 2023-08-27T20:55:35.793 INFO:tasks.rados.rados.0.smithi093.stdout:3999: done (0 left) 2023-08-27T20:55:35.795 INFO:tasks.rados.rados.0.smithi093.stderr:0 errors. 2023-08-27T20:55:35.795 INFO:tasks.rados.rados.0.smithi093.stderr: 2023-08-27T20:55:35.798 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd dump --format=json 2023-08-27T20:55:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:35 smithi093 ceph-mon[111335]: pgmap v61: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 81 op/s 2023-08-27T20:55:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:35 smithi093 ceph-mon[118902]: pgmap v61: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 81 op/s 2023-08-27T20:55:36.102 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:55:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:35 smithi136 ceph-mon[113531]: pgmap v61: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 58 MiB/s rd, 9.1 MiB/s wr, 81 op/s 2023-08-27T20:55:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3206080229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3206080229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:37.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3206080229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:37.222 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:55:37.223 INFO:teuthology.orchestra.run.smithi093.stdout:{"epoch":73,"fsid":"b79a52c0-451a-11ee-817b-1b7a5f1ba315","created":"2023-08-27T20:46:40.100445+0000","modified":"2023-08-27T20:53:38.494078+0000","last_up_change":"2023-08-27T20:50:18.452654+0000","last_in_change":"2023-08-27T20:50:04.867005+0000","flags":"sortbitwise,recovery_deletes,purged_snapdirs,pglog_hardlimit","flags_num":5799936,"flags_set":["pglog_hardlimit","purged_snapdirs","recovery_deletes","sortbitwise"],"crush_version":18,"full_ratio":0.94999998807907104,"backfillfull_ratio":0.89999997615814209,"nearfull_ratio":0.85000002384185791,"cluster_snapshot":"","pool_max":5,"max_osd":8,"require_min_compat_client":"luminous","min_compat_client":"luminous","require_osd_release":"quincy","pools":[{"pool":1,"pool_name":".mgr","create_time":"2023-08-27T20:48:59.294309+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":1,"pg_placement_num":1,"pg_placement_num_target":1,"pg_num_target":1,"pg_num_pending":1,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"22","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_num_max":32,"pg_num_min":1},"application_metadata":{"mgr":{}}},{"pool":2,"pool_name":"rbd","create_time":"2023-08-27T20:50:39.429736+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":8,"pg_placement_num":8,"pg_placement_num_target":8,"pg_num_target":8,"pg_num_pending":8,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"59","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rbd":{}}},{"pool":3,"pool_name":"cephfs.foo.meta","create_time":"2023-08-27T20:51:16.824303+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{"pg_autoscale_bias":4,"pg_num_min":16,"recovery_priority":5},"application_metadata":{"cephfs":{"metadata":"foo"}}},{"pool":4,"pool_name":"cephfs.foo.data","create_time":"2023-08-27T20:51:17.324126+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":32,"pg_placement_num":32,"pg_placement_num_target":32,"pg_num_target":32,"pg_num_pending":32,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"63","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"cephfs":{"data":"foo"}}},{"pool":5,"pool_name":"unique_pool_0","create_time":"2023-08-27T20:51:57.440290+0000","flags":1,"flags_names":"hashpspool","type":1,"size":3,"min_size":2,"crush_rule":0,"peering_crush_bucket_count":0,"peering_crush_bucket_target":0,"peering_crush_bucket_barrier":0,"peering_crush_bucket_mandatory_member":2147483647,"object_hash":2,"pg_autoscale_mode":"off","pg_num":16,"pg_placement_num":16,"pg_placement_num_target":16,"pg_num_target":16,"pg_num_pending":16,"last_pg_merge_meta":{"source_pgid":"0.0","ready_epoch":0,"last_epoch_started":0,"last_epoch_clean":0,"source_version":"0'0","target_version":"0'0"},"last_change":"70","last_force_op_resend":"0","last_force_op_resend_prenautilus":"0","last_force_op_resend_preluminous":"0","auid":0,"snap_mode":"selfmanaged","snap_seq":0,"snap_epoch":0,"pool_snaps":[],"removed_snaps":"[]","quota_max_bytes":0,"quota_max_objects":0,"tiers":[],"tier_of":-1,"read_tier":-1,"write_tier":-1,"cache_mode":"none","target_max_bytes":0,"target_max_objects":0,"cache_target_dirty_ratio_micro":400000,"cache_target_dirty_high_ratio_micro":600000,"cache_target_full_ratio_micro":800000,"cache_min_flush_age":0,"cache_min_evict_age":0,"erasure_code_profile":"","hit_set_params":{"type":"none"},"hit_set_period":0,"hit_set_count":0,"use_gmt_hitset":true,"min_read_recency_for_promote":0,"min_write_recency_for_promote":0,"hit_set_grade_decay_rate":0,"hit_set_search_last_n":0,"grade_table":[],"stripe_width":0,"expected_num_objects":0,"fast_read":false,"options":{},"application_metadata":{"rados":{}}}],"osds":[{"osd":0,"uuid":"f8288f9e-5794-4532-9495-65379f66b902","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":8,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6802","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6803","nonce":468012600}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6804","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6805","nonce":468012600}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6808","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6809","nonce":468012600}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6806","nonce":468012600},{"type":"v1","addr":"172.21.15.93:6807","nonce":468012600}]},"public_addr":"172.21.15.93:6803/468012600","cluster_addr":"172.21.15.93:6805/468012600","heartbeat_back_addr":"172.21.15.93:6809/468012600","heartbeat_front_addr":"172.21.15.93:6807/468012600","state":["exists","up"]},{"osd":1,"uuid":"703653bb-1bc5-418f-bf13-4e71cb82a4da","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":13,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6810","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6811","nonce":1906627391}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6812","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6813","nonce":1906627391}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6816","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6817","nonce":1906627391}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6814","nonce":1906627391},{"type":"v1","addr":"172.21.15.93:6815","nonce":1906627391}]},"public_addr":"172.21.15.93:6811/1906627391","cluster_addr":"172.21.15.93:6813/1906627391","heartbeat_back_addr":"172.21.15.93:6817/1906627391","heartbeat_front_addr":"172.21.15.93:6815/1906627391","state":["exists","up"]},{"osd":2,"uuid":"ca3d4d17-b4ba-402a-999d-496be3cda08d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":18,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6818","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6819","nonce":3915054168}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6820","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6821","nonce":3915054168}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6824","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6825","nonce":3915054168}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6822","nonce":3915054168},{"type":"v1","addr":"172.21.15.93:6823","nonce":3915054168}]},"public_addr":"172.21.15.93:6819/3915054168","cluster_addr":"172.21.15.93:6821/3915054168","heartbeat_back_addr":"172.21.15.93:6825/3915054168","heartbeat_front_addr":"172.21.15.93:6823/3915054168","state":["exists","up"]},{"osd":3,"uuid":"04f9b593-2f0e-4b0e-b5ba-2778d688e66d","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":25,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6826","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6827","nonce":4177593389}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6828","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6829","nonce":4177593389}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6832","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6833","nonce":4177593389}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.93:6830","nonce":4177593389},{"type":"v1","addr":"172.21.15.93:6831","nonce":4177593389}]},"public_addr":"172.21.15.93:6827/4177593389","cluster_addr":"172.21.15.93:6829/4177593389","heartbeat_back_addr":"172.21.15.93:6833/4177593389","heartbeat_front_addr":"172.21.15.93:6831/4177593389","state":["exists","up"]},{"osd":4,"uuid":"2ca1eb6c-000f-42cc-a188-e0044386e9fd","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":30,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6800","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6801","nonce":4129055894}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6802","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6803","nonce":4129055894}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6806","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6807","nonce":4129055894}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6804","nonce":4129055894},{"type":"v1","addr":"172.21.15.136:6805","nonce":4129055894}]},"public_addr":"172.21.15.136:6801/4129055894","cluster_addr":"172.21.15.136:6803/4129055894","heartbeat_back_addr":"172.21.15.136:6807/4129055894","heartbeat_front_addr":"172.21.15.136:6805/4129055894","state":["exists","up"]},{"osd":5,"uuid":"eb791c51-36bd-4562-9715-7081bdb9b50b","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":37,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6808","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6809","nonce":1032556808}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6810","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6811","nonce":1032556808}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6814","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6815","nonce":1032556808}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6812","nonce":1032556808},{"type":"v1","addr":"172.21.15.136:6813","nonce":1032556808}]},"public_addr":"172.21.15.136:6809/1032556808","cluster_addr":"172.21.15.136:6811/1032556808","heartbeat_back_addr":"172.21.15.136:6815/1032556808","heartbeat_front_addr":"172.21.15.136:6813/1032556808","state":["exists","up"]},{"osd":6,"uuid":"9d1b03b4-2611-4877-83ac-05b34cb091ce","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":44,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6816","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6817","nonce":1621616467}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6818","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6819","nonce":1621616467}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6822","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6823","nonce":1621616467}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6820","nonce":1621616467},{"type":"v1","addr":"172.21.15.136:6821","nonce":1621616467}]},"public_addr":"172.21.15.136:6817/1621616467","cluster_addr":"172.21.15.136:6819/1621616467","heartbeat_back_addr":"172.21.15.136:6823/1621616467","heartbeat_front_addr":"172.21.15.136:6821/1621616467","state":["exists","up"]},{"osd":7,"uuid":"500cecba-b803-4827-9e93-025f93e19a50","up":1,"in":1,"weight":1,"primary_affinity":1,"last_clean_begin":0,"last_clean_end":0,"up_from":51,"up_thru":66,"down_at":0,"lost_at":0,"public_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6824","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6825","nonce":4170532217}]},"cluster_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6826","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6827","nonce":4170532217}]},"heartbeat_back_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6830","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6831","nonce":4170532217}]},"heartbeat_front_addrs":{"addrvec":[{"type":"v2","addr":"172.21.15.136:6828","nonce":4170532217},{"type":"v1","addr":"172.21.15.136:6829","nonce":4170532217}]},"public_addr":"172.21.15.136:6825/4170532217","cluster_addr":"172.21.15.136:6827/4170532217","heartbeat_back_addr":"172.21.15.136:6831/4170532217","heartbeat_front_addr":"172.21.15.136:6829/4170532217","state":["exists","up"]}],"osd_xinfo":[{"osd":0,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:23.430450+0000","dead_epoch":0},{"osd":1,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:39.677984+0000","dead_epoch":0},{"osd":2,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:48:56.383706+0000","dead_epoch":0},{"osd":3,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:13.215635+0000","dead_epoch":0},{"osd":4,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:28.875590+0000","dead_epoch":0},{"osd":5,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:49:45.109289+0000","dead_epoch":0},{"osd":6,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:01.198559+0000","dead_epoch":0},{"osd":7,"down_stamp":"0.000000","laggy_probability":0,"laggy_interval":0,"features":4540138320759226367,"old_weight":0,"last_purged_snaps_scrub":"2023-08-27T20:50:17.193536+0000","dead_epoch":0}],"pg_upmap":[],"pg_upmap_items":[{"pgid":"4.9","mappings":[{"from":1,"to":5}]},{"pgid":"4.e","mappings":[{"from":1,"to":2}]},{"pgid":"4.f","mappings":[{"from":1,"to":2}]}],"pg_temp":[],"primary_temp":[],"blocklist":{"172.21.15.93:6801/4025849521":"2023-08-28T20:53:05.083465+0000","172.21.15.93:6800/4025849521":"2023-08-28T20:53:05.083465+0000","172.21.15.93:0/421269516":"2023-08-28T20:53:05.083465+0000","172.21.15.93:0/966563482":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/750179315":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/2543216731":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/1072616791":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/804147002":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/713101306":"2023-08-28T20:53:05.083465+0000","172.21.15.93:0/1009313451":"2023-08-28T20:47:20.986783+0000","172.21.15.93:6800/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1575139232":"2023-08-28T20:47:01.868764+0000","172.21.15.93:6801/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/3173026838":"2023-08-28T20:53:05.083465+0000","172.21.15.93:6801/3557251521":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/1735542319":"2023-08-28T20:50:40.950579+0000","172.21.15.93:6800/1183753159":"2023-08-28T20:50:40.950579+0000","172.21.15.93:0/3005830643":"2023-08-28T20:53:05.083465+0000","172.21.15.93:0/2843438722":"2023-08-28T20:47:20.986783+0000","172.21.15.93:0/4182115085":"2023-08-28T20:47:01.868764+0000","172.21.15.93:0/2819309041":"2023-08-28T20:50:40.950579+0000"},"range_blocklist":{},"erasure_code_profiles":{"default":{"crush-failure-domain":"osd","k":"2","m":"1","plugin":"jerasure","technique":"reed_sol_van"}},"removed_snaps_queue":[],"new_removed_snaps":[],"new_purged_snaps":[],"crush_node_flags":{},"device_class_flags":{},"stretch_mode":{"stretch_mode_enabled":false,"stretch_bucket_count":0,"degraded_stretch_mode":0,"recovering_stretch_mode":0,"stretch_mode_bucket":0}} 2023-08-27T20:55:37.632 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph pg dump --format=json 2023-08-27T20:55:37.899 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:55:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:37 smithi093 ceph-mon[111335]: pgmap v62: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 14 MiB/s wr, 126 op/s 2023-08-27T20:55:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1546734638' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:55:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:37 smithi093 ceph-mon[118902]: pgmap v62: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 14 MiB/s wr, 126 op/s 2023-08-27T20:55:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1546734638' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:55:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:37 smithi136 ceph-mon[113531]: pgmap v62: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 86 MiB/s rd, 14 MiB/s wr, 126 op/s 2023-08-27T20:55:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1546734638' entity='client.admin' cmd=[{"prefix": "osd dump", "format": "json"}]: dispatch 2023-08-27T20:55:39.074 INFO:teuthology.orchestra.run.smithi093.stdout: 2023-08-27T20:55:39.076 INFO:teuthology.orchestra.run.smithi093.stderr:dumped all 2023-08-27T20:55:39.432 INFO:teuthology.orchestra.run.smithi093.stdout:{"pg_ready":true,"pg_map":{"version":63,"stamp":"2023-08-27T20:55:38.344535+0000","last_osdmap_epoch":0,"last_pg_scan":0,"pg_stats_sum":{"stat_sum":{"num_bytes":1226195231,"num_objects":438,"num_object_clones":0,"num_object_copies":1314,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":438,"num_whiteouts":0,"num_read":14675,"num_read_kb":15344293,"num_write":9983,"num_write_kb":2766147,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7871,"ondisk_log_size":7871,"up":267,"acting":267,"num_store_stats":0},"osd_stats_sum":{"up_from":0,"seq":0,"num_pgs":267,"num_osds":8,"num_per_pool_osds":8,"num_per_pool_omap_osds":8,"kb":749961216,"kb_used":2469128,"kb_used_data":2243504,"kb_used_omap":0,"kb_used_meta":195264,"kb_avail":747492088,"statfs":{"total":767960285184,"available":765431898112,"internally_reserved":0,"allocated":2297348096,"data_stored":2281362632,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":199950336},"hb_peers":[],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[]},"pg_stats_delta":{"stat_sum":{"num_bytes":7778397,"num_objects":3,"num_object_clones":0,"num_object_copies":9,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":3,"num_whiteouts":0,"num_read":589,"num_read_kb":677155,"num_write":384,"num_write_kb":108428,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":0,"acting":0,"num_store_stats":0,"stamp_delta":"12.002452"},"pg_stats":[{"pgid":"3.1a","version":"63'1","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976707+0000","last_change":"2023-08-27T20:51:18.103078+0000","last_active":"2023-08-27T20:53:06.976707+0000","last_peered":"2023-08-27T20:53:06.976707+0000","last_clean":"2023-08-27T20:53:06.976707+0000","last_became_active":"2023-08-27T20:51:18.102289+0000","last_became_peered":"2023-08-27T20:51:18.102289+0000","last_unstale":"2023-08-27T20:53:06.976707+0000","last_undegraded":"2023-08-27T20:53:06.976707+0000","last_fullsized":"2023-08-27T20:53:06.976707+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:46:25.395790+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,1,2],"acting":[4,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.1d","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973671+0000","last_change":"2023-08-27T20:51:18.129248+0000","last_active":"2023-08-27T20:53:06.973671+0000","last_peered":"2023-08-27T20:53:06.973671+0000","last_clean":"2023-08-27T20:53:06.973671+0000","last_became_active":"2023-08-27T20:51:18.128963+0000","last_became_peered":"2023-08-27T20:51:18.128963+0000","last_unstale":"2023-08-27T20:53:06.973671+0000","last_undegraded":"2023-08-27T20:53:06.973671+0000","last_fullsized":"2023-08-27T20:53:06.973671+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:31:19.516025+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.19","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936255+0000","last_change":"2023-08-27T20:51:18.100237+0000","last_active":"2023-08-27T20:53:06.936255+0000","last_peered":"2023-08-27T20:53:06.936255+0000","last_clean":"2023-08-27T20:53:06.936255+0000","last_became_active":"2023-08-27T20:51:18.099976+0000","last_became_peered":"2023-08-27T20:51:18.099976+0000","last_unstale":"2023-08-27T20:53:06.936255+0000","last_undegraded":"2023-08-27T20:53:06.936255+0000","last_fullsized":"2023-08-27T20:53:06.936255+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:46:15.847961+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,3,4],"acting":[1,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.1e","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061859+0000","last_change":"2023-08-27T20:51:18.174303+0000","last_active":"2023-08-27T20:53:07.061859+0000","last_peered":"2023-08-27T20:53:07.061859+0000","last_clean":"2023-08-27T20:53:07.061859+0000","last_became_active":"2023-08-27T20:51:18.173979+0000","last_became_peered":"2023-08-27T20:51:18.173979+0000","last_unstale":"2023-08-27T20:53:07.061859+0000","last_undegraded":"2023-08-27T20:53:07.061859+0000","last_fullsized":"2023-08-27T20:53:07.061859+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T20:58:16.258007+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.18","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973737+0000","last_change":"2023-08-27T20:51:18.094635+0000","last_active":"2023-08-27T20:53:06.973737+0000","last_peered":"2023-08-27T20:53:06.973737+0000","last_clean":"2023-08-27T20:53:06.973737+0000","last_became_active":"2023-08-27T20:51:18.094323+0000","last_became_peered":"2023-08-27T20:51:18.094323+0000","last_unstale":"2023-08-27T20:53:06.973737+0000","last_undegraded":"2023-08-27T20:53:06.973737+0000","last_fullsized":"2023-08-27T20:53:06.973737+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:29:06.163259+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,1],"acting":[3,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.1f","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.534366+0000","last_change":"2023-08-27T20:51:18.128634+0000","last_active":"2023-08-27T20:53:07.534366+0000","last_peered":"2023-08-27T20:53:07.534366+0000","last_clean":"2023-08-27T20:53:07.534366+0000","last_became_active":"2023-08-27T20:51:18.128367+0000","last_became_peered":"2023-08-27T20:51:18.128367+0000","last_unstale":"2023-08-27T20:53:07.534366+0000","last_undegraded":"2023-08-27T20:53:07.534366+0000","last_fullsized":"2023-08-27T20:53:07.534366+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:51:25.311153+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,5,1],"acting":[6,5,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.17","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061925+0000","last_change":"2023-08-27T20:51:18.097550+0000","last_active":"2023-08-27T20:53:07.061925+0000","last_peered":"2023-08-27T20:53:07.061925+0000","last_clean":"2023-08-27T20:53:07.061925+0000","last_became_active":"2023-08-27T20:51:18.097145+0000","last_became_peered":"2023-08-27T20:51:18.097145+0000","last_unstale":"2023-08-27T20:53:07.061925+0000","last_undegraded":"2023-08-27T20:53:07.061925+0000","last_fullsized":"2023-08-27T20:53:07.061925+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:36:26.771185+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,3],"acting":[0,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.10","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973160+0000","last_change":"2023-08-27T20:51:18.129959+0000","last_active":"2023-08-27T20:53:06.973160+0000","last_peered":"2023-08-27T20:53:06.973160+0000","last_clean":"2023-08-27T20:53:06.973160+0000","last_became_active":"2023-08-27T20:51:18.129690+0000","last_became_peered":"2023-08-27T20:51:18.129690+0000","last_unstale":"2023-08-27T20:53:06.973160+0000","last_undegraded":"2023-08-27T20:53:06.973160+0000","last_fullsized":"2023-08-27T20:53:06.973160+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:35:08.487865+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,6],"acting":[3,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.16","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936468+0000","last_change":"2023-08-27T20:51:18.097098+0000","last_active":"2023-08-27T20:53:06.936468+0000","last_peered":"2023-08-27T20:53:06.936468+0000","last_clean":"2023-08-27T20:53:06.936468+0000","last_became_active":"2023-08-27T20:51:18.096745+0000","last_became_peered":"2023-08-27T20:51:18.096745+0000","last_unstale":"2023-08-27T20:53:06.936468+0000","last_undegraded":"2023-08-27T20:53:06.936468+0000","last_fullsized":"2023-08-27T20:53:06.936468+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:42:05.626352+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,1],"acting":[5,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.11","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973280+0000","last_change":"2023-08-27T20:51:18.174261+0000","last_active":"2023-08-27T20:53:06.973280+0000","last_peered":"2023-08-27T20:53:06.973280+0000","last_clean":"2023-08-27T20:53:06.973280+0000","last_became_active":"2023-08-27T20:51:18.174020+0000","last_became_peered":"2023-08-27T20:51:18.174020+0000","last_unstale":"2023-08-27T20:53:06.973280+0000","last_undegraded":"2023-08-27T20:53:06.973280+0000","last_fullsized":"2023-08-27T20:53:06.973280+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:06:33.961645+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,6],"acting":[3,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.15","version":"63'3","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067576+0000","last_change":"2023-08-27T20:51:18.102959+0000","last_active":"2023-08-27T20:53:07.067576+0000","last_peered":"2023-08-27T20:53:07.067576+0000","last_clean":"2023-08-27T20:53:07.067576+0000","last_became_active":"2023-08-27T20:51:18.102698+0000","last_became_peered":"2023-08-27T20:51:18.102698+0000","last_unstale":"2023-08-27T20:53:07.067576+0000","last_undegraded":"2023-08-27T20:53:07.067576+0000","last_fullsized":"2023-08-27T20:53:07.067576+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":3,"ondisk_log_size":3,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:00:59.591758+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,3,4],"acting":[7,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.12","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936423+0000","last_change":"2023-08-27T20:51:18.126515+0000","last_active":"2023-08-27T20:53:06.936423+0000","last_peered":"2023-08-27T20:53:06.936423+0000","last_clean":"2023-08-27T20:53:06.936423+0000","last_became_active":"2023-08-27T20:51:18.123514+0000","last_became_peered":"2023-08-27T20:51:18.123514+0000","last_unstale":"2023-08-27T20:53:06.936423+0000","last_undegraded":"2023-08-27T20:53:06.936423+0000","last_fullsized":"2023-08-27T20:53:06.936423+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:14:00.412812+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,2],"acting":[1,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.14","version":"73'12","reported_seq":33,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:08.632473+0000","last_change":"2023-08-27T20:51:18.106623+0000","last_active":"2023-08-27T20:55:08.632473+0000","last_peered":"2023-08-27T20:55:08.632473+0000","last_clean":"2023-08-27T20:55:08.632473+0000","last_became_active":"2023-08-27T20:51:18.106343+0000","last_became_peered":"2023-08-27T20:51:18.106343+0000","last_unstale":"2023-08-27T20:55:08.632473+0000","last_undegraded":"2023-08-27T20:55:08.632473+0000","last_fullsized":"2023-08-27T20:55:08.632473+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":12,"ondisk_log_size":12,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:43:08.157700+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":2608,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":9,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,7,6],"acting":[4,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.13","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976869+0000","last_change":"2023-08-27T20:51:18.131165+0000","last_active":"2023-08-27T20:53:06.976869+0000","last_peered":"2023-08-27T20:53:06.976869+0000","last_clean":"2023-08-27T20:53:06.976869+0000","last_became_active":"2023-08-27T20:51:18.130812+0000","last_became_peered":"2023-08-27T20:51:18.130812+0000","last_unstale":"2023-08-27T20:53:06.976869+0000","last_undegraded":"2023-08-27T20:53:06.976869+0000","last_fullsized":"2023-08-27T20:53:06.976869+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:17:11.521086+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.13","version":"63'2","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067673+0000","last_change":"2023-08-27T20:51:18.096913+0000","last_active":"2023-08-27T20:53:07.067673+0000","last_peered":"2023-08-27T20:53:07.067673+0000","last_clean":"2023-08-27T20:53:07.067673+0000","last_became_active":"2023-08-27T20:51:18.096600+0000","last_became_peered":"2023-08-27T20:51:18.096600+0000","last_unstale":"2023-08-27T20:53:07.067673+0000","last_undegraded":"2023-08-27T20:53:07.067673+0000","last_fullsized":"2023-08-27T20:53:07.067673+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:41:23.301928+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":34,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":6,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,2],"acting":[7,4,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.14","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973432+0000","last_change":"2023-08-27T20:51:18.174487+0000","last_active":"2023-08-27T20:53:06.973432+0000","last_peered":"2023-08-27T20:53:06.973432+0000","last_clean":"2023-08-27T20:53:06.973432+0000","last_became_active":"2023-08-27T20:51:18.174142+0000","last_became_peered":"2023-08-27T20:51:18.174142+0000","last_unstale":"2023-08-27T20:53:06.973432+0000","last_undegraded":"2023-08-27T20:53:06.973432+0000","last_fullsized":"2023-08-27T20:53:06.973432+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:21:00.888579+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.12","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061406+0000","last_change":"2023-08-27T20:51:18.097273+0000","last_active":"2023-08-27T20:53:07.061406+0000","last_peered":"2023-08-27T20:53:07.061406+0000","last_clean":"2023-08-27T20:53:07.061406+0000","last_became_active":"2023-08-27T20:51:18.096940+0000","last_became_peered":"2023-08-27T20:51:18.096940+0000","last_unstale":"2023-08-27T20:53:07.061406+0000","last_undegraded":"2023-08-27T20:53:07.061406+0000","last_fullsized":"2023-08-27T20:53:07.061406+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:01:47.450662+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,7,3],"acting":[0,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.15","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936467+0000","last_change":"2023-08-27T20:51:18.174803+0000","last_active":"2023-08-27T20:53:06.936467+0000","last_peered":"2023-08-27T20:53:06.936467+0000","last_clean":"2023-08-27T20:53:06.936467+0000","last_became_active":"2023-08-27T20:51:18.174313+0000","last_became_peered":"2023-08-27T20:51:18.174313+0000","last_unstale":"2023-08-27T20:53:06.936467+0000","last_undegraded":"2023-08-27T20:53:06.936467+0000","last_fullsized":"2023-08-27T20:53:06.936467+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:10:32.503697+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,3],"acting":[5,7,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.11","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067465+0000","last_change":"2023-08-27T20:51:18.098165+0000","last_active":"2023-08-27T20:53:07.067465+0000","last_peered":"2023-08-27T20:53:07.067465+0000","last_clean":"2023-08-27T20:53:07.067465+0000","last_became_active":"2023-08-27T20:51:18.097846+0000","last_became_peered":"2023-08-27T20:51:18.097846+0000","last_unstale":"2023-08-27T20:53:07.067465+0000","last_undegraded":"2023-08-27T20:53:07.067465+0000","last_fullsized":"2023-08-27T20:53:07.067465+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:43:32.430484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,6],"acting":[7,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.16","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061788+0000","last_change":"2023-08-27T20:51:18.176875+0000","last_active":"2023-08-27T20:53:07.061788+0000","last_peered":"2023-08-27T20:53:07.061788+0000","last_clean":"2023-08-27T20:53:07.061788+0000","last_became_active":"2023-08-27T20:51:18.176547+0000","last_became_peered":"2023-08-27T20:51:18.176547+0000","last_unstale":"2023-08-27T20:53:07.061788+0000","last_undegraded":"2023-08-27T20:53:07.061788+0000","last_fullsized":"2023-08-27T20:53:07.061788+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:13:16.094024+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,3,7],"acting":[0,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"3.10","version":"63'8","reported_seq":29,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.534129+0000","last_change":"2023-08-27T20:51:18.105714+0000","last_active":"2023-08-27T20:53:07.534129+0000","last_peered":"2023-08-27T20:53:07.534129+0000","last_clean":"2023-08-27T20:53:07.534129+0000","last_became_active":"2023-08-27T20:51:18.105364+0000","last_became_peered":"2023-08-27T20:51:18.105364+0000","last_unstale":"2023-08-27T20:53:07.534129+0000","last_undegraded":"2023-08-27T20:53:07.534129+0000","last_fullsized":"2023-08-27T20:53:07.534129+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":8,"ondisk_log_size":8,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:28:47.202084+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":4,"num_object_clones":0,"num_object_copies":12,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":4,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":12,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":4,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.17","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973873+0000","last_change":"2023-08-27T20:51:18.174292+0000","last_active":"2023-08-27T20:53:06.973873+0000","last_peered":"2023-08-27T20:53:06.973873+0000","last_clean":"2023-08-27T20:53:06.973873+0000","last_became_active":"2023-08-27T20:51:18.173994+0000","last_became_peered":"2023-08-27T20:51:18.173994+0000","last_unstale":"2023-08-27T20:53:06.973873+0000","last_undegraded":"2023-08-27T20:53:06.973873+0000","last_fullsized":"2023-08-27T20:53:06.973873+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:16:13.606362+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,1],"acting":[3,7,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.9","version":"73'444","reported_seq":838,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:30.859782+0000","last_change":"2023-08-27T20:51:59.069578+0000","last_active":"2023-08-27T20:55:30.859782+0000","last_peered":"2023-08-27T20:55:30.859782+0000","last_clean":"2023-08-27T20:55:30.859782+0000","last_became_active":"2023-08-27T20:51:59.069288+0000","last_became_peered":"2023-08-27T20:51:59.069288+0000","last_unstale":"2023-08-27T20:55:30.859782+0000","last_undegraded":"2023-08-27T20:55:30.859782+0000","last_fullsized":"2023-08-27T20:55:30.859782+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":444,"ondisk_log_size":444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:35:12.757989+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76131509,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":897,"num_read_kb":980269,"num_write":558,"num_write_kb":155808,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,6,4],"acting":[7,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.f","version":"63'1","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067775+0000","last_change":"2023-08-27T20:51:18.100527+0000","last_active":"2023-08-27T20:53:07.067775+0000","last_peered":"2023-08-27T20:53:07.067775+0000","last_clean":"2023-08-27T20:53:07.067775+0000","last_became_active":"2023-08-27T20:51:18.100150+0000","last_became_peered":"2023-08-27T20:51:18.100150+0000","last_unstale":"2023-08-27T20:53:07.067775+0000","last_undegraded":"2023-08-27T20:53:07.067775+0000","last_fullsized":"2023-08-27T20:53:07.067775+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:31:30.303986+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":22,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,0],"acting":[7,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.8","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.935783+0000","last_change":"2023-08-27T20:51:18.174616+0000","last_active":"2023-08-27T20:53:06.935783+0000","last_peered":"2023-08-27T20:53:06.935783+0000","last_clean":"2023-08-27T20:53:06.935783+0000","last_became_active":"2023-08-27T20:51:18.173970+0000","last_became_peered":"2023-08-27T20:51:18.173970+0000","last_unstale":"2023-08-27T20:53:06.935783+0000","last_undegraded":"2023-08-27T20:53:06.935783+0000","last_fullsized":"2023-08-27T20:53:06.935783+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:17:07.500477+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,7,6],"acting":[5,7,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"5.8","version":"73'435","reported_seq":860,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.210230+0000","last_change":"2023-08-27T20:51:59.068072+0000","last_active":"2023-08-27T20:55:34.210230+0000","last_peered":"2023-08-27T20:55:34.210230+0000","last_clean":"2023-08-27T20:55:34.210230+0000","last_became_active":"2023-08-27T20:51:59.067808+0000","last_became_peered":"2023-08-27T20:51:59.067808+0000","last_unstale":"2023-08-27T20:55:34.210230+0000","last_undegraded":"2023-08-27T20:55:34.210230+0000","last_fullsized":"2023-08-27T20:55:34.210230+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":435,"ondisk_log_size":435,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:43:55.774400+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75829346,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":983,"num_read_kb":1054977,"num_write":557,"num_write_kb":156411,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,1],"acting":[2,0,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.e","version":"63'2","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067622+0000","last_change":"2023-08-27T20:51:18.104712+0000","last_active":"2023-08-27T20:53:07.067622+0000","last_peered":"2023-08-27T20:53:07.067622+0000","last_clean":"2023-08-27T20:53:07.067622+0000","last_became_active":"2023-08-27T20:51:18.104420+0000","last_became_peered":"2023-08-27T20:51:18.104420+0000","last_unstale":"2023-08-27T20:53:07.067622+0000","last_undegraded":"2023-08-27T20:53:07.067622+0000","last_fullsized":"2023-08-27T20:53:07.067622+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:26:27.616548+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,4,1],"acting":[7,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.9","version":"0'0","reported_seq":30,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.977063+0000","last_change":"2023-08-27T20:51:42.784195+0000","last_active":"2023-08-27T20:53:06.977063+0000","last_peered":"2023-08-27T20:53:06.977063+0000","last_clean":"2023-08-27T20:53:06.977063+0000","last_became_active":"2023-08-27T20:51:42.783951+0000","last_became_peered":"2023-08-27T20:51:42.783951+0000","last_unstale":"2023-08-27T20:53:06.977063+0000","last_undegraded":"2023-08-27T20:53:06.977063+0000","last_fullsized":"2023-08-27T20:53:06.977063+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:45.897182+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,3],"acting":[4,5,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.b","version":"73'497","reported_seq":923,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:33.999653+0000","last_change":"2023-08-27T20:51:59.073860+0000","last_active":"2023-08-27T20:55:33.999653+0000","last_peered":"2023-08-27T20:55:33.999653+0000","last_clean":"2023-08-27T20:55:33.999653+0000","last_became_active":"2023-08-27T20:51:59.073580+0000","last_became_peered":"2023-08-27T20:51:59.073580+0000","last_unstale":"2023-08-27T20:55:33.999653+0000","last_undegraded":"2023-08-27T20:55:33.999653+0000","last_fullsized":"2023-08-27T20:55:33.999653+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":497,"ondisk_log_size":497,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:22:06.997654+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82857180,"num_objects":29,"num_object_clones":0,"num_object_copies":87,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":29,"num_whiteouts":0,"num_read":938,"num_read_kb":928427,"num_write":635,"num_write_kb":181850,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,0,5],"acting":[2,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.d","version":"63'1","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067660+0000","last_change":"2023-08-27T20:51:18.102162+0000","last_active":"2023-08-27T20:53:07.067660+0000","last_peered":"2023-08-27T20:53:07.067660+0000","last_clean":"2023-08-27T20:53:07.067660+0000","last_became_active":"2023-08-27T20:51:18.101864+0000","last_became_peered":"2023-08-27T20:51:18.101864+0000","last_unstale":"2023-08-27T20:53:07.067660+0000","last_undegraded":"2023-08-27T20:53:07.067660+0000","last_fullsized":"2023-08-27T20:53:07.067660+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:19:31.686394+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":70,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,5,6],"acting":[7,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"4.a","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.534335+0000","last_change":"2023-08-27T20:51:18.174360+0000","last_active":"2023-08-27T20:53:07.534335+0000","last_peered":"2023-08-27T20:53:07.534335+0000","last_clean":"2023-08-27T20:53:07.534335+0000","last_became_active":"2023-08-27T20:51:18.174027+0000","last_became_peered":"2023-08-27T20:51:18.174027+0000","last_unstale":"2023-08-27T20:53:07.534335+0000","last_undegraded":"2023-08-27T20:53:07.534335+0000","last_fullsized":"2023-08-27T20:53:07.534335+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:04:51.281103+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,1,7],"acting":[6,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.a","version":"73'519","reported_seq":992,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:33.059976+0000","last_change":"2023-08-27T20:51:59.063287+0000","last_active":"2023-08-27T20:55:33.059976+0000","last_peered":"2023-08-27T20:55:33.059976+0000","last_clean":"2023-08-27T20:55:33.059976+0000","last_became_active":"2023-08-27T20:51:59.062992+0000","last_became_peered":"2023-08-27T20:51:59.062992+0000","last_unstale":"2023-08-27T20:55:33.059976+0000","last_undegraded":"2023-08-27T20:55:33.059976+0000","last_fullsized":"2023-08-27T20:55:33.059976+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":519,"ondisk_log_size":519,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:03:51.314585+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":94250379,"num_objects":31,"num_object_clones":0,"num_object_copies":93,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":31,"num_whiteouts":0,"num_read":1080,"num_read_kb":1151503,"num_write":662,"num_write_kb":187170,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,4,3],"acting":[2,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.c","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936249+0000","last_change":"2023-08-27T20:51:18.102046+0000","last_active":"2023-08-27T20:53:06.936249+0000","last_peered":"2023-08-27T20:53:06.936249+0000","last_clean":"2023-08-27T20:53:06.936249+0000","last_became_active":"2023-08-27T20:51:18.101748+0000","last_became_peered":"2023-08-27T20:51:18.101748+0000","last_unstale":"2023-08-27T20:53:06.936249+0000","last_undegraded":"2023-08-27T20:53:06.936249+0000","last_fullsized":"2023-08-27T20:53:06.936249+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:23:02.841371+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,6],"acting":[5,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.b","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061466+0000","last_change":"2023-08-27T20:51:18.132475+0000","last_active":"2023-08-27T20:53:07.061466+0000","last_peered":"2023-08-27T20:53:07.061466+0000","last_clean":"2023-08-27T20:53:07.061466+0000","last_became_active":"2023-08-27T20:51:18.132202+0000","last_became_peered":"2023-08-27T20:51:18.132202+0000","last_unstale":"2023-08-27T20:53:07.061466+0000","last_undegraded":"2023-08-27T20:53:07.061466+0000","last_fullsized":"2023-08-27T20:53:07.061466+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:59:04.704593+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.d","version":"73'510","reported_seq":865,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.419884+0000","last_change":"2023-08-27T20:51:59.067440+0000","last_active":"2023-08-27T20:55:34.419884+0000","last_peered":"2023-08-27T20:55:34.419884+0000","last_clean":"2023-08-27T20:55:34.419884+0000","last_became_active":"2023-08-27T20:51:59.067165+0000","last_became_peered":"2023-08-27T20:51:59.067165+0000","last_unstale":"2023-08-27T20:55:34.419884+0000","last_undegraded":"2023-08-27T20:55:34.419884+0000","last_fullsized":"2023-08-27T20:55:34.419884+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":510,"ondisk_log_size":510,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:25:06.219630+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78568491,"num_objects":25,"num_object_clones":0,"num_object_copies":75,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":25,"num_whiteouts":0,"num_read":763,"num_read_kb":757163,"num_write":641,"num_write_kb":180742,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,7,5],"acting":[2,7,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.b","version":"63'2","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973265+0000","last_change":"2023-08-27T20:51:18.096242+0000","last_active":"2023-08-27T20:53:06.973265+0000","last_peered":"2023-08-27T20:53:06.973265+0000","last_clean":"2023-08-27T20:53:06.973265+0000","last_became_active":"2023-08-27T20:51:18.096012+0000","last_became_peered":"2023-08-27T20:51:18.096012+0000","last_unstale":"2023-08-27T20:53:06.973265+0000","last_undegraded":"2023-08-27T20:53:06.973265+0000","last_fullsized":"2023-08-27T20:53:06.973265+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:07:12.658963+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,0,4],"acting":[3,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.c","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976837+0000","last_change":"2023-08-27T20:51:18.129095+0000","last_active":"2023-08-27T20:53:06.976837+0000","last_peered":"2023-08-27T20:53:06.976837+0000","last_clean":"2023-08-27T20:53:06.976837+0000","last_became_active":"2023-08-27T20:51:18.128667+0000","last_became_peered":"2023-08-27T20:51:18.128667+0000","last_unstale":"2023-08-27T20:53:06.976837+0000","last_undegraded":"2023-08-27T20:53:06.976837+0000","last_fullsized":"2023-08-27T20:53:06.976837+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:14:50.669396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,6],"acting":[4,3,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.c","version":"73'624","reported_seq":1110,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.176446+0000","last_change":"2023-08-27T20:51:59.071007+0000","last_active":"2023-08-27T20:55:34.176446+0000","last_peered":"2023-08-27T20:55:34.176446+0000","last_clean":"2023-08-27T20:55:34.176446+0000","last_became_active":"2023-08-27T20:51:59.070778+0000","last_became_peered":"2023-08-27T20:51:59.070778+0000","last_unstale":"2023-08-27T20:55:34.176446+0000","last_undegraded":"2023-08-27T20:55:34.176446+0000","last_fullsized":"2023-08-27T20:55:34.176446+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":624,"ondisk_log_size":624,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:45:05.141905+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78216666,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":1080,"num_read_kb":1150127,"num_write":785,"num_write_kb":224858,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,4,0],"acting":[1,4,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.a","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.534428+0000","last_change":"2023-08-27T20:51:18.108839+0000","last_active":"2023-08-27T20:53:07.534428+0000","last_peered":"2023-08-27T20:53:07.534428+0000","last_clean":"2023-08-27T20:53:07.534428+0000","last_became_active":"2023-08-27T20:51:18.108560+0000","last_became_peered":"2023-08-27T20:51:18.108560+0000","last_unstale":"2023-08-27T20:53:07.534428+0000","last_undegraded":"2023-08-27T20:53:07.534428+0000","last_fullsized":"2023-08-27T20:53:07.534428+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:36:37.649085+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,1],"acting":[6,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"4.d","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976803+0000","last_change":"2023-08-27T20:51:18.127756+0000","last_active":"2023-08-27T20:53:06.976803+0000","last_peered":"2023-08-27T20:53:06.976803+0000","last_clean":"2023-08-27T20:53:06.976803+0000","last_became_active":"2023-08-27T20:51:18.127374+0000","last_became_peered":"2023-08-27T20:51:18.127374+0000","last_unstale":"2023-08-27T20:53:06.976803+0000","last_undegraded":"2023-08-27T20:53:06.976803+0000","last_fullsized":"2023-08-27T20:53:06.976803+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:56:07.099266+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,1],"acting":[4,2,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.f","version":"73'422","reported_seq":780,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.147852+0000","last_change":"2023-08-27T20:51:59.063269+0000","last_active":"2023-08-27T20:55:34.147852+0000","last_peered":"2023-08-27T20:55:34.147852+0000","last_clean":"2023-08-27T20:55:34.147852+0000","last_became_active":"2023-08-27T20:51:59.062941+0000","last_became_peered":"2023-08-27T20:51:59.062941+0000","last_unstale":"2023-08-27T20:55:34.147852+0000","last_undegraded":"2023-08-27T20:55:34.147852+0000","last_fullsized":"2023-08-27T20:55:34.147852+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":422,"ondisk_log_size":422,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:19:32.012399+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":64062994,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":793,"num_read_kb":844255,"num_write":536,"num_write_kb":150221,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.9","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976514+0000","last_change":"2023-08-27T20:51:18.104763+0000","last_active":"2023-08-27T20:53:06.976514+0000","last_peered":"2023-08-27T20:53:06.976514+0000","last_clean":"2023-08-27T20:53:06.976514+0000","last_became_active":"2023-08-27T20:51:18.104258+0000","last_became_peered":"2023-08-27T20:51:18.104258+0000","last_unstale":"2023-08-27T20:53:06.976514+0000","last_undegraded":"2023-08-27T20:53:06.976514+0000","last_fullsized":"2023-08-27T20:53:06.976514+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:23:37.009396+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,2,7],"acting":[4,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.e","version":"0'0","reported_seq":30,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976797+0000","last_change":"2023-08-27T20:51:42.749070+0000","last_active":"2023-08-27T20:53:06.976797+0000","last_peered":"2023-08-27T20:53:06.976797+0000","last_clean":"2023-08-27T20:53:06.976797+0000","last_became_active":"2023-08-27T20:51:42.748736+0000","last_became_peered":"2023-08-27T20:51:42.748736+0000","last_unstale":"2023-08-27T20:53:06.976797+0000","last_undegraded":"2023-08-27T20:53:06.976797+0000","last_fullsized":"2023-08-27T20:53:06.976797+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:58:27.930976+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,2],"acting":[4,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.e","version":"73'425","reported_seq":765,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:33.970474+0000","last_change":"2023-08-27T20:51:59.073511+0000","last_active":"2023-08-27T20:55:33.970474+0000","last_peered":"2023-08-27T20:55:33.970474+0000","last_clean":"2023-08-27T20:55:33.970474+0000","last_became_active":"2023-08-27T20:51:59.073196+0000","last_became_peered":"2023-08-27T20:51:59.073196+0000","last_unstale":"2023-08-27T20:55:33.970474+0000","last_undegraded":"2023-08-27T20:55:33.970474+0000","last_fullsized":"2023-08-27T20:55:33.970474+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":425,"ondisk_log_size":425,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:59:53.594657+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":75206959,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":755,"num_read_kb":788479,"num_write":537,"num_write_kb":150108,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,0],"acting":[4,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.8","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973461+0000","last_change":"2023-08-27T20:51:18.101235+0000","last_active":"2023-08-27T20:53:06.973461+0000","last_peered":"2023-08-27T20:53:06.973461+0000","last_clean":"2023-08-27T20:53:06.973461+0000","last_became_active":"2023-08-27T20:51:18.100949+0000","last_became_peered":"2023-08-27T20:51:18.100949+0000","last_unstale":"2023-08-27T20:53:06.973461+0000","last_undegraded":"2023-08-27T20:53:06.973461+0000","last_fullsized":"2023-08-27T20:53:06.973461+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:22:26.784792+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,7],"acting":[3,1,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.f","version":"0'0","reported_seq":18,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.938289+0000","last_change":"2023-08-27T20:51:42.784026+0000","last_active":"2023-08-27T20:53:06.938289+0000","last_peered":"2023-08-27T20:53:06.938289+0000","last_clean":"2023-08-27T20:53:06.938289+0000","last_became_active":"2023-08-27T20:51:42.783779+0000","last_became_peered":"2023-08-27T20:51:42.783779+0000","last_unstale":"2023-08-27T20:53:06.938289+0000","last_undegraded":"2023-08-27T20:53:06.938289+0000","last_fullsized":"2023-08-27T20:53:06.938289+0000","mapping_epoch":64,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":65,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:11:19.094709+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,4],"acting":[2,3,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.4","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.935409+0000","last_change":"2023-08-27T20:50:41.004078+0000","last_active":"2023-08-27T20:53:06.935409+0000","last_peered":"2023-08-27T20:53:06.935409+0000","last_clean":"2023-08-27T20:53:06.935409+0000","last_became_active":"2023-08-27T20:50:41.003803+0000","last_became_peered":"2023-08-27T20:50:41.003803+0000","last_unstale":"2023-08-27T20:53:06.935409+0000","last_undegraded":"2023-08-27T20:53:06.935409+0000","last_fullsized":"2023-08-27T20:53:06.935409+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:14:14.109789+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,0,7],"acting":[1,0,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"3.5","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936543+0000","last_change":"2023-08-27T20:51:18.098948+0000","last_active":"2023-08-27T20:53:06.936543+0000","last_peered":"2023-08-27T20:53:06.936543+0000","last_clean":"2023-08-27T20:53:06.936543+0000","last_became_active":"2023-08-27T20:51:18.098648+0000","last_became_peered":"2023-08-27T20:51:18.098648+0000","last_unstale":"2023-08-27T20:53:06.936543+0000","last_undegraded":"2023-08-27T20:53:06.936543+0000","last_fullsized":"2023-08-27T20:53:06.936543+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:57:40.046678+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,3,2],"acting":[5,3,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.2","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936485+0000","last_change":"2023-08-27T20:51:18.125800+0000","last_active":"2023-08-27T20:53:06.936485+0000","last_peered":"2023-08-27T20:53:06.936485+0000","last_clean":"2023-08-27T20:53:06.936485+0000","last_became_active":"2023-08-27T20:51:18.125533+0000","last_became_peered":"2023-08-27T20:51:18.125533+0000","last_unstale":"2023-08-27T20:53:06.936485+0000","last_undegraded":"2023-08-27T20:53:06.936485+0000","last_fullsized":"2023-08-27T20:53:06.936485+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:58:23.165148+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,4],"acting":[1,5,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.3","version":"73'573","reported_seq":1040,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.193023+0000","last_change":"2023-08-27T20:51:59.072874+0000","last_active":"2023-08-27T20:55:34.193023+0000","last_peered":"2023-08-27T20:55:34.193023+0000","last_clean":"2023-08-27T20:55:34.193023+0000","last_became_active":"2023-08-27T20:51:59.072600+0000","last_became_peered":"2023-08-27T20:51:59.072600+0000","last_unstale":"2023-08-27T20:55:34.193023+0000","last_undegraded":"2023-08-27T20:55:34.193023+0000","last_fullsized":"2023-08-27T20:55:34.193023+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":573,"ondisk_log_size":573,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:56:40.932324+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":77544551,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":1052,"num_read_kb":1161389,"num_write":723,"num_write_kb":210292,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,6,5],"acting":[0,6,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.5","version":"73'474","reported_seq":839,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:33.088137+0000","last_change":"2023-08-27T20:51:59.073077+0000","last_active":"2023-08-27T20:55:33.088137+0000","last_peered":"2023-08-27T20:55:33.088137+0000","last_clean":"2023-08-27T20:55:33.088137+0000","last_became_active":"2023-08-27T20:51:59.072850+0000","last_became_peered":"2023-08-27T20:51:59.072850+0000","last_unstale":"2023-08-27T20:55:33.088137+0000","last_undegraded":"2023-08-27T20:55:33.088137+0000","last_fullsized":"2023-08-27T20:55:33.088137+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":474,"ondisk_log_size":474,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:55:01.588093+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":66667551,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":811,"num_read_kb":904165,"num_write":596,"num_write_kb":170129,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.2","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.935951+0000","last_change":"2023-08-27T20:50:40.475080+0000","last_active":"2023-08-27T20:53:06.935951+0000","last_peered":"2023-08-27T20:53:06.935951+0000","last_clean":"2023-08-27T20:53:06.935951+0000","last_became_active":"2023-08-27T20:50:40.474840+0000","last_became_peered":"2023-08-27T20:50:40.474840+0000","last_unstale":"2023-08-27T20:53:06.935951+0000","last_undegraded":"2023-08-27T20:53:06.935951+0000","last_fullsized":"2023-08-27T20:53:06.935951+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:35:31.367035+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,6],"acting":[5,1,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"3.3","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976637+0000","last_change":"2023-08-27T20:51:18.102990+0000","last_active":"2023-08-27T20:53:06.976637+0000","last_peered":"2023-08-27T20:53:06.976637+0000","last_clean":"2023-08-27T20:53:06.976637+0000","last_became_active":"2023-08-27T20:51:18.102475+0000","last_became_peered":"2023-08-27T20:51:18.102475+0000","last_unstale":"2023-08-27T20:53:06.976637+0000","last_undegraded":"2023-08-27T20:53:06.976637+0000","last_fullsized":"2023-08-27T20:53:06.976637+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T20:52:23.666385+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,0,6],"acting":[4,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"4.4","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.935511+0000","last_change":"2023-08-27T20:51:18.133061+0000","last_active":"2023-08-27T20:53:06.935511+0000","last_peered":"2023-08-27T20:53:06.935511+0000","last_clean":"2023-08-27T20:53:06.935511+0000","last_became_active":"2023-08-27T20:51:18.132737+0000","last_became_peered":"2023-08-27T20:51:18.132737+0000","last_unstale":"2023-08-27T20:53:06.935511+0000","last_undegraded":"2023-08-27T20:53:06.935511+0000","last_fullsized":"2023-08-27T20:53:06.935511+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:38:31.433133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,3],"acting":[1,2,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"5.6","version":"73'561","reported_seq":984,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:33.090771+0000","last_change":"2023-08-27T20:51:59.067922+0000","last_active":"2023-08-27T20:55:33.090771+0000","last_peered":"2023-08-27T20:55:33.090771+0000","last_clean":"2023-08-27T20:55:33.090771+0000","last_became_active":"2023-08-27T20:51:59.067651+0000","last_became_peered":"2023-08-27T20:51:59.067651+0000","last_unstale":"2023-08-27T20:55:33.090771+0000","last_undegraded":"2023-08-27T20:55:33.090771+0000","last_fullsized":"2023-08-27T20:55:33.090771+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":561,"ondisk_log_size":561,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:36:56.705080+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":90805208,"num_objects":30,"num_object_clones":0,"num_object_copies":90,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":30,"num_whiteouts":0,"num_read":927,"num_read_kb":957144,"num_write":713,"num_write_kb":205614,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,5,7],"acting":[2,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"2.1","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.938301+0000","last_change":"2023-08-27T20:50:41.003113+0000","last_active":"2023-08-27T20:53:06.938301+0000","last_peered":"2023-08-27T20:53:06.938301+0000","last_clean":"2023-08-27T20:53:06.938301+0000","last_became_active":"2023-08-27T20:50:41.002820+0000","last_became_peered":"2023-08-27T20:50:41.002820+0000","last_unstale":"2023-08-27T20:53:06.938301+0000","last_undegraded":"2023-08-27T20:53:06.938301+0000","last_fullsized":"2023-08-27T20:53:06.938301+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:03:10.585918+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,3,0],"acting":[2,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.0","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.935129+0000","last_change":"2023-08-27T20:51:18.102189+0000","last_active":"2023-08-27T20:53:06.935129+0000","last_peered":"2023-08-27T20:53:06.935129+0000","last_clean":"2023-08-27T20:53:06.935129+0000","last_became_active":"2023-08-27T20:51:18.101928+0000","last_became_peered":"2023-08-27T20:51:18.101928+0000","last_unstale":"2023-08-27T20:53:06.935129+0000","last_undegraded":"2023-08-27T20:53:06.935129+0000","last_fullsized":"2023-08-27T20:53:06.935129+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:39:54.277133+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,6],"acting":[1,2,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.7","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.935320+0000","last_change":"2023-08-27T20:51:18.128769+0000","last_active":"2023-08-27T20:53:06.935320+0000","last_peered":"2023-08-27T20:53:06.935320+0000","last_clean":"2023-08-27T20:53:06.935320+0000","last_became_active":"2023-08-27T20:51:18.128121+0000","last_became_peered":"2023-08-27T20:51:18.128121+0000","last_unstale":"2023-08-27T20:53:06.935320+0000","last_undegraded":"2023-08-27T20:53:06.935320+0000","last_fullsized":"2023-08-27T20:53:06.935320+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:54:21.500010+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,5,0],"acting":[1,5,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"2.3","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936024+0000","last_change":"2023-08-27T20:50:40.472409+0000","last_active":"2023-08-27T20:53:06.936024+0000","last_peered":"2023-08-27T20:53:06.936024+0000","last_clean":"2023-08-27T20:53:06.936024+0000","last_became_active":"2023-08-27T20:50:40.472101+0000","last_became_peered":"2023-08-27T20:50:40.472101+0000","last_unstale":"2023-08-27T20:53:06.936024+0000","last_undegraded":"2023-08-27T20:53:06.936024+0000","last_fullsized":"2023-08-27T20:53:06.936024+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:37:42.899620+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,2,7],"acting":[5,2,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"1.0","version":"73'139","reported_seq":323,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:53:38.633336+0000","last_change":"2023-08-27T20:50:22.534100+0000","last_active":"2023-08-27T20:53:38.633336+0000","last_peered":"2023-08-27T20:53:38.633336+0000","last_clean":"2023-08-27T20:53:38.633336+0000","last_became_active":"2023-08-27T20:50:22.533604+0000","last_became_peered":"2023-08-27T20:50:22.533604+0000","last_unstale":"2023-08-27T20:53:38.633336+0000","last_undegraded":"2023-08-27T20:53:38.633336+0000","last_fullsized":"2023-08-27T20:53:38.633336+0000","mapping_epoch":54,"log_start":"0'0","ondisk_log_start":"0'0","created":19,"last_epoch_clean":55,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:48:59.426866+0000","last_clean_scrub_stamp":"2023-08-27T20:48:59.426866+0000","objects_scrubbed":0,"log_size":139,"ondisk_log_size":139,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:50:03.653850+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,6],"acting":[7,0,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.2","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973266+0000","last_change":"2023-08-27T20:51:18.105105+0000","last_active":"2023-08-27T20:53:06.973266+0000","last_peered":"2023-08-27T20:53:06.973266+0000","last_clean":"2023-08-27T20:53:06.973266+0000","last_became_active":"2023-08-27T20:51:18.104814+0000","last_became_peered":"2023-08-27T20:51:18.104814+0000","last_unstale":"2023-08-27T20:53:06.973266+0000","last_undegraded":"2023-08-27T20:53:06.973266+0000","last_fullsized":"2023-08-27T20:53:06.973266+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:48:25.708690+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,5,6],"acting":[3,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.5","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.534246+0000","last_change":"2023-08-27T20:51:18.174265+0000","last_active":"2023-08-27T20:53:07.534246+0000","last_peered":"2023-08-27T20:53:07.534246+0000","last_clean":"2023-08-27T20:53:07.534246+0000","last_became_active":"2023-08-27T20:51:18.173954+0000","last_became_peered":"2023-08-27T20:51:18.173954+0000","last_unstale":"2023-08-27T20:53:07.534246+0000","last_undegraded":"2023-08-27T20:53:07.534246+0000","last_fullsized":"2023-08-27T20:53:07.534246+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:07:21.475116+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,4,7],"acting":[6,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"5.4","version":"73'314","reported_seq":631,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.148859+0000","last_change":"2023-08-27T20:51:59.072301+0000","last_active":"2023-08-27T20:55:34.148859+0000","last_peered":"2023-08-27T20:55:34.148859+0000","last_clean":"2023-08-27T20:55:34.148859+0000","last_became_active":"2023-08-27T20:51:59.071935+0000","last_became_peered":"2023-08-27T20:51:59.071935+0000","last_unstale":"2023-08-27T20:55:34.148859+0000","last_undegraded":"2023-08-27T20:55:34.148859+0000","last_fullsized":"2023-08-27T20:55:34.148859+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":314,"ondisk_log_size":314,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:50:21.546061+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":54782065,"num_objects":17,"num_object_clones":0,"num_object_copies":51,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":17,"num_whiteouts":0,"num_read":721,"num_read_kb":791548,"num_write":400,"num_write_kb":114123,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,2,5],"acting":[7,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"5.7","version":"73'472","reported_seq":864,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.017068+0000","last_change":"2023-08-27T20:51:59.069776+0000","last_active":"2023-08-27T20:55:34.017068+0000","last_peered":"2023-08-27T20:55:34.017068+0000","last_clean":"2023-08-27T20:55:34.017068+0000","last_became_active":"2023-08-27T20:51:59.069458+0000","last_became_peered":"2023-08-27T20:51:59.069458+0000","last_unstale":"2023-08-27T20:55:34.017068+0000","last_undegraded":"2023-08-27T20:55:34.017068+0000","last_fullsized":"2023-08-27T20:55:34.017068+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":472,"ondisk_log_size":472,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T07:36:57.001115+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":78838360,"num_objects":26,"num_object_clones":0,"num_object_copies":78,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":26,"num_whiteouts":0,"num_read":891,"num_read_kb":954397,"num_write":594,"num_write_kb":164847,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,1,0],"acting":[5,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"2.0","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067420+0000","last_change":"2023-08-27T20:50:41.004114+0000","last_active":"2023-08-27T20:53:07.067420+0000","last_peered":"2023-08-27T20:53:07.067420+0000","last_clean":"2023-08-27T20:53:07.067420+0000","last_became_active":"2023-08-27T20:50:41.003717+0000","last_became_peered":"2023-08-27T20:50:41.003717+0000","last_unstale":"2023-08-27T20:53:07.067420+0000","last_undegraded":"2023-08-27T20:53:07.067420+0000","last_fullsized":"2023-08-27T20:53:07.067420+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:24:46.799377+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,1,0],"acting":[7,1,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.1","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061368+0000","last_change":"2023-08-27T20:51:18.096016+0000","last_active":"2023-08-27T20:53:07.061368+0000","last_peered":"2023-08-27T20:53:07.061368+0000","last_clean":"2023-08-27T20:53:07.061368+0000","last_became_active":"2023-08-27T20:51:18.095736+0000","last_became_peered":"2023-08-27T20:51:18.095736+0000","last_unstale":"2023-08-27T20:53:07.061368+0000","last_undegraded":"2023-08-27T20:53:07.061368+0000","last_fullsized":"2023-08-27T20:53:07.061368+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:22:49.189530+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,3],"acting":[0,4,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.6","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061928+0000","last_change":"2023-08-27T20:51:18.132942+0000","last_active":"2023-08-27T20:53:07.061928+0000","last_peered":"2023-08-27T20:53:07.061928+0000","last_clean":"2023-08-27T20:53:07.061928+0000","last_became_active":"2023-08-27T20:51:18.132641+0000","last_became_peered":"2023-08-27T20:51:18.132641+0000","last_unstale":"2023-08-27T20:53:07.061928+0000","last_undegraded":"2023-08-27T20:53:07.061928+0000","last_fullsized":"2023-08-27T20:53:07.061928+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:16:24.405549+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,2],"acting":[0,1,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"2.5","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.067631+0000","last_change":"2023-08-27T20:50:41.004048+0000","last_active":"2023-08-27T20:53:07.067631+0000","last_peered":"2023-08-27T20:53:07.067631+0000","last_clean":"2023-08-27T20:53:07.067631+0000","last_became_active":"2023-08-27T20:50:41.003731+0000","last_became_peered":"2023-08-27T20:50:41.003731+0000","last_unstale":"2023-08-27T20:53:07.067631+0000","last_undegraded":"2023-08-27T20:53:07.067631+0000","last_fullsized":"2023-08-27T20:53:07.067631+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":58,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:22:58.339578+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[7,0,4],"acting":[7,0,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":7,"acting_primary":7,"purged_snaps":[]},{"pgid":"3.4","version":"0'0","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.935474+0000","last_change":"2023-08-27T20:51:18.101498+0000","last_active":"2023-08-27T20:53:06.935474+0000","last_peered":"2023-08-27T20:53:06.935474+0000","last_clean":"2023-08-27T20:53:06.935474+0000","last_became_active":"2023-08-27T20:51:18.101220+0000","last_became_peered":"2023-08-27T20:51:18.101220+0000","last_unstale":"2023-08-27T20:53:06.935474+0000","last_undegraded":"2023-08-27T20:53:06.935474+0000","last_fullsized":"2023-08-27T20:53:06.935474+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:27:28.524484+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,2,5],"acting":[1,2,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.3","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061560+0000","last_change":"2023-08-27T20:51:18.175452+0000","last_active":"2023-08-27T20:53:07.061560+0000","last_peered":"2023-08-27T20:53:07.061560+0000","last_clean":"2023-08-27T20:53:07.061560+0000","last_became_active":"2023-08-27T20:51:18.174003+0000","last_became_peered":"2023-08-27T20:51:18.174003+0000","last_unstale":"2023-08-27T20:53:07.061560+0000","last_undegraded":"2023-08-27T20:53:07.061560+0000","last_fullsized":"2023-08-27T20:53:07.061560+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:26:59.035718+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,7],"acting":[0,5,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"5.2","version":"73'574","reported_seq":1035,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.176063+0000","last_change":"2023-08-27T20:51:59.072999+0000","last_active":"2023-08-27T20:55:34.176063+0000","last_peered":"2023-08-27T20:55:34.176063+0000","last_clean":"2023-08-27T20:55:34.176063+0000","last_became_active":"2023-08-27T20:51:59.072729+0000","last_became_peered":"2023-08-27T20:51:59.072729+0000","last_unstale":"2023-08-27T20:55:34.176063+0000","last_undegraded":"2023-08-27T20:55:34.176063+0000","last_fullsized":"2023-08-27T20:55:34.176063+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":574,"ondisk_log_size":574,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:02:14.181356+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":82183415,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":1019,"num_read_kb":1089833,"num_write":727,"num_write_kb":207231,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,0,5],"acting":[6,0,5],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.7","version":"63'2","reported_seq":22,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973947+0000","last_change":"2023-08-27T20:51:18.103183+0000","last_active":"2023-08-27T20:53:06.973947+0000","last_peered":"2023-08-27T20:53:06.973947+0000","last_clean":"2023-08-27T20:53:06.973947+0000","last_became_active":"2023-08-27T20:51:18.102921+0000","last_became_peered":"2023-08-27T20:51:18.102921+0000","last_unstale":"2023-08-27T20:53:06.973947+0000","last_undegraded":"2023-08-27T20:53:06.973947+0000","last_fullsized":"2023-08-27T20:53:06.973947+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:47:12.209253+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"2.6","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936813+0000","last_change":"2023-08-27T20:50:40.474630+0000","last_active":"2023-08-27T20:53:06.936813+0000","last_peered":"2023-08-27T20:53:06.936813+0000","last_clean":"2023-08-27T20:53:06.936813+0000","last_became_active":"2023-08-27T20:50:40.474345+0000","last_became_peered":"2023-08-27T20:50:40.474345+0000","last_unstale":"2023-08-27T20:53:06.936813+0000","last_undegraded":"2023-08-27T20:53:06.936813+0000","last_fullsized":"2023-08-27T20:53:06.936813+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T06:35:03.409338+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[1,6,4],"acting":[1,6,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":1,"acting_primary":1,"purged_snaps":[]},{"pgid":"4.0","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973057+0000","last_change":"2023-08-27T20:51:18.175070+0000","last_active":"2023-08-27T20:53:06.973057+0000","last_peered":"2023-08-27T20:53:06.973057+0000","last_clean":"2023-08-27T20:53:06.973057+0000","last_became_active":"2023-08-27T20:51:18.174799+0000","last_became_peered":"2023-08-27T20:51:18.174799+0000","last_unstale":"2023-08-27T20:53:06.973057+0000","last_undegraded":"2023-08-27T20:53:06.973057+0000","last_fullsized":"2023-08-27T20:53:06.973057+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T00:04:36.156817+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,7,0],"acting":[3,7,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"5.1","version":"73'444","reported_seq":822,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.147577+0000","last_change":"2023-08-27T20:51:59.064651+0000","last_active":"2023-08-27T20:55:34.147577+0000","last_peered":"2023-08-27T20:55:34.147577+0000","last_clean":"2023-08-27T20:55:34.147577+0000","last_became_active":"2023-08-27T20:51:59.064368+0000","last_became_peered":"2023-08-27T20:51:59.064368+0000","last_unstale":"2023-08-27T20:55:34.147577+0000","last_undegraded":"2023-08-27T20:55:34.147577+0000","last_fullsized":"2023-08-27T20:55:34.147577+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":444,"ondisk_log_size":444,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T04:43:26.522172+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":72752543,"num_objects":24,"num_object_clones":0,"num_object_copies":72,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":24,"num_whiteouts":0,"num_read":841,"num_read_kb":904653,"num_write":561,"num_write_kb":158145,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,7],"acting":[4,3,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"2.7","version":"0'0","reported_seq":25,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.534332+0000","last_change":"2023-08-27T20:50:40.472176+0000","last_active":"2023-08-27T20:53:07.534332+0000","last_peered":"2023-08-27T20:53:07.534332+0000","last_clean":"2023-08-27T20:53:07.534332+0000","last_became_active":"2023-08-27T20:50:40.471979+0000","last_became_peered":"2023-08-27T20:50:40.471979+0000","last_unstale":"2023-08-27T20:53:07.534332+0000","last_undegraded":"2023-08-27T20:53:07.534332+0000","last_fullsized":"2023-08-27T20:53:07.534332+0000","mapping_epoch":56,"log_start":"0'0","ondisk_log_start":"0'0","created":56,"last_epoch_clean":57,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:50:39.459237+0000","last_clean_scrub_stamp":"2023-08-27T20:50:39.459237+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T22:16:03.929633+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[6,7,2],"acting":[6,7,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":6,"acting_primary":6,"purged_snaps":[]},{"pgid":"3.6","version":"63'1","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061919+0000","last_change":"2023-08-27T20:51:18.097033+0000","last_active":"2023-08-27T20:53:07.061919+0000","last_peered":"2023-08-27T20:53:07.061919+0000","last_clean":"2023-08-27T20:53:07.061919+0000","last_became_active":"2023-08-27T20:51:18.096760+0000","last_became_peered":"2023-08-27T20:51:18.096760+0000","last_unstale":"2023-08-27T20:53:07.061919+0000","last_undegraded":"2023-08-27T20:53:07.061919+0000","last_fullsized":"2023-08-27T20:53:07.061919+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:22:34.415568+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":1,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,1,4],"acting":[0,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976613+0000","last_change":"2023-08-27T20:51:18.124571+0000","last_active":"2023-08-27T20:53:06.976613+0000","last_peered":"2023-08-27T20:53:06.976613+0000","last_clean":"2023-08-27T20:53:06.976613+0000","last_became_active":"2023-08-27T20:51:18.124278+0000","last_became_peered":"2023-08-27T20:51:18.124278+0000","last_unstale":"2023-08-27T20:53:06.976613+0000","last_undegraded":"2023-08-27T20:53:06.976613+0000","last_fullsized":"2023-08-27T20:53:06.976613+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:27:56.736173+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,5,6],"acting":[4,5,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"5.0","version":"73'399","reported_seq":761,"reported_epoch":73,"state":"active+clean","last_fresh":"2023-08-27T20:55:34.161801+0000","last_change":"2023-08-27T20:51:59.070471+0000","last_active":"2023-08-27T20:55:34.161801+0000","last_peered":"2023-08-27T20:55:34.161801+0000","last_clean":"2023-08-27T20:55:34.161801+0000","last_became_active":"2023-08-27T20:51:59.070161+0000","last_became_peered":"2023-08-27T20:51:59.070161+0000","last_unstale":"2023-08-27T20:55:34.161801+0000","last_undegraded":"2023-08-27T20:55:34.161801+0000","last_fullsized":"2023-08-27T20:55:34.161801+0000","mapping_epoch":66,"log_start":"0'0","ondisk_log_start":"0'0","created":66,"last_epoch_clean":67,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:58.044718+0000","last_clean_scrub_stamp":"2023-08-27T20:51:58.044718+0000","objects_scrubbed":0,"log_size":399,"ondisk_log_size":399,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T21:41:18.198347+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":76904268,"num_objects":27,"num_object_clones":0,"num_object_copies":81,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":27,"num_whiteouts":0,"num_read":816,"num_read_kb":925452,"num_write":508,"num_write_kb":146163,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,1,4],"acting":[3,1,4],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1b","version":"63'1","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.061559+0000","last_change":"2023-08-27T20:51:18.098125+0000","last_active":"2023-08-27T20:53:07.061559+0000","last_peered":"2023-08-27T20:53:07.061559+0000","last_clean":"2023-08-27T20:53:07.061559+0000","last_became_active":"2023-08-27T20:51:18.097870+0000","last_became_peered":"2023-08-27T20:51:18.097870+0000","last_unstale":"2023-08-27T20:53:07.061559+0000","last_undegraded":"2023-08-27T20:53:07.061559+0000","last_fullsized":"2023-08-27T20:53:07.061559+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:32:32.322197+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,4,7],"acting":[0,4,7],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.1c","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.938384+0000","last_change":"2023-08-27T20:51:18.131530+0000","last_active":"2023-08-27T20:53:06.938384+0000","last_peered":"2023-08-27T20:53:06.938384+0000","last_clean":"2023-08-27T20:53:06.938384+0000","last_became_active":"2023-08-27T20:51:18.131263+0000","last_became_peered":"2023-08-27T20:51:18.131263+0000","last_unstale":"2023-08-27T20:53:06.938384+0000","last_undegraded":"2023-08-27T20:53:06.938384+0000","last_fullsized":"2023-08-27T20:53:06.938384+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:10:46.350911+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[2,1,3],"acting":[2,1,3],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":2,"acting_primary":2,"purged_snaps":[]},{"pgid":"3.1c","version":"63'2","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936283+0000","last_change":"2023-08-27T20:51:18.096858+0000","last_active":"2023-08-27T20:53:06.936283+0000","last_peered":"2023-08-27T20:53:06.936283+0000","last_clean":"2023-08-27T20:53:06.936283+0000","last_became_active":"2023-08-27T20:51:18.095892+0000","last_became_peered":"2023-08-27T20:51:18.095892+0000","last_unstale":"2023-08-27T20:53:06.936283+0000","last_undegraded":"2023-08-27T20:53:06.936283+0000","last_fullsized":"2023-08-27T20:53:06.936283+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:27:14.781996+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,1],"acting":[5,4,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1b","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976797+0000","last_change":"2023-08-27T20:51:18.129448+0000","last_active":"2023-08-27T20:53:06.976797+0000","last_peered":"2023-08-27T20:53:06.976797+0000","last_clean":"2023-08-27T20:53:06.976797+0000","last_became_active":"2023-08-27T20:51:18.129152+0000","last_became_peered":"2023-08-27T20:51:18.129152+0000","last_unstale":"2023-08-27T20:53:06.976797+0000","last_undegraded":"2023-08-27T20:53:06.976797+0000","last_fullsized":"2023-08-27T20:53:06.976797+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T03:52:03.853769+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,1],"acting":[4,3,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1d","version":"63'2","reported_seq":23,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.936597+0000","last_change":"2023-08-27T20:51:18.106356+0000","last_active":"2023-08-27T20:53:06.936597+0000","last_peered":"2023-08-27T20:53:06.936597+0000","last_clean":"2023-08-27T20:53:06.936597+0000","last_became_active":"2023-08-27T20:51:18.105473+0000","last_became_peered":"2023-08-27T20:51:18.105473+0000","last_unstale":"2023-08-27T20:53:06.936597+0000","last_undegraded":"2023-08-27T20:53:06.936597+0000","last_fullsized":"2023-08-27T20:53:06.936597+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":2,"ondisk_log_size":2,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:47:12.781410+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":1,"num_object_clones":0,"num_object_copies":3,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":1,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":3,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[5,4,6],"acting":[5,4,6],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":5,"acting_primary":5,"purged_snaps":[]},{"pgid":"4.1a","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976683+0000","last_change":"2023-08-27T20:51:18.129808+0000","last_active":"2023-08-27T20:53:06.976683+0000","last_peered":"2023-08-27T20:53:06.976683+0000","last_clean":"2023-08-27T20:53:06.976683+0000","last_became_active":"2023-08-27T20:51:18.129515+0000","last_became_peered":"2023-08-27T20:51:18.129515+0000","last_unstale":"2023-08-27T20:53:06.976683+0000","last_undegraded":"2023-08-27T20:53:06.976683+0000","last_fullsized":"2023-08-27T20:53:06.976683+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T05:45:42.761867+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,3,0],"acting":[4,3,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]},{"pgid":"3.1e","version":"63'1","reported_seq":21,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973793+0000","last_change":"2023-08-27T20:51:18.102113+0000","last_active":"2023-08-27T20:53:06.973793+0000","last_peered":"2023-08-27T20:53:06.973793+0000","last_clean":"2023-08-27T20:53:06.973793+0000","last_became_active":"2023-08-27T20:51:18.101848+0000","last_became_peered":"2023-08-27T20:51:18.101848+0000","last_unstale":"2023-08-27T20:53:06.973793+0000","last_undegraded":"2023-08-27T20:53:06.973793+0000","last_fullsized":"2023-08-27T20:53:06.973793+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":1,"ondisk_log_size":1,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T02:49:21.087984+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,6,2],"acting":[3,6,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"4.19","version":"0'0","reported_seq":19,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.973351+0000","last_change":"2023-08-27T20:51:18.133000+0000","last_active":"2023-08-27T20:53:06.973351+0000","last_peered":"2023-08-27T20:53:06.973351+0000","last_clean":"2023-08-27T20:53:06.973351+0000","last_became_active":"2023-08-27T20:51:18.132719+0000","last_became_peered":"2023-08-27T20:51:18.132719+0000","last_unstale":"2023-08-27T20:53:06.973351+0000","last_undegraded":"2023-08-27T20:53:06.973351+0000","last_fullsized":"2023-08-27T20:53:06.973351+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-28T23:00:48.680834+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[3,2,0],"acting":[3,2,0],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":3,"acting_primary":3,"purged_snaps":[]},{"pgid":"3.1f","version":"63'4","reported_seq":24,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:07.062005+0000","last_change":"2023-08-27T20:51:18.097797+0000","last_active":"2023-08-27T20:53:07.062005+0000","last_peered":"2023-08-27T20:53:07.062005+0000","last_clean":"2023-08-27T20:53:07.062005+0000","last_became_active":"2023-08-27T20:51:18.097523+0000","last_became_peered":"2023-08-27T20:51:18.097523+0000","last_unstale":"2023-08-27T20:53:07.062005+0000","last_undegraded":"2023-08-27T20:53:07.062005+0000","last_fullsized":"2023-08-27T20:53:07.062005+0000","mapping_epoch":60,"log_start":"0'0","ondisk_log_start":"0'0","created":60,"last_epoch_clean":61,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:17.077146+0000","last_clean_scrub_stamp":"2023-08-27T20:51:17.077146+0000","objects_scrubbed":0,"log_size":4,"ondisk_log_size":4,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T08:10:27.963795+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":554,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":4,"num_write_kb":1,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":1,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[0,5,2],"acting":[0,5,2],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":0,"acting_primary":0,"purged_snaps":[]},{"pgid":"4.18","version":"0'0","reported_seq":20,"reported_epoch":72,"state":"active+clean","last_fresh":"2023-08-27T20:53:06.976435+0000","last_change":"2023-08-27T20:51:18.131159+0000","last_active":"2023-08-27T20:53:06.976435+0000","last_peered":"2023-08-27T20:53:06.976435+0000","last_clean":"2023-08-27T20:53:06.976435+0000","last_became_active":"2023-08-27T20:51:18.130806+0000","last_became_peered":"2023-08-27T20:51:18.130806+0000","last_unstale":"2023-08-27T20:53:06.976435+0000","last_undegraded":"2023-08-27T20:53:06.976435+0000","last_fullsized":"2023-08-27T20:53:06.976435+0000","mapping_epoch":61,"log_start":"0'0","ondisk_log_start":"0'0","created":61,"last_epoch_clean":62,"parent":"0.0","parent_split_bits":0,"last_scrub":"0'0","last_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_deep_scrub":"0'0","last_deep_scrub_stamp":"2023-08-27T20:51:18.081719+0000","last_clean_scrub_stamp":"2023-08-27T20:51:18.081719+0000","objects_scrubbed":0,"log_size":0,"ondisk_log_size":0,"stats_invalid":false,"dirty_stats_invalid":false,"omap_stats_invalid":false,"hitset_stats_invalid":false,"hitset_bytes_stats_invalid":false,"pin_stats_invalid":false,"manifest_stats_invalid":false,"snaptrimq_len":0,"last_scrub_duration":0,"scrub_schedule":"periodic scrub scheduled @ 2023-08-29T01:26:50.146935+0000","scrub_duration":0,"objects_trimmed":0,"snaptrim_duration":0,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"up":[4,6,1],"acting":[4,6,1],"avail_no_missing":[],"object_location_counts":[],"blocked_by":[],"up_primary":4,"acting_primary":4,"purged_snaps":[]}],"pool_stats":[{"poolid":5,"num_pg":16,"stat_sum":{"num_bytes":1225601485,"num_objects":414,"num_object_clones":0,"num_object_copies":1242,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":414,"num_whiteouts":0,"num_read":14367,"num_read_kb":15343781,"num_write":9733,"num_write_kb":2763712,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":2282287104,"data_stored":2269704936,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":7687,"ondisk_log_size":7687,"up":48,"acting":48,"num_store_stats":8},{"poolid":4,"num_pg":32,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":96,"acting":96,"num_store_stats":8},{"poolid":3,"num_pg":32,"stat_sum":{"num_bytes":3378,"num_objects":22,"num_object_clones":0,"num_object_copies":66,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":22,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":51,"num_write_kb":19,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":14,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":98304,"data_stored":10134,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":45,"ondisk_log_size":45,"up":96,"acting":96,"num_store_stats":8},{"poolid":1,"num_pg":1,"stat_sum":{"num_bytes":590368,"num_objects":2,"num_object_clones":0,"num_object_copies":6,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":2,"num_whiteouts":0,"num_read":308,"num_read_kb":512,"num_write":199,"num_write_kb":2416,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":6,"num_bytes_recovered":1771104,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":1781760,"data_stored":1771104,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":139,"ondisk_log_size":139,"up":3,"acting":3,"num_store_stats":3},{"poolid":2,"num_pg":8,"stat_sum":{"num_bytes":0,"num_objects":0,"num_object_clones":0,"num_object_copies":0,"num_objects_missing_on_primary":0,"num_objects_missing":0,"num_objects_degraded":0,"num_objects_misplaced":0,"num_objects_unfound":0,"num_objects_dirty":0,"num_whiteouts":0,"num_read":0,"num_read_kb":0,"num_write":0,"num_write_kb":0,"num_scrub_errors":0,"num_shallow_scrub_errors":0,"num_deep_scrub_errors":0,"num_objects_recovered":0,"num_bytes_recovered":0,"num_keys_recovered":0,"num_objects_omap":0,"num_objects_hit_set_archive":0,"num_bytes_hit_set_archive":0,"num_flush":0,"num_flush_kb":0,"num_evict":0,"num_evict_kb":0,"num_promote":0,"num_flush_mode_high":0,"num_flush_mode_low":0,"num_evict_mode_some":0,"num_evict_mode_full":0,"num_objects_pinned":0,"num_legacy_snapsets":0,"num_large_omap_objects":0,"num_objects_manifest":0,"num_omap_bytes":0,"num_omap_keys":0,"num_objects_repaired":0},"store_stats":{"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},"log_size":0,"ondisk_log_size":0,"up":24,"acting":24,"num_store_stats":8}],"osd_stats":[{"osd":7,"up_from":51,"seq":219043332156,"num_pgs":35,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":252384,"kb_used_data":228080,"kb_used_omap":0,"kb_used_meta":22784,"kb_avail":93492768,"statfs":{"total":95995035648,"available":95736594432,"internally_reserved":0,"allocated":233553920,"data_stored":231800223,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":23330816},"hb_peers":[0,1,2,3,4,5,6],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46999999999999997,"5min":0.503,"15min":0.56000000000000005},"min":{"1min":0.30199999999999999,"5min":0.22700000000000001,"15min":0.22700000000000001},"max":{"1min":0.65400000000000003,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.49399999999999999},{"interface":"front","average":{"1min":0.47299999999999998,"5min":0.497,"15min":0.53700000000000003},"min":{"1min":0.34999999999999998,"5min":0.28499999999999998,"15min":0.28499999999999998},"max":{"1min":0.63100000000000001,"5min":0.69399999999999995,"15min":0.69399999999999995},"last":0.48299999999999998}]},{"osd":1,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.62,"5min":0.54000000000000004,"15min":0.56999999999999995},"min":{"1min":0.29699999999999999,"5min":0.29699999999999999,"15min":0.29699999999999999},"max":{"1min":3.3879999999999999,"5min":3.3879999999999999,"15min":3.3879999999999999},"last":0.42699999999999999},{"interface":"front","average":{"1min":0.501,"5min":0.50900000000000001,"15min":0.55900000000000005},"min":{"1min":0.35399999999999998,"5min":0.26000000000000001,"15min":0.26000000000000001},"max":{"1min":0.71199999999999997,"5min":0.83999999999999997,"15min":0.83999999999999997},"last":0.39900000000000002}]},{"osd":2,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.46600000000000003,"5min":0.52100000000000002,"15min":0.54700000000000004},"min":{"1min":0.34499999999999997,"5min":0.312,"15min":0.312},"max":{"1min":0.60499999999999998,"5min":3.8039999999999998,"15min":3.8039999999999998},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.499,"5min":0.55800000000000005,"15min":0.58399999999999996},"min":{"1min":0.34499999999999997,"5min":0.33400000000000002,"15min":0.33400000000000002},"max":{"1min":0.63500000000000001,"5min":3.8490000000000002,"15min":3.8490000000000002},"last":0.51200000000000001}]},{"osd":3,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.49299999999999999,"5min":0.53400000000000003,"15min":0.56699999999999995},"min":{"1min":0.371,"5min":0.29999999999999999,"15min":0.29999999999999999},"max":{"1min":0.64800000000000002,"5min":2.0150000000000001,"15min":2.0150000000000001},"last":0.41199999999999998},{"interface":"front","average":{"1min":0.55600000000000005,"5min":0.57499999999999996,"15min":0.58699999999999997},"min":{"1min":0.36599999999999999,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":0.69899999999999995,"5min":3.7490000000000001,"15min":3.7490000000000001},"last":0.46999999999999997}]},{"osd":4,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.496,"5min":0.47299999999999998,"15min":0.44800000000000001},"min":{"1min":0.27200000000000002,"5min":0.27200000000000002,"15min":0.27200000000000002},"max":{"1min":0.748,"5min":0.82099999999999995,"15min":0.82099999999999995},"last":0.34599999999999997},{"interface":"front","average":{"1min":0.53100000000000003,"5min":0.49299999999999999,"15min":0.45400000000000001},"min":{"1min":0.28799999999999998,"5min":0.26700000000000002,"15min":0.26700000000000002},"max":{"1min":0.71099999999999997,"5min":0.84399999999999997,"15min":0.84399999999999997},"last":0.45700000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.51900000000000002,"15min":0.48799999999999999},"min":{"1min":0.309,"5min":0.23999999999999999,"15min":0.23999999999999999},"max":{"1min":0.70799999999999996,"5min":0.876,"15min":0.876},"last":0.36399999999999999},{"interface":"front","average":{"1min":0.57599999999999996,"5min":0.53800000000000003,"15min":0.497},"min":{"1min":0.318,"5min":0.318,"15min":0.318},"max":{"1min":0.78800000000000003,"5min":0.90000000000000002,"15min":0.90000000000000002},"last":0.44500000000000001}]},{"osd":6,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.57899999999999996,"5min":0.55800000000000005,"15min":0.51600000000000001},"min":{"1min":0.32600000000000001,"5min":0.32600000000000001,"15min":0.32600000000000001},"max":{"1min":0.81999999999999995,"5min":0.94999999999999996,"15min":0.94999999999999996},"last":0.52200000000000002},{"interface":"front","average":{"1min":0.56799999999999995,"5min":0.54700000000000004,"15min":0.50700000000000001},"min":{"1min":0.33400000000000002,"5min":0.317,"15min":0.317},"max":{"1min":0.751,"5min":1.24,"15min":1.24},"last":0.503}]}]},{"osd":6,"up_from":44,"seq":188978561088,"num_pgs":31,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":209528,"kb_used_data":185372,"kb_used_omap":0,"kb_used_meta":22272,"kb_avail":93535624,"statfs":{"total":95995035648,"available":95780478976,"internally_reserved":0,"allocated":189820928,"data_stored":188412701,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":22806528},"hb_peers":[0,1,2,3,4,5,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.498,"5min":0.497,"15min":0.51100000000000001},"min":{"1min":0.30599999999999999,"5min":0.253,"15min":0.253},"max":{"1min":0.59899999999999998,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.47399999999999998},{"interface":"front","average":{"1min":0.54100000000000004,"5min":0.51300000000000001,"15min":0.54000000000000004},"min":{"1min":0.41199999999999998,"5min":0.23499999999999999,"15min":0.23499999999999999},"max":{"1min":0.69499999999999995,"5min":1.524,"15min":1.524},"last":0.374}]},{"osd":1,"last update":"Sun Aug 27 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.51800000000000002,"5min":0.51700000000000002,"15min":0.54800000000000004},"min":{"1min":0.38600000000000001,"5min":0.26900000000000002,"15min":0.26900000000000002},"max":{"1min":0.61199999999999999,"5min":1.583,"15min":1.583},"last":0.35799999999999998},{"interface":"front","average":{"1min":0.52300000000000002,"5min":0.51700000000000002,"15min":0.53300000000000003},"min":{"1min":0.35199999999999998,"5min":0.30399999999999999,"15min":0.30399999999999999},"max":{"1min":0.72399999999999998,"5min":0.755,"15min":0.755},"last":0.439}]},{"osd":2,"last update":"Sun Aug 27 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56999999999999995,"5min":0.55900000000000005,"15min":0.64500000000000002},"min":{"1min":0.41199999999999998,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.86899999999999999,"5min":3.613,"15min":3.613},"last":0.308},{"interface":"front","average":{"1min":0.53000000000000003,"5min":0.57699999999999996,"15min":0.73499999999999999},"min":{"1min":0.36199999999999999,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.68300000000000005,"5min":5.9009999999999998,"15min":5.9009999999999998},"last":0.40799999999999997}]},{"osd":3,"last update":"Sun Aug 27 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.53200000000000003,"5min":0.55300000000000005,"15min":0.64500000000000002},"min":{"1min":0.38700000000000001,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.70799999999999996,"5min":3.7389999999999999,"15min":3.7389999999999999},"last":0.52100000000000002},{"interface":"front","average":{"1min":0.57699999999999996,"5min":0.59899999999999998,"15min":0.67500000000000004},"min":{"1min":0.443,"5min":0.34000000000000002,"15min":0.34000000000000002},"max":{"1min":1.1000000000000001,"5min":3.4100000000000001,"15min":3.4100000000000001},"last":0.34399999999999997}]},{"osd":4,"last update":"Sun Aug 27 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.56100000000000005,"5min":0.48599999999999999,"15min":0.45200000000000001},"min":{"1min":0.40100000000000002,"5min":0.26600000000000001,"15min":0.26600000000000001},"max":{"1min":0.70999999999999996,"5min":0.753,"15min":0.753},"last":0.28699999999999998},{"interface":"front","average":{"1min":0.58299999999999996,"5min":0.502,"15min":0.47599999999999998},"min":{"1min":0.42799999999999999,"5min":0.27300000000000002,"15min":0.27300000000000002},"max":{"1min":0.74299999999999999,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.42299999999999999}]},{"osd":5,"last update":"Sun Aug 27 20:55:08 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.499,"15min":0.45500000000000002},"min":{"1min":0.34399999999999997,"5min":0.28699999999999998,"15min":0.28699999999999998},"max":{"1min":0.748,"5min":0.79700000000000004,"15min":0.79700000000000004},"last":0.32200000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.53900000000000003,"15min":0.51500000000000001},"min":{"1min":0.45300000000000001,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.78100000000000003,"5min":0.82299999999999995,"15min":0.82299999999999995},"last":0.49299999999999999}]},{"osd":7,"last update":"Sun Aug 27 20:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.58599999999999997,"5min":0.54400000000000004,"15min":0.53600000000000003},"min":{"1min":0.40999999999999998,"5min":0.309,"15min":0.309},"max":{"1min":0.73499999999999999,"5min":0.86499999999999999,"15min":0.86499999999999999},"last":0.46300000000000002},{"interface":"front","average":{"1min":0.65700000000000003,"5min":0.58499999999999996,"15min":0.56499999999999995},"min":{"1min":0.373,"5min":0.318,"15min":0.318},"max":{"1min":2.0249999999999999,"5min":2.0249999999999999,"15min":2.0249999999999999},"last":0.38500000000000001}]}]},{"osd":1,"up_from":13,"seq":55834574928,"num_pgs":37,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":253376,"kb_used_data":226392,"kb_used_omap":0,"kb_used_meta":24448,"kb_avail":93491776,"statfs":{"total":95995035648,"available":95735578624,"internally_reserved":0,"allocated":231825408,"data_stored":230158451,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":25034752},"hb_peers":[0,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:54:51 2023","interfaces":[{"interface":"back","average":{"1min":0.28499999999999998,"5min":0.41299999999999998,"15min":0.372},"min":{"1min":0.20599999999999999,"5min":0.19,"15min":0.19},"max":{"1min":0.39100000000000001,"5min":8.1530000000000005,"15min":8.1530000000000005},"last":0.26700000000000002},{"interface":"front","average":{"1min":0.28799999999999998,"5min":0.40300000000000002,"15min":0.373},"min":{"1min":0.22,"5min":0.218,"15min":0.218},"max":{"1min":0.377,"5min":1.9259999999999999,"15min":1.9259999999999999},"last":0.22}]},{"osd":2,"last update":"Sun Aug 27 20:55:06 2023","interfaces":[{"interface":"back","average":{"1min":0.33100000000000002,"5min":0.42799999999999999,"15min":0.432},"min":{"1min":0.254,"5min":0.221,"15min":0.221},"max":{"1min":0.44400000000000001,"5min":3.6480000000000001,"15min":3.6480000000000001},"last":0.28299999999999997},{"interface":"front","average":{"1min":0.35199999999999998,"5min":0.40500000000000003,"15min":0.44700000000000001},"min":{"1min":0.29099999999999998,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.48899999999999999,"5min":2.754,"15min":2.754},"last":0.34999999999999998}]},{"osd":3,"last update":"Sun Aug 27 20:55:26 2023","interfaces":[{"interface":"back","average":{"1min":0.33900000000000002,"5min":0.40400000000000003,"15min":0.45100000000000001},"min":{"1min":0.27100000000000002,"5min":0.23599999999999999,"15min":0.23599999999999999},"max":{"1min":0.49199999999999999,"5min":2.9049999999999998,"15min":2.9049999999999998},"last":0.36799999999999999},{"interface":"front","average":{"1min":0.36899999999999999,"5min":0.40600000000000003,"15min":0.44},"min":{"1min":0.26300000000000001,"5min":0.255,"15min":0.255},"max":{"1min":0.55600000000000005,"5min":2.7149999999999999,"15min":2.7149999999999999},"last":0.30099999999999999}]},{"osd":4,"last update":"Sun Aug 27 20:54:43 2023","interfaces":[{"interface":"back","average":{"1min":0.55300000000000005,"5min":0.59699999999999998,"15min":0.59599999999999997},"min":{"1min":0.441,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.86599999999999999,"5min":2,"15min":2},"last":0.60799999999999998},{"interface":"front","average":{"1min":0.59199999999999997,"5min":0.60199999999999998,"15min":0.59199999999999997},"min":{"1min":0.45500000000000002,"5min":0.34599999999999997,"15min":0.34599999999999997},"max":{"1min":0.82199999999999995,"5min":2.9780000000000002,"15min":2.9780000000000002},"last":0.59899999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:54:55 2023","interfaces":[{"interface":"back","average":{"1min":0.56000000000000005,"5min":0.57699999999999996,"15min":0.57199999999999995},"min":{"1min":0.439,"5min":0.33700000000000002,"15min":0.33700000000000002},"max":{"1min":0.75600000000000001,"5min":2.6280000000000001,"15min":2.6280000000000001},"last":0.61799999999999999},{"interface":"front","average":{"1min":0.56100000000000005,"5min":0.60999999999999999,"15min":0.61299999999999999},"min":{"1min":0.39900000000000002,"5min":0.34499999999999997,"15min":0.34499999999999997},"max":{"1min":0.83999999999999997,"5min":2.032,"15min":2.032},"last":0.64100000000000001}]},{"osd":6,"last update":"Sun Aug 27 20:55:16 2023","interfaces":[{"interface":"back","average":{"1min":0.58199999999999996,"5min":0.62,"15min":0.62},"min":{"1min":0.35199999999999998,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.80800000000000005,"5min":2.0539999999999998,"15min":2.0539999999999998},"last":0.68200000000000005},{"interface":"front","average":{"1min":0.58599999999999997,"5min":0.61599999999999999,"15min":0.62},"min":{"1min":0.40300000000000002,"5min":0.40300000000000002,"15min":0.40300000000000002},"max":{"1min":0.82799999999999996,"5min":2.681,"15min":2.681},"last":0.65300000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:55:32 2023","interfaces":[{"interface":"back","average":{"1min":0.54800000000000004,"5min":0.61099999999999999,"15min":0.61499999999999999},"min":{"1min":0.38800000000000001,"5min":0.36799999999999999,"15min":0.36799999999999999},"max":{"1min":0.84599999999999997,"5min":2.0739999999999998,"15min":2.0739999999999998},"last":0.71699999999999997},{"interface":"front","average":{"1min":0.52700000000000002,"5min":0.59899999999999998,"15min":0.60499999999999998},"min":{"1min":0.40799999999999997,"5min":0.40100000000000002,"15min":0.40100000000000002},"max":{"1min":0.79000000000000004,"5min":2.9470000000000001,"15min":2.9470000000000001},"last":0.70599999999999996}]}]},{"osd":0,"up_from":8,"seq":34359738451,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":399780,"kb_used_data":374264,"kb_used_omap":0,"kb_used_meta":23552,"kb_avail":93345372,"statfs":{"total":95995035648,"available":95585660928,"internally_reserved":0,"allocated":383246336,"data_stored":380773881,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":24117248},"hb_peers":[1,2,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":1,"last update":"Sun Aug 27 20:54:54 2023","interfaces":[{"interface":"back","average":{"1min":0.26400000000000001,"5min":0.312,"15min":0.34200000000000003},"min":{"1min":0.185,"5min":0.185,"15min":0.185},"max":{"1min":0.38,"5min":0.56699999999999995,"15min":0.57699999999999996},"last":0.32800000000000001},{"interface":"front","average":{"1min":0.313,"5min":0.33800000000000002,"15min":0.378},"min":{"1min":0.20899999999999999,"5min":0.19400000000000001,"15min":0.19400000000000001},"max":{"1min":0.41099999999999998,"5min":0.69999999999999996,"15min":0.69999999999999996},"last":0.52100000000000002}]},{"osd":2,"last update":"Sun Aug 27 20:55:11 2023","interfaces":[{"interface":"back","average":{"1min":0.316,"5min":0.36599999999999999,"15min":0.40699999999999997},"min":{"1min":0.23400000000000001,"5min":0.22900000000000001,"15min":0.22900000000000001},"max":{"1min":0.44800000000000001,"5min":2.141,"15min":2.141},"last":0.36599999999999999},{"interface":"front","average":{"1min":0.31900000000000001,"5min":0.38400000000000001,"15min":0.40799999999999997},"min":{"1min":0.20699999999999999,"5min":0.20699999999999999,"15min":0.20699999999999999},"max":{"1min":0.39300000000000002,"5min":2.4590000000000001,"15min":2.4590000000000001},"last":0.46400000000000002}]},{"osd":3,"last update":"Sun Aug 27 20:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.39300000000000002,"15min":0.45100000000000001},"min":{"1min":0.246,"5min":0.246,"15min":0.246},"max":{"1min":0.79100000000000004,"5min":2.4969999999999999,"15min":2.4969999999999999},"last":0.79100000000000004},{"interface":"front","average":{"1min":0.35499999999999998,"5min":0.375,"15min":0.433},"min":{"1min":0.24399999999999999,"5min":0.24399999999999999,"15min":0.24399999999999999},"max":{"1min":0.57099999999999995,"5min":0.70199999999999996,"15min":0.70199999999999996},"last":0.56399999999999995}]},{"osd":4,"last update":"Sun Aug 27 20:54:40 2023","interfaces":[{"interface":"back","average":{"1min":0.57199999999999995,"5min":0.55900000000000005,"15min":0.56899999999999995},"min":{"1min":0.377,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.93999999999999995,"5min":1.073,"15min":1.073},"last":0.75800000000000001},{"interface":"front","average":{"1min":0.60999999999999999,"5min":0.57899999999999996,"15min":0.58299999999999996},"min":{"1min":0.38600000000000001,"5min":0.249,"15min":0.249},"max":{"1min":0.95399999999999996,"5min":2.1030000000000002,"15min":2.1030000000000002},"last":0.59899999999999998}]},{"osd":5,"last update":"Sun Aug 27 20:54:59 2023","interfaces":[{"interface":"back","average":{"1min":0.53100000000000003,"5min":0.55900000000000005,"15min":0.55700000000000005},"min":{"1min":0.33500000000000002,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.96799999999999997,"5min":2.5259999999999998,"15min":2.5259999999999998},"last":0.503},{"interface":"front","average":{"1min":0.55700000000000005,"5min":0.55600000000000005,"15min":0.55100000000000005},"min":{"1min":0.42599999999999999,"5min":0.313,"15min":0.313},"max":{"1min":0.91300000000000003,"5min":0.91300000000000003,"15min":0.91300000000000003},"last":0.81899999999999995}]},{"osd":6,"last update":"Sun Aug 27 20:55:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54900000000000004,"5min":0.58099999999999996,"15min":0.55200000000000005},"min":{"1min":0.39200000000000002,"5min":0.34200000000000003,"15min":0.34200000000000003},"max":{"1min":0.80100000000000005,"5min":2.5510000000000002,"15min":2.5510000000000002},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.55800000000000005,"5min":0.56799999999999995,"15min":0.56000000000000005},"min":{"1min":0.42199999999999999,"5min":0.373,"15min":0.373},"max":{"1min":0.85799999999999998,"5min":0.88300000000000001,"15min":0.88300000000000001},"last":0.85599999999999998}]},{"osd":7,"last update":"Sun Aug 27 20:55:29 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.58999999999999997,"15min":0.55400000000000005},"min":{"1min":0.42399999999999999,"5min":0.34799999999999998,"15min":0.34799999999999998},"max":{"1min":0.996,"5min":2.5779999999999998,"15min":2.5779999999999998},"last":0.65400000000000003},{"interface":"front","average":{"1min":0.59599999999999997,"5min":0.58399999999999996,"15min":0.56999999999999995},"min":{"1min":0.41899999999999998,"5min":0.41299999999999998,"15min":0.41299999999999998},"max":{"1min":0.93999999999999995,"5min":0.93999999999999995,"15min":0.93999999999999995},"last":0.88700000000000001}]}]},{"osd":2,"up_from":18,"seq":77309411405,"num_pgs":25,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":334804,"kb_used_data":297660,"kb_used_omap":0,"kb_used_meta":29824,"kb_avail":93410348,"statfs":{"total":95995035648,"available":95652196352,"internally_reserved":0,"allocated":304803840,"data_stored":302686531,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":30539776},"hb_peers":[0,1,3,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:55:14 2023","interfaces":[{"interface":"back","average":{"1min":0.28899999999999998,"5min":0.32400000000000001,"15min":0.39600000000000002},"min":{"1min":0.219,"5min":0.17899999999999999,"15min":0.17899999999999999},"max":{"1min":0.39100000000000001,"5min":0.73799999999999999,"15min":0.73799999999999999},"last":0.39700000000000002},{"interface":"front","average":{"1min":0.29799999999999999,"5min":0.33300000000000002,"15min":0.38700000000000001},"min":{"1min":0.216,"5min":0.19800000000000001,"15min":0.19800000000000001},"max":{"1min":0.46300000000000002,"5min":0.875,"15min":0.875},"last":0.26600000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:55:14 2023","interfaces":[{"interface":"back","average":{"1min":0.30299999999999999,"5min":0.34399999999999997,"15min":0.39000000000000001},"min":{"1min":0.23000000000000001,"5min":0.219,"15min":0.219},"max":{"1min":0.51800000000000002,"5min":0.89900000000000002,"15min":0.89900000000000002},"last":0.27900000000000003},{"interface":"front","average":{"1min":0.32700000000000001,"5min":0.37,"15min":0.41999999999999998},"min":{"1min":0.24099999999999999,"5min":0.24099999999999999,"15min":0.24099999999999999},"max":{"1min":0.42999999999999999,"5min":0.622,"15min":0.622},"last":0.42199999999999999}]},{"osd":3,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.34899999999999998,"5min":0.36699999999999999,"15min":0.439},"min":{"1min":0.29199999999999998,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":0.45300000000000001,"5min":1.1719999999999999,"15min":1.1719999999999999},"last":0.312},{"interface":"front","average":{"1min":0.36099999999999999,"5min":0.38800000000000001,"15min":0.46100000000000002},"min":{"1min":0.27600000000000002,"5min":0.25800000000000001,"15min":0.25800000000000001},"max":{"1min":0.45000000000000001,"5min":0.79200000000000004,"15min":0.79200000000000004},"last":0.35799999999999998}]},{"osd":4,"last update":"Sun Aug 27 20:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.59799999999999998,"5min":0.57399999999999995,"15min":0.58799999999999997},"min":{"1min":0.41699999999999998,"5min":0.317,"15min":0.317},"max":{"1min":0.77400000000000002,"5min":1.377,"15min":1.377},"last":0.69799999999999995},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57599999999999996,"15min":0.59999999999999998},"min":{"1min":0.39600000000000002,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.83999999999999997,"5min":0.95899999999999996,"15min":0.95899999999999996},"last":0.64700000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:54:59 2023","interfaces":[{"interface":"back","average":{"1min":0.58899999999999997,"5min":0.56999999999999995,"15min":0.59799999999999998},"min":{"1min":0.38600000000000001,"5min":0.35599999999999998,"15min":0.35599999999999998},"max":{"1min":0.753,"5min":1.321,"15min":1.321},"last":0.63100000000000001},{"interface":"front","average":{"1min":0.61299999999999999,"5min":0.58399999999999996,"15min":0.59899999999999998},"min":{"1min":0.435,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":0.97399999999999998,"5min":0.97399999999999998,"15min":0.97399999999999998},"last":0.54300000000000004}]},{"osd":6,"last update":"Sun Aug 27 20:55:14 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.58299999999999996,"15min":0.58899999999999997},"min":{"1min":0.38100000000000001,"5min":0.33500000000000002,"15min":0.33500000000000002},"max":{"1min":1.01,"5min":1.01,"15min":1.01},"last":0.57399999999999995},{"interface":"front","average":{"1min":0.60899999999999999,"5min":0.59099999999999997,"15min":0.625},"min":{"1min":0.434,"5min":0.38,"15min":0.38},"max":{"1min":0.98899999999999999,"5min":0.98899999999999999,"15min":0.98899999999999999},"last":0.59799999999999998}]},{"osd":7,"last update":"Sun Aug 27 20:55:28 2023","interfaces":[{"interface":"back","average":{"1min":0.60499999999999998,"5min":0.60099999999999998,"15min":0.61299999999999999},"min":{"1min":0.45100000000000001,"5min":0.379,"15min":0.379},"max":{"1min":0.80600000000000005,"5min":1.353,"15min":1.353},"last":0.66400000000000003},{"interface":"front","average":{"1min":0.63600000000000001,"5min":0.61399999999999999,"15min":0.62},"min":{"1min":0.42699999999999999,"5min":0.36499999999999999,"15min":0.36499999999999999},"max":{"1min":1.034,"5min":1.034,"15min":1.034},"last":0.59199999999999997}]}]},{"osd":3,"up_from":25,"seq":107374182472,"num_pgs":34,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":163956,"kb_used_data":147560,"kb_used_omap":0,"kb_used_meta":15168,"kb_avail":93581196,"statfs":{"total":95995035648,"available":95827144704,"internally_reserved":0,"allocated":151101440,"data_stored":149826248,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":15532032},"hb_peers":[0,1,2,4,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.29899999999999999,"5min":0.32200000000000001,"15min":0.35499999999999998},"min":{"1min":0.21299999999999999,"5min":0.20100000000000001,"15min":0.20100000000000001},"max":{"1min":0.39300000000000002,"5min":0.60599999999999998,"15min":0.60599999999999998},"last":0.23100000000000001},{"interface":"front","average":{"1min":0.28899999999999998,"5min":0.34000000000000002,"15min":0.41799999999999998},"min":{"1min":0.217,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.35399999999999998,"5min":0.61399999999999999,"15min":0.628},"last":0.29599999999999999}]},{"osd":1,"last update":"Sun Aug 27 20:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.30399999999999999,"5min":0.34599999999999997,"15min":0.42399999999999999},"min":{"1min":0.22600000000000001,"5min":0.22600000000000001,"15min":0.22600000000000001},"max":{"1min":0.40000000000000002,"5min":0.65100000000000002,"15min":0.65100000000000002},"last":0.33400000000000002},{"interface":"front","average":{"1min":0.32600000000000001,"5min":0.34599999999999997,"15min":0.41199999999999998},"min":{"1min":0.21099999999999999,"5min":0.21099999999999999,"15min":0.21099999999999999},"max":{"1min":0.38,"5min":0.59899999999999998,"15min":0.66800000000000004},"last":0.27800000000000002}]},{"osd":2,"last update":"Sun Aug 27 20:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.34300000000000003,"5min":0.38200000000000001,"15min":0.45700000000000002},"min":{"1min":0.23000000000000001,"5min":0.23000000000000001,"15min":0.23000000000000001},"max":{"1min":0.40899999999999997,"5min":1.1100000000000001,"15min":1.1100000000000001},"last":0.34999999999999998},{"interface":"front","average":{"1min":0.33200000000000002,"5min":0.377,"15min":0.43099999999999999},"min":{"1min":0.22800000000000001,"5min":0.22800000000000001,"15min":0.22800000000000001},"max":{"1min":0.42099999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.26400000000000001}]},{"osd":4,"last update":"Sun Aug 27 20:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.54600000000000004,"5min":0.54100000000000004,"15min":0.56499999999999995},"min":{"1min":0.371,"5min":0.36599999999999999,"15min":0.36599999999999999},"max":{"1min":0.86599999999999999,"5min":0.92700000000000005,"15min":0.92700000000000005},"last":0.38400000000000001},{"interface":"front","average":{"1min":0.56299999999999994,"5min":0.57699999999999996,"15min":0.58499999999999996},"min":{"1min":0.38200000000000001,"5min":0.34999999999999998,"15min":0.34999999999999998},"max":{"1min":0.91300000000000003,"5min":2.907,"15min":2.907},"last":0.39200000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:54:54 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.57799999999999996,"15min":0.59999999999999998},"min":{"1min":0.373,"5min":0.35799999999999998,"15min":0.35799999999999998},"max":{"1min":0.93700000000000006,"5min":0.93700000000000006,"15min":0.93700000000000006},"last":0.40500000000000003},{"interface":"front","average":{"1min":0.58499999999999996,"5min":0.57999999999999996,"15min":0.60199999999999998},"min":{"1min":0.39900000000000002,"5min":0.35999999999999999,"15min":0.35999999999999999},"max":{"1min":0.95699999999999996,"5min":1.821,"15min":1.821},"last":0.47199999999999998}]},{"osd":6,"last update":"Sun Aug 27 20:55:12 2023","interfaces":[{"interface":"back","average":{"1min":0.623,"5min":0.58599999999999997,"15min":0.59599999999999997},"min":{"1min":0.39700000000000002,"5min":0.38100000000000001,"15min":0.38100000000000001},"max":{"1min":0.86299999999999999,"5min":0.94599999999999995,"15min":0.94599999999999995},"last":0.46100000000000002},{"interface":"front","average":{"1min":0.60399999999999998,"5min":0.58599999999999997,"15min":0.59399999999999997},"min":{"1min":0.373,"5min":0.373,"15min":0.373},"max":{"1min":0.83599999999999997,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.42599999999999999}]},{"osd":7,"last update":"Sun Aug 27 20:55:27 2023","interfaces":[{"interface":"back","average":{"1min":0.59399999999999997,"5min":0.58399999999999996,"15min":0.59399999999999997},"min":{"1min":0.41599999999999998,"5min":0.38700000000000001,"15min":0.38700000000000001},"max":{"1min":0.88100000000000001,"5min":0.92300000000000004,"15min":0.92300000000000004},"last":0.51800000000000002},{"interface":"front","average":{"1min":0.61699999999999999,"5min":0.61699999999999999,"15min":0.60799999999999998},"min":{"1min":0.434,"5min":0.39000000000000001,"15min":0.39000000000000001},"max":{"1min":0.89100000000000001,"5min":2.4319999999999999,"15min":2.4319999999999999},"last":0.496}]}]},{"osd":4,"up_from":30,"seq":128849018949,"num_pgs":41,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":406136,"kb_used_data":366356,"kb_used_omap":0,"kb_used_meta":33664,"kb_avail":93339016,"statfs":{"total":95995035648,"available":95579152384,"internally_reserved":0,"allocated":375148544,"data_stored":372614252,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":34471936},"hb_peers":[0,1,2,3,5,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.51700000000000002,"5min":0.496,"15min":0.54000000000000004},"min":{"1min":0.248,"5min":0.23300000000000001,"15min":0.23300000000000001},"max":{"1min":1.0229999999999999,"5min":1.0229999999999999,"15min":1.0229999999999999},"last":0.45400000000000001},{"interface":"front","average":{"1min":0.47799999999999998,"5min":0.48999999999999999,"15min":0.52000000000000002},"min":{"1min":0.314,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.59999999999999998,"5min":1.0189999999999999,"15min":1.0189999999999999},"last":0.51400000000000001}]},{"osd":1,"last update":"Sun Aug 27 20:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.60899999999999999,"5min":0.52400000000000002,"15min":0.54000000000000004},"min":{"1min":0.39900000000000002,"5min":0.30099999999999999,"15min":0.30099999999999999},"max":{"1min":1.615,"5min":1.615,"15min":1.615},"last":0.42299999999999999},{"interface":"front","average":{"1min":0.65600000000000003,"5min":0.53500000000000003,"15min":0.54300000000000004},"min":{"1min":0.32100000000000001,"5min":0.32100000000000001,"15min":0.32100000000000001},"max":{"1min":2.7450000000000001,"5min":2.7450000000000001,"15min":2.7450000000000001},"last":0.58899999999999997}]},{"osd":2,"last update":"Sun Aug 27 20:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.86799999999999999,"5min":0.57999999999999996,"15min":0.55800000000000005},"min":{"1min":0.27900000000000003,"5min":0.27900000000000003,"15min":0.27900000000000003},"max":{"1min":6.6420000000000003,"5min":6.6420000000000003,"15min":6.6420000000000003},"last":0.55400000000000005},{"interface":"front","average":{"1min":0.67500000000000004,"5min":0.53800000000000003,"15min":0.54000000000000004},"min":{"1min":0.35999999999999999,"5min":0.315,"15min":0.315},"max":{"1min":3.7240000000000002,"5min":3.7240000000000002,"15min":3.7240000000000002},"last":0.53800000000000003}]},{"osd":3,"last update":"Sun Aug 27 20:54:42 2023","interfaces":[{"interface":"back","average":{"1min":0.69299999999999995,"5min":0.54600000000000004,"15min":0.55200000000000005},"min":{"1min":0.34399999999999997,"5min":0.32900000000000001,"15min":0.32900000000000001},"max":{"1min":3.6139999999999999,"5min":3.6139999999999999,"15min":3.6139999999999999},"last":0.43099999999999999},{"interface":"front","average":{"1min":0.72899999999999998,"5min":0.55500000000000005,"15min":0.54300000000000004},"min":{"1min":0.40999999999999998,"5min":0.29299999999999998,"15min":0.29299999999999998},"max":{"1min":2.8900000000000001,"5min":2.8900000000000001,"15min":2.8900000000000001},"last":0.52800000000000002}]},{"osd":5,"last update":"Sun Aug 27 20:54:57 2023","interfaces":[{"interface":"back","average":{"1min":0.53300000000000003,"5min":0.45300000000000001,"15min":0.42699999999999999},"min":{"1min":0.33000000000000002,"5min":0.255,"15min":0.255},"max":{"1min":1.1579999999999999,"5min":1.1579999999999999,"15min":1.1579999999999999},"last":0.38500000000000001},{"interface":"front","average":{"1min":0.46200000000000002,"5min":0.45000000000000001,"15min":0.44},"min":{"1min":0.374,"5min":0.251,"15min":0.251},"max":{"1min":0.629,"5min":0.73499999999999999,"15min":0.73499999999999999},"last":0.47899999999999998}]},{"osd":6,"last update":"Sun Aug 27 20:55:11 2023","interfaces":[{"interface":"back","average":{"1min":0.54300000000000004,"5min":0.502,"15min":0.48399999999999999},"min":{"1min":0.40000000000000002,"5min":0.29799999999999999,"15min":0.29799999999999999},"max":{"1min":0.73399999999999999,"5min":0.73399999999999999,"15min":0.73399999999999999},"last":0.56499999999999995},{"interface":"front","average":{"1min":0.53700000000000003,"5min":0.51100000000000001,"15min":0.48099999999999998},"min":{"1min":0.314,"5min":0.30599999999999999,"15min":0.30599999999999999},"max":{"1min":0.83199999999999996,"5min":0.83199999999999996,"15min":0.83199999999999996},"last":0.40200000000000002}]},{"osd":7,"last update":"Sun Aug 27 20:54:31 2023","interfaces":[{"interface":"back","average":{"1min":0.52900000000000003,"5min":0.504,"15min":0.498},"min":{"1min":0.33000000000000002,"5min":0.32400000000000001,"15min":0.32400000000000001},"max":{"1min":0.69899999999999995,"5min":0.73699999999999999,"15min":0.73699999999999999},"last":0.46500000000000002},{"interface":"front","average":{"1min":0.59099999999999997,"5min":0.51600000000000001,"15min":0.49299999999999999},"min":{"1min":0.371,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.77800000000000002,"5min":0.77800000000000002,"15min":0.77800000000000002},"last":0.498}]}]},{"osd":5,"up_from":37,"seq":158913790019,"num_pgs":30,"num_osds":1,"num_per_pool_osds":1,"num_per_pool_omap_osds":1,"kb":93745152,"kb_used":449164,"kb_used_data":417820,"kb_used_omap":0,"kb_used_meta":23552,"kb_avail":93295988,"statfs":{"total":95995035648,"available":95535091712,"internally_reserved":0,"allocated":427847680,"data_stored":425090345,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":24117248},"hb_peers":[0,1,2,3,4,6,7],"snap_trim_queue_len":0,"num_snap_trimming":0,"num_shards_repaired":0,"op_queue_age_hist":{"histogram":[],"upper_bound":1},"perf_stat":{"commit_latency_ms":0,"apply_latency_ms":0,"commit_latency_ns":0,"apply_latency_ns":0},"alerts":[],"network_ping_times":[{"osd":0,"last update":"Sun Aug 27 20:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.46800000000000003,"5min":0.49199999999999999,"15min":0.52500000000000002},"min":{"1min":0.27600000000000002,"5min":0.215,"15min":0.215},"max":{"1min":0.71299999999999997,"5min":1.5780000000000001,"15min":1.5780000000000001},"last":0.54600000000000004},{"interface":"front","average":{"1min":0.48299999999999998,"5min":0.496,"15min":0.53200000000000003},"min":{"1min":0.29599999999999999,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.60999999999999999,"5min":0.78500000000000003,"15min":0.78500000000000003},"last":0.41799999999999998}]},{"osd":1,"last update":"Sun Aug 27 20:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.47699999999999998,"5min":0.48099999999999998,"15min":0.52000000000000002},"min":{"1min":0.34699999999999998,"5min":0.28100000000000003,"15min":0.28100000000000003},"max":{"1min":0.63400000000000001,"5min":0.73899999999999999,"15min":0.73899999999999999},"last":0.53100000000000003},{"interface":"front","average":{"1min":0.47599999999999998,"5min":0.504,"15min":0.53000000000000003},"min":{"1min":0.36599999999999999,"5min":0.30299999999999999,"15min":0.30299999999999999},"max":{"1min":0.63500000000000001,"5min":0.82499999999999996,"15min":0.82499999999999996},"last":0.371}]},{"osd":2,"last update":"Sun Aug 27 20:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.48999999999999999,"5min":0.52100000000000002,"15min":0.53500000000000003},"min":{"1min":0.34399999999999997,"5min":0.33900000000000002,"15min":0.33900000000000002},"max":{"1min":0.67400000000000004,"5min":1.5429999999999999,"15min":1.5429999999999999},"last":0.39800000000000002},{"interface":"front","average":{"1min":0.48199999999999998,"5min":0.5,"15min":0.54500000000000004},"min":{"1min":0.315,"5min":0.27800000000000002,"15min":0.27800000000000002},"max":{"1min":0.66400000000000003,"5min":0.81399999999999995,"15min":0.81399999999999995},"last":0.56799999999999995}]},{"osd":3,"last update":"Sun Aug 27 20:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.55700000000000005,"5min":0.53500000000000003,"15min":0.53600000000000003},"min":{"1min":0.42099999999999999,"5min":0.29499999999999998,"15min":0.29499999999999998},"max":{"1min":1.4470000000000001,"5min":1.4470000000000001,"15min":1.4470000000000001},"last":0.501},{"interface":"front","average":{"1min":0.505,"5min":0.53900000000000003,"15min":0.55200000000000005},"min":{"1min":0.371,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.63500000000000001,"5min":1.9510000000000001,"15min":1.9510000000000001},"last":0.432}]},{"osd":4,"last update":"Sun Aug 27 20:54:58 2023","interfaces":[{"interface":"back","average":{"1min":0.497,"5min":0.47899999999999998,"15min":0.45700000000000002},"min":{"1min":0.29899999999999999,"5min":0.28000000000000003,"15min":0.28000000000000003},"max":{"1min":0.68100000000000005,"5min":0.90300000000000002,"15min":0.90300000000000002},"last":0.51400000000000001},{"interface":"front","average":{"1min":0.50900000000000001,"5min":0.51800000000000002,"15min":0.48599999999999999},"min":{"1min":0.27000000000000002,"5min":0.27000000000000002,"15min":0.27000000000000002},"max":{"1min":0.78200000000000003,"5min":1.3520000000000001,"15min":1.3520000000000001},"last":0.34799999999999998}]},{"osd":6,"last update":"Sun Aug 27 20:55:11 2023","interfaces":[{"interface":"back","average":{"1min":0.57599999999999996,"5min":0.56499999999999995,"15min":0.53500000000000003},"min":{"1min":0.34100000000000003,"5min":0.309,"15min":0.309},"max":{"1min":0.73999999999999999,"5min":1.1990000000000001,"15min":1.1990000000000001},"last":0.45300000000000001},{"interface":"front","average":{"1min":0.56699999999999995,"5min":0.55700000000000005,"15min":0.54100000000000004},"min":{"1min":0.40899999999999997,"5min":0.35199999999999998,"15min":0.35199999999999998},"max":{"1min":0.68999999999999995,"5min":0.81200000000000006,"15min":0.81200000000000006},"last":0.47999999999999998}]},{"osd":7,"last update":"Sun Aug 27 20:55:30 2023","interfaces":[{"interface":"back","average":{"1min":0.51300000000000001,"5min":0.54700000000000004,"15min":0.57099999999999995},"min":{"1min":0.316,"5min":0.307,"15min":0.307},"max":{"1min":0.751,"5min":0.86899999999999999,"15min":0.86899999999999999},"last":0.46700000000000003},{"interface":"front","average":{"1min":0.56599999999999995,"5min":0.58899999999999997,"15min":0.58599999999999997},"min":{"1min":0.38700000000000001,"5min":0.35299999999999998,"15min":0.35299999999999998},"max":{"1min":0.72999999999999998,"5min":1.444,"15min":1.444},"last":0.58099999999999996}]}]}],"pool_statfs":[{"poolid":1,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":1,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":593920,"data_stored":590368,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":2,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":576,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":588,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":4096,"data_stored":90,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":24576,"data_stored":2754,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":8192,"data_stored":624,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":16384,"data_stored":2678,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":3,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":28672,"data_stored":2824,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":4,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":0,"data_stored":0,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":0,"total":0,"available":0,"internally_reserved":0,"allocated":381886464,"data_stored":379834634,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":1,"total":0,"available":0,"internally_reserved":0,"allocated":232562688,"data_stored":231291826,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":2,"total":0,"available":0,"internally_reserved":0,"allocated":300068864,"data_stored":298387696,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":3,"total":0,"available":0,"internally_reserved":0,"allocated":150339584,"data_stored":149477855,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":4,"total":0,"available":0,"internally_reserved":0,"allocated":375963648,"data_stored":373857055,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":5,"total":0,"available":0,"internally_reserved":0,"allocated":422862848,"data_stored":420543498,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":6,"total":0,"available":0,"internally_reserved":0,"allocated":188346368,"data_stored":187363359,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0},{"poolid":5,"osd":7,"total":0,"available":0,"internally_reserved":0,"allocated":230256640,"data_stored":228949013,"data_compressed":0,"data_compressed_allocated":0,"data_compressed_original":0,"omap_allocated":0,"internal_metadata":0}]}} 2023-08-27T20:55:39.438 INFO:tasks.cephadm.ceph_manager.ceph:removing pool_name unique_pool_0 2023-08-27T20:55:39.439 DEBUG:teuthology.orchestra.run.smithi093:> sudo /home/ubuntu/cephtest/cephadm --image quay.ceph.io/ceph-ci/ceph:quincy shell --fsid b79a52c0-451a-11ee-817b-1b7a5f1ba315 -- ceph osd pool rm unique_pool_0 unique_pool_0 --yes-i-really-really-mean-it 2023-08-27T20:55:39.705 INFO:teuthology.orchestra.run.smithi093.stderr:Inferring config /var/lib/ceph/b79a52c0-451a-11ee-817b-1b7a5f1ba315/mon.a/config 2023-08-27T20:55:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[111335]: pgmap v63: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4124424887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[111335]: from='client.25304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:55:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[118902]: pgmap v63: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4124424887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:39 smithi093 ceph-mon[118902]: from='client.25304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:55:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:39 smithi136 ceph-mon[113531]: pgmap v63: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 55 MiB/s rd, 8.8 MiB/s wr, 81 op/s 2023-08-27T20:55:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4124424887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:39 smithi136 ceph-mon[113531]: from='client.25304 -' entity='client.admin' cmd=[{"prefix": "pg dump", "target": ["mon-mgr", ""], "format": "json"}]: dispatch 2023-08-27T20:55:41.535 INFO:teuthology.orchestra.run.smithi093.stderr:pool 'unique_pool_0' removed 2023-08-27T20:55:41.897 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-mon[111335]: pgmap v64: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 9.5 MiB/s wr, 85 op/s 2023-08-27T20:55:41.897 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3229926907' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T20:55:41.897 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3906938524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:41.897 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-mon[118902]: pgmap v64: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 9.5 MiB/s wr, 85 op/s 2023-08-27T20:55:41.897 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3229926907' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T20:55:41.898 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3906938524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:41.898 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:41.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:41.898 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:41.632Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:41.923 INFO:teuthology.task.sequential:In sequential, running task print... 2023-08-27T20:55:41.923 INFO:teuthology.task.print:**** done rados/readwrite 5-workload 2023-08-27T20:55:41.923 DEBUG:teuthology.parallel:result is None 2023-08-27T20:55:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:41 smithi136 ceph-mon[113531]: pgmap v64: 89 pgs: 89 active+clean; 1.1 GiB data, 2.4 GiB used, 713 GiB / 715 GiB avail; 57 MiB/s rd, 9.5 MiB/s wr, 85 op/s 2023-08-27T20:55:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3229926907' entity='client.admin' cmd=[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]: dispatch 2023-08-27T20:55:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3906938524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3229926907' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T20:55:43.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:42 smithi136 ceph-mon[113531]: osdmap e74: 8 total, 8 up, 8 in 2023-08-27T20:55:43.620 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3229926907' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T20:55:43.621 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:42 smithi093 ceph-mon[111335]: osdmap e74: 8 total, 8 up, 8 in 2023-08-27T20:55:43.621 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3229926907' entity='client.admin' cmd='[{"prefix": "osd pool rm", "pool": "unique_pool_0", "pool2": "unique_pool_0", "yes_i_really_really_mean_it": true}]': finished 2023-08-27T20:55:43.621 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:42 smithi093 ceph-mon[118902]: osdmap e74: 8 total, 8 up, 8 in 2023-08-27T20:55:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:43 smithi136 ceph-mon[113531]: pgmap v66: 73 pgs: 73 active+clean; 580 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:55:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:43 smithi093 ceph-mon[111335]: pgmap v66: 73 pgs: 73 active+clean; 580 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:55:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:43 smithi093 ceph-mon[118902]: pgmap v66: 73 pgs: 73 active+clean; 580 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:55:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:43.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:43.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3047984165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3047984165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3047984165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:45 smithi093 ceph-mon[111335]: pgmap v67: 73 pgs: 73 active+clean; 580 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:55:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:55:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:55:45] "GET /metrics HTTP/1.1" 200 242675 "" "Prometheus/2.43.0" 2023-08-27T20:55:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:45 smithi093 ceph-mon[118902]: pgmap v67: 73 pgs: 73 active+clean; 580 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:55:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:45 smithi136 ceph-mon[113531]: pgmap v67: 73 pgs: 73 active+clean; 580 KiB data, 2.4 GiB used, 713 GiB / 715 GiB avail 2023-08-27T20:55:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1149326990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1149326990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1149326990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:47 smithi136 ceph-mon[113531]: pgmap v68: 73 pgs: 73 active+clean; 580 KiB data, 204 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:47 smithi093 ceph-mon[111335]: pgmap v68: 73 pgs: 73 active+clean; 580 KiB data, 204 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:47 smithi093 ceph-mon[118902]: pgmap v68: 73 pgs: 73 active+clean; 580 KiB data, 204 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:48.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/429311158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/429311158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/429311158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:49 smithi136 ceph-mon[113531]: pgmap v69: 73 pgs: 73 active+clean; 580 KiB data, 204 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:49 smithi093 ceph-mon[111335]: pgmap v69: 73 pgs: 73 active+clean; 580 KiB data, 204 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:49 smithi093 ceph-mon[118902]: pgmap v69: 73 pgs: 73 active+clean; 580 KiB data, 204 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:51 smithi136 ceph-mon[113531]: pgmap v70: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2948894376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:51 smithi093 ceph-mon[111335]: pgmap v70: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2948894376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:51 smithi093 ceph-mon[118902]: pgmap v70: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2948894376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:51.632Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:51.632Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:53 smithi136 ceph-mon[113531]: pgmap v71: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1332524400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:53 smithi093 ceph-mon[111335]: pgmap v71: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1332524400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:53 smithi093 ceph-mon[118902]: pgmap v71: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1332524400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:53.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:55:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:55:53.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:55:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:55:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:55:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:55:55] "GET /metrics HTTP/1.1" 200 240032 "" "Prometheus/2.43.0" 2023-08-27T20:55:55.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:55 smithi093 ceph-mon[111335]: pgmap v72: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:55.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1672366315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:55.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:55 smithi093 ceph-mon[118902]: pgmap v72: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:55.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1672366315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:55 smithi136 ceph-mon[113531]: pgmap v72: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1672366315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:57 smithi136 ceph-mon[113531]: pgmap v73: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:57 smithi093 ceph-mon[111335]: pgmap v73: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:57 smithi093 ceph-mon[118902]: pgmap v73: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/748886040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/748886040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/748886040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:55:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:55:59 smithi136 ceph-mon[113531]: pgmap v74: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:55:59 smithi093 ceph-mon[111335]: pgmap v74: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:55:59.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:55:59 smithi093 ceph-mon[118902]: pgmap v74: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/652492600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/652492600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/652492600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:01 smithi136 ceph-mon[113531]: pgmap v75: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:01 smithi093 ceph-mon[111335]: pgmap v75: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:01 smithi093 ceph-mon[118902]: pgmap v75: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:01.632Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:01.633Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:03 smithi136 ceph-mon[113531]: pgmap v76: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/565232066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:03 smithi093 ceph-mon[111335]: pgmap v76: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/565232066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:03 smithi093 ceph-mon[118902]: pgmap v76: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/565232066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:03.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:03.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:56:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:56:05] "GET /metrics HTTP/1.1" 200 240032 "" "Prometheus/2.43.0" 2023-08-27T20:56:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:05 smithi093 ceph-mon[111335]: pgmap v77: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2764515690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:05 smithi093 ceph-mon[118902]: pgmap v77: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2764515690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:05 smithi136 ceph-mon[113531]: pgmap v77: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2764515690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:07 smithi136 ceph-mon[113531]: pgmap v78: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2758108272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:07 smithi093 ceph-mon[111335]: pgmap v78: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2758108272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:07 smithi093 ceph-mon[118902]: pgmap v78: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2758108272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:09 smithi136 ceph-mon[113531]: pgmap v79: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:09 smithi093 ceph-mon[111335]: pgmap v79: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:09 smithi093 ceph-mon[118902]: pgmap v79: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1279044291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1279044291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1279044291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:11 smithi136 ceph-mon[113531]: pgmap v80: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:11 smithi093 ceph-mon[111335]: pgmap v80: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:11 smithi093 ceph-mon[118902]: pgmap v80: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:11.633Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:11.634Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/512464612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/512464612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/512464612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:13 smithi136 ceph-mon[113531]: pgmap v81: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:13 smithi093 ceph-mon[111335]: pgmap v81: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:13 smithi093 ceph-mon[118902]: pgmap v81: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:13.523Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:13.524Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:56:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:56:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:56:15.546 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:15 smithi093 ceph-mon[111335]: pgmap v82: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:15.875 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2607570309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:15.875 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:56:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:56:15] "GET /metrics HTTP/1.1" 200 240031 "" "Prometheus/2.43.0" 2023-08-27T20:56:15.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:15 smithi136 ceph-mon[113531]: pgmap v82: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:16.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2607570309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:16.035 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:15 smithi093 ceph-mon[118902]: pgmap v82: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:16.039 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2607570309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:56:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:56:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:56:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:56:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:56:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:56:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:56:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:17 smithi136 ceph-mon[113531]: pgmap v83: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2705308916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:17 smithi093 ceph-mon[111335]: pgmap v83: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2705308916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:17 smithi093 ceph-mon[118902]: pgmap v83: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2705308916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:19 smithi136 ceph-mon[113531]: pgmap v84: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3311110691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:19 smithi093 ceph-mon[111335]: pgmap v84: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3311110691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:19 smithi093 ceph-mon[118902]: pgmap v84: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3311110691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:21 smithi136 ceph-mon[113531]: pgmap v85: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:21 smithi093 ceph-mon[111335]: pgmap v85: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:21 smithi093 ceph-mon[118902]: pgmap v85: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:21.634Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:21.634Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/19648836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/19648836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/19648836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:23 smithi136 ceph-mon[113531]: pgmap v86: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:23 smithi093 ceph-mon[118902]: pgmap v86: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:23 smithi093 ceph-mon[111335]: pgmap v86: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:23.523Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:23.524Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/992473447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/992473447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/992473447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:25.642 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:56:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:56:25] "GET /metrics HTTP/1.1" 200 240026 "" "Prometheus/2.43.0" 2023-08-27T20:56:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:25 smithi136 ceph-mon[113531]: pgmap v87: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:25 smithi093 ceph-mon[111335]: pgmap v87: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:25 smithi093 ceph-mon[118902]: pgmap v87: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2091653279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2091653279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2091653279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:27 smithi136 ceph-mon[113531]: pgmap v88: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:27 smithi093 ceph-mon[111335]: pgmap v88: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:27 smithi093 ceph-mon[118902]: pgmap v88: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:29 smithi136 ceph-mon[113531]: pgmap v89: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4198184846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:29 smithi093 ceph-mon[111335]: pgmap v89: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4198184846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:29 smithi093 ceph-mon[118902]: pgmap v89: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4198184846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:31 smithi136 ceph-mon[113531]: pgmap v90: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1669937561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:31 smithi093 ceph-mon[111335]: pgmap v90: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1669937561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:31 smithi093 ceph-mon[118902]: pgmap v90: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1669937561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:31.634Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:31.635Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:33 smithi136 ceph-mon[113531]: pgmap v91: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:33 smithi093 ceph-mon[111335]: pgmap v91: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:33 smithi093 ceph-mon[118902]: pgmap v91: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:33.524Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:33.525Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/541609955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/541609955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/541609955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:35.667 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:56:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:56:35] "GET /metrics HTTP/1.1" 200 240026 "" "Prometheus/2.43.0" 2023-08-27T20:56:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:35 smithi136 ceph-mon[113531]: pgmap v92: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:35 smithi093 ceph-mon[111335]: pgmap v92: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:35 smithi093 ceph-mon[118902]: pgmap v92: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2686995096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2686995096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2686995096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:37 smithi136 ceph-mon[113531]: pgmap v93: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:37 smithi093 ceph-mon[111335]: pgmap v93: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:37 smithi093 ceph-mon[118902]: pgmap v93: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:38.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1080327574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1080327574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1080327574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:39 smithi136 ceph-mon[113531]: pgmap v94: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:39 smithi093 ceph-mon[111335]: pgmap v94: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:39 smithi093 ceph-mon[118902]: pgmap v94: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:41 smithi136 ceph-mon[113531]: pgmap v95: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1233869025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:41 smithi093 ceph-mon[111335]: pgmap v95: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1233869025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:41 smithi093 ceph-mon[118902]: pgmap v95: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1233869025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:42.186 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:41.635Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:42.186 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:41.636Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:43 smithi136 ceph-mon[113531]: pgmap v96: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3709673283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:43 smithi093 ceph-mon[111335]: pgmap v96: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3709673283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:43 smithi093 ceph-mon[118902]: pgmap v96: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3709673283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:43.525Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:43.526Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:45.688 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:56:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:56:45] "GET /metrics HTTP/1.1" 200 240024 "" "Prometheus/2.43.0" 2023-08-27T20:56:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:45 smithi093 ceph-mon[111335]: pgmap v97: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:45.999 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1492737081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:45.999 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:45 smithi093 ceph-mon[118902]: pgmap v97: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:45.999 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1492737081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:45 smithi136 ceph-mon[113531]: pgmap v97: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:46.310 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1492737081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:47 smithi093 ceph-mon[111335]: pgmap v98: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:47 smithi093 ceph-mon[118902]: pgmap v98: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:47 smithi136 ceph-mon[113531]: pgmap v98: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/557807690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:49.054 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/557807690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:49.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/557807690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:49 smithi093 ceph-mon[111335]: pgmap v99: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:50.519 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:49 smithi093 ceph-mon[118902]: pgmap v99: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:50.519 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:49 smithi136 ceph-mon[113531]: pgmap v99: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1114040200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:51.748 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1114040200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:51.749 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1114040200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:51 smithi093 ceph-mon[118902]: pgmap v100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:52.065 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:51 smithi093 ceph-mon[111335]: pgmap v100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:52.065 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:51.636Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:52.065 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:51.636Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:51 smithi136 ceph-mon[113531]: pgmap v100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-mon[111335]: pgmap v101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3411906528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-mon[118902]: pgmap v101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3411906528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:53.526Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:56:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:56:53.527Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:56:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:53 smithi136 ceph-mon[113531]: pgmap v101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3411906528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:56:55.710 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:56:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:56:55] "GET /metrics HTTP/1.1" 200 240024 "" "Prometheus/2.43.0" 2023-08-27T20:56:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:55 smithi093 ceph-mon[111335]: pgmap v102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2791515197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:55 smithi093 ceph-mon[118902]: pgmap v102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2791515197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:55 smithi136 ceph-mon[113531]: pgmap v102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2791515197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:57 smithi093 ceph-mon[118902]: pgmap v103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3499271956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:57 smithi093 ceph-mon[111335]: pgmap v103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3499271956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:57 smithi136 ceph-mon[113531]: pgmap v103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3499271956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:56:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:56:59 smithi093 ceph-mon[111335]: pgmap v104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:56:59.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:56:59 smithi093 ceph-mon[118902]: pgmap v104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:56:59 smithi136 ceph-mon[113531]: pgmap v104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/968575547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/968575547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/968575547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:01 smithi093 ceph-mon[111335]: pgmap v105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:01 smithi093 ceph-mon[118902]: pgmap v105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:01.637Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:01.638Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:01 smithi136 ceph-mon[113531]: pgmap v105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/699014248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/699014248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/699014248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:03 smithi093 ceph-mon[111335]: pgmap v106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:03 smithi093 ceph-mon[118902]: pgmap v106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:03.526Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:03.527Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:03 smithi136 ceph-mon[113531]: pgmap v106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2758904808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2758904808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2758904808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:57:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:57:05] "GET /metrics HTTP/1.1" 200 240024 "" "Prometheus/2.43.0" 2023-08-27T20:57:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:05 smithi093 ceph-mon[111335]: pgmap v107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:05 smithi093 ceph-mon[118902]: pgmap v107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:05 smithi136 ceph-mon[113531]: pgmap v107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:07 smithi136 ceph-mon[113531]: pgmap v108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1198899935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:07 smithi093 ceph-mon[111335]: pgmap v108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1198899935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:07 smithi093 ceph-mon[118902]: pgmap v108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1198899935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:09 smithi136 ceph-mon[113531]: pgmap v109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/289425315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:09 smithi093 ceph-mon[111335]: pgmap v109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/289425315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:09 smithi093 ceph-mon[118902]: pgmap v109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/289425315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:11 smithi136 ceph-mon[113531]: pgmap v110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:11 smithi093 ceph-mon[111335]: pgmap v110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:11 smithi093 ceph-mon[118902]: pgmap v110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:11.638Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:11.639Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1300150978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1300150978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1300150978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:13 smithi136 ceph-mon[113531]: pgmap v111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:13 smithi093 ceph-mon[111335]: pgmap v111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:13 smithi093 ceph-mon[118902]: pgmap v111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:13.526Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:13.527Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1572860689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1572860689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1572860689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:15 smithi093 ceph-mon[111335]: pgmap v112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:57:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:57:15] "GET /metrics HTTP/1.1" 200 240024 "" "Prometheus/2.43.0" 2023-08-27T20:57:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:15 smithi093 ceph-mon[118902]: pgmap v112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:15 smithi136 ceph-mon[113531]: pgmap v112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:17 smithi136 ceph-mon[113531]: pgmap v113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1562371657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:57:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[111335]: pgmap v113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1562371657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:57:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[118902]: pgmap v113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1562371657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:57:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:19 smithi136 ceph-mon[113531]: pgmap v114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3882773194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[111335]: pgmap v114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3882773194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[118902]: pgmap v114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3882773194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:57:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:57:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:57:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:57:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:57:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:57:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:57:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:21 smithi136 ceph-mon[113531]: pgmap v115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2604968313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:21 smithi093 ceph-mon[111335]: pgmap v115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2604968313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:21 smithi093 ceph-mon[118902]: pgmap v115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2604968313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:21.638Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:21.640Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:23 smithi136 ceph-mon[113531]: pgmap v116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:23 smithi093 ceph-mon[111335]: pgmap v116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:23 smithi093 ceph-mon[118902]: pgmap v116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:23.527Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:23.528Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3585389382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3585389382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3585389382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:25.691 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:57:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:57:25] "GET /metrics HTTP/1.1" 200 240017 "" "Prometheus/2.43.0" 2023-08-27T20:57:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:25 smithi093 ceph-mon[111335]: pgmap v117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:25 smithi093 ceph-mon[118902]: pgmap v117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:25 smithi136 ceph-mon[113531]: pgmap v117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2948114341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2948114341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:27.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2948114341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:27 smithi093 ceph-mon[111335]: pgmap v118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:27 smithi093 ceph-mon[118902]: pgmap v118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:27 smithi136 ceph-mon[113531]: pgmap v118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:29 smithi136 ceph-mon[113531]: pgmap v119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2064865097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:29 smithi093 ceph-mon[111335]: pgmap v119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2064865097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:29 smithi093 ceph-mon[118902]: pgmap v119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2064865097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:31 smithi136 ceph-mon[113531]: pgmap v120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2941666707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:31 smithi093 ceph-mon[111335]: pgmap v120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2941666707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:31 smithi093 ceph-mon[118902]: pgmap v120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2941666707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:31.639Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:31.641Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:33 smithi136 ceph-mon[113531]: pgmap v121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1537931645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:33 smithi093 ceph-mon[111335]: pgmap v121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1537931645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:33 smithi093 ceph-mon[118902]: pgmap v121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1537931645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:33.528Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:33.528Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:35 smithi093 ceph-mon[111335]: pgmap v122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:35 smithi093 ceph-mon[118902]: pgmap v122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:57:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:57:35] "GET /metrics HTTP/1.1" 200 240017 "" "Prometheus/2.43.0" 2023-08-27T20:57:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:35 smithi136 ceph-mon[113531]: pgmap v122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3663245166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3663245166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3663245166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:37 smithi136 ceph-mon[113531]: pgmap v123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:37 smithi093 ceph-mon[111335]: pgmap v123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:37 smithi093 ceph-mon[118902]: pgmap v123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:38.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3530024037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3530024037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3530024037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:39 smithi136 ceph-mon[113531]: pgmap v124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:39 smithi093 ceph-mon[111335]: pgmap v124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:39 smithi093 ceph-mon[118902]: pgmap v124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:40.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2649884886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2649884886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2649884886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:41 smithi136 ceph-mon[113531]: pgmap v125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:41 smithi093 ceph-mon[111335]: pgmap v125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:41 smithi093 ceph-mon[118902]: pgmap v125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:41.640Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:41.641Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:43 smithi136 ceph-mon[113531]: pgmap v126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1807716923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:43 smithi093 ceph-mon[111335]: pgmap v126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1807716923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:43 smithi093 ceph-mon[118902]: pgmap v126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1807716923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:43.528Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:43.529Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:57:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:57:45] "GET /metrics HTTP/1.1" 200 240023 "" "Prometheus/2.43.0" 2023-08-27T20:57:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:45 smithi093 ceph-mon[111335]: pgmap v127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3519465766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:45 smithi093 ceph-mon[118902]: pgmap v127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3519465766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:45 smithi136 ceph-mon[113531]: pgmap v127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3519465766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:47 smithi136 ceph-mon[113531]: pgmap v128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:47 smithi093 ceph-mon[111335]: pgmap v128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:47 smithi093 ceph-mon[118902]: pgmap v128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:48.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2014042453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2014042453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2014042453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:49 smithi136 ceph-mon[113531]: pgmap v129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:49 smithi093 ceph-mon[111335]: pgmap v129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:49 smithi093 ceph-mon[118902]: pgmap v129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/195924302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/195924302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/195924302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:51 smithi136 ceph-mon[113531]: pgmap v130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:51 smithi093 ceph-mon[111335]: pgmap v130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:51 smithi093 ceph-mon[118902]: pgmap v130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:51.640Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:51.641Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3758180207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3758180207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3758180207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:53 smithi136 ceph-mon[113531]: pgmap v131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:53 smithi093 ceph-mon[111335]: pgmap v131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:53 smithi093 ceph-mon[118902]: pgmap v131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:53.529Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:57:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:57:53.529Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:57:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:57:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:57:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:57:55] "GET /metrics HTTP/1.1" 200 239972 "" "Prometheus/2.43.0" 2023-08-27T20:57:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:55 smithi093 ceph-mon[111335]: pgmap v132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3820661009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:55 smithi093 ceph-mon[118902]: pgmap v132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3820661009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:55 smithi136 ceph-mon[113531]: pgmap v132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3820661009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:57 smithi136 ceph-mon[113531]: pgmap v133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1719609307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:57 smithi093 ceph-mon[111335]: pgmap v133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1719609307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:57 smithi093 ceph-mon[118902]: pgmap v133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1719609307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:59 smithi136 ceph-mon[113531]: pgmap v134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:57:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/390799400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:59 smithi093 ceph-mon[111335]: pgmap v134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:57:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/390799400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:57:59.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:59 smithi093 ceph-mon[118902]: pgmap v134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:57:59.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:57:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/390799400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:01 smithi136 ceph-mon[113531]: pgmap v135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:01 smithi093 ceph-mon[111335]: pgmap v135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:01 smithi093 ceph-mon[118902]: pgmap v135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:01.641Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:01.642Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2861435353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2861435353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:02.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2861435353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:03 smithi136 ceph-mon[113531]: pgmap v136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:03 smithi093 ceph-mon[111335]: pgmap v136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:03 smithi093 ceph-mon[118902]: pgmap v136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:03.529Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:03.530Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3810940638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3810940638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3810940638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:05.629 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:58:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:58:05] "GET /metrics HTTP/1.1" 200 239972 "" "Prometheus/2.43.0" 2023-08-27T20:58:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:05 smithi136 ceph-mon[113531]: pgmap v137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:05 smithi093 ceph-mon[111335]: pgmap v137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:05 smithi093 ceph-mon[118902]: pgmap v137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:07 smithi136 ceph-mon[113531]: pgmap v138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3481058266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:07 smithi093 ceph-mon[111335]: pgmap v138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3481058266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:07 smithi093 ceph-mon[118902]: pgmap v138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3481058266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:09 smithi136 ceph-mon[113531]: pgmap v139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2531860567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:09 smithi093 ceph-mon[118902]: pgmap v139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2531860567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:09 smithi093 ceph-mon[111335]: pgmap v139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2531860567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:11 smithi136 ceph-mon[113531]: pgmap v140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/474043295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:11 smithi093 ceph-mon[111335]: pgmap v140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/474043295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:11 smithi093 ceph-mon[118902]: pgmap v140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/474043295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:11.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:11.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:13 smithi136 ceph-mon[113531]: pgmap v141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:13 smithi093 ceph-mon[111335]: pgmap v141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:13 smithi093 ceph-mon[118902]: pgmap v141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:13.530Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:13.531Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4183109515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4183109515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4183109515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:15 smithi093 ceph-mon[111335]: pgmap v142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:58:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:58:15] "GET /metrics HTTP/1.1" 200 239976 "" "Prometheus/2.43.0" 2023-08-27T20:58:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:15 smithi093 ceph-mon[118902]: pgmap v142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:15 smithi136 ceph-mon[113531]: pgmap v142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3522982687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3522982687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3522982687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:17 smithi136 ceph-mon[113531]: pgmap v143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:17 smithi093 ceph-mon[118902]: pgmap v143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:17 smithi093 ceph-mon[111335]: pgmap v143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:19 smithi136 ceph-mon[113531]: pgmap v144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/535387695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:19 smithi093 ceph-mon[111335]: pgmap v144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/535387695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:19 smithi093 ceph-mon[118902]: pgmap v144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/535387695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:58:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:58:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:58:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:21.882 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:21 smithi136 ceph-mon[113531]: pgmap v145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:21.882 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3758677940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:21 smithi093 ceph-mon[111335]: pgmap v145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3758677940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:21 smithi093 ceph-mon[118902]: pgmap v145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3758677940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:21.643Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:21.644Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:23.796 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:23.531Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:23.797 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:23.532Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:23 smithi136 ceph-mon[113531]: pgmap v146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:58:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:58:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3969840135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[111335]: pgmap v146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3969840135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[118902]: pgmap v146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:58:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:58:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:58:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3969840135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:58:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:58:25] "GET /metrics HTTP/1.1" 200 239977 "" "Prometheus/2.43.0" 2023-08-27T20:58:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:25 smithi136 ceph-mon[113531]: pgmap v147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3246514712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:25 smithi093 ceph-mon[111335]: pgmap v147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3246514712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:25 smithi093 ceph-mon[118902]: pgmap v147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3246514712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:27 smithi136 ceph-mon[113531]: pgmap v148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:27 smithi093 ceph-mon[111335]: pgmap v148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:27 smithi093 ceph-mon[118902]: pgmap v148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:29.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3479532315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3479532315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3479532315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:29 smithi136 ceph-mon[113531]: pgmap v149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:29 smithi093 ceph-mon[111335]: pgmap v149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:29 smithi093 ceph-mon[118902]: pgmap v149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3914948746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3914948746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3914948746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:31 smithi093 ceph-mon[111335]: pgmap v150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:31 smithi093 ceph-mon[118902]: pgmap v150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:31.644Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:31.645Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:31 smithi136 ceph-mon[113531]: pgmap v150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:33.815 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:33.531Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:33.816 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:33.533Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:33 smithi136 ceph-mon[113531]: pgmap v151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3759577127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:33 smithi093 ceph-mon[111335]: pgmap v151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3759577127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:33 smithi093 ceph-mon[118902]: pgmap v151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3759577127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:58:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:58:35] "GET /metrics HTTP/1.1" 200 239977 "" "Prometheus/2.43.0" 2023-08-27T20:58:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:35 smithi136 ceph-mon[113531]: pgmap v152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2525235331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:35 smithi093 ceph-mon[118902]: pgmap v152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2525235331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:35 smithi093 ceph-mon[111335]: pgmap v152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2525235331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:37 smithi136 ceph-mon[113531]: pgmap v153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1019048273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:37 smithi093 ceph-mon[111335]: pgmap v153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1019048273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:37 smithi093 ceph-mon[118902]: pgmap v153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1019048273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:39 smithi136 ceph-mon[113531]: pgmap v154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:39 smithi093 ceph-mon[111335]: pgmap v154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:39 smithi093 ceph-mon[118902]: pgmap v154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:40.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/528462191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/528462191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/528462191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:41 smithi136 ceph-mon[113531]: pgmap v155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:41 smithi093 ceph-mon[111335]: pgmap v155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:41 smithi093 ceph-mon[118902]: pgmap v155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:41.644Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:41.645Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4262467233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4262467233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4262467233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:43 smithi136 ceph-mon[113531]: pgmap v156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:43 smithi093 ceph-mon[111335]: pgmap v156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:43.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:43 smithi093 ceph-mon[118902]: pgmap v156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:43.993 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:43.532Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:43.993 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:43.533Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:58:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:58:45] "GET /metrics HTTP/1.1" 200 239974 "" "Prometheus/2.43.0" 2023-08-27T20:58:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:45 smithi093 ceph-mon[111335]: pgmap v157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1596744221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:45 smithi093 ceph-mon[118902]: pgmap v157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1596744221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:45 smithi136 ceph-mon[113531]: pgmap v157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1596744221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:47 smithi136 ceph-mon[113531]: pgmap v158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3649705385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:47 smithi093 ceph-mon[111335]: pgmap v158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3649705385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:47 smithi093 ceph-mon[118902]: pgmap v158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3649705385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:49 smithi136 ceph-mon[113531]: pgmap v159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:49 smithi093 ceph-mon[118902]: pgmap v159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:49 smithi093 ceph-mon[111335]: pgmap v159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2417504112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2417504112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2417504112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:51.647 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:51 smithi093 ceph-mon[111335]: pgmap v160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:51.647 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:51 smithi093 ceph-mon[118902]: pgmap v160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:51.647 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:51.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:51.647 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:51.646Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:51 smithi136 ceph-mon[113531]: pgmap v160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/441539768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/441539768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/441539768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:53 smithi136 ceph-mon[113531]: pgmap v161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:53 smithi093 ceph-mon[111335]: pgmap v161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:53 smithi093 ceph-mon[118902]: pgmap v161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:53.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:58:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:58:53.534Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:58:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3147874358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3147874358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:58:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3147874358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:55 smithi093 ceph-mon[111335]: pgmap v162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:55 smithi093 ceph-mon[118902]: pgmap v162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:58:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:58:55] "GET /metrics HTTP/1.1" 200 239976 "" "Prometheus/2.43.0" 2023-08-27T20:58:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:55 smithi136 ceph-mon[113531]: pgmap v162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:57 smithi136 ceph-mon[113531]: pgmap v163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3738458257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:57 smithi093 ceph-mon[111335]: pgmap v163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3738458257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:57 smithi093 ceph-mon[118902]: pgmap v163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3738458257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:59 smithi136 ceph-mon[113531]: pgmap v164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:58:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2823993024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:59 smithi093 ceph-mon[111335]: pgmap v164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:58:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2823993024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:58:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:59 smithi093 ceph-mon[118902]: pgmap v164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:58:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:58:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2823993024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:01 smithi136 ceph-mon[113531]: pgmap v165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1877359983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:01 smithi093 ceph-mon[111335]: pgmap v165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1877359983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:01 smithi093 ceph-mon[118902]: pgmap v165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1877359983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:01.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:01.646Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:03 smithi136 ceph-mon[113531]: pgmap v166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:03 smithi093 ceph-mon[111335]: pgmap v166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:03 smithi093 ceph-mon[118902]: pgmap v166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:03.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:03.535Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/944867734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/944867734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/944867734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:05.613 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:59:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:59:05] "GET /metrics HTTP/1.1" 200 239976 "" "Prometheus/2.43.0" 2023-08-27T20:59:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:05 smithi136 ceph-mon[113531]: pgmap v167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:05 smithi093 ceph-mon[111335]: pgmap v167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:05 smithi093 ceph-mon[118902]: pgmap v167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:06.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/926732512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/926732512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:06.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/926732512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:07 smithi136 ceph-mon[113531]: pgmap v168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:07 smithi093 ceph-mon[111335]: pgmap v168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:07 smithi093 ceph-mon[118902]: pgmap v168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:09 smithi136 ceph-mon[113531]: pgmap v169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3977926688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:09 smithi093 ceph-mon[111335]: pgmap v169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3977926688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:09 smithi093 ceph-mon[118902]: pgmap v169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3977926688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:11 smithi136 ceph-mon[113531]: pgmap v170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2729348790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:11 smithi093 ceph-mon[118902]: pgmap v170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2729348790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:11 smithi093 ceph-mon[111335]: pgmap v170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2729348790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:11.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:11.647Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:13 smithi136 ceph-mon[113531]: pgmap v171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3148487539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:13 smithi093 ceph-mon[111335]: pgmap v171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3148487539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:13 smithi093 ceph-mon[118902]: pgmap v171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3148487539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:13.534Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:13.535Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:15 smithi093 ceph-mon[111335]: pgmap v172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:15 smithi093 ceph-mon[118902]: pgmap v172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:59:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:59:15] "GET /metrics HTTP/1.1" 200 239973 "" "Prometheus/2.43.0" 2023-08-27T20:59:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:15 smithi136 ceph-mon[113531]: pgmap v172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1047328701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1047328701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1047328701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:17 smithi136 ceph-mon[113531]: pgmap v173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:17 smithi093 ceph-mon[111335]: pgmap v173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:17 smithi093 ceph-mon[118902]: pgmap v173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3628167760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3628167760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3628167760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:19 smithi136 ceph-mon[113531]: pgmap v174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:19 smithi093 ceph-mon[111335]: pgmap v174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:19 smithi093 ceph-mon[118902]: pgmap v174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:21 smithi136 ceph-mon[113531]: pgmap v175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1246314989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:21 smithi093 ceph-mon[111335]: pgmap v175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1246314989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:21 smithi093 ceph-mon[118902]: pgmap v175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1246314989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:21.646Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:21.647Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:23 smithi136 ceph-mon[113531]: pgmap v176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3446051091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:59:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-mon[111335]: pgmap v176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3446051091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:59:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-mon[118902]: pgmap v176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3446051091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T20:59:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:23.535Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:23.536Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:59:25] "GET /metrics HTTP/1.1" 200 239972 "" "Prometheus/2.43.0" 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: pgmap v177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1960101258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:59:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: pgmap v177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1960101258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: pgmap v177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1960101258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T20:59:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T20:59:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T20:59:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:27 smithi136 ceph-mon[113531]: pgmap v178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1877740532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:27 smithi093 ceph-mon[111335]: pgmap v178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1877740532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:27 smithi093 ceph-mon[118902]: pgmap v178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1877740532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:29 smithi136 ceph-mon[113531]: pgmap v179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:29 smithi093 ceph-mon[111335]: pgmap v179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:29 smithi093 ceph-mon[118902]: pgmap v179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3522931531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3522931531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3522931531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:31 smithi093 ceph-mon[111335]: pgmap v180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:31 smithi093 ceph-mon[118902]: pgmap v180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:31.647Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:31.648Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:31 smithi136 ceph-mon[113531]: pgmap v180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2884639509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2884639509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2884639509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:33.816 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:33.535Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:33.816 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:33.536Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:33 smithi136 ceph-mon[113531]: pgmap v181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:33 smithi093 ceph-mon[118902]: pgmap v181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:33 smithi093 ceph-mon[111335]: pgmap v181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:59:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:59:35] "GET /metrics HTTP/1.1" 200 239972 "" "Prometheus/2.43.0" 2023-08-27T20:59:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:35 smithi136 ceph-mon[113531]: pgmap v182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3589135612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:35 smithi093 ceph-mon[111335]: pgmap v182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3589135612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:35 smithi093 ceph-mon[118902]: pgmap v182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3589135612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:37 smithi136 ceph-mon[113531]: pgmap v183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1664257481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:37 smithi093 ceph-mon[111335]: pgmap v183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1664257481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:37 smithi093 ceph-mon[118902]: pgmap v183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1664257481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:39 smithi136 ceph-mon[113531]: pgmap v184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:39 smithi093 ceph-mon[111335]: pgmap v184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:39 smithi093 ceph-mon[118902]: pgmap v184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:40.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1583137499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1583137499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1583137499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:41 smithi136 ceph-mon[113531]: pgmap v185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:41 smithi093 ceph-mon[111335]: pgmap v185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:41 smithi093 ceph-mon[118902]: pgmap v185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:41.648Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:41.649Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3541386740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3541386740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3541386740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:43 smithi136 ceph-mon[113531]: pgmap v186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:43 smithi093 ceph-mon[111335]: pgmap v186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:43 smithi093 ceph-mon[118902]: pgmap v186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:43.536Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:43.537Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2859793747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2859793747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2859793747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:45 smithi093 ceph-mon[111335]: pgmap v187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:59:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:59:45] "GET /metrics HTTP/1.1" 200 239969 "" "Prometheus/2.43.0" 2023-08-27T20:59:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:45 smithi093 ceph-mon[118902]: pgmap v187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:45 smithi136 ceph-mon[113531]: pgmap v187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:47 smithi136 ceph-mon[113531]: pgmap v188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/531811704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:47 smithi093 ceph-mon[111335]: pgmap v188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/531811704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:47 smithi093 ceph-mon[118902]: pgmap v188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/531811704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:49 smithi136 ceph-mon[113531]: pgmap v189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1282753352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:49 smithi093 ceph-mon[111335]: pgmap v189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1282753352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:49 smithi093 ceph-mon[118902]: pgmap v189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1282753352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:51 smithi136 ceph-mon[113531]: pgmap v190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/993431570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:51 smithi093 ceph-mon[111335]: pgmap v190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/993431570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:51 smithi093 ceph-mon[118902]: pgmap v190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/993431570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:51.648Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:51.649Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:53 smithi136 ceph-mon[113531]: pgmap v191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:53 smithi093 ceph-mon[111335]: pgmap v191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:53 smithi093 ceph-mon[118902]: pgmap v191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T20:59:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:53.536Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 20:59:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T20:59:53.537Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T20:59:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/775457542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/775457542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/775457542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:55 smithi093 ceph-mon[111335]: pgmap v192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 20:59:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:20:59:55] "GET /metrics HTTP/1.1" 200 239970 "" "Prometheus/2.43.0" 2023-08-27T20:59:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:55 smithi093 ceph-mon[118902]: pgmap v192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:55 smithi136 ceph-mon[113531]: pgmap v192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2846899032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2846899032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2846899032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:57 smithi136 ceph-mon[113531]: pgmap v193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:57 smithi093 ceph-mon[111335]: pgmap v193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:57 smithi093 ceph-mon[118902]: pgmap v193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:59 smithi136 ceph-mon[113531]: pgmap v194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 20:59:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340087466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:59 smithi093 ceph-mon[111335]: pgmap v194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 20:59:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340087466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T20:59:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:59 smithi093 ceph-mon[118902]: pgmap v194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T20:59:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 20:59:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340087466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:00 smithi136 ceph-mon[113531]: Health detail: HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:00:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:00 smithi136 ceph-mon[113531]: [WRN] UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T21:00:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:00 smithi136 ceph-mon[113531]: The upgrade process needs to upgrade the mgr, but it needs at least one standby to proceed. 2023-08-27T21:00:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:00 smithi093 ceph-mon[118902]: Health detail: HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:00:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:00 smithi093 ceph-mon[118902]: [WRN] UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T21:00:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:00 smithi093 ceph-mon[118902]: The upgrade process needs to upgrade the mgr, but it needs at least one standby to proceed. 2023-08-27T21:00:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:00 smithi093 ceph-mon[111335]: Health detail: HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:00:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:00 smithi093 ceph-mon[111335]: [WRN] UPGRADE_NO_STANDBY_MGR: Upgrade: Need standby mgr daemon 2023-08-27T21:00:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:00 smithi093 ceph-mon[111335]: The upgrade process needs to upgrade the mgr, but it needs at least one standby to proceed. 2023-08-27T21:00:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:01 smithi136 ceph-mon[113531]: pgmap v195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3514529441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:01 smithi093 ceph-mon[111335]: pgmap v195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3514529441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:01 smithi093 ceph-mon[118902]: pgmap v195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3514529441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:01.649Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:01.650Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:03 smithi136 ceph-mon[113531]: pgmap v196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1270194481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:03 smithi093 ceph-mon[111335]: pgmap v196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1270194481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:03 smithi093 ceph-mon[118902]: pgmap v196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1270194481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:03.537Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:03.538Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:05.626 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:00:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:00:05] "GET /metrics HTTP/1.1" 200 239970 "" "Prometheus/2.43.0" 2023-08-27T21:00:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:05 smithi136 ceph-mon[113531]: pgmap v197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:05 smithi093 ceph-mon[111335]: pgmap v197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:05 smithi093 ceph-mon[118902]: pgmap v197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:06.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/7343309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/7343309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:06.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/7343309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:07 smithi136 ceph-mon[113531]: pgmap v198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:08.105 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:07 smithi093 ceph-mon[111335]: pgmap v198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:08.106 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:07 smithi093 ceph-mon[118902]: pgmap v198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1348415205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1348415205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1348415205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:09 smithi136 ceph-mon[113531]: pgmap v199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:09 smithi093 ceph-mon[111335]: pgmap v199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:09 smithi093 ceph-mon[118902]: pgmap v199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:11 smithi136 ceph-mon[113531]: pgmap v200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3269799048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:11 smithi093 ceph-mon[111335]: pgmap v200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3269799048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:11 smithi093 ceph-mon[118902]: pgmap v200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3269799048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:11.649Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:11.650Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:13 smithi136 ceph-mon[113531]: pgmap v201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/993562841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:13 smithi093 ceph-mon[111335]: pgmap v201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/993562841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:13 smithi093 ceph-mon[118902]: pgmap v201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/993562841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:13.537Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:13.538Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:15.652 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:00:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:00:15] "GET /metrics HTTP/1.1" 200 239975 "" "Prometheus/2.43.0" 2023-08-27T21:00:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:15 smithi136 ceph-mon[113531]: pgmap v202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1222873060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:15 smithi093 ceph-mon[111335]: pgmap v202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1222873060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:15 smithi093 ceph-mon[118902]: pgmap v202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1222873060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:17 smithi136 ceph-mon[113531]: pgmap v203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:17 smithi093 ceph-mon[111335]: pgmap v203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:17 smithi093 ceph-mon[118902]: pgmap v203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1363687775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1363687775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1363687775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:19 smithi136 ceph-mon[113531]: pgmap v204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:19 smithi093 ceph-mon[118902]: pgmap v204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:19 smithi093 ceph-mon[111335]: pgmap v204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/930612615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/930612615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/930612615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:21 smithi136 ceph-mon[113531]: pgmap v205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:21 smithi093 ceph-mon[111335]: pgmap v205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:21 smithi093 ceph-mon[118902]: pgmap v205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:21.650Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:21.651Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3396332786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3396332786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3396332786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:23 smithi136 ceph-mon[113531]: pgmap v206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:23 smithi093 ceph-mon[111335]: pgmap v206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:23 smithi093 ceph-mon[118902]: pgmap v206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:23.538Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:23.539Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:25.674 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:00:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:00:25] "GET /metrics HTTP/1.1" 200 239968 "" "Prometheus/2.43.0" 2023-08-27T21:00:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:25 smithi136 ceph-mon[113531]: pgmap v207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4074898250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:25 smithi093 ceph-mon[111335]: pgmap v207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4074898250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:25 smithi093 ceph-mon[118902]: pgmap v207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4074898250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:26.911 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:00:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:00:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:00:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:27 smithi136 ceph-mon[113531]: pgmap v208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1972380406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:27 smithi093 ceph-mon[111335]: pgmap v208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1972380406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:27 smithi093 ceph-mon[118902]: pgmap v208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1972380406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:29.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:00:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:00:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:00:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:00:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:00:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:29 smithi136 ceph-mon[113531]: pgmap v209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4089301695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:29 smithi093 ceph-mon[118902]: pgmap v209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4089301695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:29 smithi093 ceph-mon[111335]: pgmap v209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4089301695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:31 smithi093 ceph-mon[111335]: pgmap v210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:31 smithi093 ceph-mon[118902]: pgmap v210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:31.651Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:31.652Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:31 smithi136 ceph-mon[113531]: pgmap v210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1769164225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1769164225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1769164225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:33.801 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:33.539Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:33.801 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:33.540Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:33 smithi136 ceph-mon[113531]: pgmap v211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:33 smithi093 ceph-mon[111335]: pgmap v211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:33 smithi093 ceph-mon[118902]: pgmap v211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:35.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/973802831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/973802831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/973802831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:00:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:00:35] "GET /metrics HTTP/1.1" 200 239968 "" "Prometheus/2.43.0" 2023-08-27T21:00:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:35 smithi136 ceph-mon[113531]: pgmap v212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:35 smithi093 ceph-mon[111335]: pgmap v212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:35 smithi093 ceph-mon[118902]: pgmap v212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:37 smithi136 ceph-mon[113531]: pgmap v213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002833529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:37 smithi093 ceph-mon[111335]: pgmap v213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002833529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:37 smithi093 ceph-mon[118902]: pgmap v213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002833529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:39 smithi136 ceph-mon[113531]: pgmap v214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/145098359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:39 smithi093 ceph-mon[118902]: pgmap v214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/145098359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:39 smithi093 ceph-mon[111335]: pgmap v214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/145098359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:41 smithi136 ceph-mon[113531]: pgmap v215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:41 smithi093 ceph-mon[111335]: pgmap v215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:41 smithi093 ceph-mon[118902]: pgmap v215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:41.652Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:41.652Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4160410048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4160410048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4160410048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:43 smithi136 ceph-mon[113531]: pgmap v216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:43 smithi093 ceph-mon[111335]: pgmap v216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:43 smithi093 ceph-mon[118902]: pgmap v216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:43.539Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:43.540Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1443012089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1443012089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1443012089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:45 smithi093 ceph-mon[111335]: pgmap v217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:45 smithi093 ceph-mon[118902]: pgmap v217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:00:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:00:45] "GET /metrics HTTP/1.1" 200 239973 "" "Prometheus/2.43.0" 2023-08-27T21:00:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:45 smithi136 ceph-mon[113531]: pgmap v217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3244321849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3244321849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3244321849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:47 smithi136 ceph-mon[113531]: pgmap v218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:47 smithi093 ceph-mon[111335]: pgmap v218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:47 smithi093 ceph-mon[118902]: pgmap v218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:49 smithi136 ceph-mon[113531]: pgmap v219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2176467400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:49 smithi093 ceph-mon[111335]: pgmap v219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2176467400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:49 smithi093 ceph-mon[118902]: pgmap v219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2176467400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:51 smithi136 ceph-mon[113531]: pgmap v220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4274373237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:51 smithi093 ceph-mon[111335]: pgmap v220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4274373237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:51 smithi093 ceph-mon[118902]: pgmap v220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4274373237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:51.652Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:51.653Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:53 smithi136 ceph-mon[113531]: pgmap v221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:53.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4164982580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:53.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-mon[111335]: pgmap v221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4164982580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-mon[118902]: pgmap v221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4164982580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:00:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:53.540Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:00:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:00:53.541Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:00:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:55 smithi093 ceph-mon[111335]: pgmap v222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:55 smithi093 ceph-mon[118902]: pgmap v222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:00:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:00:55] "GET /metrics HTTP/1.1" 200 239972 "" "Prometheus/2.43.0" 2023-08-27T21:00:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:55 smithi136 ceph-mon[113531]: pgmap v222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/354246880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/354246880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/354246880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:57 smithi136 ceph-mon[113531]: pgmap v223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:57 smithi093 ceph-mon[111335]: pgmap v223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:57 smithi093 ceph-mon[118902]: pgmap v223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3620266932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3620266932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:58.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3620266932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:00:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:00:59 smithi136 ceph-mon[113531]: pgmap v224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:00:59 smithi093 ceph-mon[111335]: pgmap v224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:00:59.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:00:59 smithi093 ceph-mon[118902]: pgmap v224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:01 smithi136 ceph-mon[113531]: pgmap v225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/733409486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:01 smithi093 ceph-mon[111335]: pgmap v225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/733409486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:01 smithi093 ceph-mon[118902]: pgmap v225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/733409486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:01.653Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:01.654Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:03 smithi136 ceph-mon[113531]: pgmap v226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:04.023 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1662573498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:04.023 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:03 smithi093 ceph-mon[111335]: pgmap v226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:04.024 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1662573498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:04.024 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:03 smithi093 ceph-mon[118902]: pgmap v226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:04.024 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1662573498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:04.024 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:03.541Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:04.025 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:03.542Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:05.625 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:01:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:01:05] "GET /metrics HTTP/1.1" 200 239972 "" "Prometheus/2.43.0" 2023-08-27T21:01:05.625 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=sqlstore.transactions t=2023-08-27T21:01:05.31512382Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-08-27T21:01:05.626 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T21:01:05.317586031Z level=info msg="Completed cleanup jobs" duration=12.960463ms 2023-08-27T21:01:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:05 smithi136 ceph-mon[113531]: pgmap v227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1398502385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:05 smithi093 ceph-mon[111335]: pgmap v227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1398502385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:05 smithi093 ceph-mon[118902]: pgmap v227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1398502385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:07 smithi136 ceph-mon[113531]: pgmap v228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:07 smithi093 ceph-mon[111335]: pgmap v228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:07 smithi093 ceph-mon[118902]: pgmap v228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3958941062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3958941062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3958941062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:09 smithi136 ceph-mon[113531]: pgmap v229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:09 smithi093 ceph-mon[111335]: pgmap v229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:09 smithi093 ceph-mon[118902]: pgmap v229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1806091331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1806091331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1806091331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:11 smithi136 ceph-mon[113531]: pgmap v230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:11 smithi093 ceph-mon[111335]: pgmap v230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:11 smithi093 ceph-mon[118902]: pgmap v230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:11.653Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:11.654Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3647596768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3647596768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3647596768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:13 smithi136 ceph-mon[113531]: pgmap v231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:13 smithi093 ceph-mon[111335]: pgmap v231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:13 smithi093 ceph-mon[118902]: pgmap v231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:13.542Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:13.543Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:15.648 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:01:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:01:15] "GET /metrics HTTP/1.1" 200 239971 "" "Prometheus/2.43.0" 2023-08-27T21:01:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:15 smithi136 ceph-mon[113531]: pgmap v232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/757906117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:15 smithi093 ceph-mon[111335]: pgmap v232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/757906117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:15 smithi093 ceph-mon[118902]: pgmap v232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/757906117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:17 smithi136 ceph-mon[113531]: pgmap v233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/451060301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:17 smithi093 ceph-mon[111335]: pgmap v233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/451060301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:17 smithi093 ceph-mon[118902]: pgmap v233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/451060301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:19 smithi136 ceph-mon[113531]: pgmap v234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:19 smithi093 ceph-mon[111335]: pgmap v234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:19 smithi093 ceph-mon[118902]: pgmap v234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/705570708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/705570708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/705570708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:21 smithi136 ceph-mon[113531]: pgmap v235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:21 smithi093 ceph-mon[111335]: pgmap v235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:21 smithi093 ceph-mon[118902]: pgmap v235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:21.654Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:21.655Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2984460210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2984460210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2984460210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:23 smithi136 ceph-mon[113531]: pgmap v236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:23 smithi093 ceph-mon[111335]: pgmap v236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:23 smithi093 ceph-mon[118902]: pgmap v236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:23.542Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:23.543Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/184410759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/184410759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/184410759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:25.679 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:01:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:01:25] "GET /metrics HTTP/1.1" 200 239962 "" "Prometheus/2.43.0" 2023-08-27T21:01:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:25 smithi136 ceph-mon[113531]: pgmap v237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:25 smithi093 ceph-mon[111335]: pgmap v237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:25 smithi093 ceph-mon[118902]: pgmap v237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:27 smithi136 ceph-mon[113531]: pgmap v238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1694427168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:27 smithi093 ceph-mon[111335]: pgmap v238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1694427168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:27 smithi093 ceph-mon[118902]: pgmap v238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1694427168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:29.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:29 smithi136 ceph-mon[113531]: pgmap v239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:29.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:01:29.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1855168001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:29 smithi093 ceph-mon[111335]: pgmap v239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:01:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1855168001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:29 smithi093 ceph-mon[118902]: pgmap v239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:01:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1855168001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:31.655Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:31.657Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[111335]: pgmap v240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2686080381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[118902]: pgmap v240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:31.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2686080381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:31 smithi136 ceph-mon[113531]: pgmap v240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2686080381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:01:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:01:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:01:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:01:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:01:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:01:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:01:33.908 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:33.543Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:33.908 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:33.544Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:33 smithi136 ceph-mon[113531]: pgmap v241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:34.260 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:33 smithi093 ceph-mon[111335]: pgmap v241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:34.260 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:33 smithi093 ceph-mon[118902]: pgmap v241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:35.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4062162666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4062162666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4062162666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:01:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:01:35] "GET /metrics HTTP/1.1" 200 239962 "" "Prometheus/2.43.0" 2023-08-27T21:01:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:35 smithi136 ceph-mon[113531]: pgmap v242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:35 smithi093 ceph-mon[111335]: pgmap v242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:35 smithi093 ceph-mon[118902]: pgmap v242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:37.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2001352418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2001352418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2001352418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:37 smithi136 ceph-mon[113531]: pgmap v243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:37 smithi093 ceph-mon[111335]: pgmap v243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:37 smithi093 ceph-mon[118902]: pgmap v243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:39 smithi136 ceph-mon[113531]: pgmap v244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/930302346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:39 smithi093 ceph-mon[111335]: pgmap v244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/930302346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:39 smithi093 ceph-mon[118902]: pgmap v244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/930302346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:41 smithi136 ceph-mon[113531]: pgmap v245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/684441912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:41 smithi093 ceph-mon[111335]: pgmap v245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/684441912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:41 smithi093 ceph-mon[118902]: pgmap v245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/684441912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:41.656Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:41.657Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:43 smithi136 ceph-mon[113531]: pgmap v246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:43 smithi093 ceph-mon[118902]: pgmap v246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:43 smithi093 ceph-mon[111335]: pgmap v246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:43.544Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:43.545Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3419489973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3419489973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3419489973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:45 smithi093 ceph-mon[111335]: pgmap v247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:45 smithi093 ceph-mon[118902]: pgmap v247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:01:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:01:45] "GET /metrics HTTP/1.1" 200 239960 "" "Prometheus/2.43.0" 2023-08-27T21:01:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:45 smithi136 ceph-mon[113531]: pgmap v247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3099079123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3099079123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3099079123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:47 smithi136 ceph-mon[113531]: pgmap v248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:47 smithi093 ceph-mon[111335]: pgmap v248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:47 smithi093 ceph-mon[118902]: pgmap v248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:48.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/329005969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/329005969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/329005969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:49 smithi136 ceph-mon[113531]: pgmap v249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:49 smithi093 ceph-mon[111335]: pgmap v249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:49 smithi093 ceph-mon[118902]: pgmap v249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:51 smithi136 ceph-mon[113531]: pgmap v250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2766161549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:51 smithi093 ceph-mon[111335]: pgmap v250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2766161549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:51 smithi093 ceph-mon[118902]: pgmap v250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2766161549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:51.657Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:51.658Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:53 smithi136 ceph-mon[113531]: pgmap v251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1814557252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-mon[111335]: pgmap v251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1814557252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-mon[118902]: pgmap v251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1814557252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:01:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:53.545Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:01:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:01:53.546Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:01:55.614 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:01:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:01:55] "GET /metrics HTTP/1.1" 200 239957 "" "Prometheus/2.43.0" 2023-08-27T21:01:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:55 smithi136 ceph-mon[113531]: pgmap v252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3060599099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:55 smithi093 ceph-mon[111335]: pgmap v252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3060599099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:55 smithi093 ceph-mon[118902]: pgmap v252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3060599099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:57 smithi136 ceph-mon[113531]: pgmap v253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:57 smithi093 ceph-mon[111335]: pgmap v253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:57 smithi093 ceph-mon[118902]: pgmap v253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4020055373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4020055373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:58.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4020055373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:01:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:01:59 smithi136 ceph-mon[113531]: pgmap v254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:01:59 smithi093 ceph-mon[111335]: pgmap v254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:01:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:01:59 smithi093 ceph-mon[118902]: pgmap v254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3409428782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3409428782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3409428782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:01 smithi136 ceph-mon[113531]: pgmap v255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:01 smithi093 ceph-mon[111335]: pgmap v255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:01 smithi093 ceph-mon[118902]: pgmap v255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:01.657Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:01.658Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:03 smithi136 ceph-mon[113531]: pgmap v256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1685124895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:03 smithi093 ceph-mon[111335]: pgmap v256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1685124895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:03 smithi093 ceph-mon[118902]: pgmap v256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1685124895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:03.546Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:03.547Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:05.635 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:02:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:02:05] "GET /metrics HTTP/1.1" 200 239957 "" "Prometheus/2.43.0" 2023-08-27T21:02:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:05 smithi136 ceph-mon[113531]: pgmap v257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2925135705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:05 smithi093 ceph-mon[111335]: pgmap v257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2925135705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:05 smithi093 ceph-mon[118902]: pgmap v257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2925135705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:07 smithi136 ceph-mon[113531]: pgmap v258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2965470271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:07 smithi093 ceph-mon[111335]: pgmap v258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2965470271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:07 smithi093 ceph-mon[118902]: pgmap v258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2965470271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:09 smithi136 ceph-mon[113531]: pgmap v259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:09 smithi093 ceph-mon[111335]: pgmap v259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:09 smithi093 ceph-mon[118902]: pgmap v259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2523550991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2523550991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2523550991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:11 smithi136 ceph-mon[113531]: pgmap v260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:11 smithi093 ceph-mon[111335]: pgmap v260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:11 smithi093 ceph-mon[118902]: pgmap v260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:11.658Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:11.658Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3749342749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3749342749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3749342749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:13 smithi136 ceph-mon[113531]: pgmap v261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:13 smithi093 ceph-mon[111335]: pgmap v261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:13 smithi093 ceph-mon[118902]: pgmap v261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:13.547Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:13.548Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:02:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:02:15] "GET /metrics HTTP/1.1" 200 239960 "" "Prometheus/2.43.0" 2023-08-27T21:02:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:15 smithi093 ceph-mon[111335]: pgmap v262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2554660881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:15 smithi093 ceph-mon[118902]: pgmap v262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2554660881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:15 smithi136 ceph-mon[113531]: pgmap v262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2554660881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:17 smithi136 ceph-mon[113531]: pgmap v263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3068154829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:17 smithi093 ceph-mon[111335]: pgmap v263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3068154829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:17 smithi093 ceph-mon[118902]: pgmap v263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3068154829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:19 smithi136 ceph-mon[113531]: pgmap v264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/31134790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:19 smithi093 ceph-mon[111335]: pgmap v264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/31134790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:19 smithi093 ceph-mon[118902]: pgmap v264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/31134790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:21 smithi136 ceph-mon[113531]: pgmap v265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:21 smithi093 ceph-mon[111335]: pgmap v265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:21 smithi093 ceph-mon[118902]: pgmap v265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:21.658Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:21.659Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2806033836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2806033836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2806033836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:23 smithi136 ceph-mon[113531]: pgmap v266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:23 smithi093 ceph-mon[111335]: pgmap v266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:23 smithi093 ceph-mon[118902]: pgmap v266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:23.548Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:23.549Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1790651568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1790651568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1790651568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:25.633 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:02:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:02:25] "GET /metrics HTTP/1.1" 200 239954 "" "Prometheus/2.43.0" 2023-08-27T21:02:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:25 smithi136 ceph-mon[113531]: pgmap v267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:25 smithi093 ceph-mon[111335]: pgmap v267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:25 smithi093 ceph-mon[118902]: pgmap v267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/603531291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/603531291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/603531291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:27 smithi136 ceph-mon[113531]: pgmap v268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:27 smithi093 ceph-mon[111335]: pgmap v268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:27 smithi093 ceph-mon[118902]: pgmap v268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:29 smithi136 ceph-mon[113531]: pgmap v269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3873870537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:29 smithi093 ceph-mon[111335]: pgmap v269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3873870537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:29 smithi093 ceph-mon[118902]: pgmap v269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3873870537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:31 smithi136 ceph-mon[113531]: pgmap v270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/494188852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:31 smithi093 ceph-mon[111335]: pgmap v270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/494188852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:31 smithi093 ceph-mon[118902]: pgmap v270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/494188852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:31.659Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:31.660Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:02:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:02:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:02:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:33 smithi136 ceph-mon[113531]: pgmap v271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:33 smithi093 ceph-mon[111335]: pgmap v271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:33 smithi093 ceph-mon[118902]: pgmap v271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:33.549Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:33.550Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/689024264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/689024264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/689024264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:02:35] "GET /metrics HTTP/1.1" 200 239954 "" "Prometheus/2.43.0" 2023-08-27T21:02:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:35 smithi136 ceph-mon[113531]: pgmap v272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:02:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:02:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[111335]: pgmap v272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:02:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:02:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[118902]: pgmap v272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:02:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:02:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:02:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1648680399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1648680399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1648680399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:37 smithi136 ceph-mon[113531]: pgmap v273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:37 smithi093 ceph-mon[111335]: pgmap v273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:37 smithi093 ceph-mon[118902]: pgmap v273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3068529035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3068529035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3068529035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:39 smithi136 ceph-mon[113531]: pgmap v274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:39 smithi093 ceph-mon[111335]: pgmap v274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:39 smithi093 ceph-mon[118902]: pgmap v274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/302585262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/302585262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/302585262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:41 smithi093 ceph-mon[111335]: pgmap v275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:41 smithi093 ceph-mon[118902]: pgmap v275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:41.660Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:41.661Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:41 smithi136 ceph-mon[113531]: pgmap v275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:43.894 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:43.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:43.894 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:43.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:43 smithi136 ceph-mon[113531]: pgmap v276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1325977142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:43 smithi093 ceph-mon[111335]: pgmap v276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1325977142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:43 smithi093 ceph-mon[118902]: pgmap v276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1325977142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:02:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:02:45] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-27T21:02:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:45 smithi136 ceph-mon[113531]: pgmap v277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2023510293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:45 smithi093 ceph-mon[118902]: pgmap v277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2023510293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:45 smithi093 ceph-mon[111335]: pgmap v277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2023510293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:47 smithi136 ceph-mon[113531]: pgmap v278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:47 smithi093 ceph-mon[111335]: pgmap v278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:47 smithi093 ceph-mon[118902]: pgmap v278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:49.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3174590486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3174590486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3174590486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:49 smithi136 ceph-mon[113531]: pgmap v279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:49 smithi093 ceph-mon[111335]: pgmap v279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:49 smithi093 ceph-mon[118902]: pgmap v279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2023957570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2023957570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2023957570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:51.913 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:51.661Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:51.914 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:51.662Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:51 smithi136 ceph-mon[113531]: pgmap v280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:51 smithi093 ceph-mon[111335]: pgmap v280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:51 smithi093 ceph-mon[118902]: pgmap v280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3645692066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3645692066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3645692066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:53.919 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:53.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:53.919 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:02:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:02:53.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:02:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:53 smithi136 ceph-mon[113531]: pgmap v281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:53 smithi093 ceph-mon[111335]: pgmap v281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:53 smithi093 ceph-mon[118902]: pgmap v281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:02:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:02:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:02:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-27T21:02:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:55 smithi136 ceph-mon[113531]: pgmap v282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2974637590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:55 smithi093 ceph-mon[111335]: pgmap v282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2974637590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:55 smithi093 ceph-mon[118902]: pgmap v282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2974637590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:57 smithi136 ceph-mon[113531]: pgmap v283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/671407781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:57 smithi093 ceph-mon[111335]: pgmap v283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/671407781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:02:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:57 smithi093 ceph-mon[118902]: pgmap v283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:02:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/671407781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:59 smithi136 ceph-mon[113531]: pgmap v284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:02:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/645475412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:59 smithi093 ceph-mon[118902]: pgmap v284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:02:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/645475412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:59 smithi093 ceph-mon[111335]: pgmap v284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:02:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/645475412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:01.930 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:01.662Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:01.930 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:01.663Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:01 smithi136 ceph-mon[113531]: pgmap v285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:01 smithi093 ceph-mon[111335]: pgmap v285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:01 smithi093 ceph-mon[118902]: pgmap v285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4090360840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4090360840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4090360840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:03.935 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:03.551Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:03.935 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:03.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:03 smithi136 ceph-mon[113531]: pgmap v286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:03 smithi093 ceph-mon[111335]: pgmap v286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:03 smithi093 ceph-mon[118902]: pgmap v286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3428212705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3428212705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3428212705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:03:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:03:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-27T21:03:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:05 smithi093 ceph-mon[111335]: pgmap v287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:05 smithi093 ceph-mon[118902]: pgmap v287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:05 smithi136 ceph-mon[113531]: pgmap v287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:07 smithi093 ceph-mon[111335]: pgmap v288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3935386970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:07 smithi093 ceph-mon[118902]: pgmap v288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3935386970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:07 smithi136 ceph-mon[113531]: pgmap v288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3935386970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:09 smithi136 ceph-mon[113531]: pgmap v289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1210630809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:09 smithi093 ceph-mon[118902]: pgmap v289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1210630809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:09 smithi093 ceph-mon[111335]: pgmap v289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1210630809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:11 smithi136 ceph-mon[113531]: pgmap v290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:11 smithi093 ceph-mon[111335]: pgmap v290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:11 smithi093 ceph-mon[118902]: pgmap v290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:11.662Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:11.663Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/623316521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/623316521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/623316521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:13 smithi136 ceph-mon[113531]: pgmap v291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:13 smithi093 ceph-mon[111335]: pgmap v291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:13 smithi093 ceph-mon[118902]: pgmap v291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:13.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:13.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/627231425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/627231425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/627231425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:15 smithi093 ceph-mon[111335]: pgmap v292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:15 smithi093 ceph-mon[118902]: pgmap v292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:03:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:03:15] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-27T21:03:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:15 smithi136 ceph-mon[113531]: pgmap v292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2706040547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2706040547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2706040547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:17 smithi136 ceph-mon[113531]: pgmap v293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:17 smithi093 ceph-mon[111335]: pgmap v293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:17 smithi093 ceph-mon[118902]: pgmap v293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:19 smithi136 ceph-mon[113531]: pgmap v294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/569483919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:19 smithi093 ceph-mon[111335]: pgmap v294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/569483919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:19 smithi093 ceph-mon[118902]: pgmap v294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/569483919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:21 smithi136 ceph-mon[113531]: pgmap v295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1307730320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:21 smithi093 ceph-mon[111335]: pgmap v295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1307730320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:21 smithi093 ceph-mon[118902]: pgmap v295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1307730320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:21.662Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:21.663Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:23 smithi136 ceph-mon[113531]: pgmap v296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:23 smithi093 ceph-mon[111335]: pgmap v296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:23 smithi093 ceph-mon[118902]: pgmap v296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:23.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:23.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/622384290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/622384290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/622384290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:25.675 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:03:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:03:25] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-27T21:03:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:25 smithi136 ceph-mon[113531]: pgmap v297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:25 smithi093 ceph-mon[111335]: pgmap v297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:25 smithi093 ceph-mon[118902]: pgmap v297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2546767923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2546767923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2546767923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:27 smithi136 ceph-mon[113531]: pgmap v298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:27 smithi093 ceph-mon[111335]: pgmap v298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:27 smithi093 ceph-mon[118902]: pgmap v298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:28.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3796421955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:28.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3796421955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3796421955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:29 smithi093 ceph-mon[111335]: pgmap v299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:29 smithi093 ceph-mon[118902]: pgmap v299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:29 smithi136 ceph-mon[113531]: pgmap v299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:31 smithi093 ceph-mon[111335]: pgmap v300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3555151171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:31 smithi093 ceph-mon[118902]: pgmap v300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3555151171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:31.663Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:31.664Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:31 smithi136 ceph-mon[113531]: pgmap v300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3555151171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:33 smithi093 ceph-mon[118902]: pgmap v301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2230115655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:33 smithi093 ceph-mon[111335]: pgmap v301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2230115655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:33.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:33.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:33 smithi136 ceph-mon[113531]: pgmap v301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2230115655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:35.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:35 smithi093 ceph-mon[111335]: pgmap v302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:35.699 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:03:35.699 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:03:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:03:35] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-27T21:03:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:35 smithi093 ceph-mon[118902]: pgmap v302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:03:36.131 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:35 smithi136 ceph-mon[113531]: pgmap v302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:36.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:03:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3959301685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3959301685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3959301685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:38 smithi136 ceph-mon[113531]: pgmap v303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[111335]: pgmap v303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[118902]: pgmap v303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/978907728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/978907728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:03:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:03:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/978907728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:03:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:03:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:03:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:40 smithi093 ceph-mon[111335]: pgmap v304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:40 smithi093 ceph-mon[118902]: pgmap v304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:40 smithi136 ceph-mon[113531]: pgmap v304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/86151846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/86151846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/86151846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:41.664Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:41.665Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:42 smithi093 ceph-mon[111335]: pgmap v305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:42 smithi093 ceph-mon[118902]: pgmap v305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:42 smithi136 ceph-mon[113531]: pgmap v305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1688822820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1688822820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1688822820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:43.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:43.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:44 smithi093 ceph-mon[111335]: pgmap v306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:44 smithi093 ceph-mon[118902]: pgmap v306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:44 smithi136 ceph-mon[113531]: pgmap v306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:03:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:03:45] "GET /metrics HTTP/1.1" 200 239934 "" "Prometheus/2.43.0" 2023-08-27T21:03:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:46 smithi093 ceph-mon[111335]: pgmap v307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3953437740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:46 smithi093 ceph-mon[118902]: pgmap v307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3953437740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:46 smithi136 ceph-mon[113531]: pgmap v307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3953437740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:48 smithi093 ceph-mon[111335]: pgmap v308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1945139343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:48 smithi093 ceph-mon[118902]: pgmap v308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1945139343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:48 smithi136 ceph-mon[113531]: pgmap v308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1945139343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:50 smithi093 ceph-mon[111335]: pgmap v309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1290594475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:50 smithi093 ceph-mon[118902]: pgmap v309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1290594475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:50 smithi136 ceph-mon[113531]: pgmap v309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1290594475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:51.664Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:51.666Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:52 smithi093 ceph-mon[111335]: pgmap v310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:52 smithi093 ceph-mon[118902]: pgmap v310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:52 smithi136 ceph-mon[113531]: pgmap v310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3079009832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3079009832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3079009832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:53.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:03:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:03:53.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:03:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:54 smithi093 ceph-mon[111335]: pgmap v311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:54 smithi093 ceph-mon[118902]: pgmap v311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:54 smithi136 ceph-mon[113531]: pgmap v311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:03:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4206063245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4206063245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:03:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:03:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-27T21:03:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4206063245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:56 smithi093 ceph-mon[111335]: pgmap v312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:56 smithi093 ceph-mon[118902]: pgmap v312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:56 smithi136 ceph-mon[113531]: pgmap v312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3338011152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3338011152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3338011152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:03:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:03:58 smithi093 ceph-mon[111335]: pgmap v313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:03:58 smithi093 ceph-mon[118902]: pgmap v313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:03:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:03:58 smithi136 ceph-mon[113531]: pgmap v313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:00 smithi093 ceph-mon[118902]: pgmap v314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1879454236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:00 smithi093 ceph-mon[111335]: pgmap v314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1879454236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:00 smithi136 ceph-mon[113531]: pgmap v314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1879454236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:01.665Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:01.666Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:02 smithi093 ceph-mon[111335]: pgmap v315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3351992726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:02 smithi093 ceph-mon[118902]: pgmap v315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3351992726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:02 smithi136 ceph-mon[113531]: pgmap v315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3351992726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:03.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:03.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:04 smithi136 ceph-mon[113531]: pgmap v316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:04 smithi093 ceph-mon[111335]: pgmap v316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:04 smithi093 ceph-mon[118902]: pgmap v316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1909125173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1909125173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1909125173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:04:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:04:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-27T21:04:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:06 smithi136 ceph-mon[113531]: pgmap v317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:06 smithi093 ceph-mon[111335]: pgmap v317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:06 smithi093 ceph-mon[118902]: pgmap v317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1354461408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1354461408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1354461408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:08 smithi136 ceph-mon[113531]: pgmap v318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:08 smithi093 ceph-mon[111335]: pgmap v318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:08 smithi093 ceph-mon[118902]: pgmap v318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:09 smithi136 ceph-mon[113531]: pgmap v319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471315397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:09 smithi093 ceph-mon[111335]: pgmap v319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471315397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:09 smithi093 ceph-mon[118902]: pgmap v319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471315397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:11 smithi136 ceph-mon[113531]: pgmap v320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/293920332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:11 smithi093 ceph-mon[118902]: pgmap v320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/293920332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:11 smithi093 ceph-mon[111335]: pgmap v320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/293920332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:11.666Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:11.667Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:13 smithi136 ceph-mon[113531]: pgmap v321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:13 smithi093 ceph-mon[111335]: pgmap v321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:13 smithi093 ceph-mon[118902]: pgmap v321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:13.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:13.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/775815605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/775815605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/775815605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:15.609 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:04:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:04:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-27T21:04:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:15 smithi136 ceph-mon[113531]: pgmap v322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:15 smithi093 ceph-mon[111335]: pgmap v322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:15 smithi093 ceph-mon[118902]: pgmap v322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2297636046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2297636046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2297636046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:17 smithi136 ceph-mon[113531]: pgmap v323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:17 smithi093 ceph-mon[111335]: pgmap v323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:17 smithi093 ceph-mon[118902]: pgmap v323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3384442734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3384442734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3384442734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:19 smithi136 ceph-mon[113531]: pgmap v324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:19 smithi093 ceph-mon[111335]: pgmap v324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:19 smithi093 ceph-mon[118902]: pgmap v324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:21 smithi136 ceph-mon[113531]: pgmap v325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2955542950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:21 smithi093 ceph-mon[111335]: pgmap v325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2955542950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:21 smithi093 ceph-mon[118902]: pgmap v325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2955542950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:21.667Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:21.668Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:23 smithi136 ceph-mon[113531]: pgmap v326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1836930217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-mon[111335]: pgmap v326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1836930217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-mon[118902]: pgmap v326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1836930217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:23.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:23.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:25.631 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:04:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:04:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-27T21:04:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:25 smithi136 ceph-mon[113531]: pgmap v327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:25 smithi093 ceph-mon[111335]: pgmap v327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:25 smithi093 ceph-mon[118902]: pgmap v327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3281336962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3281336962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3281336962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:27 smithi136 ceph-mon[113531]: pgmap v328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:27 smithi093 ceph-mon[111335]: pgmap v328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:27 smithi093 ceph-mon[118902]: pgmap v328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/608405592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/608405592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/608405592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:29 smithi136 ceph-mon[113531]: pgmap v329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:29 smithi093 ceph-mon[111335]: pgmap v329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:29 smithi093 ceph-mon[118902]: pgmap v329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3754900000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3754900000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3754900000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:31 smithi136 ceph-mon[113531]: pgmap v330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:31 smithi093 ceph-mon[111335]: pgmap v330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:31 smithi093 ceph-mon[118902]: pgmap v330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:31.668Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:31.669Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:33 smithi136 ceph-mon[113531]: pgmap v331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3250120552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:33 smithi093 ceph-mon[111335]: pgmap v331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3250120552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:33 smithi093 ceph-mon[118902]: pgmap v331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3250120552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:33.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:33.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:35.656 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:04:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:04:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-27T21:04:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:35 smithi136 ceph-mon[113531]: pgmap v332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3118245288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:35 smithi093 ceph-mon[111335]: pgmap v332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3118245288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:35 smithi093 ceph-mon[118902]: pgmap v332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3118245288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:37 smithi136 ceph-mon[113531]: pgmap v333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2605674475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:37 smithi093 ceph-mon[111335]: pgmap v333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2605674475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:37 smithi093 ceph-mon[118902]: pgmap v333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2605674475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:38.921 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:04:38.921 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:38.964 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:04:38.965 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:38.965 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:04:38.965 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:39.913 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:39 smithi093 ceph-mon[111335]: pgmap v334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:39.913 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:39 smithi093 ceph-mon[118902]: pgmap v334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:39 smithi136 ceph-mon[113531]: pgmap v334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/171624422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/171624422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/171624422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[118902]: pgmap v335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:04:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:04:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[111335]: pgmap v335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:41.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:04:41.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:04:41.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:41.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:41.669Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:41.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:41.670Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:41 smithi136 ceph-mon[113531]: pgmap v335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:04:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:04:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:04:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2312741079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2312741079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2312741079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:43 smithi093 ceph-mon[111335]: pgmap v336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:43 smithi093 ceph-mon[118902]: pgmap v336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:43.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:43.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:43 smithi136 ceph-mon[113531]: pgmap v336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:04:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:04:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-27T21:04:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:45 smithi136 ceph-mon[113531]: pgmap v337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4060005216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:45 smithi093 ceph-mon[118902]: pgmap v337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4060005216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:45 smithi093 ceph-mon[111335]: pgmap v337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4060005216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:47 smithi136 ceph-mon[113531]: pgmap v338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/303542755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:47 smithi093 ceph-mon[111335]: pgmap v338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/303542755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:47 smithi093 ceph-mon[118902]: pgmap v338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/303542755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:49 smithi136 ceph-mon[113531]: pgmap v339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4251032816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:49 smithi093 ceph-mon[118902]: pgmap v339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4251032816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:49 smithi093 ceph-mon[111335]: pgmap v339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4251032816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:51 smithi093 ceph-mon[111335]: pgmap v340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:51 smithi093 ceph-mon[118902]: pgmap v340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:51.670Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:51.671Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:51 smithi136 ceph-mon[113531]: pgmap v340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1668048453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1668048453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1668048453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:53 smithi093 ceph-mon[118902]: pgmap v341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:53 smithi093 ceph-mon[111335]: pgmap v341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:53.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:04:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:04:53.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:04:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:53 smithi136 ceph-mon[113531]: pgmap v341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:04:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1508431264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1508431264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1508431264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:04:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:04:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-27T21:04:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:55 smithi136 ceph-mon[113531]: pgmap v342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:55 smithi093 ceph-mon[111335]: pgmap v342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:55 smithi093 ceph-mon[118902]: pgmap v342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/763371420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/763371420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/763371420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:04:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:57 smithi136 ceph-mon[113531]: pgmap v343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:57 smithi093 ceph-mon[118902]: pgmap v343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:04:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:57 smithi093 ceph-mon[111335]: pgmap v343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:59 smithi136 ceph-mon[113531]: pgmap v344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:04:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2891129733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:59 smithi093 ceph-mon[111335]: pgmap v344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:04:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2891129733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:59 smithi093 ceph-mon[118902]: pgmap v344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:04:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2891129733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:01 smithi093 ceph-mon[111335]: pgmap v345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:02.043 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2485343514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:02.043 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:01 smithi093 ceph-mon[118902]: pgmap v345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:02.044 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2485343514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:02.044 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:01.670Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:02.044 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:01.671Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:01 smithi136 ceph-mon[113531]: pgmap v345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2485343514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:03 smithi093 ceph-mon[111335]: pgmap v346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:03 smithi093 ceph-mon[118902]: pgmap v346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:03.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:03.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:03 smithi136 ceph-mon[113531]: pgmap v346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2939593434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2939593434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2939593434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:05:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:05:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-27T21:05:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:05 smithi136 ceph-mon[113531]: pgmap v347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:05 smithi093 ceph-mon[111335]: pgmap v347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:05 smithi093 ceph-mon[118902]: pgmap v347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/486997972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:07.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/486997972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:07.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/486997972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:07 smithi136 ceph-mon[113531]: pgmap v348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:07 smithi093 ceph-mon[111335]: pgmap v348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:07 smithi093 ceph-mon[118902]: pgmap v348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/939478738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/939478738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/939478738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:09 smithi136 ceph-mon[113531]: pgmap v349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:09 smithi093 ceph-mon[111335]: pgmap v349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:09 smithi093 ceph-mon[118902]: pgmap v349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:11 smithi093 ceph-mon[118902]: pgmap v350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1035530803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:11 smithi093 ceph-mon[111335]: pgmap v350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1035530803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:11.671Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:11.672Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:11 smithi136 ceph-mon[113531]: pgmap v350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1035530803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:13.823 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:13.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:13.823 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:13.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:13 smithi136 ceph-mon[113531]: pgmap v351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/587515436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:13 smithi093 ceph-mon[111335]: pgmap v351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/587515436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:13 smithi093 ceph-mon[118902]: pgmap v351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/587515436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:05:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:05:15] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-27T21:05:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:15 smithi136 ceph-mon[113531]: pgmap v352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3566167382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:15 smithi093 ceph-mon[111335]: pgmap v352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3566167382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:15 smithi093 ceph-mon[118902]: pgmap v352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3566167382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:17 smithi136 ceph-mon[113531]: pgmap v353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:17 smithi093 ceph-mon[111335]: pgmap v353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:17 smithi093 ceph-mon[118902]: pgmap v353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1925515341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1925515341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1925515341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:19 smithi136 ceph-mon[113531]: pgmap v354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:19 smithi093 ceph-mon[111335]: pgmap v354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:19 smithi093 ceph-mon[118902]: pgmap v354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/401921644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/401921644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/401921644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:21 smithi093 ceph-mon[111335]: pgmap v355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:21 smithi093 ceph-mon[118902]: pgmap v355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:21.671Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:21.672Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:21 smithi136 ceph-mon[113531]: pgmap v355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:23.846 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:23.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:23.846 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:23.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:23 smithi136 ceph-mon[113531]: pgmap v356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2638281457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-mon[118902]: pgmap v356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2638281457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-mon[111335]: pgmap v356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2638281457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:05:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:05:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-27T21:05:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:25 smithi136 ceph-mon[113531]: pgmap v357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3189065753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:26.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:25 smithi093 ceph-mon[118902]: pgmap v357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:26.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3189065753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:26.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:25 smithi093 ceph-mon[111335]: pgmap v357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:26.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3189065753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:27 smithi136 ceph-mon[113531]: pgmap v358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:28.212 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2170663348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:27 smithi093 ceph-mon[111335]: pgmap v358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2170663348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:27 smithi093 ceph-mon[118902]: pgmap v358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2170663348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:29 smithi136 ceph-mon[113531]: pgmap v359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:29 smithi093 ceph-mon[111335]: pgmap v359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:29 smithi093 ceph-mon[118902]: pgmap v359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2754112551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2754112551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2754112551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:31 smithi093 ceph-mon[118902]: pgmap v360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:31 smithi093 ceph-mon[111335]: pgmap v360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:31.672Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:31.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:31 smithi136 ceph-mon[113531]: pgmap v360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3558689057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3558689057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3558689057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:33.871 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:33.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:33.871 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:33.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:33 smithi136 ceph-mon[113531]: pgmap v361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:33 smithi093 ceph-mon[111335]: pgmap v361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:33 smithi093 ceph-mon[118902]: pgmap v361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2548582120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2548582120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2548582120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:05:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:05:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-27T21:05:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:35 smithi136 ceph-mon[113531]: pgmap v362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:35 smithi093 ceph-mon[111335]: pgmap v362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:35 smithi093 ceph-mon[118902]: pgmap v362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:37 smithi136 ceph-mon[113531]: pgmap v363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2172709027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:37 smithi093 ceph-mon[111335]: pgmap v363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2172709027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:37 smithi093 ceph-mon[118902]: pgmap v363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2172709027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:39 smithi136 ceph-mon[113531]: pgmap v364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/88990763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:39 smithi093 ceph-mon[111335]: pgmap v364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/88990763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:39 smithi093 ceph-mon[118902]: pgmap v364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/88990763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:41.953 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:41 smithi093 ceph-mon[111335]: pgmap v365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:41.953 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:05:41.954 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:41 smithi093 ceph-mon[118902]: pgmap v365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:41.954 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:05:41.954 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:41.673Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:41.954 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:41.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:41 smithi136 ceph-mon[113531]: pgmap v365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:05:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2394695550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2394695550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2394695550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:43.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:43.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:44 smithi136 ceph-mon[113531]: pgmap v366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[118902]: pgmap v366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[111335]: pgmap v366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:05:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:05:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3959670264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:05:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:05:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3959670264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:05:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:05:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:05:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3959670264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:05:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:05:45] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-27T21:05:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:46 smithi093 ceph-mon[111335]: pgmap v367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:46 smithi093 ceph-mon[118902]: pgmap v367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:46 smithi136 ceph-mon[113531]: pgmap v367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/837376704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/837376704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/837376704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:48 smithi093 ceph-mon[111335]: pgmap v368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:48 smithi093 ceph-mon[118902]: pgmap v368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:48 smithi136 ceph-mon[113531]: pgmap v368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:49 smithi136 ceph-mon[113531]: pgmap v369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4217855960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:49 smithi093 ceph-mon[118902]: pgmap v369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4217855960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:49 smithi093 ceph-mon[111335]: pgmap v369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4217855960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:51 smithi136 ceph-mon[113531]: pgmap v370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3724664138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:51 smithi093 ceph-mon[111335]: pgmap v370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3724664138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:51 smithi093 ceph-mon[118902]: pgmap v370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3724664138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:51.673Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:51.675Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:53 smithi136 ceph-mon[113531]: pgmap v371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:53 smithi093 ceph-mon[111335]: pgmap v371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:53 smithi093 ceph-mon[118902]: pgmap v371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:05:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:53.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:05:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:05:53.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:05:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1716526166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1716526166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1716526166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:55.639 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:05:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:05:55] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-27T21:05:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:55 smithi136 ceph-mon[113531]: pgmap v372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:55 smithi093 ceph-mon[111335]: pgmap v372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:55 smithi093 ceph-mon[118902]: pgmap v372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:56.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2344420549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2344420549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2344420549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:57 smithi136 ceph-mon[113531]: pgmap v373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:57 smithi093 ceph-mon[111335]: pgmap v373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:57 smithi093 ceph-mon[118902]: pgmap v373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:59 smithi136 ceph-mon[113531]: pgmap v374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:05:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1130027879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:59 smithi093 ceph-mon[111335]: pgmap v374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:05:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1130027879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:05:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:59 smithi093 ceph-mon[118902]: pgmap v374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:05:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:05:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1130027879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:01 smithi136 ceph-mon[113531]: pgmap v375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3685050969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:01 smithi093 ceph-mon[111335]: pgmap v375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3685050969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:01 smithi093 ceph-mon[118902]: pgmap v375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3685050969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:01.675Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:01.676Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:03 smithi136 ceph-mon[113531]: pgmap v376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2046824503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:03 smithi093 ceph-mon[111335]: pgmap v376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2046824503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:03 smithi093 ceph-mon[118902]: pgmap v376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2046824503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:03.563Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:03.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:04.939 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:06:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=sqlstore.transactions t=2023-08-27T21:06:04.564999968Z level=info msg="Database locked, sleeping then retrying" error="database is locked" retry=0 code="database is locked" 2023-08-27T21:06:05.659 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:06:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:06:05] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-27T21:06:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:05 smithi136 ceph-mon[113531]: pgmap v377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:05 smithi093 ceph-mon[111335]: pgmap v377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:05 smithi093 ceph-mon[118902]: pgmap v377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4261292896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4261292896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4261292896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:07 smithi136 ceph-mon[113531]: pgmap v378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:07 smithi093 ceph-mon[111335]: pgmap v378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:07 smithi093 ceph-mon[118902]: pgmap v378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/426557306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/426557306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/426557306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:09 smithi136 ceph-mon[113531]: pgmap v379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:09 smithi093 ceph-mon[111335]: pgmap v379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:09 smithi093 ceph-mon[118902]: pgmap v379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2296770624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2296770624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2296770624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:11 smithi136 ceph-mon[113531]: pgmap v380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:11 smithi093 ceph-mon[111335]: pgmap v380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:11 smithi093 ceph-mon[118902]: pgmap v380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:11.675Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:11.676Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:13 smithi136 ceph-mon[113531]: pgmap v381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1823855082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:13 smithi093 ceph-mon[111335]: pgmap v381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1823855082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:13 smithi093 ceph-mon[118902]: pgmap v381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1823855082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:13.564Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:13.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:15.681 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:06:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:06:15] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-27T21:06:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:15 smithi136 ceph-mon[113531]: pgmap v382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3217892003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:15 smithi093 ceph-mon[111335]: pgmap v382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3217892003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:15 smithi093 ceph-mon[118902]: pgmap v382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3217892003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:17 smithi136 ceph-mon[113531]: pgmap v383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:17 smithi093 ceph-mon[111335]: pgmap v383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:17 smithi093 ceph-mon[118902]: pgmap v383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1273165557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1273165557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1273165557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:19 smithi093 ceph-mon[111335]: pgmap v384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:19 smithi093 ceph-mon[118902]: pgmap v384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:19 smithi136 ceph-mon[113531]: pgmap v384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2655834897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2655834897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2655834897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:21 smithi093 ceph-mon[111335]: pgmap v385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:21 smithi093 ceph-mon[118902]: pgmap v385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:21.676Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:21.677Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:21 smithi136 ceph-mon[113531]: pgmap v385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1588561746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1588561746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1588561746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:23 smithi093 ceph-mon[111335]: pgmap v386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:23 smithi093 ceph-mon[118902]: pgmap v386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:23.565Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:23.566Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:23 smithi136 ceph-mon[113531]: pgmap v386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:25.712 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:06:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:06:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-27T21:06:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:25 smithi093 ceph-mon[111335]: pgmap v387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2074227473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:25 smithi093 ceph-mon[118902]: pgmap v387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2074227473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:25 smithi136 ceph-mon[113531]: pgmap v387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2074227473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:27 smithi093 ceph-mon[111335]: pgmap v388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1425624342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:27 smithi093 ceph-mon[118902]: pgmap v388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1425624342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:27 smithi136 ceph-mon[113531]: pgmap v388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1425624342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:29 smithi093 ceph-mon[118902]: pgmap v389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2262464301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:29 smithi093 ceph-mon[111335]: pgmap v389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2262464301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:29 smithi136 ceph-mon[113531]: pgmap v389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2262464301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:31 smithi093 ceph-mon[111335]: pgmap v390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:31 smithi093 ceph-mon[118902]: pgmap v390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:31.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:31.677Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:31 smithi136 ceph-mon[113531]: pgmap v390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/454379013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/454379013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/454379013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:33 smithi093 ceph-mon[111335]: pgmap v391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:33 smithi093 ceph-mon[118902]: pgmap v391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:33.566Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:33.567Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:33 smithi136 ceph-mon[113531]: pgmap v391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4268489277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4268489277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4268489277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:06:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:06:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-27T21:06:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:35 smithi093 ceph-mon[118902]: pgmap v392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:35 smithi093 ceph-mon[111335]: pgmap v392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:35 smithi136 ceph-mon[113531]: pgmap v392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:37 smithi136 ceph-mon[113531]: pgmap v393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/70928061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:37 smithi093 ceph-mon[111335]: pgmap v393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/70928061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:37 smithi093 ceph-mon[118902]: pgmap v393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/70928061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:39 smithi136 ceph-mon[113531]: pgmap v394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3996902350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:39 smithi093 ceph-mon[111335]: pgmap v394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3996902350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:39 smithi093 ceph-mon[118902]: pgmap v394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3996902350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:41 smithi093 ceph-mon[111335]: pgmap v395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1362689305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:41 smithi093 ceph-mon[118902]: pgmap v395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1362689305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:41.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:41.678Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:41 smithi136 ceph-mon[113531]: pgmap v395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1362689305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:43 smithi093 ceph-mon[111335]: pgmap v396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:43 smithi093 ceph-mon[118902]: pgmap v396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:43.567Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:43.568Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:43 smithi136 ceph-mon[113531]: pgmap v396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/855394451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:06:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/855394451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:06:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/855394451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:06:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:06:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:06:45] "GET /metrics HTTP/1.1" 200 239934 "" "Prometheus/2.43.0" 2023-08-27T21:06:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:45 smithi136 ceph-mon[113531]: pgmap v397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:45 smithi093 ceph-mon[111335]: pgmap v397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:45 smithi093 ceph-mon[118902]: pgmap v397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1890269383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1890269383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1890269383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[111335]: pgmap v398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:06:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:06:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[118902]: pgmap v398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:06:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:06:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:48 smithi136 ceph-mon[113531]: pgmap v398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:06:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:06:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:06:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:49 smithi136 ceph-mon[113531]: pgmap v399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1732570581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:49 smithi093 ceph-mon[111335]: pgmap v399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1732570581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:49 smithi093 ceph-mon[118902]: pgmap v399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1732570581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:51 smithi136 ceph-mon[113531]: pgmap v400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2372714625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:51 smithi093 ceph-mon[111335]: pgmap v400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2372714625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:51 smithi093 ceph-mon[118902]: pgmap v400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2372714625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:51.678Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:51.679Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:53 smithi136 ceph-mon[113531]: pgmap v401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1119416485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-mon[111335]: pgmap v401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1119416485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-mon[118902]: pgmap v401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1119416485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:06:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:53.567Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:06:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:06:53.569Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:06:55.641 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:06:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:06:55] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-27T21:06:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:55 smithi136 ceph-mon[113531]: pgmap v402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:55 smithi093 ceph-mon[111335]: pgmap v402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:55 smithi093 ceph-mon[118902]: pgmap v402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1483149332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1483149332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1483149332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:57 smithi136 ceph-mon[113531]: pgmap v403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:57 smithi093 ceph-mon[111335]: pgmap v403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:57 smithi093 ceph-mon[118902]: pgmap v403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2462232550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2462232550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2462232550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:06:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:06:59 smithi136 ceph-mon[113531]: pgmap v404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:06:59 smithi093 ceph-mon[111335]: pgmap v404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:06:59.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:06:59 smithi093 ceph-mon[118902]: pgmap v404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3214732849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3214732849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3214732849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:01 smithi136 ceph-mon[113531]: pgmap v405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:01 smithi093 ceph-mon[111335]: pgmap v405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:01 smithi093 ceph-mon[118902]: pgmap v405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:01.679Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:01.680Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:03 smithi136 ceph-mon[113531]: pgmap v406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3665353384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:03 smithi093 ceph-mon[118902]: pgmap v406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3665353384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:03 smithi093 ceph-mon[111335]: pgmap v406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3665353384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:03.568Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:03.570Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:05.666 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:07:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:07:05] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-27T21:07:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:05 smithi136 ceph-mon[113531]: pgmap v407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/733575875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:05 smithi093 ceph-mon[111335]: pgmap v407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/733575875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:05 smithi093 ceph-mon[118902]: pgmap v407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/733575875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:07 smithi136 ceph-mon[113531]: pgmap v408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:07 smithi093 ceph-mon[111335]: pgmap v408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:07 smithi093 ceph-mon[118902]: pgmap v408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2190371736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2190371736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2190371736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:09 smithi136 ceph-mon[113531]: pgmap v409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:09 smithi093 ceph-mon[111335]: pgmap v409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:09 smithi093 ceph-mon[118902]: pgmap v409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/449298741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/449298741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/449298741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:11 smithi136 ceph-mon[113531]: pgmap v410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:11 smithi093 ceph-mon[111335]: pgmap v410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:11 smithi093 ceph-mon[118902]: pgmap v410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:11.679Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:11.680Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3158512769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3158512769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:13.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3158512769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:13 smithi093 ceph-mon[111335]: pgmap v411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:13 smithi093 ceph-mon[118902]: pgmap v411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:13.569Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:13.571Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:14.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:13 smithi136 ceph-mon[113531]: pgmap v411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:15.694 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:07:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:07:15] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-27T21:07:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:15 smithi093 ceph-mon[111335]: pgmap v412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1091028689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:15 smithi093 ceph-mon[118902]: pgmap v412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1091028689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:15 smithi136 ceph-mon[113531]: pgmap v412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1091028689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:17 smithi093 ceph-mon[111335]: pgmap v413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3967922128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:17 smithi093 ceph-mon[118902]: pgmap v413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3967922128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:17 smithi136 ceph-mon[113531]: pgmap v413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3967922128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:19 smithi093 ceph-mon[118902]: pgmap v414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/934406378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:19 smithi093 ceph-mon[111335]: pgmap v414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/934406378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:19 smithi136 ceph-mon[113531]: pgmap v414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/934406378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:21 smithi093 ceph-mon[111335]: pgmap v415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:21 smithi093 ceph-mon[118902]: pgmap v415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:21.680Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:21.681Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:21 smithi136 ceph-mon[113531]: pgmap v415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3186641317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3186641317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3186641317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:23 smithi093 ceph-mon[111335]: pgmap v416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:23 smithi093 ceph-mon[118902]: pgmap v416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:23.570Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:23.571Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:23 smithi136 ceph-mon[113531]: pgmap v416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3476636932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3476636932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3476636932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:25.715 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:07:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:07:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-27T21:07:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:25 smithi093 ceph-mon[111335]: pgmap v417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:25 smithi093 ceph-mon[118902]: pgmap v417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:25 smithi136 ceph-mon[113531]: pgmap v417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:27 smithi093 ceph-mon[111335]: pgmap v418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2255842872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:27 smithi093 ceph-mon[118902]: pgmap v418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2255842872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:27 smithi136 ceph-mon[113531]: pgmap v418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2255842872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:29 smithi093 ceph-mon[111335]: pgmap v419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3550912287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:29 smithi093 ceph-mon[118902]: pgmap v419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3550912287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:29 smithi136 ceph-mon[113531]: pgmap v419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3550912287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:31 smithi093 ceph-mon[111335]: pgmap v420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1543731615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:31 smithi093 ceph-mon[118902]: pgmap v420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1543731615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:31.681Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:31.682Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:31 smithi136 ceph-mon[113531]: pgmap v420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1543731615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:33 smithi093 ceph-mon[111335]: pgmap v421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:33 smithi093 ceph-mon[118902]: pgmap v421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:33.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:33.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:33 smithi136 ceph-mon[113531]: pgmap v421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/186433655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/186433655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/186433655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:35.736 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:07:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:07:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-27T21:07:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:35 smithi093 ceph-mon[111335]: pgmap v422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:35 smithi093 ceph-mon[118902]: pgmap v422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:35 smithi136 ceph-mon[113531]: pgmap v422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1234480515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1234480515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1234480515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:37 smithi136 ceph-mon[113531]: pgmap v423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:37 smithi093 ceph-mon[111335]: pgmap v423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:37 smithi093 ceph-mon[118902]: pgmap v423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:39 smithi136 ceph-mon[113531]: pgmap v424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1848503247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:39 smithi093 ceph-mon[111335]: pgmap v424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1848503247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:39 smithi093 ceph-mon[118902]: pgmap v424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1848503247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:41 smithi093 ceph-mon[118902]: pgmap v425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3948836988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:41 smithi093 ceph-mon[111335]: pgmap v425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3948836988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:41.682Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:41.683Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:41 smithi136 ceph-mon[113531]: pgmap v425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3948836988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:43 smithi093 ceph-mon[111335]: pgmap v426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3241932786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:43 smithi093 ceph-mon[118902]: pgmap v426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3241932786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:43.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:43.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:43 smithi136 ceph-mon[113531]: pgmap v426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3241932786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:07:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:07:45] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-27T21:07:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:45 smithi136 ceph-mon[113531]: pgmap v427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:45 smithi093 ceph-mon[111335]: pgmap v427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:45 smithi093 ceph-mon[118902]: pgmap v427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/517514188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/517514188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/517514188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:48.072 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:47 smithi136 ceph-mon[113531]: pgmap v428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:48.072 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:07:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:47 smithi093 ceph-mon[111335]: pgmap v428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:07:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:47 smithi093 ceph-mon[118902]: pgmap v428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:07:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1206634265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1206634265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1206634265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:49 smithi136 ceph-mon[113531]: pgmap v429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:49 smithi093 ceph-mon[118902]: pgmap v429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:49 smithi093 ceph-mon[111335]: pgmap v429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1959640483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:07:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:07:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1959640483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:07:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:07:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:07:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1959640483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:51.683Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:51.684Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:52 smithi136 ceph-mon[113531]: pgmap v430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:52 smithi093 ceph-mon[111335]: pgmap v430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:52 smithi093 ceph-mon[118902]: pgmap v430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:53.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:07:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:07:53.573Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:07:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:54 smithi136 ceph-mon[113531]: pgmap v431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2065128379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:54 smithi093 ceph-mon[118902]: pgmap v431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2065128379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:54 smithi093 ceph-mon[111335]: pgmap v431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2065128379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:07:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:07:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:07:55] "GET /metrics HTTP/1.1" 200 239862 "" "Prometheus/2.43.0" 2023-08-27T21:07:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:56 smithi136 ceph-mon[113531]: pgmap v432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/141707379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:56 smithi093 ceph-mon[111335]: pgmap v432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/141707379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:56 smithi093 ceph-mon[118902]: pgmap v432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/141707379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:58 smithi136 ceph-mon[113531]: pgmap v433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:07:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/685388825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:58 smithi093 ceph-mon[111335]: pgmap v433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:07:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/685388825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:07:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:58 smithi093 ceph-mon[118902]: pgmap v433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:07:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:07:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/685388825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:00 smithi136 ceph-mon[113531]: pgmap v434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:00 smithi093 ceph-mon[111335]: pgmap v434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:00 smithi093 ceph-mon[118902]: pgmap v434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3546573484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3546573484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3546573484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:01.684Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:01.685Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:02.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:02 smithi136 ceph-mon[113531]: pgmap v435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:02 smithi093 ceph-mon[111335]: pgmap v435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:02 smithi093 ceph-mon[118902]: pgmap v435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1269277458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1269277458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1269277458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:03.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:03.573Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:04 smithi136 ceph-mon[113531]: pgmap v436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:04 smithi093 ceph-mon[111335]: pgmap v436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:04 smithi093 ceph-mon[118902]: pgmap v436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:05.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4029947744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4029947744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4029947744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:08:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:08:05] "GET /metrics HTTP/1.1" 200 239862 "" "Prometheus/2.43.0" 2023-08-27T21:08:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:06 smithi136 ceph-mon[113531]: pgmap v437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:06 smithi093 ceph-mon[111335]: pgmap v437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:06 smithi093 ceph-mon[118902]: pgmap v437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:08 smithi136 ceph-mon[113531]: pgmap v438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:08.444 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1754008503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:08 smithi093 ceph-mon[118902]: pgmap v438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1754008503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:08 smithi093 ceph-mon[111335]: pgmap v438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1754008503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:10 smithi136 ceph-mon[113531]: pgmap v439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2175367279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:10 smithi093 ceph-mon[111335]: pgmap v439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2175367279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:10 smithi093 ceph-mon[118902]: pgmap v439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2175367279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:11.685Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:11.686Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:12 smithi136 ceph-mon[113531]: pgmap v440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:12 smithi093 ceph-mon[111335]: pgmap v440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:12 smithi093 ceph-mon[118902]: pgmap v440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/317458921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/317458921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/317458921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:13.573Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:13.574Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:14 smithi136 ceph-mon[113531]: pgmap v441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:14 smithi093 ceph-mon[111335]: pgmap v441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:14 smithi093 ceph-mon[118902]: pgmap v441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3617028294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3617028294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3617028294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:08:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:08:15] "GET /metrics HTTP/1.1" 200 239862 "" "Prometheus/2.43.0" 2023-08-27T21:08:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:16 smithi136 ceph-mon[113531]: pgmap v442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:16 smithi093 ceph-mon[111335]: pgmap v442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:16 smithi093 ceph-mon[118902]: pgmap v442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4272033383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4272033383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4272033383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:18 smithi136 ceph-mon[113531]: pgmap v443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:18 smithi093 ceph-mon[111335]: pgmap v443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:18 smithi093 ceph-mon[118902]: pgmap v443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:20 smithi136 ceph-mon[113531]: pgmap v444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1217494662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:20 smithi093 ceph-mon[111335]: pgmap v444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1217494662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:20 smithi093 ceph-mon[118902]: pgmap v444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1217494662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:21.686Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:21.687Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:22 smithi136 ceph-mon[113531]: pgmap v445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1979646748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:22 smithi093 ceph-mon[111335]: pgmap v445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1979646748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:22 smithi093 ceph-mon[118902]: pgmap v445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1979646748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:23.574Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:23.575Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:24 smithi136 ceph-mon[113531]: pgmap v446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/739013175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:24 smithi093 ceph-mon[118902]: pgmap v446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/739013175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:24 smithi093 ceph-mon[111335]: pgmap v446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/739013175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:08:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:08:25] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:08:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:26 smithi136 ceph-mon[113531]: pgmap v447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:26 smithi093 ceph-mon[111335]: pgmap v447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:26 smithi093 ceph-mon[118902]: pgmap v447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3929095508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3929095508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3929095508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:28 smithi136 ceph-mon[113531]: pgmap v448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:28 smithi093 ceph-mon[111335]: pgmap v448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:28 smithi093 ceph-mon[118902]: pgmap v448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:29 smithi136 ceph-mon[113531]: pgmap v449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2796565425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:29 smithi093 ceph-mon[111335]: pgmap v449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2796565425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:29 smithi093 ceph-mon[118902]: pgmap v449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2796565425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:31 smithi136 ceph-mon[113531]: pgmap v450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1974656157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:31 smithi093 ceph-mon[118902]: pgmap v450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1974656157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:31 smithi093 ceph-mon[111335]: pgmap v450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1974656157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:31.687Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:31.687Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:33 smithi136 ceph-mon[113531]: pgmap v451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3994850108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:33 smithi093 ceph-mon[111335]: pgmap v451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3994850108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:33 smithi093 ceph-mon[118902]: pgmap v451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3994850108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:33.574Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:33.575Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:35.656 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:08:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:08:35] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:08:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:35 smithi136 ceph-mon[113531]: pgmap v452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:35 smithi093 ceph-mon[111335]: pgmap v452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:35 smithi093 ceph-mon[118902]: pgmap v452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2528544118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2528544118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2528544118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:37 smithi136 ceph-mon[113531]: pgmap v453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:37 smithi093 ceph-mon[111335]: pgmap v453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:37 smithi093 ceph-mon[118902]: pgmap v453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3475762245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3475762245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3475762245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:39 smithi136 ceph-mon[113531]: pgmap v454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:39 smithi093 ceph-mon[111335]: pgmap v454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:39 smithi093 ceph-mon[118902]: pgmap v454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:41 smithi136 ceph-mon[113531]: pgmap v455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2464730501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:41 smithi093 ceph-mon[111335]: pgmap v455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2464730501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:41 smithi093 ceph-mon[118902]: pgmap v455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2464730501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:41.687Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:41.688Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:43 smithi136 ceph-mon[113531]: pgmap v456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2627342505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:43 smithi093 ceph-mon[111335]: pgmap v456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2627342505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:43 smithi093 ceph-mon[118902]: pgmap v456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2627342505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:43.575Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:43.576Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:45.683 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:08:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:08:45] "GET /metrics HTTP/1.1" 200 239861 "" "Prometheus/2.43.0" 2023-08-27T21:08:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:45 smithi136 ceph-mon[113531]: pgmap v457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4125934777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:45 smithi093 ceph-mon[118902]: pgmap v457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4125934777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:45 smithi093 ceph-mon[111335]: pgmap v457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4125934777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:47 smithi136 ceph-mon[113531]: pgmap v458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:47 smithi093 ceph-mon[111335]: pgmap v458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:47 smithi093 ceph-mon[118902]: pgmap v458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1041957667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1041957667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1041957667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:49 smithi093 ceph-mon[111335]: pgmap v459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:49 smithi093 ceph-mon[118902]: pgmap v459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:49 smithi136 ceph-mon[113531]: pgmap v459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/77035321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:08:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/77035321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:08:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/77035321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:08:51.945 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:51.688Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:51.945 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:51.689Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:51 smithi136 ceph-mon[113531]: pgmap v460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-mon[111335]: pgmap v460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-mon[118902]: pgmap v460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[111335]: pgmap v461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1556724981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[118902]: pgmap v461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1556724981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:53.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:53.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:08:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:08:53.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:08:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:53 smithi136 ceph-mon[113531]: pgmap v461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1556724981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:08:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:08:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:08:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:08:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:08:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:08:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:08:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:08:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:55 smithi093 ceph-mon[111335]: pgmap v462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2526664786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:55 smithi093 ceph-mon[118902]: pgmap v462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2526664786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:08:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:08:55] "GET /metrics HTTP/1.1" 200 239862 "" "Prometheus/2.43.0" 2023-08-27T21:08:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:55 smithi136 ceph-mon[113531]: pgmap v462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2526664786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:57 smithi136 ceph-mon[113531]: pgmap v463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/276156393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:57 smithi093 ceph-mon[111335]: pgmap v463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/276156393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:57 smithi093 ceph-mon[118902]: pgmap v463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/276156393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:08:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:08:59 smithi136 ceph-mon[113531]: pgmap v464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:08:59 smithi093 ceph-mon[111335]: pgmap v464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:08:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:08:59 smithi093 ceph-mon[118902]: pgmap v464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/128661758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/128661758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/128661758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:01 smithi136 ceph-mon[113531]: pgmap v465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:01 smithi093 ceph-mon[111335]: pgmap v465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:01 smithi093 ceph-mon[118902]: pgmap v465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:01.688Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:01.690Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3124740545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3124740545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3124740545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:03 smithi136 ceph-mon[113531]: pgmap v466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:03 smithi093 ceph-mon[111335]: pgmap v466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:03 smithi093 ceph-mon[118902]: pgmap v466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:03.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:03.578Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3942353496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3942353496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3942353496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:05.679 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:09:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:09:05] "GET /metrics HTTP/1.1" 200 239862 "" "Prometheus/2.43.0" 2023-08-27T21:09:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:05 smithi136 ceph-mon[113531]: pgmap v467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:05 smithi093 ceph-mon[111335]: pgmap v467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:05 smithi093 ceph-mon[118902]: pgmap v467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:07 smithi136 ceph-mon[113531]: pgmap v468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2245117363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:07 smithi093 ceph-mon[111335]: pgmap v468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2245117363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:07 smithi093 ceph-mon[118902]: pgmap v468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2245117363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:09 smithi093 ceph-mon[111335]: pgmap v469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2462535904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:09 smithi093 ceph-mon[118902]: pgmap v469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2462535904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:09 smithi136 ceph-mon[113531]: pgmap v469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2462535904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:11 smithi093 ceph-mon[111335]: pgmap v470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:11 smithi093 ceph-mon[118902]: pgmap v470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:11.689Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:11.690Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:11 smithi136 ceph-mon[113531]: pgmap v470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3897981618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3897981618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3897981618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:13 smithi093 ceph-mon[111335]: pgmap v471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:13 smithi093 ceph-mon[118902]: pgmap v471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:13.577Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:13.578Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:13 smithi136 ceph-mon[113531]: pgmap v471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/338065352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/338065352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/338065352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:15.712 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:09:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:09:15] "GET /metrics HTTP/1.1" 200 239860 "" "Prometheus/2.43.0" 2023-08-27T21:09:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:15 smithi093 ceph-mon[118902]: pgmap v472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:15 smithi093 ceph-mon[111335]: pgmap v472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:15 smithi136 ceph-mon[113531]: pgmap v472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/609490036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/609490036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/609490036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:17 smithi093 ceph-mon[111335]: pgmap v473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:17 smithi093 ceph-mon[118902]: pgmap v473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:17 smithi136 ceph-mon[113531]: pgmap v473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:19 smithi093 ceph-mon[111335]: pgmap v474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2272050086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:19 smithi093 ceph-mon[118902]: pgmap v474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2272050086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:19 smithi136 ceph-mon[113531]: pgmap v474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2272050086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:21 smithi093 ceph-mon[111335]: pgmap v475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1909138056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:21 smithi093 ceph-mon[118902]: pgmap v475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1909138056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:21.690Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:21.691Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:21 smithi136 ceph-mon[113531]: pgmap v475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1909138056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-mon[118902]: pgmap v476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1884164072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-mon[111335]: pgmap v476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1884164072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:23.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:23.578Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:23.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:23.579Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:23 smithi136 ceph-mon[113531]: pgmap v476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1884164072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:25.729 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:09:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:09:25] "GET /metrics HTTP/1.1" 200 239860 "" "Prometheus/2.43.0" 2023-08-27T21:09:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:25 smithi093 ceph-mon[111335]: pgmap v477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:25 smithi093 ceph-mon[118902]: pgmap v477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:25 smithi136 ceph-mon[113531]: pgmap v477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1809205278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1809205278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:27.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1809205278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:27 smithi093 ceph-mon[111335]: pgmap v478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:27 smithi093 ceph-mon[118902]: pgmap v478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:27 smithi136 ceph-mon[113531]: pgmap v478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2906262293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2906262293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2906262293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:29 smithi136 ceph-mon[113531]: pgmap v479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:29 smithi093 ceph-mon[111335]: pgmap v479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:29 smithi093 ceph-mon[118902]: pgmap v479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:31 smithi093 ceph-mon[111335]: pgmap v480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1632297484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:31 smithi093 ceph-mon[118902]: pgmap v480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1632297484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:31.691Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:31.692Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:31 smithi136 ceph-mon[113531]: pgmap v480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1632297484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:33 smithi093 ceph-mon[111335]: pgmap v481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4010611480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:33 smithi093 ceph-mon[118902]: pgmap v481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4010611480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:33.578Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:33.579Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:33 smithi136 ceph-mon[113531]: pgmap v481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4010611480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:09:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:09:35] "GET /metrics HTTP/1.1" 200 239860 "" "Prometheus/2.43.0" 2023-08-27T21:09:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:35 smithi136 ceph-mon[113531]: pgmap v482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3020338601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:35 smithi093 ceph-mon[111335]: pgmap v482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3020338601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:35 smithi093 ceph-mon[118902]: pgmap v482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3020338601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:37 smithi136 ceph-mon[113531]: pgmap v483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:37 smithi093 ceph-mon[111335]: pgmap v483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:37 smithi093 ceph-mon[118902]: pgmap v483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2602086810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2602086810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2602086810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:40.142 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:39 smithi093 ceph-mon[111335]: pgmap v484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:40.142 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:39 smithi093 ceph-mon[118902]: pgmap v484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:39 smithi136 ceph-mon[113531]: pgmap v484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:40.142Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/493786423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/493786423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/493786423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:41 smithi093 ceph-mon[111335]: pgmap v485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:41 smithi093 ceph-mon[118902]: pgmap v485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:41.691Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:41.692Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:41 smithi136 ceph-mon[113531]: pgmap v485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3153632527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3153632527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3153632527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:43 smithi093 ceph-mon[111335]: pgmap v486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:43 smithi093 ceph-mon[118902]: pgmap v486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:43.579Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:43.580Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:43 smithi136 ceph-mon[113531]: pgmap v486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:09:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:09:45] "GET /metrics HTTP/1.1" 200 239864 "" "Prometheus/2.43.0" 2023-08-27T21:09:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:45 smithi136 ceph-mon[113531]: pgmap v487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2571618954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:45 smithi093 ceph-mon[111335]: pgmap v487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2571618954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:45 smithi093 ceph-mon[118902]: pgmap v487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2571618954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:47 smithi136 ceph-mon[113531]: pgmap v488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3465201161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:47 smithi093 ceph-mon[111335]: pgmap v488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3465201161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:47 smithi093 ceph-mon[118902]: pgmap v488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3465201161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:50.141 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:49 smithi093 ceph-mon[111335]: pgmap v489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:50.142 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:49 smithi093 ceph-mon[118902]: pgmap v489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:49 smithi136 ceph-mon[113531]: pgmap v489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:50.141Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:50.142Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1991540334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1991540334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1991540334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:51 smithi093 ceph-mon[111335]: pgmap v490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:51 smithi093 ceph-mon[118902]: pgmap v490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:51.692Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:51.693Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:51 smithi136 ceph-mon[113531]: pgmap v490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3723908287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3723908287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3723908287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:53 smithi093 ceph-mon[111335]: pgmap v491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:53 smithi093 ceph-mon[118902]: pgmap v491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:53.580Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:09:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:09:53.581Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:09:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:53 smithi136 ceph-mon[113531]: pgmap v491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:09:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:09:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3269525313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:09:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3269525313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:09:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3269525313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:55.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:09:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:09:55] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:09:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:55 smithi093 ceph-mon[111335]: pgmap v492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:55 smithi093 ceph-mon[118902]: pgmap v492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:55 smithi136 ceph-mon[113531]: pgmap v492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:57 smithi136 ceph-mon[113531]: pgmap v493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:09:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:09:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/47340501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[111335]: pgmap v493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:09:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:09:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/47340501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:09:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[118902]: pgmap v493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:09:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:09:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:09:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:09:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/47340501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:00.142 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:59 smithi093 ceph-mon[118902]: pgmap v494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:00.143 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:09:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3566527879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:00.143 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:59 smithi093 ceph-mon[111335]: pgmap v494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:00.143 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:09:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3566527879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:59 smithi136 ceph-mon[113531]: pgmap v494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:09:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3566527879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:00.142Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:00.143Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:10:01.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:10:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:10:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:01 smithi093 ceph-mon[111335]: pgmap v495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2068943533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:01 smithi093 ceph-mon[118902]: pgmap v495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2068943533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:01.693Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:01.693Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:01 smithi136 ceph-mon[113531]: pgmap v495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2068943533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:03.877 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:03.581Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:03.877 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:03.582Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:03 smithi136 ceph-mon[113531]: pgmap v496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:03 smithi093 ceph-mon[111335]: pgmap v496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:03 smithi093 ceph-mon[118902]: pgmap v496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1291607796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1291607796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1291607796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:10:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:10:05] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:10:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:05 smithi136 ceph-mon[113531]: pgmap v497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:05 smithi093 ceph-mon[111335]: pgmap v497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:05 smithi093 ceph-mon[118902]: pgmap v497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1515410439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1515410439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1515410439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:07 smithi136 ceph-mon[113531]: pgmap v498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:07 smithi093 ceph-mon[111335]: pgmap v498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:07 smithi093 ceph-mon[118902]: pgmap v498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:09 smithi136 ceph-mon[113531]: pgmap v499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3252377409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:09 smithi093 ceph-mon[111335]: pgmap v499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3252377409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:09 smithi093 ceph-mon[118902]: pgmap v499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3252377409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:10.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:10.143Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:10.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:10.144Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:11 smithi093 ceph-mon[111335]: pgmap v500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3463591392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:11 smithi093 ceph-mon[118902]: pgmap v500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3463591392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:11.693Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:11.694Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:11 smithi136 ceph-mon[113531]: pgmap v500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3463591392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:13.908 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:13.581Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:13.908 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:13.582Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:13 smithi136 ceph-mon[113531]: pgmap v501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/629884407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:13 smithi093 ceph-mon[111335]: pgmap v501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/629884407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:13 smithi093 ceph-mon[118902]: pgmap v501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/629884407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:10:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:10:15] "GET /metrics HTTP/1.1" 200 239865 "" "Prometheus/2.43.0" 2023-08-27T21:10:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:15 smithi136 ceph-mon[113531]: pgmap v502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:15 smithi093 ceph-mon[111335]: pgmap v502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:15 smithi093 ceph-mon[118902]: pgmap v502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/656605990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/656605990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/656605990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:17 smithi136 ceph-mon[113531]: pgmap v503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:17 smithi093 ceph-mon[111335]: pgmap v503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:17 smithi093 ceph-mon[118902]: pgmap v503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2246414890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2246414890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2246414890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:19 smithi136 ceph-mon[113531]: pgmap v504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:19 smithi093 ceph-mon[111335]: pgmap v504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:19 smithi093 ceph-mon[118902]: pgmap v504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:20.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:20.143Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:20.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:20.144Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3531614088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3531614088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3531614088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:21 smithi093 ceph-mon[111335]: pgmap v505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:21 smithi093 ceph-mon[118902]: pgmap v505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:21.693Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:21.694Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:21 smithi136 ceph-mon[113531]: pgmap v505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:23.930 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:23.582Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:23.930 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:23.582Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:23 smithi136 ceph-mon[113531]: pgmap v506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1443606947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-mon[111335]: pgmap v506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1443606947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-mon[118902]: pgmap v506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1443606947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:10:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:10:25] "GET /metrics HTTP/1.1" 200 239860 "" "Prometheus/2.43.0" 2023-08-27T21:10:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:25 smithi136 ceph-mon[113531]: pgmap v507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2440864020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:25 smithi093 ceph-mon[111335]: pgmap v507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2440864020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:25 smithi093 ceph-mon[118902]: pgmap v507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2440864020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:27 smithi136 ceph-mon[113531]: pgmap v508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:27 smithi093 ceph-mon[111335]: pgmap v508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:27 smithi093 ceph-mon[118902]: pgmap v508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1755191281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1755191281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1755191281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:29 smithi093 ceph-mon[111335]: pgmap v509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:29 smithi093 ceph-mon[118902]: pgmap v509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:30.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:30.145Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:30.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:30.146Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:29 smithi136 ceph-mon[113531]: pgmap v509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2397634031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2397634031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2397634031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:31.955 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:31.694Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:31.955 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:31.695Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:31 smithi093 ceph-mon[111335]: pgmap v510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:31 smithi093 ceph-mon[118902]: pgmap v510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:31 smithi136 ceph-mon[113531]: pgmap v510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/526420445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/526420445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/526420445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:33.963 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:33.583Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:33.963 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:33.584Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:33 smithi093 ceph-mon[111335]: pgmap v511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:33 smithi093 ceph-mon[118902]: pgmap v511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:33 smithi136 ceph-mon[113531]: pgmap v511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:10:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:10:35] "GET /metrics HTTP/1.1" 200 239860 "" "Prometheus/2.43.0" 2023-08-27T21:10:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:35 smithi093 ceph-mon[111335]: pgmap v512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/731323312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:35 smithi093 ceph-mon[118902]: pgmap v512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/731323312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:35 smithi136 ceph-mon[113531]: pgmap v512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/731323312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:37 smithi093 ceph-mon[111335]: pgmap v513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/265084606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:37 smithi093 ceph-mon[118902]: pgmap v513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/265084606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:37 smithi136 ceph-mon[113531]: pgmap v513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/265084606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:39 smithi093 ceph-mon[111335]: pgmap v514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2506911122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:39 smithi093 ceph-mon[118902]: pgmap v514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2506911122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:40.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:40.145Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:40.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:40.146Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:39 smithi136 ceph-mon[113531]: pgmap v514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2506911122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:41.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:41.695Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:41.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:41.696Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:41 smithi093 ceph-mon[111335]: pgmap v515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:41 smithi093 ceph-mon[118902]: pgmap v515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:41 smithi136 ceph-mon[113531]: pgmap v515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2316914740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2316914740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2316914740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:43.583Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:43.584Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:43 smithi136 ceph-mon[113531]: pgmap v516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:43 smithi093 ceph-mon[111335]: pgmap v516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:43 smithi093 ceph-mon[118902]: pgmap v516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/78114629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/78114629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/78114629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:10:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:10:45] "GET /metrics HTTP/1.1" 200 239867 "" "Prometheus/2.43.0" 2023-08-27T21:10:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:46 smithi136 ceph-mon[113531]: pgmap v517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:46 smithi093 ceph-mon[111335]: pgmap v517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:46 smithi093 ceph-mon[118902]: pgmap v517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:48 smithi136 ceph-mon[113531]: pgmap v518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3703373949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:48 smithi093 ceph-mon[118902]: pgmap v518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3703373949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:48 smithi093 ceph-mon[111335]: pgmap v518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3703373949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:50 smithi136 ceph-mon[113531]: pgmap v519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1541365656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:50 smithi093 ceph-mon[111335]: pgmap v519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1541365656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:50 smithi093 ceph-mon[118902]: pgmap v519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1541365656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:50.145Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:50.146Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:51.696Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:51.697Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:52 smithi136 ceph-mon[113531]: pgmap v520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1010845257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:52 smithi093 ceph-mon[111335]: pgmap v520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1010845257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:52 smithi093 ceph-mon[118902]: pgmap v520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1010845257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:53.584Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:10:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:10:53.584Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:10:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:54 smithi136 ceph-mon[113531]: pgmap v521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:54 smithi093 ceph-mon[111335]: pgmap v521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:54 smithi093 ceph-mon[118902]: pgmap v521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:10:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3422321043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3422321043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3422321043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:10:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:10:55] "GET /metrics HTTP/1.1" 200 239864 "" "Prometheus/2.43.0" 2023-08-27T21:10:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:56 smithi136 ceph-mon[113531]: pgmap v522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:56 smithi093 ceph-mon[111335]: pgmap v522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:56 smithi093 ceph-mon[118902]: pgmap v522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2286909713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2286909713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2286909713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:10:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:58 smithi136 ceph-mon[113531]: pgmap v523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:10:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:58 smithi093 ceph-mon[111335]: pgmap v523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:10:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:58 smithi093 ceph-mon[118902]: pgmap v523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:10:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:11:00.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:00.146Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:00.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:00.147Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: pgmap v524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3035685508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:11:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: pgmap v524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3035685508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: pgmap v524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3035685508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:11:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:11:01.950 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:01.697Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:01.950 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:01.698Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:01 smithi093 ceph-mon[111335]: pgmap v525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3051041221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:01 smithi093 ceph-mon[118902]: pgmap v525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3051041221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:01 smithi136 ceph-mon[113531]: pgmap v525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3051041221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:03.953 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:03.584Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:03.953 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:03.585Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:03 smithi093 ceph-mon[111335]: pgmap v526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1620459773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:03 smithi093 ceph-mon[118902]: pgmap v526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1620459773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:03 smithi136 ceph-mon[113531]: pgmap v526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1620459773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T21:11:05.317563456Z level=info msg="Completed cleanup jobs" duration=12.969877ms 2023-08-27T21:11:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:11:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:11:05] "GET /metrics HTTP/1.1" 200 239864 "" "Prometheus/2.43.0" 2023-08-27T21:11:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:05 smithi093 ceph-mon[111335]: pgmap v527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:05 smithi093 ceph-mon[118902]: pgmap v527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:05 smithi136 ceph-mon[113531]: pgmap v527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2824438903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2824438903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2824438903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:07 smithi093 ceph-mon[111335]: pgmap v528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:07 smithi093 ceph-mon[118902]: pgmap v528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:07 smithi136 ceph-mon[113531]: pgmap v528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4260856729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4260856729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4260856729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:10.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:09 smithi093 ceph-mon[111335]: pgmap v529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:10.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:09 smithi093 ceph-mon[118902]: pgmap v529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:10.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:10.147Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:10.243 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:10.148Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:09 smithi136 ceph-mon[113531]: pgmap v529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:11.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:11.698Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:11.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:11.699Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:11 smithi093 ceph-mon[111335]: pgmap v530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3460311396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:12.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:11 smithi093 ceph-mon[118902]: pgmap v530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3460311396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:11 smithi136 ceph-mon[113531]: pgmap v530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3460311396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:13.975 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:13.584Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:13.975 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:13.585Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:13 smithi093 ceph-mon[111335]: pgmap v531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/558809375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:13 smithi093 ceph-mon[118902]: pgmap v531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/558809375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:13 smithi136 ceph-mon[113531]: pgmap v531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/558809375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:11:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:11:15] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:11:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:15 smithi093 ceph-mon[118902]: pgmap v532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/548224697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:15 smithi093 ceph-mon[111335]: pgmap v532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/548224697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:15 smithi136 ceph-mon[113531]: pgmap v532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/548224697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:17 smithi093 ceph-mon[111335]: pgmap v533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:17 smithi093 ceph-mon[118902]: pgmap v533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:17 smithi136 ceph-mon[113531]: pgmap v533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/366043381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/366043381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/366043381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:19 smithi093 ceph-mon[111335]: pgmap v534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:19 smithi093 ceph-mon[118902]: pgmap v534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:20.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:20.147Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:20.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:20.148Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:19 smithi136 ceph-mon[113531]: pgmap v534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/325251004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/325251004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/325251004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:21.699Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:21.700Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:21 smithi136 ceph-mon[113531]: pgmap v535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:21 smithi093 ceph-mon[111335]: pgmap v535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:21 smithi093 ceph-mon[118902]: pgmap v535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1150564580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1150564580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1150564580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:23.586Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:23.586Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:23 smithi136 ceph-mon[113531]: pgmap v536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:23 smithi093 ceph-mon[111335]: pgmap v536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:23 smithi093 ceph-mon[118902]: pgmap v536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:11:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:11:25] "GET /metrics HTTP/1.1" 200 239868 "" "Prometheus/2.43.0" 2023-08-27T21:11:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:26 smithi136 ceph-mon[113531]: pgmap v537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1623543699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:26 smithi093 ceph-mon[111335]: pgmap v537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1623543699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:26 smithi093 ceph-mon[118902]: pgmap v537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1623543699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:28 smithi136 ceph-mon[113531]: pgmap v538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2487611499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:28 smithi093 ceph-mon[111335]: pgmap v538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2487611499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:28 smithi093 ceph-mon[118902]: pgmap v538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2487611499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:30 smithi136 ceph-mon[113531]: pgmap v539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:30 smithi093 ceph-mon[111335]: pgmap v539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:30.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:30 smithi093 ceph-mon[118902]: pgmap v539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:30.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:30.148Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:30.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:30.149Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3249707781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3249707781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3249707781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:31.699Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:31.700Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:32 smithi136 ceph-mon[113531]: pgmap v540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:32 smithi093 ceph-mon[111335]: pgmap v540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:32 smithi093 ceph-mon[118902]: pgmap v540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2544325570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2544325570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2544325570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:33.587Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:33.588Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:34 smithi136 ceph-mon[113531]: pgmap v541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:34 smithi093 ceph-mon[111335]: pgmap v541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:34 smithi093 ceph-mon[118902]: pgmap v541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/340627281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/340627281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/340627281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:11:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:11:35] "GET /metrics HTTP/1.1" 200 239868 "" "Prometheus/2.43.0" 2023-08-27T21:11:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:36 smithi136 ceph-mon[113531]: pgmap v542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:36 smithi093 ceph-mon[111335]: pgmap v542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:36 smithi093 ceph-mon[118902]: pgmap v542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:38 smithi136 ceph-mon[113531]: pgmap v543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1080805948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:38 smithi093 ceph-mon[111335]: pgmap v543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1080805948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:38 smithi093 ceph-mon[118902]: pgmap v543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1080805948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:40 smithi136 ceph-mon[113531]: pgmap v544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4088949915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:40 smithi093 ceph-mon[111335]: pgmap v544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4088949915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:40 smithi093 ceph-mon[118902]: pgmap v544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4088949915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:40.149Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:40.150Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:41.700Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:41.701Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:42 smithi136 ceph-mon[113531]: pgmap v545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1543147332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:42 smithi093 ceph-mon[111335]: pgmap v545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1543147332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:42 smithi093 ceph-mon[118902]: pgmap v545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1543147332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:43.588Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:43.589Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:44 smithi136 ceph-mon[113531]: pgmap v546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:44 smithi093 ceph-mon[111335]: pgmap v546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:44 smithi093 ceph-mon[118902]: pgmap v546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1566743359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1566743359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1566743359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:11:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:11:45] "GET /metrics HTTP/1.1" 200 239869 "" "Prometheus/2.43.0" 2023-08-27T21:11:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:46 smithi136 ceph-mon[113531]: pgmap v547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:46 smithi093 ceph-mon[111335]: pgmap v547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:46 smithi093 ceph-mon[118902]: pgmap v547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1722446342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1722446342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1722446342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:48 smithi136 ceph-mon[113531]: pgmap v548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:48 smithi093 ceph-mon[111335]: pgmap v548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:48 smithi093 ceph-mon[118902]: pgmap v548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:50 smithi136 ceph-mon[113531]: pgmap v549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/198960119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:50 smithi093 ceph-mon[111335]: pgmap v549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/198960119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:50 smithi093 ceph-mon[118902]: pgmap v549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/198960119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:50.150Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:50.151Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:51.701Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:51.702Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:52 smithi136 ceph-mon[113531]: pgmap v550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3979681713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:52 smithi093 ceph-mon[118902]: pgmap v550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3979681713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:52 smithi093 ceph-mon[111335]: pgmap v550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3979681713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:53.588Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:11:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:11:53.589Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:11:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:54 smithi136 ceph-mon[113531]: pgmap v551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2715339966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:54 smithi093 ceph-mon[111335]: pgmap v551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2715339966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:54 smithi093 ceph-mon[118902]: pgmap v551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:11:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2715339966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:11:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:11:55] "GET /metrics HTTP/1.1" 200 239867 "" "Prometheus/2.43.0" 2023-08-27T21:11:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:56 smithi136 ceph-mon[113531]: pgmap v552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:56 smithi093 ceph-mon[111335]: pgmap v552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:56 smithi093 ceph-mon[118902]: pgmap v552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2321818211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2321818211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2321818211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:58 smithi136 ceph-mon[113531]: pgmap v553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:58 smithi093 ceph-mon[111335]: pgmap v553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:58 smithi093 ceph-mon[118902]: pgmap v553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:11:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:11:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4286101237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:11:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4286101237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:11:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:11:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4286101237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:00 smithi136 ceph-mon[113531]: pgmap v554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:12:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:00 smithi093 ceph-mon[111335]: pgmap v554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:12:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:00 smithi093 ceph-mon[118902]: pgmap v554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:12:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:00.150Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:00.152Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:01.702Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:01.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:02.329 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:02 smithi136 ceph-mon[113531]: pgmap v555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:02.329 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/486454815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:02.329 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:02.329 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[111335]: pgmap v555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/486454815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[118902]: pgmap v555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/486454815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:03 smithi136 ceph-mon[113531]: pgmap v556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:12:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:12:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1759199714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:03.588Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:03.589Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[111335]: pgmap v556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1759199714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[118902]: pgmap v556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:03.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:12:03.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:12:03.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:12:03.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1759199714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:05.689 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:12:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:12:05] "GET /metrics HTTP/1.1" 200 239867 "" "Prometheus/2.43.0" 2023-08-27T21:12:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:05 smithi093 ceph-mon[111335]: pgmap v557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:05 smithi093 ceph-mon[118902]: pgmap v557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:05 smithi136 ceph-mon[113531]: pgmap v557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3211388601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:06.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3211388601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3211388601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:07 smithi093 ceph-mon[111335]: pgmap v558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:07 smithi093 ceph-mon[118902]: pgmap v558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:07 smithi136 ceph-mon[113531]: pgmap v558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/668077640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/668077640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/668077640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:09 smithi093 ceph-mon[111335]: pgmap v559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:10.531 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:09 smithi093 ceph-mon[118902]: pgmap v559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:10.532 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:10.151Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:10.532 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:10.152Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:10.532 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:09 smithi136 ceph-mon[113531]: pgmap v559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2395446350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:11.214 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2395446350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:11.215 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2395446350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:11 smithi093 ceph-mon[111335]: pgmap v560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:11 smithi093 ceph-mon[118902]: pgmap v560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:11.703Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:11.704Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:11 smithi136 ceph-mon[113531]: pgmap v560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:13 smithi093 ceph-mon[111335]: pgmap v561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3791903203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:13 smithi093 ceph-mon[118902]: pgmap v561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3791903203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:13.589Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:13.590Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:13 smithi136 ceph-mon[113531]: pgmap v561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3791903203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:15.714 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:12:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:12:15] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:12:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:15 smithi093 ceph-mon[118902]: pgmap v562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2788231076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:15 smithi093 ceph-mon[111335]: pgmap v562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2788231076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:15 smithi136 ceph-mon[113531]: pgmap v562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2788231076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:17 smithi093 ceph-mon[111335]: pgmap v563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:17 smithi093 ceph-mon[118902]: pgmap v563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:17 smithi136 ceph-mon[113531]: pgmap v563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/995442283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/995442283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/995442283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:19 smithi093 ceph-mon[111335]: pgmap v564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:19 smithi093 ceph-mon[118902]: pgmap v564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:19 smithi136 ceph-mon[113531]: pgmap v564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:20.152Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:20.153Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/874994290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:21.430 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/874994290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:21.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/874994290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:21 smithi093 ceph-mon[111335]: pgmap v565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:21 smithi093 ceph-mon[118902]: pgmap v565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:21.703Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:21.704Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:21 smithi136 ceph-mon[113531]: pgmap v565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1891651004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1891651004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1891651004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:23 smithi093 ceph-mon[111335]: pgmap v566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:23 smithi093 ceph-mon[118902]: pgmap v566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:23.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:23.590Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:23.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:23.591Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:23 smithi136 ceph-mon[113531]: pgmap v566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:12:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:12:25] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:12:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:25 smithi136 ceph-mon[113531]: pgmap v567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4077562508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:25 smithi093 ceph-mon[111335]: pgmap v567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4077562508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:25 smithi093 ceph-mon[118902]: pgmap v567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4077562508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:27 smithi136 ceph-mon[113531]: pgmap v568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2684908435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:27 smithi093 ceph-mon[111335]: pgmap v568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2684908435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:27 smithi093 ceph-mon[118902]: pgmap v568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2684908435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:30.153 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:29 smithi093 ceph-mon[111335]: pgmap v569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:30.153 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3574121883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:30.153 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:29 smithi093 ceph-mon[118902]: pgmap v569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:30.153 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3574121883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:29 smithi136 ceph-mon[113531]: pgmap v569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3574121883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:30.153Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:30.153Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:31 smithi093 ceph-mon[111335]: pgmap v570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:31 smithi093 ceph-mon[118902]: pgmap v570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:31.704Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:31.705Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:31 smithi136 ceph-mon[113531]: pgmap v570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3987948639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3987948639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3987948639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:33 smithi093 ceph-mon[111335]: pgmap v571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:33 smithi093 ceph-mon[118902]: pgmap v571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:33.591Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:33.592Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:33 smithi136 ceph-mon[113531]: pgmap v571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4067216596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4067216596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4067216596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:12:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:12:35] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:12:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:35 smithi136 ceph-mon[113531]: pgmap v572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:35 smithi093 ceph-mon[111335]: pgmap v572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:35 smithi093 ceph-mon[118902]: pgmap v572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:37 smithi136 ceph-mon[113531]: pgmap v573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/649713916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:37 smithi093 ceph-mon[111335]: pgmap v573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/649713916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:37 smithi093 ceph-mon[118902]: pgmap v573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/649713916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:40.153 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:39 smithi093 ceph-mon[111335]: pgmap v574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:40.153 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2922710148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:40.154 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:39 smithi093 ceph-mon[118902]: pgmap v574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:40.154 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2922710148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:39 smithi136 ceph-mon[113531]: pgmap v574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2922710148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:40.153Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:40.154Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:41 smithi093 ceph-mon[118902]: pgmap v575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4243835820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:41 smithi093 ceph-mon[111335]: pgmap v575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4243835820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:41.705Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:41.706Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:41 smithi136 ceph-mon[113531]: pgmap v575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4243835820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:43 smithi093 ceph-mon[111335]: pgmap v576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:43 smithi093 ceph-mon[118902]: pgmap v576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:43.591Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:43.592Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:43 smithi136 ceph-mon[113531]: pgmap v576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1310394052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1310394052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1310394052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:12:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:12:45] "GET /metrics HTTP/1.1" 200 239865 "" "Prometheus/2.43.0" 2023-08-27T21:12:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:45 smithi136 ceph-mon[113531]: pgmap v577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:45 smithi093 ceph-mon[111335]: pgmap v577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:45 smithi093 ceph-mon[118902]: pgmap v577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/83639510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/83639510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/83639510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:47 smithi136 ceph-mon[113531]: pgmap v578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:47 smithi093 ceph-mon[111335]: pgmap v578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:47 smithi093 ceph-mon[118902]: pgmap v578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:49 smithi093 ceph-mon[111335]: pgmap v579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/427470349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:49 smithi093 ceph-mon[118902]: pgmap v579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/427470349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:49 smithi136 ceph-mon[113531]: pgmap v579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/427470349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:50.154Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:50.155Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:51 smithi093 ceph-mon[111335]: pgmap v580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2516542303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:51 smithi093 ceph-mon[118902]: pgmap v580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2516542303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:51.706Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:51.706Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:51 smithi136 ceph-mon[113531]: pgmap v580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2516542303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-mon[111335]: pgmap v581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3299091106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-mon[118902]: pgmap v581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3299091106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:53.592Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:12:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:12:53.593Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:12:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:53 smithi136 ceph-mon[113531]: pgmap v581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3299091106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:12:55.701 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:12:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:12:55] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:12:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:55 smithi093 ceph-mon[111335]: pgmap v582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:55 smithi093 ceph-mon[118902]: pgmap v582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:55 smithi136 ceph-mon[113531]: pgmap v582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2134954226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2134954226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2134954226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:57 smithi093 ceph-mon[111335]: pgmap v583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:57 smithi093 ceph-mon[118902]: pgmap v583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:57 smithi136 ceph-mon[113531]: pgmap v583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4163004472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4163004472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4163004472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:12:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:12:59 smithi093 ceph-mon[111335]: pgmap v584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:12:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:12:59 smithi093 ceph-mon[118902]: pgmap v584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:12:59 smithi136 ceph-mon[113531]: pgmap v584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:00.155Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:00.156Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1435640326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1435640326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1435640326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:01 smithi093 ceph-mon[111335]: pgmap v585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:01 smithi093 ceph-mon[118902]: pgmap v585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:01.706Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:01.707Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:01 smithi136 ceph-mon[113531]: pgmap v585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-mon[111335]: pgmap v586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:03.993 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2946717507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:03.993 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:13:03.994 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-mon[118902]: pgmap v586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:03.994 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2946717507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:03.994 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:13:03.994 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:03.593Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:03.994 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:03.594Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:04.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:03 smithi136 ceph-mon[113531]: pgmap v586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:04.156 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2946717507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:04.156 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:13:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:13:05] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:13:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[111335]: pgmap v587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1937982078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[118902]: pgmap v587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1937982078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:05 smithi136 ceph-mon[113531]: pgmap v587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1937982078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:13:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:13:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:13:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:13:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:13:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:13:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:13:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:08 smithi136 ceph-mon[113531]: pgmap v588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3513481192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:08 smithi093 ceph-mon[111335]: pgmap v588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3513481192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:08 smithi093 ceph-mon[118902]: pgmap v588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3513481192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:10 smithi136 ceph-mon[113531]: pgmap v589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:10 smithi093 ceph-mon[111335]: pgmap v589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:10.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:10 smithi093 ceph-mon[118902]: pgmap v589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:10.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:10.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:10.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:10.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4024270307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4024270307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4024270307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:11.707Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:11.708Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:12 smithi136 ceph-mon[113531]: pgmap v590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:12 smithi093 ceph-mon[111335]: pgmap v590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:12 smithi093 ceph-mon[118902]: pgmap v590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1771459740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1771459740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1771459740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:13.593Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:13.594Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:14 smithi136 ceph-mon[113531]: pgmap v591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:14 smithi093 ceph-mon[111335]: pgmap v591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:14 smithi093 ceph-mon[118902]: pgmap v591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3023215641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3023215641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3023215641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:13:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:13:15] "GET /metrics HTTP/1.1" 200 239864 "" "Prometheus/2.43.0" 2023-08-27T21:13:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:16 smithi136 ceph-mon[113531]: pgmap v592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:16 smithi093 ceph-mon[111335]: pgmap v592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:16 smithi093 ceph-mon[118902]: pgmap v592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:18 smithi136 ceph-mon[113531]: pgmap v593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3401687216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:18 smithi093 ceph-mon[111335]: pgmap v593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3401687216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:18 smithi093 ceph-mon[118902]: pgmap v593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3401687216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:20 smithi136 ceph-mon[113531]: pgmap v594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3441103721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:20 smithi093 ceph-mon[111335]: pgmap v594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3441103721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:20 smithi093 ceph-mon[118902]: pgmap v594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3441103721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:20.157Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:20.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:21.707Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:21.708Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:22 smithi136 ceph-mon[113531]: pgmap v595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:22 smithi093 ceph-mon[111335]: pgmap v595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:22 smithi093 ceph-mon[118902]: pgmap v595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2983758768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2983758768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2983758768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:23.594Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:23.595Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:24 smithi136 ceph-mon[113531]: pgmap v596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:24 smithi093 ceph-mon[111335]: pgmap v596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:24 smithi093 ceph-mon[118902]: pgmap v596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1400603222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1400603222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1400603222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:13:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:13:25] "GET /metrics HTTP/1.1" 200 239869 "" "Prometheus/2.43.0" 2023-08-27T21:13:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:26 smithi136 ceph-mon[113531]: pgmap v597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:26 smithi093 ceph-mon[111335]: pgmap v597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:26 smithi093 ceph-mon[118902]: pgmap v597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/106837383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/106837383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/106837383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:28 smithi136 ceph-mon[113531]: pgmap v598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:28 smithi093 ceph-mon[111335]: pgmap v598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:28 smithi093 ceph-mon[118902]: pgmap v598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:30 smithi136 ceph-mon[113531]: pgmap v599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3878026466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:30 smithi093 ceph-mon[118902]: pgmap v599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3878026466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:30.158Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:30.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:30 smithi093 ceph-mon[111335]: pgmap v599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3878026466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:31.708Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:31.709Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:32 smithi136 ceph-mon[113531]: pgmap v600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2580266050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:32 smithi093 ceph-mon[118902]: pgmap v600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2580266050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:32 smithi093 ceph-mon[111335]: pgmap v600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2580266050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:33.595Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:33.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:33.596Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:34 smithi136 ceph-mon[113531]: pgmap v601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2402581019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:34 smithi093 ceph-mon[111335]: pgmap v601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2402581019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:34 smithi093 ceph-mon[118902]: pgmap v601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2402581019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:13:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:13:35] "GET /metrics HTTP/1.1" 200 239869 "" "Prometheus/2.43.0" 2023-08-27T21:13:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:36 smithi136 ceph-mon[113531]: pgmap v602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:36 smithi093 ceph-mon[111335]: pgmap v602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:36 smithi093 ceph-mon[118902]: pgmap v602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2477729795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2477729795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2477729795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:38 smithi136 ceph-mon[113531]: pgmap v603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:38 smithi093 ceph-mon[111335]: pgmap v603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:38 smithi093 ceph-mon[118902]: pgmap v603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3089849098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3089849098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3089849098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:40 smithi136 ceph-mon[113531]: pgmap v604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:40 smithi093 ceph-mon[111335]: pgmap v604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:40 smithi093 ceph-mon[118902]: pgmap v604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:40.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:40.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:41.708Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:41.709Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:42 smithi136 ceph-mon[113531]: pgmap v605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1503607675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:42 smithi093 ceph-mon[111335]: pgmap v605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1503607675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:42 smithi093 ceph-mon[118902]: pgmap v605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1503607675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:43.595Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:43.596Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:44 smithi136 ceph-mon[113531]: pgmap v606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/184426966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:44 smithi093 ceph-mon[111335]: pgmap v606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/184426966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:44 smithi093 ceph-mon[118902]: pgmap v606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/184426966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:13:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:13:45] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:13:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:46 smithi136 ceph-mon[113531]: pgmap v607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3758472729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:46 smithi093 ceph-mon[111335]: pgmap v607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3758472729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:46 smithi093 ceph-mon[118902]: pgmap v607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3758472729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:48 smithi136 ceph-mon[113531]: pgmap v608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:48 smithi093 ceph-mon[111335]: pgmap v608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:48 smithi093 ceph-mon[118902]: pgmap v608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1905730681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1905730681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1905730681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:50 smithi136 ceph-mon[113531]: pgmap v609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:50 smithi093 ceph-mon[111335]: pgmap v609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:50 smithi093 ceph-mon[118902]: pgmap v609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:50.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:50.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1264296621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1264296621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1264296621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:51.709Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:51.710Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:52 smithi136 ceph-mon[113531]: pgmap v610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:52 smithi093 ceph-mon[111335]: pgmap v610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:52 smithi093 ceph-mon[118902]: pgmap v610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3489486380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3489486380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3489486380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:53.596Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:13:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:13:53.596Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:13:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:54 smithi136 ceph-mon[113531]: pgmap v611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:54 smithi093 ceph-mon[111335]: pgmap v611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:54 smithi093 ceph-mon[118902]: pgmap v611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:13:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:13:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:13:55] "GET /metrics HTTP/1.1" 200 239870 "" "Prometheus/2.43.0" 2023-08-27T21:13:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:56 smithi136 ceph-mon[113531]: pgmap v612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3554537507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:56 smithi093 ceph-mon[111335]: pgmap v612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3554537507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:56 smithi093 ceph-mon[118902]: pgmap v612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3554537507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:58 smithi136 ceph-mon[113531]: pgmap v613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:13:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1232749291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:58 smithi093 ceph-mon[118902]: pgmap v613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:13:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1232749291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:13:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:58 smithi093 ceph-mon[111335]: pgmap v613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:13:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:13:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1232749291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:00 smithi136 ceph-mon[113531]: pgmap v614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:00 smithi093 ceph-mon[111335]: pgmap v614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:00 smithi093 ceph-mon[118902]: pgmap v614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:00.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:00.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/857759602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/857759602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/857759602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:01.710Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:01.711Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:02 smithi136 ceph-mon[113531]: pgmap v615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:02 smithi093 ceph-mon[111335]: pgmap v615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:02 smithi093 ceph-mon[118902]: pgmap v615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/909214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/909214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/909214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:03.596Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:03.597Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:04 smithi136 ceph-mon[113531]: pgmap v616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:04 smithi093 ceph-mon[111335]: pgmap v616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:04 smithi093 ceph-mon[118902]: pgmap v616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2910602557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2910602557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2910602557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:14:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:14:05] "GET /metrics HTTP/1.1" 200 239870 "" "Prometheus/2.43.0" 2023-08-27T21:14:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:06 smithi136 ceph-mon[113531]: pgmap v617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:14:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:06 smithi093 ceph-mon[111335]: pgmap v617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:14:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:06 smithi093 ceph-mon[118902]: pgmap v617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:14:08.257 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:08 smithi136 ceph-mon[113531]: pgmap v618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:08.257 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1230364162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:08.257 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:08.257 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[111335]: pgmap v618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1230364162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[118902]: pgmap v618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1230364162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[111335]: pgmap v619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[118902]: pgmap v619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:14:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:14:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:09 smithi136 ceph-mon[113531]: pgmap v619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:14:10.472 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:10.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:10.472 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:10.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/667745390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/667745390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/667745390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:11 smithi093 ceph-mon[111335]: pgmap v620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:11 smithi093 ceph-mon[118902]: pgmap v620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:11.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:11.711Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:11.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:11.712Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:11 smithi136 ceph-mon[113531]: pgmap v620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/805712553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/805712553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/805712553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:13 smithi093 ceph-mon[111335]: pgmap v621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:13 smithi093 ceph-mon[118902]: pgmap v621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:13.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:13.596Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:13.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:13.597Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:13 smithi136 ceph-mon[113531]: pgmap v621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4077957467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4077957467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4077957467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:15.714 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:14:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:14:15] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:14:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:15 smithi093 ceph-mon[111335]: pgmap v622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:15 smithi093 ceph-mon[118902]: pgmap v622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:15 smithi136 ceph-mon[113531]: pgmap v622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:17 smithi093 ceph-mon[111335]: pgmap v623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3695857014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:17 smithi093 ceph-mon[118902]: pgmap v623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3695857014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:17 smithi136 ceph-mon[113531]: pgmap v623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3695857014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:19 smithi093 ceph-mon[111335]: pgmap v624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3923424144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:19 smithi093 ceph-mon[118902]: pgmap v624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3923424144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:19 smithi136 ceph-mon[113531]: pgmap v624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3923424144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:20.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:20.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:21 smithi093 ceph-mon[111335]: pgmap v625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:21 smithi093 ceph-mon[118902]: pgmap v625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:21.712Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:21.713Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:21 smithi136 ceph-mon[113531]: pgmap v625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4131112724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4131112724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4131112724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:23 smithi093 ceph-mon[111335]: pgmap v626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:23 smithi093 ceph-mon[118902]: pgmap v626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:23.597Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:23.599Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:23 smithi136 ceph-mon[113531]: pgmap v626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3275612327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3275612327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3275612327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:25.734 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:14:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:14:25] "GET /metrics HTTP/1.1" 200 239865 "" "Prometheus/2.43.0" 2023-08-27T21:14:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:25 smithi093 ceph-mon[111335]: pgmap v627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:25 smithi093 ceph-mon[118902]: pgmap v627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:25 smithi136 ceph-mon[113531]: pgmap v627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1691413310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1691413310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:27.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1691413310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:27 smithi136 ceph-mon[113531]: pgmap v628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:27 smithi093 ceph-mon[111335]: pgmap v628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:27 smithi093 ceph-mon[118902]: pgmap v628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:30.163 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:29 smithi093 ceph-mon[118902]: pgmap v629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:30.163 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/120416448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:30.164 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:29 smithi093 ceph-mon[111335]: pgmap v629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:30.164 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/120416448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:29 smithi136 ceph-mon[113531]: pgmap v629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/120416448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:30.163Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:30.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:31 smithi093 ceph-mon[111335]: pgmap v630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1478954453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:31 smithi093 ceph-mon[118902]: pgmap v630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1478954453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:31.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:31.712Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:31.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:31.713Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:31 smithi136 ceph-mon[113531]: pgmap v630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1478954453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:33 smithi093 ceph-mon[111335]: pgmap v631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2790364214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:33 smithi093 ceph-mon[118902]: pgmap v631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2790364214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:33.598Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:33.599Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:33 smithi136 ceph-mon[113531]: pgmap v631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2790364214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:14:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:14:35] "GET /metrics HTTP/1.1" 200 239865 "" "Prometheus/2.43.0" 2023-08-27T21:14:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:35 smithi136 ceph-mon[113531]: pgmap v632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:35 smithi093 ceph-mon[111335]: pgmap v632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:35 smithi093 ceph-mon[118902]: pgmap v632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1672446685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1672446685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1672446685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:37 smithi136 ceph-mon[113531]: pgmap v633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:37 smithi093 ceph-mon[111335]: pgmap v633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:37 smithi093 ceph-mon[118902]: pgmap v633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4241592438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4241592438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4241592438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:40.166 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:39 smithi093 ceph-mon[111335]: pgmap v634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:40.166 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:39 smithi093 ceph-mon[118902]: pgmap v634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:39 smithi136 ceph-mon[113531]: pgmap v634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:40.164Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:40.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:41 smithi093 ceph-mon[111335]: pgmap v635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2846398886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:41 smithi093 ceph-mon[118902]: pgmap v635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2846398886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:41.714Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:41.715Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:41 smithi136 ceph-mon[113531]: pgmap v635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2846398886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:43 smithi093 ceph-mon[111335]: pgmap v636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2751699226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:43 smithi093 ceph-mon[118902]: pgmap v636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2751699226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:43.598Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:43.599Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:43 smithi136 ceph-mon[113531]: pgmap v636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2751699226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:14:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:14:45] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:14:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:45 smithi136 ceph-mon[113531]: pgmap v637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2890435313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:45 smithi093 ceph-mon[111335]: pgmap v637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2890435313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:45 smithi093 ceph-mon[118902]: pgmap v637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2890435313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:47 smithi136 ceph-mon[113531]: pgmap v638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:47 smithi093 ceph-mon[111335]: pgmap v638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:47 smithi093 ceph-mon[118902]: pgmap v638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3822298301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3822298301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3822298301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:50.164 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:49 smithi093 ceph-mon[111335]: pgmap v639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:50.165 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:49 smithi093 ceph-mon[118902]: pgmap v639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:49 smithi136 ceph-mon[113531]: pgmap v639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:50.164Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:50.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3622516428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3622516428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3622516428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:51 smithi093 ceph-mon[111335]: pgmap v640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:51 smithi093 ceph-mon[118902]: pgmap v640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:51.715Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:51.716Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:51 smithi136 ceph-mon[113531]: pgmap v640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2161336365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2161336365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2161336365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:53 smithi093 ceph-mon[111335]: pgmap v641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:53 smithi093 ceph-mon[118902]: pgmap v641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:53.599Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:14:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:14:53.600Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:14:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:53 smithi136 ceph-mon[113531]: pgmap v641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:14:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:14:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:14:55] "GET /metrics HTTP/1.1" 200 239870 "" "Prometheus/2.43.0" 2023-08-27T21:14:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:55 smithi136 ceph-mon[113531]: pgmap v642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3017649127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:55 smithi093 ceph-mon[111335]: pgmap v642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3017649127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:55 smithi093 ceph-mon[118902]: pgmap v642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3017649127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:57 smithi136 ceph-mon[113531]: pgmap v643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3247930860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:57 smithi093 ceph-mon[111335]: pgmap v643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3247930860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:14:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:57 smithi093 ceph-mon[118902]: pgmap v643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:14:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3247930860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:00.165 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:14:59 smithi093 ceph-mon[111335]: pgmap v644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:00.166 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:14:59 smithi093 ceph-mon[118902]: pgmap v644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:14:59 smithi136 ceph-mon[113531]: pgmap v644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:00.165Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:00.166Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3539967392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3539967392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3539967392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:01 smithi093 ceph-mon[111335]: pgmap v645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:01 smithi093 ceph-mon[118902]: pgmap v645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:01.715Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:01.716Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:01 smithi136 ceph-mon[113531]: pgmap v645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1099680335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1099680335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:03.348 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1099680335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:03 smithi093 ceph-mon[111335]: pgmap v646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:03 smithi093 ceph-mon[118902]: pgmap v646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:03.600Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:03.601Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:03 smithi136 ceph-mon[113531]: pgmap v646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/736336308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/736336308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/736336308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:15:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:15:05] "GET /metrics HTTP/1.1" 200 239870 "" "Prometheus/2.43.0" 2023-08-27T21:15:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:05 smithi136 ceph-mon[113531]: pgmap v647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:05 smithi093 ceph-mon[111335]: pgmap v647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:05 smithi093 ceph-mon[118902]: pgmap v647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:07 smithi136 ceph-mon[113531]: pgmap v648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3605920202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:07 smithi093 ceph-mon[111335]: pgmap v648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3605920202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:07 smithi093 ceph-mon[118902]: pgmap v648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3605920202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:09 smithi136 ceph-mon[113531]: pgmap v649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:15:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1163997211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:10.167 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:09 smithi093 ceph-mon[111335]: pgmap v649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:10.167 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:15:10.167 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1163997211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:10.168 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:09 smithi093 ceph-mon[118902]: pgmap v649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:10.168 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:15:10.168 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1163997211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:10.166Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:10.168Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:11.716Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:11.717Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: pgmap v650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4151868138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:15:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:15:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: pgmap v650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4151868138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: pgmap v650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:12.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4151868138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:12.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:15:12.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:15:12.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:15:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:13.600Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:13.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:14 smithi136 ceph-mon[113531]: pgmap v651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:14 smithi093 ceph-mon[111335]: pgmap v651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:14 smithi093 ceph-mon[118902]: pgmap v651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3183775725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3183775725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3183775725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:15:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:15:15] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:15:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:16 smithi136 ceph-mon[113531]: pgmap v652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:16 smithi093 ceph-mon[111335]: pgmap v652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:16 smithi093 ceph-mon[118902]: pgmap v652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2922817239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2922817239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2922817239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:18 smithi136 ceph-mon[113531]: pgmap v653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:18 smithi093 ceph-mon[111335]: pgmap v653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:18 smithi093 ceph-mon[118902]: pgmap v653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:19 smithi093 ceph-mon[118902]: pgmap v654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4147537649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:19 smithi093 ceph-mon[111335]: pgmap v654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4147537649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:19 smithi136 ceph-mon[113531]: pgmap v654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4147537649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:20.167Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:20.168Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:21 smithi093 ceph-mon[111335]: pgmap v655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4207258021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:21 smithi093 ceph-mon[118902]: pgmap v655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4207258021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:21.717Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:21.718Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:21 smithi136 ceph-mon[113531]: pgmap v655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4207258021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:23 smithi093 ceph-mon[111335]: pgmap v656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:23 smithi093 ceph-mon[118902]: pgmap v656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:23.601Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:23.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:23 smithi136 ceph-mon[113531]: pgmap v656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2295802641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2295802641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2295802641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:25.732 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:15:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:15:25] "GET /metrics HTTP/1.1" 200 239862 "" "Prometheus/2.43.0" 2023-08-27T21:15:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:25 smithi093 ceph-mon[111335]: pgmap v657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:25 smithi093 ceph-mon[118902]: pgmap v657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:25 smithi136 ceph-mon[113531]: pgmap v657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2472673531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2472673531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2472673531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:27 smithi093 ceph-mon[111335]: pgmap v658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:27 smithi093 ceph-mon[118902]: pgmap v658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:27 smithi136 ceph-mon[113531]: pgmap v658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/613999842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/613999842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/613999842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:30.168 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:29 smithi093 ceph-mon[111335]: pgmap v659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:30.168 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:29 smithi093 ceph-mon[118902]: pgmap v659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:29 smithi136 ceph-mon[113531]: pgmap v659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:30.168Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:30.169Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:31 smithi093 ceph-mon[118902]: pgmap v660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3545959312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:31 smithi093 ceph-mon[111335]: pgmap v660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3545959312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:31.717Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:31.718Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:31 smithi136 ceph-mon[113531]: pgmap v660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3545959312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:33 smithi093 ceph-mon[111335]: pgmap v661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2496148124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:33 smithi093 ceph-mon[118902]: pgmap v661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2496148124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:33.601Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:33.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:33 smithi136 ceph-mon[113531]: pgmap v661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2496148124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:15:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:15:35] "GET /metrics HTTP/1.1" 200 239862 "" "Prometheus/2.43.0" 2023-08-27T21:15:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:35 smithi136 ceph-mon[113531]: pgmap v662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3819650014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:35 smithi093 ceph-mon[118902]: pgmap v662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3819650014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:35 smithi093 ceph-mon[111335]: pgmap v662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3819650014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:37 smithi136 ceph-mon[113531]: pgmap v663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:37 smithi093 ceph-mon[111335]: pgmap v663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:37 smithi093 ceph-mon[118902]: pgmap v663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1097765185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1097765185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1097765185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:40.169 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:39 smithi093 ceph-mon[111335]: pgmap v664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:40.170 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:39 smithi093 ceph-mon[118902]: pgmap v664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:39 smithi136 ceph-mon[113531]: pgmap v664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:40.169Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:40.170Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2153727973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2153727973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2153727973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:41 smithi093 ceph-mon[111335]: pgmap v665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:41 smithi093 ceph-mon[118902]: pgmap v665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:41.718Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:41.719Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:41 smithi136 ceph-mon[113531]: pgmap v665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:43 smithi093 ceph-mon[111335]: pgmap v666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3452606366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:43 smithi093 ceph-mon[118902]: pgmap v666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3452606366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:43.601Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:43.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:43 smithi136 ceph-mon[113531]: pgmap v666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3452606366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:15:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:15:45] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:15:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:45 smithi136 ceph-mon[113531]: pgmap v667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/620275292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:45 smithi093 ceph-mon[111335]: pgmap v667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/620275292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:45 smithi093 ceph-mon[118902]: pgmap v667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/620275292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:47 smithi136 ceph-mon[113531]: pgmap v668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3719366768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:47 smithi093 ceph-mon[111335]: pgmap v668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3719366768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:47 smithi093 ceph-mon[118902]: pgmap v668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3719366768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:50.170 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:49 smithi093 ceph-mon[111335]: pgmap v669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:50.170 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:49 smithi093 ceph-mon[118902]: pgmap v669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:49 smithi136 ceph-mon[113531]: pgmap v669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:50.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:50.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1897007317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1897007317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1897007317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:51 smithi093 ceph-mon[111335]: pgmap v670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:51 smithi093 ceph-mon[118902]: pgmap v670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:51.719Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:51.720Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:51 smithi136 ceph-mon[113531]: pgmap v670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2289853713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2289853713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2289853713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:53 smithi093 ceph-mon[111335]: pgmap v671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:53 smithi093 ceph-mon[118902]: pgmap v671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:53.602Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:15:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:15:53.603Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:15:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:53 smithi136 ceph-mon[113531]: pgmap v671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:15:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/946066891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/946066891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/946066891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:15:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:15:55] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:15:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:55 smithi136 ceph-mon[113531]: pgmap v672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:55 smithi093 ceph-mon[111335]: pgmap v672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:55 smithi093 ceph-mon[118902]: pgmap v672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:57 smithi136 ceph-mon[113531]: pgmap v673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2219524487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:57 smithi093 ceph-mon[111335]: pgmap v673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2219524487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:15:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:57 smithi093 ceph-mon[118902]: pgmap v673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:15:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2219524487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:00.170 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:59 smithi093 ceph-mon[111335]: pgmap v674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:00.171 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:15:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2572467048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:00.171 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:59 smithi093 ceph-mon[118902]: pgmap v674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:00.171 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:15:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2572467048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:59 smithi136 ceph-mon[113531]: pgmap v674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:15:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2572467048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:00.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:00.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:01 smithi093 ceph-mon[111335]: pgmap v675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:01 smithi093 ceph-mon[118902]: pgmap v675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:01.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:01.720Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:01 smithi136 ceph-mon[113531]: pgmap v675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/782261188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/782261188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/782261188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:03 smithi093 ceph-mon[111335]: pgmap v676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:03 smithi093 ceph-mon[118902]: pgmap v676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:03.603Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:03.604Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:03 smithi136 ceph-mon[113531]: pgmap v676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1068325204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1068325204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1068325204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:16:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:16:05] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:16:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:05 smithi136 ceph-mon[113531]: pgmap v677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:05 smithi093 ceph-mon[111335]: pgmap v677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:05 smithi093 ceph-mon[118902]: pgmap v677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/706235971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/706235971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/706235971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:07 smithi136 ceph-mon[113531]: pgmap v678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:07 smithi093 ceph-mon[111335]: pgmap v678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:07 smithi093 ceph-mon[118902]: pgmap v678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:10.170 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:09 smithi093 ceph-mon[111335]: pgmap v679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:10.170 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2352575987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:10.171 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:09 smithi093 ceph-mon[118902]: pgmap v679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:10.171 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2352575987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:09 smithi136 ceph-mon[113531]: pgmap v679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2352575987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:10.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:10.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:11 smithi093 ceph-mon[111335]: pgmap v680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1931661334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:11 smithi093 ceph-mon[118902]: pgmap v680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1931661334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:11.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:11.722Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:11 smithi136 ceph-mon[113531]: pgmap v680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1931661334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:16:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:16:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:16:13.849 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:13.604Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:13.850 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:13.605Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:13 smithi136 ceph-mon[113531]: pgmap v681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/264958030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[111335]: pgmap v681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/264958030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[118902]: pgmap v681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/264958030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:15 smithi136 ceph-mon[113531]: pgmap v682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:16:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:16:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:16:15] "GET /metrics HTTP/1.1" 200 239864 "" "Prometheus/2.43.0" 2023-08-27T21:16:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[111335]: pgmap v682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:16:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:16:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[118902]: pgmap v682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:16:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:16:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:16:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1215805392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1215805392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1215805392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:17 smithi136 ceph-mon[113531]: pgmap v683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:17 smithi093 ceph-mon[111335]: pgmap v683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:17 smithi093 ceph-mon[118902]: pgmap v683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111727941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111727941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111727941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:19 smithi093 ceph-mon[111335]: pgmap v684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:19 smithi093 ceph-mon[118902]: pgmap v684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:19 smithi136 ceph-mon[113531]: pgmap v684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:20.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:20.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:21 smithi093 ceph-mon[111335]: pgmap v685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1681355245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:21 smithi093 ceph-mon[118902]: pgmap v685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1681355245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:21.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:21.721Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:21 smithi136 ceph-mon[113531]: pgmap v685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1681355245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-mon[111335]: pgmap v686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3734668043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-mon[118902]: pgmap v686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3734668043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:23.605Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:23.606Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:23 smithi136 ceph-mon[113531]: pgmap v686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3734668043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:16:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:16:25] "GET /metrics HTTP/1.1" 200 239870 "" "Prometheus/2.43.0" 2023-08-27T21:16:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:25 smithi136 ceph-mon[113531]: pgmap v687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3771110341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:25 smithi093 ceph-mon[111335]: pgmap v687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3771110341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:25 smithi093 ceph-mon[118902]: pgmap v687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3771110341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:27 smithi136 ceph-mon[113531]: pgmap v688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:27 smithi093 ceph-mon[111335]: pgmap v688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:27 smithi093 ceph-mon[118902]: pgmap v688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2703738926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2703738926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2703738926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:30.171 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:29 smithi093 ceph-mon[111335]: pgmap v689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:30.172 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:29 smithi093 ceph-mon[118902]: pgmap v689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:29 smithi136 ceph-mon[113531]: pgmap v689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:30.171Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:30.173Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2046392689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2046392689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2046392689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:31 smithi093 ceph-mon[111335]: pgmap v690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:31.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:31 smithi093 ceph-mon[118902]: pgmap v690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:31.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:31.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:31.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:31.721Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:31 smithi136 ceph-mon[113531]: pgmap v690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:33 smithi093 ceph-mon[111335]: pgmap v691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2531096442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:33 smithi093 ceph-mon[118902]: pgmap v691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2531096442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:33.606Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:33.607Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:33 smithi136 ceph-mon[113531]: pgmap v691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2531096442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:16:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:16:35] "GET /metrics HTTP/1.1" 200 239870 "" "Prometheus/2.43.0" 2023-08-27T21:16:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:35 smithi136 ceph-mon[113531]: pgmap v692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3849058200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:35 smithi093 ceph-mon[111335]: pgmap v692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3849058200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:35 smithi093 ceph-mon[118902]: pgmap v692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3849058200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:37 smithi136 ceph-mon[113531]: pgmap v693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2007270227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:37 smithi093 ceph-mon[111335]: pgmap v693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2007270227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:37 smithi093 ceph-mon[118902]: pgmap v693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2007270227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:40.173 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:39 smithi093 ceph-mon[111335]: pgmap v694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:40.173 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:39 smithi093 ceph-mon[118902]: pgmap v694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:39 smithi136 ceph-mon[113531]: pgmap v694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:40.173Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:40.173Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2816298274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2816298274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2816298274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:41 smithi093 ceph-mon[111335]: pgmap v695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:41 smithi093 ceph-mon[118902]: pgmap v695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:41.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:41.721Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:41.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:41.722Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:41 smithi136 ceph-mon[113531]: pgmap v695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1924884899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1924884899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1924884899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:43 smithi093 ceph-mon[118902]: pgmap v696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:43 smithi093 ceph-mon[111335]: pgmap v696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:43.606Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:43.607Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:43 smithi136 ceph-mon[113531]: pgmap v696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:16:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:16:45] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:16:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:45 smithi136 ceph-mon[113531]: pgmap v697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/171281868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:45 smithi093 ceph-mon[111335]: pgmap v697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/171281868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:45 smithi093 ceph-mon[118902]: pgmap v697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/171281868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:47 smithi136 ceph-mon[113531]: pgmap v698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4260725541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:47 smithi093 ceph-mon[111335]: pgmap v698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4260725541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:47 smithi093 ceph-mon[118902]: pgmap v698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4260725541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:50.173 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:49 smithi093 ceph-mon[111335]: pgmap v699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:50.174 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2405154645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:50.174 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:49 smithi093 ceph-mon[118902]: pgmap v699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:50.174 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2405154645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:49 smithi136 ceph-mon[113531]: pgmap v699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2405154645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:50.173Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:50.174Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:51 smithi093 ceph-mon[111335]: pgmap v700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:51 smithi093 ceph-mon[118902]: pgmap v700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:51.721Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:51.722Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:51 smithi136 ceph-mon[113531]: pgmap v700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3974929058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3974929058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3974929058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:53 smithi093 ceph-mon[111335]: pgmap v701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:53 smithi093 ceph-mon[118902]: pgmap v701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:53.607Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:16:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:16:53.608Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:16:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:53 smithi136 ceph-mon[113531]: pgmap v701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:16:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3173966792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3173966792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3173966792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:16:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:16:55] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:16:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:55 smithi136 ceph-mon[113531]: pgmap v702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:55 smithi093 ceph-mon[111335]: pgmap v702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:55 smithi093 ceph-mon[118902]: pgmap v702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4247153685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4247153685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4247153685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:16:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:57 smithi136 ceph-mon[113531]: pgmap v703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:57 smithi093 ceph-mon[111335]: pgmap v703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:16:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:57 smithi093 ceph-mon[118902]: pgmap v703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:00.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:59 smithi093 ceph-mon[111335]: pgmap v704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:00.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:16:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1340591913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:00.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:59 smithi093 ceph-mon[118902]: pgmap v704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:00.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:16:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1340591913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:59 smithi136 ceph-mon[113531]: pgmap v704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:16:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1340591913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:00.174Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:00.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:01 smithi093 ceph-mon[111335]: pgmap v705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2514515025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:01 smithi093 ceph-mon[118902]: pgmap v705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2514515025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:01.722Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:01.723Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:01 smithi136 ceph-mon[113531]: pgmap v705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2514515025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:03 smithi093 ceph-mon[111335]: pgmap v706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:03 smithi093 ceph-mon[118902]: pgmap v706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:03.608Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:03.609Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:03 smithi136 ceph-mon[113531]: pgmap v706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2587499581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2587499581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2587499581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:17:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:17:05] "GET /metrics HTTP/1.1" 200 239863 "" "Prometheus/2.43.0" 2023-08-27T21:17:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:05 smithi136 ceph-mon[113531]: pgmap v707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:05 smithi093 ceph-mon[111335]: pgmap v707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:05 smithi093 ceph-mon[118902]: pgmap v707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2513697968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2513697968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2513697968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:07 smithi136 ceph-mon[113531]: pgmap v708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:07 smithi093 ceph-mon[111335]: pgmap v708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:07 smithi093 ceph-mon[118902]: pgmap v708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3692582500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:09.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:09.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3692582500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:09.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:09.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3692582500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:10.176 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:09 smithi093 ceph-mon[111335]: pgmap v709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:10.177 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:09 smithi093 ceph-mon[118902]: pgmap v709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:09 smithi136 ceph-mon[113531]: pgmap v709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:10.176Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:10.177Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:11 smithi093 ceph-mon[111335]: pgmap v710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/437326567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:11 smithi093 ceph-mon[118902]: pgmap v710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/437326567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:11.723Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:11.724Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:11 smithi136 ceph-mon[113531]: pgmap v710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/437326567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:13.861 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:13 smithi093 ceph-mon[111335]: pgmap v711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:13.862 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1374718730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:13.862 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:13.609Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:13.862 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:13.610Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:13 smithi136 ceph-mon[113531]: pgmap v711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1374718730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:13 smithi093 ceph-mon[118902]: pgmap v711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1374718730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:17:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:17:15] "GET /metrics HTTP/1.1" 200 239870 "" "Prometheus/2.43.0" 2023-08-27T21:17:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:15 smithi136 ceph-mon[113531]: pgmap v712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:17:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2692849098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:15 smithi093 ceph-mon[111335]: pgmap v712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:17:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2692849098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:15 smithi093 ceph-mon[118902]: pgmap v712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:17:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2692849098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: pgmap v713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: pgmap v713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:17:18.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:17:18.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: pgmap v713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:17:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:17:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2530533569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2530533569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2530533569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:19 smithi093 ceph-mon[111335]: pgmap v714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:20.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:19 smithi093 ceph-mon[118902]: pgmap v714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:20.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:20.176Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:20.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:20.177Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:19 smithi136 ceph-mon[113531]: pgmap v714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3663111803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3663111803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3663111803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:21.723Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:21.724Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:21 smithi136 ceph-mon[113531]: pgmap v715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:21 smithi093 ceph-mon[111335]: pgmap v715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:21 smithi093 ceph-mon[118902]: pgmap v715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/708877848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/708877848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/708877848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:23.610Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:23.611Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:24 smithi136 ceph-mon[113531]: pgmap v716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:24 smithi093 ceph-mon[111335]: pgmap v716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:24 smithi093 ceph-mon[118902]: pgmap v716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:17:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:17:25] "GET /metrics HTTP/1.1" 200 239868 "" "Prometheus/2.43.0" 2023-08-27T21:17:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:26 smithi136 ceph-mon[113531]: pgmap v717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1172584012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:26 smithi093 ceph-mon[111335]: pgmap v717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1172584012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:26 smithi093 ceph-mon[118902]: pgmap v717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1172584012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:28 smithi136 ceph-mon[113531]: pgmap v718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2875749925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:28 smithi093 ceph-mon[111335]: pgmap v718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2875749925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:28 smithi093 ceph-mon[118902]: pgmap v718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2875749925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:30 smithi136 ceph-mon[113531]: pgmap v719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:30 smithi093 ceph-mon[111335]: pgmap v719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:30 smithi093 ceph-mon[118902]: pgmap v719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:30.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:30.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1021267053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1021267053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1021267053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:31.724Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:31.725Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:32 smithi136 ceph-mon[113531]: pgmap v720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:32 smithi093 ceph-mon[111335]: pgmap v720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:32 smithi093 ceph-mon[118902]: pgmap v720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/489350028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/489350028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/489350028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:33.610Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:33.611Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:34 smithi136 ceph-mon[113531]: pgmap v721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:34 smithi093 ceph-mon[111335]: pgmap v721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:34 smithi093 ceph-mon[118902]: pgmap v721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1154365581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1154365581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1154365581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:17:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:17:35] "GET /metrics HTTP/1.1" 200 239868 "" "Prometheus/2.43.0" 2023-08-27T21:17:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:36 smithi136 ceph-mon[113531]: pgmap v722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:36 smithi093 ceph-mon[111335]: pgmap v722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:36 smithi093 ceph-mon[118902]: pgmap v722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:38 smithi136 ceph-mon[113531]: pgmap v723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2489546328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:38 smithi093 ceph-mon[111335]: pgmap v723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2489546328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:38 smithi093 ceph-mon[118902]: pgmap v723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2489546328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:40 smithi136 ceph-mon[113531]: pgmap v724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3921553526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:40 smithi093 ceph-mon[111335]: pgmap v724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3921553526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:40 smithi093 ceph-mon[118902]: pgmap v724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3921553526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:40.178Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:40.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:41.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:41.726Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:42 smithi136 ceph-mon[113531]: pgmap v725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1521342189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:42 smithi093 ceph-mon[118902]: pgmap v725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1521342189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:42 smithi093 ceph-mon[111335]: pgmap v725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1521342189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:43.611Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:43.612Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:44 smithi136 ceph-mon[113531]: pgmap v726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:44 smithi093 ceph-mon[118902]: pgmap v726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:44 smithi093 ceph-mon[111335]: pgmap v726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4155787084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4155787084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4155787084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:17:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:17:45] "GET /metrics HTTP/1.1" 200 239868 "" "Prometheus/2.43.0" 2023-08-27T21:17:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:46 smithi136 ceph-mon[113531]: pgmap v727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:46 smithi093 ceph-mon[111335]: pgmap v727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:46 smithi093 ceph-mon[118902]: pgmap v727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2623672947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2623672947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2623672947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:48 smithi136 ceph-mon[113531]: pgmap v728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:48 smithi093 ceph-mon[111335]: pgmap v728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:48 smithi093 ceph-mon[118902]: pgmap v728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:50 smithi136 ceph-mon[113531]: pgmap v729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1272169397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:50 smithi093 ceph-mon[111335]: pgmap v729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1272169397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:50 smithi093 ceph-mon[118902]: pgmap v729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1272169397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:50.178Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:50.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:51.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:51.726Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:52 smithi136 ceph-mon[113531]: pgmap v730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1310341476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:52 smithi093 ceph-mon[111335]: pgmap v730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1310341476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:52 smithi093 ceph-mon[118902]: pgmap v730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1310341476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:53.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:17:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:17:53.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:17:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:54 smithi136 ceph-mon[113531]: pgmap v731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3345538705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:54 smithi093 ceph-mon[111335]: pgmap v731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3345538705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:54 smithi093 ceph-mon[118902]: pgmap v731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:17:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3345538705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:17:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:17:55] "GET /metrics HTTP/1.1" 200 239864 "" "Prometheus/2.43.0" 2023-08-27T21:17:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:56 smithi136 ceph-mon[113531]: pgmap v732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:56 smithi093 ceph-mon[111335]: pgmap v732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:56 smithi093 ceph-mon[118902]: pgmap v732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4093970487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4093970487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4093970487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:58 smithi136 ceph-mon[113531]: pgmap v733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:58 smithi093 ceph-mon[111335]: pgmap v733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:58 smithi093 ceph-mon[118902]: pgmap v733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:17:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:17:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2618397324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:17:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2618397324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:17:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:17:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2618397324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:00 smithi136 ceph-mon[113531]: pgmap v734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:00 smithi093 ceph-mon[111335]: pgmap v734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:00 smithi093 ceph-mon[118902]: pgmap v734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:00.179Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:00.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2952978062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2952978062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2952978062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:01.726Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:01.727Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:02 smithi136 ceph-mon[113531]: pgmap v735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:02 smithi093 ceph-mon[111335]: pgmap v735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:02 smithi093 ceph-mon[118902]: pgmap v735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:03.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:03.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:04 smithi136 ceph-mon[113531]: pgmap v736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2900566812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:04 smithi093 ceph-mon[111335]: pgmap v736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2900566812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:04 smithi093 ceph-mon[118902]: pgmap v736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2900566812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:18:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:18:05] "GET /metrics HTTP/1.1" 200 239864 "" "Prometheus/2.43.0" 2023-08-27T21:18:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:06 smithi136 ceph-mon[113531]: pgmap v737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3245006620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:06 smithi093 ceph-mon[111335]: pgmap v737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3245006620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:06 smithi093 ceph-mon[118902]: pgmap v737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3245006620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:08 smithi136 ceph-mon[113531]: pgmap v738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:08 smithi093 ceph-mon[111335]: pgmap v738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:08 smithi093 ceph-mon[118902]: pgmap v738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3257537435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3257537435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3257537435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:10 smithi136 ceph-mon[113531]: pgmap v739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:10 smithi093 ceph-mon[111335]: pgmap v739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:10 smithi093 ceph-mon[118902]: pgmap v739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:10.180Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:10.181Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2947257698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2947257698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2947257698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:11.727Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:11.728Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:12 smithi136 ceph-mon[113531]: pgmap v740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:12 smithi093 ceph-mon[111335]: pgmap v740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:12 smithi093 ceph-mon[118902]: pgmap v740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2090201228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2090201228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2090201228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:13.613Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:13.614Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:14 smithi136 ceph-mon[113531]: pgmap v741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:14 smithi093 ceph-mon[111335]: pgmap v741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:14 smithi093 ceph-mon[118902]: pgmap v741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:18:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:18:15] "GET /metrics HTTP/1.1" 200 239866 "" "Prometheus/2.43.0" 2023-08-27T21:18:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:16 smithi136 ceph-mon[113531]: pgmap v742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2049748341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:16 smithi093 ceph-mon[111335]: pgmap v742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2049748341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:16 smithi093 ceph-mon[118902]: pgmap v742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2049748341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:18 smithi136 ceph-mon[113531]: pgmap v743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2745733183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:18:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:18 smithi093 ceph-mon[118902]: pgmap v743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2745733183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:18:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:18 smithi093 ceph-mon[111335]: pgmap v743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2745733183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:18:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:20 smithi136 ceph-mon[113531]: pgmap v744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/256916940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[111335]: pgmap v744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/256916940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[118902]: pgmap v744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/256916940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:20.180Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:20.182Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:18:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:18:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.728 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.728 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.728 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:18:21.728 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:18:21.728 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.729 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.729 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.729 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:18:21.729 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:18:21.729 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:18:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:21.728Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:21.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:21.729Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:22 smithi136 ceph-mon[113531]: pgmap v745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:22 smithi093 ceph-mon[111335]: pgmap v745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:22 smithi093 ceph-mon[118902]: pgmap v745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:23.613 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/603083063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:23.614 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/603083063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/603083063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:23.613Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:23.614Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:24 smithi136 ceph-mon[113531]: pgmap v746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:24 smithi093 ceph-mon[118902]: pgmap v746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:24 smithi093 ceph-mon[111335]: pgmap v746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/269327972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/269327972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:18:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:18:25] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:18:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/269327972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:26 smithi136 ceph-mon[113531]: pgmap v747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:26 smithi093 ceph-mon[118902]: pgmap v747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:26 smithi093 ceph-mon[111335]: pgmap v747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2879571447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2879571447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2879571447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:28 smithi136 ceph-mon[113531]: pgmap v748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:28 smithi093 ceph-mon[111335]: pgmap v748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:28 smithi093 ceph-mon[118902]: pgmap v748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:30 smithi093 ceph-mon[111335]: pgmap v749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1031836787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:30 smithi093 ceph-mon[118902]: pgmap v749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1031836787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:30.181Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:30.182Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:30 smithi136 ceph-mon[113531]: pgmap v749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1031836787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:31.728Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:31.729Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:32 smithi136 ceph-mon[113531]: pgmap v750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/863146318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:32 smithi093 ceph-mon[111335]: pgmap v750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/863146318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:32 smithi093 ceph-mon[118902]: pgmap v750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/863146318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:33.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:33.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:34 smithi136 ceph-mon[113531]: pgmap v751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:34 smithi093 ceph-mon[111335]: pgmap v751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:34 smithi093 ceph-mon[118902]: pgmap v751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/258834310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/258834310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/258834310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:18:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:18:35] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:18:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:36 smithi136 ceph-mon[113531]: pgmap v752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:36 smithi093 ceph-mon[111335]: pgmap v752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:36 smithi093 ceph-mon[118902]: pgmap v752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1436337790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1436337790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1436337790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:38 smithi136 ceph-mon[113531]: pgmap v753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:38 smithi093 ceph-mon[111335]: pgmap v753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:38.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:38 smithi093 ceph-mon[118902]: pgmap v753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1693694475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1693694475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1693694475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:40 smithi093 ceph-mon[111335]: pgmap v754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:40 smithi093 ceph-mon[118902]: pgmap v754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:40.182Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:40.183Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:40 smithi136 ceph-mon[113531]: pgmap v754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:41.729Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:41.730Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:42 smithi136 ceph-mon[113531]: pgmap v755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1766184875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:42 smithi093 ceph-mon[118902]: pgmap v755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1766184875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:42 smithi093 ceph-mon[111335]: pgmap v755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1766184875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:43.615Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:43.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:44 smithi136 ceph-mon[113531]: pgmap v756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1844128608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:44 smithi093 ceph-mon[111335]: pgmap v756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1844128608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:44 smithi093 ceph-mon[118902]: pgmap v756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1844128608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:18:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:18:45] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:18:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:46 smithi136 ceph-mon[113531]: pgmap v757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1005278191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:46 smithi093 ceph-mon[111335]: pgmap v757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1005278191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:46 smithi093 ceph-mon[118902]: pgmap v757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1005278191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:48 smithi136 ceph-mon[113531]: pgmap v758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:48 smithi093 ceph-mon[111335]: pgmap v758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:48 smithi093 ceph-mon[118902]: pgmap v758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/187723292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/187723292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/187723292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:50 smithi093 ceph-mon[111335]: pgmap v759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:50 smithi093 ceph-mon[118902]: pgmap v759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:50.182Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:50.183Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:50 smithi136 ceph-mon[113531]: pgmap v759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4137748751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:51.730 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4137748751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:51.731 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4137748751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:51.730Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:51.731Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:52 smithi136 ceph-mon[113531]: pgmap v760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:52 smithi093 ceph-mon[111335]: pgmap v760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:52 smithi093 ceph-mon[118902]: pgmap v760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:53.615Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:18:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:18:53.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:18:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:54 smithi136 ceph-mon[113531]: pgmap v761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1324832084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:54 smithi093 ceph-mon[111335]: pgmap v761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1324832084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:54 smithi093 ceph-mon[118902]: pgmap v761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1324832084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:18:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:18:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:18:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:18:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:56 smithi136 ceph-mon[113531]: pgmap v762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/871312670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:56 smithi093 ceph-mon[111335]: pgmap v762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/871312670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:56 smithi093 ceph-mon[118902]: pgmap v762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/871312670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:58 smithi136 ceph-mon[113531]: pgmap v763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:18:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2688072455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:58 smithi093 ceph-mon[111335]: pgmap v763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:18:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2688072455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:18:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:58 smithi093 ceph-mon[118902]: pgmap v763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:18:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:18:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2688072455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:00 smithi093 ceph-mon[118902]: pgmap v764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:00 smithi093 ceph-mon[111335]: pgmap v764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:00.183Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:00.184Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:00 smithi136 ceph-mon[113531]: pgmap v764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3082357844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:01.731 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3082357844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:01.731 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3082357844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:01.731Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:01.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:02 smithi136 ceph-mon[113531]: pgmap v765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:02 smithi093 ceph-mon[111335]: pgmap v765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:02.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:02 smithi093 ceph-mon[118902]: pgmap v765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3197765425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3197765425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3197765425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:03.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:03.616Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:03.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:03.617Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:04 smithi136 ceph-mon[113531]: pgmap v766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:04 smithi093 ceph-mon[111335]: pgmap v766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:04 smithi093 ceph-mon[118902]: pgmap v766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:05 smithi093 ceph-mon[111335]: pgmap v767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/985172219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:19:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:19:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:19:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:05 smithi093 ceph-mon[118902]: pgmap v767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/985172219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:05 smithi136 ceph-mon[113531]: pgmap v767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/985172219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:07 smithi136 ceph-mon[113531]: pgmap v768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:07 smithi093 ceph-mon[111335]: pgmap v768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:07 smithi093 ceph-mon[118902]: pgmap v768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3572303392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3572303392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3572303392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:10.184 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:09 smithi093 ceph-mon[111335]: pgmap v769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:10.184 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:09 smithi093 ceph-mon[118902]: pgmap v769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:10.185 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:10.184Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:09 smithi136 ceph-mon[113531]: pgmap v769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:10.185Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/397413425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/397413425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/397413425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:11 smithi093 ceph-mon[111335]: pgmap v770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:11 smithi093 ceph-mon[118902]: pgmap v770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:11.731Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:11.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:11 smithi136 ceph-mon[113531]: pgmap v770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1140618709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1140618709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1140618709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:13 smithi093 ceph-mon[111335]: pgmap v771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:13 smithi093 ceph-mon[118902]: pgmap v771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:13.616Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:13.617Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:13 smithi136 ceph-mon[113531]: pgmap v771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:15.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:19:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:19:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:19:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:15 smithi136 ceph-mon[113531]: pgmap v772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3038657459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:15 smithi093 ceph-mon[118902]: pgmap v772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3038657459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:15 smithi093 ceph-mon[111335]: pgmap v772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3038657459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:17 smithi136 ceph-mon[113531]: pgmap v773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3749738428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:17 smithi093 ceph-mon[111335]: pgmap v773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3749738428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:17 smithi093 ceph-mon[118902]: pgmap v773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3749738428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:20.185 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:19 smithi093 ceph-mon[118902]: pgmap v774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:20.185 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2633890187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:20.185 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:19 smithi093 ceph-mon[111335]: pgmap v774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:20.185 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2633890187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:19 smithi136 ceph-mon[113531]: pgmap v774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2633890187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:20.185Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:20.185Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[111335]: pgmap v775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:19:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[118902]: pgmap v775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:19:21.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:21.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:21.731Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:21.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:22.135 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:21 smithi136 ceph-mon[113531]: pgmap v775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:22.136 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:19:22.136 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:22.136 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:23.120 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3579076361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:23.120 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:23.121 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3579076361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3579076361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:23.788 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:23.617Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:23.789 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:23.618Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:24.110 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[111335]: pgmap v776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:24.110 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:24.110 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:24.111 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:24.111 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[118902]: pgmap v776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:24.111 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:24.111 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:24.111 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:23 smithi136 ceph-mon[113531]: pgmap v776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1903518736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1903518736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1903518736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:19:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:19:25] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:19:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:26 smithi136 ceph-mon[113531]: pgmap v777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:19:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:19:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[118902]: pgmap v777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:19:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:19:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[111335]: pgmap v777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:26.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:26.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:19:26.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:19:26.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:19:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2921401048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2921401048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2921401048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:28 smithi136 ceph-mon[113531]: pgmap v778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:28 smithi093 ceph-mon[111335]: pgmap v778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:28 smithi093 ceph-mon[118902]: pgmap v778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:30 smithi136 ceph-mon[113531]: pgmap v779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3862596773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:30 smithi093 ceph-mon[118902]: pgmap v779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3862596773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:30 smithi093 ceph-mon[111335]: pgmap v779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3862596773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:30.185Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:30.186Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:31.732Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:31.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:32 smithi136 ceph-mon[113531]: pgmap v780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2258535457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:32 smithi093 ceph-mon[111335]: pgmap v780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2258535457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:32 smithi093 ceph-mon[118902]: pgmap v780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2258535457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:33.618Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:33.619Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:34 smithi136 ceph-mon[113531]: pgmap v781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:34 smithi093 ceph-mon[111335]: pgmap v781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:34 smithi093 ceph-mon[118902]: pgmap v781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1233700265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1233700265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1233700265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:19:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:19:35] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:19:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:36 smithi136 ceph-mon[113531]: pgmap v782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:36 smithi093 ceph-mon[111335]: pgmap v782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:36 smithi093 ceph-mon[118902]: pgmap v782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3409737953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3409737953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3409737953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:38 smithi136 ceph-mon[113531]: pgmap v783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:38 smithi093 ceph-mon[111335]: pgmap v783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:38 smithi093 ceph-mon[118902]: pgmap v783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/61882916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/61882916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/61882916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:40 smithi136 ceph-mon[113531]: pgmap v784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:40 smithi093 ceph-mon[111335]: pgmap v784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:40 smithi093 ceph-mon[118902]: pgmap v784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:40.186Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:40.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:40.187Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:41.732Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:41.733Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:42 smithi136 ceph-mon[113531]: pgmap v785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1032168049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:42 smithi093 ceph-mon[111335]: pgmap v785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1032168049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:42 smithi093 ceph-mon[118902]: pgmap v785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1032168049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:43.618Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:43.619Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:44 smithi136 ceph-mon[113531]: pgmap v786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2480464944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:44 smithi093 ceph-mon[111335]: pgmap v786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2480464944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:44 smithi093 ceph-mon[118902]: pgmap v786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2480464944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:19:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:19:45] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:19:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:46 smithi136 ceph-mon[113531]: pgmap v787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2805099277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:46 smithi093 ceph-mon[111335]: pgmap v787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2805099277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:46 smithi093 ceph-mon[118902]: pgmap v787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2805099277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:48 smithi136 ceph-mon[113531]: pgmap v788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:48 smithi093 ceph-mon[111335]: pgmap v788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:48 smithi093 ceph-mon[118902]: pgmap v788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4263431286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4263431286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4263431286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:50 smithi136 ceph-mon[113531]: pgmap v789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:50 smithi093 ceph-mon[111335]: pgmap v789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:50 smithi093 ceph-mon[118902]: pgmap v789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:50.187Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:50.188Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/883526300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/883526300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/883526300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:51.733Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:51.734Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:52 smithi136 ceph-mon[113531]: pgmap v790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:52 smithi093 ceph-mon[111335]: pgmap v790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:52 smithi093 ceph-mon[118902]: pgmap v790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:53.619Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:19:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:19:53.620Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:19:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:54 smithi136 ceph-mon[113531]: pgmap v791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2126575491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:54 smithi093 ceph-mon[111335]: pgmap v791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2126575491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:54 smithi093 ceph-mon[118902]: pgmap v791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2126575491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:19:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:19:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:19:55] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:19:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:56 smithi136 ceph-mon[113531]: pgmap v792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/178630843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:56 smithi093 ceph-mon[111335]: pgmap v792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/178630843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:56 smithi093 ceph-mon[118902]: pgmap v792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/178630843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:58 smithi136 ceph-mon[113531]: pgmap v793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:19:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/553462943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:58 smithi093 ceph-mon[111335]: pgmap v793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:19:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/553462943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:19:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:58 smithi093 ceph-mon[118902]: pgmap v793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:19:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:19:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/553462943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:00 smithi136 ceph-mon[113531]: pgmap v794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:20:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:00 smithi093 ceph-mon[111335]: pgmap v794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:20:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:00 smithi093 ceph-mon[118902]: pgmap v794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:20:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:00.188Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:00.189Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/817287276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/817287276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/817287276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:01.733Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:01.734Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:02 smithi136 ceph-mon[113531]: pgmap v795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:02 smithi093 ceph-mon[111335]: pgmap v795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:02 smithi093 ceph-mon[118902]: pgmap v795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2053776614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2053776614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2053776614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:03.620Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:03.621Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:04 smithi136 ceph-mon[113531]: pgmap v796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:04 smithi093 ceph-mon[111335]: pgmap v796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:04 smithi093 ceph-mon[118902]: pgmap v796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:05.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2732283717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2732283717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2732283717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:20:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:20:05] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:20:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:06 smithi136 ceph-mon[113531]: pgmap v797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:06 smithi093 ceph-mon[111335]: pgmap v797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:06 smithi093 ceph-mon[118902]: pgmap v797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:08 smithi136 ceph-mon[113531]: pgmap v798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1489029887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:08 smithi093 ceph-mon[111335]: pgmap v798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1489029887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:08 smithi093 ceph-mon[118902]: pgmap v798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1489029887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:10 smithi136 ceph-mon[113531]: pgmap v799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2433264090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:10 smithi093 ceph-mon[111335]: pgmap v799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2433264090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:10 smithi093 ceph-mon[118902]: pgmap v799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2433264090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:10.189Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:10.190Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:11.733Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:11.734Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:12 smithi136 ceph-mon[113531]: pgmap v800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:12 smithi093 ceph-mon[111335]: pgmap v800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:12 smithi093 ceph-mon[118902]: pgmap v800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/692205908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/692205908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/692205908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:13.620Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:13.621Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:14 smithi136 ceph-mon[113531]: pgmap v801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:14 smithi093 ceph-mon[111335]: pgmap v801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:14 smithi093 ceph-mon[118902]: pgmap v801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/28443031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/28443031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/28443031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:20:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:20:15] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:20:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:16 smithi136 ceph-mon[113531]: pgmap v802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:16 smithi093 ceph-mon[111335]: pgmap v802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:16 smithi093 ceph-mon[118902]: pgmap v802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3014516881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3014516881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3014516881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:18 smithi136 ceph-mon[113531]: pgmap v803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:18 smithi093 ceph-mon[111335]: pgmap v803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:18 smithi093 ceph-mon[118902]: pgmap v803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:20 smithi136 ceph-mon[113531]: pgmap v804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/862880137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:20 smithi093 ceph-mon[111335]: pgmap v804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/862880137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:20 smithi093 ceph-mon[118902]: pgmap v804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/862880137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:20.189Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:20.190Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:21.734Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:21.735Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:22 smithi136 ceph-mon[113531]: pgmap v805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/60940025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:22 smithi093 ceph-mon[111335]: pgmap v805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/60940025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:22 smithi093 ceph-mon[118902]: pgmap v805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/60940025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:23.621Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:23.622Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:24 smithi136 ceph-mon[113531]: pgmap v806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2750558059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:24 smithi093 ceph-mon[111335]: pgmap v806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2750558059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:24 smithi093 ceph-mon[118902]: pgmap v806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2750558059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:20:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:20:25] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:20:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:26 smithi136 ceph-mon[113531]: pgmap v807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:20:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:26 smithi093 ceph-mon[111335]: pgmap v807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:20:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:26 smithi093 ceph-mon[118902]: pgmap v807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:20:27.346 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1380416389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:27.346 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:27.346 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1380416389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1380416389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:28 smithi136 ceph-mon[113531]: pgmap v808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:20:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:20:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[111335]: pgmap v808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[118902]: pgmap v808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:20:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:20:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:20:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:29 smithi136 ceph-mon[113531]: pgmap v809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3045131619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:30.191 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:29 smithi093 ceph-mon[111335]: pgmap v809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:30.192 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3045131619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:30.192 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:29 smithi093 ceph-mon[118902]: pgmap v809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:30.192 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3045131619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:30.190Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:30.192Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:31 smithi093 ceph-mon[111335]: pgmap v810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4104693713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:31 smithi093 ceph-mon[118902]: pgmap v810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4104693713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:31.735Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:31.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:31.736Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:31 smithi136 ceph-mon[113531]: pgmap v810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4104693713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:33 smithi093 ceph-mon[118902]: pgmap v811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1237714220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:33 smithi093 ceph-mon[111335]: pgmap v811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1237714220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:33.622Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:33.623Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:33 smithi136 ceph-mon[113531]: pgmap v811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1237714220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:20:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:20:35] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:20:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:35 smithi136 ceph-mon[113531]: pgmap v812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:35 smithi093 ceph-mon[111335]: pgmap v812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:35 smithi093 ceph-mon[118902]: pgmap v812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3125600065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3125600065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3125600065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:37 smithi136 ceph-mon[113531]: pgmap v813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:37 smithi093 ceph-mon[111335]: pgmap v813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:37 smithi093 ceph-mon[118902]: pgmap v813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1370312672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1370312672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1370312672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:39 smithi136 ceph-mon[113531]: pgmap v814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:40.191 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:39 smithi093 ceph-mon[111335]: pgmap v814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:40.191 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:39 smithi093 ceph-mon[118902]: pgmap v814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:40.191Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:40.192Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:41 smithi093 ceph-mon[111335]: pgmap v815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1470087926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:41 smithi093 ceph-mon[118902]: pgmap v815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1470087926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:41.735Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:41.736Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:41 smithi136 ceph-mon[113531]: pgmap v815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1470087926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:43 smithi093 ceph-mon[111335]: pgmap v816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1469580675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:43 smithi093 ceph-mon[118902]: pgmap v816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1469580675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:43.623Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:43.624Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:43 smithi136 ceph-mon[113531]: pgmap v816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1469580675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:20:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:20:45] "GET /metrics HTTP/1.1" 200 239873 "" "Prometheus/2.43.0" 2023-08-27T21:20:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:45 smithi136 ceph-mon[113531]: pgmap v817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12047850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:45 smithi093 ceph-mon[111335]: pgmap v817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12047850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:45 smithi093 ceph-mon[118902]: pgmap v817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12047850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:47 smithi136 ceph-mon[113531]: pgmap v818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:47 smithi093 ceph-mon[111335]: pgmap v818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:47 smithi093 ceph-mon[118902]: pgmap v818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1102841839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1102841839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1102841839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:49 smithi136 ceph-mon[113531]: pgmap v819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:50.192 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:49 smithi093 ceph-mon[111335]: pgmap v819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:50.192 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:49 smithi093 ceph-mon[118902]: pgmap v819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:50.192Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:50.193Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/730208065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/730208065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/730208065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:51 smithi093 ceph-mon[111335]: pgmap v820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:51 smithi093 ceph-mon[118902]: pgmap v820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:51.736Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:51.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:51.737Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:51 smithi136 ceph-mon[113531]: pgmap v820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2513789913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2513789913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2513789913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:53.940 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:53.623Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:53.940 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:20:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:20:53.624Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:20:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:53 smithi093 ceph-mon[111335]: pgmap v821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:53 smithi093 ceph-mon[118902]: pgmap v821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:53 smithi136 ceph-mon[113531]: pgmap v821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:20:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:20:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:20:55] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:20:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:55 smithi093 ceph-mon[111335]: pgmap v822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3411591895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:55 smithi093 ceph-mon[118902]: pgmap v822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3411591895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:55 smithi136 ceph-mon[113531]: pgmap v822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3411591895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:57 smithi093 ceph-mon[111335]: pgmap v823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/172277416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:57 smithi093 ceph-mon[118902]: pgmap v823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/172277416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:20:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:57 smithi136 ceph-mon[113531]: pgmap v823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:20:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/172277416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:59 smithi093 ceph-mon[111335]: pgmap v824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:20:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1505401287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:59 smithi093 ceph-mon[118902]: pgmap v824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:20:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1505401287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:00.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:00.192Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:00.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:00.193Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:59 smithi136 ceph-mon[113531]: pgmap v824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:20:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1505401287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:01 smithi093 ceph-mon[111335]: pgmap v825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:01 smithi093 ceph-mon[118902]: pgmap v825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:01.737Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:01.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:01.738Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:01 smithi136 ceph-mon[113531]: pgmap v825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3846303142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3846303142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3846303142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:03.958 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:03.624Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:03.958 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:03.625Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:03 smithi093 ceph-mon[111335]: pgmap v826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:03 smithi093 ceph-mon[118902]: pgmap v826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:03 smithi136 ceph-mon[113531]: pgmap v826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1997438845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1997438845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:05.319 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1997438845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T21:21:05.317960508Z level=info msg="Completed cleanup jobs" duration=12.992424ms 2023-08-27T21:21:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:21:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:21:05] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:21:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:05 smithi093 ceph-mon[111335]: pgmap v827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:05 smithi093 ceph-mon[118902]: pgmap v827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:05 smithi136 ceph-mon[113531]: pgmap v827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:07 smithi093 ceph-mon[111335]: pgmap v828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2819407567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:07 smithi093 ceph-mon[118902]: pgmap v828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:08.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2819407567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:07 smithi136 ceph-mon[113531]: pgmap v828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2819407567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:09 smithi093 ceph-mon[111335]: pgmap v829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2435466369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:09 smithi093 ceph-mon[118902]: pgmap v829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2435466369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:10.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:10.193Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:10.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:10.194Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:09 smithi136 ceph-mon[113531]: pgmap v829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2435466369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:11 smithi093 ceph-mon[118902]: pgmap v830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/849427854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:11 smithi093 ceph-mon[111335]: pgmap v830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/849427854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:11.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:11.738Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:11.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:11.738Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:11 smithi136 ceph-mon[113531]: pgmap v830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/849427854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:13.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:13.625Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:13.982 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:13.626Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:13 smithi093 ceph-mon[111335]: pgmap v831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:13 smithi093 ceph-mon[118902]: pgmap v831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:13 smithi136 ceph-mon[113531]: pgmap v831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1456155768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1456155768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1456155768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:21:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:21:15] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:21:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:15 smithi093 ceph-mon[111335]: pgmap v832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:15 smithi093 ceph-mon[118902]: pgmap v832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:15 smithi136 ceph-mon[113531]: pgmap v832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1634475648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1634475648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1634475648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:17 smithi136 ceph-mon[113531]: pgmap v833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:17 smithi093 ceph-mon[111335]: pgmap v833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:18 smithi093 ceph-mon[118902]: pgmap v833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:19 smithi136 ceph-mon[113531]: pgmap v834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2068366213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:20.193 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:19 smithi093 ceph-mon[111335]: pgmap v834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:20.193 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2068366213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:20.194 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:19 smithi093 ceph-mon[118902]: pgmap v834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:20.194 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2068366213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:20.193Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:20.194Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:21 smithi093 ceph-mon[111335]: pgmap v835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/90397923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:21 smithi093 ceph-mon[118902]: pgmap v835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:21.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/90397923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:21.738Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:21.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:21.739Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:21 smithi136 ceph-mon[113531]: pgmap v835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/90397923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:23.626Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:23.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-mon[111335]: pgmap v836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1597102430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-mon[118902]: pgmap v836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1597102430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:23 smithi136 ceph-mon[113531]: pgmap v836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1597102430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:21:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:21:25] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:21:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:25 smithi136 ceph-mon[113531]: pgmap v837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:25 smithi093 ceph-mon[118902]: pgmap v837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:25 smithi093 ceph-mon[111335]: pgmap v837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/50478413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/50478413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/50478413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:27 smithi136 ceph-mon[113531]: pgmap v838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:27 smithi093 ceph-mon[111335]: pgmap v838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:27 smithi093 ceph-mon[118902]: pgmap v838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:21:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2517989233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:21:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2517989233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:29.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:21:29.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2517989233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:29 smithi136 ceph-mon[113531]: pgmap v839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:29 smithi093 ceph-mon[111335]: pgmap v839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:29 smithi093 ceph-mon[118902]: pgmap v839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:30.194Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:30.195Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:31 smithi136 ceph-mon[113531]: pgmap v840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/855539113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:21:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:21:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[111335]: pgmap v840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/855539113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[118902]: pgmap v840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/855539113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:21:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:21:32.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:31.739Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:32.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:31.740Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:33 smithi093 ceph-mon[111335]: pgmap v841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1627741205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:33 smithi093 ceph-mon[118902]: pgmap v841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1627741205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:33.626Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:33.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:33 smithi136 ceph-mon[113531]: pgmap v841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1627741205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:21:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:21:35] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:21:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:35 smithi136 ceph-mon[113531]: pgmap v842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/895038193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:35 smithi093 ceph-mon[118902]: pgmap v842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/895038193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:35 smithi093 ceph-mon[111335]: pgmap v842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/895038193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:37 smithi136 ceph-mon[113531]: pgmap v843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:37 smithi093 ceph-mon[111335]: pgmap v843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:37 smithi093 ceph-mon[118902]: pgmap v843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3846142062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3846142062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3846142062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:39 smithi136 ceph-mon[113531]: pgmap v844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:40.194 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:39 smithi093 ceph-mon[111335]: pgmap v844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:40.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:39 smithi093 ceph-mon[118902]: pgmap v844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:40.194Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:40.195Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1439476701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1439476701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1439476701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:41 smithi136 ceph-mon[113531]: pgmap v845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:41 smithi093 ceph-mon[111335]: pgmap v845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:41 smithi093 ceph-mon[118902]: pgmap v845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:41.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:41.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/216333648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/216333648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/216333648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:43 smithi093 ceph-mon[118902]: pgmap v846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:43 smithi093 ceph-mon[111335]: pgmap v846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:43.627Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:43.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:43 smithi136 ceph-mon[113531]: pgmap v846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:21:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:21:45] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:21:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:45 smithi136 ceph-mon[113531]: pgmap v847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3301809437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:45 smithi093 ceph-mon[118902]: pgmap v847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3301809437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:45 smithi093 ceph-mon[111335]: pgmap v847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3301809437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:47 smithi136 ceph-mon[113531]: pgmap v848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3118426365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:47 smithi093 ceph-mon[111335]: pgmap v848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3118426365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:47 smithi093 ceph-mon[118902]: pgmap v848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3118426365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:49 smithi136 ceph-mon[113531]: pgmap v849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:50.195 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:49 smithi093 ceph-mon[111335]: pgmap v849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:50.195 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:49 smithi093 ceph-mon[118902]: pgmap v849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:50.195Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:50.196Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/713478071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/713478071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/713478071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:51 smithi136 ceph-mon[113531]: pgmap v850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:51 smithi093 ceph-mon[111335]: pgmap v850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:51 smithi093 ceph-mon[118902]: pgmap v850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:51.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:51.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4129363771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4129363771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4129363771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:53 smithi093 ceph-mon[111335]: pgmap v851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:53 smithi093 ceph-mon[118902]: pgmap v851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:53.627Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:21:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:21:53.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:21:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:53 smithi136 ceph-mon[113531]: pgmap v851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:21:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3693418048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3693418048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3693418048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:21:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:21:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:21:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:55 smithi136 ceph-mon[113531]: pgmap v852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:55 smithi093 ceph-mon[111335]: pgmap v852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:55 smithi093 ceph-mon[118902]: pgmap v852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:57 smithi136 ceph-mon[113531]: pgmap v853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:58.196 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4270317205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:57 smithi093 ceph-mon[111335]: pgmap v853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4270317205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:21:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:57 smithi093 ceph-mon[118902]: pgmap v853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:21:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4270317205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:59 smithi136 ceph-mon[113531]: pgmap v854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:21:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1646342211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:00.197 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:59 smithi093 ceph-mon[111335]: pgmap v854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:00.197 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:21:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1646342211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:00.198 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:59 smithi093 ceph-mon[118902]: pgmap v854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:00.198 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:21:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1646342211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:00.195Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:00.196Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:01 smithi136 ceph-mon[113531]: pgmap v855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3806841722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:01 smithi093 ceph-mon[111335]: pgmap v855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3806841722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:01 smithi093 ceph-mon[118902]: pgmap v855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3806841722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:01.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:01.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:03.882 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:03.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:03.882 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:03.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:03 smithi136 ceph-mon[113531]: pgmap v856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:03 smithi093 ceph-mon[111335]: pgmap v856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:03 smithi093 ceph-mon[118902]: pgmap v856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2433036179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2433036179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2433036179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:22:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:22:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:22:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:05 smithi136 ceph-mon[113531]: pgmap v857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:05 smithi093 ceph-mon[111335]: pgmap v857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:05 smithi093 ceph-mon[118902]: pgmap v857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3311175133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3311175133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3311175133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:07 smithi136 ceph-mon[113531]: pgmap v858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:07 smithi093 ceph-mon[111335]: pgmap v858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:07 smithi093 ceph-mon[118902]: pgmap v858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:09 smithi136 ceph-mon[113531]: pgmap v859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3947900886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:10.196 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:09 smithi093 ceph-mon[111335]: pgmap v859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:10.196 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3947900886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:10.197 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:09 smithi093 ceph-mon[118902]: pgmap v859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:10.197 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3947900886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:10.196Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:10.197Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:11 smithi136 ceph-mon[113531]: pgmap v860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/544968884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:11 smithi093 ceph-mon[111335]: pgmap v860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/544968884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:11 smithi093 ceph-mon[118902]: pgmap v860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/544968884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:12.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:11.741Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:12.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:11.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:13.909 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:13.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:13.909 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:13.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:13 smithi136 ceph-mon[113531]: pgmap v861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2628528774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:13 smithi093 ceph-mon[111335]: pgmap v861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2628528774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:13 smithi093 ceph-mon[118902]: pgmap v861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2628528774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:22:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:22:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:22:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:15 smithi136 ceph-mon[113531]: pgmap v862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:15 smithi093 ceph-mon[111335]: pgmap v862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:15 smithi093 ceph-mon[118902]: pgmap v862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:17.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3799327134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3799327134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3799327134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:17 smithi136 ceph-mon[113531]: pgmap v863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:17 smithi093 ceph-mon[111335]: pgmap v863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:17 smithi093 ceph-mon[118902]: pgmap v863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3423841090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3423841090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3423841090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:19 smithi136 ceph-mon[113531]: pgmap v864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:20.197 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:19 smithi093 ceph-mon[111335]: pgmap v864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:20.197 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:19 smithi093 ceph-mon[118902]: pgmap v864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:20.197Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:20.198Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1634271768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1634271768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1634271768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:21 smithi136 ceph-mon[113531]: pgmap v865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:21 smithi093 ceph-mon[111335]: pgmap v865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:21 smithi093 ceph-mon[118902]: pgmap v865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:21.742Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:21.743Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:23.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:23.629Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:23.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:23.630Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:23 smithi136 ceph-mon[113531]: pgmap v866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2476039041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-mon[111335]: pgmap v866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2476039041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-mon[118902]: pgmap v866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2476039041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:22:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:22:25] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:22:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:25 smithi093 ceph-mon[111335]: pgmap v867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2875074464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:25 smithi093 ceph-mon[118902]: pgmap v867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2875074464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:25 smithi136 ceph-mon[113531]: pgmap v867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2875074464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:27 smithi093 ceph-mon[111335]: pgmap v868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:27 smithi093 ceph-mon[118902]: pgmap v868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:27 smithi136 ceph-mon[113531]: pgmap v868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2012007437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2012007437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2012007437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:29 smithi093 ceph-mon[111335]: pgmap v869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:29 smithi093 ceph-mon[118902]: pgmap v869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:30.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:30.197Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:30.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:30.198Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:29 smithi136 ceph-mon[113531]: pgmap v869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/52259626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/52259626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/52259626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:31 smithi136 ceph-mon[113531]: pgmap v870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:22:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:31 smithi093 ceph-mon[111335]: pgmap v870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:22:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:31 smithi093 ceph-mon[118902]: pgmap v870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:22:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:31.742Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:31.743Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1742547279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1742547279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1742547279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:33.629Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:33.630Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:34 smithi136 ceph-mon[113531]: pgmap v871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[118902]: pgmap v871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[111335]: pgmap v871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:35.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:22:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:22:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:22:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:22:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:22:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:22:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:22:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:22:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:22:35] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:22:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:36 smithi136 ceph-mon[113531]: pgmap v872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3944999558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:36 smithi093 ceph-mon[111335]: pgmap v872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3944999558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:36 smithi093 ceph-mon[118902]: pgmap v872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3944999558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:38 smithi093 ceph-mon[111335]: pgmap v873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1050590906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:38 smithi093 ceph-mon[118902]: pgmap v873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1050590906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:38 smithi136 ceph-mon[113531]: pgmap v873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1050590906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:40 smithi093 ceph-mon[118902]: pgmap v874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3763991093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:40 smithi093 ceph-mon[111335]: pgmap v874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3763991093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:40.198Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:40.198Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:40 smithi136 ceph-mon[113531]: pgmap v874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3763991093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:42.198 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:41.743Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:42.198 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:41.744Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:42 smithi093 ceph-mon[118902]: pgmap v875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:42 smithi093 ceph-mon[111335]: pgmap v875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:42 smithi136 ceph-mon[113531]: pgmap v875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2429782034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2429782034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2429782034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:43.993 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:43.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:43.993 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:43.631Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:44 smithi093 ceph-mon[111335]: pgmap v876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:44 smithi093 ceph-mon[118902]: pgmap v876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:44 smithi136 ceph-mon[113531]: pgmap v876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1806805763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:22:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:22:45] "GET /metrics HTTP/1.1" 200 239875 "" "Prometheus/2.43.0" 2023-08-27T21:22:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1806805763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1806805763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:46 smithi093 ceph-mon[111335]: pgmap v877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:46 smithi093 ceph-mon[118902]: pgmap v877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:46 smithi136 ceph-mon[113531]: pgmap v877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3542067023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:47.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3542067023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3542067023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:48 smithi093 ceph-mon[111335]: pgmap v878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:48 smithi093 ceph-mon[118902]: pgmap v878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:48 smithi136 ceph-mon[113531]: pgmap v878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:50 smithi093 ceph-mon[118902]: pgmap v879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3672000564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:50 smithi093 ceph-mon[111335]: pgmap v879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3672000564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:50.198Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:50.199Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:50 smithi136 ceph-mon[113531]: pgmap v879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3672000564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:52.226 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:51.744Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:52.226 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:51.744Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:52 smithi093 ceph-mon[111335]: pgmap v880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/627785581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:52 smithi093 ceph-mon[118902]: pgmap v880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/627785581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:52 smithi136 ceph-mon[113531]: pgmap v880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/627785581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:53.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:22:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:22:53.631Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:22:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:54 smithi093 ceph-mon[111335]: pgmap v881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:54 smithi093 ceph-mon[118902]: pgmap v881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:54 smithi136 ceph-mon[113531]: pgmap v881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:22:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/844462374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/844462374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:22:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:22:55] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:22:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/844462374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:56 smithi136 ceph-mon[113531]: pgmap v882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:56 smithi093 ceph-mon[111335]: pgmap v882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:56 smithi093 ceph-mon[118902]: pgmap v882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/494025211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/494025211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/494025211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:22:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:58 smithi136 ceph-mon[113531]: pgmap v883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:58 smithi093 ceph-mon[111335]: pgmap v883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:22:58.806 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:58 smithi093 ceph-mon[118902]: pgmap v883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:59 smithi136 ceph-mon[113531]: pgmap v884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:22:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2999628798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:00.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:59 smithi093 ceph-mon[111335]: pgmap v884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:00.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:22:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2999628798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:00.199 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:59 smithi093 ceph-mon[118902]: pgmap v884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:00.199 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:22:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2999628798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:00.199Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:00.200Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:01 smithi136 ceph-mon[113531]: pgmap v885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/363084995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:01 smithi093 ceph-mon[111335]: pgmap v885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/363084995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:01 smithi093 ceph-mon[118902]: pgmap v885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/363084995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:02.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:01.745Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:02.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:01.746Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:03 smithi093 ceph-mon[111335]: pgmap v886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/948618091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:03 smithi093 ceph-mon[118902]: pgmap v886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:03.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/948618091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:03.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:03.631Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:03.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:03.632Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:03 smithi136 ceph-mon[113531]: pgmap v886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/948618091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:23:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:23:05] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:23:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:05 smithi136 ceph-mon[113531]: pgmap v887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:05 smithi093 ceph-mon[111335]: pgmap v887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:05 smithi093 ceph-mon[118902]: pgmap v887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3140933648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3140933648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3140933648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:07 smithi136 ceph-mon[113531]: pgmap v888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:07 smithi093 ceph-mon[111335]: pgmap v888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:07 smithi093 ceph-mon[118902]: pgmap v888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1881767753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1881767753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1881767753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:09 smithi136 ceph-mon[113531]: pgmap v889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:10.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:09 smithi093 ceph-mon[111335]: pgmap v889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:10.200 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:09 smithi093 ceph-mon[118902]: pgmap v889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:10.199Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:10.201Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:11 smithi136 ceph-mon[113531]: pgmap v890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/124304761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:11 smithi093 ceph-mon[111335]: pgmap v890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:12.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/124304761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:11 smithi093 ceph-mon[118902]: pgmap v890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/124304761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:12.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:11.745Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:12.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:11.746Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:13 smithi093 ceph-mon[111335]: pgmap v891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2363983179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:13 smithi093 ceph-mon[118902]: pgmap v891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2363983179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:13.632Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:13.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:13.633Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:13 smithi136 ceph-mon[113531]: pgmap v891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2363983179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:23:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:23:15] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:23:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:15 smithi136 ceph-mon[113531]: pgmap v892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/775912520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:15 smithi093 ceph-mon[111335]: pgmap v892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/775912520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:15 smithi093 ceph-mon[118902]: pgmap v892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/775912520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:17 smithi136 ceph-mon[113531]: pgmap v893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:17 smithi093 ceph-mon[111335]: pgmap v893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:17 smithi093 ceph-mon[118902]: pgmap v893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3500690346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3500690346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3500690346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:19 smithi136 ceph-mon[113531]: pgmap v894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:20.200 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:19 smithi093 ceph-mon[111335]: pgmap v894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:20.201 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:19 smithi093 ceph-mon[118902]: pgmap v894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:20.203 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:20.200Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:20.203 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:20.201Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/622678123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:21.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/622678123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/622678123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:21 smithi136 ceph-mon[113531]: pgmap v895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:22.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:21 smithi093 ceph-mon[111335]: pgmap v895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:22.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:21 smithi093 ceph-mon[118902]: pgmap v895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:22.246 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:21.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:22.246 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:21.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1413163992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1413163992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1413163992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:23 smithi093 ceph-mon[118902]: pgmap v896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:23 smithi093 ceph-mon[111335]: pgmap v896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:23.633Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:23.634Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:23 smithi136 ceph-mon[113531]: pgmap v896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:23:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:23:25] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:23:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:25 smithi136 ceph-mon[113531]: pgmap v897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1140160411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:25 smithi093 ceph-mon[111335]: pgmap v897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1140160411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:25 smithi093 ceph-mon[118902]: pgmap v897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1140160411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:27 smithi136 ceph-mon[113531]: pgmap v898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3651729341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:27 smithi093 ceph-mon[118902]: pgmap v898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3651729341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:27 smithi093 ceph-mon[111335]: pgmap v898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3651729341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:29 smithi136 ceph-mon[113531]: pgmap v899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:30.318 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:29 smithi093 ceph-mon[111335]: pgmap v899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:30.318 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:29 smithi093 ceph-mon[118902]: pgmap v899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:30.201Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:30.202Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2268466016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2268466016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2268466016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:31 smithi136 ceph-mon[113531]: pgmap v900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:31 smithi093 ceph-mon[111335]: pgmap v900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:31 smithi093 ceph-mon[118902]: pgmap v900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:31.747Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:31.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:33.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/171127025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/171127025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/171127025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:33 smithi093 ceph-mon[111335]: pgmap v901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:33.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:33 smithi093 ceph-mon[118902]: pgmap v901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:33.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:33.634Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:33.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:33.635Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:33 smithi136 ceph-mon[113531]: pgmap v901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:23:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2874839357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:23:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2874839357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:23:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2874839357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:23:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:23:35] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:23:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:35 smithi136 ceph-mon[113531]: pgmap v902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:37.133 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:35 smithi093 ceph-mon[111335]: pgmap v902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:37.133 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:35 smithi093 ceph-mon[118902]: pgmap v902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:38 smithi136 ceph-mon[113531]: pgmap v903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3386760586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:23:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:23:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[111335]: pgmap v903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3386760586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:23:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:23:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[118902]: pgmap v903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3386760586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:38.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:23:38.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:23:38.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:23:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:40 smithi136 ceph-mon[113531]: pgmap v904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778827535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:40 smithi093 ceph-mon[111335]: pgmap v904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778827535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:40 smithi093 ceph-mon[118902]: pgmap v904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778827535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:40.201Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:40.202Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:42.061 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:41.747Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:42.061 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:41.748Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:42 smithi136 ceph-mon[113531]: pgmap v905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3068561700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:42 smithi093 ceph-mon[111335]: pgmap v905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3068561700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:42 smithi093 ceph-mon[118902]: pgmap v905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3068561700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:43.635Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:43.636Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:44 smithi136 ceph-mon[113531]: pgmap v906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:44 smithi093 ceph-mon[118902]: pgmap v906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:44 smithi093 ceph-mon[111335]: pgmap v906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3228181704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3228181704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3228181704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:23:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:23:45] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:23:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:46 smithi136 ceph-mon[113531]: pgmap v907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:46 smithi093 ceph-mon[111335]: pgmap v907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:46 smithi093 ceph-mon[118902]: pgmap v907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2472689970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2472689970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2472689970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:48 smithi136 ceph-mon[113531]: pgmap v908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:48 smithi093 ceph-mon[118902]: pgmap v908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:48 smithi093 ceph-mon[111335]: pgmap v908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:50 smithi136 ceph-mon[113531]: pgmap v909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1166331225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:50 smithi093 ceph-mon[111335]: pgmap v909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1166331225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:50.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:50 smithi093 ceph-mon[118902]: pgmap v909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:50.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1166331225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:50.202Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:50.203Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:52.088 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:51.748Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:52.088 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:51.749Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:52 smithi136 ceph-mon[113531]: pgmap v910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/29961638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:52 smithi093 ceph-mon[111335]: pgmap v910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/29961638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:52 smithi093 ceph-mon[118902]: pgmap v910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/29961638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:53.635Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:23:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:23:53.636Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:23:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:54 smithi136 ceph-mon[113531]: pgmap v911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1550850100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:54 smithi093 ceph-mon[111335]: pgmap v911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1550850100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:54 smithi093 ceph-mon[118902]: pgmap v911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:23:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1550850100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:23:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:23:55] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:23:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:56 smithi136 ceph-mon[113531]: pgmap v912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:56 smithi093 ceph-mon[118902]: pgmap v912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:56 smithi093 ceph-mon[111335]: pgmap v912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1675305193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1675305193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1675305193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:58 smithi136 ceph-mon[113531]: pgmap v913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:58 smithi093 ceph-mon[111335]: pgmap v913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:58 smithi093 ceph-mon[118902]: pgmap v913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:23:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:23:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3311375353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:23:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3311375353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:23:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:23:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3311375353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:00 smithi136 ceph-mon[113531]: pgmap v914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:00 smithi093 ceph-mon[111335]: pgmap v914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:00 smithi093 ceph-mon[118902]: pgmap v914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:00.203Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:00.205Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3713644693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3713644693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3713644693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:02.115 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:01.749Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:02.115 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:01.750Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:02 smithi136 ceph-mon[113531]: pgmap v915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:02 smithi093 ceph-mon[111335]: pgmap v915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:02 smithi093 ceph-mon[118902]: pgmap v915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:03.636Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:03.638Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:04 smithi136 ceph-mon[113531]: pgmap v916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1403535352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:04 smithi093 ceph-mon[111335]: pgmap v916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1403535352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:04 smithi093 ceph-mon[118902]: pgmap v916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1403535352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:24:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:24:05] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:24:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:06 smithi136 ceph-mon[113531]: pgmap v917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4023284108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:06 smithi093 ceph-mon[111335]: pgmap v917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4023284108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:06 smithi093 ceph-mon[118902]: pgmap v917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4023284108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:08 smithi136 ceph-mon[113531]: pgmap v918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:08 smithi093 ceph-mon[111335]: pgmap v918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:08 smithi093 ceph-mon[118902]: pgmap v918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/837996355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/837996355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/837996355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:10 smithi136 ceph-mon[113531]: pgmap v919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:10 smithi093 ceph-mon[111335]: pgmap v919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:10 smithi093 ceph-mon[118902]: pgmap v919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:10.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:10.205Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3054448487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3054448487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3054448487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:12.140 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:11.750Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:12.140 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:11.751Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:12 smithi136 ceph-mon[113531]: pgmap v920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:12 smithi093 ceph-mon[111335]: pgmap v920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:12 smithi093 ceph-mon[118902]: pgmap v920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2534861265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2534861265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2534861265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:13.637Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:13.638Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:14 smithi136 ceph-mon[113531]: pgmap v921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:14 smithi093 ceph-mon[111335]: pgmap v921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:14 smithi093 ceph-mon[118902]: pgmap v921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:24:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:24:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:24:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:16 smithi136 ceph-mon[113531]: pgmap v922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:16.497 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1397037705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:16.498 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:16 smithi093 ceph-mon[111335]: pgmap v922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:16.498 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1397037705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:16.498 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:16 smithi093 ceph-mon[118902]: pgmap v922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:16.498 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1397037705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:18 smithi136 ceph-mon[113531]: pgmap v923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/483495691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:18 smithi093 ceph-mon[111335]: pgmap v923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/483495691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:18 smithi093 ceph-mon[118902]: pgmap v923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/483495691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:20 smithi136 ceph-mon[113531]: pgmap v924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2828966459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:20.506 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:20 smithi093 ceph-mon[111335]: pgmap v924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:20.507 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2828966459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:20.507 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:20 smithi093 ceph-mon[118902]: pgmap v924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:20.507 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2828966459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:20.508 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:20.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:20.508 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:20.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:22.157 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:21.751Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:22.158 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:21.751Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:22 smithi136 ceph-mon[113531]: pgmap v925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:22 smithi093 ceph-mon[111335]: pgmap v925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:22 smithi093 ceph-mon[118902]: pgmap v925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2821394010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2821394010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2821394010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:23.638Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:23.639Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:24 smithi136 ceph-mon[113531]: pgmap v926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:24 smithi093 ceph-mon[111335]: pgmap v926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:24 smithi093 ceph-mon[118902]: pgmap v926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3520302244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3520302244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3520302244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:24:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:24:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:24:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:26 smithi136 ceph-mon[113531]: pgmap v927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:26 smithi093 ceph-mon[118902]: pgmap v927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:26 smithi093 ceph-mon[111335]: pgmap v927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:28 smithi136 ceph-mon[113531]: pgmap v928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3890459987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:28 smithi093 ceph-mon[118902]: pgmap v928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3890459987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:28 smithi093 ceph-mon[111335]: pgmap v928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3890459987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:30 smithi136 ceph-mon[113531]: pgmap v929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/259233987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:30 smithi093 ceph-mon[111335]: pgmap v929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/259233987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:30 smithi093 ceph-mon[118902]: pgmap v929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/259233987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:30.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:30.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:32.183 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:31.751Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:32.184 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:31.752Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:32 smithi136 ceph-mon[113531]: pgmap v930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3090227574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:32 smithi093 ceph-mon[111335]: pgmap v930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3090227574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:32 smithi093 ceph-mon[118902]: pgmap v930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3090227574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:33.638Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:33.640Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:34 smithi136 ceph-mon[113531]: pgmap v931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:34 smithi093 ceph-mon[111335]: pgmap v931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:34 smithi093 ceph-mon[118902]: pgmap v931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3426209168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:24:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:24:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:24:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3426209168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3426209168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:36 smithi093 ceph-mon[111335]: pgmap v932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:36 smithi093 ceph-mon[118902]: pgmap v932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:36 smithi136 ceph-mon[113531]: pgmap v932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1727957494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1727957494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1727957494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:38.356 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:38 smithi136 ceph-mon[113531]: pgmap v933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:38.356 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:24:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:38 smithi093 ceph-mon[111335]: pgmap v933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:24:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:38 smithi093 ceph-mon[118902]: pgmap v933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:24:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/634200742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/634200742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/634200742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[118902]: pgmap v934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:40.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:40.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:40.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[111335]: pgmap v934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:40.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:40 smithi136 ceph-mon[113531]: pgmap v934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:41 smithi136 ceph-mon[113531]: pgmap v935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1976551465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:41.752Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:41.753Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[111335]: pgmap v935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1976551465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[118902]: pgmap v935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1976551465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:43 smithi136 ceph-mon[113531]: pgmap v936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:43.639Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:43.640Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[111335]: pgmap v936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:24:43.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:24:43.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:43 smithi093 ceph-mon[118902]: pgmap v936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/64603997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/64603997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/64603997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:24:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:24:45] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:24:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:45 smithi136 ceph-mon[113531]: pgmap v937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:45 smithi093 ceph-mon[111335]: pgmap v937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:45 smithi093 ceph-mon[118902]: pgmap v937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1366686461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1366686461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1366686461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:47 smithi136 ceph-mon[113531]: pgmap v938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:47 smithi093 ceph-mon[111335]: pgmap v938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:47 smithi093 ceph-mon[118902]: pgmap v938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/916291084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/916291084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/916291084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:49 smithi136 ceph-mon[113531]: pgmap v939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:50.208 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:49 smithi093 ceph-mon[111335]: pgmap v939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:50.208 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:49 smithi093 ceph-mon[118902]: pgmap v939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:50.207Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:50.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:51 smithi136 ceph-mon[113531]: pgmap v940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3548918037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:51 smithi093 ceph-mon[111335]: pgmap v940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3548918037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:51 smithi093 ceph-mon[118902]: pgmap v940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3548918037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:51.753Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:51.754Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-mon[111335]: pgmap v941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1393330911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-mon[118902]: pgmap v941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1393330911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:53.639Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:24:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:24:53.640Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:24:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:53 smithi136 ceph-mon[113531]: pgmap v941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1393330911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:24:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:24:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:24:55] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:24:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:55 smithi136 ceph-mon[113531]: pgmap v942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2928719520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:55 smithi093 ceph-mon[111335]: pgmap v942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2928719520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:55 smithi093 ceph-mon[118902]: pgmap v942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2928719520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:57 smithi136 ceph-mon[113531]: pgmap v943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:57 smithi093 ceph-mon[111335]: pgmap v943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:57 smithi093 ceph-mon[118902]: pgmap v943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:24:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3552270515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3552270515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:24:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3552270515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:24:59 smithi136 ceph-mon[113531]: pgmap v944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:00.209 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:24:59 smithi093 ceph-mon[111335]: pgmap v944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:00.209 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:24:59 smithi093 ceph-mon[118902]: pgmap v944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:00.208Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:00.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1606591040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1606591040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1606591040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:01 smithi136 ceph-mon[113531]: pgmap v945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:01 smithi093 ceph-mon[111335]: pgmap v945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:01 smithi093 ceph-mon[118902]: pgmap v945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:01.753Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:01.754Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:03 smithi093 ceph-mon[111335]: pgmap v946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1192131091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:03 smithi093 ceph-mon[118902]: pgmap v946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1192131091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:03.640Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:03.641Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:03 smithi136 ceph-mon[113531]: pgmap v946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1192131091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:25:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:25:05] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:25:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:05 smithi136 ceph-mon[113531]: pgmap v947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4219377929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:05 smithi093 ceph-mon[111335]: pgmap v947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4219377929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:05 smithi093 ceph-mon[118902]: pgmap v947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4219377929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:07 smithi136 ceph-mon[113531]: pgmap v948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2448670850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:07 smithi093 ceph-mon[111335]: pgmap v948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2448670850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:07 smithi093 ceph-mon[118902]: pgmap v948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2448670850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:09 smithi136 ceph-mon[113531]: pgmap v949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:10.209 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:09 smithi093 ceph-mon[111335]: pgmap v949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:10.210 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:09 smithi093 ceph-mon[118902]: pgmap v949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:10.209Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:10.210Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/790522531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/790522531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/790522531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:11 smithi136 ceph-mon[113531]: pgmap v950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:11 smithi093 ceph-mon[111335]: pgmap v950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:11 smithi093 ceph-mon[118902]: pgmap v950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:11.753Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:11.755Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3377368676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3377368676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3377368676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:13 smithi093 ceph-mon[111335]: pgmap v951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:13 smithi093 ceph-mon[118902]: pgmap v951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:13.641Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:13.642Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:13 smithi136 ceph-mon[113531]: pgmap v951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:25:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:25:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:25:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:15 smithi136 ceph-mon[113531]: pgmap v952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2576964541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:15 smithi093 ceph-mon[111335]: pgmap v952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2576964541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:15 smithi093 ceph-mon[118902]: pgmap v952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2576964541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:17 smithi136 ceph-mon[113531]: pgmap v953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/971346324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:17 smithi093 ceph-mon[111335]: pgmap v953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/971346324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:17 smithi093 ceph-mon[118902]: pgmap v953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/971346324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:19 smithi136 ceph-mon[113531]: pgmap v954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/54637012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:20.210 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:19 smithi093 ceph-mon[111335]: pgmap v954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:20.210 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/54637012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:20.211 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:19 smithi093 ceph-mon[118902]: pgmap v954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:20.211 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/54637012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:20.210Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:20.211Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:21 smithi136 ceph-mon[113531]: pgmap v955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:21 smithi093 ceph-mon[111335]: pgmap v955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:21 smithi093 ceph-mon[118902]: pgmap v955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:21.754Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:21.754Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/780216059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/780216059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/780216059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:23.906 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:23.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:23.906 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:23.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:23 smithi136 ceph-mon[113531]: pgmap v956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:23 smithi093 ceph-mon[118902]: pgmap v956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:23 smithi093 ceph-mon[111335]: pgmap v956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3556594569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3556594569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3556594569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:25:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:25:25] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:25:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:25 smithi136 ceph-mon[113531]: pgmap v957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:25 smithi093 ceph-mon[111335]: pgmap v957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:25 smithi093 ceph-mon[118902]: pgmap v957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3305179133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3305179133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3305179133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:27 smithi136 ceph-mon[113531]: pgmap v958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:27 smithi093 ceph-mon[111335]: pgmap v958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:27 smithi093 ceph-mon[118902]: pgmap v958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:29 smithi136 ceph-mon[113531]: pgmap v959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3829680322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:30.211 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:29 smithi093 ceph-mon[111335]: pgmap v959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:30.212 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3829680322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:30.212 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:29 smithi093 ceph-mon[118902]: pgmap v959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:30.212 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3829680322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:30.211Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:30.212Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:31 smithi136 ceph-mon[113531]: pgmap v960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2325379117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:31 smithi093 ceph-mon[118902]: pgmap v960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2325379117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:31 smithi093 ceph-mon[111335]: pgmap v960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2325379117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:32.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:31.754Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:32.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:31.755Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:33.927 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:33.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:33.927 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:33.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:33 smithi136 ceph-mon[113531]: pgmap v961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:33 smithi093 ceph-mon[111335]: pgmap v961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:33 smithi093 ceph-mon[118902]: pgmap v961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/689503293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/689503293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/689503293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:25:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:25:35] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:25:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:35 smithi136 ceph-mon[113531]: pgmap v962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:35 smithi093 ceph-mon[111335]: pgmap v962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:35 smithi093 ceph-mon[118902]: pgmap v962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2710877847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2710877847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2710877847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:37 smithi093 ceph-mon[111335]: pgmap v963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:37 smithi093 ceph-mon[118902]: pgmap v963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:37 smithi136 ceph-mon[113531]: pgmap v963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1743313118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1743313118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1743313118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:40.212 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:39 smithi093 ceph-mon[111335]: pgmap v964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:40.212 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:39 smithi093 ceph-mon[118902]: pgmap v964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:39 smithi136 ceph-mon[113531]: pgmap v964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:40.212Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:40.213Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:41 smithi093 ceph-mon[111335]: pgmap v965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2367517022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:41 smithi093 ceph-mon[118902]: pgmap v965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2367517022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:41.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:41.756Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:41 smithi136 ceph-mon[113531]: pgmap v965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2367517022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:25:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:25:43.365 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:25:43.912 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:43.643Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:43.912 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:43.645Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:43 smithi136 ceph-mon[113531]: pgmap v966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/471716073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:43 smithi093 ceph-mon[118902]: pgmap v966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/471716073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:43 smithi093 ceph-mon[111335]: pgmap v966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/471716073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[111335]: pgmap v967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:25:45] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[118902]: pgmap v967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:45 smithi136 ceph-mon[113531]: pgmap v967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:25:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:25:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:46.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3450274321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:25:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:25:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3450274321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:25:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:25:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:25:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3450274321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:47 smithi136 ceph-mon[113531]: pgmap v968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:47 smithi093 ceph-mon[111335]: pgmap v968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:47 smithi093 ceph-mon[118902]: pgmap v968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4158718895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4158718895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4158718895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:49 smithi136 ceph-mon[113531]: pgmap v969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:50.213 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:49 smithi093 ceph-mon[111335]: pgmap v969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:50.213 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:49 smithi093 ceph-mon[118902]: pgmap v969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:50.213Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:50.214Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3594037791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3594037791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3594037791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:51 smithi136 ceph-mon[113531]: pgmap v970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:51 smithi093 ceph-mon[111335]: pgmap v970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:51 smithi093 ceph-mon[118902]: pgmap v970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:51.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:51.756Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-mon[111335]: pgmap v971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3588076888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-mon[118902]: pgmap v971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:53.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3588076888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:53.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:53.644Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:25:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:25:53.645Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:25:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:53 smithi136 ceph-mon[113531]: pgmap v971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3588076888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:25:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:25:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:25:55] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:25:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:55 smithi136 ceph-mon[113531]: pgmap v972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1217796007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:55 smithi093 ceph-mon[111335]: pgmap v972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1217796007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:55 smithi093 ceph-mon[118902]: pgmap v972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1217796007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:57 smithi136 ceph-mon[113531]: pgmap v973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/658789693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:57 smithi093 ceph-mon[111335]: pgmap v973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/658789693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:25:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:57 smithi093 ceph-mon[118902]: pgmap v973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:25:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/658789693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:25:59 smithi136 ceph-mon[113531]: pgmap v974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:00.214 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:25:59 smithi093 ceph-mon[111335]: pgmap v974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:00.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:25:59 smithi093 ceph-mon[118902]: pgmap v974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:00.214 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:00.213Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:00.214Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2315093500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2315093500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2315093500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:01 smithi136 ceph-mon[113531]: pgmap v975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:01 smithi093 ceph-mon[111335]: pgmap v975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:01 smithi093 ceph-mon[118902]: pgmap v975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:01.756Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:01.757Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2085141606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2085141606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2085141606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:03 smithi093 ceph-mon[111335]: pgmap v976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:03 smithi093 ceph-mon[118902]: pgmap v976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:03.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:03.646Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:03 smithi136 ceph-mon[113531]: pgmap v976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:26:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:26:05] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:26:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:05 smithi136 ceph-mon[113531]: pgmap v977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3114321902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:05 smithi093 ceph-mon[111335]: pgmap v977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3114321902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:05 smithi093 ceph-mon[118902]: pgmap v977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3114321902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:07 smithi136 ceph-mon[113531]: pgmap v978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3148924786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:07 smithi093 ceph-mon[111335]: pgmap v978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3148924786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:07 smithi093 ceph-mon[118902]: pgmap v978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3148924786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:09 smithi136 ceph-mon[113531]: pgmap v979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3180199377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:10.214 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:09 smithi093 ceph-mon[111335]: pgmap v979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:10.215 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3180199377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:10.215 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:09 smithi093 ceph-mon[118902]: pgmap v979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:10.215 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3180199377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:10.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:10.215Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:10.215Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:11 smithi136 ceph-mon[113531]: pgmap v980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:11 smithi093 ceph-mon[111335]: pgmap v980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:11 smithi093 ceph-mon[118902]: pgmap v980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:11.756Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:11.758Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/122353242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/122353242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/122353242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:13 smithi093 ceph-mon[111335]: pgmap v981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:13 smithi093 ceph-mon[118902]: pgmap v981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:13.646Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:13.647Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:13 smithi136 ceph-mon[113531]: pgmap v981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3822999404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3822999404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3822999404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:26:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:26:15] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:26:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:15 smithi136 ceph-mon[113531]: pgmap v982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:15 smithi093 ceph-mon[111335]: pgmap v982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:15 smithi093 ceph-mon[118902]: pgmap v982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3399721586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3399721586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3399721586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:17 smithi136 ceph-mon[113531]: pgmap v983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:17 smithi093 ceph-mon[111335]: pgmap v983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:17 smithi093 ceph-mon[118902]: pgmap v983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:19 smithi136 ceph-mon[113531]: pgmap v984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3587796383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:20.216 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:19 smithi093 ceph-mon[111335]: pgmap v984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:20.216 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3587796383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:20.216 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:19 smithi093 ceph-mon[118902]: pgmap v984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:20.216 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3587796383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:20.215Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:20.216Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:21 smithi136 ceph-mon[113531]: pgmap v985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4198097111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:21 smithi093 ceph-mon[111335]: pgmap v985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4198097111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:21 smithi093 ceph-mon[118902]: pgmap v985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4198097111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:21.757Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:21.758Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:23.914 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:23.647Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:23.914 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:23.648Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:23 smithi136 ceph-mon[113531]: pgmap v986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:23 smithi093 ceph-mon[111335]: pgmap v986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:23 smithi093 ceph-mon[118902]: pgmap v986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2012110272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2012110272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2012110272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:26:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:26:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:26:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:25 smithi136 ceph-mon[113531]: pgmap v987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:25 smithi093 ceph-mon[111335]: pgmap v987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:25 smithi093 ceph-mon[118902]: pgmap v987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/447985473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/447985473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/447985473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:27 smithi136 ceph-mon[113531]: pgmap v988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:27 smithi093 ceph-mon[111335]: pgmap v988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:27 smithi093 ceph-mon[118902]: pgmap v988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/72441633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/72441633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/72441633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:29 smithi136 ceph-mon[113531]: pgmap v989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:30.216 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:29 smithi093 ceph-mon[111335]: pgmap v989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:30.217 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:29 smithi093 ceph-mon[118902]: pgmap v989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:30.216Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:30.217Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:31 smithi136 ceph-mon[113531]: pgmap v990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3025105955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:31 smithi093 ceph-mon[111335]: pgmap v990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3025105955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:31 smithi093 ceph-mon[118902]: pgmap v990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3025105955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:31.757Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:31.758Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:33.935 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:33.647Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:33.935 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:33.648Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:33 smithi136 ceph-mon[113531]: pgmap v991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1510544116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:33 smithi093 ceph-mon[118902]: pgmap v991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1510544116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:33 smithi093 ceph-mon[111335]: pgmap v991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1510544116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:26:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:26:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:26:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:35 smithi136 ceph-mon[113531]: pgmap v992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:35 smithi093 ceph-mon[111335]: pgmap v992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:35 smithi093 ceph-mon[118902]: pgmap v992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1745373314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1745373314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1745373314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:37 smithi093 ceph-mon[111335]: pgmap v993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:37 smithi093 ceph-mon[118902]: pgmap v993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:37 smithi136 ceph-mon[113531]: pgmap v993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/23386474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/23386474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/23386474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:40.217 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:39 smithi093 ceph-mon[111335]: pgmap v994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:40.217 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:39 smithi093 ceph-mon[118902]: pgmap v994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:39 smithi136 ceph-mon[113531]: pgmap v994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:40.217Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:40.218Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3962907103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3962907103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3962907103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:41 smithi093 ceph-mon[111335]: pgmap v995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:41 smithi093 ceph-mon[118902]: pgmap v995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:41.758Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:41.759Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:41 smithi136 ceph-mon[113531]: pgmap v995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:43.960 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:43.648Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:43.960 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:43.649Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:43 smithi093 ceph-mon[111335]: pgmap v996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2928472970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:43 smithi093 ceph-mon[118902]: pgmap v996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2928472970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:43 smithi136 ceph-mon[113531]: pgmap v996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2928472970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:26:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:26:45] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:26:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:45 smithi093 ceph-mon[111335]: pgmap v997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/412534095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:26:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:45 smithi093 ceph-mon[118902]: pgmap v997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/412534095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:26:46.433 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:45 smithi136 ceph-mon[113531]: pgmap v997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:46.433 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/412534095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:46.433 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:26:48.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:47 smithi136 ceph-mon[113531]: pgmap v998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:48.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:48.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:48.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/933455639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[111335]: pgmap v998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/933455639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[118902]: pgmap v998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/933455639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:26:49.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:26:49.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:26:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:26:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:49.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:26:49.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:26:49.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:26:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:50 smithi093 ceph-mon[111335]: pgmap v999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2533461429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:50 smithi093 ceph-mon[118902]: pgmap v999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2533461429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:50.217Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:50.218Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:50 smithi136 ceph-mon[113531]: pgmap v999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2533461429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:51.759Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:51.760Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:52 smithi136 ceph-mon[113531]: pgmap v1000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:52 smithi093 ceph-mon[111335]: pgmap v1000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:52 smithi093 ceph-mon[118902]: pgmap v1000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:53.649 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/529444055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:53.649 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/529444055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/529444055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:53.649Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:26:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:26:53.650Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:26:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:54 smithi136 ceph-mon[113531]: pgmap v1001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:54 smithi093 ceph-mon[111335]: pgmap v1001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:54 smithi093 ceph-mon[118902]: pgmap v1001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:26:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/697100682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/697100682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:26:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:26:55] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:26:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/697100682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:56 smithi136 ceph-mon[113531]: pgmap v1002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:56 smithi093 ceph-mon[111335]: pgmap v1002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:56 smithi093 ceph-mon[118902]: pgmap v1002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:58 smithi136 ceph-mon[113531]: pgmap v1003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:26:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3059703466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:58 smithi093 ceph-mon[111335]: pgmap v1003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:26:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3059703466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:26:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:58 smithi093 ceph-mon[118902]: pgmap v1003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:26:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:26:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3059703466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:00 smithi093 ceph-mon[111335]: pgmap v1004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1735765236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:00 smithi093 ceph-mon[118902]: pgmap v1004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1735765236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:00.218Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:00.219Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:00 smithi136 ceph-mon[113531]: pgmap v1004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1735765236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:01.759Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:01.761Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:02 smithi136 ceph-mon[113531]: pgmap v1005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/131339955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:02 smithi093 ceph-mon[111335]: pgmap v1005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/131339955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:02 smithi093 ceph-mon[118902]: pgmap v1005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/131339955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:03.649Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:03.650Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:04 smithi136 ceph-mon[113531]: pgmap v1006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:04.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:04 smithi093 ceph-mon[111335]: pgmap v1006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:04.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:04 smithi093 ceph-mon[118902]: pgmap v1006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2973213826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2973213826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2973213826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:27:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:27:05] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:27:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:06 smithi136 ceph-mon[113531]: pgmap v1007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:06 smithi093 ceph-mon[111335]: pgmap v1007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:06 smithi093 ceph-mon[118902]: pgmap v1007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4143623781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4143623781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4143623781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:08 smithi136 ceph-mon[113531]: pgmap v1008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:08 smithi093 ceph-mon[111335]: pgmap v1008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:08 smithi093 ceph-mon[118902]: pgmap v1008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/436366766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/436366766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/436366766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:10 smithi093 ceph-mon[111335]: pgmap v1009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:10 smithi093 ceph-mon[118902]: pgmap v1009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:10.220Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:10.221Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:10 smithi136 ceph-mon[113531]: pgmap v1009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:11.760Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:11.761Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:12 smithi136 ceph-mon[113531]: pgmap v1010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1564096704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:12 smithi093 ceph-mon[111335]: pgmap v1010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1564096704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:12 smithi093 ceph-mon[118902]: pgmap v1010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1564096704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:13.650Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:13.651Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:14 smithi136 ceph-mon[113531]: pgmap v1011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2853946031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:14 smithi093 ceph-mon[118902]: pgmap v1011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2853946031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:14 smithi093 ceph-mon[111335]: pgmap v1011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2853946031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:27:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:27:15] "GET /metrics HTTP/1.1" 200 239875 "" "Prometheus/2.43.0" 2023-08-27T21:27:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:16 smithi136 ceph-mon[113531]: pgmap v1012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:16 smithi093 ceph-mon[118902]: pgmap v1012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:16 smithi093 ceph-mon[111335]: pgmap v1012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1130227312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1130227312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1130227312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:18 smithi093 ceph-mon[111335]: pgmap v1013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:18 smithi093 ceph-mon[118902]: pgmap v1013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:18 smithi136 ceph-mon[113531]: pgmap v1013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:19 smithi136 ceph-mon[113531]: pgmap v1014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3975486220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:20.221 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:19 smithi093 ceph-mon[118902]: pgmap v1014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:20.221 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3975486220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:20.221 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:19 smithi093 ceph-mon[111335]: pgmap v1014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:20.221 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3975486220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:20.221Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:20.222Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:21 smithi136 ceph-mon[113531]: pgmap v1015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/889820811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:22.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:21 smithi093 ceph-mon[118902]: pgmap v1015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:22.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/889820811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:21 smithi093 ceph-mon[111335]: pgmap v1015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/889820811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:21.761Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:21.762Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-mon[111335]: pgmap v1016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3795270506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-mon[118902]: pgmap v1016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3795270506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:23.650Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:23.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:23.651Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:23 smithi136 ceph-mon[113531]: pgmap v1016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3795270506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:27:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:27:25] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:27:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:25 smithi136 ceph-mon[113531]: pgmap v1017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:25 smithi093 ceph-mon[111335]: pgmap v1017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:25 smithi093 ceph-mon[118902]: pgmap v1017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3908637357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3908637357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3908637357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:27 smithi136 ceph-mon[113531]: pgmap v1018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:27 smithi093 ceph-mon[111335]: pgmap v1018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:27 smithi093 ceph-mon[118902]: pgmap v1018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2000996238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2000996238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2000996238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:29 smithi136 ceph-mon[113531]: pgmap v1019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:30.221 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:29 smithi093 ceph-mon[111335]: pgmap v1019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:30.222 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:29 smithi093 ceph-mon[118902]: pgmap v1019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:30.221Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:30.223Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3483357096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3483357096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3483357096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:31 smithi136 ceph-mon[113531]: pgmap v1020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:31 smithi093 ceph-mon[111335]: pgmap v1020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:31 smithi093 ceph-mon[118902]: pgmap v1020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:31.762Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:31.763Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:33 smithi093 ceph-mon[111335]: pgmap v1021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1218038893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:33 smithi093 ceph-mon[118902]: pgmap v1021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1218038893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:33.651Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:33.652Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:33 smithi136 ceph-mon[113531]: pgmap v1021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1218038893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:27:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:27:35] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:27:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:35 smithi136 ceph-mon[113531]: pgmap v1022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/539765908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:35 smithi093 ceph-mon[111335]: pgmap v1022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/539765908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:35 smithi093 ceph-mon[118902]: pgmap v1022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/539765908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:37 smithi136 ceph-mon[113531]: pgmap v1023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:37 smithi093 ceph-mon[111335]: pgmap v1023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:37 smithi093 ceph-mon[118902]: pgmap v1023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/916014556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/916014556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/916014556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:39 smithi136 ceph-mon[113531]: pgmap v1024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:40.222 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:39 smithi093 ceph-mon[111335]: pgmap v1024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:40.223 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:39 smithi093 ceph-mon[118902]: pgmap v1024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:40.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:40.222Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:40.223Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2202558543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2202558543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2202558543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:41 smithi136 ceph-mon[113531]: pgmap v1025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:41 smithi093 ceph-mon[111335]: pgmap v1025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:41 smithi093 ceph-mon[118902]: pgmap v1025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:41.763Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:41.764Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4149574826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4149574826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4149574826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:43 smithi093 ceph-mon[111335]: pgmap v1026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:43 smithi093 ceph-mon[118902]: pgmap v1026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:43.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:43.652Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:43.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:43.653Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:43 smithi136 ceph-mon[113531]: pgmap v1026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:27:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:27:45] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:27:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:45 smithi136 ceph-mon[113531]: pgmap v1027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/921918193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:45 smithi093 ceph-mon[111335]: pgmap v1027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/921918193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:45 smithi093 ceph-mon[118902]: pgmap v1027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/921918193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:47 smithi136 ceph-mon[113531]: pgmap v1028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/528714390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:47 smithi093 ceph-mon[111335]: pgmap v1028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/528714390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:47 smithi093 ceph-mon[118902]: pgmap v1028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/528714390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:49 smithi136 ceph-mon[113531]: pgmap v1029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:27:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2079945630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:50.224 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:49 smithi093 ceph-mon[111335]: pgmap v1029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:50.224 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:27:50.224 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2079945630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:50.224 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:49 smithi093 ceph-mon[118902]: pgmap v1029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:50.224 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:27:50.224 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2079945630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:50.223Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:50.224Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:51 smithi136 ceph-mon[113531]: pgmap v1030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[111335]: pgmap v1030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[118902]: pgmap v1030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:52.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:51.764Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:52.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:51.765Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:27:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:27:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1988526244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:27:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:27:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1988526244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:27:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:27:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:27:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1988526244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:53.984 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:53.653Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:53.985 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:27:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:27:53.653Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:27:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:53 smithi093 ceph-mon[111335]: pgmap v1031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:53 smithi093 ceph-mon[118902]: pgmap v1031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:53 smithi136 ceph-mon[113531]: pgmap v1031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:27:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2672564209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2672564209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2672564209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:27:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:27:55] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:27:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:55 smithi136 ceph-mon[113531]: pgmap v1032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:55 smithi093 ceph-mon[111335]: pgmap v1032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:55 smithi093 ceph-mon[118902]: pgmap v1032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2301055733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2301055733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2301055733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:27:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:27:57 smithi136 ceph-mon[113531]: pgmap v1033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:27:57 smithi093 ceph-mon[111335]: pgmap v1033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:27:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:27:57 smithi093 ceph-mon[118902]: pgmap v1033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:00 smithi136 ceph-mon[113531]: pgmap v1034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/15375362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:00 smithi093 ceph-mon[111335]: pgmap v1034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/15375362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:00 smithi093 ceph-mon[118902]: pgmap v1034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/15375362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:00.224Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:00.225Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:02 smithi093 ceph-mon[118902]: pgmap v1035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/396902133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:02 smithi093 ceph-mon[111335]: pgmap v1035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/396902133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:01.765Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:02.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:01.766Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:02 smithi136 ceph-mon[113531]: pgmap v1035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/396902133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:03.654Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:03.655Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:04 smithi136 ceph-mon[113531]: pgmap v1036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:04 smithi093 ceph-mon[111335]: pgmap v1036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:04 smithi093 ceph-mon[118902]: pgmap v1036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1847722873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1847722873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1847722873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:28:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:28:05] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:28:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:06 smithi136 ceph-mon[113531]: pgmap v1037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:06 smithi093 ceph-mon[111335]: pgmap v1037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:06 smithi093 ceph-mon[118902]: pgmap v1037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1265801095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1265801095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1265801095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:08 smithi136 ceph-mon[113531]: pgmap v1038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:08 smithi093 ceph-mon[111335]: pgmap v1038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:08 smithi093 ceph-mon[118902]: pgmap v1038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:09.034 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:09.034 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2267530968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2267530968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2267530968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:10 smithi136 ceph-mon[113531]: pgmap v1039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:10 smithi093 ceph-mon[111335]: pgmap v1039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:10 smithi093 ceph-mon[118902]: pgmap v1039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:10.225Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:10.226Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:12.037 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:11.765Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:12.037 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:11.766Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:12 smithi136 ceph-mon[113531]: pgmap v1040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2370264695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:12 smithi093 ceph-mon[111335]: pgmap v1040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2370264695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:12 smithi093 ceph-mon[118902]: pgmap v1040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2370264695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:13.654Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:13.655Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:14 smithi136 ceph-mon[113531]: pgmap v1041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1502708491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:14 smithi093 ceph-mon[111335]: pgmap v1041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1502708491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:14 smithi093 ceph-mon[118902]: pgmap v1041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1502708491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:28:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:28:15] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:28:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:16 smithi136 ceph-mon[113531]: pgmap v1042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:16 smithi093 ceph-mon[111335]: pgmap v1042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:16 smithi093 ceph-mon[118902]: pgmap v1042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2182407381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2182407381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2182407381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:18 smithi136 ceph-mon[113531]: pgmap v1043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:18 smithi093 ceph-mon[111335]: pgmap v1043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:18 smithi093 ceph-mon[118902]: pgmap v1043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/133918551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/133918551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/133918551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:20 smithi136 ceph-mon[113531]: pgmap v1044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:20 smithi093 ceph-mon[111335]: pgmap v1044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:20 smithi093 ceph-mon[118902]: pgmap v1044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:20.226Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:20.227Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2410409727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2410409727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2410409727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:22.064 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:21.766Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:22.064 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:21.767Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:22 smithi136 ceph-mon[113531]: pgmap v1045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:22 smithi093 ceph-mon[111335]: pgmap v1045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:22 smithi093 ceph-mon[118902]: pgmap v1045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:23.655Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:23.656Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:24 smithi136 ceph-mon[113531]: pgmap v1046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3271016799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:24 smithi093 ceph-mon[111335]: pgmap v1046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3271016799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:24 smithi093 ceph-mon[118902]: pgmap v1046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3271016799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:28:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:28:25] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:28:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:26 smithi136 ceph-mon[113531]: pgmap v1047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1118753822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:26 smithi093 ceph-mon[111335]: pgmap v1047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1118753822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:26 smithi093 ceph-mon[118902]: pgmap v1047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1118753822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:28 smithi136 ceph-mon[113531]: pgmap v1048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1964955185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:28 smithi093 ceph-mon[111335]: pgmap v1048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1964955185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:28 smithi093 ceph-mon[118902]: pgmap v1048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1964955185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:30 smithi136 ceph-mon[113531]: pgmap v1049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:30 smithi093 ceph-mon[118902]: pgmap v1049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:30 smithi093 ceph-mon[111335]: pgmap v1049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:30.226Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:30.227Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/806360998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/806360998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/806360998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:32.086 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:31.767Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:32.086 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:31.768Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:32 smithi136 ceph-mon[113531]: pgmap v1050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:32 smithi093 ceph-mon[111335]: pgmap v1050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:32 smithi093 ceph-mon[118902]: pgmap v1050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2524111659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2524111659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2524111659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:33.656Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:33.657Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:34 smithi136 ceph-mon[113531]: pgmap v1051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:34 smithi093 ceph-mon[111335]: pgmap v1051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:34 smithi093 ceph-mon[118902]: pgmap v1051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:28:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:28:35] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:28:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:36 smithi136 ceph-mon[113531]: pgmap v1052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2487432948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:36 smithi093 ceph-mon[118902]: pgmap v1052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2487432948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:36 smithi093 ceph-mon[111335]: pgmap v1052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2487432948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:38 smithi136 ceph-mon[113531]: pgmap v1053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2758538432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:38 smithi093 ceph-mon[111335]: pgmap v1053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2758538432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:38 smithi093 ceph-mon[118902]: pgmap v1053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2758538432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:40 smithi136 ceph-mon[113531]: pgmap v1054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/926273028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:40 smithi093 ceph-mon[111335]: pgmap v1054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/926273028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:40 smithi093 ceph-mon[118902]: pgmap v1054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/926273028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:40.227Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:40.228Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:42.110 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:41.768Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:42.110 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:41.769Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:42 smithi136 ceph-mon[113531]: pgmap v1055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:42 smithi093 ceph-mon[111335]: pgmap v1055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:42 smithi093 ceph-mon[118902]: pgmap v1055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3945927155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3945927155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3945927155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:43.657Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:43.658Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:44 smithi136 ceph-mon[113531]: pgmap v1056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:44 smithi093 ceph-mon[111335]: pgmap v1056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:44 smithi093 ceph-mon[118902]: pgmap v1056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2291275552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2291275552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2291275552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:28:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:28:45] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:28:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:46 smithi136 ceph-mon[113531]: pgmap v1057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:46 smithi093 ceph-mon[111335]: pgmap v1057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:46 smithi093 ceph-mon[118902]: pgmap v1057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3524304483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3524304483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3524304483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:48 smithi136 ceph-mon[113531]: pgmap v1058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:48 smithi093 ceph-mon[111335]: pgmap v1058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:48 smithi093 ceph-mon[118902]: pgmap v1058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:50 smithi136 ceph-mon[113531]: pgmap v1059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4206044271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:50 smithi093 ceph-mon[118902]: pgmap v1059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4206044271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:50 smithi093 ceph-mon[111335]: pgmap v1059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4206044271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:50.228Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:50.229Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:52.140 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:51.768Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:52.140 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:51.769Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:52 smithi136 ceph-mon[113531]: pgmap v1060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1600995311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:28:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:52 smithi093 ceph-mon[111335]: pgmap v1060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1600995311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:28:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:52 smithi093 ceph-mon[118902]: pgmap v1060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1600995311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:28:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:53.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:53.657Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:53.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:28:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:28:53.659Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:28:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[111335]: pgmap v1061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1493245816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[118902]: pgmap v1061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1493245816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:54 smithi136 ceph-mon[113531]: pgmap v1061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:28:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1493245816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:28:55] "GET /metrics HTTP/1.1" 200 239876 "" "Prometheus/2.43.0" 2023-08-27T21:28:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:55 smithi136 ceph-mon[113531]: pgmap v1062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:28:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:28:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[118902]: pgmap v1062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[111335]: pgmap v1062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:28:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:28:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/438837810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/438837810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/438837810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:28:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:57 smithi136 ceph-mon[113531]: pgmap v1063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:57 smithi093 ceph-mon[111335]: pgmap v1063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:28:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:57 smithi093 ceph-mon[118902]: pgmap v1063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:59 smithi136 ceph-mon[113531]: pgmap v1064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:28:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3742903409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:00.229 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:59 smithi093 ceph-mon[111335]: pgmap v1064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:00.230 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:28:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3742903409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:00.230 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:59 smithi093 ceph-mon[118902]: pgmap v1064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:00.230 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:28:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3742903409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:00.229Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:00.230Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:01 smithi136 ceph-mon[113531]: pgmap v1065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3152248114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:01 smithi093 ceph-mon[111335]: pgmap v1065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3152248114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:01 smithi093 ceph-mon[118902]: pgmap v1065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3152248114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:02.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:01.768Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:02.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:01.769Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:03 smithi093 ceph-mon[118902]: pgmap v1066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4141518038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:03.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:03 smithi093 ceph-mon[111335]: pgmap v1066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:03.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4141518038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:03.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:03.658Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:03.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:03.659Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:03 smithi136 ceph-mon[113531]: pgmap v1066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4141518038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:05.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:29:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:29:05] "GET /metrics HTTP/1.1" 200 239876 "" "Prometheus/2.43.0" 2023-08-27T21:29:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:05 smithi136 ceph-mon[113531]: pgmap v1067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:05 smithi093 ceph-mon[111335]: pgmap v1067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:05 smithi093 ceph-mon[118902]: pgmap v1067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3548897155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3548897155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3548897155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:07 smithi136 ceph-mon[113531]: pgmap v1068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:07 smithi093 ceph-mon[111335]: pgmap v1068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:07 smithi093 ceph-mon[118902]: pgmap v1068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1914239061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1914239061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1914239061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:09 smithi136 ceph-mon[113531]: pgmap v1069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:10.230 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:09 smithi093 ceph-mon[111335]: pgmap v1069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:10.230 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:09 smithi093 ceph-mon[118902]: pgmap v1069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:10.494 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:10.230Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:10.495 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:10.231Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2952592093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2952592093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2952592093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:11 smithi136 ceph-mon[113531]: pgmap v1070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:11 smithi093 ceph-mon[111335]: pgmap v1070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:11 smithi093 ceph-mon[118902]: pgmap v1070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:11.769Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:11.770Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:13.924 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:13.659Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:13.925 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:13.660Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:13 smithi136 ceph-mon[113531]: pgmap v1071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2363471311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:13 smithi093 ceph-mon[111335]: pgmap v1071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2363471311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:13 smithi093 ceph-mon[118902]: pgmap v1071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2363471311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:29:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:29:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:29:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:15 smithi136 ceph-mon[113531]: pgmap v1072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2035152102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:15 smithi093 ceph-mon[111335]: pgmap v1072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2035152102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:15 smithi093 ceph-mon[118902]: pgmap v1072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2035152102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:17 smithi136 ceph-mon[113531]: pgmap v1073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:17 smithi093 ceph-mon[111335]: pgmap v1073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:17 smithi093 ceph-mon[118902]: pgmap v1073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3394186018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3394186018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3394186018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:19 smithi136 ceph-mon[113531]: pgmap v1074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:20.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:19 smithi093 ceph-mon[111335]: pgmap v1074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:20.231 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:19 smithi093 ceph-mon[118902]: pgmap v1074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:20.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:20.231Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:20.231Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3012464450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3012464450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3012464450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:22.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:21 smithi093 ceph-mon[111335]: pgmap v1075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:22.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:21 smithi093 ceph-mon[118902]: pgmap v1075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:22.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:21.770Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:22.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:21.771Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:21 smithi136 ceph-mon[113531]: pgmap v1075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3685213041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3685213041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3685213041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:23.948 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:23.660Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:23.948 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:23.661Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:23 smithi093 ceph-mon[118902]: pgmap v1076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:23 smithi093 ceph-mon[111335]: pgmap v1076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:23 smithi136 ceph-mon[113531]: pgmap v1076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:29:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:29:25] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:29:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:25 smithi093 ceph-mon[111335]: pgmap v1077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2686153729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:25 smithi093 ceph-mon[118902]: pgmap v1077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2686153729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:25 smithi136 ceph-mon[113531]: pgmap v1077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2686153729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:27 smithi093 ceph-mon[111335]: pgmap v1078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2778624456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:27 smithi093 ceph-mon[118902]: pgmap v1078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2778624456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:27 smithi136 ceph-mon[113531]: pgmap v1078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2778624456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:30.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:29 smithi093 ceph-mon[111335]: pgmap v1079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:30.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/642836437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:30.231 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:29 smithi093 ceph-mon[118902]: pgmap v1079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:30.232 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/642836437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:29 smithi136 ceph-mon[113531]: pgmap v1079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/642836437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:30.231Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:30.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:30.232Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:31 smithi093 ceph-mon[111335]: pgmap v1080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:31 smithi093 ceph-mon[118902]: pgmap v1080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:31.770Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:31.771Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:31 smithi136 ceph-mon[113531]: pgmap v1080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2373850965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2373850965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2373850965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:33.971 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:33.660Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:33.971 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:33.661Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:33 smithi093 ceph-mon[111335]: pgmap v1081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:33 smithi093 ceph-mon[118902]: pgmap v1081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:33 smithi136 ceph-mon[113531]: pgmap v1081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3331523342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3331523342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3331523342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:29:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:29:35] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:29:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:35 smithi093 ceph-mon[111335]: pgmap v1082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:35 smithi093 ceph-mon[118902]: pgmap v1082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:35 smithi136 ceph-mon[113531]: pgmap v1082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:37 smithi093 ceph-mon[111335]: pgmap v1083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2605911882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:37 smithi093 ceph-mon[118902]: pgmap v1083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2605911882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:37 smithi136 ceph-mon[113531]: pgmap v1083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2605911882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:39 smithi136 ceph-mon[113531]: pgmap v1084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3118335200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:39 smithi093 ceph-mon[111335]: pgmap v1084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3118335200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:39 smithi093 ceph-mon[118902]: pgmap v1084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3118335200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:40.232Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:40.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:40.233Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:41 smithi093 ceph-mon[111335]: pgmap v1085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/125504717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:41 smithi093 ceph-mon[118902]: pgmap v1085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/125504717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:41.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:41.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:41 smithi136 ceph-mon[113531]: pgmap v1085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/125504717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:43.660Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:43.662Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:43 smithi136 ceph-mon[113531]: pgmap v1086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:43 smithi093 ceph-mon[111335]: pgmap v1086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:43 smithi093 ceph-mon[118902]: pgmap v1086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4270552689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4270552689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4270552689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:29:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:29:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:29:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:46 smithi136 ceph-mon[113531]: pgmap v1087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:46 smithi093 ceph-mon[111335]: pgmap v1087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:46 smithi093 ceph-mon[118902]: pgmap v1087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2685191247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2685191247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2685191247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:48 smithi136 ceph-mon[113531]: pgmap v1088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:48 smithi093 ceph-mon[111335]: pgmap v1088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:48 smithi093 ceph-mon[118902]: pgmap v1088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:49 smithi136 ceph-mon[113531]: pgmap v1089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/105879279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:50.232 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:49 smithi093 ceph-mon[111335]: pgmap v1089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:50.233 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/105879279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:50.233 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:49 smithi093 ceph-mon[118902]: pgmap v1089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:50.233 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/105879279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:50.232Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:50.233Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:51 smithi136 ceph-mon[113531]: pgmap v1090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1385431564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:51 smithi093 ceph-mon[111335]: pgmap v1090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1385431564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:51 smithi093 ceph-mon[118902]: pgmap v1090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1385431564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:51.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:51.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-mon[111335]: pgmap v1091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1841252583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-mon[118902]: pgmap v1091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1841252583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:53.661Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:53.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:29:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:29:53.662Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:29:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:53 smithi136 ceph-mon[113531]: pgmap v1091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:29:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1841252583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:29:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:29:56.180 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[111335]: pgmap v1092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:56.180 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:29:56.180 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:56.180 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:56.181 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[118902]: pgmap v1092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:56.181 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:29:56.181 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:56.181 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:55 smithi136 ceph-mon[113531]: pgmap v1092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:29:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4217463944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:57.155 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4217463944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:57.156 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4217463944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:58 smithi136 ceph-mon[113531]: pgmap v1093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[111335]: pgmap v1093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[118902]: pgmap v1093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4022684508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:29:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:29:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4022684508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:29:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:29:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:29:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4022684508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:29:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:29:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:29:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:30:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:00 smithi093 ceph-mon[111335]: pgmap v1094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:30:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:00 smithi093 ceph-mon[118902]: pgmap v1094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:30:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:00.233Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:00.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:00.235Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:00 smithi136 ceph-mon[113531]: pgmap v1094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:30:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1781205816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1781205816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1781205816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:01.772Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:01.773Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:02 smithi136 ceph-mon[113531]: pgmap v1095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:02 smithi093 ceph-mon[111335]: pgmap v1095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:02 smithi093 ceph-mon[118902]: pgmap v1095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:03.663 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3080283268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:03.663 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3080283268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3080283268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:03.662Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:03.663Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:04 smithi136 ceph-mon[113531]: pgmap v1096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:04 smithi093 ceph-mon[111335]: pgmap v1096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:04 smithi093 ceph-mon[118902]: pgmap v1096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:30:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:30:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:30:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:06 smithi136 ceph-mon[113531]: pgmap v1097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3127976686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:06 smithi093 ceph-mon[111335]: pgmap v1097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3127976686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:06 smithi093 ceph-mon[118902]: pgmap v1097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3127976686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:08 smithi136 ceph-mon[113531]: pgmap v1098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/791256214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:08 smithi093 ceph-mon[111335]: pgmap v1098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/791256214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:08 smithi093 ceph-mon[118902]: pgmap v1098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/791256214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:10 smithi093 ceph-mon[111335]: pgmap v1099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:10 smithi093 ceph-mon[118902]: pgmap v1099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:10.234Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:10.235Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:10 smithi136 ceph-mon[113531]: pgmap v1099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2196007940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2196007940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2196007940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:11.772Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:11.773Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:12 smithi136 ceph-mon[113531]: pgmap v1100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:12 smithi093 ceph-mon[111335]: pgmap v1100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:12 smithi093 ceph-mon[118902]: pgmap v1100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2992039869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2992039869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2992039869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:13.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:13.663Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:13.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:13.664Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:14 smithi136 ceph-mon[113531]: pgmap v1101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:14 smithi093 ceph-mon[111335]: pgmap v1101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:14 smithi093 ceph-mon[118902]: pgmap v1101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3505096042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3505096042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3505096042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:30:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:30:15] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T21:30:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:16 smithi136 ceph-mon[113531]: pgmap v1102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:16 smithi093 ceph-mon[111335]: pgmap v1102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:16 smithi093 ceph-mon[118902]: pgmap v1102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:18 smithi093 ceph-mon[111335]: pgmap v1103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3147254157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:18 smithi093 ceph-mon[118902]: pgmap v1103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3147254157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:18 smithi136 ceph-mon[113531]: pgmap v1103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3147254157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:20 smithi093 ceph-mon[111335]: pgmap v1104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3272117685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:20 smithi093 ceph-mon[118902]: pgmap v1104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3272117685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:20.235Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:20.236Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:20 smithi136 ceph-mon[113531]: pgmap v1104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3272117685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:21.773Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:21.774Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:22 smithi093 ceph-mon[111335]: pgmap v1105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/112759251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:22 smithi093 ceph-mon[118902]: pgmap v1105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/112759251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:22 smithi136 ceph-mon[113531]: pgmap v1105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/112759251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:23.664Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:23.665Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:24 smithi093 ceph-mon[111335]: pgmap v1106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:24 smithi093 ceph-mon[118902]: pgmap v1106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:24 smithi136 ceph-mon[113531]: pgmap v1106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1782570956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:30:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:30:25] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:30:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1782570956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1782570956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:26 smithi093 ceph-mon[111335]: pgmap v1107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:26 smithi093 ceph-mon[118902]: pgmap v1107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:26 smithi136 ceph-mon[113531]: pgmap v1107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3742514415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3742514415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3742514415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:28 smithi093 ceph-mon[111335]: pgmap v1108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:28 smithi093 ceph-mon[118902]: pgmap v1108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:28 smithi136 ceph-mon[113531]: pgmap v1108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:30 smithi093 ceph-mon[111335]: pgmap v1109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/330521083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:30 smithi093 ceph-mon[118902]: pgmap v1109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/330521083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:30.236Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:30.237Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:30 smithi136 ceph-mon[113531]: pgmap v1109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/330521083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:31.773Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:31.774Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:32 smithi093 ceph-mon[118902]: pgmap v1110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/432212065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:32 smithi093 ceph-mon[111335]: pgmap v1110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/432212065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:32 smithi136 ceph-mon[113531]: pgmap v1110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/432212065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:33.665Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:33.665Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:34 smithi093 ceph-mon[111335]: pgmap v1111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4091057355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:34 smithi093 ceph-mon[118902]: pgmap v1111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4091057355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:34 smithi136 ceph-mon[113531]: pgmap v1111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4091057355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:30:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:30:35] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:30:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:36 smithi093 ceph-mon[111335]: pgmap v1112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:36 smithi093 ceph-mon[118902]: pgmap v1112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:36 smithi136 ceph-mon[113531]: pgmap v1112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3301781214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3301781214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3301781214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:38 smithi093 ceph-mon[111335]: pgmap v1113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:38 smithi093 ceph-mon[118902]: pgmap v1113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:38 smithi136 ceph-mon[113531]: pgmap v1113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2765995837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2765995837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2765995837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:40.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:40 smithi093 ceph-mon[111335]: pgmap v1114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:40.487 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:40.236Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:40.487 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:40.237Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:40 smithi093 ceph-mon[118902]: pgmap v1114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:40 smithi136 ceph-mon[113531]: pgmap v1114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1180688678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1180688678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1180688678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:41.774Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:41.775Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:42 smithi136 ceph-mon[113531]: pgmap v1115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:42 smithi093 ceph-mon[111335]: pgmap v1115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:42 smithi093 ceph-mon[118902]: pgmap v1115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:43.665Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:43.666Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:44 smithi136 ceph-mon[113531]: pgmap v1116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2764972253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:44 smithi093 ceph-mon[111335]: pgmap v1116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2764972253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:44 smithi093 ceph-mon[118902]: pgmap v1116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2764972253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:30:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:30:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:30:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:46 smithi136 ceph-mon[113531]: pgmap v1117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1338739932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:46 smithi093 ceph-mon[111335]: pgmap v1117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1338739932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:46 smithi093 ceph-mon[118902]: pgmap v1117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1338739932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:48 smithi136 ceph-mon[113531]: pgmap v1118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:48 smithi093 ceph-mon[111335]: pgmap v1118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:48 smithi093 ceph-mon[118902]: pgmap v1118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1083507589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1083507589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1083507589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:50.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:50.237Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:50.238Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:50 smithi136 ceph-mon[113531]: pgmap v1119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:50 smithi093 ceph-mon[111335]: pgmap v1119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:50 smithi093 ceph-mon[118902]: pgmap v1119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:51.775 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2986744021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:51.775 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2986744021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2986744021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:51.775Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:51.775Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:52 smithi136 ceph-mon[113531]: pgmap v1120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:52 smithi093 ceph-mon[111335]: pgmap v1120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:52 smithi093 ceph-mon[118902]: pgmap v1120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:53 smithi136 ceph-mon[113531]: pgmap v1121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4173140790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:53 smithi093 ceph-mon[111335]: pgmap v1121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4173140790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:53 smithi093 ceph-mon[118902]: pgmap v1121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4173140790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:53.666Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:30:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:30:53.667Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:30:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:30:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:55 smithi093 ceph-mon[111335]: pgmap v1122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:55 smithi093 ceph-mon[118902]: pgmap v1122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:30:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:30:55] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:30:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:55 smithi136 ceph-mon[113531]: pgmap v1122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2941341585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2941341585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2941341585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:57 smithi136 ceph-mon[113531]: pgmap v1123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:57 smithi093 ceph-mon[111335]: pgmap v1123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:57 smithi093 ceph-mon[118902]: pgmap v1123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/992636218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/992636218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:58.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/992636218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:30:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:30:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:30:59 smithi136 ceph-mon[113531]: pgmap v1124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:30:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:30:59 smithi093 ceph-mon[111335]: pgmap v1124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:30:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:30:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:30:59 smithi093 ceph-mon[118902]: pgmap v1124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:00.238Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:00.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:00.239Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1308215971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1308215971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1308215971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:01.775Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:01.776Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[111335]: pgmap v1125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:31:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:31:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[118902]: pgmap v1125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:31:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:31:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:02 smithi136 ceph-mon[113531]: pgmap v1125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:31:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:31:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:31:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3155350399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3155350399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:03.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:03.666Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:03.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:03.667Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3155350399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:04 smithi093 ceph-mon[111335]: pgmap v1126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:04 smithi093 ceph-mon[118902]: pgmap v1126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:04 smithi136 ceph-mon[113531]: pgmap v1126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2959391720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T21:31:05.318354978Z level=info msg="Completed cleanup jobs" duration=13.041561ms 2023-08-27T21:31:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:31:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:31:05] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:31:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2959391720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2959391720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:06 smithi093 ceph-mon[111335]: pgmap v1127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:06 smithi093 ceph-mon[118902]: pgmap v1127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:06 smithi136 ceph-mon[113531]: pgmap v1127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:08 smithi093 ceph-mon[111335]: pgmap v1128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/723164826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:08 smithi093 ceph-mon[118902]: pgmap v1128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/723164826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:08 smithi136 ceph-mon[113531]: pgmap v1128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/723164826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:10 smithi093 ceph-mon[111335]: pgmap v1129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2169474644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:10 smithi093 ceph-mon[118902]: pgmap v1129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2169474644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:10.239Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:10.239Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:10 smithi136 ceph-mon[113531]: pgmap v1129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2169474644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:11.776Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:11.777Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:12 smithi136 ceph-mon[113531]: pgmap v1130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/430393952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:12 smithi093 ceph-mon[111335]: pgmap v1130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/430393952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:12 smithi093 ceph-mon[118902]: pgmap v1130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/430393952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:13.667Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:13.668Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:14 smithi136 ceph-mon[113531]: pgmap v1131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:14 smithi093 ceph-mon[111335]: pgmap v1131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:14 smithi093 ceph-mon[118902]: pgmap v1131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3513714730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3513714730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:31:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:31:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:31:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3513714730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:16 smithi136 ceph-mon[113531]: pgmap v1132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:16 smithi093 ceph-mon[111335]: pgmap v1132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:16 smithi093 ceph-mon[118902]: pgmap v1132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2501196196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2501196196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2501196196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:18 smithi136 ceph-mon[113531]: pgmap v1133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:18 smithi093 ceph-mon[111335]: pgmap v1133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:19.637 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:18 smithi093 ceph-mon[118902]: pgmap v1133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:20.507 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:20.240Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:20.507 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:20.241Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:20 smithi136 ceph-mon[113531]: pgmap v1134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/430125150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:20 smithi093 ceph-mon[111335]: pgmap v1134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/430125150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:20 smithi093 ceph-mon[118902]: pgmap v1134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/430125150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:21.777Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:21.777Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:22 smithi136 ceph-mon[113531]: pgmap v1135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2174996649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:22 smithi093 ceph-mon[111335]: pgmap v1135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2174996649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:22 smithi093 ceph-mon[118902]: pgmap v1135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2174996649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:23.668Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:23.670Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:24 smithi136 ceph-mon[113531]: pgmap v1136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/151048411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:24 smithi093 ceph-mon[111335]: pgmap v1136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/151048411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:24 smithi093 ceph-mon[118902]: pgmap v1136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/151048411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:25 smithi093 ceph-mon[111335]: pgmap v1137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:25 smithi093 ceph-mon[118902]: pgmap v1137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:31:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:31:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:31:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:25 smithi136 ceph-mon[113531]: pgmap v1137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/427786034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/427786034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/427786034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:27 smithi136 ceph-mon[113531]: pgmap v1138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:27 smithi093 ceph-mon[111335]: pgmap v1138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:27 smithi093 ceph-mon[118902]: pgmap v1138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:29 smithi136 ceph-mon[113531]: pgmap v1139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3939692222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:29 smithi093 ceph-mon[111335]: pgmap v1139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3939692222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:29 smithi093 ceph-mon[118902]: pgmap v1139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3939692222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:30.241Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:30.241Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:31 smithi136 ceph-mon[113531]: pgmap v1140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2145513025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:31 smithi093 ceph-mon[111335]: pgmap v1140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2145513025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:31 smithi093 ceph-mon[118902]: pgmap v1140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2145513025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:31.777Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:32.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:31.778Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:33.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:33.669Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:33.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:33.670Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:33 smithi136 ceph-mon[113531]: pgmap v1141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3738485186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:33 smithi093 ceph-mon[111335]: pgmap v1141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3738485186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:33 smithi093 ceph-mon[118902]: pgmap v1141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3738485186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:31:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:31:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:31:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:35 smithi093 ceph-mon[111335]: pgmap v1142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:36.854 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:35 smithi093 ceph-mon[118902]: pgmap v1142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:36.854 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:35 smithi136 ceph-mon[113531]: pgmap v1142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/667409437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/667409437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/667409437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:37 smithi093 ceph-mon[118902]: pgmap v1143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:37 smithi093 ceph-mon[111335]: pgmap v1143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:37 smithi136 ceph-mon[113531]: pgmap v1143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1687968211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1687968211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1687968211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:39 smithi093 ceph-mon[111335]: pgmap v1144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:39 smithi093 ceph-mon[118902]: pgmap v1144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:39 smithi136 ceph-mon[113531]: pgmap v1144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:40.242Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:40.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:41 smithi093 ceph-mon[111335]: pgmap v1145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3331063604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:41 smithi093 ceph-mon[118902]: pgmap v1145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3331063604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:41.777Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:41.779Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:41 smithi136 ceph-mon[113531]: pgmap v1145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3331063604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:43.958 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:43.670Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:43.958 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:43.672Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:43 smithi093 ceph-mon[118902]: pgmap v1146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1051688278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:43 smithi093 ceph-mon[111335]: pgmap v1146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1051688278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:43 smithi136 ceph-mon[113531]: pgmap v1146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1051688278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:31:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:31:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:31:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:45 smithi093 ceph-mon[118902]: pgmap v1147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3023968260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:45 smithi093 ceph-mon[111335]: pgmap v1147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3023968260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:45 smithi136 ceph-mon[113531]: pgmap v1147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3023968260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:47 smithi093 ceph-mon[111335]: pgmap v1148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:47 smithi093 ceph-mon[118902]: pgmap v1148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:47 smithi136 ceph-mon[113531]: pgmap v1148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4117347274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4117347274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4117347274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:49 smithi093 ceph-mon[111335]: pgmap v1149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:49 smithi093 ceph-mon[118902]: pgmap v1149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:49 smithi136 ceph-mon[113531]: pgmap v1149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:50.242Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:50.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/895291533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/895291533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/895291533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:51 smithi093 ceph-mon[111335]: pgmap v1150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:51 smithi093 ceph-mon[118902]: pgmap v1150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:51.778Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:51.779Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:51 smithi136 ceph-mon[113531]: pgmap v1150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3288852158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3288852158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3288852158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:53.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:53.671Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:53.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:31:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:31:53.672Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:31:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:53 smithi093 ceph-mon[118902]: pgmap v1151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:53 smithi093 ceph-mon[111335]: pgmap v1151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:53 smithi136 ceph-mon[113531]: pgmap v1151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:31:55.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:31:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:31:55] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:31:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:55 smithi093 ceph-mon[111335]: pgmap v1152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3733640209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:55 smithi093 ceph-mon[118902]: pgmap v1152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3733640209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:55 smithi136 ceph-mon[113531]: pgmap v1152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3733640209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:57 smithi093 ceph-mon[111335]: pgmap v1153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/184898169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:57 smithi093 ceph-mon[118902]: pgmap v1153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/184898169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:31:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:57 smithi136 ceph-mon[113531]: pgmap v1153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:31:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/184898169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:00.244 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:31:59 smithi093 ceph-mon[111335]: pgmap v1154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:00.244 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:31:59 smithi093 ceph-mon[118902]: pgmap v1154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:31:59 smithi136 ceph-mon[113531]: pgmap v1154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:00.243Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:00.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/447288372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/447288372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/447288372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:01 smithi093 ceph-mon[111335]: pgmap v1155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:32:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:01 smithi093 ceph-mon[118902]: pgmap v1155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:32:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:01.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:01.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:01 smithi136 ceph-mon[113531]: pgmap v1155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:32:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1776840443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:03.388 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1776840443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:03.388 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1776840443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:03.671Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:03.672Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:04.173 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:04 smithi136 ceph-mon[113531]: pgmap v1156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:04.173 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:04.173 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:04 smithi093 ceph-mon[111335]: pgmap v1156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:04 smithi093 ceph-mon[118902]: pgmap v1156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:32:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:32:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3807882057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:32:05] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3807882057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:32:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:32:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:32:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3807882057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:06 smithi136 ceph-mon[113531]: pgmap v1157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:06 smithi093 ceph-mon[118902]: pgmap v1157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:06 smithi093 ceph-mon[111335]: pgmap v1157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3173839646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3173839646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3173839646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:08 smithi136 ceph-mon[113531]: pgmap v1158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:08 smithi093 ceph-mon[111335]: pgmap v1158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:08 smithi093 ceph-mon[118902]: pgmap v1158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:10 smithi136 ceph-mon[113531]: pgmap v1159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3056277432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:10 smithi093 ceph-mon[111335]: pgmap v1159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3056277432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:10 smithi093 ceph-mon[118902]: pgmap v1159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3056277432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:10.244Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:10.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:11.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:11.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:12 smithi136 ceph-mon[113531]: pgmap v1160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1402007778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:12 smithi093 ceph-mon[111335]: pgmap v1160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1402007778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:12 smithi093 ceph-mon[118902]: pgmap v1160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1402007778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:13.672Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:13.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:14 smithi136 ceph-mon[113531]: pgmap v1161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:14 smithi093 ceph-mon[111335]: pgmap v1161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:14 smithi093 ceph-mon[118902]: pgmap v1161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4011128615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4011128615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:32:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:32:15] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:32:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4011128615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:16 smithi136 ceph-mon[113531]: pgmap v1162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:16 smithi093 ceph-mon[111335]: pgmap v1162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:16 smithi093 ceph-mon[118902]: pgmap v1162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/799966042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/799966042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/799966042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:18 smithi136 ceph-mon[113531]: pgmap v1163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:18 smithi093 ceph-mon[111335]: pgmap v1163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:18 smithi093 ceph-mon[118902]: pgmap v1163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:19 smithi136 ceph-mon[113531]: pgmap v1164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/382786914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:19 smithi093 ceph-mon[111335]: pgmap v1164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/382786914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:19 smithi093 ceph-mon[118902]: pgmap v1164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/382786914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:20.245Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:20.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:21 smithi136 ceph-mon[113531]: pgmap v1165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3664317324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:21 smithi093 ceph-mon[111335]: pgmap v1165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3664317324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:21 smithi093 ceph-mon[118902]: pgmap v1165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3664317324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:22.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:21.780Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:22.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:21.781Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:23 smithi093 ceph-mon[111335]: pgmap v1166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:23 smithi093 ceph-mon[118902]: pgmap v1166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:23.673Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:23.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:23 smithi136 ceph-mon[113531]: pgmap v1166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1773096645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1773096645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1773096645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:32:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:32:25] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:32:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:25 smithi136 ceph-mon[113531]: pgmap v1167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:25 smithi093 ceph-mon[111335]: pgmap v1167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:25 smithi093 ceph-mon[118902]: pgmap v1167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1357168801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1357168801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1357168801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:27 smithi136 ceph-mon[113531]: pgmap v1168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:27 smithi093 ceph-mon[111335]: pgmap v1168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:27 smithi093 ceph-mon[118902]: pgmap v1168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/528054540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/528054540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/528054540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:29 smithi093 ceph-mon[111335]: pgmap v1169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:29 smithi093 ceph-mon[118902]: pgmap v1169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:29 smithi136 ceph-mon[113531]: pgmap v1169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:30.245Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:30.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:31 smithi093 ceph-mon[111335]: pgmap v1170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2460039305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:31 smithi093 ceph-mon[118902]: pgmap v1170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2460039305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:31.780Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:31.781Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:31 smithi136 ceph-mon[113531]: pgmap v1170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2460039305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:33.944 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:33.673Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:33.944 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:33.674Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:33 smithi093 ceph-mon[111335]: pgmap v1171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/22148923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:33 smithi093 ceph-mon[118902]: pgmap v1171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:34.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/22148923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:33 smithi136 ceph-mon[113531]: pgmap v1171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/22148923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:32:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:32:35] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:32:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:35 smithi093 ceph-mon[111335]: pgmap v1172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3814487492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:35 smithi093 ceph-mon[118902]: pgmap v1172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3814487492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:35 smithi136 ceph-mon[113531]: pgmap v1172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3814487492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:37 smithi093 ceph-mon[111335]: pgmap v1173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:37 smithi093 ceph-mon[118902]: pgmap v1173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:37 smithi136 ceph-mon[113531]: pgmap v1173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2922232220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2922232220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2922232220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:39 smithi093 ceph-mon[111335]: pgmap v1174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:39 smithi093 ceph-mon[118902]: pgmap v1174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:39 smithi136 ceph-mon[113531]: pgmap v1174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:40.246Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:40.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2828231653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2828231653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2828231653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:41 smithi093 ceph-mon[111335]: pgmap v1175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:41 smithi093 ceph-mon[118902]: pgmap v1175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:41.781Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:41.782Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:41 smithi136 ceph-mon[113531]: pgmap v1175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:43.966 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:43.674Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:43.966 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:43.674Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:43 smithi093 ceph-mon[111335]: pgmap v1176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/50148109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:43 smithi093 ceph-mon[118902]: pgmap v1176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/50148109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:43 smithi136 ceph-mon[113531]: pgmap v1176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/50148109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:32:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:32:45] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:32:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:45 smithi093 ceph-mon[118902]: pgmap v1177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3875001480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:45 smithi093 ceph-mon[111335]: pgmap v1177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3875001480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:45 smithi136 ceph-mon[113531]: pgmap v1177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3875001480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:47 smithi093 ceph-mon[118902]: pgmap v1178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2591912260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:47 smithi093 ceph-mon[111335]: pgmap v1178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2591912260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:47 smithi136 ceph-mon[113531]: pgmap v1178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2591912260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:49 smithi093 ceph-mon[111335]: pgmap v1179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:49 smithi093 ceph-mon[118902]: pgmap v1179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:49 smithi136 ceph-mon[113531]: pgmap v1179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:50.247Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:50.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/206489213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/206489213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/206489213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:51 smithi093 ceph-mon[111335]: pgmap v1180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:51 smithi093 ceph-mon[118902]: pgmap v1180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:51.782Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:51.782Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:51 smithi136 ceph-mon[113531]: pgmap v1180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2315916060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2315916060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2315916060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:53.987 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:53.675Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:53.987 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:32:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:32:53.676Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:32:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:53 smithi093 ceph-mon[111335]: pgmap v1181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:53 smithi093 ceph-mon[118902]: pgmap v1181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:53 smithi136 ceph-mon[113531]: pgmap v1181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:32:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2338396793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2338396793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2338396793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:32:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:32:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:32:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:55 smithi136 ceph-mon[113531]: pgmap v1182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:55 smithi093 ceph-mon[111335]: pgmap v1182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:55 smithi093 ceph-mon[118902]: pgmap v1182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:57 smithi136 ceph-mon[113531]: pgmap v1183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:32:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/503116198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:57 smithi093 ceph-mon[111335]: pgmap v1183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:32:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/503116198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:32:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:57 smithi093 ceph-mon[118902]: pgmap v1183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:32:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:32:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/503116198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:00 smithi136 ceph-mon[113531]: pgmap v1184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111016811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:00 smithi093 ceph-mon[111335]: pgmap v1184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111016811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:00 smithi093 ceph-mon[118902]: pgmap v1184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111016811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:00.248Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:00.249Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:02 smithi093 ceph-mon[111335]: pgmap v1185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:02 smithi093 ceph-mon[118902]: pgmap v1185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:01.782Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:01.783Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:02 smithi136 ceph-mon[113531]: pgmap v1185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1683582636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1683582636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1683582636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:03.675Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:03.676Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:04 smithi136 ceph-mon[113531]: pgmap v1186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:04 smithi093 ceph-mon[111335]: pgmap v1186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:04 smithi093 ceph-mon[118902]: pgmap v1186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1676919253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:33:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1676919253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:33:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1676919253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:33:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:33:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:33:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:33:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:06 smithi136 ceph-mon[113531]: pgmap v1187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:06.442 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:06 smithi093 ceph-mon[111335]: pgmap v1187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:06.443 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:06 smithi093 ceph-mon[118902]: pgmap v1187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[118902]: pgmap v1188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2695281680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[111335]: pgmap v1188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2695281680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:07.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:07.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:07 smithi136 ceph-mon[113531]: pgmap v1188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2695281680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:33:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:33:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:33:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:33:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:09 smithi136 ceph-mon[113531]: pgmap v1189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/31876917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:09 smithi093 ceph-mon[111335]: pgmap v1189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/31876917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:09 smithi093 ceph-mon[118902]: pgmap v1189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/31876917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:10.249Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:10.249Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:11 smithi136 ceph-mon[113531]: pgmap v1190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/325551819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:11 smithi093 ceph-mon[111335]: pgmap v1190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/325551819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:11 smithi093 ceph-mon[118902]: pgmap v1190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/325551819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:11.783Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:11.784Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:13 smithi093 ceph-mon[111335]: pgmap v1191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:13 smithi093 ceph-mon[118902]: pgmap v1191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:13.676Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:13.677Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:13 smithi136 ceph-mon[113531]: pgmap v1191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/284821155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/284821155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/284821155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:33:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:33:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:33:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:15 smithi136 ceph-mon[113531]: pgmap v1192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:15 smithi093 ceph-mon[111335]: pgmap v1192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:15 smithi093 ceph-mon[118902]: pgmap v1192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3269900955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3269900955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3269900955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:17 smithi136 ceph-mon[113531]: pgmap v1193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:17 smithi093 ceph-mon[111335]: pgmap v1193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:17 smithi093 ceph-mon[118902]: pgmap v1193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/877831179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/877831179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/877831179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:19 smithi136 ceph-mon[113531]: pgmap v1194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:19 smithi093 ceph-mon[111335]: pgmap v1194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:19 smithi093 ceph-mon[118902]: pgmap v1194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:20.250Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:20.251Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:21 smithi136 ceph-mon[113531]: pgmap v1195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1300262088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:21 smithi093 ceph-mon[111335]: pgmap v1195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1300262088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:21 smithi093 ceph-mon[118902]: pgmap v1195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1300262088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:21.783Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:21.784Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:23.941 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:23.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:23.941 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:23.678Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-mon[118902]: pgmap v1196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1197340031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-mon[111335]: pgmap v1196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1197340031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:23 smithi136 ceph-mon[113531]: pgmap v1196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1197340031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:33:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:33:25] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:33:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:25 smithi093 ceph-mon[111335]: pgmap v1197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:25 smithi093 ceph-mon[118902]: pgmap v1197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:25 smithi136 ceph-mon[113531]: pgmap v1197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1729681248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1729681248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1729681248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:27 smithi093 ceph-mon[111335]: pgmap v1198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:27 smithi093 ceph-mon[118902]: pgmap v1198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:27 smithi136 ceph-mon[113531]: pgmap v1198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1913065213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1913065213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1913065213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:29 smithi093 ceph-mon[111335]: pgmap v1199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:29 smithi093 ceph-mon[118902]: pgmap v1199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:29 smithi136 ceph-mon[113531]: pgmap v1199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:30.250Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:30.252Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2570092667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2570092667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2570092667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:31 smithi093 ceph-mon[111335]: pgmap v1200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:31 smithi093 ceph-mon[118902]: pgmap v1200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:31.784Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:31.785Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:31 smithi136 ceph-mon[113531]: pgmap v1200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:33.970 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:33.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:33.970 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:33.678Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:33 smithi093 ceph-mon[111335]: pgmap v1201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/323197557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:33 smithi093 ceph-mon[118902]: pgmap v1201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/323197557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:33 smithi136 ceph-mon[113531]: pgmap v1201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/323197557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:33:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:33:35] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:33:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:35 smithi093 ceph-mon[111335]: pgmap v1202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3672313945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:35 smithi093 ceph-mon[118902]: pgmap v1202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3672313945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:35 smithi136 ceph-mon[113531]: pgmap v1202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3672313945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:37 smithi093 ceph-mon[118902]: pgmap v1203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3634551989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:37 smithi093 ceph-mon[111335]: pgmap v1203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3634551989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:37 smithi136 ceph-mon[113531]: pgmap v1203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3634551989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:39 smithi093 ceph-mon[111335]: pgmap v1204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:39 smithi093 ceph-mon[118902]: pgmap v1204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:39 smithi136 ceph-mon[113531]: pgmap v1204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:40.251Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:40.252Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2347232391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2347232391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2347232391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:41 smithi093 ceph-mon[111335]: pgmap v1205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:41 smithi093 ceph-mon[118902]: pgmap v1205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:41.784Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:41.785Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:41 smithi136 ceph-mon[113531]: pgmap v1205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/880926083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/880926083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/880926083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:43.678Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:43.680Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:43 smithi136 ceph-mon[113531]: pgmap v1206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:43 smithi093 ceph-mon[111335]: pgmap v1206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:43 smithi093 ceph-mon[118902]: pgmap v1206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:33:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:33:45] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:33:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:45 smithi136 ceph-mon[113531]: pgmap v1207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1557695674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:45 smithi093 ceph-mon[111335]: pgmap v1207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1557695674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:45 smithi093 ceph-mon[118902]: pgmap v1207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1557695674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:47 smithi136 ceph-mon[113531]: pgmap v1208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/307992838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:47 smithi093 ceph-mon[111335]: pgmap v1208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/307992838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:47 smithi093 ceph-mon[118902]: pgmap v1208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/307992838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:49 smithi136 ceph-mon[113531]: pgmap v1209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/83174586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:49 smithi093 ceph-mon[111335]: pgmap v1209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/83174586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:49 smithi093 ceph-mon[118902]: pgmap v1209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/83174586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:50.252Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:50.252Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:51 smithi136 ceph-mon[113531]: pgmap v1210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:51 smithi093 ceph-mon[111335]: pgmap v1210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:51 smithi093 ceph-mon[118902]: pgmap v1210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:51.785Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:51.786Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4049844904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4049844904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4049844904' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:53 smithi093 ceph-mon[111335]: pgmap v1211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:53 smithi093 ceph-mon[118902]: pgmap v1211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:53.679Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:33:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:33:53.681Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:33:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:53 smithi136 ceph-mon[113531]: pgmap v1211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:33:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/550494550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/550494550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/550494550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:33:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:33:55] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:33:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:55 smithi136 ceph-mon[113531]: pgmap v1212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:55 smithi093 ceph-mon[111335]: pgmap v1212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:55 smithi093 ceph-mon[118902]: pgmap v1212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:57 smithi136 ceph-mon[113531]: pgmap v1213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/863607135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:57 smithi093 ceph-mon[111335]: pgmap v1213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/863607135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:33:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:57 smithi093 ceph-mon[118902]: pgmap v1213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:33:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/863607135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:59 smithi136 ceph-mon[113531]: pgmap v1214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:33:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1627164306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:59 smithi093 ceph-mon[118902]: pgmap v1214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:33:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1627164306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:59 smithi093 ceph-mon[111335]: pgmap v1214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:33:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1627164306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:00.253Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:00.254Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:01 smithi136 ceph-mon[113531]: pgmap v1215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2354911757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:01 smithi093 ceph-mon[111335]: pgmap v1215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2354911757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:01 smithi093 ceph-mon[118902]: pgmap v1215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2354911757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:01.786Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:01.787Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:03 smithi093 ceph-mon[111335]: pgmap v1216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:03 smithi093 ceph-mon[118902]: pgmap v1216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:03.680Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:03.682Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:03 smithi136 ceph-mon[113531]: pgmap v1216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2615261398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2615261398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2615261398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:34:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:34:05] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:34:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:05 smithi136 ceph-mon[113531]: pgmap v1217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:05 smithi093 ceph-mon[111335]: pgmap v1217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:05 smithi093 ceph-mon[118902]: pgmap v1217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3843056509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3843056509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3843056509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:07 smithi136 ceph-mon[113531]: pgmap v1218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:07 smithi093 ceph-mon[111335]: pgmap v1218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:07 smithi093 ceph-mon[118902]: pgmap v1218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:34:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3788692597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:34:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3788692597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:34:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3788692597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:09 smithi136 ceph-mon[113531]: pgmap v1219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:09 smithi093 ceph-mon[111335]: pgmap v1219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:09 smithi093 ceph-mon[118902]: pgmap v1219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:10.254Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:10.255Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:11.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/277528937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:34:11.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:34:11.788 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.788 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/277528937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:34:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:34:11.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:34:11.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/277528937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:11.786Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:11.787Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:12 smithi136 ceph-mon[113531]: pgmap v1220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:12 smithi093 ceph-mon[111335]: pgmap v1220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:12 smithi093 ceph-mon[118902]: pgmap v1220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:13.681Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:13.683Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:14 smithi136 ceph-mon[113531]: pgmap v1221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1282722401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:14 smithi093 ceph-mon[111335]: pgmap v1221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1282722401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:14 smithi093 ceph-mon[118902]: pgmap v1221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1282722401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:34:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:34:15] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:34:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:16 smithi136 ceph-mon[113531]: pgmap v1222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1074145103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:16 smithi093 ceph-mon[111335]: pgmap v1222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1074145103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:16 smithi093 ceph-mon[118902]: pgmap v1222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1074145103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:18 smithi136 ceph-mon[113531]: pgmap v1223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2429710827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:18 smithi093 ceph-mon[111335]: pgmap v1223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2429710827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:18 smithi093 ceph-mon[118902]: pgmap v1223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2429710827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:19 smithi136 ceph-mon[113531]: pgmap v1224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:19 smithi093 ceph-mon[111335]: pgmap v1224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:19 smithi093 ceph-mon[118902]: pgmap v1224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:20.255Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:20.256Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3176564513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3176564513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3176564513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:21 smithi136 ceph-mon[113531]: pgmap v1225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:21 smithi093 ceph-mon[111335]: pgmap v1225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:21 smithi093 ceph-mon[118902]: pgmap v1225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:21.787Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:21.787Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:23.934 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:23.682Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:23.934 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:23.683Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:23 smithi136 ceph-mon[113531]: pgmap v1226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3040720804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-mon[118902]: pgmap v1226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3040720804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-mon[111335]: pgmap v1226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3040720804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:34:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:34:25] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:34:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:25 smithi136 ceph-mon[113531]: pgmap v1227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1494802714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:25 smithi093 ceph-mon[111335]: pgmap v1227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1494802714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:25 smithi093 ceph-mon[118902]: pgmap v1227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1494802714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:27 smithi093 ceph-mon[111335]: pgmap v1228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:27 smithi093 ceph-mon[118902]: pgmap v1228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:27 smithi136 ceph-mon[113531]: pgmap v1228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1923361300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1923361300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1923361300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:29 smithi093 ceph-mon[111335]: pgmap v1229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:29 smithi093 ceph-mon[118902]: pgmap v1229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:29 smithi136 ceph-mon[113531]: pgmap v1229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:30.256Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:30.257Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/718359015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/718359015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/718359015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:31 smithi093 ceph-mon[111335]: pgmap v1230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:31 smithi093 ceph-mon[118902]: pgmap v1230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:31.787Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:31.788Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:31 smithi136 ceph-mon[113531]: pgmap v1230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1230111882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1230111882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1230111882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:33.955 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:33.682Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:33.956 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:33.683Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:33 smithi093 ceph-mon[111335]: pgmap v1231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:33 smithi093 ceph-mon[118902]: pgmap v1231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:33 smithi136 ceph-mon[113531]: pgmap v1231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:34:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:34:35] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:34:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:35 smithi093 ceph-mon[111335]: pgmap v1232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2162450069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:35 smithi093 ceph-mon[118902]: pgmap v1232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2162450069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:35 smithi136 ceph-mon[113531]: pgmap v1232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2162450069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:37 smithi093 ceph-mon[111335]: pgmap v1233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1727237114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:37 smithi093 ceph-mon[118902]: pgmap v1233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1727237114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:37 smithi136 ceph-mon[113531]: pgmap v1233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1727237114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:39 smithi093 ceph-mon[111335]: pgmap v1234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3952693820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:39 smithi093 ceph-mon[118902]: pgmap v1234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3952693820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:39 smithi136 ceph-mon[113531]: pgmap v1234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3952693820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:40.256Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:40.257Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:41 smithi093 ceph-mon[111335]: pgmap v1235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:41 smithi093 ceph-mon[118902]: pgmap v1235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:41.787Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:41.788Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:41 smithi136 ceph-mon[113531]: pgmap v1235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/901785792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/901785792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/901785792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:43.971 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:43.684Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:43.971 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:43.685Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:44.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:43 smithi093 ceph-mon[111335]: pgmap v1236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:44.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:43 smithi093 ceph-mon[118902]: pgmap v1236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:43 smithi136 ceph-mon[113531]: pgmap v1236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3950269362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3950269362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3950269362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:34:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:34:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:34:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:45 smithi093 ceph-mon[111335]: pgmap v1237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:45 smithi093 ceph-mon[118902]: pgmap v1237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:45 smithi136 ceph-mon[113531]: pgmap v1237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:47 smithi093 ceph-mon[111335]: pgmap v1238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1732899045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:47 smithi093 ceph-mon[118902]: pgmap v1238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1732899045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:47 smithi136 ceph-mon[113531]: pgmap v1238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1732899045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:49 smithi136 ceph-mon[113531]: pgmap v1239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1271853519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:49 smithi093 ceph-mon[111335]: pgmap v1239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1271853519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:49 smithi093 ceph-mon[118902]: pgmap v1239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1271853519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:50.257Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:50.258Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:51 smithi093 ceph-mon[111335]: pgmap v1240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3018249697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:51 smithi093 ceph-mon[118902]: pgmap v1240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3018249697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:51.788Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:52.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:51.789Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:51 smithi136 ceph-mon[113531]: pgmap v1240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3018249697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:53.982 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:53.685Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:53.982 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:34:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:34:53.686Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:34:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:53 smithi093 ceph-mon[111335]: pgmap v1241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:53 smithi093 ceph-mon[118902]: pgmap v1241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:53 smithi136 ceph-mon[113531]: pgmap v1241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:34:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2765295706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2765295706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2765295706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:34:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:34:55] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:34:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:55 smithi093 ceph-mon[111335]: pgmap v1242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:55 smithi093 ceph-mon[118902]: pgmap v1242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:55 smithi136 ceph-mon[113531]: pgmap v1242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/427741122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/427741122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/427741122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:34:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:57 smithi136 ceph-mon[113531]: pgmap v1243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:57 smithi093 ceph-mon[111335]: pgmap v1243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:34:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:57 smithi093 ceph-mon[118902]: pgmap v1243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:59 smithi136 ceph-mon[113531]: pgmap v1244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:34:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2184444304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:59 smithi093 ceph-mon[111335]: pgmap v1244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:34:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2184444304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:59 smithi093 ceph-mon[118902]: pgmap v1244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:34:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2184444304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:00.258Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:00.259Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:01 smithi093 ceph-mon[118902]: pgmap v1245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2229638951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:01 smithi093 ceph-mon[111335]: pgmap v1245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2229638951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:01.789Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:01.789Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:01 smithi136 ceph-mon[113531]: pgmap v1245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2229638951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:03.985 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:03.685Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:03.986 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:03.686Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:03 smithi093 ceph-mon[118902]: pgmap v1246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4219790893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:03 smithi093 ceph-mon[111335]: pgmap v1246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4219790893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:03 smithi136 ceph-mon[113531]: pgmap v1246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4219790893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:35:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:35:05] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:35:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:05 smithi093 ceph-mon[111335]: pgmap v1247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:05 smithi093 ceph-mon[118902]: pgmap v1247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:05 smithi136 ceph-mon[113531]: pgmap v1247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4225184544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4225184544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4225184544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:07 smithi136 ceph-mon[113531]: pgmap v1248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:07 smithi093 ceph-mon[111335]: pgmap v1248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:07 smithi093 ceph-mon[118902]: pgmap v1248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/250932532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/250932532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/250932532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:10.259 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:09 smithi093 ceph-mon[111335]: pgmap v1249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:10.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:09 smithi093 ceph-mon[118902]: pgmap v1249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:09 smithi136 ceph-mon[113531]: pgmap v1249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:10.259Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:10.260Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/142639590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/142639590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/142639590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:11.789Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:11.790Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:12 smithi136 ceph-mon[113531]: pgmap v1250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:35:12.477 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:12 smithi093 ceph-mon[111335]: pgmap v1250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:12.478 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:35:12.478 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:12 smithi093 ceph-mon[118902]: pgmap v1250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:12.478 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:35:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[111335]: pgmap v1251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/84386392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[118902]: pgmap v1251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/84386392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:13.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:13.686Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:13.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:13.686Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:13 smithi136 ceph-mon[113531]: pgmap v1251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/84386392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:35:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:35:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:35:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:35:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:35:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:35:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:35:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:35:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:35:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:35:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:15 smithi136 ceph-mon[113531]: pgmap v1252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1000261177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:15 smithi093 ceph-mon[111335]: pgmap v1252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1000261177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:15 smithi093 ceph-mon[118902]: pgmap v1252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1000261177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:17 smithi093 ceph-mon[111335]: pgmap v1253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:17 smithi093 ceph-mon[118902]: pgmap v1253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:17 smithi136 ceph-mon[113531]: pgmap v1253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:19.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/63657975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:19.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/63657975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/63657975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:20.259 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:19 smithi093 ceph-mon[111335]: pgmap v1254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:20.259 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:19 smithi093 ceph-mon[118902]: pgmap v1254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:19 smithi136 ceph-mon[113531]: pgmap v1254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:20.259Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:20.260Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/484693511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/484693511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/484693511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:21 smithi093 ceph-mon[111335]: pgmap v1255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:21 smithi093 ceph-mon[118902]: pgmap v1255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:21.789Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:21.790Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:21 smithi136 ceph-mon[113531]: pgmap v1255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/72596937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/72596937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/72596937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:23.687Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:23.688Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:24 smithi136 ceph-mon[113531]: pgmap v1256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:24 smithi093 ceph-mon[111335]: pgmap v1256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:24 smithi093 ceph-mon[118902]: pgmap v1256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:35:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:35:25] "GET /metrics HTTP/1.1" 200 239876 "" "Prometheus/2.43.0" 2023-08-27T21:35:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:26 smithi136 ceph-mon[113531]: pgmap v1257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1655841679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:26 smithi093 ceph-mon[111335]: pgmap v1257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1655841679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:26 smithi093 ceph-mon[118902]: pgmap v1257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1655841679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:28 smithi136 ceph-mon[113531]: pgmap v1258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/917503902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:28 smithi093 ceph-mon[111335]: pgmap v1258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/917503902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:28 smithi093 ceph-mon[118902]: pgmap v1258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/917503902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:29 smithi136 ceph-mon[113531]: pgmap v1259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2037506657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:29 smithi093 ceph-mon[111335]: pgmap v1259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2037506657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:29 smithi093 ceph-mon[118902]: pgmap v1259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2037506657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:30.260Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:30.260Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:31 smithi093 ceph-mon[111335]: pgmap v1260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:31 smithi093 ceph-mon[118902]: pgmap v1260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:31.790Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:31.791Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:31 smithi136 ceph-mon[113531]: pgmap v1260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3042647588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3042647588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3042647588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:33.687Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:33.688Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:33 smithi136 ceph-mon[113531]: pgmap v1261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:33 smithi093 ceph-mon[111335]: pgmap v1261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:33 smithi093 ceph-mon[118902]: pgmap v1261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:35.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/808440975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/808440975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/808440975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:35:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:35:35] "GET /metrics HTTP/1.1" 200 239876 "" "Prometheus/2.43.0" 2023-08-27T21:35:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:36 smithi136 ceph-mon[113531]: pgmap v1262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:36 smithi093 ceph-mon[111335]: pgmap v1262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:36 smithi093 ceph-mon[118902]: pgmap v1262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:38 smithi136 ceph-mon[113531]: pgmap v1263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3232542315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:38 smithi093 ceph-mon[111335]: pgmap v1263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3232542315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:38 smithi093 ceph-mon[118902]: pgmap v1263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3232542315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:40.261 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:40 smithi093 ceph-mon[111335]: pgmap v1264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:40.261 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4031081272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:40.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:40 smithi093 ceph-mon[118902]: pgmap v1264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:40.261 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4031081272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:40 smithi136 ceph-mon[113531]: pgmap v1264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4031081272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:40.261Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:40.262Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:42 smithi093 ceph-mon[111335]: pgmap v1265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/973432036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:42 smithi093 ceph-mon[118902]: pgmap v1265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/973432036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:41.790Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:41.791Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:42 smithi136 ceph-mon[113531]: pgmap v1265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/973432036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:43.688Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:43.689Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:44 smithi136 ceph-mon[113531]: pgmap v1266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:44 smithi093 ceph-mon[111335]: pgmap v1266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:44 smithi093 ceph-mon[118902]: pgmap v1266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2666683507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2666683507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2666683507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:35:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:35:45] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:35:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:46 smithi136 ceph-mon[113531]: pgmap v1267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:46 smithi093 ceph-mon[111335]: pgmap v1267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:46 smithi093 ceph-mon[118902]: pgmap v1267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1508519634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1508519634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1508519634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:48 smithi136 ceph-mon[113531]: pgmap v1268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:48 smithi093 ceph-mon[111335]: pgmap v1268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:48 smithi093 ceph-mon[118902]: pgmap v1268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:49 smithi136 ceph-mon[113531]: pgmap v1269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/603922211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:49 smithi093 ceph-mon[111335]: pgmap v1269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/603922211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:49 smithi093 ceph-mon[118902]: pgmap v1269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/603922211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:50.262Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:50.263Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:51 smithi093 ceph-mon[111335]: pgmap v1270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1853435639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:51 smithi093 ceph-mon[118902]: pgmap v1270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1853435639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:51.791Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:51.792Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:51 smithi136 ceph-mon[113531]: pgmap v1270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1853435639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:53.689Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:35:53.690Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:35:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:53 smithi136 ceph-mon[113531]: pgmap v1271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1010934218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-mon[111335]: pgmap v1271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1010934218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-mon[118902]: pgmap v1271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:35:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1010934218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:35:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:35:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:35:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:56 smithi136 ceph-mon[113531]: pgmap v1272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:56 smithi093 ceph-mon[111335]: pgmap v1272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:56 smithi093 ceph-mon[118902]: pgmap v1272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3941005466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3941005466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3941005466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:58 smithi136 ceph-mon[113531]: pgmap v1273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:58 smithi093 ceph-mon[111335]: pgmap v1273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:58 smithi093 ceph-mon[118902]: pgmap v1273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:35:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:35:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2637251760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:35:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2637251760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:35:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:35:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2637251760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:00.263 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:00 smithi093 ceph-mon[111335]: pgmap v1274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:00.263 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:00 smithi093 ceph-mon[118902]: pgmap v1274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:00 smithi136 ceph-mon[113531]: pgmap v1274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:00.263Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:00.263Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/324638288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/324638288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/324638288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:02 smithi093 ceph-mon[111335]: pgmap v1275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:02 smithi093 ceph-mon[118902]: pgmap v1275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:01.792Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:01.793Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:02 smithi136 ceph-mon[113531]: pgmap v1275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:03.690Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:03.691Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:04 smithi136 ceph-mon[113531]: pgmap v1276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1756699499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:04 smithi093 ceph-mon[111335]: pgmap v1276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1756699499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:04 smithi093 ceph-mon[118902]: pgmap v1276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1756699499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:36:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:36:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:36:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:06 smithi136 ceph-mon[113531]: pgmap v1277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1800772759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:06 smithi093 ceph-mon[111335]: pgmap v1277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1800772759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:06 smithi093 ceph-mon[118902]: pgmap v1277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1800772759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:08.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:08 smithi136 ceph-mon[113531]: pgmap v1278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:08 smithi093 ceph-mon[111335]: pgmap v1278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:08 smithi093 ceph-mon[118902]: pgmap v1278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2544745026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2544745026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2544745026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:10 smithi136 ceph-mon[113531]: pgmap v1279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:10 smithi093 ceph-mon[111335]: pgmap v1279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:10 smithi093 ceph-mon[118902]: pgmap v1279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:10.263Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:10.264Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1148740107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1148740107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1148740107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:12 smithi093 ceph-mon[111335]: pgmap v1280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:12 smithi093 ceph-mon[118902]: pgmap v1280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:11.793Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:11.794Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:12 smithi136 ceph-mon[113531]: pgmap v1280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/776987363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/776987363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/776987363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:13.690Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:13.691Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:14 smithi136 ceph-mon[113531]: pgmap v1281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:14 smithi093 ceph-mon[111335]: pgmap v1281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:14 smithi093 ceph-mon[118902]: pgmap v1281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:36:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:36:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:36:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:36:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:36:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:36:16.392 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:16 smithi136 ceph-mon[113531]: pgmap v1282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:16.392 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2991975559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:16.392 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:16.392 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[118902]: pgmap v1282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2991975559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[111335]: pgmap v1282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2991975559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[118902]: pgmap v1283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:36:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:36:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2720705016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:17.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:17.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:17.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[111335]: pgmap v1283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:17.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:36:17.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:36:17.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:17.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2720705016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:17 smithi136 ceph-mon[113531]: pgmap v1283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:36:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:36:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:36:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2720705016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:19 smithi093 ceph-mon[111335]: pgmap v1284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:19 smithi093 ceph-mon[118902]: pgmap v1284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:19 smithi136 ceph-mon[113531]: pgmap v1284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:20.264Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:20.264Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3393267923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3393267923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3393267923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:21 smithi093 ceph-mon[111335]: pgmap v1285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:22.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:21 smithi093 ceph-mon[118902]: pgmap v1285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:21.794Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:21.794Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:21 smithi136 ceph-mon[113531]: pgmap v1285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1590108509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1590108509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1590108509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:23.953 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:23.690Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:23.953 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:23.691Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:23 smithi093 ceph-mon[111335]: pgmap v1286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:23 smithi093 ceph-mon[118902]: pgmap v1286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:23 smithi136 ceph-mon[113531]: pgmap v1286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/161097565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/161097565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/161097565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:36:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:36:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:36:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:25 smithi093 ceph-mon[111335]: pgmap v1287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:25 smithi093 ceph-mon[118902]: pgmap v1287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:25 smithi136 ceph-mon[113531]: pgmap v1287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:27 smithi093 ceph-mon[111335]: pgmap v1288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1352150750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:27 smithi093 ceph-mon[118902]: pgmap v1288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1352150750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:27 smithi136 ceph-mon[113531]: pgmap v1288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1352150750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:29 smithi093 ceph-mon[111335]: pgmap v1289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2173194701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:29 smithi093 ceph-mon[118902]: pgmap v1289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2173194701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:29 smithi136 ceph-mon[113531]: pgmap v1289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2173194701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:30.265Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:30.265Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:32 smithi093 ceph-mon[111335]: pgmap v1290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3243310686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:32 smithi093 ceph-mon[118902]: pgmap v1290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3243310686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:31.794Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:31.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:32 smithi136 ceph-mon[113531]: pgmap v1290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3243310686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:33.691Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:33.692Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:34 smithi136 ceph-mon[113531]: pgmap v1291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:34 smithi093 ceph-mon[111335]: pgmap v1291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:34 smithi093 ceph-mon[118902]: pgmap v1291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:35.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1336383685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1336383685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1336383685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:36:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:36:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:36:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:36 smithi136 ceph-mon[113531]: pgmap v1292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:36 smithi093 ceph-mon[111335]: pgmap v1292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:36 smithi093 ceph-mon[118902]: pgmap v1292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2901625868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2901625868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2901625868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:38 smithi136 ceph-mon[113531]: pgmap v1293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:38 smithi093 ceph-mon[111335]: pgmap v1293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:38 smithi093 ceph-mon[118902]: pgmap v1293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:40 smithi136 ceph-mon[113531]: pgmap v1294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/241066821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:40 smithi093 ceph-mon[111335]: pgmap v1294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/241066821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:40 smithi093 ceph-mon[118902]: pgmap v1294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/241066821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:40.265Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:40.266Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:42 smithi093 ceph-mon[118902]: pgmap v1295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3402555468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:42 smithi093 ceph-mon[111335]: pgmap v1295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3402555468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:41.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:41.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:42 smithi136 ceph-mon[113531]: pgmap v1295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3402555468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:43.691Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:43.692Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:44 smithi136 ceph-mon[113531]: pgmap v1296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/843128713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:44 smithi093 ceph-mon[118902]: pgmap v1296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/843128713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:44 smithi093 ceph-mon[111335]: pgmap v1296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/843128713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:36:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:36:45] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:36:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:46 smithi136 ceph-mon[113531]: pgmap v1297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:46 smithi093 ceph-mon[111335]: pgmap v1297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:46 smithi093 ceph-mon[118902]: pgmap v1297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/332695946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/332695946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/332695946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:48 smithi136 ceph-mon[113531]: pgmap v1298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:48 smithi093 ceph-mon[111335]: pgmap v1298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:48 smithi093 ceph-mon[118902]: pgmap v1298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2690044514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2690044514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2690044514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:50 smithi136 ceph-mon[113531]: pgmap v1299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:50 smithi093 ceph-mon[111335]: pgmap v1299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:50 smithi093 ceph-mon[118902]: pgmap v1299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:50.266Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:50.267Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2416500525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2416500525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2416500525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:52.052 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:51.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:52.053 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:51.796Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:52 smithi136 ceph-mon[113531]: pgmap v1300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:52 smithi093 ceph-mon[111335]: pgmap v1300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:52 smithi093 ceph-mon[118902]: pgmap v1300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:53.692Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:36:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:36:53.693Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:36:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:54 smithi136 ceph-mon[113531]: pgmap v1301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2342094660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:54 smithi093 ceph-mon[118902]: pgmap v1301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2342094660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:54 smithi093 ceph-mon[111335]: pgmap v1301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2342094660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:36:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:36:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:36:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:36:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:56 smithi136 ceph-mon[113531]: pgmap v1302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1816366580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:56 smithi093 ceph-mon[111335]: pgmap v1302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1816366580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:56 smithi093 ceph-mon[118902]: pgmap v1302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1816366580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:58 smithi136 ceph-mon[113531]: pgmap v1303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:58 smithi093 ceph-mon[111335]: pgmap v1303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:58 smithi093 ceph-mon[118902]: pgmap v1303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:36:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:36:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3708661211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:36:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3708661211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:36:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:36:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3708661211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:00 smithi136 ceph-mon[113531]: pgmap v1304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:00 smithi093 ceph-mon[111335]: pgmap v1304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:00 smithi093 ceph-mon[118902]: pgmap v1304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:00.267Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:00.268Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1822708160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1822708160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1822708160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:02.075 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:01.796Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:02.075 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:01.797Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:02 smithi136 ceph-mon[113531]: pgmap v1305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:02 smithi093 ceph-mon[111335]: pgmap v1305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:02 smithi093 ceph-mon[118902]: pgmap v1305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2620860195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2620860195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2620860195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:03.693Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:03.694Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:04 smithi136 ceph-mon[113531]: pgmap v1306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:04 smithi093 ceph-mon[111335]: pgmap v1306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:04 smithi093 ceph-mon[118902]: pgmap v1306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:37:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:37:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:37:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:06 smithi136 ceph-mon[113531]: pgmap v1307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3037476518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:06 smithi093 ceph-mon[111335]: pgmap v1307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3037476518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:06 smithi093 ceph-mon[118902]: pgmap v1307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3037476518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:08 smithi136 ceph-mon[113531]: pgmap v1308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1918182101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:08 smithi093 ceph-mon[111335]: pgmap v1308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1918182101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:08 smithi093 ceph-mon[118902]: pgmap v1308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1918182101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:10 smithi136 ceph-mon[113531]: pgmap v1309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4073768150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:10 smithi093 ceph-mon[111335]: pgmap v1309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4073768150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:10 smithi093 ceph-mon[118902]: pgmap v1309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4073768150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:10.268Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:10.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:12.096 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:11.797Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:12.096 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:11.798Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:12 smithi136 ceph-mon[113531]: pgmap v1310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:12 smithi093 ceph-mon[111335]: pgmap v1310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:12 smithi093 ceph-mon[118902]: pgmap v1310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2567990119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2567990119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2567990119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:13.694Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:13.695Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:14 smithi136 ceph-mon[113531]: pgmap v1311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:14 smithi093 ceph-mon[111335]: pgmap v1311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:14 smithi093 ceph-mon[118902]: pgmap v1311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/130189397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/130189397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/130189397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:37:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:37:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:37:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:16 smithi136 ceph-mon[113531]: pgmap v1312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:16 smithi093 ceph-mon[111335]: pgmap v1312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:16 smithi093 ceph-mon[118902]: pgmap v1312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:18 smithi136 ceph-mon[113531]: pgmap v1313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4040315272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:37:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:18 smithi093 ceph-mon[111335]: pgmap v1313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4040315272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:37:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:18 smithi093 ceph-mon[118902]: pgmap v1313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4040315272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:37:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:19 smithi136 ceph-mon[113531]: pgmap v1314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/63822735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[111335]: pgmap v1314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/63822735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[118902]: pgmap v1314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/63822735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:20.269Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:20.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:20.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:37:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:37:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:37:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:37:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:37:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:37:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:37:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:21 smithi093 ceph-mon[111335]: pgmap v1315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:21 smithi093 ceph-mon[118902]: pgmap v1315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:21.798Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:21.799Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:21 smithi136 ceph-mon[113531]: pgmap v1315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2212234234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2212234234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2212234234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:23.962 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:23.695Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:23.962 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:23.696Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:23 smithi093 ceph-mon[118902]: pgmap v1316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:23 smithi093 ceph-mon[111335]: pgmap v1316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:23 smithi136 ceph-mon[113531]: pgmap v1316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2635492804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2635492804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2635492804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:37:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:37:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:37:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:25 smithi093 ceph-mon[111335]: pgmap v1317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:25 smithi093 ceph-mon[118902]: pgmap v1317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:25 smithi136 ceph-mon[113531]: pgmap v1317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/838831460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/838831460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/838831460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:27 smithi093 ceph-mon[111335]: pgmap v1318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:27 smithi093 ceph-mon[118902]: pgmap v1318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:27 smithi136 ceph-mon[113531]: pgmap v1318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:29 smithi093 ceph-mon[111335]: pgmap v1319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/67867938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:29 smithi093 ceph-mon[118902]: pgmap v1319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/67867938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:29 smithi136 ceph-mon[113531]: pgmap v1319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/67867938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:30.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:30.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:32 smithi093 ceph-mon[111335]: pgmap v1320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1296667760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:32 smithi093 ceph-mon[118902]: pgmap v1320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1296667760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:31.799Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:31.799Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:32 smithi136 ceph-mon[113531]: pgmap v1320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1296667760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:33.695Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:33.696Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:34 smithi136 ceph-mon[113531]: pgmap v1321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/77915446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:34 smithi093 ceph-mon[111335]: pgmap v1321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/77915446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:34 smithi093 ceph-mon[118902]: pgmap v1321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/77915446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:37:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:37:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:37:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:36 smithi136 ceph-mon[113531]: pgmap v1322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:36 smithi093 ceph-mon[118902]: pgmap v1322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:36 smithi093 ceph-mon[111335]: pgmap v1322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2408279845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2408279845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2408279845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:38 smithi136 ceph-mon[113531]: pgmap v1323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:38 smithi093 ceph-mon[111335]: pgmap v1323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:38 smithi093 ceph-mon[118902]: pgmap v1323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1216356353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1216356353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1216356353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:40 smithi136 ceph-mon[113531]: pgmap v1324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:40 smithi093 ceph-mon[111335]: pgmap v1324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:40 smithi093 ceph-mon[118902]: pgmap v1324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:40.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:40.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/986817185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/986817185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/986817185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:42 smithi093 ceph-mon[111335]: pgmap v1325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:42 smithi093 ceph-mon[118902]: pgmap v1325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:41.799Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:41.800Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:42 smithi136 ceph-mon[113531]: pgmap v1325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:43.695Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:43.696Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:44 smithi136 ceph-mon[113531]: pgmap v1326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/782933544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:44 smithi093 ceph-mon[111335]: pgmap v1326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/782933544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:44 smithi093 ceph-mon[118902]: pgmap v1326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/782933544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:37:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:37:45] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:37:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:46 smithi136 ceph-mon[113531]: pgmap v1327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3619345448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:46 smithi093 ceph-mon[118902]: pgmap v1327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3619345448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:46 smithi093 ceph-mon[111335]: pgmap v1327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3619345448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:48 smithi136 ceph-mon[113531]: pgmap v1328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:48 smithi093 ceph-mon[111335]: pgmap v1328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:48 smithi093 ceph-mon[118902]: pgmap v1328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/325790373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/325790373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/325790373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:50 smithi136 ceph-mon[113531]: pgmap v1329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:50 smithi093 ceph-mon[111335]: pgmap v1329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:50 smithi093 ceph-mon[118902]: pgmap v1329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:50.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:50.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3789010546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3789010546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3789010546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:52.064 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:51.800Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:52.064 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:51.801Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:52 smithi136 ceph-mon[113531]: pgmap v1330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:52 smithi093 ceph-mon[111335]: pgmap v1330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:52 smithi093 ceph-mon[118902]: pgmap v1330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/99386539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/99386539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/99386539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:53.697Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:37:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:37:53.698Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:37:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:54 smithi136 ceph-mon[113531]: pgmap v1331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:54 smithi093 ceph-mon[111335]: pgmap v1331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:54 smithi093 ceph-mon[118902]: pgmap v1331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:37:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:37:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:37:55] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:37:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:56 smithi136 ceph-mon[113531]: pgmap v1332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1104248044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:56 smithi093 ceph-mon[118902]: pgmap v1332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1104248044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:56 smithi093 ceph-mon[111335]: pgmap v1332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1104248044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:58 smithi136 ceph-mon[113531]: pgmap v1333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3183303355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:58 smithi093 ceph-mon[118902]: pgmap v1333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3183303355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:37:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:58 smithi093 ceph-mon[111335]: pgmap v1333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:37:58.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3183303355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:37:59 smithi093 ceph-mon[111335]: pgmap v1334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:37:59 smithi093 ceph-mon[118902]: pgmap v1334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:37:59 smithi136 ceph-mon[113531]: pgmap v1334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:00.271Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:00.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3462274805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3462274805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3462274805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:01 smithi093 ceph-mon[111335]: pgmap v1335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:01 smithi093 ceph-mon[118902]: pgmap v1335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:01.801Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:01.801Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:01 smithi136 ceph-mon[113531]: pgmap v1335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1390805053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1390805053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1390805053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:03.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:03.697Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:03.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:03.698Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:03 smithi093 ceph-mon[111335]: pgmap v1336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:03 smithi093 ceph-mon[118902]: pgmap v1336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:03 smithi136 ceph-mon[113531]: pgmap v1336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3164683369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3164683369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3164683369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:38:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:38:05] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:38:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:05 smithi136 ceph-mon[113531]: pgmap v1337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:05 smithi093 ceph-mon[111335]: pgmap v1337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:05 smithi093 ceph-mon[118902]: pgmap v1337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:07 smithi136 ceph-mon[113531]: pgmap v1338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1910455599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:07 smithi093 ceph-mon[111335]: pgmap v1338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1910455599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:07 smithi093 ceph-mon[118902]: pgmap v1338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1910455599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:10.272 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:09 smithi093 ceph-mon[111335]: pgmap v1339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:10.272 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2368657258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:10.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:09 smithi093 ceph-mon[118902]: pgmap v1339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:10.273 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2368657258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:09 smithi136 ceph-mon[113531]: pgmap v1339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2368657258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:10.272Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:10.273Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:11 smithi093 ceph-mon[111335]: pgmap v1340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:11 smithi093 ceph-mon[118902]: pgmap v1340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:11.801Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:11.802Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:11 smithi136 ceph-mon[113531]: pgmap v1340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1127180413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1127180413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1127180413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:13.698Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:13.699Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:14 smithi136 ceph-mon[113531]: pgmap v1341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:14 smithi093 ceph-mon[111335]: pgmap v1341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:14 smithi093 ceph-mon[118902]: pgmap v1341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3709288903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3709288903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3709288903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:38:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:38:15] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:38:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:16 smithi136 ceph-mon[113531]: pgmap v1342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:16 smithi093 ceph-mon[111335]: pgmap v1342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:16 smithi093 ceph-mon[118902]: pgmap v1342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/211644862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/211644862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/211644862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:18 smithi136 ceph-mon[113531]: pgmap v1343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:18 smithi093 ceph-mon[111335]: pgmap v1343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:18 smithi093 ceph-mon[118902]: pgmap v1343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:19 smithi093 ceph-mon[111335]: pgmap v1344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1798599404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:19 smithi093 ceph-mon[118902]: pgmap v1344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1798599404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:19 smithi136 ceph-mon[113531]: pgmap v1344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1798599404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:20.273Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:20.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:38:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:38:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:38:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:21 smithi136 ceph-mon[113531]: pgmap v1345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2453727766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:21 smithi093 ceph-mon[111335]: pgmap v1345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2453727766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:21 smithi093 ceph-mon[118902]: pgmap v1345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2453727766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:21.802Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:21.803Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:23.699Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:23.700Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:24 smithi136 ceph-mon[113531]: pgmap v1346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1767102149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[111335]: pgmap v1346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1767102149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[118902]: pgmap v1346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:38:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1767102149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:38:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:38:25] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:38:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:26 smithi136 ceph-mon[113531]: pgmap v1347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:26 smithi093 ceph-mon[111335]: pgmap v1347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:26 smithi093 ceph-mon[118902]: pgmap v1347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/723348843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/723348843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/723348843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:28 smithi136 ceph-mon[113531]: pgmap v1348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:28 smithi093 ceph-mon[111335]: pgmap v1348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:28 smithi093 ceph-mon[118902]: pgmap v1348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1444272954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1444272954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1444272954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:30 smithi136 ceph-mon[113531]: pgmap v1349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:30 smithi093 ceph-mon[111335]: pgmap v1349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:30 smithi093 ceph-mon[118902]: pgmap v1349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:30.274Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:30.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1007267595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1007267595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1007267595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:32.160 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:31.803Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:32.161 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:31.804Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:32 smithi136 ceph-mon[113531]: pgmap v1350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:32 smithi093 ceph-mon[111335]: pgmap v1350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:32 smithi093 ceph-mon[118902]: pgmap v1350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:33.700Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:33.701Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:34 smithi136 ceph-mon[113531]: pgmap v1351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4182212146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:34 smithi093 ceph-mon[111335]: pgmap v1351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4182212146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:34 smithi093 ceph-mon[118902]: pgmap v1351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4182212146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:38:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:38:35] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:38:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:36 smithi136 ceph-mon[113531]: pgmap v1352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/792486995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:36 smithi093 ceph-mon[111335]: pgmap v1352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/792486995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:36 smithi093 ceph-mon[118902]: pgmap v1352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/792486995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:38 smithi136 ceph-mon[113531]: pgmap v1353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:38 smithi093 ceph-mon[111335]: pgmap v1353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:38 smithi093 ceph-mon[118902]: pgmap v1353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2071164832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2071164832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2071164832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:40 smithi136 ceph-mon[113531]: pgmap v1354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:40 smithi093 ceph-mon[111335]: pgmap v1354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:40 smithi093 ceph-mon[118902]: pgmap v1354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:40.275Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:40.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1017621061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1017621061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1017621061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:42.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:41.804Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:42.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:41.805Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:42 smithi136 ceph-mon[113531]: pgmap v1355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:42 smithi093 ceph-mon[111335]: pgmap v1355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:42 smithi093 ceph-mon[118902]: pgmap v1355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/429093084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/429093084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/429093084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:43.701Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:43.702Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:44 smithi136 ceph-mon[113531]: pgmap v1356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:44 smithi093 ceph-mon[111335]: pgmap v1356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:44 smithi093 ceph-mon[118902]: pgmap v1356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:38:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:38:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:38:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:46 smithi093 ceph-mon[111335]: pgmap v1357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2429818235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:46 smithi093 ceph-mon[118902]: pgmap v1357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2429818235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:46 smithi136 ceph-mon[113531]: pgmap v1357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2429818235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:48 smithi093 ceph-mon[111335]: pgmap v1358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1346830572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:48 smithi093 ceph-mon[118902]: pgmap v1358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1346830572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:48 smithi136 ceph-mon[113531]: pgmap v1358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1346830572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:49 smithi093 ceph-mon[111335]: pgmap v1359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:49 smithi093 ceph-mon[118902]: pgmap v1359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:49 smithi136 ceph-mon[113531]: pgmap v1359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:50.276Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:50.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3585445974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3585445974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3585445974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:51 smithi093 ceph-mon[111335]: pgmap v1360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:51 smithi093 ceph-mon[118902]: pgmap v1360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:51.804Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:51.805Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:51 smithi136 ceph-mon[113531]: pgmap v1360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/304486706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/304486706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/304486706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:53.980 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:53.701Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:53.980 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:38:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:38:53.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:38:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:53 smithi093 ceph-mon[111335]: pgmap v1361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:53 smithi093 ceph-mon[118902]: pgmap v1361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:53 smithi136 ceph-mon[113531]: pgmap v1361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:38:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/863713188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/863713188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/863713188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:38:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:38:55] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T21:38:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:55 smithi093 ceph-mon[111335]: pgmap v1362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:55 smithi093 ceph-mon[118902]: pgmap v1362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:55 smithi136 ceph-mon[113531]: pgmap v1362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:57 smithi136 ceph-mon[113531]: pgmap v1363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2269433701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:57 smithi093 ceph-mon[111335]: pgmap v1363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2269433701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:38:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:57 smithi093 ceph-mon[118902]: pgmap v1363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:38:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2269433701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:59 smithi093 ceph-mon[111335]: pgmap v1364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:38:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/549788897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:59 smithi093 ceph-mon[118902]: pgmap v1364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:38:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/549788897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:59 smithi136 ceph-mon[113531]: pgmap v1364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:38:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/549788897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:00.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:00.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:02 smithi093 ceph-mon[111335]: pgmap v1365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:02 smithi093 ceph-mon[118902]: pgmap v1365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:01.805Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:01.806Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:02 smithi136 ceph-mon[113531]: pgmap v1365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2838987941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2838987941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2838987941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:03.701Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:03.702Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:04 smithi136 ceph-mon[113531]: pgmap v1366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:04 smithi093 ceph-mon[111335]: pgmap v1366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:04 smithi093 ceph-mon[118902]: pgmap v1366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:05.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1630439753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1630439753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1630439753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:39:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:39:05] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T21:39:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:06 smithi136 ceph-mon[113531]: pgmap v1367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:06 smithi093 ceph-mon[111335]: pgmap v1367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:06 smithi093 ceph-mon[118902]: pgmap v1367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/819157808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/819157808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/819157808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:08 smithi136 ceph-mon[113531]: pgmap v1368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:08 smithi093 ceph-mon[111335]: pgmap v1368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:08 smithi093 ceph-mon[118902]: pgmap v1368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:10 smithi136 ceph-mon[113531]: pgmap v1369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4163281762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:10 smithi093 ceph-mon[111335]: pgmap v1369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4163281762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:10 smithi093 ceph-mon[118902]: pgmap v1369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4163281762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:10.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:10.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:12.067 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:11.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:12.067 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:11.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:12.068 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:12 smithi093 ceph-mon[111335]: pgmap v1370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:12 smithi136 ceph-mon[113531]: pgmap v1370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3789915760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3789915760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:12 smithi093 ceph-mon[118902]: pgmap v1370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3789915760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:13.702Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:13.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:14 smithi136 ceph-mon[113531]: pgmap v1371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/800998972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:14 smithi093 ceph-mon[118902]: pgmap v1371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/800998972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:14 smithi093 ceph-mon[111335]: pgmap v1371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/800998972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:39:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:39:15] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:39:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:16 smithi136 ceph-mon[113531]: pgmap v1372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:16 smithi093 ceph-mon[111335]: pgmap v1372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:16 smithi093 ceph-mon[118902]: pgmap v1372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1532057076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1532057076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1532057076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:18 smithi136 ceph-mon[113531]: pgmap v1373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:18 smithi093 ceph-mon[111335]: pgmap v1373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:18 smithi093 ceph-mon[118902]: pgmap v1373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4245078704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4245078704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4245078704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:20 smithi136 ceph-mon[113531]: pgmap v1374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:20 smithi093 ceph-mon[111335]: pgmap v1374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:20 smithi093 ceph-mon[118902]: pgmap v1374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:20.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:20.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:22.089 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:21.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:22.089 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:21.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:22 smithi136 ceph-mon[113531]: pgmap v1375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/754358645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:22 smithi093 ceph-mon[118902]: pgmap v1375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/754358645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:22 smithi093 ceph-mon[111335]: pgmap v1375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/754358645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:23.702Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:23.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:24.433 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[111335]: pgmap v1376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:24.433 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4089063292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:24.433 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:39:24.433 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:24.433 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:24.434 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:24.434 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[118902]: pgmap v1376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:24.434 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4089063292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:24.434 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:39:24.434 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:24.434 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:24.434 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:24 smithi136 ceph-mon[113531]: pgmap v1376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4089063292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:39:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:39:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:39:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: pgmap v1377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/403981497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:39:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:39:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: pgmap v1377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/403981497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: pgmap v1377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/403981497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:26.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:39:26.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:39:26.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:39:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:28 smithi136 ceph-mon[113531]: pgmap v1378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/119513559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:28 smithi093 ceph-mon[111335]: pgmap v1378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/119513559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:28 smithi093 ceph-mon[118902]: pgmap v1378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/119513559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:29 smithi093 ceph-mon[111335]: pgmap v1379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:29 smithi093 ceph-mon[118902]: pgmap v1379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:29 smithi136 ceph-mon[113531]: pgmap v1379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:30.278Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:30.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2316419336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2316419336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2316419336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:31 smithi093 ceph-mon[111335]: pgmap v1380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:31 smithi093 ceph-mon[118902]: pgmap v1380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:31.807Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:31.808Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:31 smithi136 ceph-mon[113531]: pgmap v1380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:33.703Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:33.704Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:34 smithi136 ceph-mon[113531]: pgmap v1381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2592867719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:34 smithi093 ceph-mon[111335]: pgmap v1381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2592867719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:34 smithi093 ceph-mon[118902]: pgmap v1381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2592867719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:39:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:39:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:39:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:36 smithi136 ceph-mon[113531]: pgmap v1382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/352612510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:36 smithi093 ceph-mon[111335]: pgmap v1382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/352612510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:36 smithi093 ceph-mon[118902]: pgmap v1382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/352612510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:38 smithi136 ceph-mon[113531]: pgmap v1383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2103027361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:38 smithi093 ceph-mon[111335]: pgmap v1383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2103027361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:38 smithi093 ceph-mon[118902]: pgmap v1383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2103027361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:40 smithi136 ceph-mon[113531]: pgmap v1384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:40 smithi093 ceph-mon[111335]: pgmap v1384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:40 smithi093 ceph-mon[118902]: pgmap v1384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:40.278Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:40.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3944241163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3944241163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3944241163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:42.067 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:41.808Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:42.067 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:41.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:42 smithi136 ceph-mon[113531]: pgmap v1385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:42 smithi093 ceph-mon[111335]: pgmap v1385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:42 smithi093 ceph-mon[118902]: pgmap v1385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3829412005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3829412005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3829412005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:43.704Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:43.705Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:44 smithi136 ceph-mon[113531]: pgmap v1386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:44 smithi093 ceph-mon[111335]: pgmap v1386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:44 smithi093 ceph-mon[118902]: pgmap v1386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4236662838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4236662838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4236662838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:39:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:39:45] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:39:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:46 smithi136 ceph-mon[113531]: pgmap v1387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:46 smithi093 ceph-mon[111335]: pgmap v1387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:46 smithi093 ceph-mon[118902]: pgmap v1387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:48 smithi136 ceph-mon[113531]: pgmap v1388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2341866366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:48 smithi093 ceph-mon[111335]: pgmap v1388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2341866366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:48 smithi093 ceph-mon[118902]: pgmap v1388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2341866366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:49 smithi093 ceph-mon[111335]: pgmap v1389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2853999998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:49 smithi093 ceph-mon[118902]: pgmap v1389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2853999998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:49 smithi136 ceph-mon[113531]: pgmap v1389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2853999998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:50.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:50.743 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:50.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:52 smithi093 ceph-mon[111335]: pgmap v1390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:52 smithi093 ceph-mon[118902]: pgmap v1390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:51.808Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:51.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:52 smithi136 ceph-mon[113531]: pgmap v1390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1850509940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1850509940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1850509940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:53.705Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:39:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:39:53.706Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:39:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:54 smithi136 ceph-mon[113531]: pgmap v1391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:54.794 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:54.795 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:54 smithi093 ceph-mon[111335]: pgmap v1391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:54.795 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:54.795 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:54 smithi093 ceph-mon[118902]: pgmap v1391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:54.796 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:39:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2398378058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2398378058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2398378058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:39:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:39:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:39:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:56 smithi136 ceph-mon[113531]: pgmap v1392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:56 smithi093 ceph-mon[111335]: pgmap v1392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:56 smithi093 ceph-mon[118902]: pgmap v1392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/943957888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/943957888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/943957888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:39:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:58 smithi136 ceph-mon[113531]: pgmap v1393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:58 smithi093 ceph-mon[111335]: pgmap v1393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:39:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:58 smithi093 ceph-mon[118902]: pgmap v1393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:59 smithi093 ceph-mon[111335]: pgmap v1394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:39:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2231100731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:59 smithi093 ceph-mon[118902]: pgmap v1394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:39:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2231100731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:59 smithi136 ceph-mon[113531]: pgmap v1394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:39:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2231100731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:00.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:00.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:40:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:40:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:40:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:01 smithi093 ceph-mon[111335]: pgmap v1395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4010459987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:01 smithi093 ceph-mon[118902]: pgmap v1395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4010459987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:01.808Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:01.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:01 smithi136 ceph-mon[113531]: pgmap v1395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4010459987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:03.705Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:03.706Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:04 smithi136 ceph-mon[113531]: pgmap v1396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:04 smithi093 ceph-mon[111335]: pgmap v1396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:04 smithi093 ceph-mon[118902]: pgmap v1396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/182376799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/182376799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/182376799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:40:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:40:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:40:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:06 smithi136 ceph-mon[113531]: pgmap v1397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:06 smithi093 ceph-mon[111335]: pgmap v1397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:06 smithi093 ceph-mon[118902]: pgmap v1397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1837233210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1837233210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1837233210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:08 smithi136 ceph-mon[113531]: pgmap v1398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:08 smithi093 ceph-mon[111335]: pgmap v1398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:08 smithi093 ceph-mon[118902]: pgmap v1398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1050515022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1050515022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1050515022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:10 smithi136 ceph-mon[113531]: pgmap v1399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:10 smithi093 ceph-mon[111335]: pgmap v1399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:10 smithi093 ceph-mon[118902]: pgmap v1399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:10.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:10.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:12.081 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:11.809Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:12.082 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:11.810Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:12.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:12 smithi136 ceph-mon[113531]: pgmap v1400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:12.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/388328971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:12 smithi093 ceph-mon[111335]: pgmap v1400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/388328971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:12 smithi093 ceph-mon[118902]: pgmap v1400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/388328971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:13.706Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:13.707Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:14 smithi136 ceph-mon[113531]: pgmap v1401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/946692784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:14 smithi093 ceph-mon[111335]: pgmap v1401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/946692784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:14 smithi093 ceph-mon[118902]: pgmap v1401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/946692784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:40:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:40:15] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:40:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:16 smithi136 ceph-mon[113531]: pgmap v1402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2679602603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:16 smithi093 ceph-mon[111335]: pgmap v1402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2679602603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:16 smithi093 ceph-mon[118902]: pgmap v1402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2679602603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:18 smithi136 ceph-mon[113531]: pgmap v1403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:18 smithi093 ceph-mon[111335]: pgmap v1403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:18 smithi093 ceph-mon[118902]: pgmap v1403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/800371262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/800371262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/800371262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:20 smithi136 ceph-mon[113531]: pgmap v1404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:20.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:20 smithi093 ceph-mon[111335]: pgmap v1404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:20 smithi093 ceph-mon[118902]: pgmap v1404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:20.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:20.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:20.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:20.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1132777201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1132777201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1132777201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:22.107 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:21.809Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:22.107 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:21.810Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:22 smithi136 ceph-mon[113531]: pgmap v1405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:22 smithi093 ceph-mon[111335]: pgmap v1405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:22 smithi093 ceph-mon[118902]: pgmap v1405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:23.707Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:23.708Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:24 smithi136 ceph-mon[113531]: pgmap v1406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/808667339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:24 smithi093 ceph-mon[111335]: pgmap v1406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/808667339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:24 smithi093 ceph-mon[118902]: pgmap v1406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/808667339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:40:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:40:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:40:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:26 smithi136 ceph-mon[113531]: pgmap v1407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1538044722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:26 smithi093 ceph-mon[111335]: pgmap v1407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1538044722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:26 smithi093 ceph-mon[118902]: pgmap v1407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1538044722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:40:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:40:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:40:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:28 smithi136 ceph-mon[113531]: pgmap v1408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1258055022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:28 smithi093 ceph-mon[111335]: pgmap v1408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1258055022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:28 smithi093 ceph-mon[118902]: pgmap v1408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1258055022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:29.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:30 smithi136 ceph-mon[113531]: pgmap v1409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:40:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:40:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/780916887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[111335]: pgmap v1409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:40:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:40:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/780916887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[118902]: pgmap v1409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:40:30.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:40:30.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:40:30.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/780916887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:30.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:30.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:30.743 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:30.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:31.810Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:31.811Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:32.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:32 smithi136 ceph-mon[113531]: pgmap v1410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:32 smithi093 ceph-mon[111335]: pgmap v1410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:32 smithi093 ceph-mon[118902]: pgmap v1410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/61787672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:33.708 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/61787672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:33.708 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/61787672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:33.707Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:33.709Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:34 smithi136 ceph-mon[113531]: pgmap v1411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:34 smithi093 ceph-mon[111335]: pgmap v1411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:34 smithi093 ceph-mon[118902]: pgmap v1411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4183848504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4183848504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:40:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:40:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:40:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4183848504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:36 smithi136 ceph-mon[113531]: pgmap v1412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:36 smithi093 ceph-mon[118902]: pgmap v1412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:36 smithi093 ceph-mon[111335]: pgmap v1412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:38 smithi136 ceph-mon[113531]: pgmap v1413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/76478458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:38 smithi093 ceph-mon[111335]: pgmap v1413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/76478458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:38 smithi093 ceph-mon[118902]: pgmap v1413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/76478458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:40 smithi136 ceph-mon[113531]: pgmap v1414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3016399790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:40 smithi093 ceph-mon[118902]: pgmap v1414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3016399790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:40 smithi093 ceph-mon[111335]: pgmap v1414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3016399790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:40.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:40.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:41.810Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:41.811Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:42 smithi136 ceph-mon[113531]: pgmap v1415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3092829592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:42 smithi093 ceph-mon[118902]: pgmap v1415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3092829592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:42 smithi093 ceph-mon[111335]: pgmap v1415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3092829592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:43.708Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:43.709Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:44 smithi136 ceph-mon[113531]: pgmap v1416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:44 smithi093 ceph-mon[111335]: pgmap v1416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:44 smithi093 ceph-mon[118902]: pgmap v1416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/9724499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/9724499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:40:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:40:45] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:40:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/9724499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:46 smithi093 ceph-mon[118902]: pgmap v1417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:46 smithi093 ceph-mon[111335]: pgmap v1417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:46 smithi136 ceph-mon[113531]: pgmap v1417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2242889003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2242889003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2242889003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:48 smithi093 ceph-mon[111335]: pgmap v1418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:48 smithi093 ceph-mon[118902]: pgmap v1418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:48 smithi136 ceph-mon[113531]: pgmap v1418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:49 smithi093 ceph-mon[111335]: pgmap v1419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/228250035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:49 smithi093 ceph-mon[118902]: pgmap v1419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/228250035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:49 smithi136 ceph-mon[113531]: pgmap v1419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/228250035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:50.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:50.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:52.068 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:51.811Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:52.069 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:51.812Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:52 smithi136 ceph-mon[113531]: pgmap v1420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2530040725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:52 smithi093 ceph-mon[111335]: pgmap v1420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2530040725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:52 smithi093 ceph-mon[118902]: pgmap v1420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2530040725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:53.709Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:40:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:40:53.710Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:40:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:54 smithi136 ceph-mon[113531]: pgmap v1421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:54 smithi093 ceph-mon[118902]: pgmap v1421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:54 smithi093 ceph-mon[111335]: pgmap v1421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:40:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3006736663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3006736663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3006736663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:40:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:40:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:40:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:56 smithi136 ceph-mon[113531]: pgmap v1422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:56 smithi093 ceph-mon[111335]: pgmap v1422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:56 smithi093 ceph-mon[118902]: pgmap v1422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1991693524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1991693524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1991693524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:58 smithi136 ceph-mon[113531]: pgmap v1423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:58 smithi093 ceph-mon[111335]: pgmap v1423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:58 smithi093 ceph-mon[118902]: pgmap v1423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:40:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:40:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3515032227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:40:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3515032227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:40:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:40:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3515032227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:00 smithi136 ceph-mon[113531]: pgmap v1424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:00 smithi093 ceph-mon[111335]: pgmap v1424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:00 smithi093 ceph-mon[118902]: pgmap v1424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:00.284Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:00.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:02.090 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:01.812Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:02.091 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:01.813Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:02 smithi136 ceph-mon[113531]: pgmap v1425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1596026405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:02 smithi093 ceph-mon[111335]: pgmap v1425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1596026405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:02 smithi093 ceph-mon[118902]: pgmap v1425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1596026405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:03.709Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:03.710Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:04 smithi136 ceph-mon[113531]: pgmap v1426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2233733107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:04 smithi093 ceph-mon[111335]: pgmap v1426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2233733107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:04 smithi093 ceph-mon[118902]: pgmap v1426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2233733107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T21:41:05.317762759Z level=info msg="Completed cleanup jobs" duration=13.199753ms 2023-08-27T21:41:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:41:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:41:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:41:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:06 smithi136 ceph-mon[113531]: pgmap v1427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/927027486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:06 smithi093 ceph-mon[111335]: pgmap v1427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/927027486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:06 smithi093 ceph-mon[118902]: pgmap v1427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/927027486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:08 smithi136 ceph-mon[113531]: pgmap v1428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:08 smithi093 ceph-mon[111335]: pgmap v1428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:08 smithi093 ceph-mon[118902]: pgmap v1428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4061450011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4061450011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4061450011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:10 smithi136 ceph-mon[113531]: pgmap v1429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:10 smithi093 ceph-mon[111335]: pgmap v1429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:10 smithi093 ceph-mon[118902]: pgmap v1429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:10.285Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:10.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1194861034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1194861034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1194861034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:12.109 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:11.813Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:12.109 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:11.814Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:12 smithi136 ceph-mon[113531]: pgmap v1430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:12 smithi093 ceph-mon[118902]: pgmap v1430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:12 smithi093 ceph-mon[111335]: pgmap v1430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:13.710Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:13.711Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:14 smithi136 ceph-mon[113531]: pgmap v1431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2693493146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:14 smithi093 ceph-mon[118902]: pgmap v1431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2693493146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:14 smithi093 ceph-mon[111335]: pgmap v1431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2693493146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:41:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:41:15] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T21:41:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:16 smithi136 ceph-mon[113531]: pgmap v1432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2711129112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:16 smithi093 ceph-mon[111335]: pgmap v1432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2711129112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:16 smithi093 ceph-mon[118902]: pgmap v1432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2711129112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:18 smithi136 ceph-mon[113531]: pgmap v1433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2202347197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:18 smithi093 ceph-mon[111335]: pgmap v1433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2202347197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:18 smithi093 ceph-mon[118902]: pgmap v1433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2202347197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:20.286 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:19 smithi093 ceph-mon[111335]: pgmap v1434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:20.286 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:19 smithi093 ceph-mon[118902]: pgmap v1434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:19 smithi136 ceph-mon[113531]: pgmap v1434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:20.285Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:20.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2216569591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2216569591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2216569591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:22.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:22 smithi093 ceph-mon[111335]: pgmap v1435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:22 smithi093 ceph-mon[118902]: pgmap v1435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:22.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:21.814Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:22.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:21.815Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:22 smithi136 ceph-mon[113531]: pgmap v1435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/758956605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/758956605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/758956605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:23.711Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:23.712Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:24 smithi136 ceph-mon[113531]: pgmap v1436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:24 smithi093 ceph-mon[111335]: pgmap v1436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:24 smithi093 ceph-mon[118902]: pgmap v1436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:41:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:41:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:41:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:26 smithi136 ceph-mon[113531]: pgmap v1437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3059605089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:26 smithi093 ceph-mon[118902]: pgmap v1437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3059605089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:26 smithi093 ceph-mon[111335]: pgmap v1437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3059605089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:28 smithi136 ceph-mon[113531]: pgmap v1438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3757547711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:28 smithi093 ceph-mon[118902]: pgmap v1438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3757547711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:28.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:28 smithi093 ceph-mon[111335]: pgmap v1438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:28.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3757547711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:30.286 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:29 smithi093 ceph-mon[111335]: pgmap v1439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:30.286 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/213180964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:30.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:41:30.287 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:29 smithi093 ceph-mon[118902]: pgmap v1439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:30.287 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/213180964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:30.287 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:41:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:29 smithi136 ceph-mon[113531]: pgmap v1439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/213180964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:41:30.541 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:30.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:30.541 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:30.287Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:31.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:31.815Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:32 smithi136 ceph-mon[113531]: pgmap v1440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2434409172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[111335]: pgmap v1440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2434409172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[118902]: pgmap v1440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2434409172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:41:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:41:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:41:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:41:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:33.712Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:33.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:33.713Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:33.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:33.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:41:33.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:41:33.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:41:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:34 smithi136 ceph-mon[113531]: pgmap v1441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:34 smithi093 ceph-mon[111335]: pgmap v1441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:34 smithi093 ceph-mon[118902]: pgmap v1441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:35.613 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:41:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:41:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:41:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4065713415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4065713415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4065713415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:36 smithi136 ceph-mon[113531]: pgmap v1442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:36 smithi093 ceph-mon[111335]: pgmap v1442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:36 smithi093 ceph-mon[118902]: pgmap v1442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:37 smithi136 ceph-mon[113531]: pgmap v1443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3965242154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:37 smithi093 ceph-mon[111335]: pgmap v1443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3965242154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:37 smithi093 ceph-mon[118902]: pgmap v1443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3965242154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:40.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:40 smithi093 ceph-mon[111335]: pgmap v1444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:40.288 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1937079339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:40.288 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:40 smithi093 ceph-mon[118902]: pgmap v1444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:40.288 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1937079339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:40 smithi136 ceph-mon[113531]: pgmap v1444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1937079339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:40.287Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:40.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:42.081 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:41.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:42.081 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:41.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:42 smithi136 ceph-mon[113531]: pgmap v1445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/703757423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:42 smithi093 ceph-mon[111335]: pgmap v1445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/703757423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:42 smithi093 ceph-mon[118902]: pgmap v1445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/703757423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:43.713Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:43.714Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:44 smithi136 ceph-mon[113531]: pgmap v1446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:44 smithi093 ceph-mon[111335]: pgmap v1446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:44 smithi093 ceph-mon[118902]: pgmap v1446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/967387881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/967387881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/967387881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:41:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:41:45] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T21:41:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:46 smithi136 ceph-mon[113531]: pgmap v1447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:46 smithi093 ceph-mon[111335]: pgmap v1447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:46 smithi093 ceph-mon[118902]: pgmap v1447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2915427855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2915427855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2915427855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:48 smithi136 ceph-mon[113531]: pgmap v1448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:48 smithi093 ceph-mon[111335]: pgmap v1448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:48 smithi093 ceph-mon[118902]: pgmap v1448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3607121558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3607121558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3607121558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:50 smithi136 ceph-mon[113531]: pgmap v1449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:50 smithi093 ceph-mon[111335]: pgmap v1449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:50 smithi093 ceph-mon[118902]: pgmap v1449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:50.287Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:50.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:52.102 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:51.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:52.102 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:51.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:52 smithi136 ceph-mon[113531]: pgmap v1450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1494249088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:52 smithi093 ceph-mon[111335]: pgmap v1450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1494249088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:52 smithi093 ceph-mon[118902]: pgmap v1450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1494249088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:53.714Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:41:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:41:53.714Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:41:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:54 smithi136 ceph-mon[113531]: pgmap v1451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2332129450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:54 smithi093 ceph-mon[111335]: pgmap v1451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2332129450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:54 smithi093 ceph-mon[118902]: pgmap v1451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:41:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2332129450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:41:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:41:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:41:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:56 smithi136 ceph-mon[113531]: pgmap v1452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:56 smithi093 ceph-mon[111335]: pgmap v1452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:56 smithi093 ceph-mon[118902]: pgmap v1452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3452458785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3452458785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3452458785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:58 smithi136 ceph-mon[113531]: pgmap v1453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:58 smithi093 ceph-mon[111335]: pgmap v1453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:58 smithi093 ceph-mon[118902]: pgmap v1453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:41:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:41:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4202639963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:41:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4202639963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:41:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:41:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4202639963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:00 smithi136 ceph-mon[113531]: pgmap v1454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:00 smithi093 ceph-mon[111335]: pgmap v1454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:00 smithi093 ceph-mon[118902]: pgmap v1454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:00.288Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:00.289Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1116328732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1116328732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1116328732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:02.121 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:01.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:02.121 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:01.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:02 smithi136 ceph-mon[113531]: pgmap v1455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:02 smithi093 ceph-mon[111335]: pgmap v1455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:02 smithi093 ceph-mon[118902]: pgmap v1455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:03.714Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:03.715Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:04 smithi136 ceph-mon[113531]: pgmap v1456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2052921543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:04 smithi093 ceph-mon[111335]: pgmap v1456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2052921543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:04 smithi093 ceph-mon[118902]: pgmap v1456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2052921543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:42:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:42:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:42:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:06 smithi136 ceph-mon[113531]: pgmap v1457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/156780322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:06 smithi093 ceph-mon[111335]: pgmap v1457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/156780322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:06 smithi093 ceph-mon[118902]: pgmap v1457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/156780322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:08 smithi136 ceph-mon[113531]: pgmap v1458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1684545290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:08 smithi093 ceph-mon[111335]: pgmap v1458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1684545290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:08 smithi093 ceph-mon[118902]: pgmap v1458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1684545290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:10 smithi136 ceph-mon[113531]: pgmap v1459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:10 smithi093 ceph-mon[111335]: pgmap v1459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:10 smithi093 ceph-mon[118902]: pgmap v1459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:10.290Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:10.290Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3308559047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3308559047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3308559047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:12.146 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:11.816Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:12.146 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:11.817Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:12 smithi136 ceph-mon[113531]: pgmap v1460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:12 smithi093 ceph-mon[111335]: pgmap v1460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:12 smithi093 ceph-mon[118902]: pgmap v1460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/544431886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/544431886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/544431886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:13.715Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:13.716Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:14 smithi136 ceph-mon[113531]: pgmap v1461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:14 smithi093 ceph-mon[111335]: pgmap v1461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:14 smithi093 ceph-mon[118902]: pgmap v1461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:42:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:42:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:42:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:16 smithi136 ceph-mon[113531]: pgmap v1462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/305708050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:16 smithi093 ceph-mon[111335]: pgmap v1462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/305708050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:16 smithi093 ceph-mon[118902]: pgmap v1462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/305708050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:18 smithi136 ceph-mon[113531]: pgmap v1463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3185156882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:18 smithi093 ceph-mon[111335]: pgmap v1463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3185156882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:18 smithi093 ceph-mon[118902]: pgmap v1463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3185156882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:20.291 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:20 smithi093 ceph-mon[111335]: pgmap v1464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:20.291 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4146481151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:20.292 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:20 smithi093 ceph-mon[118902]: pgmap v1464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:20.292 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4146481151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:20 smithi136 ceph-mon[113531]: pgmap v1464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4146481151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:20.291Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:20.291Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:22.091 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:21.817Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:22.091 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:21.818Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:22 smithi136 ceph-mon[113531]: pgmap v1465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:22 smithi093 ceph-mon[111335]: pgmap v1465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:22 smithi093 ceph-mon[118902]: pgmap v1465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2866082293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2866082293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2866082293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:23.716Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:23.717Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:24 smithi136 ceph-mon[113531]: pgmap v1466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:24 smithi093 ceph-mon[111335]: pgmap v1466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:24 smithi093 ceph-mon[118902]: pgmap v1466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/81307357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/81307357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/81307357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:42:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:42:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:42:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:26 smithi136 ceph-mon[113531]: pgmap v1467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:26.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:26 smithi093 ceph-mon[111335]: pgmap v1467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:26.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:26 smithi093 ceph-mon[118902]: pgmap v1467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:28 smithi136 ceph-mon[113531]: pgmap v1468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/187963834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:28 smithi093 ceph-mon[111335]: pgmap v1468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/187963834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:28 smithi093 ceph-mon[118902]: pgmap v1468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/187963834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:30.292 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:30 smithi093 ceph-mon[111335]: pgmap v1469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:30.292 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2358407335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:30.292 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:30 smithi093 ceph-mon[118902]: pgmap v1469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:30.292 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2358407335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:30 smithi136 ceph-mon[113531]: pgmap v1469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2358407335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:30.292Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:30.293Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:32.093 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:31.818Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:32.094 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:31.819Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:32 smithi136 ceph-mon[113531]: pgmap v1470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2541220596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:32 smithi093 ceph-mon[111335]: pgmap v1470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2541220596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:32 smithi093 ceph-mon[118902]: pgmap v1470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2541220596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:33.717Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:33.718Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:34 smithi136 ceph-mon[113531]: pgmap v1471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:42:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:34 smithi093 ceph-mon[118902]: pgmap v1471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:42:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:34 smithi093 ceph-mon[111335]: pgmap v1471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:42:35.340 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1259765089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:35.341 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:35.341 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1259765089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1259765089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:42:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:42:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:42:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:36 smithi136 ceph-mon[113531]: pgmap v1472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:42:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:42:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[111335]: pgmap v1472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:42:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:42:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[118902]: pgmap v1472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:42:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:42:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:42:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4214240537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4214240537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4214240537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:38 smithi136 ceph-mon[113531]: pgmap v1473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:38 smithi093 ceph-mon[111335]: pgmap v1473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:38 smithi093 ceph-mon[118902]: pgmap v1473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3824488752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3824488752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3824488752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:40 smithi136 ceph-mon[113531]: pgmap v1474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:40 smithi093 ceph-mon[111335]: pgmap v1474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:40 smithi093 ceph-mon[118902]: pgmap v1474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:40.293Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:40.294Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:42.112 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:41.818Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:42.112 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:41.820Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:42 smithi136 ceph-mon[113531]: pgmap v1475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2469049249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:42 smithi093 ceph-mon[111335]: pgmap v1475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2469049249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:42 smithi093 ceph-mon[118902]: pgmap v1475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2469049249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:43.717Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:43.719Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:44 smithi136 ceph-mon[113531]: pgmap v1476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3438228606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:44 smithi093 ceph-mon[111335]: pgmap v1476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3438228606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:44 smithi093 ceph-mon[118902]: pgmap v1476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3438228606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:42:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:42:45] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:42:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:46 smithi136 ceph-mon[113531]: pgmap v1477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:46 smithi093 ceph-mon[111335]: pgmap v1477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:46 smithi093 ceph-mon[118902]: pgmap v1477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2428618679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2428618679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2428618679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:48 smithi136 ceph-mon[113531]: pgmap v1478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:48 smithi093 ceph-mon[111335]: pgmap v1478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:48 smithi093 ceph-mon[118902]: pgmap v1478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1172476140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1172476140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1172476140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:50 smithi136 ceph-mon[113531]: pgmap v1479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:50 smithi093 ceph-mon[111335]: pgmap v1479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:50 smithi093 ceph-mon[118902]: pgmap v1479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:50.293Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:50.294Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3803358457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3803358457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3803358457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:52.132 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:51.819Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:52.132 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:51.820Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:52 smithi136 ceph-mon[113531]: pgmap v1480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:52 smithi093 ceph-mon[111335]: pgmap v1480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:52 smithi093 ceph-mon[118902]: pgmap v1480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:53.718Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:42:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:42:53.720Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:42:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:54 smithi136 ceph-mon[113531]: pgmap v1481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/352124487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:54 smithi093 ceph-mon[111335]: pgmap v1481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/352124487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:54 smithi093 ceph-mon[118902]: pgmap v1481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/352124487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:42:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:42:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:42:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:42:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:56 smithi136 ceph-mon[113531]: pgmap v1482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/208103084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:56 smithi093 ceph-mon[111335]: pgmap v1482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/208103084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:56 smithi093 ceph-mon[118902]: pgmap v1482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/208103084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:58 smithi136 ceph-mon[113531]: pgmap v1483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:42:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3838794985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:58 smithi093 ceph-mon[111335]: pgmap v1483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:42:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3838794985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:42:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:58 smithi093 ceph-mon[118902]: pgmap v1483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:42:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:42:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3838794985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:00.294 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:00 smithi093 ceph-mon[111335]: pgmap v1484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:00.295 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:00 smithi093 ceph-mon[118902]: pgmap v1484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:00 smithi136 ceph-mon[113531]: pgmap v1484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:00.294Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:00.295Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3659657333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3659657333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:01.534 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3659657333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:02 smithi093 ceph-mon[111335]: pgmap v1485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:02 smithi093 ceph-mon[118902]: pgmap v1485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:01.820Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:02.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:01.820Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:02 smithi136 ceph-mon[113531]: pgmap v1485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2333365137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2333365137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2333365137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:03.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:03.721Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:04 smithi136 ceph-mon[113531]: pgmap v1486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:04 smithi093 ceph-mon[111335]: pgmap v1486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:04 smithi093 ceph-mon[118902]: pgmap v1486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:43:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:43:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:43:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:06 smithi136 ceph-mon[113531]: pgmap v1487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1738427463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:06 smithi093 ceph-mon[118902]: pgmap v1487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1738427463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:06 smithi093 ceph-mon[111335]: pgmap v1487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1738427463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:08 smithi136 ceph-mon[113531]: pgmap v1488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3802891145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:08 smithi093 ceph-mon[111335]: pgmap v1488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3802891145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:08 smithi093 ceph-mon[118902]: pgmap v1488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3802891145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:10 smithi136 ceph-mon[113531]: pgmap v1489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2880500187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:10 smithi093 ceph-mon[118902]: pgmap v1489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2880500187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:10 smithi093 ceph-mon[111335]: pgmap v1489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2880500187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:10.295Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:10.296Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:12.111 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:11.820Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:12.112 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:11.821Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:12 smithi136 ceph-mon[113531]: pgmap v1490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:12 smithi093 ceph-mon[111335]: pgmap v1490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:12 smithi093 ceph-mon[118902]: pgmap v1490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1739618363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1739618363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1739618363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:13.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:13.721Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:14 smithi136 ceph-mon[113531]: pgmap v1491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:14 smithi093 ceph-mon[111335]: pgmap v1491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:14 smithi093 ceph-mon[118902]: pgmap v1491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1173106434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1173106434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1173106434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:43:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:43:15] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:43:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:16 smithi136 ceph-mon[113531]: pgmap v1492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:16 smithi093 ceph-mon[111335]: pgmap v1492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:16 smithi093 ceph-mon[118902]: pgmap v1492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:18 smithi136 ceph-mon[113531]: pgmap v1493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/348230316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:18 smithi093 ceph-mon[111335]: pgmap v1493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/348230316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:18 smithi093 ceph-mon[118902]: pgmap v1493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/348230316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:20.296 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:20 smithi093 ceph-mon[111335]: pgmap v1494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:20.296 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1730822730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:20.297 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:20 smithi093 ceph-mon[118902]: pgmap v1494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:20.297 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1730822730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:20 smithi136 ceph-mon[113531]: pgmap v1494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1730822730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:20.296Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:20.297Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:22.106 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:21.821Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:22.107 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:21.822Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:22 smithi136 ceph-mon[113531]: pgmap v1495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3578832752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:22 smithi093 ceph-mon[118902]: pgmap v1495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3578832752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:22 smithi093 ceph-mon[111335]: pgmap v1495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3578832752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:23.721Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:23.722Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:24 smithi136 ceph-mon[113531]: pgmap v1496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:24 smithi093 ceph-mon[118902]: pgmap v1496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:24 smithi093 ceph-mon[111335]: pgmap v1496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2126772828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2126772828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2126772828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:43:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:43:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:43:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:26 smithi136 ceph-mon[113531]: pgmap v1497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:26 smithi093 ceph-mon[111335]: pgmap v1497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:26 smithi093 ceph-mon[118902]: pgmap v1497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1017289632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1017289632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1017289632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:28 smithi136 ceph-mon[113531]: pgmap v1498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:28 smithi093 ceph-mon[111335]: pgmap v1498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:28 smithi093 ceph-mon[118902]: pgmap v1498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:30.296 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:30 smithi093 ceph-mon[111335]: pgmap v1499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:30.297 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4235614272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:30.297 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:30 smithi093 ceph-mon[118902]: pgmap v1499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:30.297 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4235614272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:30 smithi136 ceph-mon[113531]: pgmap v1499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4235614272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:30.296Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:30.297Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:32.109 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:31.821Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:32.109 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:31.822Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:32 smithi136 ceph-mon[113531]: pgmap v1500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2164179748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:32 smithi093 ceph-mon[111335]: pgmap v1500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2164179748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:32 smithi093 ceph-mon[118902]: pgmap v1500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2164179748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:33.721Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:33.722Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:34 smithi136 ceph-mon[113531]: pgmap v1501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/924225589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:34 smithi093 ceph-mon[111335]: pgmap v1501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/924225589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:34 smithi093 ceph-mon[118902]: pgmap v1501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/924225589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:43:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:43:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:43:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:36 smithi136 ceph-mon[113531]: pgmap v1502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:36 smithi093 ceph-mon[111335]: pgmap v1502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:36 smithi093 ceph-mon[118902]: pgmap v1502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:43:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1478371338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:43:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1478371338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:43:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1478371338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:38 smithi136 ceph-mon[113531]: pgmap v1503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:38 smithi093 ceph-mon[118902]: pgmap v1503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:38 smithi093 ceph-mon[111335]: pgmap v1503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3714916793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3714916793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3714916793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:40 smithi136 ceph-mon[113531]: pgmap v1504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:43:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:43:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[111335]: pgmap v1504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:43:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:43:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[118902]: pgmap v1504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:43:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:43:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:43:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:40.297Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:40.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:40.299Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1226598537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1226598537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1226598537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:42.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:41.822Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:42.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:41.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:42 smithi136 ceph-mon[113531]: pgmap v1505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:42 smithi093 ceph-mon[111335]: pgmap v1505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:42 smithi093 ceph-mon[118902]: pgmap v1505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:43.722Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:43.723Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:44 smithi136 ceph-mon[113531]: pgmap v1506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1713832203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:44 smithi093 ceph-mon[118902]: pgmap v1506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1713832203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:44 smithi093 ceph-mon[111335]: pgmap v1506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1713832203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:43:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:43:45] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:43:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:46 smithi136 ceph-mon[113531]: pgmap v1507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1424906229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:46 smithi093 ceph-mon[111335]: pgmap v1507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1424906229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:46 smithi093 ceph-mon[118902]: pgmap v1507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1424906229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:48 smithi136 ceph-mon[113531]: pgmap v1508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:48 smithi093 ceph-mon[111335]: pgmap v1508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:48 smithi093 ceph-mon[118902]: pgmap v1508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1089187772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1089187772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1089187772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:50 smithi136 ceph-mon[113531]: pgmap v1509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:50 smithi093 ceph-mon[111335]: pgmap v1509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:50 smithi093 ceph-mon[118902]: pgmap v1509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:50.298Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:50.299Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2742059581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2742059581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2742059581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:52.156 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:51.823Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:52.156 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:51.824Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:52 smithi136 ceph-mon[113531]: pgmap v1510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:52 smithi093 ceph-mon[111335]: pgmap v1510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:52 smithi093 ceph-mon[118902]: pgmap v1510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2442817504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2442817504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2442817504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:53.723Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:43:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:43:53.724Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:43:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:54 smithi136 ceph-mon[113531]: pgmap v1511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:54 smithi093 ceph-mon[111335]: pgmap v1511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:54 smithi093 ceph-mon[118902]: pgmap v1511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:43:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:43:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:43:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:43:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:56 smithi136 ceph-mon[113531]: pgmap v1512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2804207858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:56 smithi093 ceph-mon[111335]: pgmap v1512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2804207858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:56 smithi093 ceph-mon[118902]: pgmap v1512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2804207858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:58 smithi136 ceph-mon[113531]: pgmap v1513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:43:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/354657559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:58 smithi093 ceph-mon[111335]: pgmap v1513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:43:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/354657559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:43:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:58 smithi093 ceph-mon[118902]: pgmap v1513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:43:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:43:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/354657559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:00.302 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:00 smithi093 ceph-mon[111335]: pgmap v1514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:00.302 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:00 smithi093 ceph-mon[118902]: pgmap v1514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:00 smithi136 ceph-mon[113531]: pgmap v1514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:00.299Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:00.300Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4171949993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4171949993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4171949993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:02 smithi093 ceph-mon[111335]: pgmap v1515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:02 smithi093 ceph-mon[118902]: pgmap v1515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:01.823Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:01.824Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:02 smithi136 ceph-mon[113531]: pgmap v1515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4223572683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4223572683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4223572683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:03.724Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:03.725Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:04 smithi136 ceph-mon[113531]: pgmap v1516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:04 smithi093 ceph-mon[111335]: pgmap v1516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:04 smithi093 ceph-mon[118902]: pgmap v1516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2580237706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2580237706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2580237706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:44:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:44:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:44:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:06 smithi136 ceph-mon[113531]: pgmap v1517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:06 smithi093 ceph-mon[111335]: pgmap v1517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:06 smithi093 ceph-mon[118902]: pgmap v1517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:08 smithi136 ceph-mon[113531]: pgmap v1518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3003157414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:08 smithi093 ceph-mon[111335]: pgmap v1518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3003157414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:08 smithi093 ceph-mon[118902]: pgmap v1518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3003157414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:10 smithi136 ceph-mon[113531]: pgmap v1519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/535075222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:10 smithi093 ceph-mon[111335]: pgmap v1519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/535075222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:10 smithi093 ceph-mon[118902]: pgmap v1519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/535075222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:10.300Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:10.301Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:12.123 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:11.824Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:12.123 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:11.825Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:12 smithi136 ceph-mon[113531]: pgmap v1520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1781385155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:12 smithi093 ceph-mon[111335]: pgmap v1520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1781385155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:12 smithi093 ceph-mon[118902]: pgmap v1520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1781385155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:13.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:13.726Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:14 smithi136 ceph-mon[113531]: pgmap v1521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:14 smithi093 ceph-mon[111335]: pgmap v1521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:14 smithi093 ceph-mon[118902]: pgmap v1521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2913032931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2913032931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2913032931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:44:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:44:15] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:44:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:16 smithi136 ceph-mon[113531]: pgmap v1522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:16 smithi093 ceph-mon[111335]: pgmap v1522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:16 smithi093 ceph-mon[118902]: pgmap v1522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2709000793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2709000793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2709000793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:18 smithi136 ceph-mon[113531]: pgmap v1523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:18 smithi093 ceph-mon[111335]: pgmap v1523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:18 smithi093 ceph-mon[118902]: pgmap v1523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:20.301 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:20 smithi093 ceph-mon[111335]: pgmap v1524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:20.301 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1423520777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:20.301 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:20 smithi093 ceph-mon[118902]: pgmap v1524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:20.301 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1423520777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:20 smithi136 ceph-mon[113531]: pgmap v1524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1423520777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:20.300Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:20.301Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:22.121 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:21.825Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:22.121 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:21.826Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:22 smithi136 ceph-mon[113531]: pgmap v1525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2531340643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:22 smithi093 ceph-mon[111335]: pgmap v1525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2531340643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:22 smithi093 ceph-mon[118902]: pgmap v1525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2531340643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:23.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:23.726Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:24 smithi136 ceph-mon[113531]: pgmap v1526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/578232413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:24 smithi093 ceph-mon[111335]: pgmap v1526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/578232413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:24 smithi093 ceph-mon[118902]: pgmap v1526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/578232413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:44:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:44:25] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:44:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:26 smithi136 ceph-mon[113531]: pgmap v1527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:26 smithi093 ceph-mon[111335]: pgmap v1527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:26 smithi093 ceph-mon[118902]: pgmap v1527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2839723805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2839723805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2839723805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:28 smithi136 ceph-mon[113531]: pgmap v1528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:28 smithi093 ceph-mon[111335]: pgmap v1528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:28 smithi093 ceph-mon[118902]: pgmap v1528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1352880644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1352880644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1352880644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:30 smithi136 ceph-mon[113531]: pgmap v1529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:30 smithi093 ceph-mon[118902]: pgmap v1529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:30 smithi093 ceph-mon[111335]: pgmap v1529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:30.301Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:30.302Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/465009702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/465009702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/465009702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:32.146 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:31.826Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:32.146 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:31.827Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:32 smithi136 ceph-mon[113531]: pgmap v1530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:32 smithi093 ceph-mon[111335]: pgmap v1530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:32 smithi093 ceph-mon[118902]: pgmap v1530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:33.726Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:33.727Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:34 smithi136 ceph-mon[113531]: pgmap v1531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3414119975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:34 smithi093 ceph-mon[118902]: pgmap v1531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3414119975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:34 smithi093 ceph-mon[111335]: pgmap v1531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3414119975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:44:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:44:35] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:44:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:36 smithi136 ceph-mon[113531]: pgmap v1532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/91181638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:36 smithi093 ceph-mon[111335]: pgmap v1532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/91181638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:36 smithi093 ceph-mon[118902]: pgmap v1532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/91181638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:38 smithi136 ceph-mon[113531]: pgmap v1533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:38 smithi093 ceph-mon[111335]: pgmap v1533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:38 smithi093 ceph-mon[118902]: pgmap v1533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/321478270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/321478270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/321478270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:40 smithi093 ceph-mon[111335]: pgmap v1534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:40 smithi093 ceph-mon[118902]: pgmap v1534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:40.315 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:40 smithi136 ceph-mon[113531]: pgmap v1534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:40.315 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:44:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:40.302Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:40.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/975759784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/975759784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/975759784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:42.163 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:41.826Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:42.163 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:41.828Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:42 smithi136 ceph-mon[113531]: pgmap v1535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:42 smithi093 ceph-mon[111335]: pgmap v1535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:42 smithi093 ceph-mon[118902]: pgmap v1535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:44:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:44:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/107190259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:44:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:44:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/107190259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:44:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:44:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:44:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/107190259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:43.727Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:43.728Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:44 smithi093 ceph-mon[111335]: pgmap v1536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:44 smithi093 ceph-mon[118902]: pgmap v1536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:44 smithi136 ceph-mon[113531]: pgmap v1536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:44:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:44:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:44:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:46 smithi093 ceph-mon[111335]: pgmap v1537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3391469909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:46 smithi093 ceph-mon[118902]: pgmap v1537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3391469909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:46 smithi136 ceph-mon[113531]: pgmap v1537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3391469909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:48 smithi093 ceph-mon[111335]: pgmap v1538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/176663356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:48 smithi093 ceph-mon[118902]: pgmap v1538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/176663356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:48 smithi136 ceph-mon[113531]: pgmap v1538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/176663356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:50.303 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:50 smithi093 ceph-mon[111335]: pgmap v1539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:50.304 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:50 smithi093 ceph-mon[118902]: pgmap v1539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:50 smithi136 ceph-mon[113531]: pgmap v1539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:50.303Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:50.304Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1432926122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1432926122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1432926122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:52 smithi093 ceph-mon[111335]: pgmap v1540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:52 smithi093 ceph-mon[118902]: pgmap v1540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:51.828Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:51.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:52 smithi136 ceph-mon[113531]: pgmap v1540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/868226463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/868226463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/868226463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:53.728Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:44:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:44:53.729Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:44:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:54 smithi136 ceph-mon[113531]: pgmap v1541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:54 smithi093 ceph-mon[118902]: pgmap v1541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:54 smithi093 ceph-mon[111335]: pgmap v1541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:44:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1085358849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1085358849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1085358849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:44:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:44:55] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:44:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:56 smithi136 ceph-mon[113531]: pgmap v1542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:56 smithi093 ceph-mon[111335]: pgmap v1542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:56 smithi093 ceph-mon[118902]: pgmap v1542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:58 smithi136 ceph-mon[113531]: pgmap v1543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:44:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/98977047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:58 smithi093 ceph-mon[111335]: pgmap v1543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:44:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/98977047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:44:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:58 smithi093 ceph-mon[118902]: pgmap v1543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:44:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:44:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/98977047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:00.304 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:00 smithi093 ceph-mon[111335]: pgmap v1544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:00.305 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/946680345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:00.305 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:00 smithi093 ceph-mon[118902]: pgmap v1544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:00.305 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/946680345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:00 smithi136 ceph-mon[113531]: pgmap v1544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/946680345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:00.304Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:00.305Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:02.130 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:01.828Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:02.131 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:01.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:02 smithi136 ceph-mon[113531]: pgmap v1545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3864940499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:02 smithi093 ceph-mon[118902]: pgmap v1545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3864940499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:02 smithi093 ceph-mon[111335]: pgmap v1545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3864940499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:03.729Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:03.730Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:04 smithi136 ceph-mon[113531]: pgmap v1546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:04 smithi093 ceph-mon[111335]: pgmap v1546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:04 smithi093 ceph-mon[118902]: pgmap v1546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/383070547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/383070547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:05.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/383070547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:05.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:45:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:45:05] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:45:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:06 smithi136 ceph-mon[113531]: pgmap v1547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:06 smithi093 ceph-mon[111335]: pgmap v1547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:06 smithi093 ceph-mon[118902]: pgmap v1547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1159708501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1159708501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1159708501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:08 smithi136 ceph-mon[113531]: pgmap v1548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:08 smithi093 ceph-mon[111335]: pgmap v1548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:08 smithi093 ceph-mon[118902]: pgmap v1548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:10 smithi136 ceph-mon[113531]: pgmap v1549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1440575534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:10 smithi093 ceph-mon[111335]: pgmap v1549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1440575534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:10 smithi093 ceph-mon[118902]: pgmap v1549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1440575534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:10.305Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:10.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:12.165 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:11.828Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:12.165 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:11.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:12 smithi136 ceph-mon[113531]: pgmap v1550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1575566178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:12 smithi093 ceph-mon[111335]: pgmap v1550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1575566178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:12 smithi093 ceph-mon[118902]: pgmap v1550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1575566178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:13.729Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:13.730Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:14 smithi136 ceph-mon[113531]: pgmap v1551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2566484479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:14 smithi093 ceph-mon[111335]: pgmap v1551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2566484479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:14 smithi093 ceph-mon[118902]: pgmap v1551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2566484479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:45:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:45:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:45:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:16 smithi136 ceph-mon[113531]: pgmap v1552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:16 smithi093 ceph-mon[111335]: pgmap v1552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:16 smithi093 ceph-mon[118902]: pgmap v1552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2944385926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2944385926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2944385926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:18 smithi136 ceph-mon[113531]: pgmap v1553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:18 smithi093 ceph-mon[111335]: pgmap v1553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:18 smithi093 ceph-mon[118902]: pgmap v1553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3528693505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3528693505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3528693505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:20 smithi136 ceph-mon[113531]: pgmap v1554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:20 smithi093 ceph-mon[111335]: pgmap v1554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:20 smithi093 ceph-mon[118902]: pgmap v1554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:20.306Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:20.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/672302563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/672302563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/672302563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:22.189 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:21.829Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:22.189 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:21.831Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:22 smithi093 ceph-mon[111335]: pgmap v1555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:22 smithi093 ceph-mon[118902]: pgmap v1555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:22 smithi136 ceph-mon[113531]: pgmap v1555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:23.730Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:23.731Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:24 smithi093 ceph-mon[111335]: pgmap v1556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2588883976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:24 smithi093 ceph-mon[118902]: pgmap v1556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2588883976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:24 smithi136 ceph-mon[113531]: pgmap v1556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2588883976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:45:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:45:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:45:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:26 smithi093 ceph-mon[111335]: pgmap v1557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3478787992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:26 smithi093 ceph-mon[118902]: pgmap v1557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3478787992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:26 smithi136 ceph-mon[113531]: pgmap v1557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3478787992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:28 smithi093 ceph-mon[111335]: pgmap v1558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:28 smithi093 ceph-mon[118902]: pgmap v1558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:28 smithi136 ceph-mon[113531]: pgmap v1558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/236685896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/236685896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/236685896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:30 smithi093 ceph-mon[111335]: pgmap v1559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:30 smithi093 ceph-mon[118902]: pgmap v1559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:30.306Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:30.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:30 smithi136 ceph-mon[113531]: pgmap v1559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3522316615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3522316615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3522316615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:32.208 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:31.830Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:32.209 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:31.830Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:32 smithi093 ceph-mon[111335]: pgmap v1560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:32 smithi093 ceph-mon[118902]: pgmap v1560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:32 smithi136 ceph-mon[113531]: pgmap v1560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1091768487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1091768487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1091768487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:33.731Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:33.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:34 smithi093 ceph-mon[111335]: pgmap v1561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:34 smithi093 ceph-mon[118902]: pgmap v1561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:34 smithi136 ceph-mon[113531]: pgmap v1561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:45:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:45:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:45:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:36 smithi093 ceph-mon[111335]: pgmap v1562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/721129722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:36 smithi093 ceph-mon[118902]: pgmap v1562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/721129722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:36 smithi136 ceph-mon[113531]: pgmap v1562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/721129722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:38 smithi093 ceph-mon[118902]: pgmap v1563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3345369944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:38 smithi093 ceph-mon[111335]: pgmap v1563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3345369944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:38 smithi136 ceph-mon[113531]: pgmap v1563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3345369944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:40 smithi093 ceph-mon[111335]: pgmap v1564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3415881938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:40 smithi093 ceph-mon[118902]: pgmap v1564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3415881938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:40.307Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:40.308Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:40 smithi136 ceph-mon[113531]: pgmap v1564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3415881938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:42.226 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:41.830Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:42.226 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:41.831Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:42 smithi093 ceph-mon[111335]: pgmap v1565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:42 smithi093 ceph-mon[118902]: pgmap v1565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:42 smithi136 ceph-mon[113531]: pgmap v1565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/413704581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/413704581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:45:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/413704581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:43.732Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:43.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:44 smithi093 ceph-mon[111335]: pgmap v1566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:44 smithi093 ceph-mon[118902]: pgmap v1566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:44.670 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:44 smithi136 ceph-mon[113531]: pgmap v1566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:44.670 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:44.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:45:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:45:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3535586640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:45 smithi136 ceph-mon[113531]: pgmap v1567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3535586640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[111335]: pgmap v1567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3535586640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[118902]: pgmap v1567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:45:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:45:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:48 smithi136 ceph-mon[113531]: pgmap v1568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1186207188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:48 smithi093 ceph-mon[111335]: pgmap v1568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1186207188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:48 smithi093 ceph-mon[118902]: pgmap v1568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1186207188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:50.308 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:50 smithi093 ceph-mon[111335]: pgmap v1569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:50.308 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1934325611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:50.309 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:50 smithi093 ceph-mon[118902]: pgmap v1569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:50.309 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1934325611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:50 smithi136 ceph-mon[113531]: pgmap v1569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1934325611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:50.307Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:50.308Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:52.143 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:51.831Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:52.143 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:51.832Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:52 smithi136 ceph-mon[113531]: pgmap v1570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3467846778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:52 smithi093 ceph-mon[111335]: pgmap v1570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3467846778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:52 smithi093 ceph-mon[118902]: pgmap v1570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3467846778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:53.732Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:45:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:45:53.733Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:45:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:54 smithi136 ceph-mon[113531]: pgmap v1571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:54 smithi093 ceph-mon[118902]: pgmap v1571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:54 smithi093 ceph-mon[111335]: pgmap v1571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:45:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/415644317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/415644317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/415644317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:45:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:45:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:45:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:56 smithi136 ceph-mon[113531]: pgmap v1572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:56 smithi093 ceph-mon[111335]: pgmap v1572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:56 smithi093 ceph-mon[118902]: pgmap v1572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1229362859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1229362859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1229362859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:45:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:45:58 smithi136 ceph-mon[113531]: pgmap v1573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:45:58 smithi093 ceph-mon[111335]: pgmap v1573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:45:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:45:58 smithi093 ceph-mon[118902]: pgmap v1573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:00.308 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:00 smithi093 ceph-mon[111335]: pgmap v1574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:00.308 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/55711070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:00.308 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:00 smithi093 ceph-mon[118902]: pgmap v1574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:00.309 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/55711070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:00 smithi136 ceph-mon[113531]: pgmap v1574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/55711070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:00.308Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:00.309Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:02.154 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:01.832Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:02.154 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:01.833Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:02 smithi136 ceph-mon[113531]: pgmap v1575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1777238129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:02 smithi093 ceph-mon[118902]: pgmap v1575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1777238129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:02 smithi093 ceph-mon[111335]: pgmap v1575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1777238129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:03.733Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:03.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:03.734Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:04 smithi136 ceph-mon[113531]: pgmap v1576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2717319473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:04 smithi093 ceph-mon[111335]: pgmap v1576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2717319473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:04 smithi093 ceph-mon[118902]: pgmap v1576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2717319473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:46:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:46:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:46:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:06 smithi136 ceph-mon[113531]: pgmap v1577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:06 smithi093 ceph-mon[111335]: pgmap v1577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:06 smithi093 ceph-mon[118902]: pgmap v1577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/254586712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/254586712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/254586712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:08 smithi136 ceph-mon[113531]: pgmap v1578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:08 smithi093 ceph-mon[111335]: pgmap v1578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:08 smithi093 ceph-mon[118902]: pgmap v1578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/927047100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/927047100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/927047100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:10 smithi136 ceph-mon[113531]: pgmap v1579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:10 smithi093 ceph-mon[111335]: pgmap v1579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:10 smithi093 ceph-mon[118902]: pgmap v1579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:10.309Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:10.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:12.170 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:11.833Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:12.170 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:11.834Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:12 smithi136 ceph-mon[113531]: pgmap v1580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/363343008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:12 smithi093 ceph-mon[111335]: pgmap v1580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/363343008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:12 smithi093 ceph-mon[118902]: pgmap v1580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/363343008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:13.733Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:13.734Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:14 smithi136 ceph-mon[113531]: pgmap v1581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3684515006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:14 smithi093 ceph-mon[111335]: pgmap v1581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3684515006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:14 smithi093 ceph-mon[118902]: pgmap v1581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3684515006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:46:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:46:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:46:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:16 smithi136 ceph-mon[113531]: pgmap v1582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/840505817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:16 smithi093 ceph-mon[118902]: pgmap v1582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/840505817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:16 smithi093 ceph-mon[111335]: pgmap v1582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/840505817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:18 smithi136 ceph-mon[113531]: pgmap v1583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:18 smithi093 ceph-mon[111335]: pgmap v1583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:18 smithi093 ceph-mon[118902]: pgmap v1583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2570434924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2570434924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2570434924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:20 smithi136 ceph-mon[113531]: pgmap v1584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:20 smithi093 ceph-mon[111335]: pgmap v1584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:20 smithi093 ceph-mon[118902]: pgmap v1584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:20.310Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:20.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2602632353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2602632353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2602632353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:22.188 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:21.834Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:22.188 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:21.835Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:22 smithi136 ceph-mon[113531]: pgmap v1585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:22 smithi093 ceph-mon[111335]: pgmap v1585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:22 smithi093 ceph-mon[118902]: pgmap v1585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2606925202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2606925202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2606925202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:23.734Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:23.735Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:24 smithi093 ceph-mon[111335]: pgmap v1586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:24 smithi093 ceph-mon[118902]: pgmap v1586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:24 smithi136 ceph-mon[113531]: pgmap v1586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:46:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:46:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:46:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:26 smithi093 ceph-mon[111335]: pgmap v1587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2965301743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:26 smithi093 ceph-mon[118902]: pgmap v1587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2965301743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:26 smithi136 ceph-mon[113531]: pgmap v1587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2965301743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:28 smithi093 ceph-mon[111335]: pgmap v1588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2198918028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:28 smithi093 ceph-mon[118902]: pgmap v1588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2198918028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:28 smithi136 ceph-mon[113531]: pgmap v1588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2198918028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:30.310 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:30 smithi093 ceph-mon[111335]: pgmap v1589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:30.311 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:30 smithi093 ceph-mon[118902]: pgmap v1589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:30 smithi136 ceph-mon[113531]: pgmap v1589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:30.310Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:30.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3661495793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3661495793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3661495793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:32 smithi093 ceph-mon[111335]: pgmap v1590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:32 smithi093 ceph-mon[118902]: pgmap v1590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:31.834Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:31.835Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:32 smithi136 ceph-mon[113531]: pgmap v1590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/84535857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/84535857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/84535857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:33.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:33.735Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:33.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:33.736Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:34 smithi136 ceph-mon[113531]: pgmap v1591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:34 smithi093 ceph-mon[111335]: pgmap v1591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:34 smithi093 ceph-mon[118902]: pgmap v1591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:35.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3149880946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3149880946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3149880946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:46:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:46:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:46:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:36 smithi136 ceph-mon[113531]: pgmap v1592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:36 smithi093 ceph-mon[111335]: pgmap v1592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:36 smithi093 ceph-mon[118902]: pgmap v1592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:38 smithi136 ceph-mon[113531]: pgmap v1593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3001596540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:38 smithi093 ceph-mon[111335]: pgmap v1593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3001596540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:38 smithi093 ceph-mon[118902]: pgmap v1593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3001596540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:40 smithi136 ceph-mon[113531]: pgmap v1594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3396576497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:40 smithi093 ceph-mon[111335]: pgmap v1594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3396576497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:40 smithi093 ceph-mon[118902]: pgmap v1594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3396576497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:40.311Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:40.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:42.162 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:41.835Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:42.162 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:41.836Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:42 smithi136 ceph-mon[113531]: pgmap v1595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:42 smithi093 ceph-mon[111335]: pgmap v1595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:42 smithi093 ceph-mon[118902]: pgmap v1595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/117159996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/117159996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/117159996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:43.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:43.736Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:43.737Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:44 smithi136 ceph-mon[113531]: pgmap v1596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:44 smithi093 ceph-mon[111335]: pgmap v1596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:44 smithi093 ceph-mon[118902]: pgmap v1596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2332338489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2332338489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2332338489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:45.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:46:45] "GET /metrics HTTP/1.1" 200 239875 "" "Prometheus/2.43.0" 2023-08-27T21:46:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:46 smithi136 ceph-mon[113531]: pgmap v1597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:46 smithi093 ceph-mon[111335]: pgmap v1597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:46 smithi093 ceph-mon[118902]: pgmap v1597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:46:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/919969908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/919969908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/919969908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:48 smithi136 ceph-mon[113531]: pgmap v1598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:48 smithi093 ceph-mon[111335]: pgmap v1598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:48 smithi093 ceph-mon[118902]: pgmap v1598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:46:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:50 smithi093 ceph-mon[111335]: pgmap v1599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111154028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:50 smithi093 ceph-mon[118902]: pgmap v1599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111154028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:50.312Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:50.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:50 smithi136 ceph-mon[113531]: pgmap v1599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111154028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:52.191 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:51.836Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:52.192 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:51.837Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:52 smithi093 ceph-mon[118902]: pgmap v1600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2808235903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:52 smithi093 ceph-mon[111335]: pgmap v1600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2808235903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:52 smithi136 ceph-mon[113531]: pgmap v1600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2808235903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:53.736Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:53.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:46:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:46:53.738Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:46:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:54 smithi093 ceph-mon[111335]: pgmap v1601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/175866328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:54 smithi093 ceph-mon[118902]: pgmap v1601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/175866328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:54 smithi136 ceph-mon[113531]: pgmap v1601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:46:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/175866328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:46:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:46:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:56 smithi093 ceph-mon[111335]: pgmap v1602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:56 smithi093 ceph-mon[118902]: pgmap v1602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:56 smithi136 ceph-mon[113531]: pgmap v1602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3275480507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3275480507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3275480507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:58 smithi093 ceph-mon[111335]: pgmap v1603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:58 smithi093 ceph-mon[118902]: pgmap v1603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:58 smithi136 ceph-mon[113531]: pgmap v1603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:46:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:46:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/17525681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:46:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/17525681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:46:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:46:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/17525681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:00 smithi093 ceph-mon[111335]: pgmap v1604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:00 smithi093 ceph-mon[118902]: pgmap v1604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:00.313Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:00.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:00 smithi136 ceph-mon[113531]: pgmap v1604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:02.212 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:01.836Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:02.213 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:01.837Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:02 smithi093 ceph-mon[111335]: pgmap v1605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2958905201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:02 smithi093 ceph-mon[118902]: pgmap v1605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2958905201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:02 smithi136 ceph-mon[113531]: pgmap v1605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2958905201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:03.738Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:04.029 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:03.739Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:04 smithi093 ceph-mon[118902]: pgmap v1606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1547104380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:04 smithi093 ceph-mon[111335]: pgmap v1606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1547104380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:04 smithi136 ceph-mon[113531]: pgmap v1606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1547104380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:47:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:47:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:47:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:06 smithi093 ceph-mon[111335]: pgmap v1607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2817254453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:06 smithi093 ceph-mon[118902]: pgmap v1607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2817254453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:06 smithi136 ceph-mon[113531]: pgmap v1607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2817254453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:08 smithi093 ceph-mon[111335]: pgmap v1608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:08 smithi093 ceph-mon[118902]: pgmap v1608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:08 smithi136 ceph-mon[113531]: pgmap v1608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/40502432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/40502432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/40502432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:10 smithi093 ceph-mon[111335]: pgmap v1609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:10 smithi093 ceph-mon[118902]: pgmap v1609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:10.314Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:10.315Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:10 smithi136 ceph-mon[113531]: pgmap v1609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/78519400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/78519400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/78519400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:12.233 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:11.837Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:12.233 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:11.838Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:12 smithi093 ceph-mon[111335]: pgmap v1610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:12 smithi093 ceph-mon[118902]: pgmap v1610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:12 smithi136 ceph-mon[113531]: pgmap v1610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1054329178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1054329178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1054329178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:13.738Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:13.739Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:14 smithi136 ceph-mon[113531]: pgmap v1611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:14 smithi093 ceph-mon[111335]: pgmap v1611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:14 smithi093 ceph-mon[118902]: pgmap v1611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:47:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:47:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:16 smithi136 ceph-mon[113531]: pgmap v1612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/930195762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:16 smithi093 ceph-mon[111335]: pgmap v1612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/930195762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:16 smithi093 ceph-mon[118902]: pgmap v1612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/930195762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:18 smithi136 ceph-mon[113531]: pgmap v1613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3430296318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:18 smithi093 ceph-mon[111335]: pgmap v1613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3430296318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:18 smithi093 ceph-mon[118902]: pgmap v1613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3430296318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:20.315 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:20 smithi093 ceph-mon[111335]: pgmap v1614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:20.315 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:20 smithi093 ceph-mon[118902]: pgmap v1614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:20 smithi136 ceph-mon[113531]: pgmap v1614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:20.315Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:20.316Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3875875639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3875875639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3875875639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:22.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:22 smithi093 ceph-mon[118902]: pgmap v1615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:22 smithi093 ceph-mon[111335]: pgmap v1615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:21.837Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:21.838Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:22 smithi136 ceph-mon[113531]: pgmap v1615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3397423791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3397423791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3397423791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:23.739Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:23.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:23.740Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:24 smithi136 ceph-mon[113531]: pgmap v1616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:24 smithi093 ceph-mon[111335]: pgmap v1616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:24 smithi093 ceph-mon[118902]: pgmap v1616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/288989169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/288989169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:47:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:47:25] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:47:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/288989169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:26 smithi136 ceph-mon[113531]: pgmap v1617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:26 smithi093 ceph-mon[111335]: pgmap v1617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:26 smithi093 ceph-mon[118902]: pgmap v1617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:28 smithi136 ceph-mon[113531]: pgmap v1618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1554093667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:28 smithi093 ceph-mon[111335]: pgmap v1618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1554093667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:28 smithi093 ceph-mon[118902]: pgmap v1618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1554093667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:30.317 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:30 smithi093 ceph-mon[111335]: pgmap v1619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:30.317 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3492947490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:30.317 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:30 smithi093 ceph-mon[118902]: pgmap v1619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:30.318 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3492947490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:30 smithi136 ceph-mon[113531]: pgmap v1619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3492947490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:30.316Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:30.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:32.170 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:31.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:32.170 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:31.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:32 smithi136 ceph-mon[113531]: pgmap v1620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:32 smithi093 ceph-mon[111335]: pgmap v1620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:32 smithi093 ceph-mon[118902]: pgmap v1620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/215082915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/215082915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/215082915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:34.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:33.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:34.180 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:33.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:34 smithi136 ceph-mon[113531]: pgmap v1621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:34 smithi093 ceph-mon[111335]: pgmap v1621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:34 smithi093 ceph-mon[118902]: pgmap v1621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2508116304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2508116304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2508116304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:47:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:47:35] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:47:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:36 smithi136 ceph-mon[113531]: pgmap v1622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:36 smithi093 ceph-mon[111335]: pgmap v1622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:36 smithi093 ceph-mon[118902]: pgmap v1622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1290227129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1290227129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1290227129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:38 smithi093 ceph-mon[111335]: pgmap v1623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:38 smithi093 ceph-mon[118902]: pgmap v1623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:38 smithi136 ceph-mon[113531]: pgmap v1623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:40 smithi093 ceph-mon[118902]: pgmap v1624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3342652333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:40 smithi093 ceph-mon[111335]: pgmap v1624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3342652333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:40.317Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:40.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:40 smithi136 ceph-mon[113531]: pgmap v1624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3342652333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:42.204 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:41.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:42.204 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:41.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:42 smithi093 ceph-mon[118902]: pgmap v1625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1052529287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:42 smithi093 ceph-mon[111335]: pgmap v1625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1052529287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:42 smithi136 ceph-mon[113531]: pgmap v1625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1052529287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:44.209 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:43.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:44.209 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:43.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:44 smithi093 ceph-mon[111335]: pgmap v1626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2513472467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:44 smithi093 ceph-mon[118902]: pgmap v1626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2513472467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:44 smithi136 ceph-mon[113531]: pgmap v1626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2513472467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:47:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:47:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:47:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:46 smithi093 ceph-mon[111335]: pgmap v1627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:46 smithi093 ceph-mon[118902]: pgmap v1627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:46 smithi136 ceph-mon[113531]: pgmap v1627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1133138924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1133138924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1133138924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:48 smithi093 ceph-mon[111335]: pgmap v1628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:48 smithi093 ceph-mon[118902]: pgmap v1628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:48 smithi136 ceph-mon[113531]: pgmap v1628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3448656811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3448656811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:47:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3448656811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:50.317 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:50 smithi093 ceph-mon[111335]: pgmap v1629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:50.318 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:50 smithi093 ceph-mon[118902]: pgmap v1629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:50 smithi136 ceph-mon[113531]: pgmap v1629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:50.317Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:50.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:51.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:51.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:52 smithi136 ceph-mon[113531]: pgmap v1630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3149217795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[111335]: pgmap v1630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3149217795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[118902]: pgmap v1630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3149217795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:47:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:47:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:53.741Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:47:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:47:53.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:47:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:54 smithi136 ceph-mon[113531]: pgmap v1631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3933070601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:54 smithi093 ceph-mon[111335]: pgmap v1631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3933070601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:54 smithi093 ceph-mon[118902]: pgmap v1631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:47:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3933070601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:47:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:47:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:47:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:56 smithi136 ceph-mon[113531]: pgmap v1632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/808477487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:56 smithi093 ceph-mon[111335]: pgmap v1632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/808477487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:56 smithi093 ceph-mon[118902]: pgmap v1632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/808477487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:58 smithi136 ceph-mon[113531]: pgmap v1633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:58 smithi093 ceph-mon[111335]: pgmap v1633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:58 smithi093 ceph-mon[118902]: pgmap v1633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:47:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:47:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/186059581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:47:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/186059581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:47:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:47:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/186059581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:00 smithi136 ceph-mon[113531]: pgmap v1634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:00 smithi093 ceph-mon[111335]: pgmap v1634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:00 smithi093 ceph-mon[118902]: pgmap v1634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:00.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:00.319Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4048491867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4048491867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4048491867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:01.839Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:01.840Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:02 smithi136 ceph-mon[113531]: pgmap v1635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:02 smithi093 ceph-mon[111335]: pgmap v1635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:02.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:02 smithi093 ceph-mon[118902]: pgmap v1635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2690587836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2690587836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2690587836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:03.741Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:03.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:04 smithi136 ceph-mon[113531]: pgmap v1636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:04 smithi093 ceph-mon[111335]: pgmap v1636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:04 smithi093 ceph-mon[118902]: pgmap v1636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:48:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:48:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:48:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:06 smithi136 ceph-mon[113531]: pgmap v1637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3644630301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:06 smithi093 ceph-mon[111335]: pgmap v1637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3644630301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:06 smithi093 ceph-mon[118902]: pgmap v1637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3644630301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:08 smithi136 ceph-mon[113531]: pgmap v1638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/642106094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:08 smithi093 ceph-mon[111335]: pgmap v1638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/642106094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:08 smithi093 ceph-mon[118902]: pgmap v1638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/642106094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:10 smithi136 ceph-mon[113531]: pgmap v1639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:10 smithi093 ceph-mon[111335]: pgmap v1639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:10 smithi093 ceph-mon[118902]: pgmap v1639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:10.319Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:10.320Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/550885677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/550885677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/550885677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:11.839Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:11.840Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:12 smithi136 ceph-mon[113531]: pgmap v1640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:12 smithi093 ceph-mon[111335]: pgmap v1640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:12 smithi093 ceph-mon[118902]: pgmap v1640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/174538549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/174538549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/174538549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:13.741Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:13.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:14 smithi136 ceph-mon[113531]: pgmap v1641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:14 smithi093 ceph-mon[111335]: pgmap v1641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:14 smithi093 ceph-mon[118902]: pgmap v1641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3828149955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3828149955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:48:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:48:15] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:48:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3828149955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:16 smithi136 ceph-mon[113531]: pgmap v1642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:16 smithi093 ceph-mon[111335]: pgmap v1642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:16 smithi093 ceph-mon[118902]: pgmap v1642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:18 smithi136 ceph-mon[113531]: pgmap v1643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2699219745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:18 smithi093 ceph-mon[111335]: pgmap v1643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2699219745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:18 smithi093 ceph-mon[118902]: pgmap v1643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2699219745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:20.319 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:20 smithi093 ceph-mon[118902]: pgmap v1644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:20.319 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2220232843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:20.320 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:20 smithi093 ceph-mon[111335]: pgmap v1644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:20.320 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2220232843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:20 smithi136 ceph-mon[113531]: pgmap v1644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2220232843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:20.319Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:20.320Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:22.181 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:21.840Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:22.182 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:21.841Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:22 smithi136 ceph-mon[113531]: pgmap v1645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:22 smithi093 ceph-mon[111335]: pgmap v1645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:22 smithi093 ceph-mon[118902]: pgmap v1645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1585480865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1585480865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1585480865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:24.193 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:23.741Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:24.193 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:23.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:24 smithi093 ceph-mon[111335]: pgmap v1646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:24 smithi093 ceph-mon[118902]: pgmap v1646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:24 smithi136 ceph-mon[113531]: pgmap v1646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/637711930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:48:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:48:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:48:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/637711930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/637711930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:26 smithi093 ceph-mon[111335]: pgmap v1647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:26 smithi093 ceph-mon[118902]: pgmap v1647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:26 smithi136 ceph-mon[113531]: pgmap v1647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1370416526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1370416526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1370416526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:28 smithi093 ceph-mon[111335]: pgmap v1648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:28 smithi093 ceph-mon[118902]: pgmap v1648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:28 smithi136 ceph-mon[113531]: pgmap v1648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:30.320 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:30 smithi093 ceph-mon[111335]: pgmap v1649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:30.320 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3227138596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:30.321 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:30 smithi093 ceph-mon[118902]: pgmap v1649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:30.321 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3227138596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:30 smithi136 ceph-mon[113531]: pgmap v1649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3227138596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:30.320Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:30.321Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:32.184 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:31.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:32.184 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:31.842Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:32 smithi136 ceph-mon[113531]: pgmap v1650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1570318669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:32 smithi093 ceph-mon[111335]: pgmap v1650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1570318669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:32 smithi093 ceph-mon[118902]: pgmap v1650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1570318669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:34.189 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:33.742Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:34.190 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:33.743Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:34 smithi093 ceph-mon[111335]: pgmap v1651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:34 smithi093 ceph-mon[118902]: pgmap v1651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:34 smithi136 ceph-mon[113531]: pgmap v1651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1513470961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1513470961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:48:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:48:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:48:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1513470961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:36 smithi093 ceph-mon[111335]: pgmap v1652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:36 smithi093 ceph-mon[118902]: pgmap v1652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:36 smithi136 ceph-mon[113531]: pgmap v1652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/236776039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/236776039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/236776039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:38 smithi093 ceph-mon[111335]: pgmap v1653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:38 smithi093 ceph-mon[118902]: pgmap v1653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:38 smithi136 ceph-mon[113531]: pgmap v1653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/633541801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/633541801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/633541801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:40 smithi093 ceph-mon[111335]: pgmap v1654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:40 smithi093 ceph-mon[118902]: pgmap v1654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:40.321Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:40.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:40 smithi136 ceph-mon[113531]: pgmap v1654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:42.208 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:41.842Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:42.208 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:41.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:42 smithi093 ceph-mon[111335]: pgmap v1655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1470503725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:42 smithi093 ceph-mon[118902]: pgmap v1655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1470503725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:42 smithi136 ceph-mon[113531]: pgmap v1655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1470503725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:44.211 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:43.743Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:44.211 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:43.744Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:44 smithi093 ceph-mon[111335]: pgmap v1656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1759768699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:44 smithi093 ceph-mon[118902]: pgmap v1656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1759768699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:44 smithi136 ceph-mon[113531]: pgmap v1656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1759768699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:48:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:48:45] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:48:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:46 smithi093 ceph-mon[111335]: pgmap v1657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2979746717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:46 smithi093 ceph-mon[118902]: pgmap v1657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2979746717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:46 smithi136 ceph-mon[113531]: pgmap v1657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2979746717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:48 smithi093 ceph-mon[111335]: pgmap v1658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:48 smithi093 ceph-mon[118902]: pgmap v1658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:48 smithi136 ceph-mon[113531]: pgmap v1658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/482432357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/482432357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/482432357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:50 smithi093 ceph-mon[111335]: pgmap v1659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:50 smithi093 ceph-mon[118902]: pgmap v1659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:50.322Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:50.323Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:50 smithi136 ceph-mon[113531]: pgmap v1659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1482066281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1482066281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1482066281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:52.230 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:51.843Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:52.230 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:51.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:52 smithi093 ceph-mon[118902]: pgmap v1660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:52 smithi093 ceph-mon[111335]: pgmap v1660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:52 smithi136 ceph-mon[113531]: pgmap v1660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:48:54.233 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:53.744Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:54.234 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:48:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:48:53.745Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:48:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:54 smithi136 ceph-mon[113531]: pgmap v1661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/370632248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[118902]: pgmap v1661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/370632248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[111335]: pgmap v1661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/370632248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:48:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:48:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:48:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:48:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:48:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:56 smithi136 ceph-mon[113531]: pgmap v1662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/295994193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:56 smithi093 ceph-mon[111335]: pgmap v1662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/295994193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:56 smithi093 ceph-mon[118902]: pgmap v1662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/295994193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:58 smithi136 ceph-mon[113531]: pgmap v1663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:48:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2276025197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:58 smithi093 ceph-mon[118902]: pgmap v1663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:48:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2276025197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:48:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:58 smithi093 ceph-mon[111335]: pgmap v1663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:48:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:48:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2276025197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:00.322 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:00 smithi093 ceph-mon[111335]: pgmap v1664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:00.323 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:00 smithi093 ceph-mon[118902]: pgmap v1664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:00 smithi136 ceph-mon[113531]: pgmap v1664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:00.322Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:00.323Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3861728285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3861728285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3861728285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:02 smithi093 ceph-mon[111335]: pgmap v1665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:02 smithi093 ceph-mon[118902]: pgmap v1665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:01.843Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:01.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:02 smithi136 ceph-mon[113531]: pgmap v1665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/490711426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/490711426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/490711426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:04.087 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:03.745Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:04.087 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:03.746Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:04 smithi136 ceph-mon[113531]: pgmap v1666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:04 smithi093 ceph-mon[111335]: pgmap v1666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:04 smithi093 ceph-mon[118902]: pgmap v1666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:49:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:49:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:49:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:06 smithi093 ceph-mon[111335]: pgmap v1667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4228924280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:06 smithi093 ceph-mon[118902]: pgmap v1667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4228924280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:06 smithi136 ceph-mon[113531]: pgmap v1667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4228924280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:08 smithi093 ceph-mon[111335]: pgmap v1668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1298520414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:08 smithi093 ceph-mon[118902]: pgmap v1668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1298520414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:08 smithi136 ceph-mon[113531]: pgmap v1668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1298520414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:10 smithi093 ceph-mon[111335]: pgmap v1669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:10.505 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3237996345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:10.506 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:10 smithi093 ceph-mon[118902]: pgmap v1669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:10.506 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3237996345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:10.506 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:10.323Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:10.506 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:10.324Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:10 smithi136 ceph-mon[113531]: pgmap v1669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3237996345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:12.200 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:11.843Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:12.201 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:11.844Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:12 smithi093 ceph-mon[111335]: pgmap v1670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:12 smithi093 ceph-mon[118902]: pgmap v1670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:12 smithi136 ceph-mon[113531]: pgmap v1670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1650203297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1650203297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1650203297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:14.207 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:13.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:14.208 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:13.748Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:14 smithi093 ceph-mon[111335]: pgmap v1671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:14 smithi093 ceph-mon[118902]: pgmap v1671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:14 smithi136 ceph-mon[113531]: pgmap v1671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/106782082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/106782082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:49:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:49:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:49:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/106782082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:16 smithi093 ceph-mon[111335]: pgmap v1672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:16 smithi093 ceph-mon[118902]: pgmap v1672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:16 smithi136 ceph-mon[113531]: pgmap v1672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3127099524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3127099524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3127099524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:18 smithi093 ceph-mon[111335]: pgmap v1673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:18 smithi093 ceph-mon[118902]: pgmap v1673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:18 smithi136 ceph-mon[113531]: pgmap v1673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:20.326 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:20 smithi093 ceph-mon[111335]: pgmap v1674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:20.326 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/879245865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:20.326 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:20 smithi093 ceph-mon[118902]: pgmap v1674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:20.326 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/879245865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:20.327 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:20.324Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:20 smithi136 ceph-mon[113531]: pgmap v1674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/879245865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:22.197 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:20.325Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:22.197 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:21.844Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:22.197 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:21.845Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:22 smithi093 ceph-mon[111335]: pgmap v1675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1676818954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:22 smithi093 ceph-mon[118902]: pgmap v1675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1676818954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:22 smithi136 ceph-mon[113531]: pgmap v1675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1676818954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:24.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:23.747Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:24.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:23.748Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:24 smithi093 ceph-mon[111335]: pgmap v1676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:24 smithi093 ceph-mon[118902]: pgmap v1676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:24 smithi136 ceph-mon[113531]: pgmap v1676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3327504031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3327504031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:25.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:49:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:49:25] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:49:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3327504031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:26 smithi093 ceph-mon[111335]: pgmap v1677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:26.519 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:26 smithi093 ceph-mon[118902]: pgmap v1677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:26 smithi136 ceph-mon[113531]: pgmap v1677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/536744013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/536744013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/536744013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:28 smithi093 ceph-mon[111335]: pgmap v1678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:28 smithi093 ceph-mon[118902]: pgmap v1678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:28 smithi136 ceph-mon[113531]: pgmap v1678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1709926102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:30 smithi136 ceph-mon[113531]: pgmap v1679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1709926102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:30 smithi093 ceph-mon[111335]: pgmap v1679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1709926102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:30 smithi093 ceph-mon[118902]: pgmap v1679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:30.325Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:30.326Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:32.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:31.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:32.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:31.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:32 smithi093 ceph-mon[111335]: pgmap v1680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3114589377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:32 smithi093 ceph-mon[118902]: pgmap v1680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3114589377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:32 smithi136 ceph-mon[113531]: pgmap v1680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3114589377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:34.201 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:33.748Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:34.202 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:33.749Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:34 smithi093 ceph-mon[111335]: pgmap v1681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4244544438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:34 smithi093 ceph-mon[118902]: pgmap v1681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4244544438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:34 smithi136 ceph-mon[113531]: pgmap v1681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4244544438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:49:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:49:35] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:49:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:36 smithi093 ceph-mon[111335]: pgmap v1682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:36 smithi093 ceph-mon[118902]: pgmap v1682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:36 smithi136 ceph-mon[113531]: pgmap v1682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3423064250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3423064250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3423064250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:38 smithi093 ceph-mon[111335]: pgmap v1683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:38 smithi093 ceph-mon[118902]: pgmap v1683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:38 smithi136 ceph-mon[113531]: pgmap v1683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4145580080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4145580080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4145580080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:40 smithi093 ceph-mon[111335]: pgmap v1684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:40.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:40 smithi093 ceph-mon[118902]: pgmap v1684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:40.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:40.325Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:40.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:40.326Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:40 smithi136 ceph-mon[113531]: pgmap v1684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1606624318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1606624318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1606624318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:42.224 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:41.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:42.225 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:41.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:42 smithi093 ceph-mon[111335]: pgmap v1685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:42 smithi093 ceph-mon[118902]: pgmap v1685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:42 smithi136 ceph-mon[113531]: pgmap v1685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:44.227 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:43.749Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:44.228 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:43.750Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:44 smithi093 ceph-mon[111335]: pgmap v1686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3489543542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:44 smithi093 ceph-mon[118902]: pgmap v1686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3489543542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:44 smithi136 ceph-mon[113531]: pgmap v1686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3489543542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:49:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:49:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:49:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:46 smithi093 ceph-mon[118902]: pgmap v1687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3509924518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:46 smithi093 ceph-mon[111335]: pgmap v1687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3509924518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:46 smithi136 ceph-mon[113531]: pgmap v1687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3509924518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:48 smithi093 ceph-mon[111335]: pgmap v1688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1892118592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:48 smithi093 ceph-mon[118902]: pgmap v1688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1892118592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:48 smithi136 ceph-mon[113531]: pgmap v1688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1892118592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:50 smithi136 ceph-mon[113531]: pgmap v1689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:50 smithi093 ceph-mon[111335]: pgmap v1689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:50 smithi093 ceph-mon[118902]: pgmap v1689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:50.326Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:50.327Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2503216660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2503216660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2503216660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:52 smithi093 ceph-mon[111335]: pgmap v1690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:52 smithi093 ceph-mon[118902]: pgmap v1690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:51.846Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:51.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:52 smithi136 ceph-mon[113531]: pgmap v1690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1879154014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1879154014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1879154014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:54.100 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:53.749Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:54.101 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:49:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:49:53.750Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:49:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:54 smithi136 ceph-mon[113531]: pgmap v1691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:54 smithi093 ceph-mon[111335]: pgmap v1691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:54 smithi093 ceph-mon[118902]: pgmap v1691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:49:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:49:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:49:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:49:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:49:56.670 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[111335]: pgmap v1692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:56.671 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:56.671 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:56.671 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1788305220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:56.671 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[118902]: pgmap v1692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:56.671 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:56.671 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:56.671 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1788305220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:56 smithi136 ceph-mon[113531]: pgmap v1692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1788305220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[111335]: pgmap v1693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2144856751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[118902]: pgmap v1693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:49:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2144856751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:49:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:57 smithi136 ceph-mon[113531]: pgmap v1693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:49:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:49:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:49:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2144856751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: pgmap v1694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:50:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1030970182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:00.326Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:00.327Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: pgmap v1694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1030970182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: pgmap v1694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:50:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T21:50:00.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1030970182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:01.846Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:01.847Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:02 smithi136 ceph-mon[113531]: pgmap v1695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:02 smithi093 ceph-mon[111335]: pgmap v1695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:02 smithi093 ceph-mon[118902]: pgmap v1695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/654182319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/654182319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/654182319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:03.750Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:03.752Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:04 smithi136 ceph-mon[113531]: pgmap v1696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:04 smithi093 ceph-mon[111335]: pgmap v1696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:04 smithi093 ceph-mon[118902]: pgmap v1696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1990446389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1990446389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:50:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:50:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:50:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1990446389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:06 smithi136 ceph-mon[113531]: pgmap v1697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:06 smithi093 ceph-mon[111335]: pgmap v1697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:06 smithi093 ceph-mon[118902]: pgmap v1697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4026028755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4026028755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4026028755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:08 smithi136 ceph-mon[113531]: pgmap v1698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:08 smithi093 ceph-mon[111335]: pgmap v1698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:08 smithi093 ceph-mon[118902]: pgmap v1698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:10 smithi136 ceph-mon[113531]: pgmap v1699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3995971599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:10 smithi093 ceph-mon[111335]: pgmap v1699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3995971599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:10 smithi093 ceph-mon[118902]: pgmap v1699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3995971599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:10.327Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:10.328Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:11.847Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:11.848Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:12 smithi136 ceph-mon[113531]: pgmap v1700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2558846769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:12 smithi093 ceph-mon[111335]: pgmap v1700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2558846769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:12 smithi093 ceph-mon[118902]: pgmap v1700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2558846769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:13.751Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:13.752Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:14 smithi136 ceph-mon[113531]: pgmap v1701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:14 smithi093 ceph-mon[111335]: pgmap v1701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:14 smithi093 ceph-mon[118902]: pgmap v1701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1780748216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1780748216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1780748216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:50:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:50:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:50:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:16 smithi136 ceph-mon[113531]: pgmap v1702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:16 smithi093 ceph-mon[111335]: pgmap v1702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:16 smithi093 ceph-mon[118902]: pgmap v1702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3946775757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3946775757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3946775757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:18 smithi136 ceph-mon[113531]: pgmap v1703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:18 smithi093 ceph-mon[111335]: pgmap v1703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:18 smithi093 ceph-mon[118902]: pgmap v1703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:20 smithi136 ceph-mon[113531]: pgmap v1704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3621715450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:20 smithi093 ceph-mon[111335]: pgmap v1704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3621715450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:20 smithi093 ceph-mon[118902]: pgmap v1704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3621715450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:20.328Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:20.329Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:22.219 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:21.848Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:22.219 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:21.849Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:22 smithi093 ceph-mon[111335]: pgmap v1705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3555463159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:22 smithi093 ceph-mon[118902]: pgmap v1705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3555463159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:22 smithi136 ceph-mon[113531]: pgmap v1705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3555463159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:24.214 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:23.752Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:24.214 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:23.753Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:24 smithi093 ceph-mon[111335]: pgmap v1706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2659602383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:24 smithi093 ceph-mon[118902]: pgmap v1706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2659602383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:24 smithi136 ceph-mon[113531]: pgmap v1706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2659602383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:50:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:50:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:50:26.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:26 smithi093 ceph-mon[111335]: pgmap v1707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:26.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:26 smithi093 ceph-mon[118902]: pgmap v1707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:26 smithi136 ceph-mon[113531]: pgmap v1707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3806688614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3806688614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3806688614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:28 smithi093 ceph-mon[111335]: pgmap v1708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:28 smithi093 ceph-mon[118902]: pgmap v1708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:28 smithi136 ceph-mon[113531]: pgmap v1708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3990138733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3990138733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3990138733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:30 smithi093 ceph-mon[111335]: pgmap v1709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:30 smithi093 ceph-mon[118902]: pgmap v1709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:30.329Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:30.330Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:30 smithi136 ceph-mon[113531]: pgmap v1709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3186412945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3186412945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3186412945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:32.234 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:31.849Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:32.234 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:31.850Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:32 smithi093 ceph-mon[111335]: pgmap v1710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:32 smithi093 ceph-mon[118902]: pgmap v1710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:32 smithi136 ceph-mon[113531]: pgmap v1710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:34.236 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:33.754Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:34.236 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:33.754Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:34 smithi093 ceph-mon[118902]: pgmap v1711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1256718926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:34 smithi093 ceph-mon[111335]: pgmap v1711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1256718926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:34 smithi136 ceph-mon[113531]: pgmap v1711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1256718926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:50:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:50:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:36 smithi093 ceph-mon[111335]: pgmap v1712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/332832736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:36 smithi093 ceph-mon[118902]: pgmap v1712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/332832736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:36 smithi136 ceph-mon[113531]: pgmap v1712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/332832736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:38 smithi136 ceph-mon[113531]: pgmap v1713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:38 smithi093 ceph-mon[111335]: pgmap v1713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:38 smithi093 ceph-mon[118902]: pgmap v1713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3385679541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3385679541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:39.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:39.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3385679541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:39.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:40 smithi136 ceph-mon[113531]: pgmap v1714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:40 smithi093 ceph-mon[111335]: pgmap v1714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:40 smithi093 ceph-mon[118902]: pgmap v1714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:40.330Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:40.331Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3514529820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3514529820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3514529820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:41.850Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:41.851Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:42 smithi136 ceph-mon[113531]: pgmap v1715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:42 smithi093 ceph-mon[111335]: pgmap v1715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:42 smithi093 ceph-mon[118902]: pgmap v1715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3141432967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3141432967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3141432967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:43.754Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:43.755Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:44 smithi136 ceph-mon[113531]: pgmap v1716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:44 smithi093 ceph-mon[111335]: pgmap v1716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:44 smithi093 ceph-mon[118902]: pgmap v1716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:50:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:50:45] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:50:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:46 smithi136 ceph-mon[113531]: pgmap v1717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1863369033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:46 smithi093 ceph-mon[111335]: pgmap v1717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1863369033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:46 smithi093 ceph-mon[118902]: pgmap v1717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1863369033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:48 smithi136 ceph-mon[113531]: pgmap v1718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/963968343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:48 smithi093 ceph-mon[111335]: pgmap v1718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/963968343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:48 smithi093 ceph-mon[118902]: pgmap v1718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/963968343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:50 smithi136 ceph-mon[113531]: pgmap v1719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:50 smithi093 ceph-mon[111335]: pgmap v1719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:50 smithi093 ceph-mon[118902]: pgmap v1719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:50.331Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:50.332Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/786654679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/786654679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/786654679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:52 smithi093 ceph-mon[111335]: pgmap v1720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:52 smithi093 ceph-mon[118902]: pgmap v1720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:51.850Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:51.851Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:52 smithi136 ceph-mon[113531]: pgmap v1720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2940163207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2940163207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2940163207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:54.102 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:53.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:54.102 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:50:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:50:53.756Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:50:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:54 smithi136 ceph-mon[113531]: pgmap v1721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:54 smithi093 ceph-mon[111335]: pgmap v1721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:54 smithi093 ceph-mon[118902]: pgmap v1721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:50:55.363 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1907617377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:55.363 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1907617377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1907617377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:50:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:50:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:50:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:56 smithi136 ceph-mon[113531]: pgmap v1722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:56 smithi093 ceph-mon[111335]: pgmap v1722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:56 smithi093 ceph-mon[118902]: pgmap v1722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:58 smithi093 ceph-mon[111335]: pgmap v1723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:50:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2031963032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:58 smithi093 ceph-mon[118902]: pgmap v1723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:50:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2031963032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:50:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:58 smithi136 ceph-mon[113531]: pgmap v1723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:50:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:50:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2031963032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:00.333 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[111335]: pgmap v1724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:00.333 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:00.333 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2780498903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:00.333 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:00.334 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:00.334 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[118902]: pgmap v1724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:00.334 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:00.334 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2780498903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:00.334 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:00.334 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:00.334 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:00.332Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:00.335 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:00.334Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:00.368 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:00 smithi136 ceph-mon[113531]: pgmap v1724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:00.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:51:00.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2780498903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:00.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:00.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:02.147 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:01.851Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:02.147 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:01.852Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:02 smithi136 ceph-mon[113531]: pgmap v1725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/600276197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[111335]: pgmap v1725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/600276197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[118902]: pgmap v1725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/600276197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:51:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:51:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:51:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:03.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:03.756Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:04 smithi136 ceph-mon[113531]: pgmap v1726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:04 smithi093 ceph-mon[111335]: pgmap v1726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:04 smithi093 ceph-mon[118902]: pgmap v1726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2837774233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 21:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T21:51:05.318516453Z level=info msg="Completed cleanup jobs" duration=13.16967ms 2023-08-27T21:51:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2837774233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2837774233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:51:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:51:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T21:51:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:06 smithi136 ceph-mon[113531]: pgmap v1727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:06 smithi093 ceph-mon[111335]: pgmap v1727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:06 smithi093 ceph-mon[118902]: pgmap v1727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1337473780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1337473780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1337473780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:08 smithi136 ceph-mon[113531]: pgmap v1728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:08 smithi093 ceph-mon[111335]: pgmap v1728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:08 smithi093 ceph-mon[118902]: pgmap v1728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2680736465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2680736465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2680736465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:10 smithi136 ceph-mon[113531]: pgmap v1729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:10 smithi093 ceph-mon[111335]: pgmap v1729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:10 smithi093 ceph-mon[118902]: pgmap v1729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:10.334Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:10.335Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:11.851Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:11.852Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:12 smithi136 ceph-mon[113531]: pgmap v1730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1169256842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:12 smithi093 ceph-mon[118902]: pgmap v1730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1169256842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:12 smithi093 ceph-mon[111335]: pgmap v1730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1169256842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:13.756Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:13.757Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:14 smithi136 ceph-mon[113531]: pgmap v1731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2207923443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:14 smithi093 ceph-mon[111335]: pgmap v1731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2207923443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:14 smithi093 ceph-mon[118902]: pgmap v1731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2207923443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:51:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:51:15] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:51:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:16 smithi136 ceph-mon[113531]: pgmap v1732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:16 smithi093 ceph-mon[111335]: pgmap v1732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:16 smithi093 ceph-mon[118902]: pgmap v1732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2140401591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2140401591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2140401591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:18 smithi136 ceph-mon[113531]: pgmap v1733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:18 smithi093 ceph-mon[111335]: pgmap v1733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:18 smithi093 ceph-mon[118902]: pgmap v1733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/562061168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/562061168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/562061168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:20 smithi136 ceph-mon[113531]: pgmap v1734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:20 smithi093 ceph-mon[111335]: pgmap v1734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:20 smithi093 ceph-mon[118902]: pgmap v1734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:20.334Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:20.335Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/643769640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/643769640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/643769640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:22.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:21.852Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:22.297 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:21.853Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:22 smithi136 ceph-mon[113531]: pgmap v1735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:22 smithi093 ceph-mon[111335]: pgmap v1735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:22 smithi093 ceph-mon[118902]: pgmap v1735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:23.757Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:23.758Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:24 smithi136 ceph-mon[113531]: pgmap v1736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1945226567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:24 smithi093 ceph-mon[111335]: pgmap v1736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1945226567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:24 smithi093 ceph-mon[118902]: pgmap v1736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1945226567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:51:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:51:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:51:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:26 smithi136 ceph-mon[113531]: pgmap v1737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2909458195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:26 smithi093 ceph-mon[111335]: pgmap v1737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2909458195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:26 smithi093 ceph-mon[118902]: pgmap v1737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2909458195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:28 smithi136 ceph-mon[113531]: pgmap v1738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/46839551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:28 smithi093 ceph-mon[111335]: pgmap v1738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/46839551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:28 smithi093 ceph-mon[118902]: pgmap v1738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/46839551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:30 smithi136 ceph-mon[113531]: pgmap v1739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:30 smithi093 ceph-mon[111335]: pgmap v1739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:30.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:30 smithi093 ceph-mon[118902]: pgmap v1739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:30.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:30.335Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:30.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:30.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3476262409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3476262409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3476262409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:32 smithi093 ceph-mon[111335]: pgmap v1740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:32 smithi093 ceph-mon[118902]: pgmap v1740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:31.853Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:31.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:32 smithi136 ceph-mon[113531]: pgmap v1740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1004048234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1004048234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1004048234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:34.108 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:33.758Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:34.108 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:33.759Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:34 smithi136 ceph-mon[113531]: pgmap v1741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:34 smithi093 ceph-mon[111335]: pgmap v1741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:34 smithi093 ceph-mon[118902]: pgmap v1741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:51:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:51:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:51:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:36 smithi093 ceph-mon[118902]: pgmap v1742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/819012972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:36 smithi093 ceph-mon[111335]: pgmap v1742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/819012972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:36 smithi136 ceph-mon[113531]: pgmap v1742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/819012972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:38 smithi093 ceph-mon[111335]: pgmap v1743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2222319782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:38 smithi093 ceph-mon[118902]: pgmap v1743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2222319782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:38 smithi136 ceph-mon[113531]: pgmap v1743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2222319782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:40 smithi093 ceph-mon[111335]: pgmap v1744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2873898178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:40 smithi093 ceph-mon[118902]: pgmap v1744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2873898178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:40.336Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:40.337Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:40 smithi136 ceph-mon[113531]: pgmap v1744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2873898178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:41.854Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:41.855Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:42 smithi136 ceph-mon[113531]: pgmap v1745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:42 smithi093 ceph-mon[111335]: pgmap v1745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:42 smithi093 ceph-mon[118902]: pgmap v1745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2220738701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2220738701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2220738701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:43.759Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:43.761Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:44 smithi136 ceph-mon[113531]: pgmap v1746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:44 smithi093 ceph-mon[111335]: pgmap v1746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:44 smithi093 ceph-mon[118902]: pgmap v1746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3541663475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3541663475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3541663475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:51:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:51:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:51:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:46 smithi136 ceph-mon[113531]: pgmap v1747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:46 smithi093 ceph-mon[111335]: pgmap v1747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:46 smithi093 ceph-mon[118902]: pgmap v1747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:48 smithi136 ceph-mon[113531]: pgmap v1748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1182210497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:48 smithi093 ceph-mon[111335]: pgmap v1748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1182210497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:48 smithi093 ceph-mon[118902]: pgmap v1748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1182210497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:50 smithi136 ceph-mon[113531]: pgmap v1749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/973094515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:50 smithi093 ceph-mon[111335]: pgmap v1749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/973094515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:50 smithi093 ceph-mon[118902]: pgmap v1749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/973094515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:50.336Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:50.337Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:52.234 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:51.854Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:52.234 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:51.855Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:52 smithi093 ceph-mon[118902]: pgmap v1750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2445753382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:52 smithi093 ceph-mon[111335]: pgmap v1750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2445753382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:52 smithi136 ceph-mon[113531]: pgmap v1750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2445753382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:54.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:53.760Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:51:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:51:53.760Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:51:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:54 smithi093 ceph-mon[118902]: pgmap v1751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:54 smithi093 ceph-mon[111335]: pgmap v1751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:54 smithi136 ceph-mon[113531]: pgmap v1751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:51:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1974695550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1974695550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:51:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:51:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:51:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1974695550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:56 smithi136 ceph-mon[113531]: pgmap v1752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:56 smithi093 ceph-mon[111335]: pgmap v1752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:56 smithi093 ceph-mon[118902]: pgmap v1752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2144363756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2144363756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2144363756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:51:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:51:58 smithi136 ceph-mon[113531]: pgmap v1753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:51:58 smithi093 ceph-mon[111335]: pgmap v1753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:51:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:51:58 smithi093 ceph-mon[118902]: pgmap v1753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:00 smithi136 ceph-mon[113531]: pgmap v1754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3400013897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:00 smithi093 ceph-mon[111335]: pgmap v1754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3400013897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:00 smithi093 ceph-mon[118902]: pgmap v1754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3400013897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:00.337Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:00.338Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:02.236 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:01.854Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:02.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:01.855Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:02 smithi093 ceph-mon[111335]: pgmap v1755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1168327525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:02 smithi093 ceph-mon[118902]: pgmap v1755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1168327525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:02 smithi136 ceph-mon[113531]: pgmap v1755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1168327525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:03.501 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:03.504 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:03.504 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:52:04.099 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:03.760Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:04.099 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:03.761Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:04.425 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:04 smithi093 ceph-mon[111335]: pgmap v1756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:04.426 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3621457686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:04.426 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:04 smithi093 ceph-mon[118902]: pgmap v1756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:04.426 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3621457686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:04 smithi136 ceph-mon[113531]: pgmap v1756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3621457686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:05.672 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:52:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:52:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:05.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:06 smithi136 ceph-mon[113531]: pgmap v1757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2741330807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[111335]: pgmap v1757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2741330807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[118902]: pgmap v1757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:52:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:52:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:52:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2741330807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:08 smithi136 ceph-mon[113531]: pgmap v1758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:08 smithi093 ceph-mon[111335]: pgmap v1758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:08 smithi093 ceph-mon[118902]: pgmap v1758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3531434497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3531434497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3531434497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:10.683 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:10.337Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:10.683 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:10.338Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:10 smithi136 ceph-mon[113531]: pgmap v1759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:10 smithi093 ceph-mon[111335]: pgmap v1759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:10 smithi093 ceph-mon[118902]: pgmap v1759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/65085228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/65085228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/65085228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:11.854Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:11.855Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:12 smithi093 ceph-mon[111335]: pgmap v1760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:12 smithi093 ceph-mon[118902]: pgmap v1760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:12 smithi136 ceph-mon[113531]: pgmap v1760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2677118230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2677118230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:13.762Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:13.763Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2677118230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:14 smithi093 ceph-mon[111335]: pgmap v1761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:14 smithi093 ceph-mon[118902]: pgmap v1761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:14 smithi136 ceph-mon[113531]: pgmap v1761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:52:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:52:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:52:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:16 smithi093 ceph-mon[111335]: pgmap v1762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/167089248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:16 smithi093 ceph-mon[118902]: pgmap v1762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/167089248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:16 smithi136 ceph-mon[113531]: pgmap v1762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/167089248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:18 smithi093 ceph-mon[111335]: pgmap v1763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2222936050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:18 smithi093 ceph-mon[118902]: pgmap v1763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2222936050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:18 smithi136 ceph-mon[113531]: pgmap v1763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2222936050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:20 smithi136 ceph-mon[113531]: pgmap v1764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:20 smithi093 ceph-mon[111335]: pgmap v1764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:20 smithi093 ceph-mon[118902]: pgmap v1764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:20.338Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:20.339Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3013427970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3013427970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3013427970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:22.114 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:21.855Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:22.114 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:21.856Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:22 smithi136 ceph-mon[113531]: pgmap v1765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:22 smithi093 ceph-mon[111335]: pgmap v1765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:22 smithi093 ceph-mon[118902]: pgmap v1765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2144463000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2144463000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2144463000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:23.762Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:23.763Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:24 smithi136 ceph-mon[113531]: pgmap v1766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:24 smithi093 ceph-mon[111335]: pgmap v1766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:24 smithi093 ceph-mon[118902]: pgmap v1766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:52:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:52:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:52:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:26 smithi136 ceph-mon[113531]: pgmap v1767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/119852137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:26 smithi093 ceph-mon[111335]: pgmap v1767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/119852137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:26 smithi093 ceph-mon[118902]: pgmap v1767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/119852137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:28 smithi136 ceph-mon[113531]: pgmap v1768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4139399538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:28 smithi093 ceph-mon[111335]: pgmap v1768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4139399538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:28 smithi093 ceph-mon[118902]: pgmap v1768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4139399538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:30 smithi136 ceph-mon[113531]: pgmap v1769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:30 smithi093 ceph-mon[111335]: pgmap v1769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:30 smithi093 ceph-mon[118902]: pgmap v1769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:30.339Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:30.340Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2666141415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2666141415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2666141415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:32.110 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:31.856Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:32.110 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:31.857Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:32 smithi136 ceph-mon[113531]: pgmap v1770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:32 smithi093 ceph-mon[111335]: pgmap v1770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:32 smithi093 ceph-mon[118902]: pgmap v1770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2840044507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2840044507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2840044507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:34.116 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:33.764Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:34.116 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:33.765Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:34 smithi136 ceph-mon[113531]: pgmap v1771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:34 smithi093 ceph-mon[118902]: pgmap v1771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:34 smithi093 ceph-mon[111335]: pgmap v1771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2627308381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2627308381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2627308381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:52:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:52:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:52:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:36 smithi136 ceph-mon[113531]: pgmap v1772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:36 smithi093 ceph-mon[111335]: pgmap v1772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:36 smithi093 ceph-mon[118902]: pgmap v1772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:38 smithi136 ceph-mon[113531]: pgmap v1773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1815321422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:38 smithi093 ceph-mon[111335]: pgmap v1773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1815321422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:38 smithi093 ceph-mon[118902]: pgmap v1773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1815321422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:40 smithi136 ceph-mon[113531]: pgmap v1774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2319622897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:40 smithi093 ceph-mon[111335]: pgmap v1774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2319622897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:40 smithi093 ceph-mon[118902]: pgmap v1774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2319622897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:40.340Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:40.340Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:41.857Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:41.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:42 smithi136 ceph-mon[113531]: pgmap v1775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2755884879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:42 smithi093 ceph-mon[111335]: pgmap v1775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2755884879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:42 smithi093 ceph-mon[118902]: pgmap v1775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2755884879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:43.765Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:43.766Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:44 smithi136 ceph-mon[113531]: pgmap v1776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:44 smithi093 ceph-mon[111335]: pgmap v1776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:44 smithi093 ceph-mon[118902]: pgmap v1776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3516873039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3516873039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:52:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:52:45] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:52:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3516873039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:46 smithi136 ceph-mon[113531]: pgmap v1777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:46 smithi093 ceph-mon[111335]: pgmap v1777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:46 smithi093 ceph-mon[118902]: pgmap v1777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1525822587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1525822587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1525822587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:48 smithi136 ceph-mon[113531]: pgmap v1778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:48 smithi093 ceph-mon[111335]: pgmap v1778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:48 smithi093 ceph-mon[118902]: pgmap v1778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:50 smithi136 ceph-mon[113531]: pgmap v1779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1514546837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:50 smithi093 ceph-mon[111335]: pgmap v1779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1514546837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:50.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:50 smithi093 ceph-mon[118902]: pgmap v1779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:50.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1514546837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:50.341Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:50.342Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:51.857Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:51.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:52 smithi136 ceph-mon[113531]: pgmap v1780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3915833843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:52 smithi093 ceph-mon[111335]: pgmap v1780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3915833843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:52 smithi093 ceph-mon[118902]: pgmap v1780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3915833843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:53.765Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:52:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:52:53.767Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:52:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:54 smithi136 ceph-mon[113531]: pgmap v1781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4138283676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:54 smithi093 ceph-mon[111335]: pgmap v1781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4138283676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:54 smithi093 ceph-mon[118902]: pgmap v1781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:52:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4138283676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:52:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:52:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:52:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:56 smithi136 ceph-mon[113531]: pgmap v1782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:56 smithi093 ceph-mon[111335]: pgmap v1782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:56 smithi093 ceph-mon[118902]: pgmap v1782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/83575196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/83575196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/83575196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:58 smithi136 ceph-mon[113531]: pgmap v1783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:58 smithi093 ceph-mon[111335]: pgmap v1783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:58 smithi093 ceph-mon[118902]: pgmap v1783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:52:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:52:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3994210780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:52:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3994210780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:52:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:52:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3994210780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:00 smithi136 ceph-mon[113531]: pgmap v1784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:00 smithi093 ceph-mon[111335]: pgmap v1784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:00 smithi093 ceph-mon[118902]: pgmap v1784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:00.341Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:00.342Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:01.858Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:01.859Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:02 smithi136 ceph-mon[113531]: pgmap v1785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1443408443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:02 smithi093 ceph-mon[111335]: pgmap v1785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1443408443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:02 smithi093 ceph-mon[118902]: pgmap v1785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1443408443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:03.765Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:03.766Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:04 smithi136 ceph-mon[113531]: pgmap v1786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/761894537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:04 smithi093 ceph-mon[111335]: pgmap v1786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/761894537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:04 smithi093 ceph-mon[118902]: pgmap v1786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/761894537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:53:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:53:06.600 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:06 smithi093 ceph-mon[118902]: pgmap v1787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:06.600 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/900239917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:06.600 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:06.601 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:06 smithi093 ceph-mon[111335]: pgmap v1787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:06.601 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/900239917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:06.601 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:06.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:06 smithi136 ceph-mon[113531]: pgmap v1787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:06.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/900239917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:06.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:53:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:08 smithi136 ceph-mon[113531]: pgmap v1788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:08 smithi093 ceph-mon[111335]: pgmap v1788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:08 smithi093 ceph-mon[118902]: pgmap v1788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1910010247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1910010247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1910010247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:53:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:53:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:53:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:10 smithi136 ceph-mon[113531]: pgmap v1789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:10 smithi093 ceph-mon[111335]: pgmap v1789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:10 smithi093 ceph-mon[118902]: pgmap v1789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:10.342Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:10.343Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4187084497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4187084497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4187084497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:11.858Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:11.859Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:12 smithi136 ceph-mon[113531]: pgmap v1790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:12 smithi093 ceph-mon[111335]: pgmap v1790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:12 smithi093 ceph-mon[118902]: pgmap v1790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/732382772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/732382772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/732382772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:13.766Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:13.768Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:14 smithi136 ceph-mon[113531]: pgmap v1791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:14 smithi093 ceph-mon[111335]: pgmap v1791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:14 smithi093 ceph-mon[118902]: pgmap v1791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:53:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:53:15] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:53:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:16 smithi136 ceph-mon[113531]: pgmap v1792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1377125711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:16 smithi093 ceph-mon[111335]: pgmap v1792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1377125711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:16 smithi093 ceph-mon[118902]: pgmap v1792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1377125711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:18 smithi136 ceph-mon[113531]: pgmap v1793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2971866349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:18 smithi093 ceph-mon[111335]: pgmap v1793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2971866349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:18 smithi093 ceph-mon[118902]: pgmap v1793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2971866349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:20 smithi136 ceph-mon[113531]: pgmap v1794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:20 smithi093 ceph-mon[111335]: pgmap v1794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:20 smithi093 ceph-mon[118902]: pgmap v1794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:20.343Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:20.344Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/158992413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/158992413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/158992413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:22.117 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:21.859Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:22.117 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:21.860Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:22 smithi136 ceph-mon[113531]: pgmap v1795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:22 smithi093 ceph-mon[111335]: pgmap v1795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:22 smithi093 ceph-mon[118902]: pgmap v1795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2401744858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2401744858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2401744858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:24.125 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:23.767Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:24.125 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:23.769Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:24 smithi136 ceph-mon[113531]: pgmap v1796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:24 smithi093 ceph-mon[111335]: pgmap v1796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:24 smithi093 ceph-mon[118902]: pgmap v1796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2884631386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2884631386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2884631386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:53:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:53:25] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:53:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:26 smithi136 ceph-mon[113531]: pgmap v1797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:26 smithi093 ceph-mon[111335]: pgmap v1797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:26 smithi093 ceph-mon[118902]: pgmap v1797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:28 smithi136 ceph-mon[113531]: pgmap v1798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1778351095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:28 smithi093 ceph-mon[111335]: pgmap v1798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1778351095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:28 smithi093 ceph-mon[118902]: pgmap v1798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1778351095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:30 smithi136 ceph-mon[113531]: pgmap v1799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3504862397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:30 smithi093 ceph-mon[111335]: pgmap v1799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3504862397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:30.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:30 smithi093 ceph-mon[118902]: pgmap v1799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:30.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3504862397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:30.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:30.344Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:30.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:30.345Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:31.860Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:31.860Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:32 smithi136 ceph-mon[113531]: pgmap v1800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2698572651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:32 smithi093 ceph-mon[111335]: pgmap v1800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2698572651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:32 smithi093 ceph-mon[118902]: pgmap v1800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2698572651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:33.768Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:33.769Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:34 smithi136 ceph-mon[113531]: pgmap v1801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:34 smithi093 ceph-mon[111335]: pgmap v1801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:34 smithi093 ceph-mon[118902]: pgmap v1801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/275793867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/275793867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:53:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:53:35] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:53:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/275793867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:36 smithi136 ceph-mon[113531]: pgmap v1802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:36 smithi093 ceph-mon[111335]: pgmap v1802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:36 smithi093 ceph-mon[118902]: pgmap v1802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/665861297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/665861297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/665861297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:38 smithi136 ceph-mon[113531]: pgmap v1803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:38 smithi093 ceph-mon[111335]: pgmap v1803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:38 smithi093 ceph-mon[118902]: pgmap v1803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:40 smithi136 ceph-mon[113531]: pgmap v1804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4281722914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:40 smithi093 ceph-mon[111335]: pgmap v1804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4281722914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:40 smithi093 ceph-mon[118902]: pgmap v1804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4281722914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:40.345Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:40.346Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:41.860Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:41.861Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:42 smithi136 ceph-mon[113531]: pgmap v1805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/230512547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:42 smithi093 ceph-mon[111335]: pgmap v1805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/230512547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:42 smithi093 ceph-mon[118902]: pgmap v1805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/230512547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:43.769Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:43.770Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:44 smithi136 ceph-mon[113531]: pgmap v1806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1227263146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:44 smithi093 ceph-mon[111335]: pgmap v1806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1227263146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:44 smithi093 ceph-mon[118902]: pgmap v1806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1227263146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:53:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:53:45] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:53:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:46 smithi136 ceph-mon[113531]: pgmap v1807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:46 smithi093 ceph-mon[111335]: pgmap v1807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:46 smithi093 ceph-mon[118902]: pgmap v1807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:47.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1623810811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1623810811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1623810811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:48 smithi136 ceph-mon[113531]: pgmap v1808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:48 smithi093 ceph-mon[111335]: pgmap v1808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:48 smithi093 ceph-mon[118902]: pgmap v1808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2026784599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2026784599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2026784599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:50 smithi136 ceph-mon[113531]: pgmap v1809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:50 smithi093 ceph-mon[118902]: pgmap v1809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:50 smithi093 ceph-mon[111335]: pgmap v1809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:50.346Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:50.347Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:51.861Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:51.862Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:52 smithi136 ceph-mon[113531]: pgmap v1810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1154945378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:52 smithi093 ceph-mon[111335]: pgmap v1810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1154945378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:52 smithi093 ceph-mon[118902]: pgmap v1810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1154945378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:53.769Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:53:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:53:53.770Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:53:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:54 smithi136 ceph-mon[113531]: pgmap v1811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/432385117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:54 smithi093 ceph-mon[111335]: pgmap v1811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/432385117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:54 smithi093 ceph-mon[118902]: pgmap v1811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:53:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/432385117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:53:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:53:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:53:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:56 smithi136 ceph-mon[113531]: pgmap v1812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2240742194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:56 smithi093 ceph-mon[118902]: pgmap v1812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2240742194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:56 smithi093 ceph-mon[111335]: pgmap v1812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2240742194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:58 smithi136 ceph-mon[113531]: pgmap v1813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:58 smithi093 ceph-mon[111335]: pgmap v1813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:58 smithi093 ceph-mon[118902]: pgmap v1813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:53:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:53:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3112629397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:53:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3112629397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:53:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:53:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3112629397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:00 smithi136 ceph-mon[113531]: pgmap v1814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:00 smithi093 ceph-mon[111335]: pgmap v1814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:00 smithi093 ceph-mon[118902]: pgmap v1814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:00.347Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:00.348Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3337170059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3337170059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3337170059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:01.862Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:01.863Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:02 smithi136 ceph-mon[113531]: pgmap v1815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:02 smithi093 ceph-mon[111335]: pgmap v1815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:02.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:02 smithi093 ceph-mon[118902]: pgmap v1815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2721404890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2721404890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2721404890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:03.770Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:03.771Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:04 smithi136 ceph-mon[113531]: pgmap v1816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:04 smithi093 ceph-mon[111335]: pgmap v1816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:04 smithi093 ceph-mon[118902]: pgmap v1816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:54:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:54:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:54:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:06 smithi136 ceph-mon[113531]: pgmap v1817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3576324497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:06 smithi093 ceph-mon[111335]: pgmap v1817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3576324497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:06 smithi093 ceph-mon[118902]: pgmap v1817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3576324497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:08 smithi136 ceph-mon[113531]: pgmap v1818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3947363887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:08 smithi093 ceph-mon[111335]: pgmap v1818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3947363887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:08 smithi093 ceph-mon[118902]: pgmap v1818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3947363887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:09.616 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:09.617 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:09.619 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:09.619 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:09.619 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:09.619 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:54:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:10 smithi093 ceph-mon[111335]: pgmap v1819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:10 smithi093 ceph-mon[118902]: pgmap v1819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:10.348Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:10.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:10.349Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:10 smithi136 ceph-mon[113531]: pgmap v1819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2521139373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2521139373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2521139373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:11.862Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:11.863Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[111335]: pgmap v1820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[118902]: pgmap v1820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:12 smithi136 ceph-mon[113531]: pgmap v1820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:54:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:54:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:54:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3932390506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3932390506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3932390506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:13.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:13.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:14 smithi093 ceph-mon[111335]: pgmap v1821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:14 smithi093 ceph-mon[118902]: pgmap v1821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:14 smithi136 ceph-mon[113531]: pgmap v1821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2054261236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:54:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:54:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:54:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2054261236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2054261236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:16 smithi093 ceph-mon[111335]: pgmap v1822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:16 smithi093 ceph-mon[118902]: pgmap v1822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:16 smithi136 ceph-mon[113531]: pgmap v1822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:18 smithi093 ceph-mon[111335]: pgmap v1823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4196084940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:18 smithi093 ceph-mon[118902]: pgmap v1823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4196084940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:18 smithi136 ceph-mon[113531]: pgmap v1823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4196084940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:20 smithi136 ceph-mon[113531]: pgmap v1824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/214013638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:20 smithi093 ceph-mon[111335]: pgmap v1824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/214013638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:20 smithi093 ceph-mon[118902]: pgmap v1824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/214013638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:20.348Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:20.349Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:21.863Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:21.864Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:22 smithi136 ceph-mon[113531]: pgmap v1825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:22 smithi093 ceph-mon[111335]: pgmap v1825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:22 smithi093 ceph-mon[118902]: pgmap v1825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/227181179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/227181179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/227181179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:23.772Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:23.773Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:24 smithi136 ceph-mon[113531]: pgmap v1826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:24 smithi093 ceph-mon[118902]: pgmap v1826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:24 smithi093 ceph-mon[111335]: pgmap v1826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/933107773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/933107773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:54:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:54:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:54:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/933107773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:26 smithi136 ceph-mon[113531]: pgmap v1827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:26 smithi093 ceph-mon[118902]: pgmap v1827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:26 smithi093 ceph-mon[111335]: pgmap v1827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1968590980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1968590980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1968590980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:28 smithi136 ceph-mon[113531]: pgmap v1828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:28 smithi093 ceph-mon[111335]: pgmap v1828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:28 smithi093 ceph-mon[118902]: pgmap v1828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:30 smithi136 ceph-mon[113531]: pgmap v1829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3850365921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:30 smithi093 ceph-mon[118902]: pgmap v1829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3850365921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:30 smithi093 ceph-mon[111335]: pgmap v1829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3850365921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:30.348Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:30.349Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:31.864Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:31.865Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:32 smithi136 ceph-mon[113531]: pgmap v1830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/236111027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:32 smithi093 ceph-mon[111335]: pgmap v1830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/236111027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:32 smithi093 ceph-mon[118902]: pgmap v1830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/236111027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:34.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:33.773Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:34.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:33.774Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:34 smithi136 ceph-mon[113531]: pgmap v1831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:34 smithi093 ceph-mon[111335]: pgmap v1831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:34 smithi093 ceph-mon[118902]: pgmap v1831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3862454055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3862454055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3862454055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:54:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:54:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:54:36.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:36 smithi136 ceph-mon[113531]: pgmap v1832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:36 smithi093 ceph-mon[111335]: pgmap v1832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:36 smithi093 ceph-mon[118902]: pgmap v1832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1661787773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1661787773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1661787773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:38 smithi136 ceph-mon[113531]: pgmap v1833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:38.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:38 smithi093 ceph-mon[111335]: pgmap v1833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:38.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:38 smithi093 ceph-mon[118902]: pgmap v1833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3372170926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3372170926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3372170926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:40.550 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:40 smithi136 ceph-mon[113531]: pgmap v1834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:40 smithi093 ceph-mon[111335]: pgmap v1834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:40 smithi093 ceph-mon[118902]: pgmap v1834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:40.349Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:40.350Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:40.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:40.549059+0000) 2023-08-27T21:54:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:41.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:41.549309+0000) 2023-08-27T21:54:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:41.865Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:41.866Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:42.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:42 smithi136 ceph-mon[113531]: pgmap v1835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:42.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1958276567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:42 smithi093 ceph-mon[118902]: pgmap v1835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1958276567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:42 smithi093 ceph-mon[111335]: pgmap v1835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1958276567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:42.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:42.549535+0000) 2023-08-27T21:54:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:43.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:43.549714+0000) 2023-08-27T21:54:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:43.773Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:43.774Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:44.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:44 smithi136 ceph-mon[113531]: pgmap v1836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:44.550 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3950545276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:44 smithi093 ceph-mon[111335]: pgmap v1836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3950545276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:44 smithi093 ceph-mon[118902]: pgmap v1836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3950545276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:44.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:44.550004+0000) 2023-08-27T21:54:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:54:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:54:45] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:54:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:45.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:45.550271+0000) 2023-08-27T21:54:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:46 smithi136 ceph-mon[113531]: pgmap v1837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4129176458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:46.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:46.550532+0000) 2023-08-27T21:54:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:46 smithi093 ceph-mon[111335]: pgmap v1837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4129176458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:46 smithi093 ceph-mon[118902]: pgmap v1837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4129176458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:47.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:47.550816+0000) 2023-08-27T21:54:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:48 smithi136 ceph-mon[113531]: pgmap v1838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:48.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:48.551085+0000) 2023-08-27T21:54:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:48 smithi093 ceph-mon[111335]: pgmap v1838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:48 smithi093 ceph-mon[118902]: pgmap v1838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1853129388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:49.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:49.551399+0000) 2023-08-27T21:54:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1853129388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1853129388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:50 smithi136 ceph-mon[113531]: pgmap v1839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:50.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:50.551650+0000) 2023-08-27T21:54:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:50 smithi093 ceph-mon[111335]: pgmap v1839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:50 smithi093 ceph-mon[118902]: pgmap v1839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:50.350Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:50.351Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/402316706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:51.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:51.551901+0000) 2023-08-27T21:54:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/402316706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/402316706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:51.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:51.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:52 smithi136 ceph-mon[113531]: pgmap v1840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:52.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:52.552142+0000) 2023-08-27T21:54:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:52 smithi093 ceph-mon[111335]: pgmap v1840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:52 smithi093 ceph-mon[118902]: pgmap v1840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:53.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:53.552331+0000) 2023-08-27T21:54:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:53.774Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:54:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:54:53.775Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:54:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:54.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:54.552538+0000) 2023-08-27T21:54:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:54 smithi136 ceph-mon[113531]: pgmap v1841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3463969979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:54 smithi093 ceph-mon[111335]: pgmap v1841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3463969979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:54 smithi093 ceph-mon[118902]: pgmap v1841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3463969979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:54:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:54:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:54:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:54:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:55.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:55.552773+0000) 2023-08-27T21:54:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:56 smithi136 ceph-mon[113531]: pgmap v1842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1993502218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:56.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:56.552997+0000) 2023-08-27T21:54:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:56 smithi093 ceph-mon[111335]: pgmap v1842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1993502218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:56 smithi093 ceph-mon[118902]: pgmap v1842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1993502218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:57.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:57.553253+0000) 2023-08-27T21:54:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:58 smithi136 ceph-mon[113531]: pgmap v1843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:54:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2522106005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:58.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:58.553526+0000) 2023-08-27T21:54:58.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:58 smithi093 ceph-mon[111335]: pgmap v1843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:58.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:54:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2522106005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:58.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:58 smithi093 ceph-mon[118902]: pgmap v1843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:54:58.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:54:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2522106005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:54:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:54:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:54:59.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:54:59.553790+0000) 2023-08-27T21:55:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:00 smithi136 ceph-mon[113531]: pgmap v1844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:00 smithi093 ceph-mon[111335]: pgmap v1844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:00 smithi093 ceph-mon[118902]: pgmap v1844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:00.351Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:00.352Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:00.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:00.554089+0000) 2023-08-27T21:55:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2762182987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2762182987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2762182987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:01.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:01.554344+0000) 2023-08-27T21:55:02.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:01.867Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:02.136 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:01.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:02 smithi136 ceph-mon[113531]: pgmap v1845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:02 smithi093 ceph-mon[111335]: pgmap v1845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:02 smithi093 ceph-mon[118902]: pgmap v1845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:02.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:02.554630+0000) 2023-08-27T21:55:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2191501270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2191501270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2191501270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:03.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:03.554923+0000) 2023-08-27T21:55:04.154 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:03.775Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:04.155 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:03.776Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:04 smithi136 ceph-mon[113531]: pgmap v1846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:04 smithi093 ceph-mon[111335]: pgmap v1846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:04 smithi093 ceph-mon[118902]: pgmap v1846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:04.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:04.555203+0000) 2023-08-27T21:55:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2631724658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:05.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:05.555419+0000) 2023-08-27T21:55:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2631724658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:55:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:55:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:55:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2631724658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:06 smithi136 ceph-mon[113531]: pgmap v1847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:06.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:06.555675+0000) 2023-08-27T21:55:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:06 smithi093 ceph-mon[111335]: pgmap v1847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:06 smithi093 ceph-mon[118902]: pgmap v1847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:07.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:07.555906+0000) 2023-08-27T21:55:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:08 smithi136 ceph-mon[113531]: pgmap v1848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3065998188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:08.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:08.556236+0000) 2023-08-27T21:55:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:08 smithi093 ceph-mon[111335]: pgmap v1848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3065998188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:08 smithi093 ceph-mon[118902]: pgmap v1848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3065998188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:09.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:09.556497+0000) 2023-08-27T21:55:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:10 smithi136 ceph-mon[113531]: pgmap v1849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2791113025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:10.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:10.556716+0000) 2023-08-27T21:55:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:10 smithi093 ceph-mon[111335]: pgmap v1849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2791113025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:10 smithi093 ceph-mon[118902]: pgmap v1849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2791113025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:10.351Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:10.352Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:11.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:11.556907+0000) 2023-08-27T21:55:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:11.867Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:11.868Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:12.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:12 smithi136 ceph-mon[113531]: pgmap v1850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:12.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:12.622 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:12.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:12.557095+0000) 2023-08-27T21:55:12.642 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:12 smithi093 ceph-mon[111335]: pgmap v1850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:12.642 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:12.643 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:12 smithi093 ceph-mon[118902]: pgmap v1850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:12.643 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:55:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2555736294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2555736294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2555736294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:13.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:13.557239+0000) 2023-08-27T21:55:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:13.776Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:13.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:13.777Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:14.411 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:14 smithi136 ceph-mon[113531]: pgmap v1851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:14.411 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:14.411 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:14.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:14.557432+0000) 2023-08-27T21:55:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:14 smithi093 ceph-mon[118902]: pgmap v1851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:14 smithi093 ceph-mon[111335]: pgmap v1851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:55:15] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3121420628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3121420628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.841 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.841 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.841 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.841 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.841 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3121420628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:15.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:15.842 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:15.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:15.557600+0000) 2023-08-27T21:55:16.812 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:16 smithi136 ceph-mon[113531]: pgmap v1852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:16.812 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:16.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:16.557782+0000) 2023-08-27T21:55:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:16 smithi093 ceph-mon[111335]: pgmap v1852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:16 smithi093 ceph-mon[118902]: pgmap v1852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:17.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:17.557979+0000) 2023-08-27T21:55:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3260103571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:17.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3260103571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3260103571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:55:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:55:17.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:55:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:18 smithi136 ceph-mon[113531]: pgmap v1853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:18.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:18.558236+0000) 2023-08-27T21:55:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:18 smithi093 ceph-mon[111335]: pgmap v1853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:18 smithi093 ceph-mon[118902]: pgmap v1853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:19.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:19.558533+0000) 2023-08-27T21:55:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:20 smithi136 ceph-mon[113531]: pgmap v1854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4215012725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:20 smithi093 ceph-mon[118902]: pgmap v1854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4215012725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:20.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:20 smithi093 ceph-mon[111335]: pgmap v1854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:20.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4215012725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:20.352Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:20.353Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:20.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:20.558789+0000) 2023-08-27T21:55:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:21.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:21.559137+0000) 2023-08-27T21:55:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:21.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:21.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:22.559 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:22 smithi136 ceph-mon[113531]: pgmap v1855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:22.559 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2956189933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:22 smithi093 ceph-mon[111335]: pgmap v1855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2956189933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:22 smithi093 ceph-mon[118902]: pgmap v1855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2956189933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:22.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:22.559398+0000) 2023-08-27T21:55:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:23.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:23.559643+0000) 2023-08-27T21:55:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:23.777Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:23.778Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:24.560 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:24 smithi136 ceph-mon[113531]: pgmap v1856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:24.560 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:24 smithi093 ceph-mon[111335]: pgmap v1856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:24 smithi093 ceph-mon[118902]: pgmap v1856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:24.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:24.559932+0000) 2023-08-27T21:55:25.560 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3169750924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3169750924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3169750924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:55:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:55:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:55:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:25.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:25.560178+0000) 2023-08-27T21:55:26.560 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:26 smithi136 ceph-mon[113531]: pgmap v1857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:26 smithi093 ceph-mon[111335]: pgmap v1857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:26 smithi093 ceph-mon[118902]: pgmap v1857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:26.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:26.560396+0000) 2023-08-27T21:55:27.560 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1170579626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1170579626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1170579626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:27.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:27.560653+0000) 2023-08-27T21:55:28.561 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:28 smithi136 ceph-mon[113531]: pgmap v1858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:28 smithi093 ceph-mon[111335]: pgmap v1858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:28 smithi093 ceph-mon[118902]: pgmap v1858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:28.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:28.560918+0000) 2023-08-27T21:55:29.561 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1036911473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1036911473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1036911473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:29.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:29.561208+0000) 2023-08-27T21:55:30.561 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:30 smithi136 ceph-mon[113531]: pgmap v1859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:30 smithi093 ceph-mon[111335]: pgmap v1859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:30 smithi093 ceph-mon[118902]: pgmap v1859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:30.352Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:30.353Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:30.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:30.561488+0000) 2023-08-27T21:55:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:31.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:31.561752+0000) 2023-08-27T21:55:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:31.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:31.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:32.561 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:32 smithi136 ceph-mon[113531]: pgmap v1860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:32.562 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1395610235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:32 smithi093 ceph-mon[111335]: pgmap v1860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1395610235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:32 smithi093 ceph-mon[118902]: pgmap v1860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1395610235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:32.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:32.561948+0000) 2023-08-27T21:55:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:33.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:33.562159+0000) 2023-08-27T21:55:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:33.778Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:33.779Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:34.562 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:34 smithi136 ceph-mon[113531]: pgmap v1861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:34.562 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/725669402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:34 smithi093 ceph-mon[111335]: pgmap v1861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/725669402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:34 smithi093 ceph-mon[118902]: pgmap v1861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/725669402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:34.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:34.562429+0000) 2023-08-27T21:55:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:55:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:55:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:55:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:35.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:35.562673+0000) 2023-08-27T21:55:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:36 smithi136 ceph-mon[113531]: pgmap v1862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2853732121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:36.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:36.562913+0000) 2023-08-27T21:55:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:36 smithi093 ceph-mon[111335]: pgmap v1862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2853732121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:36 smithi093 ceph-mon[118902]: pgmap v1862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2853732121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:37.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:37.563198+0000) 2023-08-27T21:55:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:38 smithi136 ceph-mon[113531]: pgmap v1863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:38.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:38.563488+0000) 2023-08-27T21:55:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:38 smithi093 ceph-mon[111335]: pgmap v1863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:38 smithi093 ceph-mon[118902]: pgmap v1863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2642065278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:39.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:39.563796+0000) 2023-08-27T21:55:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2642065278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2642065278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:40 smithi136 ceph-mon[113531]: pgmap v1864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:40.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:40.564118+0000) 2023-08-27T21:55:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:40 smithi093 ceph-mon[111335]: pgmap v1864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:40 smithi093 ceph-mon[118902]: pgmap v1864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:40.353Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:40.354Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2611377128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:41.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:41.564386+0000) 2023-08-27T21:55:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2611377128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2611377128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:41.869Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:41.870Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:42 smithi136 ceph-mon[113531]: pgmap v1865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:42.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:42.564655+0000) 2023-08-27T21:55:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:42 smithi093 ceph-mon[111335]: pgmap v1865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:42 smithi093 ceph-mon[118902]: pgmap v1865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:43.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:43.564902+0000) 2023-08-27T21:55:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:43.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:43.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:44 smithi136 ceph-mon[113531]: pgmap v1866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1852132569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:44.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:44.565198+0000) 2023-08-27T21:55:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:44 smithi093 ceph-mon[111335]: pgmap v1866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1852132569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:44 smithi093 ceph-mon[118902]: pgmap v1866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1852132569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:55:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:55:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:55:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:45.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:45.565432+0000) 2023-08-27T21:55:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:46 smithi136 ceph-mon[113531]: pgmap v1867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2384470355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:46.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:46.565661+0000) 2023-08-27T21:55:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:46 smithi093 ceph-mon[111335]: pgmap v1867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2384470355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:46 smithi093 ceph-mon[118902]: pgmap v1867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2384470355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:47.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:47.565949+0000) 2023-08-27T21:55:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:48 smithi136 ceph-mon[113531]: pgmap v1868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1723342628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:48.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:48.566200+0000) 2023-08-27T21:55:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:48 smithi093 ceph-mon[111335]: pgmap v1868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1723342628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:48 smithi093 ceph-mon[118902]: pgmap v1868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1723342628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:49.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:49.566460+0000) 2023-08-27T21:55:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:50 smithi136 ceph-mon[113531]: pgmap v1869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:50 smithi093 ceph-mon[111335]: pgmap v1869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:50 smithi093 ceph-mon[118902]: pgmap v1869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:50.354Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:50.355Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:50.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:50.566697+0000) 2023-08-27T21:55:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1144192008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1144192008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1144192008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:51.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:51.566876+0000) 2023-08-27T21:55:52.144 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:51.870Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:52.144 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:51.871Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:52 smithi136 ceph-mon[113531]: pgmap v1870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:52 smithi093 ceph-mon[111335]: pgmap v1870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:52 smithi093 ceph-mon[118902]: pgmap v1870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:52.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:52.567142+0000) 2023-08-27T21:55:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3304546808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3304546808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3304546808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:53.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:53.567423+0000) 2023-08-27T21:55:54.149 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:53.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:54.149 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:55:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:55:53.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:55:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:54 smithi136 ceph-mon[113531]: pgmap v1871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:54 smithi093 ceph-mon[111335]: pgmap v1871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:54 smithi093 ceph-mon[118902]: pgmap v1871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:55:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:54.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:54.567726+0000) 2023-08-27T21:55:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:55:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:55:55] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:55:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:55.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:55.568018+0000) 2023-08-27T21:55:56.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:56 smithi136 ceph-mon[113531]: pgmap v1872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:56.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/685896630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:56 smithi093 ceph-mon[111335]: pgmap v1872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/685896630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:56 smithi093 ceph-mon[118902]: pgmap v1872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/685896630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:56.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:56.568231+0000) 2023-08-27T21:55:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:57.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:57.568523+0000) 2023-08-27T21:55:58.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:58 smithi136 ceph-mon[113531]: pgmap v1873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:58.569 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:55:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2274819822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:58 smithi093 ceph-mon[111335]: pgmap v1873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:55:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2274819822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:58 smithi093 ceph-mon[118902]: pgmap v1873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:55:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:55:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2274819822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:55:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:58.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:58.568803+0000) 2023-08-27T21:55:59.944 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:55:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:55:59.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:55:59.569108+0000) 2023-08-27T21:56:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:00 smithi136 ceph-mon[113531]: pgmap v1874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1094478501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:00 smithi093 ceph-mon[111335]: pgmap v1874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1094478501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:00 smithi093 ceph-mon[118902]: pgmap v1874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1094478501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:00.355Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:00.355Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:00.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:00.569363+0000) 2023-08-27T21:56:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:01.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:01.569551+0000) 2023-08-27T21:56:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:01.870Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:01.871Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:02.569 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:02 smithi136 ceph-mon[113531]: pgmap v1875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:02 smithi093 ceph-mon[111335]: pgmap v1875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:02.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:02 smithi093 ceph-mon[118902]: pgmap v1875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:02.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:02.569772+0000) 2023-08-27T21:56:03.569 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1415147478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1415147478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1415147478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:03.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:03.570000+0000) 2023-08-27T21:56:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:03.780Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:03.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:04.570 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:04 smithi136 ceph-mon[113531]: pgmap v1876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:04 smithi093 ceph-mon[111335]: pgmap v1876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:04 smithi093 ceph-mon[118902]: pgmap v1876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:04.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:04.570258+0000) 2023-08-27T21:56:05.570 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4098837062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4098837062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4098837062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:56:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:56:05] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:56:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:05.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:05.570494+0000) 2023-08-27T21:56:06.570 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:06 smithi136 ceph-mon[113531]: pgmap v1877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:06 smithi093 ceph-mon[111335]: pgmap v1877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:06 smithi093 ceph-mon[118902]: pgmap v1877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:06.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:06.570718+0000) 2023-08-27T21:56:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3356952284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:07.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:07.571024+0000) 2023-08-27T21:56:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3356952284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3356952284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:08 smithi136 ceph-mon[113531]: pgmap v1878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:08.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:08.571274+0000) 2023-08-27T21:56:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:08 smithi093 ceph-mon[111335]: pgmap v1878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:08 smithi093 ceph-mon[118902]: pgmap v1878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:09.694 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:09.708 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:09.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:09.571531+0000) 2023-08-27T21:56:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:10 smithi136 ceph-mon[113531]: pgmap v1879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/385319580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:10.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:10.571746+0000) 2023-08-27T21:56:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:10 smithi093 ceph-mon[111335]: pgmap v1879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/385319580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:10 smithi093 ceph-mon[118902]: pgmap v1879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/385319580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:10.355Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:10.356Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:11.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:11.572020+0000) 2023-08-27T21:56:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:11.871Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:11.871Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:12 smithi136 ceph-mon[113531]: pgmap v1880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3094988771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:12.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:12.572291+0000) 2023-08-27T21:56:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:12 smithi093 ceph-mon[111335]: pgmap v1880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3094988771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:12 smithi093 ceph-mon[118902]: pgmap v1880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3094988771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:13.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:13.572488+0000) 2023-08-27T21:56:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:13.780Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:14.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:13.781Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:14 smithi136 ceph-mon[113531]: pgmap v1881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:14.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:14.572770+0000) 2023-08-27T21:56:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:14 smithi093 ceph-mon[111335]: pgmap v1881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:14 smithi093 ceph-mon[118902]: pgmap v1881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2929350132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:15.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:15.572989+0000) 2023-08-27T21:56:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2929350132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2929350132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:56:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:56:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T21:56:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:16 smithi136 ceph-mon[113531]: pgmap v1882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:16.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:16.573220+0000) 2023-08-27T21:56:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:16 smithi093 ceph-mon[111335]: pgmap v1882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:16 smithi093 ceph-mon[118902]: pgmap v1882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4260353813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:17.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:17.573425+0000) 2023-08-27T21:56:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4260353813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4260353813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:18.346 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:18 smithi093 ceph-mon[118902]: pgmap v1883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:18.346 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:56:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:18 smithi136 ceph-mon[113531]: pgmap v1883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:56:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:18.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:18.573593+0000) 2023-08-27T21:56:18.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:18 smithi093 ceph-mon[111335]: pgmap v1883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:18.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:56:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:19.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:19.573747+0000) 2023-08-27T21:56:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4291349545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:20 smithi136 ceph-mon[113531]: pgmap v1884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4291349545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[111335]: pgmap v1884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4291349545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[118902]: pgmap v1884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:20.356Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:20.357Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:20.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:20.573940+0000) 2023-08-27T21:56:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:56:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:56:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:56:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:56:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:56:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:56:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:56:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:21.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:21.574155+0000) 2023-08-27T21:56:22.148 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:21.871Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:22.148 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:21.872Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:22 smithi136 ceph-mon[113531]: pgmap v1885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4259425473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:22 smithi093 ceph-mon[111335]: pgmap v1885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4259425473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:22 smithi093 ceph-mon[118902]: pgmap v1885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4259425473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:22.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:22.574424+0000) 2023-08-27T21:56:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:23.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:23.574676+0000) 2023-08-27T21:56:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:23.781Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:23.782Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:24.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:24 smithi136 ceph-mon[113531]: pgmap v1886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:24.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:24.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/223787959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:24 smithi093 ceph-mon[111335]: pgmap v1886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/223787959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:24 smithi093 ceph-mon[118902]: pgmap v1886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/223787959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:24.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:24.574929+0000) 2023-08-27T21:56:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:56:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:56:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:56:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:25.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:25.575150+0000) 2023-08-27T21:56:26.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:26 smithi136 ceph-mon[113531]: pgmap v1887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:26 smithi093 ceph-mon[111335]: pgmap v1887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:26 smithi093 ceph-mon[118902]: pgmap v1887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:26.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:26.575401+0000) 2023-08-27T21:56:27.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2568360453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2568360453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2568360453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:27.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:27.575618+0000) 2023-08-27T21:56:28.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:28 smithi136 ceph-mon[113531]: pgmap v1888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:28 smithi093 ceph-mon[111335]: pgmap v1888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:28 smithi093 ceph-mon[118902]: pgmap v1888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:28.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:28.575846+0000) 2023-08-27T21:56:29.576 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1414717001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1414717001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1414717001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:29.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:29.576129+0000) 2023-08-27T21:56:30.576 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:30 smithi136 ceph-mon[113531]: pgmap v1889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:30 smithi093 ceph-mon[111335]: pgmap v1889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:30 smithi093 ceph-mon[118902]: pgmap v1889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:30.357Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:30.358Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:30.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:30.576403+0000) 2023-08-27T21:56:31.576 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/279858426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/279858426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/279858426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:31.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:31.576667+0000) 2023-08-27T21:56:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:31.872Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:31.873Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:32.576 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:32 smithi136 ceph-mon[113531]: pgmap v1890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:32 smithi093 ceph-mon[111335]: pgmap v1890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:32 smithi093 ceph-mon[118902]: pgmap v1890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:32.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:32.576930+0000) 2023-08-27T21:56:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:33.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:33.577201+0000) 2023-08-27T21:56:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:33.781Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:33.782Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:34.577 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:34 smithi136 ceph-mon[113531]: pgmap v1891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:34.577 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3627033342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:34 smithi093 ceph-mon[111335]: pgmap v1891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3627033342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:34 smithi093 ceph-mon[118902]: pgmap v1891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3627033342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:34.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:34.577467+0000) 2023-08-27T21:56:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:56:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:56:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:56:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:35.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:35.577682+0000) 2023-08-27T21:56:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:36 smithi136 ceph-mon[113531]: pgmap v1892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1009482040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:36.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:36.577856+0000) 2023-08-27T21:56:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:36 smithi093 ceph-mon[111335]: pgmap v1892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1009482040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:36 smithi093 ceph-mon[118902]: pgmap v1892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1009482040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:37.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:37.578068+0000) 2023-08-27T21:56:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:38 smithi136 ceph-mon[113531]: pgmap v1893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3913649863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:38.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:38.578327+0000) 2023-08-27T21:56:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:38 smithi093 ceph-mon[111335]: pgmap v1893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3913649863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:38 smithi093 ceph-mon[118902]: pgmap v1893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3913649863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:39.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:39.578555+0000) 2023-08-27T21:56:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:40 smithi136 ceph-mon[113531]: pgmap v1894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:40.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:40.578788+0000) 2023-08-27T21:56:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:40 smithi093 ceph-mon[111335]: pgmap v1894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:40 smithi093 ceph-mon[118902]: pgmap v1894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:40.358Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:40.359Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3282657866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:41.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:41.579022+0000) 2023-08-27T21:56:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3282657866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3282657866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:41.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:41.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:42 smithi136 ceph-mon[113531]: pgmap v1895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:42.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:42.579272+0000) 2023-08-27T21:56:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:42 smithi093 ceph-mon[111335]: pgmap v1895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:42 smithi093 ceph-mon[118902]: pgmap v1895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/579895791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:43.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:43.579486+0000) 2023-08-27T21:56:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/579895791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/579895791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:43.782Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:43.783Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:44 smithi136 ceph-mon[113531]: pgmap v1896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:44.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:44.579737+0000) 2023-08-27T21:56:44.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:44 smithi093 ceph-mon[111335]: pgmap v1896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:44 smithi093 ceph-mon[118902]: pgmap v1896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:56:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:56:45] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:56:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:45.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:45.579940+0000) 2023-08-27T21:56:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:46.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:46.580132+0000) 2023-08-27T21:56:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:46 smithi136 ceph-mon[113531]: pgmap v1897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3393940929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:46 smithi093 ceph-mon[111335]: pgmap v1897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3393940929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:46 smithi093 ceph-mon[118902]: pgmap v1897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3393940929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:47.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:47.580425+0000) 2023-08-27T21:56:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:48 smithi136 ceph-mon[113531]: pgmap v1898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3387725650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:48.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:48.580673+0000) 2023-08-27T21:56:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:48 smithi093 ceph-mon[118902]: pgmap v1898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3387725650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:48 smithi093 ceph-mon[111335]: pgmap v1898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3387725650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:49.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:49.580870+0000) 2023-08-27T21:56:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:50 smithi136 ceph-mon[113531]: pgmap v1899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:50 smithi093 ceph-mon[111335]: pgmap v1899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:50 smithi093 ceph-mon[118902]: pgmap v1899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:50.358Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:50.359Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:50.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:50.581113+0000) 2023-08-27T21:56:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2034184710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2034184710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2034184710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:51.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:51.581311+0000) 2023-08-27T21:56:52.156 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:51.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:52.156 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:51.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:52 smithi136 ceph-mon[113531]: pgmap v1900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:52 smithi093 ceph-mon[111335]: pgmap v1900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:52 smithi093 ceph-mon[118902]: pgmap v1900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:52.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:52.581541+0000) 2023-08-27T21:56:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1658361549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1658361549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1658361549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:53.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:53.581769+0000) 2023-08-27T21:56:54.163 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:53.783Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:54.163 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:56:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:56:53.784Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:56:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:54 smithi136 ceph-mon[113531]: pgmap v1901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:54 smithi093 ceph-mon[111335]: pgmap v1901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:54 smithi093 ceph-mon[118902]: pgmap v1901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:56:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:54.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:54.582000+0000) 2023-08-27T21:56:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1402941361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1402941361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:56:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:56:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:56:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1402941361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:55.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:55.582180+0000) 2023-08-27T21:56:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:56 smithi136 ceph-mon[113531]: pgmap v1902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:56 smithi093 ceph-mon[111335]: pgmap v1902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:56 smithi093 ceph-mon[118902]: pgmap v1902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:56.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:56.582418+0000) 2023-08-27T21:56:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:57.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:57.582665+0000) 2023-08-27T21:56:58.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:58 smithi136 ceph-mon[113531]: pgmap v1903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:58.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:56:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/104907846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:58 smithi093 ceph-mon[111335]: pgmap v1903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:56:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/104907846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:58 smithi093 ceph-mon[118902]: pgmap v1903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:56:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:56:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/104907846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:56:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:58.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:58.582902+0000) 2023-08-27T21:56:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:56:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:56:59.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:56:59.583170+0000) 2023-08-27T21:57:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:00 smithi136 ceph-mon[113531]: pgmap v1904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1542289593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:00 smithi093 ceph-mon[111335]: pgmap v1904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1542289593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:00 smithi093 ceph-mon[118902]: pgmap v1904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1542289593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:00.359Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:00.360Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:00.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:00.583427+0000) 2023-08-27T21:57:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:01.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:01.583599+0000) 2023-08-27T21:57:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:01.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:01.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:02.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:02 smithi136 ceph-mon[113531]: pgmap v1905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:02.584 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3585988048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:02 smithi093 ceph-mon[111335]: pgmap v1905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3585988048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:02 smithi093 ceph-mon[118902]: pgmap v1905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3585988048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:02.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:02.583861+0000) 2023-08-27T21:57:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:03.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:03.584134+0000) 2023-08-27T21:57:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:03.783Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:03.784Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:04.586 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:04 smithi136 ceph-mon[113531]: pgmap v1906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:04 smithi093 ceph-mon[111335]: pgmap v1906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:04 smithi093 ceph-mon[118902]: pgmap v1906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:04.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:04.584421+0000) 2023-08-27T21:57:05.584 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3911939183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3911939183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:57:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:57:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:57:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3911939183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:05.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:05.584626+0000) 2023-08-27T21:57:06.585 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:06 smithi136 ceph-mon[113531]: pgmap v1907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:06 smithi093 ceph-mon[111335]: pgmap v1907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:06 smithi093 ceph-mon[118902]: pgmap v1907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:06.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:06.584866+0000) 2023-08-27T21:57:07.585 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1569936533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1569936533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1569936533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:07.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:07.585155+0000) 2023-08-27T21:57:08.585 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:08 smithi136 ceph-mon[113531]: pgmap v1908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:08 smithi093 ceph-mon[111335]: pgmap v1908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:08 smithi093 ceph-mon[118902]: pgmap v1908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:08.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:08.585414+0000) 2023-08-27T21:57:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1263753234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:09.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:09.585702+0000) 2023-08-27T21:57:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1263753234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1263753234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:10 smithi136 ceph-mon[113531]: pgmap v1909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:10.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:10.585905+0000) 2023-08-27T21:57:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:10 smithi093 ceph-mon[111335]: pgmap v1909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:10 smithi093 ceph-mon[118902]: pgmap v1909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:10.359Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:10.360Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:11.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:11.586140+0000) 2023-08-27T21:57:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:11.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:11.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:12 smithi136 ceph-mon[113531]: pgmap v1910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1007170829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:12.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:12.586396+0000) 2023-08-27T21:57:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:12 smithi093 ceph-mon[111335]: pgmap v1910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1007170829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:12 smithi093 ceph-mon[118902]: pgmap v1910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1007170829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:13.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:13.586633+0000) 2023-08-27T21:57:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:13.784Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:13.785Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:14 smithi136 ceph-mon[113531]: pgmap v1911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1027197259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:14.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:14.586863+0000) 2023-08-27T21:57:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:14 smithi093 ceph-mon[111335]: pgmap v1911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1027197259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:14 smithi093 ceph-mon[118902]: pgmap v1911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1027197259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:57:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:57:15] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T21:57:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:15.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:15.587130+0000) 2023-08-27T21:57:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:16 smithi136 ceph-mon[113531]: pgmap v1912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:16.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:16.587378+0000) 2023-08-27T21:57:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:16 smithi093 ceph-mon[111335]: pgmap v1912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:16 smithi093 ceph-mon[118902]: pgmap v1912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/853954605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:17.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:17.587626+0000) 2023-08-27T21:57:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/853954605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/853954605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:18 smithi136 ceph-mon[113531]: pgmap v1913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:18.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:18.587859+0000) 2023-08-27T21:57:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:18 smithi093 ceph-mon[111335]: pgmap v1913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:18 smithi093 ceph-mon[118902]: pgmap v1913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/343246242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:19.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:19.588112+0000) 2023-08-27T21:57:19.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/343246242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:19.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/343246242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:20 smithi136 ceph-mon[113531]: pgmap v1914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:20.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:20.588311+0000) 2023-08-27T21:57:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:20 smithi093 ceph-mon[111335]: pgmap v1914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:20 smithi093 ceph-mon[118902]: pgmap v1914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:20.360Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:20.361Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2966891158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:21.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2966891158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:57:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2966891158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:21.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:21.588566+0000) 2023-08-27T21:57:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:21.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:21.875Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:22 smithi136 ceph-mon[113531]: pgmap v1915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:22.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:22.588761+0000) 2023-08-27T21:57:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:22 smithi093 ceph-mon[111335]: pgmap v1915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:22 smithi093 ceph-mon[118902]: pgmap v1915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.786 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:23.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:23.787 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:23.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:23.588979+0000) 2023-08-27T21:57:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:57:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:57:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:57:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:23.785Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:23.786Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:24 smithi136 ceph-mon[113531]: pgmap v1916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2138970847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:24.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:24.589204+0000) 2023-08-27T21:57:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:24 smithi093 ceph-mon[118902]: pgmap v1916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2138970847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:24 smithi093 ceph-mon[111335]: pgmap v1916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2138970847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:57:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:57:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:57:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:25.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:25.589418+0000) 2023-08-27T21:57:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:26 smithi136 ceph-mon[113531]: pgmap v1917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3538966939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:26.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:26.589635+0000) 2023-08-27T21:57:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:26 smithi093 ceph-mon[111335]: pgmap v1917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3538966939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:26 smithi093 ceph-mon[118902]: pgmap v1917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3538966939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:27.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:27.589926+0000) 2023-08-27T21:57:28.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:28 smithi136 ceph-mon[113531]: pgmap v1918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:28.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3290479467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:28.943 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:28.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:28.590181+0000) 2023-08-27T21:57:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:28 smithi093 ceph-mon[111335]: pgmap v1918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3290479467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:28 smithi093 ceph-mon[118902]: pgmap v1918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3290479467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:29.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:29.590394+0000) 2023-08-27T21:57:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:30 smithi136 ceph-mon[113531]: pgmap v1919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:30 smithi093 ceph-mon[111335]: pgmap v1919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:30 smithi093 ceph-mon[118902]: pgmap v1919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:30.360Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:30.361Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:30.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:30.590618+0000) 2023-08-27T21:57:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1738932897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1738932897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1738932897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:31.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:31.590861+0000) 2023-08-27T21:57:32.161 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:31.875Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:32.161 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:31.876Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:32 smithi136 ceph-mon[113531]: pgmap v1920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:32 smithi093 ceph-mon[111335]: pgmap v1920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:32 smithi093 ceph-mon[118902]: pgmap v1920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:32.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:32.591101+0000) 2023-08-27T21:57:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2013448845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2013448845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2013448845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:33.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:33.591358+0000) 2023-08-27T21:57:34.167 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:33.786Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:34.167 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:33.787Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:34 smithi136 ceph-mon[113531]: pgmap v1921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:34 smithi093 ceph-mon[111335]: pgmap v1921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:34 smithi093 ceph-mon[118902]: pgmap v1921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:34.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:34.591607+0000) 2023-08-27T21:57:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:57:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:57:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:57:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:35.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:35.591885+0000) 2023-08-27T21:57:36.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:36 smithi136 ceph-mon[113531]: pgmap v1922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:36.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2719503049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:36 smithi093 ceph-mon[111335]: pgmap v1922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2719503049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:36 smithi093 ceph-mon[118902]: pgmap v1922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2719503049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:36.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:36.592139+0000) 2023-08-27T21:57:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:37.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:37.592433+0000) 2023-08-27T21:57:38.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:38 smithi136 ceph-mon[113531]: pgmap v1923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:38.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3343238182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:38 smithi093 ceph-mon[111335]: pgmap v1923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3343238182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:38 smithi093 ceph-mon[118902]: pgmap v1923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3343238182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:38.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:38.592616+0000) 2023-08-27T21:57:39.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:39.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:39.592904+0000) 2023-08-27T21:57:40.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:40 smithi136 ceph-mon[113531]: pgmap v1924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:40.658 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3499547172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:40 smithi093 ceph-mon[111335]: pgmap v1924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3499547172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:40 smithi093 ceph-mon[118902]: pgmap v1924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3499547172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:40.360Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:40.362Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:40.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:40.593174+0000) 2023-08-27T21:57:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:41.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:41.593379+0000) 2023-08-27T21:57:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:41.876Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:41.877Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:42.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:42 smithi136 ceph-mon[113531]: pgmap v1925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:42 smithi093 ceph-mon[111335]: pgmap v1925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:42 smithi093 ceph-mon[118902]: pgmap v1925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:42.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:42.593653+0000) 2023-08-27T21:57:43.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1107367216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1107367216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1107367216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:43.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:43.593899+0000) 2023-08-27T21:57:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:43.786Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:43.788Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:44.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:44 smithi136 ceph-mon[113531]: pgmap v1926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:44 smithi093 ceph-mon[111335]: pgmap v1926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:44 smithi093 ceph-mon[118902]: pgmap v1926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:44.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:44.594170+0000) 2023-08-27T21:57:45.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3900290689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3900290689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:57:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:57:45] "GET /metrics HTTP/1.1" 200 239876 "" "Prometheus/2.43.0" 2023-08-27T21:57:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3900290689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:45.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:45.594344+0000) 2023-08-27T21:57:46.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:46 smithi136 ceph-mon[113531]: pgmap v1927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:46 smithi093 ceph-mon[111335]: pgmap v1927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:46 smithi093 ceph-mon[118902]: pgmap v1927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:46.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:46.594562+0000) 2023-08-27T21:57:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:47.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:47.594805+0000) 2023-08-27T21:57:48.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:48 smithi136 ceph-mon[113531]: pgmap v1928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:48.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/204330054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:48 smithi093 ceph-mon[111335]: pgmap v1928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/204330054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:48 smithi093 ceph-mon[118902]: pgmap v1928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/204330054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:48.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:48.595050+0000) 2023-08-27T21:57:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:49.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:49.595259+0000) 2023-08-27T21:57:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:50 smithi136 ceph-mon[113531]: pgmap v1929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4123750905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:50 smithi093 ceph-mon[118902]: pgmap v1929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4123750905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:50 smithi093 ceph-mon[111335]: pgmap v1929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4123750905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:50.361Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:50.362Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:50.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:50.595495+0000) 2023-08-27T21:57:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:51.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:51.595732+0000) 2023-08-27T21:57:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:51.877Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:51.878Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:52.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:52 smithi136 ceph-mon[113531]: pgmap v1930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:52.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3356940966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:52.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:52 smithi093 ceph-mon[111335]: pgmap v1930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:52.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3356940966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:52.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:52 smithi093 ceph-mon[118902]: pgmap v1930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:52.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3356940966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:52.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:52.595981+0000) 2023-08-27T21:57:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:53.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:53.596250+0000) 2023-08-27T21:57:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:53.787Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:57:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:57:53.789Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:57:54.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:54 smithi136 ceph-mon[113531]: pgmap v1931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:54.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:54 smithi093 ceph-mon[111335]: pgmap v1931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:54 smithi093 ceph-mon[118902]: pgmap v1931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:57:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:54.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:54.596518+0000) 2023-08-27T21:57:55.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4114375009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4114375009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4114375009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:57:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:57:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:57:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:55.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:55.596732+0000) 2023-08-27T21:57:56.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:56 smithi136 ceph-mon[113531]: pgmap v1932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:56 smithi093 ceph-mon[111335]: pgmap v1932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:56 smithi093 ceph-mon[118902]: pgmap v1932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:56.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:56.596947+0000) 2023-08-27T21:57:57.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1453871029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1453871029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1453871029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:57:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:57.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:57.597155+0000) 2023-08-27T21:57:58.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:57:58 smithi136 ceph-mon[113531]: pgmap v1933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:57:58 smithi093 ceph-mon[111335]: pgmap v1933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:57:58 smithi093 ceph-mon[118902]: pgmap v1933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:57:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:58.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:58.597398+0000) 2023-08-27T21:57:59.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:57:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:57:59.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:57:59.597637+0000) 2023-08-27T21:58:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:00 smithi136 ceph-mon[113531]: pgmap v1934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4132556057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:00 smithi093 ceph-mon[111335]: pgmap v1934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4132556057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:00 smithi093 ceph-mon[118902]: pgmap v1934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4132556057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:00.361Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:00.362Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:00.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:00.597909+0000) 2023-08-27T21:58:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:01.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:01.598142+0000) 2023-08-27T21:58:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:01.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:01.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:02.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:02 smithi136 ceph-mon[113531]: pgmap v1935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:02.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3219773856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:02 smithi093 ceph-mon[111335]: pgmap v1935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3219773856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:02 smithi093 ceph-mon[118902]: pgmap v1935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3219773856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:02.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:02.598359+0000) 2023-08-27T21:58:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:03.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:03.598603+0000) 2023-08-27T21:58:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:03.788Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:03.789Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:04.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:04 smithi136 ceph-mon[113531]: pgmap v1936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:04.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/488134674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:04 smithi093 ceph-mon[111335]: pgmap v1936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/488134674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:04 smithi093 ceph-mon[118902]: pgmap v1936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/488134674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:04.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:04.598844+0000) 2023-08-27T21:58:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:58:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:58:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:58:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:05.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:05.599112+0000) 2023-08-27T21:58:06.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:06 smithi136 ceph-mon[113531]: pgmap v1937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:06 smithi093 ceph-mon[111335]: pgmap v1937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:06 smithi093 ceph-mon[118902]: pgmap v1937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:06.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:06.599326+0000) 2023-08-27T21:58:07.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1741076654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1741076654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1741076654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:07.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:07.599571+0000) 2023-08-27T21:58:08.602 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:08 smithi136 ceph-mon[113531]: pgmap v1938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:08 smithi093 ceph-mon[111335]: pgmap v1938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:08 smithi093 ceph-mon[118902]: pgmap v1938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:08.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:08.599739+0000) 2023-08-27T21:58:09.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:09.600 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002645432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002645432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002645432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:09.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:09.599996+0000) 2023-08-27T21:58:10.600 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:10 smithi136 ceph-mon[113531]: pgmap v1939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:10 smithi093 ceph-mon[111335]: pgmap v1939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:10 smithi093 ceph-mon[118902]: pgmap v1939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:10.362Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:10.363Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:10.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:10.600223+0000) 2023-08-27T21:58:11.600 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1153620156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1153620156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1153620156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:11.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:11.600482+0000) 2023-08-27T21:58:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:11.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:11.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:12 smithi136 ceph-mon[113531]: pgmap v1940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:12.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:12.600704+0000) 2023-08-27T21:58:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:12 smithi093 ceph-mon[111335]: pgmap v1940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:12 smithi093 ceph-mon[118902]: pgmap v1940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:13.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:13.600941+0000) 2023-08-27T21:58:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:13.789Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:13.790Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:14 smithi136 ceph-mon[113531]: pgmap v1941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2173050147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:14.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:14.601205+0000) 2023-08-27T21:58:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:14 smithi093 ceph-mon[111335]: pgmap v1941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2173050147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:14 smithi093 ceph-mon[118902]: pgmap v1941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2173050147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:58:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:58:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:58:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:15.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:15.601388+0000) 2023-08-27T21:58:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:16 smithi136 ceph-mon[113531]: pgmap v1942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/210895271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:16.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:16.601558+0000) 2023-08-27T21:58:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:16 smithi093 ceph-mon[111335]: pgmap v1942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/210895271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:16 smithi093 ceph-mon[118902]: pgmap v1942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/210895271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:17.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:17.601811+0000) 2023-08-27T21:58:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:18 smithi136 ceph-mon[113531]: pgmap v1943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:18.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:18.602092+0000) 2023-08-27T21:58:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:18 smithi093 ceph-mon[111335]: pgmap v1943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:18 smithi093 ceph-mon[118902]: pgmap v1943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2745605506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:19.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:19.602357+0000) 2023-08-27T21:58:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2745605506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2745605506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:20 smithi136 ceph-mon[113531]: pgmap v1944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:20.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:20.602655+0000) 2023-08-27T21:58:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:20 smithi093 ceph-mon[111335]: pgmap v1944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:20 smithi093 ceph-mon[118902]: pgmap v1944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:20.362Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:20.363Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4009731762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:21.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:21.602906+0000) 2023-08-27T21:58:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4009731762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4009731762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:21.879Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:21.880Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:22 smithi136 ceph-mon[113531]: pgmap v1945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:22.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:22.603169+0000) 2023-08-27T21:58:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:22 smithi093 ceph-mon[111335]: pgmap v1945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:22 smithi093 ceph-mon[118902]: pgmap v1945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2939846902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:23.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:23.603344+0000) 2023-08-27T21:58:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2939846902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2939846902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:24.103 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:23.790Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:24.103 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:23.791Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:24.649 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:24 smithi093 ceph-mon[111335]: pgmap v1946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:24.649 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:24.649 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:24.649 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:24 smithi093 ceph-mon[118902]: pgmap v1946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:24.649 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:24.649 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:24 smithi136 ceph-mon[113531]: pgmap v1946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:58:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:24.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:24.603534+0000) 2023-08-27T21:58:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:58:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:58:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:58:25.885 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:25.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:25.603747+0000) 2023-08-27T21:58:26.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:26 smithi136 ceph-mon[113531]: pgmap v1947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:26.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1832470002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:26.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[111335]: pgmap v1947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1832470002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[118902]: pgmap v1947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1832470002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:26.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:26.603926+0000) 2023-08-27T21:58:27.604 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:27.604 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:58:27.605 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:58:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:58:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:58:27.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:58:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:27.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:27.604170+0000) 2023-08-27T21:58:28.604 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:28 smithi136 ceph-mon[113531]: pgmap v1948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:28.604 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4202433019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:28 smithi093 ceph-mon[111335]: pgmap v1948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4202433019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:28 smithi093 ceph-mon[118902]: pgmap v1948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4202433019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:28.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:28.604378+0000) 2023-08-27T21:58:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:29.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:29.604651+0000) 2023-08-27T21:58:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:30 smithi136 ceph-mon[113531]: pgmap v1949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:30 smithi093 ceph-mon[111335]: pgmap v1949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:30 smithi093 ceph-mon[118902]: pgmap v1949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:30.362Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:30.363Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:30.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:30.604880+0000) 2023-08-27T21:58:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2508469033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2508469033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2508469033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:31.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:31.605136+0000) 2023-08-27T21:58:32.170 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:31.880Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:32.171 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:31.881Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:32 smithi136 ceph-mon[113531]: pgmap v1950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:32 smithi093 ceph-mon[111335]: pgmap v1950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:32 smithi093 ceph-mon[118902]: pgmap v1950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:32.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:32.605385+0000) 2023-08-27T21:58:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2135161488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2135161488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2135161488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:33.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:33.605585+0000) 2023-08-27T21:58:34.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:33.790Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:34.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:33.791Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:34 smithi136 ceph-mon[113531]: pgmap v1951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:34 smithi093 ceph-mon[111335]: pgmap v1951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:34 smithi093 ceph-mon[118902]: pgmap v1951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:34.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:34.605851+0000) 2023-08-27T21:58:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4150796044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4150796044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4150796044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:58:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:58:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T21:58:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:35.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:35.606145+0000) 2023-08-27T21:58:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:36 smithi136 ceph-mon[113531]: pgmap v1952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:36 smithi093 ceph-mon[111335]: pgmap v1952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:36 smithi093 ceph-mon[118902]: pgmap v1952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:36.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:36.606356+0000) 2023-08-27T21:58:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:37.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:37.606566+0000) 2023-08-27T21:58:38.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:38 smithi136 ceph-mon[113531]: pgmap v1953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:38.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1579809671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:38 smithi093 ceph-mon[111335]: pgmap v1953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1579809671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:38 smithi093 ceph-mon[118902]: pgmap v1953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1579809671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:38.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:38.606789+0000) 2023-08-27T21:58:39.607 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:39.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:39.607077+0000) 2023-08-27T21:58:40.607 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:40 smithi136 ceph-mon[113531]: pgmap v1954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:40.607 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1602436253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:40 smithi093 ceph-mon[111335]: pgmap v1954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1602436253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:40 smithi093 ceph-mon[118902]: pgmap v1954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1602436253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:40.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:40.364Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:40.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:40.364Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:40.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:40.607355+0000) 2023-08-27T21:58:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:41.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:41.607613+0000) 2023-08-27T21:58:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:41.881Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:41.882Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:42.607 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:42 smithi136 ceph-mon[113531]: pgmap v1955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:42.608 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4047640726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:42 smithi093 ceph-mon[111335]: pgmap v1955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4047640726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:42 smithi093 ceph-mon[118902]: pgmap v1955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4047640726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:42.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:42.607893+0000) 2023-08-27T21:58:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:43.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:43.608154+0000) 2023-08-27T21:58:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:43.792Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:43.793Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:44.608 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:44 smithi136 ceph-mon[113531]: pgmap v1956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:44 smithi093 ceph-mon[111335]: pgmap v1956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:44 smithi093 ceph-mon[118902]: pgmap v1956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:44.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:44.608421+0000) 2023-08-27T21:58:45.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3823392756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3823392756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:58:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:58:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T21:58:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3823392756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:45.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:45.608655+0000) 2023-08-27T21:58:46.608 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:46 smithi136 ceph-mon[113531]: pgmap v1957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:46 smithi093 ceph-mon[111335]: pgmap v1957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:46 smithi093 ceph-mon[118902]: pgmap v1957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:46.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:46.608862+0000) 2023-08-27T21:58:47.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2316099691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2316099691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2316099691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:47.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:47.609126+0000) 2023-08-27T21:58:48.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:48 smithi136 ceph-mon[113531]: pgmap v1958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:48 smithi093 ceph-mon[118902]: pgmap v1958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:48 smithi093 ceph-mon[111335]: pgmap v1958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:48.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:48.609323+0000) 2023-08-27T21:58:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:49.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:49.609576+0000) 2023-08-27T21:58:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:50 smithi136 ceph-mon[113531]: pgmap v1959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/17108978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:50 smithi093 ceph-mon[111335]: pgmap v1959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/17108978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:50 smithi093 ceph-mon[118902]: pgmap v1959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/17108978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:50.364Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:50.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:50.365Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:50.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:50.609817+0000) 2023-08-27T21:58:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:51.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:51.609972+0000) 2023-08-27T21:58:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:51.881Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:51.882Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:52.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:52 smithi136 ceph-mon[113531]: pgmap v1960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:52.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3241792945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:52 smithi093 ceph-mon[111335]: pgmap v1960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3241792945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:52 smithi093 ceph-mon[118902]: pgmap v1960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3241792945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:52.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:52.610231+0000) 2023-08-27T21:58:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:53.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:53.610473+0000) 2023-08-27T21:58:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:53.792Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:58:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:58:53.794Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:58:54.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:54 smithi136 ceph-mon[113531]: pgmap v1961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:54.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:54.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1875773273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:54 smithi093 ceph-mon[111335]: pgmap v1961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1875773273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:54 smithi093 ceph-mon[118902]: pgmap v1961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:58:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1875773273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:54.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:54.610751+0000) 2023-08-27T21:58:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:58:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:58:55] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:58:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:55.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:55.610931+0000) 2023-08-27T21:58:56.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:56 smithi136 ceph-mon[113531]: pgmap v1962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:56 smithi093 ceph-mon[111335]: pgmap v1962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:56 smithi093 ceph-mon[118902]: pgmap v1962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:56.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:56.611152+0000) 2023-08-27T21:58:57.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2174596634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2174596634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2174596634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:57.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:57.611371+0000) 2023-08-27T21:58:58.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:58 smithi136 ceph-mon[113531]: pgmap v1963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:58 smithi093 ceph-mon[111335]: pgmap v1963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:58 smithi093 ceph-mon[118902]: pgmap v1963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:58:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:58.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:58.611576+0000) 2023-08-27T21:58:59.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:58:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/684221595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:58:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/684221595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:58:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/684221595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:58:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:58:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:58:59.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:58:59.611818+0000) 2023-08-27T21:59:00.612 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:00 smithi136 ceph-mon[113531]: pgmap v1964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:00 smithi093 ceph-mon[111335]: pgmap v1964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:00 smithi093 ceph-mon[118902]: pgmap v1964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:00.365Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:00.366Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:00.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:00.612024+0000) 2023-08-27T21:59:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/673150550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:01.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:01.612228+0000) 2023-08-27T21:59:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/673150550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/673150550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:01.882Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:01.883Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:02 smithi136 ceph-mon[113531]: pgmap v1965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:02.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:02.612478+0000) 2023-08-27T21:59:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:02 smithi093 ceph-mon[111335]: pgmap v1965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:02 smithi093 ceph-mon[118902]: pgmap v1965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:03.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:03.612760+0000) 2023-08-27T21:59:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:03.793Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:03.794Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:04 smithi136 ceph-mon[113531]: pgmap v1966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3232334376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:04.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:04.613053+0000) 2023-08-27T21:59:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:04 smithi093 ceph-mon[118902]: pgmap v1966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3232334376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:04 smithi093 ceph-mon[111335]: pgmap v1966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3232334376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:59:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:59:05] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T21:59:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:05.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:05.613247+0000) 2023-08-27T21:59:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:06 smithi136 ceph-mon[113531]: pgmap v1967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/675390785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:06.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:06.613448+0000) 2023-08-27T21:59:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:06 smithi093 ceph-mon[111335]: pgmap v1967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/675390785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:06 smithi093 ceph-mon[118902]: pgmap v1967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/675390785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:07.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:07.613715+0000) 2023-08-27T21:59:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:08 smithi136 ceph-mon[113531]: pgmap v1968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:08.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:08.613975+0000) 2023-08-27T21:59:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:08 smithi093 ceph-mon[111335]: pgmap v1968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:08 smithi093 ceph-mon[118902]: pgmap v1968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3973949038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:09.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:09.614212+0000) 2023-08-27T21:59:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3973949038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3973949038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:10 smithi136 ceph-mon[113531]: pgmap v1969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:10.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:10.614419+0000) 2023-08-27T21:59:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:10 smithi093 ceph-mon[111335]: pgmap v1969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:10.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:10 smithi093 ceph-mon[118902]: pgmap v1969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:10.366Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:10.367Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3906188825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:11.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:11.614594+0000) 2023-08-27T21:59:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3906188825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3906188825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:11.882Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:11.883Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:12 smithi136 ceph-mon[113531]: pgmap v1970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:12.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:12.614839+0000) 2023-08-27T21:59:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:12 smithi093 ceph-mon[111335]: pgmap v1970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:12 smithi093 ceph-mon[118902]: pgmap v1970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/240553855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:13.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:13.615020+0000) 2023-08-27T21:59:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/240553855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/240553855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:13.794Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:13.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:14 smithi136 ceph-mon[113531]: pgmap v1971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:14.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:14.615250+0000) 2023-08-27T21:59:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:14 smithi093 ceph-mon[111335]: pgmap v1971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:14 smithi093 ceph-mon[118902]: pgmap v1971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:59:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:59:15] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T21:59:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:15.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:15.615468+0000) 2023-08-27T21:59:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:16.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:16.615656+0000) 2023-08-27T21:59:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:16 smithi136 ceph-mon[113531]: pgmap v1972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2850085562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:16 smithi093 ceph-mon[111335]: pgmap v1972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2850085562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:16 smithi093 ceph-mon[118902]: pgmap v1972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2850085562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:17.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:17.615871+0000) 2023-08-27T21:59:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:18.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:18.616129+0000) 2023-08-27T21:59:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:18 smithi136 ceph-mon[113531]: pgmap v1973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2885973830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:18 smithi093 ceph-mon[111335]: pgmap v1973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2885973830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:18 smithi093 ceph-mon[118902]: pgmap v1973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2885973830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:19.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:19.616326+0000) 2023-08-27T21:59:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:20 smithi136 ceph-mon[113531]: pgmap v1974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:20 smithi093 ceph-mon[111335]: pgmap v1974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:20 smithi093 ceph-mon[118902]: pgmap v1974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:20.367Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:20.368Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:20.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:20.616505+0000) 2023-08-27T21:59:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1991222171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1991222171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1991222171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:21.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:21.616724+0000) 2023-08-27T21:59:22.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:21.883Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:22.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:21.884Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:22 smithi136 ceph-mon[113531]: pgmap v1975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:22 smithi093 ceph-mon[111335]: pgmap v1975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:22 smithi093 ceph-mon[118902]: pgmap v1975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:22.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:22.616976+0000) 2023-08-27T21:59:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2100390078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2100390078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2100390078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:23.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:23.617220+0000) 2023-08-27T21:59:24.184 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:23.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:24.184 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:23.796Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:24 smithi136 ceph-mon[113531]: pgmap v1976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:24 smithi093 ceph-mon[111335]: pgmap v1976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:24 smithi093 ceph-mon[118902]: pgmap v1976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:24.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:24.617455+0000) 2023-08-27T21:59:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/249104249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/249104249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/249104249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:59:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:59:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:59:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:25.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:25.617717+0000) 2023-08-27T21:59:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:26 smithi093 ceph-mon[111335]: pgmap v1977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:26 smithi093 ceph-mon[118902]: pgmap v1977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:26.617 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:26 smithi136 ceph-mon[113531]: pgmap v1977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:26.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:26.617941+0000) 2023-08-27T21:59:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:27.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T21:59:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:27.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:27.618136+0000) 2023-08-27T21:59:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:28 smithi093 ceph-mon[111335]: pgmap v1978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3846044745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:28 smithi093 ceph-mon[118902]: pgmap v1978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3846044745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:28.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:28 smithi136 ceph-mon[113531]: pgmap v1978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:28.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3846044745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:28.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:28.618285+0000) 2023-08-27T21:59:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:29.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:29.618485+0000) 2023-08-27T21:59:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[111335]: pgmap v1979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3417406425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[118902]: pgmap v1979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3417406425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:30.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:30.368Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:30.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:30.369Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:30 smithi136 ceph-mon[113531]: pgmap v1979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T21:59:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T21:59:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T21:59:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3417406425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:30.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:30.618710+0000) 2023-08-27T21:59:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:31.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:31.618950+0000) 2023-08-27T21:59:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:31.884Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:31.885Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:32 smithi093 ceph-mon[111335]: pgmap v1980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3729404988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:32 smithi093 ceph-mon[118902]: pgmap v1980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3729404988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:32 smithi136 ceph-mon[113531]: pgmap v1980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3729404988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:32.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:32.619185+0000) 2023-08-27T21:59:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:33.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:33.619435+0000) 2023-08-27T21:59:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:33.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:33.796Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:34 smithi093 ceph-mon[111335]: pgmap v1981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:34 smithi093 ceph-mon[118902]: pgmap v1981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:34 smithi136 ceph-mon[113531]: pgmap v1981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:34.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:34.619725+0000) 2023-08-27T21:59:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1680353519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:59:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:59:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:59:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1680353519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1680353519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:35.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:35.619927+0000) 2023-08-27T21:59:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:36 smithi093 ceph-mon[111335]: pgmap v1982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:36 smithi093 ceph-mon[118902]: pgmap v1982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:36 smithi136 ceph-mon[113531]: pgmap v1982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:36.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:36.620167+0000) 2023-08-27T21:59:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1066111567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1066111567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1066111567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:37.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:37.620412+0000) 2023-08-27T21:59:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:38 smithi093 ceph-mon[111335]: pgmap v1983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:38 smithi093 ceph-mon[118902]: pgmap v1983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:38 smithi136 ceph-mon[113531]: pgmap v1983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:38.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:38.620622+0000) 2023-08-27T21:59:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1397851633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1397851633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1397851633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:39.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:39.620872+0000) 2023-08-27T21:59:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:40 smithi093 ceph-mon[111335]: pgmap v1984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:40 smithi093 ceph-mon[118902]: pgmap v1984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:40.369Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:40.370Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:40 smithi136 ceph-mon[113531]: pgmap v1984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:40.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:40.621159+0000) 2023-08-27T21:59:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:41.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:41.621417+0000) 2023-08-27T21:59:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:41.885Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:41.886Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:42 smithi093 ceph-mon[111335]: pgmap v1985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3743707646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:42 smithi093 ceph-mon[118902]: pgmap v1985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3743707646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:42 smithi136 ceph-mon[113531]: pgmap v1985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3743707646' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:42.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:42.621656+0000) 2023-08-27T21:59:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:43.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:43.621900+0000) 2023-08-27T21:59:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:43.796Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:43.797Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:44 smithi093 ceph-mon[111335]: pgmap v1986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3739257096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:44 smithi093 ceph-mon[118902]: pgmap v1986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3739257096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:44 smithi136 ceph-mon[113531]: pgmap v1986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3739257096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:44.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:44.622190+0000) 2023-08-27T21:59:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:59:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:59:45] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T21:59:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:45.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:45.622365+0000) 2023-08-27T21:59:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:46 smithi093 ceph-mon[111335]: pgmap v1987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:46 smithi093 ceph-mon[118902]: pgmap v1987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:46 smithi136 ceph-mon[113531]: pgmap v1987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:46.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:46.622570+0000) 2023-08-27T21:59:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4262962808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4262962808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4262962808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:47.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:47.622816+0000) 2023-08-27T21:59:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:48 smithi093 ceph-mon[111335]: pgmap v1988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:48 smithi093 ceph-mon[118902]: pgmap v1988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:48 smithi136 ceph-mon[113531]: pgmap v1988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:48.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:48.623058+0000) 2023-08-27T21:59:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2189900938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:49.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:49.623300+0000) 2023-08-27T21:59:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2189900938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2189900938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:50 smithi093 ceph-mon[111335]: pgmap v1989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:50 smithi093 ceph-mon[118902]: pgmap v1989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:50.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:50.369Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:50.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:50.371Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:50 smithi136 ceph-mon[113531]: pgmap v1989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:50.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:50.623539+0000) 2023-08-27T21:59:51.886 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3776805296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:51.886 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3776805296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3776805296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:51.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:51.623720+0000) 2023-08-27T21:59:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:51.886Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:51.886Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:52 smithi136 ceph-mon[113531]: pgmap v1990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:52.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:52.623933+0000) 2023-08-27T21:59:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:52 smithi093 ceph-mon[111335]: pgmap v1990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:52 smithi093 ceph-mon[118902]: pgmap v1990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:53.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:53.624113+0000) 2023-08-27T21:59:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:53.797Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 21:59:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T21:59:53.798Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T21:59:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:54 smithi136 ceph-mon[113531]: pgmap v1991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/632411850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:54.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:54.624368+0000) 2023-08-27T21:59:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:54 smithi093 ceph-mon[111335]: pgmap v1991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/632411850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:54 smithi093 ceph-mon[118902]: pgmap v1991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T21:59:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/632411850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 21:59:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:21:59:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T21:59:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:55.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:55.624602+0000) 2023-08-27T21:59:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:56 smithi136 ceph-mon[113531]: pgmap v1992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2152766189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:56.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:56.624858+0000) 2023-08-27T21:59:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:56 smithi093 ceph-mon[111335]: pgmap v1992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2152766189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:56 smithi093 ceph-mon[118902]: pgmap v1992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2152766189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:57.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:57.625108+0000) 2023-08-27T21:59:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:58 smithi136 ceph-mon[113531]: pgmap v1993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 21:59:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2985818183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:58.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:58.625354+0000) 2023-08-27T21:59:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:58 smithi093 ceph-mon[111335]: pgmap v1993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 21:59:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2985818183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:58 smithi093 ceph-mon[118902]: pgmap v1993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T21:59:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 21:59:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2985818183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T21:59:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 21:59:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T21:59:59.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T20:59:59.625596+0000) 2023-08-27T22:00:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:00 smithi136 ceph-mon[113531]: pgmap v1994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:00 smithi093 ceph-mon[111335]: pgmap v1994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:00 smithi093 ceph-mon[118902]: pgmap v1994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:00:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:00.370Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:00.372Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:00.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:00.625795+0000) 2023-08-27T22:00:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3378200968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3378200968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3378200968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:01.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:01.626026+0000) 2023-08-27T22:00:02.182 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:01.886Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:02.182 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:01.887Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:02 smithi136 ceph-mon[113531]: pgmap v1995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:02 smithi093 ceph-mon[111335]: pgmap v1995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:02 smithi093 ceph-mon[118902]: pgmap v1995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:02.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:02.626307+0000) 2023-08-27T22:00:03.626 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/967159061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/967159061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/967159061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:03.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:03.626550+0000) 2023-08-27T22:00:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:03.798Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:03.799Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:04.626 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:04 smithi136 ceph-mon[113531]: pgmap v1996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:04 smithi093 ceph-mon[111335]: pgmap v1996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:04 smithi093 ceph-mon[118902]: pgmap v1996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:04.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:04.626850+0000) 2023-08-27T22:00:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:00:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:00:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:00:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:05.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:05.627161+0000) 2023-08-27T22:00:06.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:06 smithi136 ceph-mon[113531]: pgmap v1997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:06.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3247483945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:06 smithi093 ceph-mon[111335]: pgmap v1997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3247483945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:06 smithi093 ceph-mon[118902]: pgmap v1997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3247483945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:06.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:06.627422+0000) 2023-08-27T22:00:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:07.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:07.627666+0000) 2023-08-27T22:00:08.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:08 smithi136 ceph-mon[113531]: pgmap v1998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:08.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/975949342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:08 smithi093 ceph-mon[111335]: pgmap v1998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/975949342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:08 smithi093 ceph-mon[118902]: pgmap v1998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/975949342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:08.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:08.627853+0000) 2023-08-27T22:00:09.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:09.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:09.628096+0000) 2023-08-27T22:00:10.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:10 smithi136 ceph-mon[113531]: pgmap v1999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:10 smithi093 ceph-mon[111335]: pgmap v1999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:10 smithi093 ceph-mon[118902]: pgmap v1999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:10.371Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:10.372Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:10.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:10.628294+0000) 2023-08-27T22:00:11.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/90910273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/90910273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/90910273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:11.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:11.628501+0000) 2023-08-27T22:00:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:11.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:11.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:12.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:12 smithi136 ceph-mon[113531]: pgmap v2000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:12 smithi093 ceph-mon[111335]: pgmap v2000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:12 smithi093 ceph-mon[118902]: pgmap v2000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:12.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:12.628701+0000) 2023-08-27T22:00:13.629 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2339290244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2339290244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2339290244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:13.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:13.628917+0000) 2023-08-27T22:00:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:13.799Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:13.800Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:14 smithi136 ceph-mon[113531]: pgmap v2001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:14.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:14.629203+0000) 2023-08-27T22:00:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:14 smithi093 ceph-mon[111335]: pgmap v2001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:14 smithi093 ceph-mon[118902]: pgmap v2001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1128865916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:15.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:15.629405+0000) 2023-08-27T22:00:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1128865916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1128865916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:00:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:00:15] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:00:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:16 smithi136 ceph-mon[113531]: pgmap v2002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:16.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:16.629550+0000) 2023-08-27T22:00:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:16 smithi093 ceph-mon[111335]: pgmap v2002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:16 smithi093 ceph-mon[118902]: pgmap v2002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:17.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:17.629806+0000) 2023-08-27T22:00:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:18 smithi136 ceph-mon[113531]: pgmap v2003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/204530377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:18.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:18.630085+0000) 2023-08-27T22:00:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:18 smithi093 ceph-mon[111335]: pgmap v2003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/204530377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:18 smithi093 ceph-mon[118902]: pgmap v2003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/204530377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:19.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:19.630355+0000) 2023-08-27T22:00:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:20 smithi136 ceph-mon[113531]: pgmap v2004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/355958573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:20 smithi093 ceph-mon[118902]: pgmap v2004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/355958573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:20 smithi093 ceph-mon[111335]: pgmap v2004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/355958573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:20.371Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:20.372Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:20.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:20.630580+0000) 2023-08-27T22:00:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:21.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:21.630849+0000) 2023-08-27T22:00:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:21.888Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:21.889Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:22.631 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:22 smithi136 ceph-mon[113531]: pgmap v2005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:22.631 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1622616905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:22 smithi093 ceph-mon[111335]: pgmap v2005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1622616905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:22 smithi093 ceph-mon[118902]: pgmap v2005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1622616905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:22.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:22.631168+0000) 2023-08-27T22:00:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:23.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:23.631426+0000) 2023-08-27T22:00:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:23.800Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:23.801Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:24.631 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:24 smithi136 ceph-mon[113531]: pgmap v2006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:24.631 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:24 smithi093 ceph-mon[111335]: pgmap v2006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:24 smithi093 ceph-mon[118902]: pgmap v2006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:24.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:24.631598+0000) 2023-08-27T22:00:25.631 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/331377240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/331377240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:00:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:00:25] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:00:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/331377240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:25.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:25.631890+0000) 2023-08-27T22:00:26.632 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:26 smithi136 ceph-mon[113531]: pgmap v2007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:26 smithi093 ceph-mon[111335]: pgmap v2007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:26 smithi093 ceph-mon[118902]: pgmap v2007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:26.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:26.632165+0000) 2023-08-27T22:00:27.632 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3772403109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3772403109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3772403109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:27.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:27.632384+0000) 2023-08-27T22:00:28.632 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:28 smithi136 ceph-mon[113531]: pgmap v2008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:28 smithi093 ceph-mon[111335]: pgmap v2008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:28.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:28 smithi093 ceph-mon[118902]: pgmap v2008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:28.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:28.632649+0000) 2023-08-27T22:00:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:29.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:29.632905+0000) 2023-08-27T22:00:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:30 smithi136 ceph-mon[113531]: pgmap v2009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1806489144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:00:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[111335]: pgmap v2009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1806489144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:00:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[118902]: pgmap v2009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1806489144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:30.372Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:30.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:30.373Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:30.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:30.633156+0000) 2023-08-27T22:00:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:31.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:31.633321+0000) 2023-08-27T22:00:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:31.889Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:31.890Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:32.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:32 smithi136 ceph-mon[113531]: pgmap v2010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:32.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3732730806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:32.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[111335]: pgmap v2010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3732730806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[118902]: pgmap v2010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3732730806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:32.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:32.633534+0000) 2023-08-27T22:00:33.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:00:33.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:00:33.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:00:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:00:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:33.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:00:33.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:00:33.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:00:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:33.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:33.633726+0000) 2023-08-27T22:00:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:33.801Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:33.802Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:34 smithi136 ceph-mon[113531]: pgmap v2011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/712274921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:34.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:34.633979+0000) 2023-08-27T22:00:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:34 smithi093 ceph-mon[111335]: pgmap v2011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/712274921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:34 smithi093 ceph-mon[118902]: pgmap v2011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/712274921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:00:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:00:35] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:00:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:35.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:35.634252+0000) 2023-08-27T22:00:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:36 smithi136 ceph-mon[113531]: pgmap v2012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1397622599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:36.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:36.634548+0000) 2023-08-27T22:00:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:36 smithi093 ceph-mon[111335]: pgmap v2012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1397622599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:36 smithi093 ceph-mon[118902]: pgmap v2012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1397622599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:37.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:37.634882+0000) 2023-08-27T22:00:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:38.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:38.635144+0000) 2023-08-27T22:00:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:38 smithi136 ceph-mon[113531]: pgmap v2013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:38 smithi093 ceph-mon[111335]: pgmap v2013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:38 smithi093 ceph-mon[118902]: pgmap v2013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4182337284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:39.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:39.635361+0000) 2023-08-27T22:00:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4182337284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4182337284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:40.690 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:40.373Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:40.690 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:40.374Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:40 smithi136 ceph-mon[113531]: pgmap v2014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:40.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:40.635628+0000) 2023-08-27T22:00:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:40 smithi093 ceph-mon[111335]: pgmap v2014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:40 smithi093 ceph-mon[118902]: pgmap v2014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3350930095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:41.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:41.635899+0000) 2023-08-27T22:00:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3350930095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:41.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3350930095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:41.889Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:41.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:41.890Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:42 smithi136 ceph-mon[113531]: pgmap v2015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:42.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:42.636179+0000) 2023-08-27T22:00:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:42 smithi093 ceph-mon[111335]: pgmap v2015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:42 smithi093 ceph-mon[118902]: pgmap v2015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:43.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:43.636452+0000) 2023-08-27T22:00:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:43.802Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:43.803Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:44 smithi136 ceph-mon[113531]: pgmap v2016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3928819173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:44.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:44.636740+0000) 2023-08-27T22:00:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:44 smithi093 ceph-mon[111335]: pgmap v2016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3928819173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:44 smithi093 ceph-mon[118902]: pgmap v2016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3928819173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:00:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:00:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:00:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:45.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:45.636965+0000) 2023-08-27T22:00:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:46 smithi136 ceph-mon[113531]: pgmap v2017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4237115423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:46.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:46.637211+0000) 2023-08-27T22:00:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:46 smithi093 ceph-mon[111335]: pgmap v2017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4237115423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:46 smithi093 ceph-mon[118902]: pgmap v2017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4237115423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:47.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:47.637477+0000) 2023-08-27T22:00:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:48 smithi136 ceph-mon[113531]: pgmap v2018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1492931232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:48.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:48.637758+0000) 2023-08-27T22:00:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:48 smithi093 ceph-mon[111335]: pgmap v2018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1492931232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:48 smithi093 ceph-mon[118902]: pgmap v2018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1492931232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:49.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:49.638019+0000) 2023-08-27T22:00:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:50 smithi136 ceph-mon[113531]: pgmap v2019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:50 smithi093 ceph-mon[111335]: pgmap v2019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:50 smithi093 ceph-mon[118902]: pgmap v2019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:50.373Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:50.374Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:50.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:50.638312+0000) 2023-08-27T22:00:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/960995570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/960995570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/960995570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:51.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:51.638585+0000) 2023-08-27T22:00:52.190 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:51.889Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:52.190 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:51.890Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:52 smithi093 ceph-mon[111335]: pgmap v2020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:52 smithi093 ceph-mon[118902]: pgmap v2020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:52.638 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:52 smithi136 ceph-mon[113531]: pgmap v2020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:52.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:52.638830+0000) 2023-08-27T22:00:53.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2836253344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2836253344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2836253344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:53.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:53.639134+0000) 2023-08-27T22:00:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:53.803Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:00:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:00:53.804Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:00:54.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:54 smithi136 ceph-mon[113531]: pgmap v2021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:54.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:54 smithi093 ceph-mon[111335]: pgmap v2021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:54.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:54 smithi093 ceph-mon[118902]: pgmap v2021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:54.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:00:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:54.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:54.639438+0000) 2023-08-27T22:00:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:00:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:00:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:00:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:55.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:55.639625+0000) 2023-08-27T22:00:56.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:56 smithi136 ceph-mon[113531]: pgmap v2022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:56.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2643587437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:56 smithi093 ceph-mon[111335]: pgmap v2022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2643587437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:56 smithi093 ceph-mon[118902]: pgmap v2022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2643587437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:56.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:56.639828+0000) 2023-08-27T22:00:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:57.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:57.640113+0000) 2023-08-27T22:00:58.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:58 smithi136 ceph-mon[113531]: pgmap v2023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:58.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:00:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1469501868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:58 smithi093 ceph-mon[111335]: pgmap v2023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:00:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1469501868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:58 smithi093 ceph-mon[118902]: pgmap v2023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:00:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:00:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1469501868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:00:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:58.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:58.640403+0000) 2023-08-27T22:00:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:00:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:00:59.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:00:59.640641+0000) 2023-08-27T22:01:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:00 smithi136 ceph-mon[113531]: pgmap v2024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:00 smithi093 ceph-mon[111335]: pgmap v2024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:00 smithi093 ceph-mon[118902]: pgmap v2024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:00.374Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:00.375Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:00.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:00.640862+0000) 2023-08-27T22:01:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2770703465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2770703465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2770703465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:01.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:01.641136+0000) 2023-08-27T22:01:02.192 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:01.890Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:02.193 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:01.891Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:02 smithi093 ceph-mon[111335]: pgmap v2025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:02 smithi093 ceph-mon[118902]: pgmap v2025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:02.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:02 smithi136 ceph-mon[113531]: pgmap v2025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:02.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:02.641358+0000) 2023-08-27T22:01:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4175922313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4175922313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:03.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4175922313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:03.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:03.641619+0000) 2023-08-27T22:01:04.198 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:03.804Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:04.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:03.805Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:04 smithi093 ceph-mon[118902]: pgmap v2026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:04 smithi093 ceph-mon[111335]: pgmap v2026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:04.642 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:04 smithi136 ceph-mon[113531]: pgmap v2026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:04.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:04.641937+0000) 2023-08-27T22:01:05.642 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3230205656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:05.642 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 22:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T22:01:05.317548593Z level=info msg="Completed cleanup jobs" duration=12.594045ms 2023-08-27T22:01:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3230205656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3230205656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:01:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:01:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:01:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:05.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:05.642253+0000) 2023-08-27T22:01:06.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:06 smithi136 ceph-mon[113531]: pgmap v2027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:06 smithi093 ceph-mon[111335]: pgmap v2027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:06 smithi093 ceph-mon[118902]: pgmap v2027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:06.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:06.642463+0000) 2023-08-27T22:01:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:07.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:07.642729+0000) 2023-08-27T22:01:08.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:08 smithi136 ceph-mon[113531]: pgmap v2028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:08.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1620447462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:08 smithi093 ceph-mon[111335]: pgmap v2028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1620447462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:08 smithi093 ceph-mon[118902]: pgmap v2028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1620447462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:08.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:08.642985+0000) 2023-08-27T22:01:09.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:09.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:09.643292+0000) 2023-08-27T22:01:10.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:10 smithi136 ceph-mon[113531]: pgmap v2029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:10.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2574086738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:10 smithi093 ceph-mon[111335]: pgmap v2029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2574086738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:10 smithi093 ceph-mon[118902]: pgmap v2029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2574086738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:10.374Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:10.375Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:10.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:10.643543+0000) 2023-08-27T22:01:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:11.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:11.643730+0000) 2023-08-27T22:01:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:11.891Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:11.892Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:12.643 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:12 smithi136 ceph-mon[113531]: pgmap v2030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:12 smithi093 ceph-mon[111335]: pgmap v2030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:12 smithi093 ceph-mon[118902]: pgmap v2030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:12.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:12.643973+0000) 2023-08-27T22:01:13.644 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/901080353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/901080353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/901080353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:13.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:13.644260+0000) 2023-08-27T22:01:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:13.805Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:13.806Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:14.644 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:14 smithi136 ceph-mon[113531]: pgmap v2031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:14 smithi093 ceph-mon[111335]: pgmap v2031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:14 smithi093 ceph-mon[118902]: pgmap v2031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:14.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:14.644529+0000) 2023-08-27T22:01:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2403484774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:15.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:15.644805+0000) 2023-08-27T22:01:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2403484774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:01:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:01:15] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:01:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2403484774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:16 smithi136 ceph-mon[113531]: pgmap v2032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:16.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:16.645008+0000) 2023-08-27T22:01:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:16 smithi093 ceph-mon[111335]: pgmap v2032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:16 smithi093 ceph-mon[118902]: pgmap v2032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2268107615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:17.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:17.645291+0000) 2023-08-27T22:01:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2268107615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2268107615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:18 smithi136 ceph-mon[113531]: pgmap v2033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:18.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:18.645485+0000) 2023-08-27T22:01:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:18 smithi093 ceph-mon[111335]: pgmap v2033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:18 smithi093 ceph-mon[118902]: pgmap v2033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:19.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:19.645735+0000) 2023-08-27T22:01:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:20 smithi093 ceph-mon[111335]: pgmap v2034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1027437225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:20 smithi093 ceph-mon[118902]: pgmap v2034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1027437225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:20.376Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:20.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:20.376Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:20.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:20 smithi136 ceph-mon[113531]: pgmap v2034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:20.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1027437225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:20.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:20.646007+0000) 2023-08-27T22:01:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:21.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:21.646246+0000) 2023-08-27T22:01:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:21.892Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:21.893Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:22.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:22 smithi136 ceph-mon[113531]: pgmap v2035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:22.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2717574078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:22 smithi093 ceph-mon[111335]: pgmap v2035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2717574078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:22 smithi093 ceph-mon[118902]: pgmap v2035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2717574078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:22.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:22.646490+0000) 2023-08-27T22:01:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:23.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:23.646752+0000) 2023-08-27T22:01:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:23.805Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:23.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:24.647 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:24 smithi136 ceph-mon[113531]: pgmap v2036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:24.647 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:24.647 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2075031775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:24 smithi093 ceph-mon[118902]: pgmap v2036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2075031775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:24 smithi093 ceph-mon[111335]: pgmap v2036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2075031775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:24.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:24.647070+0000) 2023-08-27T22:01:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:01:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:01:25] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T22:01:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:25.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:25.647356+0000) 2023-08-27T22:01:26.647 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:26 smithi136 ceph-mon[113531]: pgmap v2037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:26 smithi093 ceph-mon[111335]: pgmap v2037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:26 smithi093 ceph-mon[118902]: pgmap v2037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:26.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:26.647630+0000) 2023-08-27T22:01:27.647 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2333958580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2333958580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2333958580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:27.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:27.647897+0000) 2023-08-27T22:01:28.648 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:28 smithi136 ceph-mon[113531]: pgmap v2038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:28 smithi093 ceph-mon[111335]: pgmap v2038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:28 smithi093 ceph-mon[118902]: pgmap v2038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:28.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:28.648130+0000) 2023-08-27T22:01:29.648 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/403886214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/403886214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/403886214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:29.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:29.648404+0000) 2023-08-27T22:01:30.648 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:30 smithi136 ceph-mon[113531]: pgmap v2039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:30 smithi093 ceph-mon[111335]: pgmap v2039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:30 smithi093 ceph-mon[118902]: pgmap v2039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:30.376Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:30.377Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:30.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:30.648658+0000) 2023-08-27T22:01:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:31.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:31.648864+0000) 2023-08-27T22:01:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:31.892Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:31.893Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:32.649 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:32 smithi136 ceph-mon[113531]: pgmap v2040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:32.649 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3008848224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:32 smithi093 ceph-mon[111335]: pgmap v2040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3008848224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:32 smithi093 ceph-mon[118902]: pgmap v2040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3008848224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:32.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:32.649110+0000) 2023-08-27T22:01:33.649 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:01:33.649 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:33.649 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:01:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:01:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:33.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:33.649264+0000) 2023-08-27T22:01:34.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:33.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:34.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:33.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:34 smithi093 ceph-mon[111335]: pgmap v2041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2805584678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:34 smithi093 ceph-mon[118902]: pgmap v2041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2805584678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:34 smithi136 ceph-mon[113531]: pgmap v2041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2805584678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:34.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:34.649447+0000) 2023-08-27T22:01:35.649 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:35.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:35.649616+0000) 2023-08-27T22:01:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:01:35] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T22:01:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:36 smithi136 ceph-mon[113531]: pgmap v2042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:01:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:01:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2988980975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:36.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:36.649838+0000) 2023-08-27T22:01:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[111335]: pgmap v2042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2988980975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[118902]: pgmap v2042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:01:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2988980975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:37.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:37.650116+0000) 2023-08-27T22:01:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:38 smithi136 ceph-mon[113531]: pgmap v2043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1336051432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:38.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:38.650341+0000) 2023-08-27T22:01:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:38 smithi093 ceph-mon[111335]: pgmap v2043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1336051432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:38 smithi093 ceph-mon[118902]: pgmap v2043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1336051432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:39.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:39.650619+0000) 2023-08-27T22:01:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:40 smithi093 ceph-mon[111335]: pgmap v2044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:40 smithi093 ceph-mon[118902]: pgmap v2044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:40.377Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:40.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:40.378Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:40.650 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:40 smithi136 ceph-mon[113531]: pgmap v2044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:40.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:40.650884+0000) 2023-08-27T22:01:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/458502471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/458502471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:41.651 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/458502471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:41.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:41.651107+0000) 2023-08-27T22:01:42.198 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:41.893Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:42.198 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:41.894Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:42 smithi093 ceph-mon[111335]: pgmap v2045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:42 smithi093 ceph-mon[118902]: pgmap v2045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:42.651 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:42 smithi136 ceph-mon[113531]: pgmap v2045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:42.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:42.651342+0000) 2023-08-27T22:01:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:43.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:43.651642+0000) 2023-08-27T22:01:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:43.807Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:43.808Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:44.651 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:44 smithi136 ceph-mon[113531]: pgmap v2046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:44.652 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4099126814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:44 smithi093 ceph-mon[111335]: pgmap v2046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4099126814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:44 smithi093 ceph-mon[118902]: pgmap v2046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4099126814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:44.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:44.651939+0000) 2023-08-27T22:01:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:01:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:01:45] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:01:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:45.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:45.652174+0000) 2023-08-27T22:01:46.652 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:46 smithi136 ceph-mon[113531]: pgmap v2047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:46.652 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1758940089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:46 smithi093 ceph-mon[111335]: pgmap v2047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1758940089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:46 smithi093 ceph-mon[118902]: pgmap v2047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1758940089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:46.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:46.652417+0000) 2023-08-27T22:01:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:47.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:47.652663+0000) 2023-08-27T22:01:48.652 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:48 smithi136 ceph-mon[113531]: pgmap v2048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:48.653 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2090028933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:48 smithi093 ceph-mon[111335]: pgmap v2048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2090028933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:48 smithi093 ceph-mon[118902]: pgmap v2048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2090028933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:48.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:48.652914+0000) 2023-08-27T22:01:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:49.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:49.653139+0000) 2023-08-27T22:01:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:50 smithi093 ceph-mon[111335]: pgmap v2049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:50 smithi093 ceph-mon[118902]: pgmap v2049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:50.377Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:50.378Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:50.653 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:50 smithi136 ceph-mon[113531]: pgmap v2049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:50.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:50.653323+0000) 2023-08-27T22:01:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/600952252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/600952252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:51.653 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/600952252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:51.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:51.653606+0000) 2023-08-27T22:01:52.201 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:51.894Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:52.201 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:51.895Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:52 smithi093 ceph-mon[111335]: pgmap v2050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:52 smithi093 ceph-mon[118902]: pgmap v2050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:52.653 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:52 smithi136 ceph-mon[113531]: pgmap v2050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:52.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:52.653853+0000) 2023-08-27T22:01:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/939830939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/939830939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:53.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/939830939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:53.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:53.654094+0000) 2023-08-27T22:01:54.206 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:53.808Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:54.206 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:01:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:01:53.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:01:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:54 smithi093 ceph-mon[111335]: pgmap v2051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:54 smithi093 ceph-mon[118902]: pgmap v2051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:54.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:54 smithi136 ceph-mon[113531]: pgmap v2051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:54.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:01:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:54.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:54.654376+0000) 2023-08-27T22:01:55.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3704196319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3704196319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3704196319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:01:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:01:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:01:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:55.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:55.654640+0000) 2023-08-27T22:01:56.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:56 smithi136 ceph-mon[113531]: pgmap v2052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:56 smithi093 ceph-mon[111335]: pgmap v2052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:56 smithi093 ceph-mon[118902]: pgmap v2052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:56.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:56.654876+0000) 2023-08-27T22:01:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:57.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:57.655168+0000) 2023-08-27T22:01:58.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:58 smithi136 ceph-mon[113531]: pgmap v2053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:58.655 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:01:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3483169782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:58 smithi093 ceph-mon[118902]: pgmap v2053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:01:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3483169782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:58 smithi093 ceph-mon[111335]: pgmap v2053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:01:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:01:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3483169782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:01:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:58.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:58.655443+0000) 2023-08-27T22:01:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:01:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:01:59.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:01:59.655685+0000) 2023-08-27T22:02:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:00 smithi093 ceph-mon[111335]: pgmap v2054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4115351987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:00 smithi093 ceph-mon[118902]: pgmap v2054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4115351987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:00.378Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:00.379Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:00.656 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:00 smithi136 ceph-mon[113531]: pgmap v2054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:00.656 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4115351987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:00.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:00.655953+0000) 2023-08-27T22:02:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:01.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:01.656200+0000) 2023-08-27T22:02:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:01.895Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:01.896Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:02.656 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:02 smithi136 ceph-mon[113531]: pgmap v2055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:02 smithi093 ceph-mon[111335]: pgmap v2055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:02 smithi093 ceph-mon[118902]: pgmap v2055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:02.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:02.656450+0000) 2023-08-27T22:02:03.656 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3440279280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3440279280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3440279280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:03.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:03.656702+0000) 2023-08-27T22:02:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:03.808Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:03.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:04.656 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:04 smithi136 ceph-mon[113531]: pgmap v2056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:04 smithi093 ceph-mon[111335]: pgmap v2056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:04 smithi093 ceph-mon[118902]: pgmap v2056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:04.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:04.656983+0000) 2023-08-27T22:02:05.657 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/190496503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/190496503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:02:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:02:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:02:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/190496503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:05.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:05.657240+0000) 2023-08-27T22:02:06.657 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:06 smithi136 ceph-mon[113531]: pgmap v2057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:06 smithi093 ceph-mon[111335]: pgmap v2057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:06 smithi093 ceph-mon[118902]: pgmap v2057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:06.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:06.657522+0000) 2023-08-27T22:02:07.657 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/903919166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/903919166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/903919166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:07.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:07.657800+0000) 2023-08-27T22:02:08.658 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:08 smithi136 ceph-mon[113531]: pgmap v2058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:08 smithi093 ceph-mon[111335]: pgmap v2058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:08 smithi093 ceph-mon[118902]: pgmap v2058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:08.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:08.658068+0000) 2023-08-27T22:02:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:09.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:09.658351+0000) 2023-08-27T22:02:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:10 smithi136 ceph-mon[113531]: pgmap v2059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/430618187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:10.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:10.658585+0000) 2023-08-27T22:02:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:10 smithi093 ceph-mon[118902]: pgmap v2059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/430618187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:10 smithi093 ceph-mon[111335]: pgmap v2059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/430618187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:10.379Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:10.380Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:11.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:11.658801+0000) 2023-08-27T22:02:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:11.895Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:11.896Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:12 smithi136 ceph-mon[113531]: pgmap v2060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/749525368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:12.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:12.659112+0000) 2023-08-27T22:02:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:12 smithi093 ceph-mon[118902]: pgmap v2060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/749525368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:12 smithi093 ceph-mon[111335]: pgmap v2060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/749525368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:13.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:13.659412+0000) 2023-08-27T22:02:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:13.809Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:13.811Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:14 smithi136 ceph-mon[113531]: pgmap v2061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/103183837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:14.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:14.659715+0000) 2023-08-27T22:02:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:14 smithi093 ceph-mon[111335]: pgmap v2061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/103183837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:14 smithi093 ceph-mon[118902]: pgmap v2061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/103183837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:02:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:02:15] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T22:02:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:15.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:15.659987+0000) 2023-08-27T22:02:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:16 smithi136 ceph-mon[113531]: pgmap v2062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:16.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:16.660240+0000) 2023-08-27T22:02:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:16 smithi093 ceph-mon[111335]: pgmap v2062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:16 smithi093 ceph-mon[118902]: pgmap v2062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2340586248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:17.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:17.660508+0000) 2023-08-27T22:02:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2340586248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2340586248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:18 smithi136 ceph-mon[113531]: pgmap v2063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:18.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:18.660677+0000) 2023-08-27T22:02:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:18 smithi093 ceph-mon[111335]: pgmap v2063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:18 smithi093 ceph-mon[118902]: pgmap v2063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/662642311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:19.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:19.660910+0000) 2023-08-27T22:02:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/662642311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/662642311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:20 smithi136 ceph-mon[113531]: pgmap v2064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:20.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:20.661160+0000) 2023-08-27T22:02:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:20 smithi093 ceph-mon[111335]: pgmap v2064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:20 smithi093 ceph-mon[118902]: pgmap v2064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:20.380Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:20.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:20.381Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:21.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:21.661397+0000) 2023-08-27T22:02:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:21.896Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:21.897Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:22 smithi136 ceph-mon[113531]: pgmap v2065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/901805447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:22.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:22.661651+0000) 2023-08-27T22:02:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:22 smithi093 ceph-mon[111335]: pgmap v2065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/901805447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:22 smithi093 ceph-mon[118902]: pgmap v2065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/901805447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:23.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:23.661897+0000) 2023-08-27T22:02:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:23.810Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:23.811Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:24 smithi093 ceph-mon[111335]: pgmap v2066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4260486691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:24 smithi093 ceph-mon[118902]: pgmap v2066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4260486691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:24 smithi136 ceph-mon[113531]: pgmap v2066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4260486691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:24.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:24.662113+0000) 2023-08-27T22:02:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:02:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:02:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:02:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:25.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:25.662372+0000) 2023-08-27T22:02:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:26 smithi093 ceph-mon[111335]: pgmap v2067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2906812408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:26 smithi093 ceph-mon[118902]: pgmap v2067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2906812408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:26 smithi136 ceph-mon[113531]: pgmap v2067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2906812408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:26.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:26.662557+0000) 2023-08-27T22:02:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:27.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:27.662830+0000) 2023-08-27T22:02:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:28 smithi093 ceph-mon[111335]: pgmap v2068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:28 smithi093 ceph-mon[118902]: pgmap v2068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:28 smithi136 ceph-mon[113531]: pgmap v2068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:28.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:28.663013+0000) 2023-08-27T22:02:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3921387854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3921387854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3921387854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:29.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:29.663260+0000) 2023-08-27T22:02:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:30 smithi093 ceph-mon[111335]: pgmap v2069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:30 smithi093 ceph-mon[118902]: pgmap v2069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:30.381Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:30.382Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:30 smithi136 ceph-mon[113531]: pgmap v2069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:30.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:30.663519+0000) 2023-08-27T22:02:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3054060366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3054060366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3054060366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:31.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:31.663766+0000) 2023-08-27T22:02:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:31.897Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:31.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:32 smithi093 ceph-mon[111335]: pgmap v2070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:32 smithi093 ceph-mon[118902]: pgmap v2070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:32 smithi136 ceph-mon[113531]: pgmap v2070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:32.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:32.664028+0000) 2023-08-27T22:02:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:33.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:33.664348+0000) 2023-08-27T22:02:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:33.810Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:33.811Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:34 smithi093 ceph-mon[111335]: pgmap v2071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3430814636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:34 smithi093 ceph-mon[118902]: pgmap v2071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3430814636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:34 smithi136 ceph-mon[113531]: pgmap v2071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3430814636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:34.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:34.664615+0000) 2023-08-27T22:02:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:02:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:02:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:02:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:35.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:35.664890+0000) 2023-08-27T22:02:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:36 smithi093 ceph-mon[111335]: pgmap v2072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1074670384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:02:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:36 smithi093 ceph-mon[118902]: pgmap v2072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1074670384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:02:36.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:36 smithi136 ceph-mon[113531]: pgmap v2072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:36.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1074670384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:36.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:02:36.769 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:36.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:36.665368+0000) 2023-08-27T22:02:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:37.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:37.665620+0000) 2023-08-27T22:02:38.665 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:38 smithi136 ceph-mon[113531]: pgmap v2073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:38.666 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:38.666 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:38.666 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3196836412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:38.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[118902]: pgmap v2073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3196836412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[111335]: pgmap v2073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:38.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:38.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3196836412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:38.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:38.665816+0000) 2023-08-27T22:02:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:39.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:39.666007+0000) 2023-08-27T22:02:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:02:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:02:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:02:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:02:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:40 smithi093 ceph-mon[111335]: pgmap v2074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:40 smithi093 ceph-mon[118902]: pgmap v2074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:40.381Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:40.383Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:40 smithi136 ceph-mon[113531]: pgmap v2074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:40.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:40.666261+0000) 2023-08-27T22:02:41.899 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2897110965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:41.899 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2897110965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2897110965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:41.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:41.666497+0000) 2023-08-27T22:02:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:41.898Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:41.899Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:42 smithi136 ceph-mon[113531]: pgmap v2075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:42.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:42.666775+0000) 2023-08-27T22:02:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:42 smithi093 ceph-mon[111335]: pgmap v2075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:42 smithi093 ceph-mon[118902]: pgmap v2075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4043880598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:43.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:43.667027+0000) 2023-08-27T22:02:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4043880598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4043880598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:43.811Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:43.812Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:44 smithi136 ceph-mon[113531]: pgmap v2076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:44.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:44.667292+0000) 2023-08-27T22:02:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:44 smithi093 ceph-mon[111335]: pgmap v2076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:44 smithi093 ceph-mon[118902]: pgmap v2076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:45.633 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:02:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:02:45] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:02:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/652067750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:45.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:45.667550+0000) 2023-08-27T22:02:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/652067750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:45.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/652067750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:46 smithi136 ceph-mon[113531]: pgmap v2077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:46.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:46.667849+0000) 2023-08-27T22:02:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:46 smithi093 ceph-mon[111335]: pgmap v2077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:46 smithi093 ceph-mon[118902]: pgmap v2077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:47.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:47.668080+0000) 2023-08-27T22:02:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:48 smithi136 ceph-mon[113531]: pgmap v2078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3761899968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:48.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:48.668277+0000) 2023-08-27T22:02:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:48 smithi093 ceph-mon[111335]: pgmap v2078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3761899968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:48 smithi093 ceph-mon[118902]: pgmap v2078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3761899968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:49.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:49.668542+0000) 2023-08-27T22:02:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:50 smithi093 ceph-mon[111335]: pgmap v2079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/984302183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:50 smithi093 ceph-mon[118902]: pgmap v2079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/984302183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:50.382Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:50.383Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:50.668 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:50 smithi136 ceph-mon[113531]: pgmap v2079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:50.669 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/984302183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:50.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:50.668818+0000) 2023-08-27T22:02:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:51.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:51.669007+0000) 2023-08-27T22:02:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:51.899Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:51.900Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:52.669 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:52 smithi136 ceph-mon[113531]: pgmap v2080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:52 smithi093 ceph-mon[111335]: pgmap v2080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:52 smithi093 ceph-mon[118902]: pgmap v2080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:52.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:52.669288+0000) 2023-08-27T22:02:53.669 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3093296901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3093296901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3093296901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:53.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:53.669534+0000) 2023-08-27T22:02:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:53.812Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:02:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:02:53.813Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:02:54.669 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:54 smithi136 ceph-mon[113531]: pgmap v2081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:54.670 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:54 smithi093 ceph-mon[118902]: pgmap v2081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:54 smithi093 ceph-mon[111335]: pgmap v2081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:02:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:54.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:54.669820+0000) 2023-08-27T22:02:55.670 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2188203860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2188203860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2188203860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:02:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:02:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:02:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:55.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:55.670097+0000) 2023-08-27T22:02:56.670 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:56 smithi136 ceph-mon[113531]: pgmap v2082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:56 smithi093 ceph-mon[111335]: pgmap v2082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:56 smithi093 ceph-mon[118902]: pgmap v2082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:56.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:56.670348+0000) 2023-08-27T22:02:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3069585387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:57.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:57.670593+0000) 2023-08-27T22:02:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3069585387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3069585387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:02:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:02:58 smithi136 ceph-mon[113531]: pgmap v2083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:58.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:58.670843+0000) 2023-08-27T22:02:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:02:58 smithi093 ceph-mon[111335]: pgmap v2083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:02:58 smithi093 ceph-mon[118902]: pgmap v2083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:02:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:02:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:02:59.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:02:59.671127+0000) 2023-08-27T22:03:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:00 smithi093 ceph-mon[111335]: pgmap v2084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1558989543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:00 smithi093 ceph-mon[118902]: pgmap v2084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1558989543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:00.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:00.383Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:00.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:00.384Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:00.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:00 smithi136 ceph-mon[113531]: pgmap v2084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:00.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1558989543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:00.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:00.671308+0000) 2023-08-27T22:03:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:01.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:01.671556+0000) 2023-08-27T22:03:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:01.899Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:01.900Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:02.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:02 smithi136 ceph-mon[113531]: pgmap v2085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:02.672 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1956986635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:02 smithi093 ceph-mon[111335]: pgmap v2085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1956986635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:02 smithi093 ceph-mon[118902]: pgmap v2085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1956986635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:02.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:02.671788+0000) 2023-08-27T22:03:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:03.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:03.671953+0000) 2023-08-27T22:03:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:03.813Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:03.814Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:04.672 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:04 smithi136 ceph-mon[113531]: pgmap v2086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:04 smithi093 ceph-mon[111335]: pgmap v2086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:04 smithi093 ceph-mon[118902]: pgmap v2086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:04.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:04.672220+0000) 2023-08-27T22:03:05.672 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1702652270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1702652270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1702652270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:03:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:03:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:03:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:05.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:05.672443+0000) 2023-08-27T22:03:06.672 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:06 smithi136 ceph-mon[113531]: pgmap v2087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:06 smithi093 ceph-mon[111335]: pgmap v2087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:06 smithi093 ceph-mon[118902]: pgmap v2087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:06.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:06.672670+0000) 2023-08-27T22:03:07.672 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1360410802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1360410802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1360410802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:07.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:07.672867+0000) 2023-08-27T22:03:08.673 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:08 smithi136 ceph-mon[113531]: pgmap v2088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:08.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:08 smithi093 ceph-mon[111335]: pgmap v2088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:08.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:08 smithi093 ceph-mon[118902]: pgmap v2088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:08.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:08.673129+0000) 2023-08-27T22:03:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3750209397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:09.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:09.673318+0000) 2023-08-27T22:03:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3750209397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3750209397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:10 smithi136 ceph-mon[113531]: pgmap v2089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:10.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:10.673561+0000) 2023-08-27T22:03:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:10 smithi093 ceph-mon[111335]: pgmap v2089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:10 smithi093 ceph-mon[118902]: pgmap v2089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:10.384Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:10.385Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:11.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:11.673832+0000) 2023-08-27T22:03:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:11.900Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:11.901Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:12 smithi136 ceph-mon[113531]: pgmap v2090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4038783084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:12.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:12.674115+0000) 2023-08-27T22:03:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:12 smithi093 ceph-mon[111335]: pgmap v2090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4038783084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:12 smithi093 ceph-mon[118902]: pgmap v2090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4038783084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:13.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:13.674345+0000) 2023-08-27T22:03:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:13.813Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:13.815Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:14 smithi136 ceph-mon[113531]: pgmap v2091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1363208277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:14.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:14.674601+0000) 2023-08-27T22:03:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:14 smithi093 ceph-mon[111335]: pgmap v2091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1363208277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:14 smithi093 ceph-mon[118902]: pgmap v2091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1363208277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:03:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:03:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:03:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:15.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:15.674818+0000) 2023-08-27T22:03:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:16 smithi136 ceph-mon[113531]: pgmap v2092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3622649796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:16.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:16.675001+0000) 2023-08-27T22:03:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:16 smithi093 ceph-mon[111335]: pgmap v2092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3622649796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:16 smithi093 ceph-mon[118902]: pgmap v2092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3622649796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:17.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:17.675266+0000) 2023-08-27T22:03:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:18 smithi136 ceph-mon[113531]: pgmap v2093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:18.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:18.675548+0000) 2023-08-27T22:03:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:18 smithi093 ceph-mon[111335]: pgmap v2093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:18 smithi093 ceph-mon[118902]: pgmap v2093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2639197523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2639197523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2639197523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:19.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:19.675867+0000) 2023-08-27T22:03:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:20 smithi093 ceph-mon[111335]: pgmap v2094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:20 smithi093 ceph-mon[118902]: pgmap v2094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:20.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:20.385Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:20.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:20.386Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:20 smithi136 ceph-mon[113531]: pgmap v2094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:20.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:20.676140+0000) 2023-08-27T22:03:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2497150613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2497150613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2497150613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:21.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:21.676360+0000) 2023-08-27T22:03:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:21.900Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:21.901Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:22 smithi093 ceph-mon[111335]: pgmap v2095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:22 smithi093 ceph-mon[118902]: pgmap v2095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:22 smithi136 ceph-mon[113531]: pgmap v2095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:22.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:22.676610+0000) 2023-08-27T22:03:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:23.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:23.676867+0000) 2023-08-27T22:03:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:23.814Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:23.815Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:24 smithi093 ceph-mon[118902]: pgmap v2096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3616231484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:24 smithi093 ceph-mon[111335]: pgmap v2096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3616231484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:24 smithi136 ceph-mon[113531]: pgmap v2096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3616231484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:24.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:24.677183+0000) 2023-08-27T22:03:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:03:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:03:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:03:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:25.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:25.677440+0000) 2023-08-27T22:03:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:26 smithi093 ceph-mon[111335]: pgmap v2097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2091670349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:26 smithi093 ceph-mon[118902]: pgmap v2097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2091670349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:26 smithi136 ceph-mon[113531]: pgmap v2097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2091670349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:26.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:26.677685+0000) 2023-08-27T22:03:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:27.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:27.677983+0000) 2023-08-27T22:03:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:28 smithi093 ceph-mon[111335]: pgmap v2098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3584038140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:28 smithi093 ceph-mon[118902]: pgmap v2098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3584038140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:28 smithi136 ceph-mon[113531]: pgmap v2098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3584038140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:28.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:28.678191+0000) 2023-08-27T22:03:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:29.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:29.678469+0000) 2023-08-27T22:03:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:30 smithi093 ceph-mon[111335]: pgmap v2099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:30 smithi093 ceph-mon[118902]: pgmap v2099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:30.385Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:30.386Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:30.679 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:30 smithi136 ceph-mon[113531]: pgmap v2099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:30.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:30.678786+0000) 2023-08-27T22:03:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2848767363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2848767363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:31.679 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2848767363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:31.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:31.679023+0000) 2023-08-27T22:03:32.218 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:31.901Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:32.218 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:31.902Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:32 smithi093 ceph-mon[111335]: pgmap v2100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:32 smithi093 ceph-mon[118902]: pgmap v2100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:32.679 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:32 smithi136 ceph-mon[113531]: pgmap v2100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:32.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:32.679291+0000) 2023-08-27T22:03:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2669142428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2669142428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:33.679 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2669142428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:33.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:33.679538+0000) 2023-08-27T22:03:34.227 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:33.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:34.227 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:33.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:34 smithi093 ceph-mon[111335]: pgmap v2101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:34 smithi093 ceph-mon[118902]: pgmap v2101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:34.679 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:34 smithi136 ceph-mon[113531]: pgmap v2101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:34.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:34.679806+0000) 2023-08-27T22:03:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:03:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:03:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:03:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:35.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:35.680020+0000) 2023-08-27T22:03:36.680 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:36 smithi136 ceph-mon[113531]: pgmap v2102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:36.680 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1573484151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:36 smithi093 ceph-mon[111335]: pgmap v2102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1573484151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:36 smithi093 ceph-mon[118902]: pgmap v2102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1573484151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:36.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:36.680208+0000) 2023-08-27T22:03:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:37.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:37.680482+0000) 2023-08-27T22:03:38.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:38 smithi136 ceph-mon[113531]: pgmap v2103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:38.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/289120821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:38 smithi093 ceph-mon[111335]: pgmap v2103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/289120821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:38 smithi093 ceph-mon[118902]: pgmap v2103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/289120821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:38.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:38.680775+0000) 2023-08-27T22:03:39.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:39.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:03:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:03:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:03:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:39.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:39.680914+0000) 2023-08-27T22:03:40.663 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:40 smithi136 ceph-mon[113531]: pgmap v2104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:40.663 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3612891551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:40 smithi093 ceph-mon[111335]: pgmap v2104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3612891551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:40.386Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:40.387Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:40 smithi093 ceph-mon[118902]: pgmap v2104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3612891551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:40.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:40.681076+0000) 2023-08-27T22:03:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:41.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:41.681226+0000) 2023-08-27T22:03:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[111335]: pgmap v2105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[118902]: pgmap v2105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:41.902Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:42.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:41.903Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:42 smithi136 ceph-mon[113531]: pgmap v2105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:42.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:42.681478+0000) 2023-08-27T22:03:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:03:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:03:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3727751067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:03:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:03:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3727751067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:03:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:03:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:03:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3727751067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:43.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:43.681739+0000) 2023-08-27T22:03:44.107 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:43.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:44.107 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:43.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:44 smithi136 ceph-mon[113531]: pgmap v2106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:44 smithi093 ceph-mon[111335]: pgmap v2106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:44 smithi093 ceph-mon[118902]: pgmap v2106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:44.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:44.681994+0000) 2023-08-27T22:03:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1310618659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:03:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:03:45] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T22:03:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1310618659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:45.682 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1310618659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:45.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:45.682187+0000) 2023-08-27T22:03:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:46 smithi093 ceph-mon[111335]: pgmap v2107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:46 smithi093 ceph-mon[118902]: pgmap v2107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:46.682 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:46 smithi136 ceph-mon[113531]: pgmap v2107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:46.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:46.682367+0000) 2023-08-27T22:03:47.682 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1008864396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1008864396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1008864396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:47.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:47.682652+0000) 2023-08-27T22:03:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:48 smithi136 ceph-mon[113531]: pgmap v2108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:48.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:48.682932+0000) 2023-08-27T22:03:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:48 smithi093 ceph-mon[111335]: pgmap v2108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:48 smithi093 ceph-mon[118902]: pgmap v2108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:49.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:49.683183+0000) 2023-08-27T22:03:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:50 smithi093 ceph-mon[111335]: pgmap v2109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2040495637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:50 smithi093 ceph-mon[118902]: pgmap v2109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2040495637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:50.386Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:50.387Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:50.683 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:50 smithi136 ceph-mon[113531]: pgmap v2109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:50.683 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2040495637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:50.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:50.683493+0000) 2023-08-27T22:03:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:51.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:51.683737+0000) 2023-08-27T22:03:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:51.903Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:51.904Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:52.684 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:52 smithi136 ceph-mon[113531]: pgmap v2110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:52.684 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3794782165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:52 smithi093 ceph-mon[118902]: pgmap v2110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3794782165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:52 smithi093 ceph-mon[111335]: pgmap v2110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3794782165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:52.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:52.683988+0000) 2023-08-27T22:03:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:53.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:53.684240+0000) 2023-08-27T22:03:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:53.816Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:03:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:03:53.817Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:03:54.684 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:54 smithi136 ceph-mon[113531]: pgmap v2111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:54.684 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:54 smithi093 ceph-mon[111335]: pgmap v2111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:54 smithi093 ceph-mon[118902]: pgmap v2111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:03:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:54.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:54.684483+0000) 2023-08-27T22:03:55.684 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/218123938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/218123938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/218123938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:03:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:03:55] "GET /metrics HTTP/1.1" 200 239891 "" "Prometheus/2.43.0" 2023-08-27T22:03:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:55.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:55.684735+0000) 2023-08-27T22:03:56.685 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:56 smithi136 ceph-mon[113531]: pgmap v2112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:56 smithi093 ceph-mon[111335]: pgmap v2112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:56 smithi093 ceph-mon[118902]: pgmap v2112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:56.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:56.684980+0000) 2023-08-27T22:03:57.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4287665472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4287665472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4287665472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:03:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:57.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:57.685244+0000) 2023-08-27T22:03:58.685 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:03:58 smithi136 ceph-mon[113531]: pgmap v2113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:03:58 smithi093 ceph-mon[111335]: pgmap v2113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:03:58 smithi093 ceph-mon[118902]: pgmap v2113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:03:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:58.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:58.685497+0000) 2023-08-27T22:03:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:03:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:03:59.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:03:59.685708+0000) 2023-08-27T22:04:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3678394018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:00 smithi093 ceph-mon[111335]: pgmap v2114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3678394018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:00 smithi093 ceph-mon[118902]: pgmap v2114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:00.387Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:00.388Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:00.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3678394018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:00.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:00 smithi136 ceph-mon[113531]: pgmap v2114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:00.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:00.685969+0000) 2023-08-27T22:04:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:01.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:01.686206+0000) 2023-08-27T22:04:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:01.903Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:01.904Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:02.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:02 smithi136 ceph-mon[113531]: pgmap v2115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:02.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1720707186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:02 smithi093 ceph-mon[111335]: pgmap v2115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1720707186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:02 smithi093 ceph-mon[118902]: pgmap v2115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1720707186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:02.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:02.686466+0000) 2023-08-27T22:04:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:03.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:03.686714+0000) 2023-08-27T22:04:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:03.817Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:03.818Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:04.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:04 smithi136 ceph-mon[113531]: pgmap v2116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:04.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1366934112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:04 smithi093 ceph-mon[111335]: pgmap v2116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1366934112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:04 smithi093 ceph-mon[118902]: pgmap v2116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1366934112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:04.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:04.686985+0000) 2023-08-27T22:04:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:04:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:04:05] "GET /metrics HTTP/1.1" 200 239891 "" "Prometheus/2.43.0" 2023-08-27T22:04:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:05.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:05.687251+0000) 2023-08-27T22:04:06.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:06 smithi136 ceph-mon[113531]: pgmap v2117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:06 smithi093 ceph-mon[111335]: pgmap v2117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:06 smithi093 ceph-mon[118902]: pgmap v2117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:06.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:06.687530+0000) 2023-08-27T22:04:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/900471152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/900471152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/900471152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:07.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:07.687864+0000) 2023-08-27T22:04:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:08 smithi136 ceph-mon[113531]: pgmap v2118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:08 smithi093 ceph-mon[111335]: pgmap v2118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:08 smithi093 ceph-mon[118902]: pgmap v2118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:08.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:08.688176+0000) 2023-08-27T22:04:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1114807711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1114807711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1114807711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:09.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:09.688465+0000) 2023-08-27T22:04:10.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:10 smithi136 ceph-mon[113531]: pgmap v2119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:10 smithi093 ceph-mon[111335]: pgmap v2119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:10 smithi093 ceph-mon[118902]: pgmap v2119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:10.388Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:10.389Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:10.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:10.688688+0000) 2023-08-27T22:04:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/753271045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/753271045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/753271045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:11.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:11.688931+0000) 2023-08-27T22:04:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:11.904Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:11.905Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:12 smithi093 ceph-mon[111335]: pgmap v2120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:12 smithi093 ceph-mon[118902]: pgmap v2120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:12 smithi136 ceph-mon[113531]: pgmap v2120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:12.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:12.689244+0000) 2023-08-27T22:04:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:13.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:13.689506+0000) 2023-08-27T22:04:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:13.818Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:13.819Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:14 smithi093 ceph-mon[111335]: pgmap v2121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1337706957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:14.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:14 smithi093 ceph-mon[118902]: pgmap v2121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:14.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1337706957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:14 smithi136 ceph-mon[113531]: pgmap v2121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1337706957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:14.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:14.689739+0000) 2023-08-27T22:04:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:04:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:04:15] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:04:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:15.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:15.690009+0000) 2023-08-27T22:04:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:16 smithi093 ceph-mon[111335]: pgmap v2122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1268542380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:16 smithi093 ceph-mon[118902]: pgmap v2122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1268542380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:16 smithi136 ceph-mon[113531]: pgmap v2122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1268542380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:16.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:16.690337+0000) 2023-08-27T22:04:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:17.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:17.690636+0000) 2023-08-27T22:04:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:18 smithi093 ceph-mon[111335]: pgmap v2123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/851836741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:18 smithi093 ceph-mon[118902]: pgmap v2123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/851836741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:18 smithi136 ceph-mon[113531]: pgmap v2123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/851836741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:18.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:18.690889+0000) 2023-08-27T22:04:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:19.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:19.691109+0000) 2023-08-27T22:04:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:20 smithi093 ceph-mon[111335]: pgmap v2124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:20 smithi093 ceph-mon[118902]: pgmap v2124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:20.389Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:20.390Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:20 smithi136 ceph-mon[113531]: pgmap v2124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:20.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:20.691288+0000) 2023-08-27T22:04:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2291012204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2291012204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2291012204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:21.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:21.691494+0000) 2023-08-27T22:04:22.225 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:21.905Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:22.225 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:21.906Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:22 smithi093 ceph-mon[111335]: pgmap v2125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:22 smithi093 ceph-mon[118902]: pgmap v2125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:22 smithi136 ceph-mon[113531]: pgmap v2125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:22.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:22.691764+0000) 2023-08-27T22:04:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1701646587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1701646587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1701646587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:23.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:23.692045+0000) 2023-08-27T22:04:24.236 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:23.819Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:24.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:23.820Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:24 smithi093 ceph-mon[111335]: pgmap v2126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:24 smithi093 ceph-mon[118902]: pgmap v2126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:24 smithi136 ceph-mon[113531]: pgmap v2126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:24.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:24.692253+0000) 2023-08-27T22:04:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:04:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:04:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:04:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:25.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:25.692477+0000) 2023-08-27T22:04:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:26 smithi136 ceph-mon[113531]: pgmap v2127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4138542434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:26 smithi093 ceph-mon[111335]: pgmap v2127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4138542434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:26 smithi093 ceph-mon[118902]: pgmap v2127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4138542434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:26.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:26.692731+0000) 2023-08-27T22:04:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:27.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:27.693026+0000) 2023-08-27T22:04:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:28 smithi136 ceph-mon[113531]: pgmap v2128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2865379601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:28 smithi093 ceph-mon[111335]: pgmap v2128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2865379601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:28 smithi093 ceph-mon[118902]: pgmap v2128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2865379601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:28.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:28.693251+0000) 2023-08-27T22:04:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:29.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:29.693402+0000) 2023-08-27T22:04:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:30 smithi093 ceph-mon[111335]: pgmap v2129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:30 smithi093 ceph-mon[118902]: pgmap v2129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:30.390Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:30.391Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:30 smithi136 ceph-mon[113531]: pgmap v2129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:30.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:30.693605+0000) 2023-08-27T22:04:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3353199285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3353199285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3353199285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:31.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:31.693871+0000) 2023-08-27T22:04:32.226 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:31.906Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:32.226 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:31.907Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:32 smithi093 ceph-mon[111335]: pgmap v2130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:32 smithi093 ceph-mon[118902]: pgmap v2130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:32 smithi136 ceph-mon[113531]: pgmap v2130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:32.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:32.694154+0000) 2023-08-27T22:04:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1838071912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1838071912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1838071912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:33.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:33.694391+0000) 2023-08-27T22:04:34.233 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:33.819Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:34.234 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:33.821Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:34 smithi093 ceph-mon[111335]: pgmap v2131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:34 smithi093 ceph-mon[118902]: pgmap v2131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:34 smithi136 ceph-mon[113531]: pgmap v2131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:34.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:34.694636+0000) 2023-08-27T22:04:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2862630192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2862630192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:04:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:04:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:04:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2862630192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:35.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:35.694867+0000) 2023-08-27T22:04:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:36 smithi093 ceph-mon[111335]: pgmap v2132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:36 smithi093 ceph-mon[118902]: pgmap v2132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:36 smithi136 ceph-mon[113531]: pgmap v2132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:36.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:36.695064+0000) 2023-08-27T22:04:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:37.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:37.695317+0000) 2023-08-27T22:04:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:38 smithi136 ceph-mon[113531]: pgmap v2133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3962605504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:38 smithi093 ceph-mon[111335]: pgmap v2133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3962605504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:38 smithi093 ceph-mon[118902]: pgmap v2133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3962605504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:38.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:38.695526+0000) 2023-08-27T22:04:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:39.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:39.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:39.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:39.695728+0000) 2023-08-27T22:04:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:40 smithi136 ceph-mon[113531]: pgmap v2134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1665084354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:40 smithi093 ceph-mon[118902]: pgmap v2134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1665084354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:40 smithi093 ceph-mon[111335]: pgmap v2134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1665084354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:40.391Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:40.392Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:40.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:40.695995+0000) 2023-08-27T22:04:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:41.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:41.696226+0000) 2023-08-27T22:04:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:41.907Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:41.908Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:42 smithi136 ceph-mon[113531]: pgmap v2135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3484453686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:04:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:42 smithi093 ceph-mon[111335]: pgmap v2135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3484453686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:04:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:42 smithi093 ceph-mon[118902]: pgmap v2135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3484453686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:04:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:42.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:42.696445+0000) 2023-08-27T22:04:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:43.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:43.696612+0000) 2023-08-27T22:04:43.945 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:43.820Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:43.945 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:43.821Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:44 smithi136 ceph-mon[113531]: pgmap v2136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:44 smithi093 ceph-mon[111335]: pgmap v2136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:44 smithi093 ceph-mon[118902]: pgmap v2136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:44.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:44.696771+0000) 2023-08-27T22:04:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/240597626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:45.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:04:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:04:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:04:45] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:04:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/240597626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/240597626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:04:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:04:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:45.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:45.696979+0000) 2023-08-27T22:04:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:46 smithi136 ceph-mon[113531]: pgmap v2137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:46 smithi093 ceph-mon[111335]: pgmap v2137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:46 smithi093 ceph-mon[118902]: pgmap v2137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:46.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:46.697188+0000) 2023-08-27T22:04:47.697 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3907429742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:47.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3907429742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:47.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3907429742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:47.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:47.697421+0000) 2023-08-27T22:04:48.697 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:48 smithi136 ceph-mon[113531]: pgmap v2138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:48 smithi093 ceph-mon[111335]: pgmap v2138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:48 smithi093 ceph-mon[118902]: pgmap v2138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:48.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:48.697647+0000) 2023-08-27T22:04:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:49.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:49.697888+0000) 2023-08-27T22:04:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3891918597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:50 smithi093 ceph-mon[111335]: pgmap v2139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3891918597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:50 smithi093 ceph-mon[118902]: pgmap v2139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:50.392Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:50.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:50.393Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3891918597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:50 smithi136 ceph-mon[113531]: pgmap v2139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:50.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:50.698142+0000) 2023-08-27T22:04:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:51.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:51.698375+0000) 2023-08-27T22:04:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:51.907Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:51.908Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:52 smithi136 ceph-mon[113531]: pgmap v2140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1300211799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:52 smithi093 ceph-mon[111335]: pgmap v2140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1300211799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:52 smithi093 ceph-mon[118902]: pgmap v2140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1300211799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:52.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:52.698606+0000) 2023-08-27T22:04:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:53.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:53.698825+0000) 2023-08-27T22:04:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:53.821Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:04:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:04:53.822Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:04:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:54 smithi136 ceph-mon[113531]: pgmap v2141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/506959132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:54 smithi093 ceph-mon[118902]: pgmap v2141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/506959132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:54 smithi093 ceph-mon[111335]: pgmap v2141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:04:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/506959132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:54.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:54.699110+0000) 2023-08-27T22:04:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:04:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:04:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:04:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:55.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:55.699317+0000) 2023-08-27T22:04:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:56 smithi136 ceph-mon[113531]: pgmap v2142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:56 smithi093 ceph-mon[111335]: pgmap v2142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:56 smithi093 ceph-mon[118902]: pgmap v2142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:56.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:56.699561+0000) 2023-08-27T22:04:57.700 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020793548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020793548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020793548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:57.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:57.699786+0000) 2023-08-27T22:04:58.700 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:58 smithi136 ceph-mon[113531]: pgmap v2143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:58 smithi093 ceph-mon[111335]: pgmap v2143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:58 smithi093 ceph-mon[118902]: pgmap v2143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:04:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:58.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:58.700014+0000) 2023-08-27T22:04:59.700 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:04:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2743137954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:04:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2743137954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:04:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:04:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2743137954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:04:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:04:59.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:04:59.700241+0000) 2023-08-27T22:05:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:00 smithi093 ceph-mon[111335]: pgmap v2144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:00 smithi093 ceph-mon[118902]: pgmap v2144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:00.393Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:00.394Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:00 smithi136 ceph-mon[113531]: pgmap v2144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:00.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:00.700453+0000) 2023-08-27T22:05:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1881759162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1881759162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1881759162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:01.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:01.700690+0000) 2023-08-27T22:05:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:01.908Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:01.909Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:02 smithi093 ceph-mon[111335]: pgmap v2145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:02 smithi093 ceph-mon[118902]: pgmap v2145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:02 smithi136 ceph-mon[113531]: pgmap v2145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:02.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:02.700912+0000) 2023-08-27T22:05:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:03.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:03.701209+0000) 2023-08-27T22:05:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:03.822Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:03.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:04 smithi093 ceph-mon[111335]: pgmap v2146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/404387206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:04 smithi093 ceph-mon[118902]: pgmap v2146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/404387206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:04 smithi136 ceph-mon[113531]: pgmap v2146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/404387206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:04.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:04.701448+0000) 2023-08-27T22:05:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:05:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:05:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:05:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:05.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:05.701686+0000) 2023-08-27T22:05:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:06 smithi093 ceph-mon[111335]: pgmap v2147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2658033746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:06 smithi093 ceph-mon[118902]: pgmap v2147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2658033746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:06 smithi136 ceph-mon[113531]: pgmap v2147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2658033746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:06.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:06.701887+0000) 2023-08-27T22:05:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:07.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:07.702150+0000) 2023-08-27T22:05:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:08 smithi093 ceph-mon[111335]: pgmap v2148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3926693457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:08 smithi093 ceph-mon[118902]: pgmap v2148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3926693457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:08 smithi136 ceph-mon[113531]: pgmap v2148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3926693457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:08.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:08.702400+0000) 2023-08-27T22:05:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:09.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:09.702667+0000) 2023-08-27T22:05:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:10 smithi093 ceph-mon[111335]: pgmap v2149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:10 smithi093 ceph-mon[118902]: pgmap v2149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:10.394Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:10.395Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:10 smithi136 ceph-mon[113531]: pgmap v2149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:10.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:10.702924+0000) 2023-08-27T22:05:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/34944550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/34944550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/34944550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:11.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:11.703188+0000) 2023-08-27T22:05:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:11.909Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:11.909Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:12 smithi093 ceph-mon[111335]: pgmap v2150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:12 smithi093 ceph-mon[118902]: pgmap v2150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:12 smithi136 ceph-mon[113531]: pgmap v2150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:12.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:12.703443+0000) 2023-08-27T22:05:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1842836239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1842836239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1842836239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:13.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:13.703702+0000) 2023-08-27T22:05:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:13.822Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:13.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:14 smithi093 ceph-mon[111335]: pgmap v2151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:14 smithi093 ceph-mon[118902]: pgmap v2151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:14 smithi136 ceph-mon[113531]: pgmap v2151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:14.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:14.703918+0000) 2023-08-27T22:05:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:05:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:05:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:05:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:15.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:15.704205+0000) 2023-08-27T22:05:16.744 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:16 smithi093 ceph-mon[118902]: pgmap v2152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:16.745 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/597967187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:16.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:16 smithi093 ceph-mon[111335]: pgmap v2152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:16.745 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/597967187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:16 smithi136 ceph-mon[113531]: pgmap v2152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/597967187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:16.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:16.704452+0000) 2023-08-27T22:05:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:17.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:17.704704+0000) 2023-08-27T22:05:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:18 smithi093 ceph-mon[111335]: pgmap v2153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3192080902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:18 smithi093 ceph-mon[118902]: pgmap v2153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3192080902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:18 smithi136 ceph-mon[113531]: pgmap v2153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3192080902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:18.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:18.704962+0000) 2023-08-27T22:05:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:19.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:19.705216+0000) 2023-08-27T22:05:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:20 smithi093 ceph-mon[111335]: pgmap v2154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:20 smithi093 ceph-mon[118902]: pgmap v2154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:20.394Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:20.395Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:20 smithi136 ceph-mon[113531]: pgmap v2154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:20.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:20.705403+0000) 2023-08-27T22:05:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/136365050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/136365050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/136365050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:21.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:21.705579+0000) 2023-08-27T22:05:22.233 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:21.909Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:22.233 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:21.910Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:22 smithi093 ceph-mon[111335]: pgmap v2155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:22 smithi093 ceph-mon[118902]: pgmap v2155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:22 smithi136 ceph-mon[113531]: pgmap v2155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:22.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:22.705848+0000) 2023-08-27T22:05:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1526501137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1526501137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1526501137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:23.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:23.706142+0000) 2023-08-27T22:05:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:23.822Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:23.824Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:24 smithi093 ceph-mon[111335]: pgmap v2156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:24 smithi093 ceph-mon[118902]: pgmap v2156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:24 smithi136 ceph-mon[113531]: pgmap v2156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:24.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:24.706405+0000) 2023-08-27T22:05:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3443085025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:05:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:05:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:05:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3443085025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3443085025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:25.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:25.706688+0000) 2023-08-27T22:05:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:26 smithi136 ceph-mon[113531]: pgmap v2157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:26 smithi093 ceph-mon[111335]: pgmap v2157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:26 smithi093 ceph-mon[118902]: pgmap v2157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:26.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:26.706955+0000) 2023-08-27T22:05:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:27.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:27.707185+0000) 2023-08-27T22:05:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:28 smithi136 ceph-mon[113531]: pgmap v2158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1672040872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:28 smithi093 ceph-mon[111335]: pgmap v2158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1672040872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:28 smithi093 ceph-mon[118902]: pgmap v2158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1672040872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:28.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:28.707383+0000) 2023-08-27T22:05:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:29.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:29.707625+0000) 2023-08-27T22:05:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:30 smithi093 ceph-mon[111335]: pgmap v2159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2182186258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:30 smithi093 ceph-mon[118902]: pgmap v2159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2182186258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:30.395Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:30.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:30.396Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:30 smithi136 ceph-mon[113531]: pgmap v2159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2182186258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:30.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:30.707914+0000) 2023-08-27T22:05:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:31.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:31.708146+0000) 2023-08-27T22:05:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:31.910Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:31.911Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:32 smithi136 ceph-mon[113531]: pgmap v2160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2062648599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:32 smithi093 ceph-mon[111335]: pgmap v2160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2062648599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:32 smithi093 ceph-mon[118902]: pgmap v2160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2062648599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:32.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:32.708355+0000) 2023-08-27T22:05:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:33.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:33.708601+0000) 2023-08-27T22:05:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:33.823Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:33.824Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:34.709 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:34 smithi136 ceph-mon[113531]: pgmap v2161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:34 smithi093 ceph-mon[111335]: pgmap v2161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:34 smithi093 ceph-mon[118902]: pgmap v2161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:34.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:34.708877+0000) 2023-08-27T22:05:35.709 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/216754627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/216754627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:05:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:05:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:05:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/216754627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:35.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:35.709168+0000) 2023-08-27T22:05:36.709 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:36 smithi136 ceph-mon[113531]: pgmap v2162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:36 smithi093 ceph-mon[111335]: pgmap v2162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:36 smithi093 ceph-mon[118902]: pgmap v2162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:36.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:36.709404+0000) 2023-08-27T22:05:37.709 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3393194679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3393194679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3393194679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:37.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:37.709607+0000) 2023-08-27T22:05:38.710 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:38 smithi136 ceph-mon[113531]: pgmap v2163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:38 smithi093 ceph-mon[111335]: pgmap v2163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:38 smithi093 ceph-mon[118902]: pgmap v2163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:38.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:38.709848+0000) 2023-08-27T22:05:39.710 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:39.710 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3917548307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3917548307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3917548307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:39.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:39.710088+0000) 2023-08-27T22:05:40.710 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:40 smithi136 ceph-mon[113531]: pgmap v2164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:40 smithi093 ceph-mon[111335]: pgmap v2164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:40 smithi093 ceph-mon[118902]: pgmap v2164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:40.395Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:40.396Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:40.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:40.710299+0000) 2023-08-27T22:05:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:41.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:41.710536+0000) 2023-08-27T22:05:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:41.910Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:41.912Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:42.710 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:42 smithi136 ceph-mon[113531]: pgmap v2165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:42.711 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1931850892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:42 smithi093 ceph-mon[111335]: pgmap v2165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1931850892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:42 smithi093 ceph-mon[118902]: pgmap v2165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1931850892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:42.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:42.710792+0000) 2023-08-27T22:05:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:43.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:43.711060+0000) 2023-08-27T22:05:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:43.823Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:43.824Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:44 smithi093 ceph-mon[111335]: pgmap v2166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2365970406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:44 smithi093 ceph-mon[118902]: pgmap v2166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2365970406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:44 smithi136 ceph-mon[113531]: pgmap v2166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2365970406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:44.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:44.711338+0000) 2023-08-27T22:05:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:05:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:05:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:05:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:05:45] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T22:05:45.890 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:05:45.891 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:45.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:45.711509+0000) 2023-08-27T22:05:46.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:46 smithi136 ceph-mon[113531]: pgmap v2167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:46 smithi093 ceph-mon[111335]: pgmap v2167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:46 smithi093 ceph-mon[118902]: pgmap v2167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:46.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:46.711738+0000) 2023-08-27T22:05:47.472 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1474590220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:47.472 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:47.472 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1474590220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1474590220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:47.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:47.711929+0000) 2023-08-27T22:05:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:48.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:48.712133+0000) 2023-08-27T22:05:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:48 smithi136 ceph-mon[113531]: pgmap v2168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:05:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:05:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[111335]: pgmap v2168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[118902]: pgmap v2168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:05:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:05:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:05:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1922919121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:49.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:49.712331+0000) 2023-08-27T22:05:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1922919121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1922919121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:50 smithi093 ceph-mon[111335]: pgmap v2169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:50 smithi093 ceph-mon[118902]: pgmap v2169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:50.395Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:50.397Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:50 smithi136 ceph-mon[113531]: pgmap v2169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:50.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:50.712542+0000) 2023-08-27T22:05:51.911 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2443419411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:51.912 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2443419411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2443419411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:51.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:51.712751+0000) 2023-08-27T22:05:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:51.911Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:51.912Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:52 smithi136 ceph-mon[113531]: pgmap v2170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:52.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:52.712954+0000) 2023-08-27T22:05:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:52 smithi093 ceph-mon[111335]: pgmap v2170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:52 smithi093 ceph-mon[118902]: pgmap v2170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:53.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:53.713159+0000) 2023-08-27T22:05:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:53.824Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:05:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:05:53.825Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:05:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:54 smithi136 ceph-mon[113531]: pgmap v2171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1608452310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:54.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:54.713362+0000) 2023-08-27T22:05:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:54 smithi093 ceph-mon[111335]: pgmap v2171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1608452310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:54 smithi093 ceph-mon[118902]: pgmap v2171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:05:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1608452310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:05:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:05:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:05:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:55.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:55.713639+0000) 2023-08-27T22:05:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:56 smithi136 ceph-mon[113531]: pgmap v2172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/39320163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:56.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:56.713864+0000) 2023-08-27T22:05:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:56 smithi093 ceph-mon[111335]: pgmap v2172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/39320163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:56 smithi093 ceph-mon[118902]: pgmap v2172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/39320163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:57.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:57.714104+0000) 2023-08-27T22:05:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:58 smithi136 ceph-mon[113531]: pgmap v2173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:05:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/93067192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:58.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:58.714263+0000) 2023-08-27T22:05:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:58 smithi093 ceph-mon[111335]: pgmap v2173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:05:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/93067192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:05:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:58 smithi093 ceph-mon[118902]: pgmap v2173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:05:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:05:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/93067192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:05:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:05:59.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:05:59.714489+0000) 2023-08-27T22:06:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:00 smithi093 ceph-mon[111335]: pgmap v2174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:00 smithi093 ceph-mon[118902]: pgmap v2174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:00.396Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:00.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:00.397Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:00 smithi136 ceph-mon[113531]: pgmap v2174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:00.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:00.714741+0000) 2023-08-27T22:06:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1431931588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1431931588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1431931588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:01.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:01.714953+0000) 2023-08-27T22:06:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:01.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:01.914Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:02 smithi136 ceph-mon[113531]: pgmap v2175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:02 smithi093 ceph-mon[111335]: pgmap v2175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:02 smithi093 ceph-mon[118902]: pgmap v2175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:02.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:02.715202+0000) 2023-08-27T22:06:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1006595859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1006595859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1006595859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:03.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:03.715440+0000) 2023-08-27T22:06:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:03.825Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:03.826Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:04 smithi136 ceph-mon[113531]: pgmap v2176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:04 smithi093 ceph-mon[111335]: pgmap v2176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:04 smithi093 ceph-mon[118902]: pgmap v2176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:04.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:04.715658+0000) 2023-08-27T22:06:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:06:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:06:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:06:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:05.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:05.715815+0000) 2023-08-27T22:06:06.716 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:06 smithi136 ceph-mon[113531]: pgmap v2177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:06.716 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/691578762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:06 smithi093 ceph-mon[111335]: pgmap v2177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/691578762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:06 smithi093 ceph-mon[118902]: pgmap v2177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/691578762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:06.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:06.716051+0000) 2023-08-27T22:06:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:07.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:07.716201+0000) 2023-08-27T22:06:08.716 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:08 smithi136 ceph-mon[113531]: pgmap v2178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:08.716 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/146382877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:08 smithi093 ceph-mon[111335]: pgmap v2178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/146382877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:08 smithi093 ceph-mon[118902]: pgmap v2178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/146382877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:09.716 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:09.717 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:08.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:08.716433+0000) 2023-08-27T22:06:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:09.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:09.716725+0000) 2023-08-27T22:06:10.717 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:10 smithi136 ceph-mon[113531]: pgmap v2179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:10.718 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2881467789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:10 smithi093 ceph-mon[111335]: pgmap v2179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2881467789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:10 smithi093 ceph-mon[118902]: pgmap v2179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2881467789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:10.397Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:10.398Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:10.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:10.716956+0000) 2023-08-27T22:06:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:11.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:11.717206+0000) 2023-08-27T22:06:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:11.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:11.913Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:12.718 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:12 smithi136 ceph-mon[113531]: pgmap v2180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:12 smithi093 ceph-mon[111335]: pgmap v2180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:12 smithi093 ceph-mon[118902]: pgmap v2180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:12.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:12.717375+0000) 2023-08-27T22:06:13.717 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/808547727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/808547727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/808547727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:13.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:13.717544+0000) 2023-08-27T22:06:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:13.825Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:13.826Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:14.718 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:14 smithi136 ceph-mon[113531]: pgmap v2181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:14 smithi093 ceph-mon[118902]: pgmap v2181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:14 smithi093 ceph-mon[111335]: pgmap v2181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:14.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:14.717813+0000) 2023-08-27T22:06:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2729505514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:06:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:06:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:06:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2729505514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2729505514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:15.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:15.718109+0000) 2023-08-27T22:06:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:16 smithi093 ceph-mon[111335]: pgmap v2182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:16 smithi093 ceph-mon[118902]: pgmap v2182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:16 smithi136 ceph-mon[113531]: pgmap v2182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:16.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:16.718352+0000) 2023-08-27T22:06:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:17.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:17.718582+0000) 2023-08-27T22:06:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:18 smithi093 ceph-mon[111335]: pgmap v2183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1791687250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:18 smithi093 ceph-mon[118902]: pgmap v2183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1791687250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:18 smithi136 ceph-mon[113531]: pgmap v2183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1791687250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:18.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:18.718832+0000) 2023-08-27T22:06:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:19.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:19.719083+0000) 2023-08-27T22:06:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:20 smithi093 ceph-mon[118902]: pgmap v2184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3602673422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:20 smithi093 ceph-mon[111335]: pgmap v2184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3602673422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:20.398Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:20.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:20.399Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:20 smithi136 ceph-mon[113531]: pgmap v2184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3602673422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:20.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:20.719290+0000) 2023-08-27T22:06:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:21.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:21.719472+0000) 2023-08-27T22:06:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:21.913Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:21.914Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:22.720 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:22 smithi136 ceph-mon[113531]: pgmap v2185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:22.720 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4173472598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:22 smithi093 ceph-mon[111335]: pgmap v2185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4173472598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:22 smithi093 ceph-mon[118902]: pgmap v2185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4173472598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:22.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:22.719738+0000) 2023-08-27T22:06:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:23.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:23.719960+0000) 2023-08-27T22:06:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:23.826Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:23.827Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:24.720 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:24 smithi136 ceph-mon[113531]: pgmap v2186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:24.720 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:24 smithi093 ceph-mon[118902]: pgmap v2186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:24 smithi093 ceph-mon[111335]: pgmap v2186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:24.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:24.720196+0000) 2023-08-27T22:06:25.720 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1763421167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1763421167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:06:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:06:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:06:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1763421167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:25.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:25.720443+0000) 2023-08-27T22:06:26.720 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:26 smithi136 ceph-mon[113531]: pgmap v2187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:26 smithi093 ceph-mon[111335]: pgmap v2187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:26 smithi093 ceph-mon[118902]: pgmap v2187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:26.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:26.720673+0000) 2023-08-27T22:06:27.721 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2082232364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2082232364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2082232364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:27.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:27.720921+0000) 2023-08-27T22:06:28.721 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:28 smithi136 ceph-mon[113531]: pgmap v2188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:28 smithi093 ceph-mon[111335]: pgmap v2188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:28 smithi093 ceph-mon[118902]: pgmap v2188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:28.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:28.721184+0000) 2023-08-27T22:06:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:29.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:29.721425+0000) 2023-08-27T22:06:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:30 smithi136 ceph-mon[113531]: pgmap v2189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/50669198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:30 smithi093 ceph-mon[111335]: pgmap v2189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/50669198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:30 smithi093 ceph-mon[118902]: pgmap v2189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/50669198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:30.398Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:30.399Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:30.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:30.721693+0000) 2023-08-27T22:06:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:31.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:31.721920+0000) 2023-08-27T22:06:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:31.913Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:31.914Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:32.722 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:32 smithi136 ceph-mon[113531]: pgmap v2190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:32.722 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/377033084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:32 smithi093 ceph-mon[111335]: pgmap v2190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/377033084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:32 smithi093 ceph-mon[118902]: pgmap v2190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/377033084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:32.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:32.722166+0000) 2023-08-27T22:06:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:33.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:33.722380+0000) 2023-08-27T22:06:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:33.827Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:33.828Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:34.722 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:34 smithi136 ceph-mon[113531]: pgmap v2191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:34.722 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1635406543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:34 smithi093 ceph-mon[118902]: pgmap v2191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1635406543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:34 smithi093 ceph-mon[111335]: pgmap v2191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1635406543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:34.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:34.722640+0000) 2023-08-27T22:06:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:06:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:06:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:06:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:35.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:35.722829+0000) 2023-08-27T22:06:36.724 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:36 smithi136 ceph-mon[113531]: pgmap v2192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:36 smithi093 ceph-mon[111335]: pgmap v2192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:36 smithi093 ceph-mon[118902]: pgmap v2192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:36.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:36.723076+0000) 2023-08-27T22:06:37.723 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3175696849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3175696849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3175696849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:37.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:37.723307+0000) 2023-08-27T22:06:38.723 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:38 smithi136 ceph-mon[113531]: pgmap v2193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:38 smithi093 ceph-mon[111335]: pgmap v2193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:38 smithi093 ceph-mon[118902]: pgmap v2193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:38.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:38.723579+0000) 2023-08-27T22:06:39.723 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:39.724 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3090825392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3090825392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3090825392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:39.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:39.723802+0000) 2023-08-27T22:06:40.724 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:40 smithi136 ceph-mon[113531]: pgmap v2194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:40 smithi093 ceph-mon[111335]: pgmap v2194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:40 smithi093 ceph-mon[118902]: pgmap v2194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:40.398Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:40.399Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:40.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:40.724017+0000) 2023-08-27T22:06:41.724 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3985975119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3985975119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3985975119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:41.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:41.724271+0000) 2023-08-27T22:06:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:41.914Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:41.915Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:42 smithi093 ceph-mon[111335]: pgmap v2195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:42 smithi093 ceph-mon[118902]: pgmap v2195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:42 smithi136 ceph-mon[113531]: pgmap v2195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:42.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:42.724514+0000) 2023-08-27T22:06:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:43.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:43.724765+0000) 2023-08-27T22:06:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:43.827Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:43.828Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:44 smithi093 ceph-mon[111335]: pgmap v2196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3118742810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:44 smithi093 ceph-mon[118902]: pgmap v2196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3118742810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:44 smithi136 ceph-mon[113531]: pgmap v2196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3118742810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:44.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:44.725064+0000) 2023-08-27T22:06:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:06:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:06:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:06:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:45.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:45.725310+0000) 2023-08-27T22:06:46.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:46 smithi093 ceph-mon[111335]: pgmap v2197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:46.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/548195252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:46.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:46 smithi093 ceph-mon[118902]: pgmap v2197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:46.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/548195252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:46.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:46.725558+0000) 2023-08-27T22:06:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:46 smithi136 ceph-mon[113531]: pgmap v2197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/548195252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:47.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:47.725841+0000) 2023-08-27T22:06:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:48 smithi093 ceph-mon[111335]: pgmap v2198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:06:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:48 smithi093 ceph-mon[118902]: pgmap v2198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:06:48.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:48 smithi136 ceph-mon[113531]: pgmap v2198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:48.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:06:48.842 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:48.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:48.726091+0000) 2023-08-27T22:06:49.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1363034714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1363034714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1363034714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:49.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:49.726259+0000) 2023-08-27T22:06:50.726 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:50 smithi136 ceph-mon[113531]: pgmap v2199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:50.727 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:50.727 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-mon[118902]: pgmap v2199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-mon[111335]: pgmap v2199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:50.399Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:50.400Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:50.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:50.726423+0000) 2023-08-27T22:06:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/923730862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:06:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:06:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:51.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:51.726628+0000) 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/923730862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/923730862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:52.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:06:52.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:06:52.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:06:52.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:51.915Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:52.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:51.916Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:52 smithi136 ceph-mon[113531]: pgmap v2200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:52.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:52.726868+0000) 2023-08-27T22:06:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:52 smithi093 ceph-mon[111335]: pgmap v2200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:52 smithi093 ceph-mon[118902]: pgmap v2200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/214637859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:53.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:53.727083+0000) 2023-08-27T22:06:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/214637859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/214637859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:06:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:53.828Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:06:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:06:53.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:06:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:54 smithi136 ceph-mon[113531]: pgmap v2201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:54.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:54.727309+0000) 2023-08-27T22:06:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:54 smithi093 ceph-mon[118902]: pgmap v2201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:54 smithi093 ceph-mon[111335]: pgmap v2201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:06:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:06:55] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:06:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/347978307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:55.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:55.727544+0000) 2023-08-27T22:06:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/347978307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/347978307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:56 smithi136 ceph-mon[113531]: pgmap v2202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:56.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:56.727778+0000) 2023-08-27T22:06:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:56 smithi093 ceph-mon[111335]: pgmap v2202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:56 smithi093 ceph-mon[118902]: pgmap v2202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:57.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:57.728070+0000) 2023-08-27T22:06:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:58 smithi136 ceph-mon[113531]: pgmap v2203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:59.207 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:06:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/667204710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:59.207 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:58.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:58.728275+0000) 2023-08-27T22:06:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:58 smithi093 ceph-mon[111335]: pgmap v2203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:06:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/667204710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:06:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:58 smithi093 ceph-mon[118902]: pgmap v2203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:06:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:06:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/667204710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:06:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:06:59.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:06:59.728512+0000) 2023-08-27T22:07:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:00 smithi136 ceph-mon[113531]: pgmap v2204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:00 smithi093 ceph-mon[111335]: pgmap v2204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:00 smithi093 ceph-mon[118902]: pgmap v2204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:00.400Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:00.400Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:00.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:00.728764+0000) 2023-08-27T22:07:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2496701708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2496701708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2496701708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:01.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:01.729001+0000) 2023-08-27T22:07:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:01.916Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:01.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:02 smithi136 ceph-mon[113531]: pgmap v2205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:02 smithi093 ceph-mon[111335]: pgmap v2205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:02 smithi093 ceph-mon[118902]: pgmap v2205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:02.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:02.729154+0000) 2023-08-27T22:07:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1774573706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:03.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1774573706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:03.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1774573706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:03.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:03.729346+0000) 2023-08-27T22:07:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:03.828Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:03.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:04 smithi136 ceph-mon[113531]: pgmap v2206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:04 smithi093 ceph-mon[111335]: pgmap v2206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:04 smithi093 ceph-mon[118902]: pgmap v2206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:04.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:04.729596+0000) 2023-08-27T22:07:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2548806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2548806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2548806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:07:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:07:05] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:07:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:05.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:05.729835+0000) 2023-08-27T22:07:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:06 smithi136 ceph-mon[113531]: pgmap v2207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:06 smithi093 ceph-mon[111335]: pgmap v2207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:06 smithi093 ceph-mon[118902]: pgmap v2207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:06.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:06.729995+0000) 2023-08-27T22:07:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:07.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:07.730300+0000) 2023-08-27T22:07:08.730 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:08 smithi136 ceph-mon[113531]: pgmap v2208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:08.730 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2863139857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:08 smithi093 ceph-mon[111335]: pgmap v2208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2863139857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:08 smithi093 ceph-mon[118902]: pgmap v2208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2863139857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:08.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:08.730545+0000) 2023-08-27T22:07:09.730 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:09.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:09.730852+0000) 2023-08-27T22:07:10.731 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:10 smithi136 ceph-mon[113531]: pgmap v2209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:10.731 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1045917862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:10 smithi093 ceph-mon[111335]: pgmap v2209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1045917862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:10 smithi093 ceph-mon[118902]: pgmap v2209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1045917862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:10.400Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:10.401Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:10.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:10.731175+0000) 2023-08-27T22:07:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:11.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:11.731401+0000) 2023-08-27T22:07:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:11.917Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:11.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:12.731 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:12 smithi136 ceph-mon[113531]: pgmap v2210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:12.732 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3927642466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:12 smithi093 ceph-mon[111335]: pgmap v2210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3927642466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:12 smithi093 ceph-mon[118902]: pgmap v2210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3927642466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:12.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:12.731689+0000) 2023-08-27T22:07:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:13.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:13.731965+0000) 2023-08-27T22:07:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:13.829Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:13.830Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:14.732 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:14 smithi136 ceph-mon[113531]: pgmap v2211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:14 smithi093 ceph-mon[111335]: pgmap v2211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:14 smithi093 ceph-mon[118902]: pgmap v2211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:14.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:14.732289+0000) 2023-08-27T22:07:15.732 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2951413315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2951413315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:07:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:07:15] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:07:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2951413315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:15.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:15.732555+0000) 2023-08-27T22:07:16.733 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:16 smithi136 ceph-mon[113531]: pgmap v2212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:16 smithi093 ceph-mon[111335]: pgmap v2212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:16 smithi093 ceph-mon[118902]: pgmap v2212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:16.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:16.732820+0000) 2023-08-27T22:07:17.733 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3429387592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3429387592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3429387592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:17.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:17.733053+0000) 2023-08-27T22:07:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:18 smithi093 ceph-mon[111335]: pgmap v2213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:18 smithi093 ceph-mon[118902]: pgmap v2213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:18 smithi136 ceph-mon[113531]: pgmap v2213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:18.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:18.733317+0000) 2023-08-27T22:07:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:19.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:19.733549+0000) 2023-08-27T22:07:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:20 smithi136 ceph-mon[113531]: pgmap v2214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2007951740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:20 smithi093 ceph-mon[111335]: pgmap v2214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2007951740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:20 smithi093 ceph-mon[118902]: pgmap v2214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2007951740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:20.401Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:20.402Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:20.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:20.733809+0000) 2023-08-27T22:07:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:21.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:21.734116+0000) 2023-08-27T22:07:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:21.918Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:21.919Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:22.734 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:22 smithi136 ceph-mon[113531]: pgmap v2215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:22.734 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1360030272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:22 smithi093 ceph-mon[111335]: pgmap v2215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1360030272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:22 smithi093 ceph-mon[118902]: pgmap v2215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1360030272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:22.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:22.734375+0000) 2023-08-27T22:07:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:23.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:23.734637+0000) 2023-08-27T22:07:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:23.829Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:23.831Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:24.735 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:24 smithi136 ceph-mon[113531]: pgmap v2216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:24.735 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:24.735 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3901027570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:24 smithi093 ceph-mon[111335]: pgmap v2216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3901027570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:24 smithi093 ceph-mon[118902]: pgmap v2216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3901027570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:24.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:24.734916+0000) 2023-08-27T22:07:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:07:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:07:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:07:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:25.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:25.735181+0000) 2023-08-27T22:07:26.735 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:26 smithi136 ceph-mon[113531]: pgmap v2217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:26 smithi093 ceph-mon[111335]: pgmap v2217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:26 smithi093 ceph-mon[118902]: pgmap v2217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:26.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:26.735447+0000) 2023-08-27T22:07:27.735 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1273729108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1273729108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1273729108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:27.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:27.735720+0000) 2023-08-27T22:07:28.736 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:28 smithi136 ceph-mon[113531]: pgmap v2218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:28 smithi093 ceph-mon[111335]: pgmap v2218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:28 smithi093 ceph-mon[118902]: pgmap v2218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:28.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:28.736003+0000) 2023-08-27T22:07:29.736 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/349628273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/349628273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/349628273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:29.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:29.736306+0000) 2023-08-27T22:07:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:30 smithi093 ceph-mon[111335]: pgmap v2219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:30 smithi093 ceph-mon[118902]: pgmap v2219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:30.401Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:30.402Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:30 smithi136 ceph-mon[113531]: pgmap v2219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:30.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:30.736529+0000) 2023-08-27T22:07:31.919 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1293405872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:31.919 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1293405872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1293405872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:31.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:31.736769+0000) 2023-08-27T22:07:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:31.919Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:31.920Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:32 smithi136 ceph-mon[113531]: pgmap v2220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:32.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:32.737007+0000) 2023-08-27T22:07:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:32 smithi093 ceph-mon[111335]: pgmap v2220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:32 smithi093 ceph-mon[118902]: pgmap v2220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:33.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:33.737262+0000) 2023-08-27T22:07:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:33.830Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:33.831Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:34 smithi136 ceph-mon[113531]: pgmap v2221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1710370355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:34.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:34.737588+0000) 2023-08-27T22:07:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:34 smithi093 ceph-mon[111335]: pgmap v2221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1710370355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:34 smithi093 ceph-mon[118902]: pgmap v2221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1710370355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:07:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:07:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:07:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:35.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:35.737874+0000) 2023-08-27T22:07:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:36 smithi136 ceph-mon[113531]: pgmap v2222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1488672292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:36.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:36.738161+0000) 2023-08-27T22:07:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:36 smithi093 ceph-mon[111335]: pgmap v2222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1488672292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:36 smithi093 ceph-mon[118902]: pgmap v2222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1488672292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:37.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:37.738431+0000) 2023-08-27T22:07:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:38.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:38.738685+0000) 2023-08-27T22:07:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:38 smithi136 ceph-mon[113531]: pgmap v2223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:38 smithi093 ceph-mon[111335]: pgmap v2223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:38 smithi093 ceph-mon[118902]: pgmap v2223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/544534153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:39.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:39.738909+0000) 2023-08-27T22:07:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/544534153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/544534153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:40 smithi093 ceph-mon[111335]: pgmap v2224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:40 smithi093 ceph-mon[118902]: pgmap v2224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:40.401Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:40.402Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:40 smithi136 ceph-mon[113531]: pgmap v2224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:40.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:40.739117+0000) 2023-08-27T22:07:41.919 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2084672368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:41.920 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2084672368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2084672368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:41.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:41.739333+0000) 2023-08-27T22:07:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:41.919Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:41.920Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:42 smithi136 ceph-mon[113531]: pgmap v2225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:42.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:42.739599+0000) 2023-08-27T22:07:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:42 smithi093 ceph-mon[111335]: pgmap v2225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:42 smithi093 ceph-mon[118902]: pgmap v2225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:43.831 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1102644179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:43.831 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1102644179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1102644179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:43.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:43.739904+0000) 2023-08-27T22:07:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:43.830Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:43.832Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:44 smithi136 ceph-mon[113531]: pgmap v2226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:44.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:44.740168+0000) 2023-08-27T22:07:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:44 smithi093 ceph-mon[111335]: pgmap v2226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:44 smithi093 ceph-mon[118902]: pgmap v2226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:07:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:07:45] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:07:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:45.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:45.740410+0000) 2023-08-27T22:07:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:46 smithi136 ceph-mon[113531]: pgmap v2227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1475904567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:46.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:46.740652+0000) 2023-08-27T22:07:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:46 smithi093 ceph-mon[111335]: pgmap v2227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1475904567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:46 smithi093 ceph-mon[118902]: pgmap v2227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1475904567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:47.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:47.740917+0000) 2023-08-27T22:07:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:48 smithi136 ceph-mon[113531]: pgmap v2228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1695706629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:48.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:48.741133+0000) 2023-08-27T22:07:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:48 smithi093 ceph-mon[111335]: pgmap v2228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1695706629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:48 smithi093 ceph-mon[118902]: pgmap v2228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1695706629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:49.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:49.741385+0000) 2023-08-27T22:07:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:50 smithi136 ceph-mon[113531]: pgmap v2229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:50 smithi093 ceph-mon[111335]: pgmap v2229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:50 smithi093 ceph-mon[118902]: pgmap v2229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:50.402Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:50.403Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:50.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:50.741637+0000) 2023-08-27T22:07:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/509184149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/509184149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/509184149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:51.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:51.741845+0000) 2023-08-27T22:07:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:51.920Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:51.921Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:07:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:52 smithi136 ceph-mon[113531]: pgmap v2230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:07:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:52 smithi093 ceph-mon[111335]: pgmap v2230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:07:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:52 smithi093 ceph-mon[118902]: pgmap v2230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:52.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:52.741984+0000) 2023-08-27T22:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4066701285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4066701285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4066701285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:53.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:53.742197+0000) 2023-08-27T22:07:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:53.831Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:07:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:07:53.832Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:07:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-mon[113531]: pgmap v2231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:07:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:07:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[118902]: pgmap v2231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:07:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[111335]: pgmap v2231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:07:54.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:07:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:54.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:54.742446+0000) 2023-08-27T22:07:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4255286510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4255286510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:07:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:07:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:07:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4255286510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:55.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:55.742662+0000) 2023-08-27T22:07:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:56 smithi136 ceph-mon[113531]: pgmap v2232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:56 smithi093 ceph-mon[111335]: pgmap v2232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:56 smithi093 ceph-mon[118902]: pgmap v2232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:56.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:56.742939+0000) 2023-08-27T22:07:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:57.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:57.743228+0000) 2023-08-27T22:07:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:58 smithi093 ceph-mon[111335]: pgmap v2233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:07:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4163725737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:58 smithi093 ceph-mon[118902]: pgmap v2233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:07:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4163725737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:58.743 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:58 smithi136 ceph-mon[113531]: pgmap v2233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:07:58.743 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:07:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4163725737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:07:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:58.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:58.743515+0000) 2023-08-27T22:08:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:07:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:07:59.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:07:59.743707+0000) 2023-08-27T22:08:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:00 smithi136 ceph-mon[113531]: pgmap v2234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2002732813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:00 smithi093 ceph-mon[111335]: pgmap v2234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2002732813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:00 smithi093 ceph-mon[118902]: pgmap v2234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2002732813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:00.402Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:00.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:00.403Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:00.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:00.744005+0000) 2023-08-27T22:08:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:01.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:01.744300+0000) 2023-08-27T22:08:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:01.920Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:01.921Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:02 smithi093 ceph-mon[111335]: pgmap v2235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2795968303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:02 smithi093 ceph-mon[118902]: pgmap v2235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2795968303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:02.744 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:02 smithi136 ceph-mon[113531]: pgmap v2235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:02.745 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2795968303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:02.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:02.744530+0000) 2023-08-27T22:08:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:03.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:03.744781+0000) 2023-08-27T22:08:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:03.832Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:03.833Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:04 smithi093 ceph-mon[111335]: pgmap v2236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:04 smithi093 ceph-mon[118902]: pgmap v2236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:04.744 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:04 smithi136 ceph-mon[113531]: pgmap v2236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:04.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:04.745053+0000) 2023-08-27T22:08:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1905201007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1905201007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:08:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:08:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:08:05.745 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1905201007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:05.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:05.745280+0000) 2023-08-27T22:08:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:06 smithi093 ceph-mon[111335]: pgmap v2237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:06 smithi093 ceph-mon[118902]: pgmap v2237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:06.745 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:06 smithi136 ceph-mon[113531]: pgmap v2237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:06.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:06.745527+0000) 2023-08-27T22:08:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2796158137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2796158137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:07.745 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2796158137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:07.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:07.745795+0000) 2023-08-27T22:08:08.746 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:08 smithi136 ceph-mon[113531]: pgmap v2238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:08 smithi093 ceph-mon[111335]: pgmap v2238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:08 smithi093 ceph-mon[118902]: pgmap v2238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:08.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:08.746085+0000) 2023-08-27T22:08:09.746 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:09.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:09.746314+0000) 2023-08-27T22:08:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:10 smithi093 ceph-mon[118902]: pgmap v2239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2670887994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:10 smithi093 ceph-mon[111335]: pgmap v2239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2670887994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:10.402Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:10.403Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:10 smithi136 ceph-mon[113531]: pgmap v2239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2670887994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:10.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:10.746573+0000) 2023-08-27T22:08:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:11.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:11.746843+0000) 2023-08-27T22:08:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:11.920Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:11.921Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:12 smithi136 ceph-mon[113531]: pgmap v2240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/903316546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:12.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:12.747173+0000) 2023-08-27T22:08:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:12 smithi093 ceph-mon[111335]: pgmap v2240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/903316546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:12 smithi093 ceph-mon[118902]: pgmap v2240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/903316546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:13.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:13.747350+0000) 2023-08-27T22:08:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:13.833Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:13.834Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:14 smithi136 ceph-mon[113531]: pgmap v2241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2067217048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:14.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:14.747637+0000) 2023-08-27T22:08:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:14 smithi093 ceph-mon[111335]: pgmap v2241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2067217048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:14 smithi093 ceph-mon[118902]: pgmap v2241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2067217048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:08:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:08:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:08:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:15.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:15.747901+0000) 2023-08-27T22:08:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:16 smithi136 ceph-mon[113531]: pgmap v2242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:16.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:16.748168+0000) 2023-08-27T22:08:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:16 smithi093 ceph-mon[111335]: pgmap v2242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:16 smithi093 ceph-mon[118902]: pgmap v2242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3253509772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:17.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:17.748407+0000) 2023-08-27T22:08:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3253509772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3253509772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:18 smithi136 ceph-mon[113531]: pgmap v2243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:18.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:18.748656+0000) 2023-08-27T22:08:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:18 smithi093 ceph-mon[111335]: pgmap v2243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:18 smithi093 ceph-mon[118902]: pgmap v2243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3233630677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:19.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:19.748888+0000) 2023-08-27T22:08:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3233630677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3233630677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:20 smithi093 ceph-mon[111335]: pgmap v2244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:20 smithi093 ceph-mon[118902]: pgmap v2244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:20.403Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:20.404Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:20 smithi136 ceph-mon[113531]: pgmap v2244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:20.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:20.749158+0000) 2023-08-27T22:08:21.922 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2407859497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:21.922 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2407859497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2407859497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:21.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:21.749419+0000) 2023-08-27T22:08:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:21.921Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:21.922Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:22 smithi136 ceph-mon[113531]: pgmap v2245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:22.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:22.749680+0000) 2023-08-27T22:08:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:22 smithi093 ceph-mon[111335]: pgmap v2245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:22 smithi093 ceph-mon[118902]: pgmap v2245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:23.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:23.749996+0000) 2023-08-27T22:08:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:23.833Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:23.834Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:24 smithi136 ceph-mon[113531]: pgmap v2246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4230765405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:24.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:24.750263+0000) 2023-08-27T22:08:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:24 smithi093 ceph-mon[111335]: pgmap v2246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4230765405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:24 smithi093 ceph-mon[118902]: pgmap v2246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4230765405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:08:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:08:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:08:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:25.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:25.750526+0000) 2023-08-27T22:08:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:26 smithi136 ceph-mon[113531]: pgmap v2247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1019207153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:26.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:26.750810+0000) 2023-08-27T22:08:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:26 smithi093 ceph-mon[111335]: pgmap v2247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1019207153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:26 smithi093 ceph-mon[118902]: pgmap v2247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1019207153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:27.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:27.751042+0000) 2023-08-27T22:08:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:28 smithi136 ceph-mon[113531]: pgmap v2248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:28.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:28.751320+0000) 2023-08-27T22:08:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:28 smithi093 ceph-mon[118902]: pgmap v2248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:28 smithi093 ceph-mon[111335]: pgmap v2248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3625434830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:29.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:29.751604+0000) 2023-08-27T22:08:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3625434830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3625434830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:30 smithi093 ceph-mon[111335]: pgmap v2249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:30 smithi093 ceph-mon[118902]: pgmap v2249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:30.403Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:30.404Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:30 smithi136 ceph-mon[113531]: pgmap v2249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:30.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:30.751899+0000) 2023-08-27T22:08:31.922 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/720514958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:31.923 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/720514958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/720514958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:31.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:31.752175+0000) 2023-08-27T22:08:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:31.922Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:31.923Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:32 smithi136 ceph-mon[113531]: pgmap v2250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:32.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:32.752400+0000) 2023-08-27T22:08:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:32 smithi093 ceph-mon[111335]: pgmap v2250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:32 smithi093 ceph-mon[118902]: pgmap v2250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:33.834 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2017843117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:33.835 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2017843117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2017843117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:33.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:33.752671+0000) 2023-08-27T22:08:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:33.834Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:33.835Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:34 smithi136 ceph-mon[113531]: pgmap v2251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:34.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:34.752983+0000) 2023-08-27T22:08:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:34 smithi093 ceph-mon[111335]: pgmap v2251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:34 smithi093 ceph-mon[118902]: pgmap v2251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:08:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:08:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:08:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:35.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:35.753297+0000) 2023-08-27T22:08:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:36 smithi136 ceph-mon[113531]: pgmap v2252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3754543375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:36.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:36.753543+0000) 2023-08-27T22:08:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:36 smithi093 ceph-mon[111335]: pgmap v2252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3754543375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:36 smithi093 ceph-mon[118902]: pgmap v2252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3754543375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:37.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:37.753772+0000) 2023-08-27T22:08:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:38 smithi136 ceph-mon[113531]: pgmap v2253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2547071699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:38.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:38.754054+0000) 2023-08-27T22:08:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:38 smithi093 ceph-mon[118902]: pgmap v2253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2547071699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:38 smithi093 ceph-mon[111335]: pgmap v2253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2547071699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:39.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:39.754310+0000) 2023-08-27T22:08:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:40 smithi093 ceph-mon[111335]: pgmap v2254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:40 smithi093 ceph-mon[118902]: pgmap v2254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:40.404Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:40.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:40.405Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:40 smithi136 ceph-mon[113531]: pgmap v2254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:40.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:40.754567+0000) 2023-08-27T22:08:41.923 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3265271725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:41.923 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3265271725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3265271725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:41.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:41.754876+0000) 2023-08-27T22:08:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:41.923Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:41.924Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:42 smithi136 ceph-mon[113531]: pgmap v2255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:42.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:42.755174+0000) 2023-08-27T22:08:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:42 smithi093 ceph-mon[111335]: pgmap v2255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:42 smithi093 ceph-mon[118902]: pgmap v2255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:43.836 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/414540321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:43.836 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/414540321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/414540321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:43.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:43.755396+0000) 2023-08-27T22:08:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:43.835Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:43.836Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:44 smithi136 ceph-mon[113531]: pgmap v2256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:44.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:44.755696+0000) 2023-08-27T22:08:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:44 smithi093 ceph-mon[111335]: pgmap v2256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:44 smithi093 ceph-mon[118902]: pgmap v2256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2535289839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2535289839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:08:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:08:45] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:08:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2535289839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:45.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:45.756010+0000) 2023-08-27T22:08:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:46 smithi136 ceph-mon[113531]: pgmap v2257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:46.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:46.756314+0000) 2023-08-27T22:08:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:46 smithi093 ceph-mon[118902]: pgmap v2257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:46 smithi093 ceph-mon[111335]: pgmap v2257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:47.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:47.756600+0000) 2023-08-27T22:08:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:48 smithi136 ceph-mon[113531]: pgmap v2258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2970837119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:48.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:48.756901+0000) 2023-08-27T22:08:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:48 smithi093 ceph-mon[111335]: pgmap v2258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2970837119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:48 smithi093 ceph-mon[118902]: pgmap v2258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2970837119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:49.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:49.757198+0000) 2023-08-27T22:08:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:50 smithi136 ceph-mon[113531]: pgmap v2259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/628749840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:50 smithi093 ceph-mon[111335]: pgmap v2259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/628749840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:50 smithi093 ceph-mon[118902]: pgmap v2259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/628749840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:50.404Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:50.406Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:50.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:50.757471+0000) 2023-08-27T22:08:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:51.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:51.757693+0000) 2023-08-27T22:08:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:51.924Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:51.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:52 smithi093 ceph-mon[111335]: pgmap v2260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:52 smithi093 ceph-mon[118902]: pgmap v2260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:52.759 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:52 smithi136 ceph-mon[113531]: pgmap v2260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:52.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:52.757963+0000) 2023-08-27T22:08:53.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3501561964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:53.836 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3501561964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:53.836 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3501561964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:54.192 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:53.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:53.758224+0000) 2023-08-27T22:08:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:53.836Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:08:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:08:53.837Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:08:54.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:54 smithi136 ceph-mon[113531]: pgmap v2261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:54.759 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:54.759 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:08:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:54 smithi093 ceph-mon[111335]: pgmap v2261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:08:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:54 smithi093 ceph-mon[118902]: pgmap v2261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:08:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:08:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:54.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:54.758449+0000) 2023-08-27T22:08:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3395837590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3395837590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3395837590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:08:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:08:55] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:08:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:55.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:55.758677+0000) 2023-08-27T22:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:56 smithi136 ceph-mon[113531]: pgmap v2262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:56 smithi093 ceph-mon[111335]: pgmap v2262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:56 smithi093 ceph-mon[118902]: pgmap v2262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:56.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:56.758979+0000) 2023-08-27T22:08:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:57.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:57.759268+0000) 2023-08-27T22:08:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2518844309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:08:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:08:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-mon[113531]: pgmap v2263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2518844309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:08:57 smithi093 ceph-mon[111335]: pgmap v2263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2518844309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:08:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:08:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:08:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:08:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:08:58 smithi093 ceph-mon[118902]: pgmap v2263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:08:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:58.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:58.759521+0000) 2023-08-27T22:09:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:08:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:08:59.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:08:59.759805+0000) 2023-08-27T22:09:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:00 smithi136 ceph-mon[113531]: pgmap v2264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/491566334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:00 smithi093 ceph-mon[111335]: pgmap v2264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/491566334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:00 smithi093 ceph-mon[118902]: pgmap v2264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/491566334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:00.405Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:00.406Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:00.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:00.760102+0000) 2023-08-27T22:09:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:01.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:01.760378+0000) 2023-08-27T22:09:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:01.925Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:01.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:02 smithi093 ceph-mon[111335]: pgmap v2265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/616734526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:02 smithi093 ceph-mon[118902]: pgmap v2265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/616734526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:02.760 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:02 smithi136 ceph-mon[113531]: pgmap v2265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:02.760 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/616734526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:02.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:02.760656+0000) 2023-08-27T22:09:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:03.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:03.760935+0000) 2023-08-27T22:09:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:03.837Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:03.838Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:04.761 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:04 smithi136 ceph-mon[113531]: pgmap v2266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:04.761 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3397916761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:04 smithi093 ceph-mon[111335]: pgmap v2266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3397916761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:04 smithi093 ceph-mon[118902]: pgmap v2266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3397916761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:04.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:04.761195+0000) 2023-08-27T22:09:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:09:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:09:05] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:09:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:05.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:05.761416+0000) 2023-08-27T22:09:06.761 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:06 smithi136 ceph-mon[113531]: pgmap v2267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:06 smithi093 ceph-mon[111335]: pgmap v2267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:06 smithi093 ceph-mon[118902]: pgmap v2267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:06.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:06.761683+0000) 2023-08-27T22:09:07.762 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/677301129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/677301129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/677301129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:07.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:07.761976+0000) 2023-08-27T22:09:08.762 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:08 smithi136 ceph-mon[113531]: pgmap v2268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:08 smithi093 ceph-mon[111335]: pgmap v2268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:08 smithi093 ceph-mon[118902]: pgmap v2268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:08.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:08.762262+0000) 2023-08-27T22:09:09.762 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:09.762 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/898750995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/898750995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/898750995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:09.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:09.762547+0000) 2023-08-27T22:09:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:10 smithi093 ceph-mon[111335]: pgmap v2269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:10 smithi093 ceph-mon[118902]: pgmap v2269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:10.405Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:10.407Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:10.762 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:10 smithi136 ceph-mon[113531]: pgmap v2269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:10.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:10.762838+0000) 2023-08-27T22:09:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:11.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:11.763130+0000) 2023-08-27T22:09:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:11.926Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:11.927Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:12.763 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:12 smithi136 ceph-mon[113531]: pgmap v2270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:12.763 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2451539473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:12 smithi093 ceph-mon[111335]: pgmap v2270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2451539473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:12 smithi093 ceph-mon[118902]: pgmap v2270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2451539473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:12.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:12.763376+0000) 2023-08-27T22:09:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:13.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:13.763619+0000) 2023-08-27T22:09:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:13.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:13.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:14.764 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:14 smithi136 ceph-mon[113531]: pgmap v2271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:14.764 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1605076222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:14 smithi093 ceph-mon[111335]: pgmap v2271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1605076222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:14 smithi093 ceph-mon[118902]: pgmap v2271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1605076222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:14.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:14.763951+0000) 2023-08-27T22:09:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:09:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:09:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:09:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:15.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:15.764208+0000) 2023-08-27T22:09:16.764 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:16 smithi136 ceph-mon[113531]: pgmap v2272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:16.764 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/7884684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:16 smithi093 ceph-mon[111335]: pgmap v2272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/7884684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:16 smithi093 ceph-mon[118902]: pgmap v2272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/7884684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:16.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:16.764381+0000) 2023-08-27T22:09:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:17.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:17.764642+0000) 2023-08-27T22:09:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:18 smithi136 ceph-mon[113531]: pgmap v2273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:18.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:18.764912+0000) 2023-08-27T22:09:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:18 smithi093 ceph-mon[111335]: pgmap v2273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:18 smithi093 ceph-mon[118902]: pgmap v2273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3053331588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:19.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:19.765188+0000) 2023-08-27T22:09:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3053331588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3053331588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:20 smithi093 ceph-mon[111335]: pgmap v2274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:20 smithi093 ceph-mon[118902]: pgmap v2274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:20.406Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:20.407Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:20 smithi136 ceph-mon[113531]: pgmap v2274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:20.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:20.765409+0000) 2023-08-27T22:09:21.927 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2073563410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:21.927 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2073563410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2073563410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:21.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:21.765604+0000) 2023-08-27T22:09:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:21.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:21.928Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:22 smithi136 ceph-mon[113531]: pgmap v2275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:22.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:22.765814+0000) 2023-08-27T22:09:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:22 smithi093 ceph-mon[111335]: pgmap v2275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:22 smithi093 ceph-mon[118902]: pgmap v2275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:23.838 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2441107905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:23.839 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2441107905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2441107905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:23.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:23.766098+0000) 2023-08-27T22:09:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:23.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:23.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:24 smithi136 ceph-mon[113531]: pgmap v2276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:24.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:24.766382+0000) 2023-08-27T22:09:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:24 smithi093 ceph-mon[111335]: pgmap v2276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:24 smithi093 ceph-mon[118902]: pgmap v2276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:09:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:09:25] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:09:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:25.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:25.766663+0000) 2023-08-27T22:09:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:26 smithi136 ceph-mon[113531]: pgmap v2277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1897096574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:26.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:26.766928+0000) 2023-08-27T22:09:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:26 smithi093 ceph-mon[111335]: pgmap v2277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1897096574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:26 smithi093 ceph-mon[118902]: pgmap v2277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1897096574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:27.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:27.767125+0000) 2023-08-27T22:09:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:28 smithi136 ceph-mon[113531]: pgmap v2278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3038322707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:28.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:28.767376+0000) 2023-08-27T22:09:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:28 smithi093 ceph-mon[111335]: pgmap v2278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3038322707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:28 smithi093 ceph-mon[118902]: pgmap v2278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3038322707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:29.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:29.767650+0000) 2023-08-27T22:09:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:30 smithi136 ceph-mon[113531]: pgmap v2279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:30 smithi093 ceph-mon[111335]: pgmap v2279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:30 smithi093 ceph-mon[118902]: pgmap v2279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:30.406Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:30.407Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:30.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:30.767845+0000) 2023-08-27T22:09:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2093701329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2093701329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2093701329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:31.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:31.768104+0000) 2023-08-27T22:09:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:31.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:31.928Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:32 smithi136 ceph-mon[113531]: pgmap v2280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:32 smithi093 ceph-mon[111335]: pgmap v2280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:32 smithi093 ceph-mon[118902]: pgmap v2280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:32.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:32.768310+0000) 2023-08-27T22:09:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4238524381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4238524381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4238524381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:33.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:33.768522+0000) 2023-08-27T22:09:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:33.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:33.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:34 smithi136 ceph-mon[113531]: pgmap v2281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:34 smithi093 ceph-mon[111335]: pgmap v2281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:34 smithi093 ceph-mon[118902]: pgmap v2281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:34.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:34.768735+0000) 2023-08-27T22:09:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/72453100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/72453100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:09:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:09:35] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:09:35.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/72453100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:35.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:35.769010+0000) 2023-08-27T22:09:36.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:36 smithi136 ceph-mon[113531]: pgmap v2282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:36 smithi093 ceph-mon[111335]: pgmap v2282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:36 smithi093 ceph-mon[118902]: pgmap v2282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:36.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:36.769190+0000) 2023-08-27T22:09:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:37.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:37.769348+0000) 2023-08-27T22:09:38.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:38 smithi136 ceph-mon[113531]: pgmap v2283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:38.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1913111266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:38 smithi093 ceph-mon[111335]: pgmap v2283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1913111266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:38 smithi093 ceph-mon[118902]: pgmap v2283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1913111266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:38.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:38.769496+0000) 2023-08-27T22:09:39.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:39.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:39.769648+0000) 2023-08-27T22:09:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:40 smithi093 ceph-mon[111335]: pgmap v2284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2905541556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:40 smithi093 ceph-mon[118902]: pgmap v2284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2905541556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:40.407Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:40.408Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:40.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:40 smithi136 ceph-mon[113531]: pgmap v2284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:40.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2905541556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:40.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:40.769833+0000) 2023-08-27T22:09:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:41.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:41.769971+0000) 2023-08-27T22:09:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:41.928Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:41.929Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:42.770 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:42 smithi136 ceph-mon[113531]: pgmap v2285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:42 smithi093 ceph-mon[111335]: pgmap v2285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:42 smithi093 ceph-mon[118902]: pgmap v2285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:42.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:42.770134+0000) 2023-08-27T22:09:43.770 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2606806181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:43.839 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2606806181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:43.840 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2606806181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:43.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:43.770350+0000) 2023-08-27T22:09:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:43.839Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:43.840Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:44.770 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:44 smithi136 ceph-mon[113531]: pgmap v2286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:44 smithi093 ceph-mon[111335]: pgmap v2286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:44 smithi093 ceph-mon[118902]: pgmap v2286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:44.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:44.770554+0000) 2023-08-27T22:09:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/196418117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:09:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:09:45] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:09:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/196418117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/196418117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:45.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:45.770726+0000) 2023-08-27T22:09:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:46 smithi136 ceph-mon[113531]: pgmap v2287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:46.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:46.770873+0000) 2023-08-27T22:09:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:46 smithi093 ceph-mon[111335]: pgmap v2287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:46 smithi093 ceph-mon[118902]: pgmap v2287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1743330403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:47.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:47.771020+0000) 2023-08-27T22:09:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1743330403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1743330403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:48 smithi136 ceph-mon[113531]: pgmap v2288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:48.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:48.771223+0000) 2023-08-27T22:09:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:48 smithi093 ceph-mon[111335]: pgmap v2288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:48 smithi093 ceph-mon[118902]: pgmap v2288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:49.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:49.771394+0000) 2023-08-27T22:09:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:50 smithi136 ceph-mon[113531]: pgmap v2289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/86685734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:50 smithi093 ceph-mon[111335]: pgmap v2289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/86685734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:50 smithi093 ceph-mon[118902]: pgmap v2289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/86685734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:50.407Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:50.408Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:50.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:50.771587+0000) 2023-08-27T22:09:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:51.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:51.771760+0000) 2023-08-27T22:09:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:51.929Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:51.930Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:52.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:52 smithi136 ceph-mon[113531]: pgmap v2290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:52.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3646139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:52 smithi093 ceph-mon[118902]: pgmap v2290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3646139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:52 smithi093 ceph-mon[111335]: pgmap v2290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3646139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:52.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:52.771932+0000) 2023-08-27T22:09:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:53.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:53.772092+0000) 2023-08-27T22:09:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:53.839Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:09:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:09:53.840Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:09:54.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:54 smithi136 ceph-mon[113531]: pgmap v2291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:54.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:54.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1415551618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:54 smithi093 ceph-mon[111335]: pgmap v2291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1415551618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:54 smithi093 ceph-mon[118902]: pgmap v2291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:09:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1415551618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:54.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:54.772314+0000) 2023-08-27T22:09:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:09:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:09:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:09:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:55.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:55.772500+0000) 2023-08-27T22:09:56.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:56 smithi136 ceph-mon[113531]: pgmap v2292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:56 smithi093 ceph-mon[111335]: pgmap v2292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:56 smithi093 ceph-mon[118902]: pgmap v2292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:56.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:56.772720+0000) 2023-08-27T22:09:57.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3286248200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3286248200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3286248200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:58.185 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:57.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:57.772929+0000) 2023-08-27T22:09:58.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:58 smithi136 ceph-mon[113531]: pgmap v2293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:58.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:58 smithi093 ceph-mon[111335]: pgmap v2293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:09:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:58 smithi093 ceph-mon[118902]: pgmap v2293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:09:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:09:59.041 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:58.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:58.773072+0000) 2023-08-27T22:09:59.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1037212376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:59.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:09:59.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:09:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1037212376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:09:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:09:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1037212376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:09:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:09:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:09:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:09:59.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:09:59.773209+0000) 2023-08-27T22:10:00.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:00.408Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:00.409Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[111335]: pgmap v2294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:10:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[118902]: pgmap v2294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:10:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-mon[113531]: pgmap v2294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:00.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:10:00.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:00.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:10:00.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:10:00.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:10:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:00.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:00.773349+0000) 2023-08-27T22:10:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:01.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:01.773514+0000) 2023-08-27T22:10:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:01.930Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:01.931Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:02.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:02 smithi136 ceph-mon[113531]: pgmap v2295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:02.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1412201913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:02 smithi093 ceph-mon[111335]: pgmap v2295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1412201913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:02 smithi093 ceph-mon[118902]: pgmap v2295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1412201913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:02.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:02.773675+0000) 2023-08-27T22:10:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:03.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:03.773852+0000) 2023-08-27T22:10:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:03.840Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:03.841Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:04.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:04 smithi136 ceph-mon[113531]: pgmap v2296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:04.774 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2221154075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:04 smithi093 ceph-mon[111335]: pgmap v2296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2221154075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:04 smithi093 ceph-mon[118902]: pgmap v2296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2221154075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:04.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:04.774071+0000) 2023-08-27T22:10:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:10:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:10:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:10:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:05.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:05.774234+0000) 2023-08-27T22:10:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:06 smithi136 ceph-mon[113531]: pgmap v2297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/64909828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:06.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:06.774436+0000) 2023-08-27T22:10:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:06 smithi093 ceph-mon[111335]: pgmap v2297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/64909828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:06 smithi093 ceph-mon[118902]: pgmap v2297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/64909828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:07.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:07.774700+0000) 2023-08-27T22:10:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:08.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:08.774936+0000) 2023-08-27T22:10:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:08 smithi136 ceph-mon[113531]: pgmap v2298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:08 smithi093 ceph-mon[111335]: pgmap v2298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:08 smithi093 ceph-mon[118902]: pgmap v2298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2145872863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:09.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:09.775144+0000) 2023-08-27T22:10:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2145872863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2145872863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:10 smithi093 ceph-mon[111335]: pgmap v2299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:10 smithi093 ceph-mon[118902]: pgmap v2299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:10.409Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:10.410Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:10 smithi136 ceph-mon[113531]: pgmap v2299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:10.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:10.775365+0000) 2023-08-27T22:10:11.931 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3058940522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:11.931 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3058940522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3058940522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:11.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:11.775539+0000) 2023-08-27T22:10:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:11.931Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:11.932Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:12 smithi136 ceph-mon[113531]: pgmap v2300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:12.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:12.775724+0000) 2023-08-27T22:10:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:12 smithi093 ceph-mon[111335]: pgmap v2300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:12 smithi093 ceph-mon[118902]: pgmap v2300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:13.841 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3390580268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:13.842 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3390580268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3390580268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:13.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:13.775934+0000) 2023-08-27T22:10:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:13.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:13.842Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:14 smithi136 ceph-mon[113531]: pgmap v2301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:14.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:14.776136+0000) 2023-08-27T22:10:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:14 smithi093 ceph-mon[111335]: pgmap v2301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:14 smithi093 ceph-mon[118902]: pgmap v2301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:10:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:10:15] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:10:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:15.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:15.776347+0000) 2023-08-27T22:10:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:16 smithi136 ceph-mon[113531]: pgmap v2302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2364655234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:16.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:16.776633+0000) 2023-08-27T22:10:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:16 smithi093 ceph-mon[111335]: pgmap v2302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2364655234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:16 smithi093 ceph-mon[118902]: pgmap v2302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2364655234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:17.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:17.776825+0000) 2023-08-27T22:10:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:18 smithi136 ceph-mon[113531]: pgmap v2303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2123246961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:18.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:18.777102+0000) 2023-08-27T22:10:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:18 smithi093 ceph-mon[111335]: pgmap v2303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2123246961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:18 smithi093 ceph-mon[118902]: pgmap v2303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2123246961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:19.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:19.777262+0000) 2023-08-27T22:10:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:20 smithi136 ceph-mon[113531]: pgmap v2304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:20 smithi093 ceph-mon[111335]: pgmap v2304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:20 smithi093 ceph-mon[118902]: pgmap v2304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:20.410Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:20.410Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:20.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:20.777478+0000) 2023-08-27T22:10:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2523673567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2523673567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:21.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2523673567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:21.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:21.777743+0000) 2023-08-27T22:10:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:21.932Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:21.933Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:22 smithi136 ceph-mon[113531]: pgmap v2305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:22 smithi093 ceph-mon[111335]: pgmap v2305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:22 smithi093 ceph-mon[118902]: pgmap v2305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:22.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:22.777944+0000) 2023-08-27T22:10:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4135906947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4135906947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4135906947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:23.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:23.778170+0000) 2023-08-27T22:10:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:23.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:23.842Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:24 smithi136 ceph-mon[113531]: pgmap v2306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:24 smithi093 ceph-mon[111335]: pgmap v2306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:24 smithi093 ceph-mon[118902]: pgmap v2306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:24.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:24.778397+0000) 2023-08-27T22:10:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1789187035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1789187035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:10:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:10:25] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:10:25.778 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1789187035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:26.194 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:25.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:25.778647+0000) 2023-08-27T22:10:26.778 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:26 smithi136 ceph-mon[113531]: pgmap v2307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:26 smithi093 ceph-mon[111335]: pgmap v2307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:26.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:26 smithi093 ceph-mon[118902]: pgmap v2307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:26.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:26.778878+0000) 2023-08-27T22:10:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:27.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:27.779085+0000) 2023-08-27T22:10:28.779 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:28 smithi136 ceph-mon[113531]: pgmap v2308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:28.779 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2596118501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:28 smithi093 ceph-mon[111335]: pgmap v2308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2596118501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:28 smithi093 ceph-mon[118902]: pgmap v2308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2596118501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:28.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:28.779311+0000) 2023-08-27T22:10:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:29.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:29.779592+0000) 2023-08-27T22:10:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:30 smithi136 ceph-mon[113531]: pgmap v2309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/76409721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:30 smithi093 ceph-mon[111335]: pgmap v2309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/76409721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:30 smithi093 ceph-mon[118902]: pgmap v2309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/76409721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:30.410Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:30.411Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:30.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:30.779890+0000) 2023-08-27T22:10:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:31.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:31.780186+0000) 2023-08-27T22:10:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:31.933Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:31.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:32.780 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:32 smithi136 ceph-mon[113531]: pgmap v2310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:32 smithi093 ceph-mon[111335]: pgmap v2310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:32 smithi093 ceph-mon[118902]: pgmap v2310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:32.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:32.780435+0000) 2023-08-27T22:10:33.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2149959659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:33.843 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2149959659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:33.843 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2149959659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:33.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:33.780702+0000) 2023-08-27T22:10:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:33.842Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:33.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:34.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:34 smithi136 ceph-mon[113531]: pgmap v2311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:34 smithi093 ceph-mon[111335]: pgmap v2311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:34 smithi093 ceph-mon[118902]: pgmap v2311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:34.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:34.780978+0000) 2023-08-27T22:10:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3558946588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3558946588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:10:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:10:35] "GET /metrics HTTP/1.1" 200 239888 "" "Prometheus/2.43.0" 2023-08-27T22:10:35.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3558946588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:35.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:35.781242+0000) 2023-08-27T22:10:36.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:36 smithi136 ceph-mon[113531]: pgmap v2312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:36 smithi093 ceph-mon[111335]: pgmap v2312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:36 smithi093 ceph-mon[118902]: pgmap v2312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:36.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:36.781507+0000) 2023-08-27T22:10:37.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/203714670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/203714670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/203714670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:37.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:37.781730+0000) 2023-08-27T22:10:38.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:38 smithi136 ceph-mon[113531]: pgmap v2313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:38 smithi093 ceph-mon[111335]: pgmap v2313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:38 smithi093 ceph-mon[118902]: pgmap v2313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:38.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:38.782002+0000) 2023-08-27T22:10:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:39.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:39.782274+0000) 2023-08-27T22:10:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:40 smithi093 ceph-mon[111335]: pgmap v2314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2855139366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:40 smithi093 ceph-mon[118902]: pgmap v2314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2855139366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:40.410Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:40.411Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:40 smithi136 ceph-mon[113531]: pgmap v2314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2855139366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:40.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:40.782519+0000) 2023-08-27T22:10:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:41.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:41.782810+0000) 2023-08-27T22:10:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:41.933Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:41.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:42 smithi136 ceph-mon[113531]: pgmap v2315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/116391216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:42.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:42.783080+0000) 2023-08-27T22:10:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:42 smithi093 ceph-mon[111335]: pgmap v2315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/116391216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:42 smithi093 ceph-mon[118902]: pgmap v2315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/116391216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:43.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:43.783350+0000) 2023-08-27T22:10:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:43.843Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:43.844Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:44 smithi136 ceph-mon[113531]: pgmap v2316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2065321154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:44.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:44.783622+0000) 2023-08-27T22:10:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:44 smithi093 ceph-mon[118902]: pgmap v2316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2065321154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:44 smithi093 ceph-mon[111335]: pgmap v2316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2065321154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:10:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:10:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:10:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:45.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:45.783897+0000) 2023-08-27T22:10:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:46 smithi136 ceph-mon[113531]: pgmap v2317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:46.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:46.784186+0000) 2023-08-27T22:10:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:46 smithi093 ceph-mon[111335]: pgmap v2317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:46 smithi093 ceph-mon[118902]: pgmap v2317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/562405565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:47.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:47.784495+0000) 2023-08-27T22:10:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/562405565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/562405565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:48 smithi136 ceph-mon[113531]: pgmap v2318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:48.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:48.784802+0000) 2023-08-27T22:10:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:48 smithi093 ceph-mon[111335]: pgmap v2318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:48 smithi093 ceph-mon[118902]: pgmap v2318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:49.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:49.785101+0000) 2023-08-27T22:10:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2392059876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:50 smithi136 ceph-mon[113531]: pgmap v2319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2392059876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:50 smithi093 ceph-mon[111335]: pgmap v2319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2392059876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:50 smithi093 ceph-mon[118902]: pgmap v2319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:50.411Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:50.412Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:50.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:50.785366+0000) 2023-08-27T22:10:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:51.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:51.785633+0000) 2023-08-27T22:10:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:51.933Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:51.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:52.786 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:52 smithi136 ceph-mon[113531]: pgmap v2320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:52.786 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3111177008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:52 smithi093 ceph-mon[111335]: pgmap v2320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3111177008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:52 smithi093 ceph-mon[118902]: pgmap v2320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3111177008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:52.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:52.785862+0000) 2023-08-27T22:10:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:53.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:53.786123+0000) 2023-08-27T22:10:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:53.843Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:10:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:10:53.844Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:10:54.786 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:54 smithi136 ceph-mon[113531]: pgmap v2321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:54.786 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:54.786 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3283635561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:54 smithi093 ceph-mon[111335]: pgmap v2321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3283635561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:54 smithi093 ceph-mon[118902]: pgmap v2321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:10:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3283635561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:54.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:54.786362+0000) 2023-08-27T22:10:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:10:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:10:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:10:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:55.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:55.786557+0000) 2023-08-27T22:10:56.786 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:56 smithi136 ceph-mon[113531]: pgmap v2322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:56.787 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2858221849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:56 smithi093 ceph-mon[111335]: pgmap v2322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2858221849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:56 smithi093 ceph-mon[118902]: pgmap v2322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2858221849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:56.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:56.786791+0000) 2023-08-27T22:10:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:57.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:57.787067+0000) 2023-08-27T22:10:58.787 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:58 smithi136 ceph-mon[113531]: pgmap v2323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:58 smithi093 ceph-mon[111335]: pgmap v2323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:58 smithi093 ceph-mon[118902]: pgmap v2323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:10:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:58.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:58.787326+0000) 2023-08-27T22:10:59.787 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:10:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2699190231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:10:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2699190231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:10:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:10:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2699190231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:10:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:10:59.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:10:59.787611+0000) 2023-08-27T22:11:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:00 smithi093 ceph-mon[111335]: pgmap v2324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:00 smithi093 ceph-mon[118902]: pgmap v2324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:00.411Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:00.412Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:00.787 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:00 smithi136 ceph-mon[113531]: pgmap v2324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:00.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:00.787815+0000) 2023-08-27T22:11:01.788 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:11:01.788 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:01.788 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:01.789 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1531947005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:01.934 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:11:01.935 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:01.935 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:01.935 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1531947005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:01.935 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:11:01.935 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:01.935 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:01.935 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1531947005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:02.123 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:01.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:01.788001+0000) 2023-08-27T22:11:02.216 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:01.934Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:02.216 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:01.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:02.788 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:02 smithi136 ceph-mon[113531]: pgmap v2325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:02 smithi093 ceph-mon[111335]: pgmap v2325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:02 smithi093 ceph-mon[118902]: pgmap v2325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:02.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:02.788185+0000) 2023-08-27T22:11:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:03.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:03.788376+0000) 2023-08-27T22:11:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4043976880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:11:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:11:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:03.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:03.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4043976880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:04.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:11:04.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:11:04.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:04.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4043976880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:04.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:11:04.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:11:04.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:11:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:04 smithi136 ceph-mon[113531]: pgmap v2326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:04.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:04.788658+0000) 2023-08-27T22:11:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:04 smithi093 ceph-mon[111335]: pgmap v2326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:04 smithi093 ceph-mon[118902]: pgmap v2326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 22:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T22:11:05.31842587Z level=info msg="Completed cleanup jobs" duration=13.070477ms 2023-08-27T22:11:05.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:11:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:11:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:11:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:05.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:05.788922+0000) 2023-08-27T22:11:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:06 smithi136 ceph-mon[113531]: pgmap v2327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/340765744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:06.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:06.789160+0000) 2023-08-27T22:11:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:06 smithi093 ceph-mon[111335]: pgmap v2327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/340765744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:06 smithi093 ceph-mon[118902]: pgmap v2327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/340765744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:07.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:07.789392+0000) 2023-08-27T22:11:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:08 smithi136 ceph-mon[113531]: pgmap v2328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3511486833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:08.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:08.789653+0000) 2023-08-27T22:11:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:08 smithi093 ceph-mon[111335]: pgmap v2328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3511486833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:08 smithi093 ceph-mon[118902]: pgmap v2328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3511486833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:09.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:09.789929+0000) 2023-08-27T22:11:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:10 smithi136 ceph-mon[113531]: pgmap v2329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:10 smithi093 ceph-mon[111335]: pgmap v2329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:10 smithi093 ceph-mon[118902]: pgmap v2329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:10.412Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:10.413Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:10.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:10.790174+0000) 2023-08-27T22:11:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/742330490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/742330490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/742330490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:11.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:11.790388+0000) 2023-08-27T22:11:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:11.934Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:11.935Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:12 smithi136 ceph-mon[113531]: pgmap v2330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:12 smithi093 ceph-mon[111335]: pgmap v2330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:12 smithi093 ceph-mon[118902]: pgmap v2330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:12.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:12.790597+0000) 2023-08-27T22:11:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2251599336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2251599336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2251599336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:13.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:13.790860+0000) 2023-08-27T22:11:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:13.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:13.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:14 smithi136 ceph-mon[113531]: pgmap v2331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:14 smithi093 ceph-mon[111335]: pgmap v2331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:14 smithi093 ceph-mon[118902]: pgmap v2331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:14.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:14.791118+0000) 2023-08-27T22:11:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/500247621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/500247621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:11:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:11:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:11:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/500247621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:15.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:15.791336+0000) 2023-08-27T22:11:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:16 smithi136 ceph-mon[113531]: pgmap v2332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:16.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:16.791589+0000) 2023-08-27T22:11:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:16 smithi093 ceph-mon[111335]: pgmap v2332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:16 smithi093 ceph-mon[118902]: pgmap v2332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:17.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:17.791808+0000) 2023-08-27T22:11:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:18 smithi136 ceph-mon[113531]: pgmap v2333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3026024371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:18.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:18.792097+0000) 2023-08-27T22:11:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:18 smithi093 ceph-mon[111335]: pgmap v2333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3026024371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:18 smithi093 ceph-mon[118902]: pgmap v2333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3026024371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:19.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:19.792321+0000) 2023-08-27T22:11:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:20 smithi136 ceph-mon[113531]: pgmap v2334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3760038976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:20 smithi093 ceph-mon[111335]: pgmap v2334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3760038976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:20 smithi093 ceph-mon[118902]: pgmap v2334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3760038976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:20.413Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:20.414Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:20.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:20.792523+0000) 2023-08-27T22:11:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:21.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:21.792764+0000) 2023-08-27T22:11:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:21.935Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:21.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:22.793 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:22 smithi136 ceph-mon[113531]: pgmap v2335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:22 smithi093 ceph-mon[111335]: pgmap v2335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:22 smithi093 ceph-mon[118902]: pgmap v2335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:22.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:22.793059+0000) 2023-08-27T22:11:23.793 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4222013520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:23.846 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4222013520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:23.847 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4222013520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:23.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:23.793267+0000) 2023-08-27T22:11:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:23.846Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:23.847Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:24.793 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:24 smithi136 ceph-mon[113531]: pgmap v2336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:24.793 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:24 smithi093 ceph-mon[111335]: pgmap v2336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:24 smithi093 ceph-mon[118902]: pgmap v2336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:24.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:24.793510+0000) 2023-08-27T22:11:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1109684281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:11:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:11:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:11:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1109684281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:25.793 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1109684281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:25.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:25.793763+0000) 2023-08-27T22:11:26.794 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:26 smithi136 ceph-mon[113531]: pgmap v2337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:26 smithi093 ceph-mon[111335]: pgmap v2337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:26 smithi093 ceph-mon[118902]: pgmap v2337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:26.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:26.794005+0000) 2023-08-27T22:11:27.794 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3084745536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3084745536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3084745536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:27.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:27.794294+0000) 2023-08-27T22:11:28.794 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:28 smithi136 ceph-mon[113531]: pgmap v2338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:28 smithi093 ceph-mon[111335]: pgmap v2338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:28 smithi093 ceph-mon[118902]: pgmap v2338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:28.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:28.794554+0000) 2023-08-27T22:11:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:29.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:29.794807+0000) 2023-08-27T22:11:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:30 smithi136 ceph-mon[113531]: pgmap v2339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3164429435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:30 smithi093 ceph-mon[111335]: pgmap v2339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3164429435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:30 smithi093 ceph-mon[118902]: pgmap v2339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3164429435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:30.413Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:30.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:30.414Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:30.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:30.795027+0000) 2023-08-27T22:11:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:31.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:31.795210+0000) 2023-08-27T22:11:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:31.936Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:31.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:32.795 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:32 smithi136 ceph-mon[113531]: pgmap v2340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:32.795 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2910068473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:32 smithi093 ceph-mon[111335]: pgmap v2340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2910068473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:32 smithi093 ceph-mon[118902]: pgmap v2340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2910068473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:32.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:32.795436+0000) 2023-08-27T22:11:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:33.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:33.795611+0000) 2023-08-27T22:11:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:33.847Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:33.848Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:34.796 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:34 smithi136 ceph-mon[113531]: pgmap v2341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:34 smithi093 ceph-mon[111335]: pgmap v2341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:34 smithi093 ceph-mon[118902]: pgmap v2341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:34.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:34.795872+0000) 2023-08-27T22:11:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4220804762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4220804762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:11:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:11:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:11:35.798 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4220804762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:35.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:35.796132+0000) 2023-08-27T22:11:36.796 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:36 smithi136 ceph-mon[113531]: pgmap v2342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:36 smithi093 ceph-mon[111335]: pgmap v2342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:36 smithi093 ceph-mon[118902]: pgmap v2342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:36.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:36.796373+0000) 2023-08-27T22:11:37.796 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/907904011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/907904011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/907904011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:37.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:37.796595+0000) 2023-08-27T22:11:38.796 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:38 smithi136 ceph-mon[113531]: pgmap v2343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:38 smithi093 ceph-mon[111335]: pgmap v2343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:38 smithi093 ceph-mon[118902]: pgmap v2343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:38.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:38.796840+0000) 2023-08-27T22:11:39.797 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:39.797 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2129203166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2129203166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2129203166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:39.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:39.797108+0000) 2023-08-27T22:11:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:40 smithi093 ceph-mon[111335]: pgmap v2344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:40 smithi093 ceph-mon[118902]: pgmap v2344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:40.414Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:40.415Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:40.798 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:40 smithi136 ceph-mon[113531]: pgmap v2344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:40.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:40.797278+0000) 2023-08-27T22:11:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:41.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:41.797481+0000) 2023-08-27T22:11:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:41.936Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:41.937Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:42 smithi136 ceph-mon[113531]: pgmap v2345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/412809882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:42.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:42.797782+0000) 2023-08-27T22:11:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:42 smithi093 ceph-mon[111335]: pgmap v2345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/412809882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:42 smithi093 ceph-mon[118902]: pgmap v2345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/412809882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:43.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:43.798002+0000) 2023-08-27T22:11:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:43.848Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:43.849Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:44 smithi136 ceph-mon[113531]: pgmap v2346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1138577508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:44.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:44.798279+0000) 2023-08-27T22:11:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:44 smithi093 ceph-mon[111335]: pgmap v2346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1138577508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:44 smithi093 ceph-mon[118902]: pgmap v2346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1138577508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:11:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:11:45] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T22:11:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:45.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:45.798543+0000) 2023-08-27T22:11:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:46 smithi136 ceph-mon[113531]: pgmap v2347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/769777335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:46.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:46.798775+0000) 2023-08-27T22:11:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:46 smithi093 ceph-mon[111335]: pgmap v2347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/769777335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:46 smithi093 ceph-mon[118902]: pgmap v2347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/769777335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:47.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:47.799066+0000) 2023-08-27T22:11:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:48 smithi136 ceph-mon[113531]: pgmap v2348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:48.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:48.799292+0000) 2023-08-27T22:11:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:48 smithi093 ceph-mon[118902]: pgmap v2348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:48 smithi093 ceph-mon[111335]: pgmap v2348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3545651581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:49.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:49.799513+0000) 2023-08-27T22:11:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3545651581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3545651581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:50 smithi093 ceph-mon[111335]: pgmap v2349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:50 smithi093 ceph-mon[118902]: pgmap v2349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:50.415Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:50.416Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:50 smithi136 ceph-mon[113531]: pgmap v2349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:50.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:50.799787+0000) 2023-08-27T22:11:51.938 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1551494612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:51.938 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1551494612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1551494612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:51.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:51.800024+0000) 2023-08-27T22:11:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:51.938Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:51.940Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:52 smithi136 ceph-mon[113531]: pgmap v2350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:52.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:52.800276+0000) 2023-08-27T22:11:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:52 smithi093 ceph-mon[111335]: pgmap v2350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:52 smithi093 ceph-mon[118902]: pgmap v2350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:53.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:53.800543+0000) 2023-08-27T22:11:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:53.848Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:11:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:11:53.849Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:11:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:54 smithi136 ceph-mon[113531]: pgmap v2351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1776491414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:54.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:54.800760+0000) 2023-08-27T22:11:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:54 smithi093 ceph-mon[111335]: pgmap v2351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1776491414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:54 smithi093 ceph-mon[118902]: pgmap v2351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1776491414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:11:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:11:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:11:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:11:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:55.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:55.801056+0000) 2023-08-27T22:11:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:56 smithi136 ceph-mon[113531]: pgmap v2352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4260577724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:56.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:56.801309+0000) 2023-08-27T22:11:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:56 smithi093 ceph-mon[111335]: pgmap v2352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4260577724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:56 smithi093 ceph-mon[118902]: pgmap v2352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4260577724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:57.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:57.801599+0000) 2023-08-27T22:11:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:58 smithi136 ceph-mon[113531]: pgmap v2353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:11:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2220967567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:58.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:58.801849+0000) 2023-08-27T22:11:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:58 smithi093 ceph-mon[111335]: pgmap v2353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:11:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2220967567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:11:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:58 smithi093 ceph-mon[118902]: pgmap v2353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:11:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:11:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2220967567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:11:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:11:59.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:11:59.802092+0000) 2023-08-27T22:12:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:00 smithi136 ceph-mon[113531]: pgmap v2354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:00 smithi093 ceph-mon[111335]: pgmap v2354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:00 smithi093 ceph-mon[118902]: pgmap v2354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:00.415Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:00.417Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:00.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:00.802369+0000) 2023-08-27T22:12:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4291022773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:01.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4291022773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:01.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4291022773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:01.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:01.802672+0000) 2023-08-27T22:12:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:01.939Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:01.940Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:02 smithi136 ceph-mon[113531]: pgmap v2355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:02 smithi093 ceph-mon[111335]: pgmap v2355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:02.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:02 smithi093 ceph-mon[118902]: pgmap v2355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:02.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:02.802922+0000) 2023-08-27T22:12:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2232895716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2232895716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2232895716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:03.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:03.803135+0000) 2023-08-27T22:12:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:03.849Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:03.849Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:04.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:04 smithi136 ceph-mon[113531]: pgmap v2356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:04.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:12:04.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:04.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[118902]: pgmap v2356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:12:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[111335]: pgmap v2356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:12:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:04.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:04.803292+0000) 2023-08-27T22:12:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:12:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:12:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:12:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:05.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:05.803459+0000) 2023-08-27T22:12:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:06.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:06.803646+0000) 2023-08-27T22:12:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:06 smithi136 ceph-mon[113531]: pgmap v2357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/585965496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[111335]: pgmap v2357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/585965496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[118902]: pgmap v2357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/585965496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:12:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:12:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:07.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:07.803903+0000) 2023-08-27T22:12:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:12:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:12:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:12:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:12:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:12:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:08 smithi136 ceph-mon[113531]: pgmap v2358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1237570867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:08.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:08.804195+0000) 2023-08-27T22:12:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:08 smithi093 ceph-mon[111335]: pgmap v2358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1237570867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:08 smithi093 ceph-mon[118902]: pgmap v2358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1237570867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:09.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:09.804461+0000) 2023-08-27T22:12:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:10 smithi136 ceph-mon[113531]: pgmap v2359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:10 smithi093 ceph-mon[111335]: pgmap v2359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:10 smithi093 ceph-mon[118902]: pgmap v2359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:10.417Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:10.418Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:10.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:10.804712+0000) 2023-08-27T22:12:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1627345950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:12.315 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1627345950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:12.315 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:11.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:11.804988+0000) 2023-08-27T22:12:12.315 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1627345950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:12.316 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:11.940Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:12.316 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:11.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:12 smithi136 ceph-mon[113531]: pgmap v2360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:13.052 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:12 smithi093 ceph-mon[111335]: pgmap v2360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:13.052 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:12 smithi093 ceph-mon[118902]: pgmap v2360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:12.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:12.805266+0000) 2023-08-27T22:12:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/408096389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/408096389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/408096389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:13.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:13.805470+0000) 2023-08-27T22:12:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:13.849Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:13.850Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:14 smithi136 ceph-mon[113531]: pgmap v2361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:14 smithi093 ceph-mon[111335]: pgmap v2361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:14 smithi093 ceph-mon[118902]: pgmap v2361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:14.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:14.805736+0000) 2023-08-27T22:12:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3157037341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3157037341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3157037341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:12:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:12:15] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:12:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:15.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:15.806061+0000) 2023-08-27T22:12:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:16 smithi136 ceph-mon[113531]: pgmap v2362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:16 smithi093 ceph-mon[111335]: pgmap v2362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:16 smithi093 ceph-mon[118902]: pgmap v2362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:16.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:16.806312+0000) 2023-08-27T22:12:17.806 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4081048563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4081048563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4081048563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:17.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:17.806548+0000) 2023-08-27T22:12:18.806 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:18 smithi136 ceph-mon[113531]: pgmap v2363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:18 smithi093 ceph-mon[111335]: pgmap v2363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:18 smithi093 ceph-mon[118902]: pgmap v2363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:18.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:18.806784+0000) 2023-08-27T22:12:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:19.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:19.806996+0000) 2023-08-27T22:12:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:20 smithi136 ceph-mon[113531]: pgmap v2364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1477786167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:20 smithi093 ceph-mon[111335]: pgmap v2364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1477786167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:20 smithi093 ceph-mon[118902]: pgmap v2364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1477786167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:20.417Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:20.418Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:20.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:20.807250+0000) 2023-08-27T22:12:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:21.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:21.807530+0000) 2023-08-27T22:12:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:21.940Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:21.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:22.808 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:22 smithi136 ceph-mon[113531]: pgmap v2365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:22.808 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2983196513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:22 smithi093 ceph-mon[111335]: pgmap v2365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2983196513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:22 smithi093 ceph-mon[118902]: pgmap v2365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2983196513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:22.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:22.807863+0000) 2023-08-27T22:12:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:23.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:23.808181+0000) 2023-08-27T22:12:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:23.850Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:23.851Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:24.808 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:24 smithi136 ceph-mon[113531]: pgmap v2366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:24.808 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:24 smithi093 ceph-mon[111335]: pgmap v2366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:24 smithi093 ceph-mon[118902]: pgmap v2366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:24.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:24.808429+0000) 2023-08-27T22:12:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2369949688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2369949688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:12:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:12:25] "GET /metrics HTTP/1.1" 200 239876 "" "Prometheus/2.43.0" 2023-08-27T22:12:25.808 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2369949688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:25.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:25.808655+0000) 2023-08-27T22:12:26.809 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:26 smithi136 ceph-mon[113531]: pgmap v2367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:26 smithi093 ceph-mon[111335]: pgmap v2367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:26 smithi093 ceph-mon[118902]: pgmap v2367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:26.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:26.808927+0000) 2023-08-27T22:12:27.809 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4155802993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4155802993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4155802993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:27.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:27.809199+0000) 2023-08-27T22:12:28.809 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:28 smithi136 ceph-mon[113531]: pgmap v2368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:28 smithi093 ceph-mon[111335]: pgmap v2368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:28 smithi093 ceph-mon[118902]: pgmap v2368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:28.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:28.809427+0000) 2023-08-27T22:12:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:29.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:29.809686+0000) 2023-08-27T22:12:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/821581693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:30 smithi136 ceph-mon[113531]: pgmap v2369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/821581693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:30 smithi093 ceph-mon[111335]: pgmap v2369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/821581693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:30 smithi093 ceph-mon[118902]: pgmap v2369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:30.418Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:30.419Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:30.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:30.809944+0000) 2023-08-27T22:12:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:31.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:31.810205+0000) 2023-08-27T22:12:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:31.940Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:31.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:32.810 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:32 smithi136 ceph-mon[113531]: pgmap v2370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:32.810 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1872013887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:32 smithi093 ceph-mon[111335]: pgmap v2370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1872013887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:32 smithi093 ceph-mon[118902]: pgmap v2370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1872013887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:32.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:32.810440+0000) 2023-08-27T22:12:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:33.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:33.810674+0000) 2023-08-27T22:12:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:33.850Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:33.851Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:34.810 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:34 smithi136 ceph-mon[113531]: pgmap v2371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:34.811 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/120807981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:34 smithi093 ceph-mon[111335]: pgmap v2371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/120807981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:34 smithi093 ceph-mon[118902]: pgmap v2371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:34.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/120807981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:34.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:34.810912+0000) 2023-08-27T22:12:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:12:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:12:35] "GET /metrics HTTP/1.1" 200 239876 "" "Prometheus/2.43.0" 2023-08-27T22:12:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:35.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:35.811179+0000) 2023-08-27T22:12:36.811 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:36 smithi136 ceph-mon[113531]: pgmap v2372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:36 smithi093 ceph-mon[111335]: pgmap v2372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:36 smithi093 ceph-mon[118902]: pgmap v2372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:36.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:36.811370+0000) 2023-08-27T22:12:37.811 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1574871018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1574871018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1574871018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:37.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:37.811621+0000) 2023-08-27T22:12:38.812 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:38 smithi136 ceph-mon[113531]: pgmap v2373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:38 smithi093 ceph-mon[111335]: pgmap v2373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:38 smithi093 ceph-mon[118902]: pgmap v2373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:38.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:38.811913+0000) 2023-08-27T22:12:39.812 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:39.812 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3874625712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3874625712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3874625712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:39.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:39.812111+0000) 2023-08-27T22:12:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:40 smithi093 ceph-mon[118902]: pgmap v2374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:40 smithi093 ceph-mon[111335]: pgmap v2374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:40.419Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:40.420Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:40.812 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:40 smithi136 ceph-mon[113531]: pgmap v2374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:40.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:40.812374+0000) 2023-08-27T22:12:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3503952466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:41.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:41.812611+0000) 2023-08-27T22:12:41.941 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3503952466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:41.941 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3503952466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:41.941Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:41.942Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:42.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:42 smithi136 ceph-mon[113531]: pgmap v2375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:42.943 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:42.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:42.812888+0000) 2023-08-27T22:12:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:42 smithi093 ceph-mon[111335]: pgmap v2375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:42 smithi093 ceph-mon[118902]: pgmap v2375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:43.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:43.813153+0000) 2023-08-27T22:12:44.269 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:43.851Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:44.269 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:43.852Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:44 smithi136 ceph-mon[113531]: pgmap v2376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1873473279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:44.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:44.813415+0000) 2023-08-27T22:12:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:44 smithi093 ceph-mon[111335]: pgmap v2376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1873473279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:44 smithi093 ceph-mon[118902]: pgmap v2376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1873473279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:12:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:12:45] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:12:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:45.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:45.813717+0000) 2023-08-27T22:12:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:46 smithi136 ceph-mon[113531]: pgmap v2377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/308512529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:46.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:46.813929+0000) 2023-08-27T22:12:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:46 smithi093 ceph-mon[111335]: pgmap v2377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/308512529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:46 smithi093 ceph-mon[118902]: pgmap v2377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/308512529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:47.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:47.814192+0000) 2023-08-27T22:12:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:48 smithi136 ceph-mon[113531]: pgmap v2378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1855442289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:48.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:48.814419+0000) 2023-08-27T22:12:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:48 smithi093 ceph-mon[111335]: pgmap v2378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1855442289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:48 smithi093 ceph-mon[118902]: pgmap v2378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1855442289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:49.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:49.814684+0000) 2023-08-27T22:12:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:50 smithi136 ceph-mon[113531]: pgmap v2379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:50 smithi093 ceph-mon[111335]: pgmap v2379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:50 smithi093 ceph-mon[118902]: pgmap v2379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:50.420Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:50.421Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:50.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:50.814907+0000) 2023-08-27T22:12:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1459424978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1459424978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1459424978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:51.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:51.815144+0000) 2023-08-27T22:12:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:51.942Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:51.943Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:52 smithi136 ceph-mon[113531]: pgmap v2380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:52 smithi093 ceph-mon[111335]: pgmap v2380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:52 smithi093 ceph-mon[118902]: pgmap v2380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:52.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:52.815394+0000) 2023-08-27T22:12:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2946737943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2946737943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2946737943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:53.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:53.815599+0000) 2023-08-27T22:12:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:53.852Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:12:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:12:53.853Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:12:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:54 smithi136 ceph-mon[113531]: pgmap v2381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:54 smithi093 ceph-mon[111335]: pgmap v2381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:54 smithi093 ceph-mon[118902]: pgmap v2381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:12:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:54.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:54.815868+0000) 2023-08-27T22:12:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:12:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:12:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:12:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:55.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:55.816103+0000) 2023-08-27T22:12:56.816 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:56 smithi136 ceph-mon[113531]: pgmap v2382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:56.816 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1012399868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:56 smithi093 ceph-mon[111335]: pgmap v2382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1012399868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:56 smithi093 ceph-mon[118902]: pgmap v2382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1012399868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:56.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:56.816335+0000) 2023-08-27T22:12:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:57.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:57.816593+0000) 2023-08-27T22:12:58.816 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:58 smithi136 ceph-mon[113531]: pgmap v2383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:58.817 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:12:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/585825701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:58 smithi093 ceph-mon[111335]: pgmap v2383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:12:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/585825701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:58 smithi093 ceph-mon[118902]: pgmap v2383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:12:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:12:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/585825701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:12:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:58.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:58.816823+0000) 2023-08-27T22:13:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:12:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:12:59.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:12:59.817078+0000) 2023-08-27T22:13:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:00 smithi136 ceph-mon[113531]: pgmap v2384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:00 smithi093 ceph-mon[111335]: pgmap v2384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:00 smithi093 ceph-mon[118902]: pgmap v2384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:00.421Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:00.422Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:00.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:00.817288+0000) 2023-08-27T22:13:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2062024508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2062024508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:01.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2062024508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:01.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:01.817528+0000) 2023-08-27T22:13:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:01.943Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:01.944Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:02 smithi136 ceph-mon[113531]: pgmap v2385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:02 smithi093 ceph-mon[111335]: pgmap v2385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:02 smithi093 ceph-mon[118902]: pgmap v2385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:02.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:02.817769+0000) 2023-08-27T22:13:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1927169049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1927169049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1927169049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:03.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:03.818000+0000) 2023-08-27T22:13:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:03.852Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:03.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:04 smithi136 ceph-mon[113531]: pgmap v2386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:04 smithi093 ceph-mon[111335]: pgmap v2386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:04 smithi093 ceph-mon[118902]: pgmap v2386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:04.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:04.818265+0000) 2023-08-27T22:13:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2168739606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2168739606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2168739606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:13:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:13:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:13:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:05.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:05.818521+0000) 2023-08-27T22:13:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:06 smithi136 ceph-mon[113531]: pgmap v2387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:06 smithi093 ceph-mon[111335]: pgmap v2387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:06 smithi093 ceph-mon[118902]: pgmap v2387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:06.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:06.818756+0000) 2023-08-27T22:13:07.606 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:13:07.639 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:13:07.639 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:13:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:07.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:07.818923+0000) 2023-08-27T22:13:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:08 smithi136 ceph-mon[113531]: pgmap v2388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3812880499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:08 smithi093 ceph-mon[111335]: pgmap v2388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3812880499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:08 smithi093 ceph-mon[118902]: pgmap v2388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3812880499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:09.078 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:08.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:08.819093+0000) 2023-08-27T22:13:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:09.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:09.819234+0000) 2023-08-27T22:13:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[111335]: pgmap v2389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2834515878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[118902]: pgmap v2389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2834515878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:10.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:13:10.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:13:10.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:10.422Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:10.423Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:10.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:10.819445+0000) 2023-08-27T22:13:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-mon[113531]: pgmap v2389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2834515878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:13:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:13:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:13:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:11.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:11.819710+0000) 2023-08-27T22:13:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:11.944Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:11.945Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:12 smithi136 ceph-mon[113531]: pgmap v2390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2819880885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:12.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:12.819990+0000) 2023-08-27T22:13:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:12 smithi093 ceph-mon[118902]: pgmap v2390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2819880885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:12 smithi093 ceph-mon[111335]: pgmap v2390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2819880885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:13.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:13.820253+0000) 2023-08-27T22:13:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:13.852Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:13.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:14 smithi136 ceph-mon[113531]: pgmap v2391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:14.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:14.820503+0000) 2023-08-27T22:13:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:14 smithi093 ceph-mon[111335]: pgmap v2391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:14 smithi093 ceph-mon[118902]: pgmap v2391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3554204112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:15.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:13:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:13:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:13:15.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3554204112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3554204112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:15.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:15.820766+0000) 2023-08-27T22:13:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:16 smithi136 ceph-mon[113531]: pgmap v2392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:16.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:16.821099+0000) 2023-08-27T22:13:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:16 smithi093 ceph-mon[111335]: pgmap v2392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:16 smithi093 ceph-mon[118902]: pgmap v2392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/703371074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:17.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:17.821267+0000) 2023-08-27T22:13:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/703371074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/703371074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:18 smithi136 ceph-mon[113531]: pgmap v2393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:18.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:18.821504+0000) 2023-08-27T22:13:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:18 smithi093 ceph-mon[111335]: pgmap v2393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:18 smithi093 ceph-mon[118902]: pgmap v2393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:19.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:19.821767+0000) 2023-08-27T22:13:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2860566826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:20 smithi136 ceph-mon[113531]: pgmap v2394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2860566826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:20 smithi093 ceph-mon[111335]: pgmap v2394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2860566826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:20 smithi093 ceph-mon[118902]: pgmap v2394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:20.423Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:20.425Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:20.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:20.821931+0000) 2023-08-27T22:13:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:21.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:21.822123+0000) 2023-08-27T22:13:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:21.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:21.945Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:22.822 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:22 smithi136 ceph-mon[113531]: pgmap v2395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:22.822 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3905807873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:22 smithi093 ceph-mon[111335]: pgmap v2395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3905807873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:22 smithi093 ceph-mon[118902]: pgmap v2395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3905807873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:22.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:22.822400+0000) 2023-08-27T22:13:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:23.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:23.822661+0000) 2023-08-27T22:13:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:23.853Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:23.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:24.823 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:24 smithi136 ceph-mon[113531]: pgmap v2396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:24.823 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:24.824 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2221953766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:24 smithi093 ceph-mon[111335]: pgmap v2396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2221953766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:24 smithi093 ceph-mon[118902]: pgmap v2396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2221953766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:24.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:24.822926+0000) 2023-08-27T22:13:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:13:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:13:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:13:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:25.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:25.823201+0000) 2023-08-27T22:13:26.823 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:26 smithi136 ceph-mon[113531]: pgmap v2397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:26 smithi093 ceph-mon[111335]: pgmap v2397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:26 smithi093 ceph-mon[118902]: pgmap v2397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:26.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:26.823386+0000) 2023-08-27T22:13:27.826 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3968184037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3968184037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3968184037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:27.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:27.823634+0000) 2023-08-27T22:13:28.823 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:28 smithi136 ceph-mon[113531]: pgmap v2398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:28 smithi093 ceph-mon[111335]: pgmap v2398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:28 smithi093 ceph-mon[118902]: pgmap v2398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:28.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:28.823855+0000) 2023-08-27T22:13:29.824 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1207510348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1207510348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1207510348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:29.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:29.824079+0000) 2023-08-27T22:13:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:30 smithi093 ceph-mon[111335]: pgmap v2399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:30 smithi093 ceph-mon[118902]: pgmap v2399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:30.425Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:30.425Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:30 smithi136 ceph-mon[113531]: pgmap v2399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:30.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:30.824298+0000) 2023-08-27T22:13:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/760039362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:31.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:31.824541+0000) 2023-08-27T22:13:31.945 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/760039362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:31.946 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/760039362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:31.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:31.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:32 smithi136 ceph-mon[113531]: pgmap v2400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:32.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:32.824750+0000) 2023-08-27T22:13:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:32 smithi093 ceph-mon[111335]: pgmap v2400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:32 smithi093 ceph-mon[118902]: pgmap v2400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:33.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:33.824968+0000) 2023-08-27T22:13:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:33.853Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:33.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:34 smithi136 ceph-mon[113531]: pgmap v2401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/902281050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:34.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:34.825160+0000) 2023-08-27T22:13:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:34 smithi093 ceph-mon[111335]: pgmap v2401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/902281050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:34 smithi093 ceph-mon[118902]: pgmap v2401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/902281050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:13:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:13:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:13:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:35.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:35.825400+0000) 2023-08-27T22:13:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:36 smithi136 ceph-mon[113531]: pgmap v2402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1864720125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:36.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:36.825620+0000) 2023-08-27T22:13:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:36 smithi093 ceph-mon[111335]: pgmap v2402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1864720125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:36 smithi093 ceph-mon[118902]: pgmap v2402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1864720125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:37.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:37.825859+0000) 2023-08-27T22:13:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:38 smithi136 ceph-mon[113531]: pgmap v2403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2770114722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:38.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:38.826122+0000) 2023-08-27T22:13:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:38 smithi093 ceph-mon[111335]: pgmap v2403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2770114722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:38 smithi093 ceph-mon[118902]: pgmap v2403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2770114722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:39.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:39.826315+0000) 2023-08-27T22:13:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:40 smithi093 ceph-mon[111335]: pgmap v2404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:40 smithi093 ceph-mon[118902]: pgmap v2404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:40.425Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:40.427Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:40 smithi136 ceph-mon[113531]: pgmap v2404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:40.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:40.826561+0000) 2023-08-27T22:13:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2967999999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:41.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:41.826837+0000) 2023-08-27T22:13:41.946 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2967999999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:41.946 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2967999999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:41.946Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:41.947Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:42 smithi136 ceph-mon[113531]: pgmap v2405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:42.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:42.827110+0000) 2023-08-27T22:13:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:42 smithi093 ceph-mon[111335]: pgmap v2405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:42 smithi093 ceph-mon[118902]: pgmap v2405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2208373087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:43.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:43.827316+0000) 2023-08-27T22:13:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2208373087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2208373087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:43.854Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:43.855Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:44 smithi136 ceph-mon[113531]: pgmap v2406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:44.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:44.827572+0000) 2023-08-27T22:13:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:44 smithi093 ceph-mon[111335]: pgmap v2406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:44 smithi093 ceph-mon[118902]: pgmap v2406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:13:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:13:45] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T22:13:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:45.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:45.827830+0000) 2023-08-27T22:13:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:46 smithi136 ceph-mon[113531]: pgmap v2407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:46.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2042942581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:46.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:46.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:46.828119+0000) 2023-08-27T22:13:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:46 smithi093 ceph-mon[118902]: pgmap v2407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2042942581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:46 smithi093 ceph-mon[111335]: pgmap v2407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2042942581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:47.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:47.828344+0000) 2023-08-27T22:13:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:48 smithi136 ceph-mon[113531]: pgmap v2408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1314955158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:48.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:48.828571+0000) 2023-08-27T22:13:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:48 smithi093 ceph-mon[118902]: pgmap v2408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1314955158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:48 smithi093 ceph-mon[111335]: pgmap v2408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1314955158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:49.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:49.828763+0000) 2023-08-27T22:13:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:50 smithi136 ceph-mon[113531]: pgmap v2409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:50.860 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:50 smithi093 ceph-mon[111335]: pgmap v2409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:50.861 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:50 smithi093 ceph-mon[118902]: pgmap v2409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:50.861 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:50.426Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:50.861 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:50.427Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:50.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:50.829076+0000) 2023-08-27T22:13:51.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1681982683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1681982683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1681982683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:51.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:51.829320+0000) 2023-08-27T22:13:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:51.947Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:51.948Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:52 smithi136 ceph-mon[113531]: pgmap v2410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:52 smithi093 ceph-mon[111335]: pgmap v2410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:52 smithi093 ceph-mon[118902]: pgmap v2410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:52.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:52.829559+0000) 2023-08-27T22:13:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3049982625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3049982625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3049982625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:53.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:53.829770+0000) 2023-08-27T22:13:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:53.855Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:13:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:13:53.856Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:13:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:54 smithi136 ceph-mon[113531]: pgmap v2411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:54 smithi093 ceph-mon[111335]: pgmap v2411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:54 smithi093 ceph-mon[118902]: pgmap v2411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:13:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:54.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:54.830054+0000) 2023-08-27T22:13:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2401134363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2401134363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:13:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:13:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:13:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2401134363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:55.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:55.830279+0000) 2023-08-27T22:13:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:56 smithi136 ceph-mon[113531]: pgmap v2412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:56 smithi093 ceph-mon[111335]: pgmap v2412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:56 smithi093 ceph-mon[118902]: pgmap v2412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:56.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:56.830473+0000) 2023-08-27T22:13:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:57.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:57.830735+0000) 2023-08-27T22:13:58.831 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:58 smithi136 ceph-mon[113531]: pgmap v2413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:58.831 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:13:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/800261545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:58 smithi093 ceph-mon[111335]: pgmap v2413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:13:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/800261545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:58 smithi093 ceph-mon[118902]: pgmap v2413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:13:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:13:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/800261545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:13:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:58.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:58.831021+0000) 2023-08-27T22:14:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:13:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:13:59.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:13:59.831211+0000) 2023-08-27T22:14:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:00 smithi136 ceph-mon[113531]: pgmap v2414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3575749780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:00 smithi093 ceph-mon[111335]: pgmap v2414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3575749780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:00 smithi093 ceph-mon[118902]: pgmap v2414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3575749780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:00.427Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:00.428Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:00.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:00.831460+0000) 2023-08-27T22:14:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:01.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:01.831705+0000) 2023-08-27T22:14:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:01.948Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:01.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:02.832 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:02 smithi136 ceph-mon[113531]: pgmap v2415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:02.832 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2499388558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:02 smithi093 ceph-mon[111335]: pgmap v2415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2499388558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:02 smithi093 ceph-mon[118902]: pgmap v2415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2499388558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:02.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:02.831905+0000) 2023-08-27T22:14:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:03.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:03.832150+0000) 2023-08-27T22:14:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:03.856Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:03.857Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:04.832 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:04 smithi136 ceph-mon[113531]: pgmap v2416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:04 smithi093 ceph-mon[111335]: pgmap v2416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:04 smithi093 ceph-mon[118902]: pgmap v2416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:04.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:04.832343+0000) 2023-08-27T22:14:05.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3739184640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:05.743 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:14:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:14:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:14:05.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3739184640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:05.833 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3739184640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:05.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:05.832558+0000) 2023-08-27T22:14:06.832 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:06 smithi136 ceph-mon[113531]: pgmap v2417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:06 smithi093 ceph-mon[111335]: pgmap v2417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:06 smithi093 ceph-mon[118902]: pgmap v2417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:06.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:06.832761+0000) 2023-08-27T22:14:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2808384247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:07.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:07.832997+0000) 2023-08-27T22:14:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2808384247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2808384247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:08.836 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:08 smithi136 ceph-mon[113531]: pgmap v2418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:08 smithi093 ceph-mon[111335]: pgmap v2418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:08 smithi093 ceph-mon[118902]: pgmap v2418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:08.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:08.836817+0000) 2023-08-27T22:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4196432170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:09.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:09.837065+0000) 2023-08-27T22:14:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4196432170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4196432170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:10 smithi093 ceph-mon[111335]: pgmap v2419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:14:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:10 smithi093 ceph-mon[118902]: pgmap v2419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:14:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:10.427Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:10.428Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:10 smithi136 ceph-mon[113531]: pgmap v2419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:14:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:10.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:10.837221+0000) 2023-08-27T22:14:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:11.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:11.837477+0000) 2023-08-27T22:14:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:11.949Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:11.950Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:12.837 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:12.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:12.837647+0000) 2023-08-27T22:14:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:12 smithi136 ceph-mon[113531]: pgmap v2420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4254499705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[111335]: pgmap v2420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4254499705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[118902]: pgmap v2420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4254499705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:13.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:13.837831+0000) 2023-08-27T22:14:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:14:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:14:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:14:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:14:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:14:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:14:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:14:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:13.856Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:13.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:14 smithi136 ceph-mon[113531]: pgmap v2421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3606040986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:14.838+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:14.838120+0000) 2023-08-27T22:14:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:14 smithi093 ceph-mon[111335]: pgmap v2421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3606040986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:14 smithi093 ceph-mon[118902]: pgmap v2421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3606040986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:14:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:14:15] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T22:14:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:15.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:15.838328+0000) 2023-08-27T22:14:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:16 smithi136 ceph-mon[113531]: pgmap v2422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2740852790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:16.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:16.838575+0000) 2023-08-27T22:14:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:16 smithi093 ceph-mon[111335]: pgmap v2422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2740852790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:16 smithi093 ceph-mon[118902]: pgmap v2422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2740852790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:17.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:17.838898+0000) 2023-08-27T22:14:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:18 smithi136 ceph-mon[113531]: pgmap v2423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:18.838+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:18.839213+0000) 2023-08-27T22:14:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:18 smithi093 ceph-mon[111335]: pgmap v2423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:18 smithi093 ceph-mon[118902]: pgmap v2423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2318234059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:19.838+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:19.839450+0000) 2023-08-27T22:14:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2318234059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2318234059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:20.428Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:20.429Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:20 smithi136 ceph-mon[113531]: pgmap v2424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:20.838+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:20.839735+0000) 2023-08-27T22:14:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:20 smithi093 ceph-mon[111335]: pgmap v2424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:20 smithi093 ceph-mon[118902]: pgmap v2424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3593859171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:21.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:21.840012+0000) 2023-08-27T22:14:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3593859171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3593859171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:21.949Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:21.950Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:22 smithi136 ceph-mon[113531]: pgmap v2425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:22.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:22.840319+0000) 2023-08-27T22:14:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:22 smithi093 ceph-mon[111335]: pgmap v2425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:22 smithi093 ceph-mon[118902]: pgmap v2425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2412208160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:23.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:23.841407+0000) 2023-08-27T22:14:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2412208160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2412208160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:23.857Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:23.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:24 smithi136 ceph-mon[113531]: pgmap v2426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:24.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:24.841621+0000) 2023-08-27T22:14:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:24 smithi093 ceph-mon[111335]: pgmap v2426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:24 smithi093 ceph-mon[118902]: pgmap v2426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:14:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:14:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:14:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:25.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:25.841881+0000) 2023-08-27T22:14:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:26 smithi136 ceph-mon[113531]: pgmap v2427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3444274533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:26.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:26.842125+0000) 2023-08-27T22:14:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:26 smithi093 ceph-mon[111335]: pgmap v2427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3444274533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:26 smithi093 ceph-mon[118902]: pgmap v2427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3444274533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:27.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:27.842372+0000) 2023-08-27T22:14:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:28 smithi136 ceph-mon[113531]: pgmap v2428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/474950034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:28.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:28.842578+0000) 2023-08-27T22:14:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:28 smithi093 ceph-mon[111335]: pgmap v2428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/474950034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:28 smithi093 ceph-mon[118902]: pgmap v2428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/474950034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:29.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:29.842832+0000) 2023-08-27T22:14:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:30 smithi136 ceph-mon[113531]: pgmap v2429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:30 smithi093 ceph-mon[111335]: pgmap v2429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:30 smithi093 ceph-mon[118902]: pgmap v2429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:30.428Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:30.429Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:30.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:30.843136+0000) 2023-08-27T22:14:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2107049589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2107049589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2107049589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:31.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:31.843395+0000) 2023-08-27T22:14:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:31.950Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:31.952Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:32 smithi136 ceph-mon[113531]: pgmap v2430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:32 smithi093 ceph-mon[111335]: pgmap v2430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:32 smithi093 ceph-mon[118902]: pgmap v2430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:32.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:32.843671+0000) 2023-08-27T22:14:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2171664589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2171664589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2171664589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:33.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:33.843954+0000) 2023-08-27T22:14:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:33.859Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:33.860Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:34 smithi136 ceph-mon[113531]: pgmap v2431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:34 smithi093 ceph-mon[111335]: pgmap v2431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:34 smithi093 ceph-mon[118902]: pgmap v2431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:34.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:34.844207+0000) 2023-08-27T22:14:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:14:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:14:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:14:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:35.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:35.844408+0000) 2023-08-27T22:14:36.844 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:36 smithi136 ceph-mon[113531]: pgmap v2432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:36.845 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/372092863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:36 smithi093 ceph-mon[111335]: pgmap v2432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/372092863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:36 smithi093 ceph-mon[118902]: pgmap v2432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/372092863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:36.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:36.844685+0000) 2023-08-27T22:14:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:37.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:37.844952+0000) 2023-08-27T22:14:38.845 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:38 smithi136 ceph-mon[113531]: pgmap v2433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:38.845 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/814731400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:38 smithi093 ceph-mon[111335]: pgmap v2433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/814731400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:38 smithi093 ceph-mon[118902]: pgmap v2433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/814731400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:38.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:38.845180+0000) 2023-08-27T22:14:39.845 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:39.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:39.845407+0000) 2023-08-27T22:14:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:40 smithi093 ceph-mon[111335]: pgmap v2434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1992463781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:40 smithi093 ceph-mon[118902]: pgmap v2434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1992463781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:40.429Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:40.429Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:40.845 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:40 smithi136 ceph-mon[113531]: pgmap v2434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:40.845 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1992463781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:40.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:40.845626+0000) 2023-08-27T22:14:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:41.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:41.845898+0000) 2023-08-27T22:14:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:41.951Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:41.952Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:42.846 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:42 smithi136 ceph-mon[113531]: pgmap v2435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:42 smithi093 ceph-mon[111335]: pgmap v2435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:42 smithi093 ceph-mon[118902]: pgmap v2435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:42.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:42.846106+0000) 2023-08-27T22:14:43.846 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12116375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:43.860 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12116375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:43.860 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12116375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:43.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:43.846342+0000) 2023-08-27T22:14:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:43.859Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:43.861Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:44.846 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:44 smithi136 ceph-mon[113531]: pgmap v2436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:44 smithi093 ceph-mon[111335]: pgmap v2436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:44 smithi093 ceph-mon[118902]: pgmap v2436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:44.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:44.846555+0000) 2023-08-27T22:14:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3635551539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3635551539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:14:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:14:45] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:14:45.846 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3635551539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:45.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:45.846769+0000) 2023-08-27T22:14:46.847 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:46 smithi136 ceph-mon[113531]: pgmap v2437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:46 smithi093 ceph-mon[111335]: pgmap v2437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:46 smithi093 ceph-mon[118902]: pgmap v2437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:46.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:46.846992+0000) 2023-08-27T22:14:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:47.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:47.847214+0000) 2023-08-27T22:14:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:48 smithi136 ceph-mon[113531]: pgmap v2438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/743753597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:48.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:48.847455+0000) 2023-08-27T22:14:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:48 smithi093 ceph-mon[111335]: pgmap v2438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/743753597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:48 smithi093 ceph-mon[118902]: pgmap v2438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/743753597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:49.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:49.847711+0000) 2023-08-27T22:14:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:50 smithi136 ceph-mon[113531]: pgmap v2439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2374668511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:50 smithi093 ceph-mon[118902]: pgmap v2439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2374668511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:50 smithi093 ceph-mon[111335]: pgmap v2439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2374668511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:50.429Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:50.430Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:50.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:50.847982+0000) 2023-08-27T22:14:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:51.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:51.848180+0000) 2023-08-27T22:14:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:51.952Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:51.953Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:52.848 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:52 smithi136 ceph-mon[113531]: pgmap v2440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:52.848 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/590666722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:52 smithi093 ceph-mon[111335]: pgmap v2440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/590666722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:52 smithi093 ceph-mon[118902]: pgmap v2440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/590666722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:52.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:52.848402+0000) 2023-08-27T22:14:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:53.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:53.848661+0000) 2023-08-27T22:14:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:53.860Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:14:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:14:53.861Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:14:54.849 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:54 smithi136 ceph-mon[113531]: pgmap v2441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:54.849 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:54 smithi093 ceph-mon[111335]: pgmap v2441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:54 smithi093 ceph-mon[118902]: pgmap v2441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:14:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:54.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:54.848932+0000) 2023-08-27T22:14:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1138604169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1138604169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:14:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:14:55] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:14:55.849 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1138604169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:55.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:55.849196+0000) 2023-08-27T22:14:56.849 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:56 smithi136 ceph-mon[113531]: pgmap v2442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:56 smithi093 ceph-mon[111335]: pgmap v2442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:56 smithi093 ceph-mon[118902]: pgmap v2442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:56.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:56.849412+0000) 2023-08-27T22:14:57.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/547574918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/547574918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/547574918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:14:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:57.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:57.849641+0000) 2023-08-27T22:14:58.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:14:58 smithi136 ceph-mon[113531]: pgmap v2443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:58.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:14:58 smithi093 ceph-mon[118902]: pgmap v2443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:14:58 smithi093 ceph-mon[111335]: pgmap v2443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:14:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:58.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:58.849901+0000) 2023-08-27T22:15:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:14:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:14:59.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:14:59.850134+0000) 2023-08-27T22:15:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:00 smithi136 ceph-mon[113531]: pgmap v2444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/476712207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:00 smithi093 ceph-mon[111335]: pgmap v2444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/476712207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:00 smithi093 ceph-mon[118902]: pgmap v2444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/476712207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:00.430Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:00.432Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:00.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:00.850401+0000) 2023-08-27T22:15:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:01.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:01.850656+0000) 2023-08-27T22:15:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:01.952Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:01.953Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:02.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:02 smithi136 ceph-mon[113531]: pgmap v2445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:02.851 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/791815576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:02 smithi093 ceph-mon[111335]: pgmap v2445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/791815576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:02 smithi093 ceph-mon[118902]: pgmap v2445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/791815576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:02.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:02.850901+0000) 2023-08-27T22:15:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:03.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:03.851157+0000) 2023-08-27T22:15:04.434 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:03.860Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:04.435 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:03.861Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:04.851 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:04 smithi136 ceph-mon[113531]: pgmap v2446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:04.851 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/783883396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:04 smithi093 ceph-mon[111335]: pgmap v2446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/783883396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:04 smithi093 ceph-mon[118902]: pgmap v2446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/783883396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:04.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:04.851417+0000) 2023-08-27T22:15:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:15:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:15:05] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:15:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:05.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:05.851674+0000) 2023-08-27T22:15:06.852 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:06 smithi136 ceph-mon[113531]: pgmap v2447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:06 smithi093 ceph-mon[111335]: pgmap v2447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:06.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:06 smithi093 ceph-mon[118902]: pgmap v2447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:06.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:06.851909+0000) 2023-08-27T22:15:07.852 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2719640388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2719640388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2719640388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:07.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:07.852111+0000) 2023-08-27T22:15:08.852 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:08 smithi136 ceph-mon[113531]: pgmap v2448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:08 smithi093 ceph-mon[111335]: pgmap v2448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:08 smithi093 ceph-mon[118902]: pgmap v2448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:08.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:08.852359+0000) 2023-08-27T22:15:09.852 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:09.852 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1145038453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1145038453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1145038453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:09.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:09.852609+0000) 2023-08-27T22:15:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:10 smithi093 ceph-mon[111335]: pgmap v2449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:10 smithi093 ceph-mon[118902]: pgmap v2449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:10.432Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:10.433Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:10.852 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:10 smithi136 ceph-mon[113531]: pgmap v2449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:10.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:10.852859+0000) 2023-08-27T22:15:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3278736913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:11.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:11.853155+0000) 2023-08-27T22:15:11.953 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3278736913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:11.954 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3278736913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:11.953Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:12.328 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:11.954Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:12 smithi136 ceph-mon[113531]: pgmap v2450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:12.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:12.853352+0000) 2023-08-27T22:15:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:12 smithi093 ceph-mon[111335]: pgmap v2450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:12 smithi093 ceph-mon[118902]: pgmap v2450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:15:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:13.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:13.853526+0000) 2023-08-27T22:15:13.954 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:15:13.954 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:15:13.954 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:13.861Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:13.955 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:13.862Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:14 smithi136 ceph-mon[113531]: pgmap v2451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2813384293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:14.910 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:14 smithi093 ceph-mon[111335]: pgmap v2451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:14.910 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2813384293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:14.910 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:14 smithi093 ceph-mon[118902]: pgmap v2451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:14.911 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2813384293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:14.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:14.853718+0000) 2023-08-27T22:15:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:15:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:15:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:15:16.162 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:15.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:15.853908+0000) 2023-08-27T22:15:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:16 smithi136 ceph-mon[113531]: pgmap v2452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[111335]: pgmap v2452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[118902]: pgmap v2452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:17.166 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:16.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:16.854127+0000) 2023-08-27T22:15:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2048735124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:15:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:15:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2048735124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:15:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:15:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2048735124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:15:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:15:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:15:18.173 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:17.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:17.854382+0000) 2023-08-27T22:15:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:18 smithi136 ceph-mon[113531]: pgmap v2453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:18 smithi093 ceph-mon[111335]: pgmap v2453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:18 smithi093 ceph-mon[118902]: pgmap v2453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:19.172 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:18.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:18.854638+0000) 2023-08-27T22:15:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3313887286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3313887286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3313887286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:19.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:19.854878+0000) 2023-08-27T22:15:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:20 smithi136 ceph-mon[113531]: pgmap v2454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:20 smithi093 ceph-mon[111335]: pgmap v2454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:20 smithi093 ceph-mon[118902]: pgmap v2454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:20.432Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:20.433Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:20.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:20.855164+0000) 2023-08-27T22:15:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3378648051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3378648051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3378648051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:21.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:21.855396+0000) 2023-08-27T22:15:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:21.954Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:21.955Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:22 smithi136 ceph-mon[113531]: pgmap v2455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:22 smithi093 ceph-mon[111335]: pgmap v2455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:22 smithi093 ceph-mon[118902]: pgmap v2455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:22.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:22.855667+0000) 2023-08-27T22:15:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/180610881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/180610881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:23.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/180610881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:23.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:23.855921+0000) 2023-08-27T22:15:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:23.862Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:23.863Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:24 smithi093 ceph-mon[111335]: pgmap v2456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:24 smithi093 ceph-mon[118902]: pgmap v2456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:24.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:24 smithi136 ceph-mon[113531]: pgmap v2456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:24.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:24.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:24.856226+0000) 2023-08-27T22:15:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:15:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:15:25] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:15:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:25.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:25.856461+0000) 2023-08-27T22:15:26.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:26 smithi136 ceph-mon[113531]: pgmap v2457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:26.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2412367889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:26 smithi093 ceph-mon[111335]: pgmap v2457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2412367889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:26 smithi093 ceph-mon[118902]: pgmap v2457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2412367889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:26.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:26.856731+0000) 2023-08-27T22:15:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:27.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:27.856926+0000) 2023-08-27T22:15:28.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:28 smithi136 ceph-mon[113531]: pgmap v2458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:28.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2130057333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:28 smithi093 ceph-mon[111335]: pgmap v2458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2130057333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:28 smithi093 ceph-mon[118902]: pgmap v2458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2130057333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:28.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:28.857159+0000) 2023-08-27T22:15:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:29.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:29.857407+0000) 2023-08-27T22:15:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:30 smithi136 ceph-mon[113531]: pgmap v2459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:30 smithi093 ceph-mon[111335]: pgmap v2459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:30 smithi093 ceph-mon[118902]: pgmap v2459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:30.434Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:30.434Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:30.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:30.857627+0000) 2023-08-27T22:15:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3785085845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3785085845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3785085845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:31.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:31.857887+0000) 2023-08-27T22:15:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:31.955Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:31.956Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:32 smithi136 ceph-mon[113531]: pgmap v2460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:32 smithi093 ceph-mon[111335]: pgmap v2460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:32 smithi093 ceph-mon[118902]: pgmap v2460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:32.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:32.858126+0000) 2023-08-27T22:15:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3862151379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3862151379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3862151379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:33.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:33.858385+0000) 2023-08-27T22:15:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:33.863Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:33.864Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:34 smithi136 ceph-mon[113531]: pgmap v2461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:35.806 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:34 smithi093 ceph-mon[111335]: pgmap v2461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:35.807 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:34.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:34.858642+0000) 2023-08-27T22:15:35.807 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:34 smithi093 ceph-mon[118902]: pgmap v2461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:35.807 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3368054683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:35.807 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3368054683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:35.808 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:15:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:15:35] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:15:36.362 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3368054683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:36.362 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:35.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:35.858914+0000) 2023-08-27T22:15:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:36 smithi136 ceph-mon[113531]: pgmap v2462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:36 smithi093 ceph-mon[111335]: pgmap v2462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:36 smithi093 ceph-mon[118902]: pgmap v2462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:36.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:36.859160+0000) 2023-08-27T22:15:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:37.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:37.859405+0000) 2023-08-27T22:15:38.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:38 smithi136 ceph-mon[113531]: pgmap v2463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:38.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1840950307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:38 smithi093 ceph-mon[111335]: pgmap v2463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1840950307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:38 smithi093 ceph-mon[118902]: pgmap v2463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1840950307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:38.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:38.859703+0000) 2023-08-27T22:15:39.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:39.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:39.859900+0000) 2023-08-27T22:15:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:40 smithi093 ceph-mon[111335]: pgmap v2464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2982877185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:40 smithi093 ceph-mon[118902]: pgmap v2464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2982877185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:40.434Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:40.435Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:40.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:40 smithi136 ceph-mon[113531]: pgmap v2464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:40.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2982877185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:40.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:40.860154+0000) 2023-08-27T22:15:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:41.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:41.860403+0000) 2023-08-27T22:15:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:41.956Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:41.957Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:42.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:42 smithi136 ceph-mon[113531]: pgmap v2465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:42.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2756646809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:42 smithi093 ceph-mon[111335]: pgmap v2465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2756646809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:42 smithi093 ceph-mon[118902]: pgmap v2465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2756646809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:42.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:42.860663+0000) 2023-08-27T22:15:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:43.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:43.860910+0000) 2023-08-27T22:15:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:43.864Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:43.865Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:44.861 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:44 smithi136 ceph-mon[113531]: pgmap v2466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:44 smithi093 ceph-mon[111335]: pgmap v2466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:44 smithi093 ceph-mon[118902]: pgmap v2466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:44.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:44.861155+0000) 2023-08-27T22:15:45.613 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:15:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:15:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:15:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1549843865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:45.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:45.861413+0000) 2023-08-27T22:15:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1549843865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1549843865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:46 smithi136 ceph-mon[113531]: pgmap v2467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:46.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:46.861630+0000) 2023-08-27T22:15:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:46 smithi093 ceph-mon[111335]: pgmap v2467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:46 smithi093 ceph-mon[118902]: pgmap v2467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4138472152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:47.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:47.861875+0000) 2023-08-27T22:15:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4138472152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4138472152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:48 smithi136 ceph-mon[113531]: pgmap v2468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:48.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:48.862103+0000) 2023-08-27T22:15:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:48 smithi093 ceph-mon[111335]: pgmap v2468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:48 smithi093 ceph-mon[118902]: pgmap v2468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:49.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:49.862351+0000) 2023-08-27T22:15:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:50 smithi136 ceph-mon[113531]: pgmap v2469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3110126428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:50 smithi093 ceph-mon[111335]: pgmap v2469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3110126428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:50 smithi093 ceph-mon[118902]: pgmap v2469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3110126428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:50.434Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:50.435Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:50.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:50.862612+0000) 2023-08-27T22:15:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:51.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:51.862840+0000) 2023-08-27T22:15:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:51.957Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:51.957Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:52.863 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:52 smithi136 ceph-mon[113531]: pgmap v2470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:52.863 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1889432990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:52 smithi093 ceph-mon[111335]: pgmap v2470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1889432990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:52 smithi093 ceph-mon[118902]: pgmap v2470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1889432990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:52.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:52.863113+0000) 2023-08-27T22:15:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:53.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:53.863392+0000) 2023-08-27T22:15:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:53.864Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:15:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:15:53.865Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:15:54.863 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:54 smithi136 ceph-mon[113531]: pgmap v2471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:54.864 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:54.864 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2454570106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:54 smithi093 ceph-mon[118902]: pgmap v2471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2454570106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:54 smithi093 ceph-mon[111335]: pgmap v2471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:15:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2454570106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:54.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:54.863642+0000) 2023-08-27T22:15:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:15:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:15:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:15:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:55.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:55.863889+0000) 2023-08-27T22:15:56.864 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:56 smithi136 ceph-mon[113531]: pgmap v2472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:56 smithi093 ceph-mon[111335]: pgmap v2472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:56 smithi093 ceph-mon[118902]: pgmap v2472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:56.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:56.864127+0000) 2023-08-27T22:15:57.864 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1178232188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1178232188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1178232188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:57.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:57.864331+0000) 2023-08-27T22:15:58.864 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:58 smithi136 ceph-mon[113531]: pgmap v2473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:58 smithi093 ceph-mon[111335]: pgmap v2473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:58 smithi093 ceph-mon[118902]: pgmap v2473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:15:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:58.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:58.864583+0000) 2023-08-27T22:15:59.865 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:15:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3948089254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:15:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3948089254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:15:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:15:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3948089254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:15:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:15:59.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:15:59.864834+0000) 2023-08-27T22:16:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:00 smithi093 ceph-mon[111335]: pgmap v2474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:00 smithi093 ceph-mon[118902]: pgmap v2474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:00.435Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:00.436Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:00.865 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:00 smithi136 ceph-mon[113531]: pgmap v2474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:00.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:00.865109+0000) 2023-08-27T22:16:01.865 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1203491717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:01.957 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1203491717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:01.958 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1203491717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:01.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:01.865345+0000) 2023-08-27T22:16:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:01.957Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:01.958Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:02 smithi136 ceph-mon[113531]: pgmap v2475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:02.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:02.865656+0000) 2023-08-27T22:16:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:02 smithi093 ceph-mon[111335]: pgmap v2475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:02 smithi093 ceph-mon[118902]: pgmap v2475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:03.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:03.865871+0000) 2023-08-27T22:16:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:03.864Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:03.865Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:04 smithi136 ceph-mon[113531]: pgmap v2476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3386667371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:04.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:04.866159+0000) 2023-08-27T22:16:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:04 smithi093 ceph-mon[118902]: pgmap v2476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3386667371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:04 smithi093 ceph-mon[111335]: pgmap v2476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3386667371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:16:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:16:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:16:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:05.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:05.866367+0000) 2023-08-27T22:16:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:06 smithi136 ceph-mon[113531]: pgmap v2477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3299824301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:06.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:06.866581+0000) 2023-08-27T22:16:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:06 smithi093 ceph-mon[111335]: pgmap v2477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3299824301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:06 smithi093 ceph-mon[118902]: pgmap v2477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3299824301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:07.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:07.866811+0000) 2023-08-27T22:16:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:08 smithi136 ceph-mon[113531]: pgmap v2478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:08.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:08.867128+0000) 2023-08-27T22:16:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:08 smithi093 ceph-mon[111335]: pgmap v2478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:08 smithi093 ceph-mon[118902]: pgmap v2478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:09.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:09.867369+0000) 2023-08-27T22:16:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/619767887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/619767887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/619767887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:10 smithi093 ceph-mon[111335]: pgmap v2479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:10 smithi093 ceph-mon[118902]: pgmap v2479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:10.436Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:10.437Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:10 smithi136 ceph-mon[113531]: pgmap v2479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:10.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:10.867622+0000) 2023-08-27T22:16:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3907089257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:11.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:11.867874+0000) 2023-08-27T22:16:11.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3907089257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:11.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3907089257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:11.958Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:11.959Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:12 smithi136 ceph-mon[113531]: pgmap v2480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:12.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:12.868109+0000) 2023-08-27T22:16:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:12 smithi093 ceph-mon[111335]: pgmap v2480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:12 smithi093 ceph-mon[118902]: pgmap v2480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2091966533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:13.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:13.868405+0000) 2023-08-27T22:16:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2091966533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2091966533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:13.865Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:13.866Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:14 smithi136 ceph-mon[113531]: pgmap v2481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:14.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:14.868661+0000) 2023-08-27T22:16:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:14 smithi093 ceph-mon[111335]: pgmap v2481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:14 smithi093 ceph-mon[118902]: pgmap v2481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:16:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:16:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:16:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:15.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:15.868906+0000) 2023-08-27T22:16:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:16 smithi136 ceph-mon[113531]: pgmap v2482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3133345482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:16:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:16.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:16.869108+0000) 2023-08-27T22:16:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:16 smithi093 ceph-mon[111335]: pgmap v2482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3133345482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:16:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:16 smithi093 ceph-mon[118902]: pgmap v2482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3133345482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:16:17.869 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:17.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:17.869265+0000) 2023-08-27T22:16:18.779 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:18 smithi136 ceph-mon[113531]: pgmap v2483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:18.780 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:18.780 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:18.780 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3480160327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[118902]: pgmap v2483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3480160327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[111335]: pgmap v2483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3480160327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:18.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:18.869442+0000) 2023-08-27T22:16:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:19.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:19.869648+0000) 2023-08-27T22:16:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:16:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:16:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:20 smithi136 ceph-mon[113531]: pgmap v2484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:20.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:16:20.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:16:20.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.438 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[111335]: pgmap v2484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:20.438 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.439 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.439 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:16:20.439 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:16:20.439 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:16:20.439 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-mon[118902]: pgmap v2484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:20.437Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:20.438Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:20.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:20.869921+0000) 2023-08-27T22:16:21.870 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/788198567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:21.958 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/788198567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:21.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/788198567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:21.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:21.870141+0000) 2023-08-27T22:16:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:21.958Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:21.959Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:22.870 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:22 smithi136 ceph-mon[113531]: pgmap v2485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:22 smithi093 ceph-mon[111335]: pgmap v2485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:22 smithi093 ceph-mon[118902]: pgmap v2485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:22.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:22.870382+0000) 2023-08-27T22:16:23.866 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1111838174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:23.866 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1111838174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:23.870 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1111838174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:23.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:23.870613+0000) 2023-08-27T22:16:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:23.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:23.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:24.871 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:24 smithi136 ceph-mon[113531]: pgmap v2486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:24.871 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:24 smithi093 ceph-mon[111335]: pgmap v2486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:24 smithi093 ceph-mon[118902]: pgmap v2486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:24.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:24.870866+0000) 2023-08-27T22:16:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2860769135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:16:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:16:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:16:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2860769135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:25.871 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2860769135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:25.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:25.871145+0000) 2023-08-27T22:16:26.871 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:26 smithi136 ceph-mon[113531]: pgmap v2487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:26 smithi093 ceph-mon[111335]: pgmap v2487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:26 smithi093 ceph-mon[118902]: pgmap v2487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:26.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:26.871421+0000) 2023-08-27T22:16:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:27.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:27.871676+0000) 2023-08-27T22:16:28.871 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:28 smithi136 ceph-mon[113531]: pgmap v2488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:28.872 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2836244816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:28 smithi093 ceph-mon[111335]: pgmap v2488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2836244816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:28 smithi093 ceph-mon[118902]: pgmap v2488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2836244816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:28.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:28.871928+0000) 2023-08-27T22:16:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:29.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:29.872174+0000) 2023-08-27T22:16:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:30 smithi136 ceph-mon[113531]: pgmap v2489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1005621968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:30 smithi093 ceph-mon[111335]: pgmap v2489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1005621968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:30 smithi093 ceph-mon[118902]: pgmap v2489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1005621968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:30.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:30.438Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:30.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:30.872481+0000) 2023-08-27T22:16:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:31.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:31.872740+0000) 2023-08-27T22:16:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:31.959Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:31.960Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:32.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:32 smithi136 ceph-mon[113531]: pgmap v2490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:32.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3062300699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:32 smithi093 ceph-mon[111335]: pgmap v2490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3062300699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:32 smithi093 ceph-mon[118902]: pgmap v2490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3062300699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:32.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:32.873010+0000) 2023-08-27T22:16:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:33.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:33.873308+0000) 2023-08-27T22:16:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:33.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:33.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:34.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:34 smithi136 ceph-mon[113531]: pgmap v2491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:34 smithi093 ceph-mon[111335]: pgmap v2491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:34 smithi093 ceph-mon[118902]: pgmap v2491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:34.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:34.873583+0000) 2023-08-27T22:16:35.611 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:16:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:16:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:16:35.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3817004924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3817004924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3817004924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:35.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:35.873833+0000) 2023-08-27T22:16:36.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:36 smithi136 ceph-mon[113531]: pgmap v2492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:36 smithi093 ceph-mon[111335]: pgmap v2492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:36 smithi093 ceph-mon[118902]: pgmap v2492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:36.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:36.874111+0000) 2023-08-27T22:16:37.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4175190978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4175190978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4175190978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:37.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:37.874389+0000) 2023-08-27T22:16:38.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:38 smithi136 ceph-mon[113531]: pgmap v2493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:38 smithi093 ceph-mon[111335]: pgmap v2493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:38 smithi093 ceph-mon[118902]: pgmap v2493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:38.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:38.874654+0000) 2023-08-27T22:16:39.875 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:39.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:39.874853+0000) 2023-08-27T22:16:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:40 smithi093 ceph-mon[111335]: pgmap v2494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3637646915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:40 smithi093 ceph-mon[118902]: pgmap v2494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3637646915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:40.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:40.439Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:40.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:40.875126+0000) 2023-08-27T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:40 smithi136 ceph-mon[113531]: pgmap v2494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3637646915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:41.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:41.875356+0000) 2023-08-27T22:16:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:41.959Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:41.960Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:42.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:42.875590+0000) 2023-08-27T22:16:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:42 smithi136 ceph-mon[113531]: pgmap v2495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3930149788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:42 smithi093 ceph-mon[111335]: pgmap v2495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3930149788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:42 smithi093 ceph-mon[118902]: pgmap v2495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3930149788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:43.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:43.875785+0000) 2023-08-27T22:16:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:43.867Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:43.868Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:44 smithi136 ceph-mon[113531]: pgmap v2496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2499706363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:44.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:44.876103+0000) 2023-08-27T22:16:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:44 smithi093 ceph-mon[118902]: pgmap v2496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2499706363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:44 smithi093 ceph-mon[111335]: pgmap v2496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2499706363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:16:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:16:45] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:16:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:45.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:45.876362+0000) 2023-08-27T22:16:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:46 smithi136 ceph-mon[113531]: pgmap v2497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:46.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:46.876564+0000) 2023-08-27T22:16:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:46 smithi093 ceph-mon[111335]: pgmap v2497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:46 smithi093 ceph-mon[118902]: pgmap v2497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2803319484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:47.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:47.876808+0000) 2023-08-27T22:16:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2803319484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2803319484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:48 smithi136 ceph-mon[113531]: pgmap v2498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:48.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:48.877120+0000) 2023-08-27T22:16:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:48 smithi093 ceph-mon[111335]: pgmap v2498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:48 smithi093 ceph-mon[118902]: pgmap v2498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1868874895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:49.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:49.877349+0000) 2023-08-27T22:16:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1868874895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1868874895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:50 smithi093 ceph-mon[111335]: pgmap v2499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:50 smithi093 ceph-mon[118902]: pgmap v2499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:50.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:50.440Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:50 smithi136 ceph-mon[113531]: pgmap v2499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:50.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:50.877631+0000) 2023-08-27T22:16:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/261023221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:51.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:51.877870+0000) 2023-08-27T22:16:51.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/261023221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:51.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/261023221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:51.960Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:51.961Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:52 smithi136 ceph-mon[113531]: pgmap v2500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:52.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:52.878153+0000) 2023-08-27T22:16:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:52 smithi093 ceph-mon[111335]: pgmap v2500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:52 smithi093 ceph-mon[118902]: pgmap v2500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:53.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:53.878457+0000) 2023-08-27T22:16:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:53.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:16:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:16:53.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:16:54.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:54 smithi136 ceph-mon[113531]: pgmap v2501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:54.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:54.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/959412967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:54.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:54.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:54.878709+0000) 2023-08-27T22:16:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:54 smithi093 ceph-mon[111335]: pgmap v2501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/959412967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:54 smithi093 ceph-mon[118902]: pgmap v2501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:16:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/959412967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:16:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:16:55] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T22:16:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:55.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:55.878980+0000) 2023-08-27T22:16:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:56 smithi136 ceph-mon[113531]: pgmap v2502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3297389522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:56.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:56.879240+0000) 2023-08-27T22:16:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:56 smithi093 ceph-mon[111335]: pgmap v2502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3297389522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:56 smithi093 ceph-mon[118902]: pgmap v2502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3297389522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:57.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:57.879484+0000) 2023-08-27T22:16:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:58 smithi136 ceph-mon[113531]: pgmap v2503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:58.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:58.879762+0000) 2023-08-27T22:16:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:58 smithi093 ceph-mon[111335]: pgmap v2503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:58 smithi093 ceph-mon[118902]: pgmap v2503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:16:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:16:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3092278925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:16:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:16:59.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:16:59.880002+0000) 2023-08-27T22:16:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:16:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3092278925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:16:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:16:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3092278925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:00 smithi093 ceph-mon[111335]: pgmap v2504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:00 smithi093 ceph-mon[118902]: pgmap v2504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:00.439Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:00.440Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:00 smithi136 ceph-mon[113531]: pgmap v2504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:00.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:00.880253+0000) 2023-08-27T22:17:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2954243836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:01.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:01.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:01.880477+0000) 2023-08-27T22:17:01.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2954243836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:01.962 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2954243836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:01.961Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:01.962Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:02 smithi136 ceph-mon[113531]: pgmap v2505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:02.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:02.880734+0000) 2023-08-27T22:17:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:02 smithi093 ceph-mon[111335]: pgmap v2505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:02 smithi093 ceph-mon[118902]: pgmap v2505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/407420820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:03.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:03.880969+0000) 2023-08-27T22:17:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/407420820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/407420820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:03.869Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:03.870Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:04 smithi136 ceph-mon[113531]: pgmap v2506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:04.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:04.881236+0000) 2023-08-27T22:17:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:04 smithi093 ceph-mon[111335]: pgmap v2506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:04 smithi093 ceph-mon[118902]: pgmap v2506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:17:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:17:05] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T22:17:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:05.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:05.881485+0000) 2023-08-27T22:17:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:06 smithi136 ceph-mon[113531]: pgmap v2507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1902032013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:06.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:06.881728+0000) 2023-08-27T22:17:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:06 smithi093 ceph-mon[111335]: pgmap v2507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1902032013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:06 smithi093 ceph-mon[118902]: pgmap v2507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1902032013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:07.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:07.881974+0000) 2023-08-27T22:17:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:08 smithi136 ceph-mon[113531]: pgmap v2508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1434038704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:08.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:08.882184+0000) 2023-08-27T22:17:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:08 smithi093 ceph-mon[118902]: pgmap v2508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1434038704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:08 smithi093 ceph-mon[111335]: pgmap v2508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1434038704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:09.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:09.882395+0000) 2023-08-27T22:17:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:10 smithi136 ceph-mon[113531]: pgmap v2509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:10 smithi093 ceph-mon[111335]: pgmap v2509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:10 smithi093 ceph-mon[118902]: pgmap v2509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:10.440Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:10.441Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:10.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:10.882668+0000) 2023-08-27T22:17:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3783079095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3783079095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3783079095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:11.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:11.882953+0000) 2023-08-27T22:17:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:11.962Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:11.963Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:12 smithi136 ceph-mon[113531]: pgmap v2510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:12 smithi093 ceph-mon[111335]: pgmap v2510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:12 smithi093 ceph-mon[118902]: pgmap v2510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:12.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:12.883193+0000) 2023-08-27T22:17:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4114741089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4114741089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4114741089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:13.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:13.883453+0000) 2023-08-27T22:17:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:13.869Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:13.870Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:14 smithi136 ceph-mon[113531]: pgmap v2511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:14 smithi093 ceph-mon[111335]: pgmap v2511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:14 smithi093 ceph-mon[118902]: pgmap v2511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:14.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:14.883773+0000) 2023-08-27T22:17:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1638024679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1638024679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:17:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:17:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:17:15.884 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1638024679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:15.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:15.884060+0000) 2023-08-27T22:17:16.884 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:16 smithi136 ceph-mon[113531]: pgmap v2512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:16 smithi093 ceph-mon[118902]: pgmap v2512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:16 smithi093 ceph-mon[111335]: pgmap v2512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:16.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:16.884307+0000) 2023-08-27T22:17:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:17.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:17.884570+0000) 2023-08-27T22:17:18.884 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:18 smithi136 ceph-mon[113531]: pgmap v2513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:18.885 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4074430792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:18 smithi093 ceph-mon[111335]: pgmap v2513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4074430792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:18 smithi093 ceph-mon[118902]: pgmap v2513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4074430792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:18.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:18.884838+0000) 2023-08-27T22:17:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:19.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:19.885000+0000) 2023-08-27T22:17:20.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:20 smithi136 ceph-mon[113531]: pgmap v2514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:20.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:17:20.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3268270541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-mon[111335]: pgmap v2514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:17:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3268270541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-mon[118902]: pgmap v2514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:17:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3268270541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:20.441Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:20.442Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:20.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:20.885256+0000) 2023-08-27T22:17:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:21.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:21.885445+0000) 2023-08-27T22:17:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:21.963Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:21.963Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[111335]: pgmap v2515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[118902]: pgmap v2515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.885 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:22 smithi136 ceph-mon[113531]: pgmap v2515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:22.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:22.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:22.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:22.885678+0000) 2023-08-27T22:17:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:17:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:17:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1544769686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:17:23.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:17:23.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:23.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1544769686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:23.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:17:23.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:17:23.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:17:23.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1544769686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:23.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:23.885944+0000) 2023-08-27T22:17:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:23.870Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:23.870Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:24 smithi093 ceph-mon[111335]: pgmap v2516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:24 smithi093 ceph-mon[118902]: pgmap v2516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:24.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:24 smithi136 ceph-mon[113531]: pgmap v2516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:24.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:24.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:24.886209+0000) 2023-08-27T22:17:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1176328169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1176328169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:17:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:17:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:17:25.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1176328169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:25.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:25.886430+0000) 2023-08-27T22:17:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:26 smithi093 ceph-mon[118902]: pgmap v2517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:26 smithi093 ceph-mon[111335]: pgmap v2517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:26.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:26 smithi136 ceph-mon[113531]: pgmap v2517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:26.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:26.886663+0000) 2023-08-27T22:17:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/766923822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/766923822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:27.886 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/766923822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:27.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:27.886856+0000) 2023-08-27T22:17:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:28 smithi093 ceph-mon[111335]: pgmap v2518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:28 smithi093 ceph-mon[118902]: pgmap v2518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:28.887 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:28 smithi136 ceph-mon[113531]: pgmap v2518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:28.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:28.887076+0000) 2023-08-27T22:17:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:29.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:29.887347+0000) 2023-08-27T22:17:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:30 smithi136 ceph-mon[113531]: pgmap v2519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1065312639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:30 smithi093 ceph-mon[111335]: pgmap v2519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1065312639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:30 smithi093 ceph-mon[118902]: pgmap v2519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1065312639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:30.442Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:30.443Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:30.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:30.887598+0000) 2023-08-27T22:17:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:31.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:31.887799+0000) 2023-08-27T22:17:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:31.963Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:31.964Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:32.888 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:32 smithi136 ceph-mon[113531]: pgmap v2520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:32.888 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3861434833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:32 smithi093 ceph-mon[111335]: pgmap v2520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3861434833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:32 smithi093 ceph-mon[118902]: pgmap v2520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3861434833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:32.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:32.888078+0000) 2023-08-27T22:17:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:33.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:33.888360+0000) 2023-08-27T22:17:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:33.871Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:33.872Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:34.888 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:34 smithi136 ceph-mon[113531]: pgmap v2521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:34.889 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2169799342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:34 smithi093 ceph-mon[111335]: pgmap v2521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2169799342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:34 smithi093 ceph-mon[118902]: pgmap v2521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2169799342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:34.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:34.888618+0000) 2023-08-27T22:17:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:17:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:17:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:17:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:35.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:35.888872+0000) 2023-08-27T22:17:36.889 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:36 smithi136 ceph-mon[113531]: pgmap v2522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:36 smithi093 ceph-mon[111335]: pgmap v2522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:36 smithi093 ceph-mon[118902]: pgmap v2522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:36.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:36.889120+0000) 2023-08-27T22:17:37.889 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1840912373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1840912373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1840912373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:37.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:37.889373+0000) 2023-08-27T22:17:38.890 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:38 smithi136 ceph-mon[113531]: pgmap v2523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:38 smithi093 ceph-mon[111335]: pgmap v2523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:38 smithi093 ceph-mon[118902]: pgmap v2523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:38.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:38.889644+0000) 2023-08-27T22:17:39.890 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:39.890 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1508831133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1508831133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1508831133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:39.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:39.889901+0000) 2023-08-27T22:17:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:40 smithi093 ceph-mon[111335]: pgmap v2524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:40 smithi093 ceph-mon[118902]: pgmap v2524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:40.442Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:40.443Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:40.890 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:40 smithi136 ceph-mon[113531]: pgmap v2524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:40.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:40.890159+0000) 2023-08-27T22:17:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:41.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:41.890426+0000) 2023-08-27T22:17:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:41.965Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:41.965Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:42.890 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:42 smithi136 ceph-mon[113531]: pgmap v2525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:42.891 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4139035911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:42 smithi093 ceph-mon[111335]: pgmap v2525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4139035911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:42 smithi093 ceph-mon[118902]: pgmap v2525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4139035911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:42.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:42.890683+0000) 2023-08-27T22:17:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:43.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:43.890943+0000) 2023-08-27T22:17:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:43.871Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:43.872Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:44 smithi136 ceph-mon[113531]: pgmap v2526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/58404881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:44.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:44.891202+0000) 2023-08-27T22:17:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:44 smithi093 ceph-mon[111335]: pgmap v2526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/58404881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:44 smithi093 ceph-mon[118902]: pgmap v2526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/58404881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:17:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:17:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:17:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:45.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:45.891450+0000) 2023-08-27T22:17:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:46 smithi136 ceph-mon[113531]: pgmap v2527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3749486206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:46.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:46.891632+0000) 2023-08-27T22:17:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:46 smithi093 ceph-mon[111335]: pgmap v2527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3749486206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:46 smithi093 ceph-mon[118902]: pgmap v2527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3749486206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:47.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:47.891881+0000) 2023-08-27T22:17:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:48 smithi136 ceph-mon[113531]: pgmap v2528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:48.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:48.892159+0000) 2023-08-27T22:17:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:48 smithi093 ceph-mon[111335]: pgmap v2528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:48 smithi093 ceph-mon[118902]: pgmap v2528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4133030266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:49.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:49.892414+0000) 2023-08-27T22:17:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4133030266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4133030266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:50 smithi093 ceph-mon[111335]: pgmap v2529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:50 smithi093 ceph-mon[118902]: pgmap v2529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:50.443Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:50.444Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:50 smithi136 ceph-mon[113531]: pgmap v2529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:50.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:50.892668+0000) 2023-08-27T22:17:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4157258255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:51.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:51.892917+0000) 2023-08-27T22:17:51.965 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4157258255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:51.966 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4157258255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:51.965Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:51.966Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:52 smithi136 ceph-mon[113531]: pgmap v2530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:52.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:52.893150+0000) 2023-08-27T22:17:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:52 smithi093 ceph-mon[111335]: pgmap v2530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:52 smithi093 ceph-mon[118902]: pgmap v2530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2493928011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:53.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:53.893449+0000) 2023-08-27T22:17:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2493928011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2493928011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:53.871Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:17:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:17:53.872Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:17:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:54 smithi136 ceph-mon[113531]: pgmap v2531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:54.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:54.893704+0000) 2023-08-27T22:17:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:54 smithi093 ceph-mon[111335]: pgmap v2531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:54 smithi093 ceph-mon[118902]: pgmap v2531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:17:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:17:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:17:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:17:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:55.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:55.893942+0000) 2023-08-27T22:17:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:56 smithi136 ceph-mon[113531]: pgmap v2532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1035690977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:56.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:56.894195+0000) 2023-08-27T22:17:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:56 smithi093 ceph-mon[111335]: pgmap v2532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1035690977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:56 smithi093 ceph-mon[118902]: pgmap v2532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1035690977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:57.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:57.894452+0000) 2023-08-27T22:17:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:58.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:58.894679+0000) 2023-08-27T22:17:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:58 smithi136 ceph-mon[113531]: pgmap v2533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:17:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/183589108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:58 smithi093 ceph-mon[111335]: pgmap v2533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:17:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/183589108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:17:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:58 smithi093 ceph-mon[118902]: pgmap v2533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:17:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:17:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/183589108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:17:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:17:59.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:17:59.894954+0000) 2023-08-27T22:18:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:00 smithi136 ceph-mon[113531]: pgmap v2534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:00 smithi093 ceph-mon[111335]: pgmap v2534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:00 smithi093 ceph-mon[118902]: pgmap v2534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:00.444Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:00.445Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:00.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:00.895178+0000) 2023-08-27T22:18:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/335103709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/335103709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/335103709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:01.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:01.895441+0000) 2023-08-27T22:18:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:01.966Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:01.967Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:02 smithi136 ceph-mon[113531]: pgmap v2535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:02 smithi093 ceph-mon[111335]: pgmap v2535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:02 smithi093 ceph-mon[118902]: pgmap v2535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:02.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:02.895649+0000) 2023-08-27T22:18:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3689285070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3689285070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3689285070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:03.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:03.895927+0000) 2023-08-27T22:18:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:03.872Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:03.873Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:04 smithi136 ceph-mon[113531]: pgmap v2536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:04 smithi093 ceph-mon[111335]: pgmap v2536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:04 smithi093 ceph-mon[118902]: pgmap v2536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:04.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:04.896211+0000) 2023-08-27T22:18:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2084349446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:18:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:18:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:18:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2084349446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:05.896 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2084349446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:05.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:05.896445+0000) 2023-08-27T22:18:06.896 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:06 smithi136 ceph-mon[113531]: pgmap v2537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:06 smithi093 ceph-mon[111335]: pgmap v2537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:06 smithi093 ceph-mon[118902]: pgmap v2537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:06.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:06.896681+0000) 2023-08-27T22:18:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:07.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:07.896928+0000) 2023-08-27T22:18:08.897 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:08 smithi136 ceph-mon[113531]: pgmap v2538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:08.897 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3841213686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:08 smithi093 ceph-mon[111335]: pgmap v2538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3841213686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:08 smithi093 ceph-mon[118902]: pgmap v2538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3841213686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:08.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:08.897128+0000) 2023-08-27T22:18:09.897 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:09.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:09.897350+0000) 2023-08-27T22:18:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:10 smithi093 ceph-mon[111335]: pgmap v2539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3447046873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:10 smithi093 ceph-mon[118902]: pgmap v2539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3447046873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:10.445Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:10.446Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:10.897 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:10 smithi136 ceph-mon[113531]: pgmap v2539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:10.897 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3447046873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:10.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:10.897571+0000) 2023-08-27T22:18:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:11.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:11.897818+0000) 2023-08-27T22:18:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:11.967Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:11.968Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:12.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:12 smithi136 ceph-mon[113531]: pgmap v2540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:12 smithi093 ceph-mon[111335]: pgmap v2540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:12 smithi093 ceph-mon[118902]: pgmap v2540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:12.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:12.898050+0000) 2023-08-27T22:18:13.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4168522647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4168522647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4168522647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:13.872Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:13.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:13.873Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:13.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:13.898247+0000) 2023-08-27T22:18:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:14 smithi136 ceph-mon[113531]: pgmap v2541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:14.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:14.898559+0000) 2023-08-27T22:18:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:14 smithi093 ceph-mon[111335]: pgmap v2541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:14 smithi093 ceph-mon[118902]: pgmap v2541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:15.653 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:18:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:18:15] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:18:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2294825473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:15.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:15.898834+0000) 2023-08-27T22:18:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2294825473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2294825473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:16 smithi136 ceph-mon[113531]: pgmap v2542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:16.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:16.899108+0000) 2023-08-27T22:18:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:16 smithi093 ceph-mon[111335]: pgmap v2542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:16 smithi093 ceph-mon[118902]: pgmap v2542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1121428953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:17.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:17.899393+0000) 2023-08-27T22:18:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1121428953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1121428953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:18 smithi136 ceph-mon[113531]: pgmap v2543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:18.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:18.899707+0000) 2023-08-27T22:18:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:18 smithi093 ceph-mon[111335]: pgmap v2543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:18 smithi093 ceph-mon[118902]: pgmap v2543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:19.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:19.899935+0000) 2023-08-27T22:18:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:20 smithi136 ceph-mon[113531]: pgmap v2544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3459404470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:20 smithi093 ceph-mon[111335]: pgmap v2544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3459404470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:20 smithi093 ceph-mon[118902]: pgmap v2544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3459404470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:20.445Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:20.446Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:20.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:20.900192+0000) 2023-08-27T22:18:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:21.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:21.900416+0000) 2023-08-27T22:18:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:21.967Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:21.968Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:22.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:22 smithi136 ceph-mon[113531]: pgmap v2545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:22.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3207378519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:22.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:18:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:22 smithi093 ceph-mon[111335]: pgmap v2545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3207378519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:18:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:22 smithi093 ceph-mon[118902]: pgmap v2545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3207378519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:18:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:22.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:22.900591+0000) 2023-08-27T22:18:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:23.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:23.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:23.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:23.900746+0000) 2023-08-27T22:18:24.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:24 smithi136 ceph-mon[113531]: pgmap v2546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:24.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:24.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:24.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:24.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1267039174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[111335]: pgmap v2546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1267039174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[118902]: pgmap v2546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1267039174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:24.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:24.900997+0000) 2023-08-27T22:18:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:18:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:18:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:18:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:25.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:25.901285+0000) 2023-08-27T22:18:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[111335]: pgmap v2547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[118902]: pgmap v2547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:26.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:18:26.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:18:26.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:26 smithi136 ceph-mon[113531]: pgmap v2547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:18:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:26.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:26.901470+0000) 2023-08-27T22:18:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1369680535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1369680535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1369680535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:27.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:27.901710+0000) 2023-08-27T22:18:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:28 smithi136 ceph-mon[113531]: pgmap v2548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:28 smithi093 ceph-mon[111335]: pgmap v2548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:28 smithi093 ceph-mon[118902]: pgmap v2548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:28.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:28.901970+0000) 2023-08-27T22:18:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:29.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:29.902233+0000) 2023-08-27T22:18:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2550933840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:30 smithi136 ceph-mon[113531]: pgmap v2549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2550933840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:30 smithi093 ceph-mon[111335]: pgmap v2549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2550933840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:30 smithi093 ceph-mon[118902]: pgmap v2549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:30.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:30.446Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:30.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:30.447Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:30.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:30.902458+0000) 2023-08-27T22:18:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:31.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:31.902655+0000) 2023-08-27T22:18:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:31.968Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:31.968Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:32.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:32 smithi136 ceph-mon[113531]: pgmap v2550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:32.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2654407301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:32 smithi093 ceph-mon[118902]: pgmap v2550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2654407301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:32 smithi093 ceph-mon[111335]: pgmap v2550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2654407301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:32.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:32.902940+0000) 2023-08-27T22:18:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:33.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:33.903186+0000) 2023-08-27T22:18:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:33.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:34.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:33.875Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:34.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:34 smithi136 ceph-mon[113531]: pgmap v2551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:34.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1038891513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:34 smithi093 ceph-mon[111335]: pgmap v2551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1038891513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:34 smithi093 ceph-mon[118902]: pgmap v2551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1038891513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:34.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:34.903381+0000) 2023-08-27T22:18:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:18:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:18:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:18:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:35.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:35.903600+0000) 2023-08-27T22:18:36.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:36 smithi136 ceph-mon[113531]: pgmap v2552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:36.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1523243123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:36 smithi093 ceph-mon[111335]: pgmap v2552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1523243123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:36 smithi093 ceph-mon[118902]: pgmap v2552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1523243123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:36.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:36.903837+0000) 2023-08-27T22:18:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:37.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:37.904064+0000) 2023-08-27T22:18:38.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:38 smithi136 ceph-mon[113531]: pgmap v2553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:38 smithi093 ceph-mon[118902]: pgmap v2553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:38 smithi093 ceph-mon[111335]: pgmap v2553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:38.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:38.904335+0000) 2023-08-27T22:18:39.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:39.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1384830954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1384830954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1384830954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:39.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:39.904573+0000) 2023-08-27T22:18:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:40 smithi093 ceph-mon[111335]: pgmap v2554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:40 smithi093 ceph-mon[118902]: pgmap v2554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:40.447Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:40.448Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:40.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:40 smithi136 ceph-mon[113531]: pgmap v2554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:40.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:40.904815+0000) 2023-08-27T22:18:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2779011076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:41.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:41.905096+0000) 2023-08-27T22:18:41.968 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2779011076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:41.968 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2779011076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:41.968Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:41.969Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:42 smithi136 ceph-mon[113531]: pgmap v2555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:42.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:42.905345+0000) 2023-08-27T22:18:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:42 smithi093 ceph-mon[111335]: pgmap v2555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:42 smithi093 ceph-mon[118902]: pgmap v2555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:43.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:43.905570+0000) 2023-08-27T22:18:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:43.875Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:43.876Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:44 smithi136 ceph-mon[113531]: pgmap v2556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3376936590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:44.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:44.905859+0000) 2023-08-27T22:18:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:44 smithi093 ceph-mon[111335]: pgmap v2556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3376936590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:44 smithi093 ceph-mon[118902]: pgmap v2556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3376936590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:18:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:18:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:18:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:45.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:45.906154+0000) 2023-08-27T22:18:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:46 smithi136 ceph-mon[113531]: pgmap v2557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3699791170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:46.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:46.906364+0000) 2023-08-27T22:18:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:46 smithi093 ceph-mon[111335]: pgmap v2557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3699791170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:46 smithi093 ceph-mon[118902]: pgmap v2557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3699791170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:47.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:47.906616+0000) 2023-08-27T22:18:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:48 smithi136 ceph-mon[113531]: pgmap v2558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3577267899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:48.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:48.906862+0000) 2023-08-27T22:18:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:48 smithi093 ceph-mon[118902]: pgmap v2558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3577267899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:48 smithi093 ceph-mon[111335]: pgmap v2558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3577267899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:50.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:49.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:49.907067+0000) 2023-08-27T22:18:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:50 smithi136 ceph-mon[113531]: pgmap v2559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:50 smithi093 ceph-mon[111335]: pgmap v2559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:50 smithi093 ceph-mon[118902]: pgmap v2559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:50.447Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:50.448Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:50.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:50.907295+0000) 2023-08-27T22:18:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3419563682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3419563682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3419563682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:51.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:51.907477+0000) 2023-08-27T22:18:52.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:51.969Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:52.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:51.970Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:52 smithi136 ceph-mon[113531]: pgmap v2560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:52 smithi093 ceph-mon[111335]: pgmap v2560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:52 smithi093 ceph-mon[118902]: pgmap v2560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:52.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:52.907734+0000) 2023-08-27T22:18:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2786571529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2786571529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2786571529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:53.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:53.907997+0000) 2023-08-27T22:18:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:53.875Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:18:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:18:53.876Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:18:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:54 smithi136 ceph-mon[113531]: pgmap v2561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:54 smithi093 ceph-mon[118902]: pgmap v2561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:54 smithi093 ceph-mon[111335]: pgmap v2561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:18:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:54.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:54.908268+0000) 2023-08-27T22:18:55.662 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:18:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:18:55] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:18:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3780050004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:55.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:55.908517+0000) 2023-08-27T22:18:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3780050004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3780050004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:56 smithi136 ceph-mon[113531]: pgmap v2562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:56.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:56.908770+0000) 2023-08-27T22:18:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:56 smithi093 ceph-mon[111335]: pgmap v2562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:56 smithi093 ceph-mon[118902]: pgmap v2562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:57.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:57.908987+0000) 2023-08-27T22:18:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:58 smithi136 ceph-mon[113531]: pgmap v2563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:18:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3980761653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:58.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:58.909224+0000) 2023-08-27T22:18:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:58 smithi093 ceph-mon[111335]: pgmap v2563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:18:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3980761653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:18:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:58 smithi093 ceph-mon[118902]: pgmap v2563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:18:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:18:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3980761653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:18:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:18:59.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:18:59.909470+0000) 2023-08-27T22:19:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:00 smithi136 ceph-mon[113531]: pgmap v2564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:00 smithi093 ceph-mon[111335]: pgmap v2564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:00 smithi093 ceph-mon[118902]: pgmap v2564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:00.448Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:00.449Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:00.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:00.909743+0000) 2023-08-27T22:19:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1347134157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1347134157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1347134157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:01.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:01.909948+0000) 2023-08-27T22:19:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:01.970Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:01.970Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:02 smithi136 ceph-mon[113531]: pgmap v2565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:02 smithi093 ceph-mon[111335]: pgmap v2565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:02 smithi093 ceph-mon[118902]: pgmap v2565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:02.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:02.910196+0000) 2023-08-27T22:19:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/335353771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/335353771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/335353771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:03.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:03.910421+0000) 2023-08-27T22:19:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:03.876Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:03.877Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:04 smithi136 ceph-mon[113531]: pgmap v2566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:04 smithi093 ceph-mon[111335]: pgmap v2566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:04 smithi093 ceph-mon[118902]: pgmap v2566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:04.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:04.910669+0000) 2023-08-27T22:19:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3417158730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3417158730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:19:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:19:05] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:19:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3417158730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:05.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:05.910913+0000) 2023-08-27T22:19:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:06 smithi136 ceph-mon[113531]: pgmap v2567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:06 smithi093 ceph-mon[111335]: pgmap v2567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:06 smithi093 ceph-mon[118902]: pgmap v2567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:06.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:06.911182+0000) 2023-08-27T22:19:07.911 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3117494936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3117494936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3117494936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:07.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:07.911433+0000) 2023-08-27T22:19:08.911 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:08 smithi136 ceph-mon[113531]: pgmap v2568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:08 smithi093 ceph-mon[118902]: pgmap v2568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:08 smithi093 ceph-mon[111335]: pgmap v2568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:08.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:08.911694+0000) 2023-08-27T22:19:09.912 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:09.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:09.911968+0000) 2023-08-27T22:19:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:10 smithi093 ceph-mon[118902]: pgmap v2569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2745169253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:10 smithi093 ceph-mon[111335]: pgmap v2569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2745169253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:10.449Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:10.450Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:10.912 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:10 smithi136 ceph-mon[113531]: pgmap v2569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:10.912 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2745169253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:10.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:10.912227+0000) 2023-08-27T22:19:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:11.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:11.912473+0000) 2023-08-27T22:19:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:11.970Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:11.971Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:12.912 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:12 smithi136 ceph-mon[113531]: pgmap v2570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:12.912 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/296889569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:12 smithi093 ceph-mon[111335]: pgmap v2570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/296889569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:12 smithi093 ceph-mon[118902]: pgmap v2570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/296889569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:12.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:12.912726+0000) 2023-08-27T22:19:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:13.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:13.912973+0000) 2023-08-27T22:19:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:13.876Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:13.877Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:14.913 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:14 smithi136 ceph-mon[113531]: pgmap v2571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:14 smithi093 ceph-mon[111335]: pgmap v2571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:14 smithi093 ceph-mon[118902]: pgmap v2571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:14.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:14.913237+0000) 2023-08-27T22:19:15.652 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:19:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:19:15] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:19:15.913 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/939387282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/939387282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/939387282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:15.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:15.913439+0000) 2023-08-27T22:19:16.913 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:16 smithi136 ceph-mon[113531]: pgmap v2572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:16 smithi093 ceph-mon[111335]: pgmap v2572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:16 smithi093 ceph-mon[118902]: pgmap v2572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:16.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:16.913674+0000) 2023-08-27T22:19:17.913 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/672249696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/672249696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/672249696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:17.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:17.913913+0000) 2023-08-27T22:19:18.914 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:18 smithi136 ceph-mon[113531]: pgmap v2573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:18 smithi093 ceph-mon[111335]: pgmap v2573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:18 smithi093 ceph-mon[118902]: pgmap v2573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:18.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:18.914168+0000) 2023-08-27T22:19:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:19.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:19.914460+0000) 2023-08-27T22:19:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2342601490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:20 smithi136 ceph-mon[113531]: pgmap v2574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2342601490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:20 smithi093 ceph-mon[111335]: pgmap v2574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2342601490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:20 smithi093 ceph-mon[118902]: pgmap v2574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:20.450Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:20.451Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:20.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:20.914741+0000) 2023-08-27T22:19:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:21.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:21.914985+0000) 2023-08-27T22:19:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:21.970Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:21.971Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:22.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:22 smithi136 ceph-mon[113531]: pgmap v2575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:22.916 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/115162342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:22 smithi093 ceph-mon[111335]: pgmap v2575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/115162342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:22 smithi093 ceph-mon[118902]: pgmap v2575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/115162342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:22.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:22.915286+0000) 2023-08-27T22:19:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:23.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:23.915527+0000) 2023-08-27T22:19:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:23.877Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:23.878Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:24.915 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:24 smithi136 ceph-mon[113531]: pgmap v2576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:24.916 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:24.916 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3619023288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:24 smithi093 ceph-mon[111335]: pgmap v2576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3619023288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:24 smithi093 ceph-mon[118902]: pgmap v2576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3619023288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:24.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:24.915819+0000) 2023-08-27T22:19:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:19:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:19:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:19:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:25.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:25.916055+0000) 2023-08-27T22:19:26.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:26 smithi136 ceph-mon[113531]: pgmap v2577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:26.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:19:26.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3343350518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:26 smithi093 ceph-mon[118902]: pgmap v2577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:19:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3343350518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:26 smithi093 ceph-mon[111335]: pgmap v2577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:19:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3343350518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:26.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:26.916277+0000) 2023-08-27T22:19:28.009 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:27.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:27.916457+0000) 2023-08-27T22:19:28.916 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:28 smithi136 ceph-mon[113531]: pgmap v2578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:28.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[111335]: pgmap v2578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[118902]: pgmap v2578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:28.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:28.916623+0000) 2023-08-27T22:19:29.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:19:29.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:19:29.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:29.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2814267719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:19:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:19:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2814267719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:19:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:19:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:19:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2814267719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:29.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:29.916885+0000) 2023-08-27T22:19:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:30 smithi093 ceph-mon[111335]: pgmap v2579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:30 smithi093 ceph-mon[118902]: pgmap v2579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:30.451Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:30.452Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:30.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:30 smithi136 ceph-mon[113531]: pgmap v2579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:30.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:30.917154+0000) 2023-08-27T22:19:31.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/872501966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:31.971 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/872501966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:31.972 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/872501966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:31.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:31.917338+0000) 2023-08-27T22:19:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:31.971Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:31.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:32.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:32 smithi136 ceph-mon[113531]: pgmap v2580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:32 smithi093 ceph-mon[111335]: pgmap v2580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:32 smithi093 ceph-mon[118902]: pgmap v2580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:32.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:32.917600+0000) 2023-08-27T22:19:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:33.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:33.917897+0000) 2023-08-27T22:19:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:33.877Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:33.878Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:34.918 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:34 smithi136 ceph-mon[113531]: pgmap v2581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:34.918 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3756041421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:34 smithi093 ceph-mon[118902]: pgmap v2581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3756041421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:34 smithi093 ceph-mon[111335]: pgmap v2581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3756041421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:34.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:34.918161+0000) 2023-08-27T22:19:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:19:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:19:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:19:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:35.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:35.918365+0000) 2023-08-27T22:19:36.918 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:36 smithi136 ceph-mon[113531]: pgmap v2582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:36.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3543814222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:36 smithi093 ceph-mon[111335]: pgmap v2582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3543814222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:36 smithi093 ceph-mon[118902]: pgmap v2582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3543814222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:36.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:36.918597+0000) 2023-08-27T22:19:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:37.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:37.918877+0000) 2023-08-27T22:19:38.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:38 smithi136 ceph-mon[113531]: pgmap v2583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:38.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1863111968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:38 smithi093 ceph-mon[111335]: pgmap v2583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1863111968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:38 smithi093 ceph-mon[118902]: pgmap v2583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1863111968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:38.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:38.919083+0000) 2023-08-27T22:19:39.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:39.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:39.919315+0000) 2023-08-27T22:19:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:40 smithi093 ceph-mon[111335]: pgmap v2584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:40 smithi093 ceph-mon[118902]: pgmap v2584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:40.452Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:40.453Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:40.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:40 smithi136 ceph-mon[113531]: pgmap v2584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:40.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:40.919570+0000) 2023-08-27T22:19:41.920 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2123301774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:41.973 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2123301774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:41.973 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2123301774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:41.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:41.919859+0000) 2023-08-27T22:19:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:41.972Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:41.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:42.920 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:42 smithi136 ceph-mon[113531]: pgmap v2585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:42 smithi093 ceph-mon[111335]: pgmap v2585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:42 smithi093 ceph-mon[118902]: pgmap v2585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:42.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:42.920167+0000) 2023-08-27T22:19:43.920 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2019641043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2019641043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2019641043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:43.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:43.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:43.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:43.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:43.920384+0000) 2023-08-27T22:19:44.921 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:44 smithi136 ceph-mon[113531]: pgmap v2586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:44 smithi093 ceph-mon[111335]: pgmap v2586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:44 smithi093 ceph-mon[118902]: pgmap v2586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:44.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:44.920666+0000) 2023-08-27T22:19:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:19:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:19:45] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T22:19:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:45.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:45.920923+0000) 2023-08-27T22:19:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:46 smithi136 ceph-mon[113531]: pgmap v2587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1567661967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:46.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:46.921123+0000) 2023-08-27T22:19:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:46 smithi093 ceph-mon[111335]: pgmap v2587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1567661967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:46 smithi093 ceph-mon[118902]: pgmap v2587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1567661967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:47.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:47.921369+0000) 2023-08-27T22:19:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:48 smithi136 ceph-mon[113531]: pgmap v2588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2735412977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:48.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:48.921614+0000) 2023-08-27T22:19:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:48 smithi093 ceph-mon[111335]: pgmap v2588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2735412977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:48 smithi093 ceph-mon[118902]: pgmap v2588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2735412977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:49.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:49.921829+0000) 2023-08-27T22:19:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:50 smithi136 ceph-mon[113531]: pgmap v2589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:50 smithi093 ceph-mon[111335]: pgmap v2589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:50 smithi093 ceph-mon[118902]: pgmap v2589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:50.452Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:50.453Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:50.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:50.922088+0000) 2023-08-27T22:19:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1093817003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1093817003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1093817003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:51.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:51.922324+0000) 2023-08-27T22:19:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:51.973Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:51.974Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:52 smithi136 ceph-mon[113531]: pgmap v2590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:52 smithi093 ceph-mon[111335]: pgmap v2590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:52 smithi093 ceph-mon[118902]: pgmap v2590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:52.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:52.922547+0000) 2023-08-27T22:19:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/982267841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/982267841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/982267841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:53.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:53.922765+0000) 2023-08-27T22:19:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:53.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:19:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:19:53.880Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:19:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:54 smithi136 ceph-mon[113531]: pgmap v2591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:54 smithi093 ceph-mon[111335]: pgmap v2591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:54 smithi093 ceph-mon[118902]: pgmap v2591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:19:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:54.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:54.922998+0000) 2023-08-27T22:19:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/63842835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/63842835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/63842835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:19:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:19:55] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T22:19:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:55.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:55.923243+0000) 2023-08-27T22:19:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:56 smithi136 ceph-mon[113531]: pgmap v2592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:56 smithi093 ceph-mon[111335]: pgmap v2592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:56 smithi093 ceph-mon[118902]: pgmap v2592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:56.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:56.923475+0000) 2023-08-27T22:19:57.923 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/528924043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/528924043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/528924043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:19:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:57.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:57.923735+0000) 2023-08-27T22:19:58.924 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:19:58 smithi136 ceph-mon[113531]: pgmap v2593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:19:58 smithi093 ceph-mon[111335]: pgmap v2593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:19:58 smithi093 ceph-mon[118902]: pgmap v2593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:19:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:58.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:58.923980+0000) 2023-08-27T22:20:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:19:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:19:59.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:19:59.924245+0000) 2023-08-27T22:20:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:00 smithi136 ceph-mon[113531]: pgmap v2594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3485410204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:20:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-mon[111335]: pgmap v2594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3485410204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:20:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:00.453Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:00.454Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-mon[118902]: pgmap v2594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3485410204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:20:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:00.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:00.924527+0000) 2023-08-27T22:20:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:01.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:01.924775+0000) 2023-08-27T22:20:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:01.974Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:01.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:02.925 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:02 smithi136 ceph-mon[113531]: pgmap v2595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:02.925 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3172515986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:02 smithi093 ceph-mon[111335]: pgmap v2595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3172515986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:02 smithi093 ceph-mon[118902]: pgmap v2595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3172515986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:02.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:02.925067+0000) 2023-08-27T22:20:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:03.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:03.925299+0000) 2023-08-27T22:20:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:03.879Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:03.881Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:04.925 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:04 smithi136 ceph-mon[113531]: pgmap v2596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:04 smithi093 ceph-mon[111335]: pgmap v2596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:04 smithi093 ceph-mon[118902]: pgmap v2596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:04.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:04.925576+0000) 2023-08-27T22:20:05.664 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:20:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:20:05] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T22:20:05.926 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2817057154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2817057154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2817057154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:05.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:05.925848+0000) 2023-08-27T22:20:06.926 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:06 smithi136 ceph-mon[113531]: pgmap v2597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:06 smithi093 ceph-mon[111335]: pgmap v2597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:06 smithi093 ceph-mon[118902]: pgmap v2597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:06.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:06.926100+0000) 2023-08-27T22:20:07.926 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4179263177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4179263177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4179263177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:07.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:07.926307+0000) 2023-08-27T22:20:08.926 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:08 smithi136 ceph-mon[113531]: pgmap v2598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:08 smithi093 ceph-mon[111335]: pgmap v2598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:08 smithi093 ceph-mon[118902]: pgmap v2598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:08.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:08.926531+0000) 2023-08-27T22:20:09.926 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:09.927 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1635007291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1635007291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1635007291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:09.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:09.926786+0000) 2023-08-27T22:20:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:10 smithi093 ceph-mon[111335]: pgmap v2599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:10 smithi093 ceph-mon[118902]: pgmap v2599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:10.454Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:10.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:10.927 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:10 smithi136 ceph-mon[113531]: pgmap v2599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:10.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:10.927027+0000) 2023-08-27T22:20:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:11.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:11.927264+0000) 2023-08-27T22:20:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:11.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:11.976Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:12 smithi136 ceph-mon[113531]: pgmap v2600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1597506253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:12.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:12.927441+0000) 2023-08-27T22:20:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:12 smithi093 ceph-mon[111335]: pgmap v2600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1597506253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:12 smithi093 ceph-mon[118902]: pgmap v2600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1597506253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:13.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:13.927643+0000) 2023-08-27T22:20:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:13.880Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:13.881Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:14 smithi136 ceph-mon[113531]: pgmap v2601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3302036562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:14.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:14.927906+0000) 2023-08-27T22:20:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:14 smithi093 ceph-mon[111335]: pgmap v2601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3302036562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:14 smithi093 ceph-mon[118902]: pgmap v2601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3302036562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:20:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:20:15] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T22:20:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:15.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:15.928190+0000) 2023-08-27T22:20:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:16 smithi136 ceph-mon[113531]: pgmap v2602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1710514975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:16.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:16.928403+0000) 2023-08-27T22:20:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:16 smithi093 ceph-mon[111335]: pgmap v2602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1710514975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:16 smithi093 ceph-mon[118902]: pgmap v2602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1710514975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:17.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:17.928661+0000) 2023-08-27T22:20:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:18 smithi136 ceph-mon[113531]: pgmap v2603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:18.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:18.928915+0000) 2023-08-27T22:20:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:18 smithi093 ceph-mon[111335]: pgmap v2603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:18 smithi093 ceph-mon[118902]: pgmap v2603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/532329092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:19.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:19.929161+0000) 2023-08-27T22:20:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/532329092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/532329092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:20 smithi093 ceph-mon[111335]: pgmap v2604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:20 smithi093 ceph-mon[118902]: pgmap v2604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:20.454Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:20.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:20 smithi136 ceph-mon[113531]: pgmap v2604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:20.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:20.929407+0000) 2023-08-27T22:20:21.975 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4138267109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:21.976 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4138267109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4138267109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:21.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:21.929660+0000) 2023-08-27T22:20:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:21.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:21.976Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:22 smithi093 ceph-mon[111335]: pgmap v2605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:22 smithi093 ceph-mon[118902]: pgmap v2605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:22 smithi136 ceph-mon[113531]: pgmap v2605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:22.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:22.929922+0000) 2023-08-27T22:20:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:23.881Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:23.882Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:23.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:23.930165+0000) 2023-08-27T22:20:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:24 smithi093 ceph-mon[111335]: pgmap v2606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/465887416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:24 smithi093 ceph-mon[118902]: pgmap v2606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/465887416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:24 smithi136 ceph-mon[113531]: pgmap v2606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/465887416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:24.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:24.930396+0000) 2023-08-27T22:20:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:20:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:20:25] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T22:20:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:25.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:25.930665+0000) 2023-08-27T22:20:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:26 smithi093 ceph-mon[111335]: pgmap v2607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3088588189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:26 smithi093 ceph-mon[118902]: pgmap v2607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3088588189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:26 smithi136 ceph-mon[113531]: pgmap v2607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3088588189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:26.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:26.930868+0000) 2023-08-27T22:20:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:27.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:27.931131+0000) 2023-08-27T22:20:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:28 smithi093 ceph-mon[111335]: pgmap v2608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2235319558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:28 smithi093 ceph-mon[118902]: pgmap v2608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2235319558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:28.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:28.931294+0000) 2023-08-27T22:20:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:28 smithi136 ceph-mon[113531]: pgmap v2608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2235319558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:29.711 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:20:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:20:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:20:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:29.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:29.931511+0000) 2023-08-27T22:20:30.712 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-mon[118902]: pgmap v2609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:30.712 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:30.712 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:30.713 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:30.455Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:30.713 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:30.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-mon[111335]: pgmap v2609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:31.100 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:30 smithi136 ceph-mon[113531]: pgmap v2609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:31.100 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:31.100 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:31.100 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:30.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:30.931694+0000) 2023-08-27T22:20:31.977 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4091400711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:31.978 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:31.978 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:31.978 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4091400711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:31.978 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:31.978 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4091400711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:31.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:31.931873+0000) 2023-08-27T22:20:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:31.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:31.977Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:32 smithi093 ceph-mon[111335]: pgmap v2610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:32 smithi093 ceph-mon[118902]: pgmap v2610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:32 smithi136 ceph-mon[113531]: pgmap v2610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:32.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:32.932152+0000) 2023-08-27T22:20:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:20:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:20:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3188079152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:33.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:33.932372+0000) 2023-08-27T22:20:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:20:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:20:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3188079152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:20:34.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:20:34.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:20:34.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3188079152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:34.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:33.881Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:34.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:33.882Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:34 smithi136 ceph-mon[113531]: pgmap v2611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:34.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:34.932648+0000) 2023-08-27T22:20:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:34 smithi093 ceph-mon[111335]: pgmap v2611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:34 smithi093 ceph-mon[118902]: pgmap v2611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:20:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:20:35] "GET /metrics HTTP/1.1" 200 239887 "" "Prometheus/2.43.0" 2023-08-27T22:20:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3015037398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:35.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:35.932928+0000) 2023-08-27T22:20:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3015037398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3015037398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:36 smithi136 ceph-mon[113531]: pgmap v2612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:36.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:36.933119+0000) 2023-08-27T22:20:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:36 smithi093 ceph-mon[111335]: pgmap v2612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:36 smithi093 ceph-mon[118902]: pgmap v2612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:37.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:37.933386+0000) 2023-08-27T22:20:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:38.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:38.933669+0000) 2023-08-27T22:20:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:38 smithi136 ceph-mon[113531]: pgmap v2613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2584912460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:38 smithi093 ceph-mon[111335]: pgmap v2613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2584912460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:38 smithi093 ceph-mon[118902]: pgmap v2613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2584912460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:39.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:39.933931+0000) 2023-08-27T22:20:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:40 smithi136 ceph-mon[113531]: pgmap v2614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:40 smithi093 ceph-mon[111335]: pgmap v2614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:40 smithi093 ceph-mon[118902]: pgmap v2614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:40.455Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:40.456Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:40.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:40.934215+0000) 2023-08-27T22:20:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/377023012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/377023012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/377023012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:41.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:41.934457+0000) 2023-08-27T22:20:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:41.977Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:41.977Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:42 smithi136 ceph-mon[113531]: pgmap v2615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:42 smithi093 ceph-mon[111335]: pgmap v2615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:42 smithi093 ceph-mon[118902]: pgmap v2615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:42.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:42.934663+0000) 2023-08-27T22:20:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3513851629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3513851629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:43.996 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3513851629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:43.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:43.934891+0000) 2023-08-27T22:20:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:43.881Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:43.882Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:44 smithi136 ceph-mon[113531]: pgmap v2616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:44 smithi093 ceph-mon[111335]: pgmap v2616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:44 smithi093 ceph-mon[118902]: pgmap v2616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:44.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:44.935114+0000) 2023-08-27T22:20:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4030226282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4030226282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4030226282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:20:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:20:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:20:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:45.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:45.935333+0000) 2023-08-27T22:20:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:46 smithi136 ceph-mon[113531]: pgmap v2617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:46 smithi093 ceph-mon[111335]: pgmap v2617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:46 smithi093 ceph-mon[118902]: pgmap v2617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:46.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:46.935592+0000) 2023-08-27T22:20:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2303062614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2303062614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2303062614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:47.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:47.935810+0000) 2023-08-27T22:20:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:48 smithi093 ceph-mon[118902]: pgmap v2618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:48 smithi093 ceph-mon[111335]: pgmap v2618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:48 smithi136 ceph-mon[113531]: pgmap v2618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:48.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:48.936100+0000) 2023-08-27T22:20:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:49.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:49.936368+0000) 2023-08-27T22:20:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:50 smithi136 ceph-mon[113531]: pgmap v2619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3261039213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:50 smithi093 ceph-mon[111335]: pgmap v2619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3261039213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:50 smithi093 ceph-mon[118902]: pgmap v2619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3261039213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:50.455Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:50.456Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:50.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:50.936617+0000) 2023-08-27T22:20:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:51.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:51.936892+0000) 2023-08-27T22:20:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:51.977Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:51.978Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:52.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:52 smithi136 ceph-mon[113531]: pgmap v2620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:52.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/515236551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:52 smithi093 ceph-mon[111335]: pgmap v2620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/515236551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:52 smithi093 ceph-mon[118902]: pgmap v2620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/515236551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:52.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:52.937138+0000) 2023-08-27T22:20:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:53.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:53.937364+0000) 2023-08-27T22:20:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:53.882Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:20:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:20:53.883Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:20:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:54 smithi136 ceph-mon[113531]: pgmap v2621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:54 smithi093 ceph-mon[111335]: pgmap v2621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:54 smithi093 ceph-mon[118902]: pgmap v2621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:20:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:54.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:54.937625+0000) 2023-08-27T22:20:55.678 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:20:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:20:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:20:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3870742921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3870742921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3870742921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:55.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:55.937894+0000) 2023-08-27T22:20:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:56 smithi136 ceph-mon[113531]: pgmap v2622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:56 smithi093 ceph-mon[111335]: pgmap v2622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:56 smithi093 ceph-mon[118902]: pgmap v2622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:56.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:56.938098+0000) 2023-08-27T22:20:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1452293545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1452293545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1452293545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:20:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:57.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:57.938350+0000) 2023-08-27T22:20:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:20:58 smithi136 ceph-mon[113531]: pgmap v2623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:20:58 smithi093 ceph-mon[111335]: pgmap v2623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:20:58 smithi093 ceph-mon[118902]: pgmap v2623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:20:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:58.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:58.938570+0000) 2023-08-27T22:21:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:20:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:20:59.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:20:59.938825+0000) 2023-08-27T22:21:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/466905328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:00 smithi136 ceph-mon[113531]: pgmap v2624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/466905328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:00 smithi093 ceph-mon[111335]: pgmap v2624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/466905328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:00 smithi093 ceph-mon[118902]: pgmap v2624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:00.456Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:00.457Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:00.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:00.939102+0000) 2023-08-27T22:21:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:01.978Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:01.979Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:01.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:01.939338+0000) 2023-08-27T22:21:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:02 smithi136 ceph-mon[113531]: pgmap v2625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2764660754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:02 smithi093 ceph-mon[111335]: pgmap v2625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2764660754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:02 smithi093 ceph-mon[118902]: pgmap v2625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2764660754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:02.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:02.939575+0000) 2023-08-27T22:21:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:03.882Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:03.883Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:03.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:03.939781+0000) 2023-08-27T22:21:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:04 smithi136 ceph-mon[113531]: pgmap v2626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:05.065 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3047122955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:05.066 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:04 smithi093 ceph-mon[111335]: pgmap v2626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:05.066 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3047122955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:05.066 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:04 smithi093 ceph-mon[118902]: pgmap v2626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:05.066 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3047122955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:05.317 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:04.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:04.939996+0000) 2023-08-27T22:21:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 22:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T22:21:05.317682857Z level=info msg="Completed cleanup jobs" duration=12.973107ms 2023-08-27T22:21:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:21:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:21:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:21:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:05.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:05.940236+0000) 2023-08-27T22:21:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:06 smithi136 ceph-mon[113531]: pgmap v2627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:06 smithi093 ceph-mon[111335]: pgmap v2627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:06 smithi093 ceph-mon[118902]: pgmap v2627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:06.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:06.940492+0000) 2023-08-27T22:21:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1095291183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1095291183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1095291183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:07.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:07.940759+0000) 2023-08-27T22:21:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:08 smithi136 ceph-mon[113531]: pgmap v2628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:08 smithi093 ceph-mon[111335]: pgmap v2628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:08 smithi093 ceph-mon[118902]: pgmap v2628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:08.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:08.941048+0000) 2023-08-27T22:21:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3901000848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3901000848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3901000848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:09.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:09.941247+0000) 2023-08-27T22:21:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:10 smithi093 ceph-mon[111335]: pgmap v2629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:10 smithi093 ceph-mon[118902]: pgmap v2629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:10.457Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:10.459Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:10.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:10 smithi136 ceph-mon[113531]: pgmap v2629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:10.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:10.941510+0000) 2023-08-27T22:21:11.979 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3925310416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:11.979 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3925310416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3925310416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:11.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:11.941791+0000) 2023-08-27T22:21:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:11.979Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:11.980Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:12 smithi093 ceph-mon[111335]: pgmap v2630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:12.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:12 smithi093 ceph-mon[118902]: pgmap v2630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:12 smithi136 ceph-mon[113531]: pgmap v2630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:12.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:12.942062+0000) 2023-08-27T22:21:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:13.883Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:13.884Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:13.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:13.942306+0000) 2023-08-27T22:21:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:14 smithi093 ceph-mon[111335]: pgmap v2631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2462154571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:14 smithi093 ceph-mon[118902]: pgmap v2631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2462154571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:14 smithi136 ceph-mon[113531]: pgmap v2631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2462154571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:14.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:14.942538+0000) 2023-08-27T22:21:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:21:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:21:15] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:21:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:15.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:15.942767+0000) 2023-08-27T22:21:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:16 smithi093 ceph-mon[111335]: pgmap v2632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3676818016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:16 smithi093 ceph-mon[118902]: pgmap v2632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3676818016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:16 smithi136 ceph-mon[113531]: pgmap v2632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3676818016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:16.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:16.943016+0000) 2023-08-27T22:21:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:17.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:17.943322+0000) 2023-08-27T22:21:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:18 smithi093 ceph-mon[111335]: pgmap v2633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2293017264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:18 smithi093 ceph-mon[118902]: pgmap v2633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2293017264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:18 smithi136 ceph-mon[113531]: pgmap v2633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2293017264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:18.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:18.943574+0000) 2023-08-27T22:21:20.388 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:19.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:19.943816+0000) 2023-08-27T22:21:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:20 smithi136 ceph-mon[113531]: pgmap v2634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:20 smithi093 ceph-mon[111335]: pgmap v2634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:20 smithi093 ceph-mon[118902]: pgmap v2634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:20.458Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:20.459Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:21.389 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:20.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:20.944080+0000) 2023-08-27T22:21:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1937677876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1937677876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1937677876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:21.980Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:21.981Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:22.394 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:21.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:21.944250+0000) 2023-08-27T22:21:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:22 smithi136 ceph-mon[113531]: pgmap v2635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:22 smithi093 ceph-mon[111335]: pgmap v2635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:22 smithi093 ceph-mon[118902]: pgmap v2635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:23.395 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:22.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:22.944467+0000) 2023-08-27T22:21:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2642415978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2642415978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2642415978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:23.884Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:23.885Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:24.402 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:23.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:23.944720+0000) 2023-08-27T22:21:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:24 smithi136 ceph-mon[113531]: pgmap v2636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:24 smithi093 ceph-mon[111335]: pgmap v2636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:24 smithi093 ceph-mon[118902]: pgmap v2636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:24.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:24.944991+0000) 2023-08-27T22:21:25.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:21:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:21:25] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:21:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:25.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:25.945248+0000) 2023-08-27T22:21:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:26 smithi136 ceph-mon[113531]: pgmap v2637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3903651881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:26 smithi093 ceph-mon[118902]: pgmap v2637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3903651881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:26 smithi093 ceph-mon[111335]: pgmap v2637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3903651881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:26.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:26.945460+0000) 2023-08-27T22:21:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:27.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:27.945705+0000) 2023-08-27T22:21:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:28 smithi136 ceph-mon[113531]: pgmap v2638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1159482771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:28 smithi093 ceph-mon[118902]: pgmap v2638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1159482771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:28 smithi093 ceph-mon[111335]: pgmap v2638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1159482771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:28.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:28.945940+0000) 2023-08-27T22:21:30.389 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:29.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:29.946208+0000) 2023-08-27T22:21:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:30 smithi136 ceph-mon[113531]: pgmap v2639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:30 smithi093 ceph-mon[111335]: pgmap v2639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:30 smithi093 ceph-mon[118902]: pgmap v2639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:30.459Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:30.459Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:31.390 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:30.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:30.946445+0000) 2023-08-27T22:21:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3371507965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3371507965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3371507965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:31.980Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:31.981Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:32.396 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:31.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:31.946708+0000) 2023-08-27T22:21:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:32 smithi136 ceph-mon[113531]: pgmap v2640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:32 smithi093 ceph-mon[111335]: pgmap v2640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:32 smithi093 ceph-mon[118902]: pgmap v2640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:33.397 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:32.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:32.946970+0000) 2023-08-27T22:21:33.680 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2820874636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:33.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:21:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2820874636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:21:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2820874636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:21:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:33.885Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:33.885Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:34.422 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:33.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:33.947153+0000) 2023-08-27T22:21:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:34 smithi136 ceph-mon[113531]: pgmap v2641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:34 smithi093 ceph-mon[111335]: pgmap v2641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:34 smithi093 ceph-mon[118902]: pgmap v2641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:35.281 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:34.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:34.947310+0000) 2023-08-27T22:21:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:21:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:21:35] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:21:36.198 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:35.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:35.947492+0000) 2023-08-27T22:21:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/384093180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: pgmap v2642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:21:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/384093180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: pgmap v2642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:21:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/384093180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: pgmap v2642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:21:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:21:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:21:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:36.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:36.947720+0000) 2023-08-27T22:21:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:37.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:37.947973+0000) 2023-08-27T22:21:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:38 smithi136 ceph-mon[113531]: pgmap v2643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/133756413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:38 smithi093 ceph-mon[111335]: pgmap v2643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/133756413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:38 smithi093 ceph-mon[118902]: pgmap v2643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/133756413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:38.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:38.948221+0000) 2023-08-27T22:21:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:39.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:39.948423+0000) 2023-08-27T22:21:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:40 smithi093 ceph-mon[118902]: pgmap v2644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2820406819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:40 smithi093 ceph-mon[111335]: pgmap v2644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2820406819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:40.459Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:40.460Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:40 smithi136 ceph-mon[113531]: pgmap v2644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2820406819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:40.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:40.948668+0000) 2023-08-27T22:21:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:41.980Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:41.981Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:41.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:41.948928+0000) 2023-08-27T22:21:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:42 smithi136 ceph-mon[113531]: pgmap v2645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/352292234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:42 smithi093 ceph-mon[118902]: pgmap v2645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/352292234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:42 smithi093 ceph-mon[111335]: pgmap v2645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/352292234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:42.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:42.949242+0000) 2023-08-27T22:21:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:43.886Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:43.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:43.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:43.949474+0000) 2023-08-27T22:21:44.949 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:44 smithi136 ceph-mon[113531]: pgmap v2646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:44 smithi093 ceph-mon[111335]: pgmap v2646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:44 smithi093 ceph-mon[118902]: pgmap v2646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:44.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:44.949698+0000) 2023-08-27T22:21:45.700 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:21:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:21:45] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:21:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2691859070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2691859070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2691859070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:45.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:45.949971+0000) 2023-08-27T22:21:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:46 smithi093 ceph-mon[111335]: pgmap v2647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:46 smithi093 ceph-mon[118902]: pgmap v2647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:46 smithi136 ceph-mon[113531]: pgmap v2647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:46.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:46.950218+0000) 2023-08-27T22:21:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2875465440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2875465440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2875465440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:47.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:47.950468+0000) 2023-08-27T22:21:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:48 smithi093 ceph-mon[111335]: pgmap v2648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:48 smithi093 ceph-mon[118902]: pgmap v2648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:48 smithi136 ceph-mon[113531]: pgmap v2648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:48.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:48.950687+0000) 2023-08-27T22:21:50.392 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:49.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:49.950917+0000) 2023-08-27T22:21:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4269940988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:50 smithi136 ceph-mon[113531]: pgmap v2649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4269940988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:50 smithi093 ceph-mon[111335]: pgmap v2649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4269940988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:50 smithi093 ceph-mon[118902]: pgmap v2649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:50.460Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:50.461Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:50.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:50.951139+0000) 2023-08-27T22:21:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:51.981Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:52.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:51.982Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:51.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:51.951332+0000) 2023-08-27T22:21:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:52 smithi136 ceph-mon[113531]: pgmap v2650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1624375841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:52 smithi093 ceph-mon[111335]: pgmap v2650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:53.004 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1624375841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:53.004 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:52 smithi093 ceph-mon[118902]: pgmap v2650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:53.004 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1624375841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:52.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:52.951569+0000) 2023-08-27T22:21:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:53.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:21:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:21:53.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:21:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:21:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:53.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:53.951773+0000) 2023-08-27T22:21:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:54 smithi093 ceph-mon[111335]: pgmap v2651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1531734050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:54 smithi093 ceph-mon[118902]: pgmap v2651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1531734050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:54 smithi136 ceph-mon[113531]: pgmap v2651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1531734050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:54.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:54.952028+0000) 2023-08-27T22:21:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:21:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:21:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:21:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:55.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:55.952309+0000) 2023-08-27T22:21:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:56 smithi136 ceph-mon[113531]: pgmap v2652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:57.282 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:56.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:56.952560+0000) 2023-08-27T22:21:57.282 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:56 smithi093 ceph-mon[111335]: pgmap v2652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:57.283 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:56 smithi093 ceph-mon[118902]: pgmap v2652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2836311095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:57.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:57.952769+0000) 2023-08-27T22:21:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2836311095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2836311095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:21:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:58 smithi136 ceph-mon[113531]: pgmap v2653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:58.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:58.953027+0000) 2023-08-27T22:21:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:58 smithi093 ceph-mon[111335]: pgmap v2653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:21:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:58 smithi093 ceph-mon[118902]: pgmap v2653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:21:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/167030923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:21:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:21:59.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:21:59.953319+0000) 2023-08-27T22:22:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:21:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/167030923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:21:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/167030923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:00.460Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:00.462Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:00 smithi136 ceph-mon[113531]: pgmap v2654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:00.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:00.953560+0000) 2023-08-27T22:22:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:00 smithi093 ceph-mon[111335]: pgmap v2654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:00 smithi093 ceph-mon[118902]: pgmap v2654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3920997762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:01.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:01.953833+0000) 2023-08-27T22:22:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3920997762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3920997762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:01.982Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:01.983Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:02 smithi136 ceph-mon[113531]: pgmap v2655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:02.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:02.954065+0000) 2023-08-27T22:22:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:02 smithi093 ceph-mon[111335]: pgmap v2655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:02 smithi093 ceph-mon[118902]: pgmap v2655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:03.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:03.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:03.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:03.954310+0000) 2023-08-27T22:22:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:04 smithi136 ceph-mon[113531]: pgmap v2656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/827232131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:04.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:04.954599+0000) 2023-08-27T22:22:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:04 smithi093 ceph-mon[111335]: pgmap v2656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/827232131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:04 smithi093 ceph-mon[118902]: pgmap v2656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/827232131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:22:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:22:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:22:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:05.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:05.954853+0000) 2023-08-27T22:22:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:06 smithi136 ceph-mon[113531]: pgmap v2657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2858313857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:06.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:06.955140+0000) 2023-08-27T22:22:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:06 smithi093 ceph-mon[118902]: pgmap v2657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2858313857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:06 smithi093 ceph-mon[111335]: pgmap v2657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2858313857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:07.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:07.955370+0000) 2023-08-27T22:22:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:08 smithi136 ceph-mon[113531]: pgmap v2658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1448388776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:08.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:08.955637+0000) 2023-08-27T22:22:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:08 smithi093 ceph-mon[111335]: pgmap v2658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1448388776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:08 smithi093 ceph-mon[118902]: pgmap v2658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1448388776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:10.397 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:09.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:09.955835+0000) 2023-08-27T22:22:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:10 smithi136 ceph-mon[113531]: pgmap v2659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:10 smithi093 ceph-mon[111335]: pgmap v2659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:10 smithi093 ceph-mon[118902]: pgmap v2659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:10.462Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:10.463Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:11.398 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:10.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:10.956104+0000) 2023-08-27T22:22:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1659275415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1659275415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1659275415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:11.983Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:11.984Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:12.402 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:11.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:11.956379+0000) 2023-08-27T22:22:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:12 smithi136 ceph-mon[113531]: pgmap v2660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:12 smithi093 ceph-mon[111335]: pgmap v2660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:12 smithi093 ceph-mon[118902]: pgmap v2660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:12.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:12.956632+0000) 2023-08-27T22:22:13.887 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2220488153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:13.887 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2220488153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2220488153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:13.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:13.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:13.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:13.956902+0000) 2023-08-27T22:22:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:14 smithi136 ceph-mon[113531]: pgmap v2661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:14 smithi093 ceph-mon[111335]: pgmap v2661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:14 smithi093 ceph-mon[118902]: pgmap v2661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:14.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:14.957172+0000) 2023-08-27T22:22:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:22:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:22:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:22:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:15.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:15.957370+0000) 2023-08-27T22:22:16.957 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:16 smithi136 ceph-mon[113531]: pgmap v2662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:16.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/725155209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:16 smithi093 ceph-mon[111335]: pgmap v2662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/725155209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:16 smithi093 ceph-mon[118902]: pgmap v2662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/725155209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:16.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:16.957611+0000) 2023-08-27T22:22:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:17.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:17.957878+0000) 2023-08-27T22:22:18.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:18 smithi136 ceph-mon[113531]: pgmap v2663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:18.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2237141371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:18 smithi093 ceph-mon[111335]: pgmap v2663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2237141371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:18 smithi093 ceph-mon[118902]: pgmap v2663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2237141371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:18.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:18.958159+0000) 2023-08-27T22:22:20.401 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:19.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:19.958432+0000) 2023-08-27T22:22:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:20 smithi136 ceph-mon[113531]: pgmap v2664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:20 smithi093 ceph-mon[111335]: pgmap v2664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:20 smithi093 ceph-mon[118902]: pgmap v2664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:20.462Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:20.463Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:21.400 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:20.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:20.958703+0000) 2023-08-27T22:22:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1266826266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1266826266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1266826266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:21.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:21.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:22.407 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:21.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:21.958947+0000) 2023-08-27T22:22:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:22 smithi136 ceph-mon[113531]: pgmap v2665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:22 smithi093 ceph-mon[111335]: pgmap v2665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:22 smithi093 ceph-mon[118902]: pgmap v2665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:23.407 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:22.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:22.959199+0000) 2023-08-27T22:22:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2314561453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2314561453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2314561453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:23.888Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:23.889Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:24.415 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:23.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:23.959447+0000) 2023-08-27T22:22:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:24 smithi136 ceph-mon[113531]: pgmap v2666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:24 smithi093 ceph-mon[111335]: pgmap v2666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:24 smithi093 ceph-mon[118902]: pgmap v2666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:25.414 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:24.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:24.959707+0000) 2023-08-27T22:22:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3019214126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3019214126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:22:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:22:25] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:22:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3019214126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:26.419 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:25.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:25.959982+0000) 2023-08-27T22:22:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:26 smithi136 ceph-mon[113531]: pgmap v2667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:26 smithi093 ceph-mon[111335]: pgmap v2667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:26 smithi093 ceph-mon[118902]: pgmap v2667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:26.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:26.960163+0000) 2023-08-27T22:22:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:27.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:27.960392+0000) 2023-08-27T22:22:28.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:28 smithi136 ceph-mon[113531]: pgmap v2668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:28.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2415743044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:28 smithi093 ceph-mon[111335]: pgmap v2668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2415743044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:28 smithi093 ceph-mon[118902]: pgmap v2668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2415743044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:28.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:28.960599+0000) 2023-08-27T22:22:30.399 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:29.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:29.960874+0000) 2023-08-27T22:22:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:30 smithi136 ceph-mon[113531]: pgmap v2669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3423117964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:30 smithi093 ceph-mon[111335]: pgmap v2669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3423117964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:30 smithi093 ceph-mon[118902]: pgmap v2669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3423117964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:30.463Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:30.464Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:30.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:30.961155+0000) 2023-08-27T22:22:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:31.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:31.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:31.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:31.961361+0000) 2023-08-27T22:22:32.961 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:32 smithi136 ceph-mon[113531]: pgmap v2670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:32.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/445239389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:32 smithi093 ceph-mon[111335]: pgmap v2670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/445239389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:32 smithi093 ceph-mon[118902]: pgmap v2670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/445239389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:32.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:32.961606+0000) 2023-08-27T22:22:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:33.889Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:33.890Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:33.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:33.961864+0000) 2023-08-27T22:22:34.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:34 smithi136 ceph-mon[113531]: pgmap v2671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:34 smithi093 ceph-mon[111335]: pgmap v2671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:34 smithi093 ceph-mon[118902]: pgmap v2671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:34.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:34.962151+0000) 2023-08-27T22:22:35.707 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:22:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:22:35] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:22:35.962 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2971916485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2971916485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2971916485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:35.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:35.962404+0000) 2023-08-27T22:22:36.963 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:36 smithi136 ceph-mon[113531]: pgmap v2672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:36.963 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:22:36.963 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:36.963 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[118902]: pgmap v2672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:22:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[111335]: pgmap v2672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:22:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:37.414 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:36.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:36.962593+0000) 2023-08-27T22:22:37.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/183659961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/183659961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/183659961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:37.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:37.962744+0000) 2023-08-27T22:22:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:38 smithi136 ceph-mon[113531]: pgmap v2673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[111335]: pgmap v2673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[118902]: pgmap v2673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:38.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:38.962939+0000) 2023-08-27T22:22:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:39.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:39.963134+0000) 2023-08-27T22:22:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:22:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:22:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1876816698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1876816698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:22:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:22:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:22:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1876816698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:40.464Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:40.465Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:40 smithi136 ceph-mon[113531]: pgmap v2674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:40.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:40.963389+0000) 2023-08-27T22:22:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:40 smithi093 ceph-mon[111335]: pgmap v2674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:40 smithi093 ceph-mon[118902]: pgmap v2674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:42.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:41.985Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:41.986Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:41.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:41.963651+0000) 2023-08-27T22:22:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:42 smithi136 ceph-mon[113531]: pgmap v2675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3647191292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:42.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:42.963898+0000) 2023-08-27T22:22:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:42 smithi093 ceph-mon[111335]: pgmap v2675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3647191292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:42 smithi093 ceph-mon[118902]: pgmap v2675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3647191292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:43.890Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:43.891Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:43.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:43.964201+0000) 2023-08-27T22:22:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:44 smithi136 ceph-mon[113531]: pgmap v2676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:45.891 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1741571127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:45.891 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:44.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:44.964436+0000) 2023-08-27T22:22:45.892 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:44 smithi093 ceph-mon[111335]: pgmap v2676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:45.892 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1741571127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:45.896 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:44 smithi093 ceph-mon[118902]: pgmap v2676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:45.896 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1741571127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:45.896 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:22:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:22:45] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:22:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:45.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:45.964694+0000) 2023-08-27T22:22:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:46 smithi136 ceph-mon[113531]: pgmap v2677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3419856856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:46.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:46.964920+0000) 2023-08-27T22:22:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:46 smithi093 ceph-mon[111335]: pgmap v2677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3419856856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:46 smithi093 ceph-mon[118902]: pgmap v2677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3419856856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:47.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:47.965166+0000) 2023-08-27T22:22:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:48 smithi136 ceph-mon[113531]: pgmap v2678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:48.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:48.965396+0000) 2023-08-27T22:22:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:48 smithi093 ceph-mon[111335]: pgmap v2678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:48 smithi093 ceph-mon[118902]: pgmap v2678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4012616187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:49.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:49.965646+0000) 2023-08-27T22:22:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4012616187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:50.265 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4012616187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:50.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:50.465Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:50.466Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:50 smithi136 ceph-mon[113531]: pgmap v2679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:50.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:50.965903+0000) 2023-08-27T22:22:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:50 smithi093 ceph-mon[111335]: pgmap v2679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:50 smithi093 ceph-mon[118902]: pgmap v2679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2858964758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:51.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:51.966149+0000) 2023-08-27T22:22:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2858964758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2858964758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:51.986Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:51.987Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:52 smithi136 ceph-mon[113531]: pgmap v2680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:52.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:52.966432+0000) 2023-08-27T22:22:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:52 smithi093 ceph-mon[111335]: pgmap v2680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:52 smithi093 ceph-mon[118902]: pgmap v2680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:53.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:53.966690+0000) 2023-08-27T22:22:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:22:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:53.890Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:22:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:22:53.891Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:22:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:54 smithi136 ceph-mon[113531]: pgmap v2681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4108102799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:54.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:54.966956+0000) 2023-08-27T22:22:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:54 smithi093 ceph-mon[111335]: pgmap v2681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4108102799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:54 smithi093 ceph-mon[118902]: pgmap v2681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4108102799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:22:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:22:55] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:22:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:55.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:55.967213+0000) 2023-08-27T22:22:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:56 smithi136 ceph-mon[113531]: pgmap v2682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/918277613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:56.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:56.967461+0000) 2023-08-27T22:22:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:56 smithi093 ceph-mon[111335]: pgmap v2682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/918277613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:56 smithi093 ceph-mon[118902]: pgmap v2682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/918277613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:57.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:57.967716+0000) 2023-08-27T22:22:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:58 smithi136 ceph-mon[113531]: pgmap v2683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:22:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2168192284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:58.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:58.967924+0000) 2023-08-27T22:22:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:58 smithi093 ceph-mon[111335]: pgmap v2683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:22:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2168192284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:22:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:58 smithi093 ceph-mon[118902]: pgmap v2683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:22:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:22:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2168192284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:00.409 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:22:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:22:59.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:22:59.968167+0000) 2023-08-27T22:23:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:00 smithi136 ceph-mon[113531]: pgmap v2684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:00 smithi093 ceph-mon[111335]: pgmap v2684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:00 smithi093 ceph-mon[118902]: pgmap v2684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:00.465Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:00.466Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:01.410 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:00.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:00.968426+0000) 2023-08-27T22:23:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2278951656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2278951656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2278951656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:02.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:01.986Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:01.987Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:02.416 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:01.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:01.968684+0000) 2023-08-27T22:23:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:02 smithi136 ceph-mon[113531]: pgmap v2685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:02 smithi093 ceph-mon[111335]: pgmap v2685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:02 smithi093 ceph-mon[118902]: pgmap v2685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:02.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:02.968969+0000) 2023-08-27T22:23:03.891 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1449940891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:03.892 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1449940891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1449940891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:03.891Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:03.892Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:03.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:03.969221+0000) 2023-08-27T22:23:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:04 smithi136 ceph-mon[113531]: pgmap v2686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:04 smithi093 ceph-mon[111335]: pgmap v2686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:04 smithi093 ceph-mon[118902]: pgmap v2686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:04.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:04.969478+0000) 2023-08-27T22:23:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:23:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:23:05] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:23:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:05.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:05.969724+0000) 2023-08-27T22:23:06.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:06 smithi136 ceph-mon[113531]: pgmap v2687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:06.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3826836414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:06 smithi093 ceph-mon[111335]: pgmap v2687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3826836414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:06 smithi093 ceph-mon[118902]: pgmap v2687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3826836414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:06.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:06.969980+0000) 2023-08-27T22:23:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:07.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:07.970203+0000) 2023-08-27T22:23:08.970 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:08 smithi136 ceph-mon[113531]: pgmap v2688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:08.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3860932256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:08.971 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:08 smithi093 ceph-mon[111335]: pgmap v2688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3860932256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:08 smithi093 ceph-mon[118902]: pgmap v2688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3860932256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:08.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:08.970461+0000) 2023-08-27T22:23:10.405 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:09.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:09.970747+0000) 2023-08-27T22:23:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:10 smithi136 ceph-mon[113531]: pgmap v2689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:10 smithi093 ceph-mon[111335]: pgmap v2689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:10.849 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:10 smithi093 ceph-mon[118902]: pgmap v2689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:10.850 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:10.466Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:10.850 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:10.467Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:11.406 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:10.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:10.970936+0000) 2023-08-27T22:23:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2724420567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2724420567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:11.869 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2724420567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:11.987Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:11.988Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:12.410 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:11.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:11.971196+0000) 2023-08-27T22:23:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:12 smithi136 ceph-mon[113531]: pgmap v2690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:12 smithi093 ceph-mon[111335]: pgmap v2690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:12 smithi093 ceph-mon[118902]: pgmap v2690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:13.412 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:12.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:12.971414+0000) 2023-08-27T22:23:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/330455716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/330455716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/330455716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:13.892Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:13.893Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:14.415 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:13.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:13.971617+0000) 2023-08-27T22:23:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:14 smithi136 ceph-mon[113531]: pgmap v2691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:14 smithi093 ceph-mon[111335]: pgmap v2691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:14 smithi093 ceph-mon[118902]: pgmap v2691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:14.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:14.971874+0000) 2023-08-27T22:23:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/912498187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/912498187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:23:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:23:15] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:23:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/912498187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:15.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:15.972146+0000) 2023-08-27T22:23:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:16 smithi136 ceph-mon[113531]: pgmap v2692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:16 smithi093 ceph-mon[111335]: pgmap v2692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:16 smithi093 ceph-mon[118902]: pgmap v2692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:16.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:16.972405+0000) 2023-08-27T22:23:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:17.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:17.972644+0000) 2023-08-27T22:23:18.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:18 smithi136 ceph-mon[113531]: pgmap v2693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:18.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1709535101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:18 smithi093 ceph-mon[111335]: pgmap v2693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1709535101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:18 smithi093 ceph-mon[118902]: pgmap v2693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1709535101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:18.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:18.972871+0000) 2023-08-27T22:23:20.406 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:19.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:19.973156+0000) 2023-08-27T22:23:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:20 smithi136 ceph-mon[113531]: pgmap v2694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2120674796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:20 smithi093 ceph-mon[111335]: pgmap v2694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2120674796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:20 smithi093 ceph-mon[118902]: pgmap v2694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2120674796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:20.467Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:20.468Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:20.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:20.973415+0000) 2023-08-27T22:23:22.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:21.988Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:21.989Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:21.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:21.973635+0000) 2023-08-27T22:23:22.973 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:22 smithi136 ceph-mon[113531]: pgmap v2695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:22.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3297517009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:22 smithi093 ceph-mon[111335]: pgmap v2695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3297517009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:22 smithi093 ceph-mon[118902]: pgmap v2695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3297517009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:22.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:22.973824+0000) 2023-08-27T22:23:23.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:23.892Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:23.893Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:23.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:23.974145+0000) 2023-08-27T22:23:24.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:24 smithi136 ceph-mon[113531]: pgmap v2696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:24 smithi093 ceph-mon[111335]: pgmap v2696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:24 smithi093 ceph-mon[118902]: pgmap v2696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:24.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:24.974409+0000) 2023-08-27T22:23:25.723 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:23:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:23:25] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:23:25.974 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2284009588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2284009588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2284009588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:25.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:25.974664+0000) 2023-08-27T22:23:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:26 smithi093 ceph-mon[111335]: pgmap v2697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:26 smithi093 ceph-mon[118902]: pgmap v2697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:26 smithi136 ceph-mon[113531]: pgmap v2697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:26.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:26.974914+0000) 2023-08-27T22:23:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1042862839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1042862839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1042862839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:27.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:27.975157+0000) 2023-08-27T22:23:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:28 smithi093 ceph-mon[111335]: pgmap v2698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:28 smithi093 ceph-mon[118902]: pgmap v2698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:28 smithi136 ceph-mon[113531]: pgmap v2698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:28.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:28.975380+0000) 2023-08-27T22:23:30.410 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:29.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:29.975603+0000) 2023-08-27T22:23:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:30 smithi136 ceph-mon[113531]: pgmap v2699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1891635328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:30 smithi093 ceph-mon[118902]: pgmap v2699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1891635328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:30 smithi093 ceph-mon[111335]: pgmap v2699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1891635328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:30.468Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:30.469Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:30.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:30.975883+0000) 2023-08-27T22:23:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:31.988Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:32.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:31.989Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:31.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:31.976129+0000) 2023-08-27T22:23:32.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:32 smithi136 ceph-mon[113531]: pgmap v2700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:32.976 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2557255386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:32 smithi093 ceph-mon[111335]: pgmap v2700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2557255386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:32 smithi093 ceph-mon[118902]: pgmap v2700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2557255386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:32.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:32.976394+0000) 2023-08-27T22:23:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:33.893Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:33.894Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:33.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:33.976689+0000) 2023-08-27T22:23:34.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:34 smithi136 ceph-mon[113531]: pgmap v2701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:34.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/106412826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:34 smithi093 ceph-mon[111335]: pgmap v2701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/106412826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:34 smithi093 ceph-mon[118902]: pgmap v2701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/106412826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:34.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:34.976956+0000) 2023-08-27T22:23:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:23:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:23:35] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:23:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:35.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:35.977209+0000) 2023-08-27T22:23:36.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:36 smithi136 ceph-mon[113531]: pgmap v2702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:36 smithi093 ceph-mon[111335]: pgmap v2702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:36 smithi093 ceph-mon[118902]: pgmap v2702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:36.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:36.977412+0000) 2023-08-27T22:23:37.977 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4139075179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4139075179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4139075179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:37.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:37.977656+0000) 2023-08-27T22:23:38.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:38 smithi136 ceph-mon[113531]: pgmap v2703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:38.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:38 smithi093 ceph-mon[111335]: pgmap v2703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:38 smithi093 ceph-mon[118902]: pgmap v2703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:38.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:38.977903+0000) 2023-08-27T22:23:39.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3872156470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:39.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:23:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3872156470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:23:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3872156470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:23:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:39.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:39.978102+0000) 2023-08-27T22:23:40.728 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:40.468Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:40.728 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:40.470Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:40.978 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:40 smithi136 ceph-mon[113531]: pgmap v2704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:40 smithi093 ceph-mon[111335]: pgmap v2704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:40 smithi093 ceph-mon[118902]: pgmap v2704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:41.397 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:40.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:40.978322+0000) 2023-08-27T22:23:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:41.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:41.978490+0000) 2023-08-27T22:23:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/541957314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:42 smithi136 ceph-mon[113531]: pgmap v2705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:41.989Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:41.990Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/541957314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[111335]: pgmap v2705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/541957314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[118902]: pgmap v2705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:23:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:23:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:42.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:42.978690+0000) 2023-08-27T22:23:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:23:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:23:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:23:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:23:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:23:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:44 smithi093 ceph-mon[118902]: pgmap v2706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3777441936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:44 smithi093 ceph-mon[111335]: pgmap v2706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3777441936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:43.894Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:43.895Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:44 smithi136 ceph-mon[113531]: pgmap v2706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3777441936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:43.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:43.979007+0000) 2023-08-27T22:23:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:44.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:44.979329+0000) 2023-08-27T22:23:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:23:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:23:45] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:23:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:45.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:45.979586+0000) 2023-08-27T22:23:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:46 smithi136 ceph-mon[113531]: pgmap v2707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:46.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2804212956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:46 smithi093 ceph-mon[118902]: pgmap v2707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2804212956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:46 smithi093 ceph-mon[111335]: pgmap v2707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2804212956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:46.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:46.979865+0000) 2023-08-27T22:23:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:47.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:47.980165+0000) 2023-08-27T22:23:48.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:48 smithi136 ceph-mon[113531]: pgmap v2708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:48 smithi093 ceph-mon[111335]: pgmap v2708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:48 smithi093 ceph-mon[118902]: pgmap v2708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:48.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:48.980432+0000) 2023-08-27T22:23:49.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/749701256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/749701256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/749701256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:49.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:49.980678+0000) 2023-08-27T22:23:50.724 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:50.470Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:50.724 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:50.471Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:50.980 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:50 smithi136 ceph-mon[113531]: pgmap v2709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:50 smithi093 ceph-mon[111335]: pgmap v2709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:50 smithi093 ceph-mon[118902]: pgmap v2709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:50.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:50.980899+0000) 2023-08-27T22:23:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1601516293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1601516293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1601516293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:51.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:51.981171+0000) 2023-08-27T22:23:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:51.989Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:51.990Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:52 smithi093 ceph-mon[118902]: pgmap v2710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:52 smithi093 ceph-mon[111335]: pgmap v2710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:52 smithi136 ceph-mon[113531]: pgmap v2710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:52.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:52.981418+0000) 2023-08-27T22:23:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3141960399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3141960399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:53.895Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:53.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:23:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:23:53.896Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:23:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3141960399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:23:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:53.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:53.981634+0000) 2023-08-27T22:23:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:54 smithi136 ceph-mon[113531]: pgmap v2711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:54.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:54.981934+0000) 2023-08-27T22:23:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:54 smithi093 ceph-mon[111335]: pgmap v2711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:54 smithi093 ceph-mon[118902]: pgmap v2711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:23:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:23:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:23:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:55.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:55.982140+0000) 2023-08-27T22:23:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:56 smithi136 ceph-mon[113531]: pgmap v2712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1047405414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:56.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:56.982364+0000) 2023-08-27T22:23:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:56 smithi093 ceph-mon[111335]: pgmap v2712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1047405414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:56 smithi093 ceph-mon[118902]: pgmap v2712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1047405414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:57.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:57.982555+0000) 2023-08-27T22:23:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:58 smithi136 ceph-mon[113531]: pgmap v2713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:23:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2901126338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:58.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:58.982746+0000) 2023-08-27T22:23:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:58 smithi093 ceph-mon[118902]: pgmap v2713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:23:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2901126338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:23:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:58 smithi093 ceph-mon[111335]: pgmap v2713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:23:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:23:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2901126338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:00.419 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:23:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:23:59.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:23:59.982937+0000) 2023-08-27T22:24:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:00 smithi136 ceph-mon[113531]: pgmap v2714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:00 smithi093 ceph-mon[111335]: pgmap v2714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:00 smithi093 ceph-mon[118902]: pgmap v2714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:00.470Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:00.471Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:01.420 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:00.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:00.983128+0000) 2023-08-27T22:24:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2023479998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2023479998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2023479998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:02.424 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:01.990Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:02.424 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:01.991Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:02.424 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:01.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:01.983299+0000) 2023-08-27T22:24:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:02 smithi136 ceph-mon[113531]: pgmap v2715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:02 smithi093 ceph-mon[111335]: pgmap v2715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:02.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:02 smithi093 ceph-mon[118902]: pgmap v2715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:03.426 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:02.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:02.983467+0000) 2023-08-27T22:24:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1637715975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1637715975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1637715975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:03.896Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:03.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:04.429 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:03.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:03.983667+0000) 2023-08-27T22:24:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:04 smithi136 ceph-mon[113531]: pgmap v2716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:04.851 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:04 smithi093 ceph-mon[111335]: pgmap v2716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:04.851 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:04 smithi093 ceph-mon[118902]: pgmap v2716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:04.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:04.983877+0000) 2023-08-27T22:24:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/661598760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:24:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:24:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:24:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/661598760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/661598760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:05.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:05.984141+0000) 2023-08-27T22:24:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:06 smithi136 ceph-mon[113531]: pgmap v2717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:07.508 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:06 smithi093 ceph-mon[111335]: pgmap v2717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:07.509 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:06.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:06.984319+0000) 2023-08-27T22:24:07.509 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:06 smithi093 ceph-mon[118902]: pgmap v2717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:07.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:07.984553+0000) 2023-08-27T22:24:08.984 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:08 smithi136 ceph-mon[113531]: pgmap v2718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:08.985 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1490763881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:08.985 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:08 smithi093 ceph-mon[111335]: pgmap v2718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1490763881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:08 smithi093 ceph-mon[118902]: pgmap v2718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1490763881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:08.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:08.984786+0000) 2023-08-27T22:24:10.417 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:09.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:09.985004+0000) 2023-08-27T22:24:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:10 smithi136 ceph-mon[113531]: pgmap v2719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2986658551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:10 smithi093 ceph-mon[111335]: pgmap v2719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2986658551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:10 smithi093 ceph-mon[118902]: pgmap v2719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2986658551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:10.471Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:10.472Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:10.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:10.985249+0000) 2023-08-27T22:24:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:11.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:11.985469+0000) 2023-08-27T22:24:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:11.990Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:11.991Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:12.985 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:12 smithi136 ceph-mon[113531]: pgmap v2720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:12.986 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1916943237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:12 smithi093 ceph-mon[111335]: pgmap v2720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1916943237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:12 smithi093 ceph-mon[118902]: pgmap v2720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1916943237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:12.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:12.985707+0000) 2023-08-27T22:24:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:13.897Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:13.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:13.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:13.985964+0000) 2023-08-27T22:24:14.986 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:14 smithi136 ceph-mon[113531]: pgmap v2721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:14 smithi093 ceph-mon[111335]: pgmap v2721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:14 smithi093 ceph-mon[118902]: pgmap v2721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:14.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:14.986174+0000) 2023-08-27T22:24:15.732 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:24:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:24:15] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:24:15.986 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2333959638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2333959638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2333959638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:15.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:15.986434+0000) 2023-08-27T22:24:16.986 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:16 smithi136 ceph-mon[113531]: pgmap v2722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:16 smithi093 ceph-mon[111335]: pgmap v2722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:16 smithi093 ceph-mon[118902]: pgmap v2722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:16.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:16.986680+0000) 2023-08-27T22:24:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/910430226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:17.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:17.986890+0000) 2023-08-27T22:24:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/910430226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/910430226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:18 smithi136 ceph-mon[113531]: pgmap v2723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:18.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:18.987162+0000) 2023-08-27T22:24:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:18 smithi093 ceph-mon[111335]: pgmap v2723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:18 smithi093 ceph-mon[118902]: pgmap v2723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:20.422 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:19.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:19.987372+0000) 2023-08-27T22:24:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:20 smithi136 ceph-mon[113531]: pgmap v2724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2743579582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:20 smithi093 ceph-mon[111335]: pgmap v2724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2743579582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:20 smithi093 ceph-mon[118902]: pgmap v2724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2743579582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:20.472Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:20.473Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:20.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:20.987562+0000) 2023-08-27T22:24:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:21.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:21.987800+0000) 2023-08-27T22:24:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:21.991Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:21.992Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:22.988 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:22 smithi136 ceph-mon[113531]: pgmap v2725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:22.988 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/78399277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:22 smithi093 ceph-mon[111335]: pgmap v2725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/78399277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:22 smithi093 ceph-mon[118902]: pgmap v2725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/78399277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:22.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:22.988046+0000) 2023-08-27T22:24:23.988 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:23.897Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:23.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:23.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:23.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:23.988302+0000) 2023-08-27T22:24:24.988 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:24 smithi136 ceph-mon[113531]: pgmap v2726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:24.988 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/969872442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:24 smithi093 ceph-mon[111335]: pgmap v2726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/969872442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:24 smithi093 ceph-mon[118902]: pgmap v2726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/969872442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:24.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:24.988556+0000) 2023-08-27T22:24:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:24:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:24:25] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:24:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:25.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:25.988833+0000) 2023-08-27T22:24:26.989 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:26 smithi136 ceph-mon[113531]: pgmap v2727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:26 smithi093 ceph-mon[111335]: pgmap v2727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:26 smithi093 ceph-mon[118902]: pgmap v2727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:26.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:26.989063+0000) 2023-08-27T22:24:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3011678109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:27.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:27.989230+0000) 2023-08-27T22:24:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3011678109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3011678109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:28 smithi136 ceph-mon[113531]: pgmap v2728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:28.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:28.989457+0000) 2023-08-27T22:24:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:28 smithi093 ceph-mon[111335]: pgmap v2728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:28 smithi093 ceph-mon[118902]: pgmap v2728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2664848897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:29.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:29.989717+0000) 2023-08-27T22:24:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2664848897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2664848897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:30.473Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:30.474Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:30 smithi136 ceph-mon[113531]: pgmap v2729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:30.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:30.989953+0000) 2023-08-27T22:24:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:30 smithi093 ceph-mon[111335]: pgmap v2729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:30 smithi093 ceph-mon[118902]: pgmap v2729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2717960598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:31.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:31.990166+0000) 2023-08-27T22:24:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2717960598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2717960598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:31.992Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:31.993Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:32 smithi136 ceph-mon[113531]: pgmap v2730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:32.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:32.990439+0000) 2023-08-27T22:24:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:32 smithi093 ceph-mon[111335]: pgmap v2730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:32 smithi093 ceph-mon[118902]: pgmap v2730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:33.898Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:33.899Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:33.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:33.990715+0000) 2023-08-27T22:24:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:34 smithi136 ceph-mon[113531]: pgmap v2731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/976446801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:34.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:34.990969+0000) 2023-08-27T22:24:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:34 smithi093 ceph-mon[111335]: pgmap v2731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/976446801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:34 smithi093 ceph-mon[118902]: pgmap v2731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/976446801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:24:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:24:35] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:24:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:35.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:35.991198+0000) 2023-08-27T22:24:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:36 smithi136 ceph-mon[113531]: pgmap v2732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1721610773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:36.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:36.991405+0000) 2023-08-27T22:24:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:36 smithi093 ceph-mon[111335]: pgmap v2732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1721610773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:36 smithi093 ceph-mon[118902]: pgmap v2732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1721610773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:37.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:37.991642+0000) 2023-08-27T22:24:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:38 smithi136 ceph-mon[113531]: pgmap v2733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:38.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:38.991880+0000) 2023-08-27T22:24:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:38 smithi093 ceph-mon[111335]: pgmap v2733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:38 smithi093 ceph-mon[118902]: pgmap v2733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/85909696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:39.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:39.992146+0000) 2023-08-27T22:24:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/85909696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/85909696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:40.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:40.474Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:40.475Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:40 smithi136 ceph-mon[113531]: pgmap v2734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:40.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:40.992405+0000) 2023-08-27T22:24:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:40 smithi093 ceph-mon[111335]: pgmap v2734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:40 smithi093 ceph-mon[118902]: pgmap v2734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2869061135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:41.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:41.992656+0000) 2023-08-27T22:24:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2869061135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2869061135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:41.993Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:41.993Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:43.116 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:42 smithi136 ceph-mon[113531]: pgmap v2735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:43.117 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:24:43.117 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:42.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:42.992912+0000) 2023-08-27T22:24:43.213 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:42 smithi093 ceph-mon[111335]: pgmap v2735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:43.213 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:24:43.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:42 smithi093 ceph-mon[118902]: pgmap v2735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:43.214 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:24:44.044 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2834562402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:44.045 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:43.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:43.993144+0000) 2023-08-27T22:24:44.130 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2834562402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:44.130 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2834562402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:44.130 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:43.899Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:44.130 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:43.900Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:44.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:44.993302+0000) 2023-08-27T22:24:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:44 smithi136 ceph-mon[113531]: pgmap v2736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:44 smithi093 ceph-mon[111335]: pgmap v2736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:44 smithi093 ceph-mon[118902]: pgmap v2736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:24:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:24:45] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:24:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:45.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:45.993498+0000) 2023-08-27T22:24:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:24:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:24:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-mon[113531]: pgmap v2737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2021797288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[111335]: pgmap v2737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2021797288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:24:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:24:46.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:24:46.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[118902]: pgmap v2737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:46.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2021797288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:46.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:46.993686+0000) 2023-08-27T22:24:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:47.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:47.993920+0000) 2023-08-27T22:24:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:48 smithi093 ceph-mon[111335]: pgmap v2738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3529000202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:48 smithi093 ceph-mon[118902]: pgmap v2738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3529000202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:48.994 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:48 smithi136 ceph-mon[113531]: pgmap v2738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:48.994 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3529000202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:48.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:48.994177+0000) 2023-08-27T22:24:50.423 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:49.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:49.994433+0000) 2023-08-27T22:24:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:50 smithi136 ceph-mon[113531]: pgmap v2739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:50 smithi093 ceph-mon[111335]: pgmap v2739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:50 smithi093 ceph-mon[118902]: pgmap v2739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:50.475Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:50.476Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:51.423 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:50.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:50.994621+0000) 2023-08-27T22:24:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/404955364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/404955364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/404955364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:52.430 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:51.993Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:52.430 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:51.994Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:52.430 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:51.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:51.994881+0000) 2023-08-27T22:24:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:52 smithi136 ceph-mon[113531]: pgmap v2740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:52 smithi093 ceph-mon[111335]: pgmap v2740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:52 smithi093 ceph-mon[118902]: pgmap v2740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:53.432 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:52.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:52.995180+0000) 2023-08-27T22:24:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1604268556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1604268556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1604268556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:53.900Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:24:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:24:53.901Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:24:54.436 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:53.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:53.995403+0000) 2023-08-27T22:24:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:54 smithi136 ceph-mon[113531]: pgmap v2741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:54 smithi093 ceph-mon[111335]: pgmap v2741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:54 smithi093 ceph-mon[118902]: pgmap v2741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:24:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:54.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:54.995669+0000) 2023-08-27T22:24:55.610 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:24:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:24:55] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:24:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2193619428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2193619428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2193619428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:55.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:55.995903+0000) 2023-08-27T22:24:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:56 smithi136 ceph-mon[113531]: pgmap v2742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:56 smithi093 ceph-mon[118902]: pgmap v2742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:56 smithi093 ceph-mon[111335]: pgmap v2742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:56.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:56.996167+0000) 2023-08-27T22:24:58.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:57.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:57.996386+0000) 2023-08-27T22:24:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:58 smithi093 ceph-mon[111335]: pgmap v2743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:24:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2479058971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:58 smithi093 ceph-mon[118902]: pgmap v2743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:24:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2479058971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:58.996 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:58 smithi136 ceph-mon[113531]: pgmap v2743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:24:58.996 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:24:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2479058971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:24:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:58.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:58.996598+0000) 2023-08-27T22:25:00.425 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:24:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:24:59.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:24:59.996812+0000) 2023-08-27T22:25:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:00 smithi136 ceph-mon[113531]: pgmap v2744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/307620686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:00 smithi093 ceph-mon[111335]: pgmap v2744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/307620686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:00 smithi093 ceph-mon[118902]: pgmap v2744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/307620686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:00.476Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:00.477Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:00.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:00.997078+0000) 2023-08-27T22:25:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:01.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:01.997320+0000) 2023-08-27T22:25:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:01.994Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:01.995Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:02 smithi093 ceph-mon[111335]: pgmap v2745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1215194667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:02 smithi093 ceph-mon[118902]: pgmap v2745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1215194667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:02.997 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:02 smithi136 ceph-mon[113531]: pgmap v2745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:02.997 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1215194667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:02.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:02.997632+0000) 2023-08-27T22:25:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:03.901Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:03.902Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:03.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:03.997925+0000) 2023-08-27T22:25:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:04 smithi093 ceph-mon[111335]: pgmap v2746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:04 smithi093 ceph-mon[118902]: pgmap v2746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:04.998 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:04 smithi136 ceph-mon[113531]: pgmap v2746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:04.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:04.998177+0000) 2023-08-27T22:25:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:25:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:25:05] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:25:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2433867686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2433867686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:05.998 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2433867686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:05.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:05.998428+0000) 2023-08-27T22:25:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:06 smithi136 ceph-mon[113531]: pgmap v2747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:06.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:06.998662+0000) 2023-08-27T22:25:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:06 smithi093 ceph-mon[111335]: pgmap v2747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:06 smithi093 ceph-mon[118902]: pgmap v2747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/569808976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:07.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:07.998863+0000) 2023-08-27T22:25:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/569808976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/569808976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:08 smithi136 ceph-mon[113531]: pgmap v2748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:08.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:08.999094+0000) 2023-08-27T22:25:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:08 smithi093 ceph-mon[111335]: pgmap v2748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:08 smithi093 ceph-mon[118902]: pgmap v2748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:10.427 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:09.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:09.999278+0000) 2023-08-27T22:25:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:10 smithi136 ceph-mon[113531]: pgmap v2749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1331832477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:10 smithi093 ceph-mon[118902]: pgmap v2749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1331832477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:10 smithi093 ceph-mon[111335]: pgmap v2749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1331832477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:10.477Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:10.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:10.478Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:10.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:10.999500+0000) 2023-08-27T22:25:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:11.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:11.999713+0000) 2023-08-27T22:25:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:11.994Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:11.995Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:12 smithi093 ceph-mon[111335]: pgmap v2750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2598202157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:12 smithi093 ceph-mon[118902]: pgmap v2750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2598202157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:13.000 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:12 smithi136 ceph-mon[113531]: pgmap v2750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:13.000 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2598202157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:12.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:12.999967+0000) 2023-08-27T22:25:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:13.902Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:13.903Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:13.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:14.000240+0000) 2023-08-27T22:25:15.000 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:14 smithi136 ceph-mon[113531]: pgmap v2751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:15.000 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/190095768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:14 smithi093 ceph-mon[118902]: pgmap v2751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/190095768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:14 smithi093 ceph-mon[111335]: pgmap v2751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/190095768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:14.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:15.000507+0000) 2023-08-27T22:25:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:25:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:25:15] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:25:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:15.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:16.000748+0000) 2023-08-27T22:25:17.001 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:16 smithi136 ceph-mon[113531]: pgmap v2752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:16 smithi093 ceph-mon[111335]: pgmap v2752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:16 smithi093 ceph-mon[118902]: pgmap v2752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:17.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:17.001016+0000) 2023-08-27T22:25:18.001 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2674297315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2674297315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2674297315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:18.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:18.001312+0000) 2023-08-27T22:25:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:18 smithi136 ceph-mon[113531]: pgmap v2753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:19.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:19.001547+0000) 2023-08-27T22:25:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:18 smithi093 ceph-mon[111335]: pgmap v2753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:18 smithi093 ceph-mon[118902]: pgmap v2753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/444601095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:20.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:20.001765+0000) 2023-08-27T22:25:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/444601095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/444601095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:20.477Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:20.478Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:20 smithi136 ceph-mon[113531]: pgmap v2754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:21.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:21.002021+0000) 2023-08-27T22:25:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:20 smithi093 ceph-mon[111335]: pgmap v2754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:20 smithi093 ceph-mon[118902]: pgmap v2754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:22.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:22.002312+0000) 2023-08-27T22:25:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:21.995Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:21.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:22 smithi136 ceph-mon[113531]: pgmap v2755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2180540720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:23.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:23.002547+0000) 2023-08-27T22:25:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:22 smithi093 ceph-mon[111335]: pgmap v2755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2180540720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:22 smithi093 ceph-mon[118902]: pgmap v2755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2180540720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:24.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:24.002789+0000) 2023-08-27T22:25:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:23.902Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:23.903Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:24 smithi136 ceph-mon[113531]: pgmap v2756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3669255320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:25.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:25.003090+0000) 2023-08-27T22:25:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:24 smithi093 ceph-mon[111335]: pgmap v2756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3669255320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:24 smithi093 ceph-mon[118902]: pgmap v2756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3669255320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:25:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:25:25] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:25:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:26.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:26.003339+0000) 2023-08-27T22:25:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:26 smithi136 ceph-mon[113531]: pgmap v2757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3548449202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:27.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:27.003593+0000) 2023-08-27T22:25:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:26 smithi093 ceph-mon[111335]: pgmap v2757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3548449202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:26 smithi093 ceph-mon[118902]: pgmap v2757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3548449202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:28.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:28.003828+0000) 2023-08-27T22:25:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:28 smithi136 ceph-mon[113531]: pgmap v2758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:29.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:29.004091+0000) 2023-08-27T22:25:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:28 smithi093 ceph-mon[111335]: pgmap v2758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:28 smithi093 ceph-mon[118902]: pgmap v2758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1078738263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:30.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:30.004308+0000) 2023-08-27T22:25:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1078738263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1078738263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:30.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:30.477Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:30.478Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:30 smithi136 ceph-mon[113531]: pgmap v2759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:31.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:31.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:31.004548+0000) 2023-08-27T22:25:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:30 smithi093 ceph-mon[111335]: pgmap v2759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:30 smithi093 ceph-mon[118902]: pgmap v2759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3299665699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:32.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:32.004799+0000) 2023-08-27T22:25:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3299665699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3299665699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:31.995Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:31.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:32 smithi136 ceph-mon[113531]: pgmap v2760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:33.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:33.005085+0000) 2023-08-27T22:25:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:32 smithi093 ceph-mon[111335]: pgmap v2760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:32 smithi093 ceph-mon[118902]: pgmap v2760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1522866843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:34.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:34.005319+0000) 2023-08-27T22:25:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1522866843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1522866843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:33.903Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:33.904Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:34 smithi136 ceph-mon[113531]: pgmap v2761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:35.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:35.005563+0000) 2023-08-27T22:25:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:34 smithi093 ceph-mon[111335]: pgmap v2761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:34 smithi093 ceph-mon[118902]: pgmap v2761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:25:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:25:35] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:25:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:36.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:36.005768+0000) 2023-08-27T22:25:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:36 smithi136 ceph-mon[113531]: pgmap v2762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2565394725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:37.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:37.006013+0000) 2023-08-27T22:25:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:36 smithi093 ceph-mon[111335]: pgmap v2762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2565394725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:36 smithi093 ceph-mon[118902]: pgmap v2762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2565394725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:38.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:38.006231+0000) 2023-08-27T22:25:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:38 smithi136 ceph-mon[113531]: pgmap v2763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/131867963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:39.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:39.006484+0000) 2023-08-27T22:25:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:38 smithi093 ceph-mon[111335]: pgmap v2763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/131867963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:38 smithi093 ceph-mon[118902]: pgmap v2763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/131867963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:40.433 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:40.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:40.006676+0000) 2023-08-27T22:25:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:40 smithi136 ceph-mon[113531]: pgmap v2764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:40 smithi093 ceph-mon[111335]: pgmap v2764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:40 smithi093 ceph-mon[118902]: pgmap v2764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:40.478Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:40.479Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:41.434 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:41.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:41.006921+0000) 2023-08-27T22:25:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1249529827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1249529827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1249529827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:42.437 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:41.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:42.437 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:41.997Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:42.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:42.007154+0000) 2023-08-27T22:25:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:42 smithi093 ceph-mon[111335]: pgmap v2765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:42 smithi093 ceph-mon[118902]: pgmap v2765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:42 smithi136 ceph-mon[113531]: pgmap v2765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:43.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:43.007377+0000) 2023-08-27T22:25:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1703574048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1703574048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1703574048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:43.904Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:43.905Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:44.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:44.007627+0000) 2023-08-27T22:25:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:44 smithi093 ceph-mon[111335]: pgmap v2766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:44 smithi093 ceph-mon[118902]: pgmap v2766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:44 smithi136 ceph-mon[113531]: pgmap v2766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:45.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:45.007839+0000) 2023-08-27T22:25:45.655 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:25:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:25:45] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:25:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:25:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/129637756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:25:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/129637756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:25:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/129637756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:46.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:46.007998+0000) 2023-08-27T22:25:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:46 smithi136 ceph-mon[113531]: pgmap v2767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:46 smithi093 ceph-mon[111335]: pgmap v2767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:46 smithi093 ceph-mon[118902]: pgmap v2767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:47.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:47.008155+0000) 2023-08-27T22:25:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:48.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:48.008288+0000) 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: pgmap v2768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/75151205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: pgmap v2768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/75151205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: pgmap v2768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/75151205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:49.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:49.008484+0000) 2023-08-27T22:25:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:50.387 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:50.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:50.008641+0000) 2023-08-27T22:25:50.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[118902]: pgmap v2769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:50.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2537842755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:50.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[111335]: pgmap v2769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:50.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2537842755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:50.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:50.697 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:50.697 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:25:50.697 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:25:50.697 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:50.697 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:50.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:50.697 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:50.480Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:25:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:25:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:51.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-mon[113531]: pgmap v2769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:51.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2537842755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:51.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:51.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:51.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:25:51.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:25:51.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:25:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:51.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:51.008881+0000) 2023-08-27T22:25:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:52.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:52.009151+0000) 2023-08-27T22:25:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:51.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:51.997Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:53.009 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:52 smithi136 ceph-mon[113531]: pgmap v2770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:52 smithi093 ceph-mon[111335]: pgmap v2770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:52 smithi093 ceph-mon[118902]: pgmap v2770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:53.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:53.009397+0000) 2023-08-27T22:25:54.009 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3315813207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:54.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3315813207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3315813207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:25:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:53.905Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:25:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:25:53.906Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:25:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:54.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:54.009670+0000) 2023-08-27T22:25:55.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:54 smithi136 ceph-mon[113531]: pgmap v2771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:54 smithi093 ceph-mon[111335]: pgmap v2771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:54 smithi093 ceph-mon[118902]: pgmap v2771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:55.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:55.009923+0000) 2023-08-27T22:25:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:25:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:25:55] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:25:56.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1961216381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1961216381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1961216381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:56.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:56.010098+0000) 2023-08-27T22:25:57.010 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:56 smithi136 ceph-mon[113531]: pgmap v2772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:56 smithi093 ceph-mon[111335]: pgmap v2772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:56 smithi093 ceph-mon[118902]: pgmap v2772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:57.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:57.010317+0000) 2023-08-27T22:25:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1744228849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:58.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:58.010564+0000) 2023-08-27T22:25:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1744228849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1744228849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:25:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:25:58 smithi136 ceph-mon[113531]: pgmap v2773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:25:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:25:59.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:25:59.010821+0000) 2023-08-27T22:25:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:25:58 smithi093 ceph-mon[111335]: pgmap v2773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:25:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:25:58 smithi093 ceph-mon[118902]: pgmap v2773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:00.435 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:00.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:00.011107+0000) 2023-08-27T22:26:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:00 smithi136 ceph-mon[113531]: pgmap v2774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/377055911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:00 smithi093 ceph-mon[111335]: pgmap v2774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/377055911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:00 smithi093 ceph-mon[118902]: pgmap v2774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/377055911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:00.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:00.480Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:01.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:01.011327+0000) 2023-08-27T22:26:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:02.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:02.011506+0000) 2023-08-27T22:26:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:01.997Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:01.998Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:03.011 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:02 smithi136 ceph-mon[113531]: pgmap v2775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:03.012 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1062168060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:02 smithi093 ceph-mon[111335]: pgmap v2775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1062168060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:02 smithi093 ceph-mon[118902]: pgmap v2775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1062168060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:03.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:03.011760+0000) 2023-08-27T22:26:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:03.906Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:03.907Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:04.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:04.011971+0000) 2023-08-27T22:26:05.012 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:04 smithi136 ceph-mon[113531]: pgmap v2776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:05.012 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3377811168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:04 smithi093 ceph-mon[111335]: pgmap v2776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3377811168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:04 smithi093 ceph-mon[118902]: pgmap v2776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3377811168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:05.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:05.012238+0000) 2023-08-27T22:26:05.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:26:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:26:05] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:26:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:06.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:06.012471+0000) 2023-08-27T22:26:07.012 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:06 smithi136 ceph-mon[113531]: pgmap v2777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:06 smithi093 ceph-mon[111335]: pgmap v2777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:06 smithi093 ceph-mon[118902]: pgmap v2777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:07.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:07.012741+0000) 2023-08-27T22:26:08.013 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4016307749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4016307749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4016307749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:08.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:08.012969+0000) 2023-08-27T22:26:09.013 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:08 smithi136 ceph-mon[113531]: pgmap v2778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:09.013 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:08 smithi093 ceph-mon[111335]: pgmap v2778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:08 smithi093 ceph-mon[118902]: pgmap v2778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:09.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:09.013221+0000) 2023-08-27T22:26:10.434 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:10.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:10.013431+0000) 2023-08-27T22:26:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3660043679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:10 smithi136 ceph-mon[113531]: pgmap v2779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3660043679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:10 smithi093 ceph-mon[111335]: pgmap v2779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3660043679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:10 smithi093 ceph-mon[118902]: pgmap v2779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:10.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:10.480Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:11.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:11.013667+0000) 2023-08-27T22:26:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:12.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:12.013918+0000) 2023-08-27T22:26:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:11.998Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:11.998Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:13.014 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:12 smithi136 ceph-mon[113531]: pgmap v2780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:13.014 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2883066619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:12 smithi093 ceph-mon[118902]: pgmap v2780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2883066619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:12 smithi093 ceph-mon[111335]: pgmap v2780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2883066619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:13.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:13.014133+0000) 2023-08-27T22:26:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:13.906Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:13.907Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:14.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:14.014348+0000) 2023-08-27T22:26:15.014 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:14 smithi136 ceph-mon[113531]: pgmap v2781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:15.014 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1795816059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:14 smithi093 ceph-mon[111335]: pgmap v2781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1795816059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:14 smithi093 ceph-mon[118902]: pgmap v2781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1795816059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:15.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:15.014608+0000) 2023-08-27T22:26:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:26:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:26:15] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T22:26:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:16.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:16.014847+0000) 2023-08-27T22:26:17.015 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:16 smithi136 ceph-mon[113531]: pgmap v2782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:17.015 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/912466599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:16 smithi093 ceph-mon[111335]: pgmap v2782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/912466599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:16 smithi093 ceph-mon[118902]: pgmap v2782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/912466599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:17.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:17.015133+0000) 2023-08-27T22:26:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:18.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:18.015377+0000) 2023-08-27T22:26:19.015 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:18 smithi136 ceph-mon[113531]: pgmap v2783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:18 smithi093 ceph-mon[111335]: pgmap v2783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:18 smithi093 ceph-mon[118902]: pgmap v2783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:19.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:19.015611+0000) 2023-08-27T22:26:20.016 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3425792079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3425792079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3425792079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:20.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:20.015844+0000) 2023-08-27T22:26:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:20.480Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:20.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:20.481Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:21.016 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:20 smithi136 ceph-mon[113531]: pgmap v2784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:20 smithi093 ceph-mon[111335]: pgmap v2784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:20 smithi093 ceph-mon[118902]: pgmap v2784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:21.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:21.016071+0000) 2023-08-27T22:26:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4132216827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:22.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:22.016318+0000) 2023-08-27T22:26:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4132216827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4132216827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:21.998Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:21.999Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:22 smithi136 ceph-mon[113531]: pgmap v2785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:23.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:23.016582+0000) 2023-08-27T22:26:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:22 smithi093 ceph-mon[111335]: pgmap v2785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:22 smithi093 ceph-mon[118902]: pgmap v2785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/630720993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:24.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:24.016786+0000) 2023-08-27T22:26:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/630720993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/630720993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:23.907Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:23.909Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:24 smithi136 ceph-mon[113531]: pgmap v2786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:25.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:25.017088+0000) 2023-08-27T22:26:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:24 smithi093 ceph-mon[111335]: pgmap v2786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:24 smithi093 ceph-mon[118902]: pgmap v2786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:26:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:26:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:26:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:26.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:26.017410+0000) 2023-08-27T22:26:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:27.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:27.017659+0000) 2023-08-27T22:26:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:26 smithi136 ceph-mon[113531]: pgmap v2787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/946551759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:26 smithi093 ceph-mon[111335]: pgmap v2787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/946551759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:26 smithi093 ceph-mon[118902]: pgmap v2787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/946551759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:28.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:28.017895+0000) 2023-08-27T22:26:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:29.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:29.018160+0000) 2023-08-27T22:26:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:28 smithi136 ceph-mon[113531]: pgmap v2788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3556985678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:28 smithi093 ceph-mon[118902]: pgmap v2788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3556985678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:28 smithi093 ceph-mon[111335]: pgmap v2788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3556985678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:30.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:30.018418+0000) 2023-08-27T22:26:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:30 smithi093 ceph-mon[111335]: pgmap v2789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:30 smithi093 ceph-mon[118902]: pgmap v2789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:30.481Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:30.482Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:30 smithi136 ceph-mon[113531]: pgmap v2789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:31.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:31.018674+0000) 2023-08-27T22:26:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/437044136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/437044136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/437044136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:32.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:32.018912+0000) 2023-08-27T22:26:32.443 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:31.999Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:32.443 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:31.999Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:32 smithi093 ceph-mon[111335]: pgmap v2790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:32 smithi093 ceph-mon[118902]: pgmap v2790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:32 smithi136 ceph-mon[113531]: pgmap v2790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:33.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:33.019156+0000) 2023-08-27T22:26:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2889994447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2889994447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2889994447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:33.908Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:33.909Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:34.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:34.019412+0000) 2023-08-27T22:26:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:34 smithi093 ceph-mon[111335]: pgmap v2791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:34 smithi093 ceph-mon[118902]: pgmap v2791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:34 smithi136 ceph-mon[113531]: pgmap v2791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:35.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:35.019668+0000) 2023-08-27T22:26:35.733 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:26:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:26:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:26:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3159884363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3159884363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:36.020 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3159884363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:36.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:36.019945+0000) 2023-08-27T22:26:37.020 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:36 smithi136 ceph-mon[113531]: pgmap v2792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:36 smithi093 ceph-mon[111335]: pgmap v2792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:36 smithi093 ceph-mon[118902]: pgmap v2792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:37.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:37.020205+0000) 2023-08-27T22:26:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:38.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:38.020444+0000) 2023-08-27T22:26:39.020 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:38 smithi136 ceph-mon[113531]: pgmap v2793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:39.021 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3015947924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:39.021 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:38 smithi093 ceph-mon[111335]: pgmap v2793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3015947924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:38 smithi093 ceph-mon[118902]: pgmap v2793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3015947924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:39.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:39.020683+0000) 2023-08-27T22:26:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:40.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:40.020934+0000) 2023-08-27T22:26:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:40 smithi093 ceph-mon[111335]: pgmap v2794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:40 smithi093 ceph-mon[118902]: pgmap v2794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:40.482Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:40.483Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:40 smithi136 ceph-mon[113531]: pgmap v2794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:41.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:41.021204+0000) 2023-08-27T22:26:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2562291823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2562291823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2562291823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:42.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:42.021420+0000) 2023-08-27T22:26:42.448 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:41.999Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:42.449 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:42.000Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:42 smithi093 ceph-mon[111335]: pgmap v2795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:42 smithi093 ceph-mon[118902]: pgmap v2795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:42 smithi136 ceph-mon[113531]: pgmap v2795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:43.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:43.021677+0000) 2023-08-27T22:26:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4076466384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4076466384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4076466384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:44.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:43.909Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:44.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:43.910Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:44.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:44.021929+0000) 2023-08-27T22:26:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:44 smithi093 ceph-mon[111335]: pgmap v2796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:44 smithi093 ceph-mon[118902]: pgmap v2796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:44 smithi136 ceph-mon[113531]: pgmap v2796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:45.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:45.022232+0000) 2023-08-27T22:26:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3858780941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:26:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:26:45] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:26:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3858780941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3858780941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:46.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:46.022480+0000) 2023-08-27T22:26:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:46 smithi093 ceph-mon[111335]: pgmap v2797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:46 smithi093 ceph-mon[118902]: pgmap v2797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:46 smithi136 ceph-mon[113531]: pgmap v2797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:47.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:47.022730+0000) 2023-08-27T22:26:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2461544076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2461544076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2461544076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:48.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:48.022994+0000) 2023-08-27T22:26:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:48 smithi136 ceph-mon[113531]: pgmap v2798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:48 smithi093 ceph-mon[111335]: pgmap v2798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:48 smithi093 ceph-mon[118902]: pgmap v2798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:49.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:49.023256+0000) 2023-08-27T22:26:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:50.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:50.023447+0000) 2023-08-27T22:26:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:50 smithi093 ceph-mon[111335]: pgmap v2799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2970121924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:50 smithi093 ceph-mon[118902]: pgmap v2799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2970121924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:50.482Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:50.483Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:50 smithi136 ceph-mon[113531]: pgmap v2799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2970121924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:51.353 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:51.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:51.023621+0000) 2023-08-27T22:26:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:26:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:26:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:26:52.367 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:51.999Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:52.368 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:52.000Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:52.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:52.023771+0000) 2023-08-27T22:26:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:52 smithi093 ceph-mon[111335]: pgmap v2800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2746705734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:52 smithi093 ceph-mon[118902]: pgmap v2800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2746705734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:52.913 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:52 smithi136 ceph-mon[113531]: pgmap v2800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:52.913 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2746705734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:53.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:53.024023+0000) 2023-08-27T22:26:53.911 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.911 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.911 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.911 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.911 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:26:53.911 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:26:53.912 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:26:53.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:26:53.913 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:26:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:26:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:26:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:53.910Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:26:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:26:53.911Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:26:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:54.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:54.024284+0000) 2023-08-27T22:26:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:54 smithi136 ceph-mon[113531]: pgmap v2801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:54 smithi093 ceph-mon[118902]: pgmap v2801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:26:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:54 smithi093 ceph-mon[111335]: pgmap v2801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:55.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:55.024539+0000) 2023-08-27T22:26:55.636 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:26:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:26:55] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:26:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3574463020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3574463020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3574463020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:56.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:56.024798+0000) 2023-08-27T22:26:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:56 smithi136 ceph-mon[113531]: pgmap v2802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:56 smithi093 ceph-mon[111335]: pgmap v2802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:56 smithi093 ceph-mon[118902]: pgmap v2802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:57.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:57.025064+0000) 2023-08-27T22:26:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2842786637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2842786637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2842786637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:26:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:58.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:58.025321+0000) 2023-08-27T22:26:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:26:58 smithi136 ceph-mon[113531]: pgmap v2803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:26:58 smithi093 ceph-mon[111335]: pgmap v2803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:26:58 smithi093 ceph-mon[118902]: pgmap v2803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:26:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:26:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:26:59.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:26:59.025596+0000) 2023-08-27T22:27:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:00.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:00.025804+0000) 2023-08-27T22:27:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1141716715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:00 smithi093 ceph-mon[111335]: pgmap v2804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1141716715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:00 smithi093 ceph-mon[118902]: pgmap v2804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:00.484Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:00.485Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1141716715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:00 smithi136 ceph-mon[113531]: pgmap v2804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:01.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:01.026084+0000) 2023-08-27T22:27:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:02.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:02.026333+0000) 2023-08-27T22:27:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:02.000Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:02.001Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:03.026 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:02 smithi136 ceph-mon[113531]: pgmap v2805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:03.026 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1168867866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:02 smithi093 ceph-mon[118902]: pgmap v2805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1168867866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:02 smithi093 ceph-mon[111335]: pgmap v2805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1168867866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:03.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:03.026556+0000) 2023-08-27T22:27:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:03.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:03.913Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:04.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:04.026812+0000) 2023-08-27T22:27:05.027 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:04 smithi136 ceph-mon[113531]: pgmap v2806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:05.027 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1735377176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:04 smithi093 ceph-mon[111335]: pgmap v2806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1735377176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:04 smithi093 ceph-mon[118902]: pgmap v2806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1735377176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:05.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:05.027089+0000) 2023-08-27T22:27:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:27:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:27:05] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:27:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:06.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:06.027368+0000) 2023-08-27T22:27:07.027 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:06 smithi136 ceph-mon[113531]: pgmap v2807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:07.027 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3509664702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:06 smithi093 ceph-mon[111335]: pgmap v2807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3509664702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:06 smithi093 ceph-mon[118902]: pgmap v2807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3509664702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:07.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:07.027608+0000) 2023-08-27T22:27:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:08.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:08.027847+0000) 2023-08-27T22:27:09.028 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:08 smithi136 ceph-mon[113531]: pgmap v2808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:09.028 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:08 smithi093 ceph-mon[111335]: pgmap v2808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:08 smithi093 ceph-mon[118902]: pgmap v2808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:09.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:09.028128+0000) 2023-08-27T22:27:10.028 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340201348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340201348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340201348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:10.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:10.028373+0000) 2023-08-27T22:27:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:10.485Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:10.486Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:11.028 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:10 smithi136 ceph-mon[113531]: pgmap v2809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:10 smithi093 ceph-mon[111335]: pgmap v2809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:10 smithi093 ceph-mon[118902]: pgmap v2809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:11.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:11.028634+0000) 2023-08-27T22:27:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2540999285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:12.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:12.028896+0000) 2023-08-27T22:27:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2540999285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2540999285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:12.001Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:12.002Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:12 smithi136 ceph-mon[113531]: pgmap v2810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:13.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:13.029164+0000) 2023-08-27T22:27:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:12 smithi093 ceph-mon[111335]: pgmap v2810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:12 smithi093 ceph-mon[118902]: pgmap v2810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:13.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:13.913Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:14.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:14.029418+0000) 2023-08-27T22:27:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:14 smithi136 ceph-mon[113531]: pgmap v2811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2070384414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:15.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:15.029673+0000) 2023-08-27T22:27:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:14 smithi093 ceph-mon[111335]: pgmap v2811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2070384414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:14 smithi093 ceph-mon[118902]: pgmap v2811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2070384414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:27:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:27:15] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:27:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:16.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:16.029912+0000) 2023-08-27T22:27:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:16 smithi136 ceph-mon[113531]: pgmap v2812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2132174586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:17.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:17.030098+0000) 2023-08-27T22:27:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:16 smithi093 ceph-mon[111335]: pgmap v2812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2132174586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:16 smithi093 ceph-mon[118902]: pgmap v2812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2132174586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:18.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:18.030366+0000) 2023-08-27T22:27:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:18 smithi136 ceph-mon[113531]: pgmap v2813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4069977957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:19.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:19.030597+0000) 2023-08-27T22:27:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:18 smithi093 ceph-mon[118902]: pgmap v2813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4069977957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:18 smithi093 ceph-mon[111335]: pgmap v2813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4069977957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:20.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:20.030789+0000) 2023-08-27T22:27:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:20 smithi093 ceph-mon[111335]: pgmap v2814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:20 smithi093 ceph-mon[118902]: pgmap v2814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:20.486Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:20.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:20.486Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:20 smithi136 ceph-mon[113531]: pgmap v2814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:21.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:21.031064+0000) 2023-08-27T22:27:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/420948754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/420948754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/420948754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:22.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:22.031248+0000) 2023-08-27T22:27:22.456 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:22.001Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:22.456 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:22.002Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:22 smithi093 ceph-mon[118902]: pgmap v2815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:22.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:22 smithi093 ceph-mon[111335]: pgmap v2815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:22 smithi136 ceph-mon[113531]: pgmap v2815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:23.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:23.031472+0000) 2023-08-27T22:27:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/852892858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:23.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/852892858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/852892858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:23.913Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:23.914Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:24.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:24.031743+0000) 2023-08-27T22:27:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:24 smithi093 ceph-mon[111335]: pgmap v2816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:24 smithi093 ceph-mon[118902]: pgmap v2816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:24 smithi136 ceph-mon[113531]: pgmap v2816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:25.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:25.032012+0000) 2023-08-27T22:27:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:27:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:27:25] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T22:27:26.032 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2327670439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2327670439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2327670439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:26.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:26.032303+0000) 2023-08-27T22:27:27.032 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:26 smithi136 ceph-mon[113531]: pgmap v2817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:26 smithi093 ceph-mon[111335]: pgmap v2817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:26 smithi093 ceph-mon[118902]: pgmap v2817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:27.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:27.032539+0000) 2023-08-27T22:27:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:28.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:28.032783+0000) 2023-08-27T22:27:29.033 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:28 smithi136 ceph-mon[113531]: pgmap v2818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:29.033 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2337838892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:28 smithi093 ceph-mon[111335]: pgmap v2818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2337838892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:28 smithi093 ceph-mon[118902]: pgmap v2818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2337838892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:29.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:29.033013+0000) 2023-08-27T22:27:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:30.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:30.033291+0000) 2023-08-27T22:27:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:30 smithi093 ceph-mon[111335]: pgmap v2819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:30 smithi093 ceph-mon[118902]: pgmap v2819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:30.486Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:30.487Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:30 smithi136 ceph-mon[113531]: pgmap v2819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:31.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:31.033518+0000) 2023-08-27T22:27:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2220044010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2220044010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2220044010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:32.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:32.033758+0000) 2023-08-27T22:27:32.456 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:32.002Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:32.457 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:32.003Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:32 smithi093 ceph-mon[111335]: pgmap v2820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:32 smithi093 ceph-mon[118902]: pgmap v2820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:32 smithi136 ceph-mon[113531]: pgmap v2820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:33.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:33.034022+0000) 2023-08-27T22:27:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1333122157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1333122157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1333122157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:33.914Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:33.915Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:34.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:34.034280+0000) 2023-08-27T22:27:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:34 smithi093 ceph-mon[111335]: pgmap v2821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:34 smithi093 ceph-mon[118902]: pgmap v2821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:34 smithi136 ceph-mon[113531]: pgmap v2821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:35.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:35.034514+0000) 2023-08-27T22:27:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4249038224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4249038224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:27:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:27:35] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T22:27:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4249038224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:36.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:36.034786+0000) 2023-08-27T22:27:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:36 smithi093 ceph-mon[111335]: pgmap v2822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:36 smithi093 ceph-mon[118902]: pgmap v2822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:36 smithi136 ceph-mon[113531]: pgmap v2822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:37.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:37.035014+0000) 2023-08-27T22:27:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/223510879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/223510879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/223510879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:38.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:38.035277+0000) 2023-08-27T22:27:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:38 smithi136 ceph-mon[113531]: pgmap v2823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:38 smithi093 ceph-mon[111335]: pgmap v2823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:38 smithi093 ceph-mon[118902]: pgmap v2823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:39.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:39.035502+0000) 2023-08-27T22:27:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:40.035 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:40.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:40.035754+0000) 2023-08-27T22:27:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:40 smithi093 ceph-mon[111335]: pgmap v2824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:40.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2189595575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:40 smithi093 ceph-mon[118902]: pgmap v2824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2189595575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:40.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:40.487Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:40.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:40.488Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:41.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:40 smithi136 ceph-mon[113531]: pgmap v2824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:41.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2189595575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:41.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:41.036010+0000) 2023-08-27T22:27:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:42.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:42.036296+0000) 2023-08-27T22:27:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:42.002Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:42.003Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:43.036 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:42 smithi136 ceph-mon[113531]: pgmap v2825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:43.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/362186359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:42 smithi093 ceph-mon[111335]: pgmap v2825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/362186359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:42 smithi093 ceph-mon[118902]: pgmap v2825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/362186359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:43.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:43.036578+0000) 2023-08-27T22:27:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:43.915Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:43.916Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:44.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:44.036844+0000) 2023-08-27T22:27:45.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:44 smithi136 ceph-mon[113531]: pgmap v2826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:44 smithi093 ceph-mon[111335]: pgmap v2826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:44 smithi093 ceph-mon[118902]: pgmap v2826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:45.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:45.037114+0000) 2023-08-27T22:27:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:27:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:27:45] "GET /metrics HTTP/1.1" 200 239886 "" "Prometheus/2.43.0" 2023-08-27T22:27:46.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/982757979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/982757979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/982757979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:46.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:46.037382+0000) 2023-08-27T22:27:47.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:46 smithi136 ceph-mon[113531]: pgmap v2827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:46 smithi093 ceph-mon[111335]: pgmap v2827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:46 smithi093 ceph-mon[118902]: pgmap v2827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:47.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:47.037596+0000) 2023-08-27T22:27:48.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/18804591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/18804591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/18804591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:48.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:48.037829+0000) 2023-08-27T22:27:49.038 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:48 smithi136 ceph-mon[113531]: pgmap v2828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:48 smithi093 ceph-mon[111335]: pgmap v2828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:48 smithi093 ceph-mon[118902]: pgmap v2828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:49.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:49.038085+0000) 2023-08-27T22:27:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:50.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:50.038302+0000) 2023-08-27T22:27:50.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2982538055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:50.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:50 smithi093 ceph-mon[111335]: pgmap v2829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:50.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2982538055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:50.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:50 smithi093 ceph-mon[118902]: pgmap v2829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:50.743 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:50.488Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:50.743 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:50.489Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2982538055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:50 smithi136 ceph-mon[113531]: pgmap v2829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:51.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:51.038559+0000) 2023-08-27T22:27:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:52.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:52.038801+0000) 2023-08-27T22:27:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:52.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:52.003Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:53.039 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:52 smithi136 ceph-mon[113531]: pgmap v2830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:53.039 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/813414598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:52 smithi093 ceph-mon[111335]: pgmap v2830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/813414598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:52 smithi093 ceph-mon[118902]: pgmap v2830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/813414598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:53.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:53.039090+0000) 2023-08-27T22:27:54.039 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:54.121 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:27:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:27:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:27:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:27:54.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:53.915Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:54.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:27:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:27:53.916Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:27:54.295 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:54.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:54.039271+0000) 2023-08-27T22:27:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:54 smithi136 ceph-mon[113531]: pgmap v2831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2303993070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:54 smithi093 ceph-mon[111335]: pgmap v2831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2303993070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:54 smithi093 ceph-mon[118902]: pgmap v2831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2303993070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:55.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:55.039507+0000) 2023-08-27T22:27:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:27:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:27:55] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T22:27:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:56.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:56.039789+0000) 2023-08-27T22:27:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:56 smithi136 ceph-mon[113531]: pgmap v2832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[111335]: pgmap v2832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[118902]: pgmap v2832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:57.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:57.039991+0000) 2023-08-27T22:27:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:27:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:27:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/967142877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:27:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:27:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/967142877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:27:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:27:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:27:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/967142877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:58.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:58.040254+0000) 2023-08-27T22:27:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:58 smithi136 ceph-mon[113531]: pgmap v2833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:58 smithi093 ceph-mon[111335]: pgmap v2833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:58.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:58 smithi093 ceph-mon[118902]: pgmap v2833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:27:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:27:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:27:59.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:27:59.040516+0000) 2023-08-27T22:27:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:27:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1632463138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:27:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1632463138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:27:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:27:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1632463138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:00.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:00.040791+0000) 2023-08-27T22:28:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:00 smithi093 ceph-mon[111335]: pgmap v2834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:00 smithi093 ceph-mon[118902]: pgmap v2834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:00.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:00.488Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:00.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:00.489Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:00 smithi136 ceph-mon[113531]: pgmap v2834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:01.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:01.041010+0000) 2023-08-27T22:28:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/992707788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/992707788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/992707788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:02.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:02.041279+0000) 2023-08-27T22:28:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:02.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:02.004Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:02 smithi136 ceph-mon[113531]: pgmap v2835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:02 smithi093 ceph-mon[111335]: pgmap v2835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:02 smithi093 ceph-mon[118902]: pgmap v2835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:03.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:03.041531+0000) 2023-08-27T22:28:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:03.916Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:03.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:04.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:04.041789+0000) 2023-08-27T22:28:05.042 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:04 smithi136 ceph-mon[113531]: pgmap v2836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:05.042 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/784089937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:04 smithi093 ceph-mon[111335]: pgmap v2836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/784089937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:04 smithi093 ceph-mon[118902]: pgmap v2836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/784089937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:05.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:05.042096+0000) 2023-08-27T22:28:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:28:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:28:05] "GET /metrics HTTP/1.1" 200 239880 "" "Prometheus/2.43.0" 2023-08-27T22:28:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:06.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:06.042338+0000) 2023-08-27T22:28:07.042 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:06 smithi136 ceph-mon[113531]: pgmap v2837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:07.043 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3987930699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:06 smithi093 ceph-mon[111335]: pgmap v2837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3987930699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:06 smithi093 ceph-mon[118902]: pgmap v2837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3987930699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:07.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:07.042575+0000) 2023-08-27T22:28:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:08.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:08.042847+0000) 2023-08-27T22:28:09.043 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:08 smithi136 ceph-mon[113531]: pgmap v2838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:09.043 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/369115502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:09.043 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:08 smithi093 ceph-mon[111335]: pgmap v2838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/369115502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:08 smithi093 ceph-mon[118902]: pgmap v2838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/369115502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:09.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:09.043089+0000) 2023-08-27T22:28:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:10.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:10.043307+0000) 2023-08-27T22:28:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:10 smithi093 ceph-mon[111335]: pgmap v2839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:10 smithi093 ceph-mon[118902]: pgmap v2839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:10.489Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:10.490Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:10 smithi136 ceph-mon[113531]: pgmap v2839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:11.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:11.043564+0000) 2023-08-27T22:28:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/78705806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/78705806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/78705806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:12.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:12.043825+0000) 2023-08-27T22:28:12.460 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:12.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:12.460 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:12.004Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:12 smithi093 ceph-mon[111335]: pgmap v2840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:12 smithi093 ceph-mon[118902]: pgmap v2840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:12 smithi136 ceph-mon[113531]: pgmap v2840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:13.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:13.044131+0000) 2023-08-27T22:28:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2196036798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2196036798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2196036798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:13.917Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:13.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:14.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:14.044366+0000) 2023-08-27T22:28:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:14 smithi093 ceph-mon[111335]: pgmap v2841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:14 smithi093 ceph-mon[118902]: pgmap v2841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:14 smithi136 ceph-mon[113531]: pgmap v2841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:15.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:15.044583+0000) 2023-08-27T22:28:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:28:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:28:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:28:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:16.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:16.044777+0000) 2023-08-27T22:28:17.045 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:16 smithi136 ceph-mon[113531]: pgmap v2842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:17.045 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2767604900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:16 smithi093 ceph-mon[111335]: pgmap v2842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2767604900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:16 smithi093 ceph-mon[118902]: pgmap v2842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2767604900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:17.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:17.044996+0000) 2023-08-27T22:28:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:18.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:18.045282+0000) 2023-08-27T22:28:19.045 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:18 smithi136 ceph-mon[113531]: pgmap v2843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:19.045 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2165113480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:18 smithi093 ceph-mon[111335]: pgmap v2843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2165113480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:18 smithi093 ceph-mon[118902]: pgmap v2843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2165113480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:19.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:19.045499+0000) 2023-08-27T22:28:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:20.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:20.045747+0000) 2023-08-27T22:28:20.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:20 smithi093 ceph-mon[111335]: pgmap v2844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:20.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:20 smithi093 ceph-mon[118902]: pgmap v2844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:20.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:20.490Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:20.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:20.491Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:20 smithi136 ceph-mon[113531]: pgmap v2844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:21.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:21.045969+0000) 2023-08-27T22:28:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3631113304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3631113304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3631113304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:22.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:22.046239+0000) 2023-08-27T22:28:22.467 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:22.004Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:22.467 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:22.005Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:22 smithi093 ceph-mon[111335]: pgmap v2845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:22 smithi093 ceph-mon[118902]: pgmap v2845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:22 smithi136 ceph-mon[113531]: pgmap v2845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:23.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:23.046491+0000) 2023-08-27T22:28:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1489545706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1489545706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1489545706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:23.917Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:23.918Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:24.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:24.046750+0000) 2023-08-27T22:28:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:24 smithi093 ceph-mon[111335]: pgmap v2846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:24 smithi093 ceph-mon[118902]: pgmap v2846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:24 smithi136 ceph-mon[113531]: pgmap v2846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:25.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:25.046966+0000) 2023-08-27T22:28:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1445841212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:28:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:28:25] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:28:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1445841212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1445841212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:26.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:26.047163+0000) 2023-08-27T22:28:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:26 smithi093 ceph-mon[111335]: pgmap v2847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:26 smithi093 ceph-mon[118902]: pgmap v2847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:26 smithi136 ceph-mon[113531]: pgmap v2847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:27.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:27.047409+0000) 2023-08-27T22:28:28.047 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/7294098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/7294098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/7294098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:28.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:28.047660+0000) 2023-08-27T22:28:29.047 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:28 smithi136 ceph-mon[113531]: pgmap v2848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:28 smithi093 ceph-mon[111335]: pgmap v2848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:28 smithi093 ceph-mon[118902]: pgmap v2848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:29.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:29.047878+0000) 2023-08-27T22:28:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:30.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:30.048098+0000) 2023-08-27T22:28:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:30 smithi093 ceph-mon[111335]: pgmap v2849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2967971290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:30 smithi093 ceph-mon[118902]: pgmap v2849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2967971290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:30.490Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:30.491Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:30 smithi136 ceph-mon[113531]: pgmap v2849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2967971290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:31.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:31.048356+0000) 2023-08-27T22:28:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:32.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:32.048599+0000) 2023-08-27T22:28:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:32.004Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:32.006Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:33.048 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:32 smithi136 ceph-mon[113531]: pgmap v2850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:33.049 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3448155842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:32 smithi093 ceph-mon[111335]: pgmap v2850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3448155842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:32 smithi093 ceph-mon[118902]: pgmap v2850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3448155842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:33.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:33.048843+0000) 2023-08-27T22:28:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:33.918Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:33.919Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:34.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:34.049135+0000) 2023-08-27T22:28:35.049 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:34 smithi136 ceph-mon[113531]: pgmap v2851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:34 smithi093 ceph-mon[111335]: pgmap v2851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:34 smithi093 ceph-mon[118902]: pgmap v2851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:35.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:35.049419+0000) 2023-08-27T22:28:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:28:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:28:35] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:28:36.049 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3400044505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3400044505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3400044505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:36.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:36.049644+0000) 2023-08-27T22:28:37.050 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:36 smithi136 ceph-mon[113531]: pgmap v2852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:36 smithi093 ceph-mon[111335]: pgmap v2852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:36 smithi093 ceph-mon[118902]: pgmap v2852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:37.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:37.049885+0000) 2023-08-27T22:28:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3345176983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:38.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:38.050109+0000) 2023-08-27T22:28:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3345176983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3345176983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:39.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:39.050369+0000) 2023-08-27T22:28:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:38 smithi136 ceph-mon[113531]: pgmap v2853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:38 smithi093 ceph-mon[111335]: pgmap v2853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:38 smithi093 ceph-mon[118902]: pgmap v2853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:40.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:40.050623+0000) 2023-08-27T22:28:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1682701242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:40 smithi093 ceph-mon[111335]: pgmap v2854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1682701242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:40 smithi093 ceph-mon[118902]: pgmap v2854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:40.491Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:40.492Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1682701242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:40 smithi136 ceph-mon[113531]: pgmap v2854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:41.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:41.050880+0000) 2023-08-27T22:28:42.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:42.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:42.051168+0000) 2023-08-27T22:28:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:42.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:42.006Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:43.051 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:42 smithi136 ceph-mon[113531]: pgmap v2855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:43.051 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2738719018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:42 smithi093 ceph-mon[111335]: pgmap v2855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2738719018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:42 smithi093 ceph-mon[118902]: pgmap v2855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2738719018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:43.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:43.051405+0000) 2023-08-27T22:28:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:43.919Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:43.920Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:44.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:44.051671+0000) 2023-08-27T22:28:45.052 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:44 smithi136 ceph-mon[113531]: pgmap v2856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:45.052 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4085013584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:44 smithi093 ceph-mon[111335]: pgmap v2856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4085013584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:44 smithi093 ceph-mon[118902]: pgmap v2856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4085013584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:45.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:45.051892+0000) 2023-08-27T22:28:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:28:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:28:45] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T22:28:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:46.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:46.052163+0000) 2023-08-27T22:28:47.052 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:46 smithi136 ceph-mon[113531]: pgmap v2857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:47.052 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2920713463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:46 smithi093 ceph-mon[111335]: pgmap v2857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2920713463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:46 smithi093 ceph-mon[118902]: pgmap v2857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2920713463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:47.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:47.052415+0000) 2023-08-27T22:28:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:48.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:48.052650+0000) 2023-08-27T22:28:49.052 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:48 smithi136 ceph-mon[113531]: pgmap v2858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:48 smithi093 ceph-mon[111335]: pgmap v2858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:48 smithi093 ceph-mon[118902]: pgmap v2858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:49.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:49.052894+0000) 2023-08-27T22:28:50.053 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4098416761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4098416761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4098416761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:50.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:50.053117+0000) 2023-08-27T22:28:50.800 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:50.492Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:50.800 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:50.493Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:51.053 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:50 smithi136 ceph-mon[113531]: pgmap v2859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:50 smithi093 ceph-mon[111335]: pgmap v2859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:50 smithi093 ceph-mon[118902]: pgmap v2859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:51.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:51.053316+0000) 2023-08-27T22:28:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1512616731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:52.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:52.053557+0000) 2023-08-27T22:28:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1512616731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1512616731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:52.006Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:52.007Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:52 smithi136 ceph-mon[113531]: pgmap v2860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:53.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:53.053819+0000) 2023-08-27T22:28:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:52 smithi093 ceph-mon[111335]: pgmap v2860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:52 smithi093 ceph-mon[118902]: pgmap v2860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:54.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:54.054106+0000) 2023-08-27T22:28:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:28:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:53.919Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:28:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:28:53.920Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:28:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:54 smithi136 ceph-mon[113531]: pgmap v2861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/215844968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:55.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:55.054368+0000) 2023-08-27T22:28:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:54 smithi093 ceph-mon[111335]: pgmap v2861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/215844968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:54 smithi093 ceph-mon[118902]: pgmap v2861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/215844968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:28:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:28:55] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:28:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:56.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:56.054645+0000) 2023-08-27T22:28:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:56 smithi136 ceph-mon[113531]: pgmap v2862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3876688804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:28:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:57.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:57.054801+0000) 2023-08-27T22:28:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:56 smithi093 ceph-mon[111335]: pgmap v2862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3876688804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:28:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:56 smithi093 ceph-mon[118902]: pgmap v2862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3876688804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:28:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:58.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:58.054971+0000) 2023-08-27T22:28:59.026 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:58 smithi136 ceph-mon[113531]: pgmap v2863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:59.026 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:28:59.026 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:28:59.026 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:28:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3124141034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[111335]: pgmap v2863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:28:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:28:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3124141034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[118902]: pgmap v2863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:28:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:28:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:28:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:28:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3124141034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:28:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:28:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:28:59.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:28:59.055161+0000) 2023-08-27T22:29:00.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:00.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:00.055372+0000) 2023-08-27T22:29:00.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:29:00.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:29:00.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:00 smithi136 ceph-mon[113531]: pgmap v2864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:29:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:29:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[111335]: pgmap v2864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:00.493Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:00.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:00.494Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:29:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:29:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:29:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:00 smithi093 ceph-mon[118902]: pgmap v2864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:01.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:01.055624+0000) 2023-08-27T22:29:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3641067408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3641067408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:02.055 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3641067408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:02.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:02.055868+0000) 2023-08-27T22:29:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:02.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:02.008Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:02 smithi093 ceph-mon[111335]: pgmap v2865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:02 smithi093 ceph-mon[118902]: pgmap v2865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:03.056 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:02 smithi136 ceph-mon[113531]: pgmap v2865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:03.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:03.056118+0000) 2023-08-27T22:29:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3010442843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3010442843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:03.920Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:03.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:03.921Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:04.056 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3010442843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:04.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:04.056280+0000) 2023-08-27T22:29:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:04 smithi093 ceph-mon[118902]: pgmap v2866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:04 smithi093 ceph-mon[111335]: pgmap v2866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:05.056 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:04 smithi136 ceph-mon[113531]: pgmap v2866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:05.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:05.056568+0000) 2023-08-27T22:29:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:29:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:29:05] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:29:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:06.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:06.056818+0000) 2023-08-27T22:29:07.057 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:06 smithi136 ceph-mon[113531]: pgmap v2867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:07.057 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3245892944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:06 smithi093 ceph-mon[111335]: pgmap v2867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3245892944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:06 smithi093 ceph-mon[118902]: pgmap v2867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3245892944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:07.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:07.056995+0000) 2023-08-27T22:29:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:08.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:08.057216+0000) 2023-08-27T22:29:09.057 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:08 smithi136 ceph-mon[113531]: pgmap v2868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:09.057 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3527363081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:09.057 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:08 smithi093 ceph-mon[111335]: pgmap v2868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3527363081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:08 smithi093 ceph-mon[118902]: pgmap v2868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3527363081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:09.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:09.057431+0000) 2023-08-27T22:29:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:10.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:10.057685+0000) 2023-08-27T22:29:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:10 smithi093 ceph-mon[111335]: pgmap v2869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:10 smithi093 ceph-mon[118902]: pgmap v2869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:10.494Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:10.495Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:10 smithi136 ceph-mon[113531]: pgmap v2869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:11.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:11.057940+0000) 2023-08-27T22:29:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2289192484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2289192484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2289192484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:12.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:12.058194+0000) 2023-08-27T22:29:12.474 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:12.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:12.474 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:12.008Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:12 smithi093 ceph-mon[111335]: pgmap v2870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:12 smithi093 ceph-mon[118902]: pgmap v2870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:12 smithi136 ceph-mon[113531]: pgmap v2870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:13.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:13.058388+0000) 2023-08-27T22:29:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2714869565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2714869565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2714869565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:13.922Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:13.923Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:14.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:14.058619+0000) 2023-08-27T22:29:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:14 smithi093 ceph-mon[111335]: pgmap v2871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:14 smithi093 ceph-mon[118902]: pgmap v2871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:14 smithi136 ceph-mon[113531]: pgmap v2871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:15.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:15.058868+0000) 2023-08-27T22:29:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1388398584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:29:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:29:15] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:29:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1388398584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1388398584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:16.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:16.059158+0000) 2023-08-27T22:29:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:16 smithi093 ceph-mon[111335]: pgmap v2872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:16 smithi093 ceph-mon[118902]: pgmap v2872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:16 smithi136 ceph-mon[113531]: pgmap v2872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:17.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:17.059334+0000) 2023-08-27T22:29:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1364938481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:18.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:18.059586+0000) 2023-08-27T22:29:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1364938481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1364938481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:18 smithi136 ceph-mon[113531]: pgmap v2873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:19.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:19.059859+0000) 2023-08-27T22:29:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:18 smithi093 ceph-mon[111335]: pgmap v2873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:18 smithi093 ceph-mon[118902]: pgmap v2873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:20.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:20.060094+0000) 2023-08-27T22:29:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:20 smithi093 ceph-mon[111335]: pgmap v2874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3955464404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:20 smithi093 ceph-mon[118902]: pgmap v2874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3955464404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:20.495Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:20.496Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:20 smithi136 ceph-mon[113531]: pgmap v2874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3955464404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:21.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:21.060358+0000) 2023-08-27T22:29:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:22.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:22.060587+0000) 2023-08-27T22:29:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:22.008Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:22.009Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:23.060 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:22 smithi136 ceph-mon[113531]: pgmap v2875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:23.061 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3047035559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:22 smithi093 ceph-mon[111335]: pgmap v2875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3047035559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:22 smithi093 ceph-mon[118902]: pgmap v2875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3047035559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:23.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:23.060791+0000) 2023-08-27T22:29:24.061 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:23.922Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:23.924Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:24.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:24.061078+0000) 2023-08-27T22:29:25.061 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:24 smithi136 ceph-mon[113531]: pgmap v2876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:24 smithi093 ceph-mon[111335]: pgmap v2876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:24 smithi093 ceph-mon[118902]: pgmap v2876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:25.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:25.061405+0000) 2023-08-27T22:29:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:29:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:29:25] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:29:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1495158448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:26.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:26.061681+0000) 2023-08-27T22:29:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1495158448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1495158448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:26 smithi136 ceph-mon[113531]: pgmap v2877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:27.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:27.061970+0000) 2023-08-27T22:29:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:26 smithi093 ceph-mon[111335]: pgmap v2877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:26 smithi093 ceph-mon[118902]: pgmap v2877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/181096662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:28.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:28.062189+0000) 2023-08-27T22:29:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/181096662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/181096662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:28 smithi136 ceph-mon[113531]: pgmap v2878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:29.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:29.062410+0000) 2023-08-27T22:29:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:28 smithi093 ceph-mon[111335]: pgmap v2878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:28 smithi093 ceph-mon[118902]: pgmap v2878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:30.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:30.062666+0000) 2023-08-27T22:29:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2834016922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:30 smithi093 ceph-mon[111335]: pgmap v2879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2834016922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:30 smithi093 ceph-mon[118902]: pgmap v2879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:30.495Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:30.496Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2834016922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:30 smithi136 ceph-mon[113531]: pgmap v2879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:31.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:31.062886+0000) 2023-08-27T22:29:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:32.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:32.063149+0000) 2023-08-27T22:29:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:32.008Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:32.009Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:33.063 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:32 smithi136 ceph-mon[113531]: pgmap v2880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:33.063 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2731241390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:32 smithi093 ceph-mon[111335]: pgmap v2880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2731241390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:32 smithi093 ceph-mon[118902]: pgmap v2880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2731241390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:33.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:33.063434+0000) 2023-08-27T22:29:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:33.923Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:33.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:34.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:34.063695+0000) 2023-08-27T22:29:35.064 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:34 smithi136 ceph-mon[113531]: pgmap v2881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:35.064 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1908010094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:34 smithi093 ceph-mon[111335]: pgmap v2881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1908010094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:34 smithi093 ceph-mon[118902]: pgmap v2881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1908010094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:35.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:35.063977+0000) 2023-08-27T22:29:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:29:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:29:35] "GET /metrics HTTP/1.1" 200 239885 "" "Prometheus/2.43.0" 2023-08-27T22:29:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:36.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:36.064232+0000) 2023-08-27T22:29:37.064 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:36 smithi136 ceph-mon[113531]: pgmap v2882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:36 smithi093 ceph-mon[111335]: pgmap v2882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:36 smithi093 ceph-mon[118902]: pgmap v2882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:37.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:37.064482+0000) 2023-08-27T22:29:38.064 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2630015076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2630015076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2630015076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:38.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:38.064775+0000) 2023-08-27T22:29:39.065 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:38 smithi136 ceph-mon[113531]: pgmap v2883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:39.065 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:38 smithi093 ceph-mon[111335]: pgmap v2883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:38 smithi093 ceph-mon[118902]: pgmap v2883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:39.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:39.064991+0000) 2023-08-27T22:29:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/136695885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:40.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:40.065262+0000) 2023-08-27T22:29:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/136695885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/136695885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:40.817 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:40.496Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:40.817 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:40.497Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:40 smithi136 ceph-mon[113531]: pgmap v2884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:41.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:41.065541+0000) 2023-08-27T22:29:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:40 smithi093 ceph-mon[111335]: pgmap v2884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:40 smithi093 ceph-mon[118902]: pgmap v2884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4106583959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:42.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:42.065763+0000) 2023-08-27T22:29:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4106583959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4106583959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:42.008Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:42.009Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:42 smithi136 ceph-mon[113531]: pgmap v2885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:43.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:43.065996+0000) 2023-08-27T22:29:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:42 smithi093 ceph-mon[111335]: pgmap v2885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:42 smithi093 ceph-mon[118902]: pgmap v2885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:43.924Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:43.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:44.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:44.066206+0000) 2023-08-27T22:29:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:44 smithi136 ceph-mon[113531]: pgmap v2886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/444410819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:45.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:45.066470+0000) 2023-08-27T22:29:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:44 smithi093 ceph-mon[111335]: pgmap v2886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/444410819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:44 smithi093 ceph-mon[118902]: pgmap v2886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/444410819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:29:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:29:45] "GET /metrics HTTP/1.1" 200 239879 "" "Prometheus/2.43.0" 2023-08-27T22:29:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:46.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:46.066653+0000) 2023-08-27T22:29:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:46 smithi136 ceph-mon[113531]: pgmap v2887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/828817274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:47.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:47.066918+0000) 2023-08-27T22:29:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:46 smithi093 ceph-mon[111335]: pgmap v2887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/828817274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:46 smithi093 ceph-mon[118902]: pgmap v2887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/828817274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:48.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:48.067174+0000) 2023-08-27T22:29:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:48 smithi136 ceph-mon[113531]: pgmap v2888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020183118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:49.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:49.067388+0000) 2023-08-27T22:29:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:48 smithi093 ceph-mon[111335]: pgmap v2888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020183118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:48 smithi093 ceph-mon[118902]: pgmap v2888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020183118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:50.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:50.067603+0000) 2023-08-27T22:29:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:50 smithi093 ceph-mon[111335]: pgmap v2889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:50 smithi093 ceph-mon[118902]: pgmap v2889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:50.496Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:50.497Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:50 smithi136 ceph-mon[113531]: pgmap v2889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:51.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:51.067862+0000) 2023-08-27T22:29:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1661540630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1661540630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1661540630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:52.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:52.068116+0000) 2023-08-27T22:29:52.485 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:52.009Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:52.485 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:52.010Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:52 smithi093 ceph-mon[111335]: pgmap v2890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:52 smithi093 ceph-mon[118902]: pgmap v2890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:52 smithi136 ceph-mon[113531]: pgmap v2890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:53.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:53.068379+0000) 2023-08-27T22:29:53.924 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3236360050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:53.925 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3236360050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3236360050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:53.924Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:29:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:29:53.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:29:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:54.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:54.068642+0000) 2023-08-27T22:29:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:54 smithi136 ceph-mon[113531]: pgmap v2891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:54 smithi093 ceph-mon[111335]: pgmap v2891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:29:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:54 smithi093 ceph-mon[118902]: pgmap v2891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:55.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:55.068852+0000) 2023-08-27T22:29:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:29:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:29:55] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:29:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:56.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:56.069078+0000) 2023-08-27T22:29:57.069 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:56 smithi136 ceph-mon[113531]: pgmap v2892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:57.069 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3108442936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:56 smithi093 ceph-mon[111335]: pgmap v2892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3108442936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:56 smithi093 ceph-mon[118902]: pgmap v2892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3108442936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:57.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:57.069323+0000) 2023-08-27T22:29:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:58.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:58.069566+0000) 2023-08-27T22:29:59.069 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:58 smithi136 ceph-mon[113531]: pgmap v2893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:59.070 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:29:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020997384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:58 smithi093 ceph-mon[111335]: pgmap v2893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:29:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020997384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:58 smithi093 ceph-mon[118902]: pgmap v2893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:29:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:29:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020997384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:29:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:29:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:29:59.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:29:59.069779+0000) 2023-08-27T22:30:00.421 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:00.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:00.069957+0000) 2023-08-27T22:30:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:00 smithi136 ceph-mon[113531]: pgmap v2894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:30:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:30:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-mon[111335]: pgmap v2894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:30:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:30:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-mon[118902]: pgmap v2894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:30:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:30:00.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:00.497Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:00.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:00.499Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:01.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:01.070126+0000) 2023-08-27T22:30:01.731 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2906430143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2906430143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2906430143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:02.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:02.070306+0000) 2023-08-27T22:30:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:02.010Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:02.011Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:02 smithi136 ceph-mon[113531]: pgmap v2895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[118902]: pgmap v2895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[111335]: pgmap v2895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:03.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:03.070548+0000) 2023-08-27T22:30:03.925 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:30:03.925 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:30:03.926 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:03.926 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3802213950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:03.926 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:30:03.926 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:30:03.926 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:03.926 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3802213950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:30:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:30:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:30:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3802213950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:03.925Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:03.926Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:04.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:04.070802+0000) 2023-08-27T22:30:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:04 smithi136 ceph-mon[113531]: pgmap v2896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:04 smithi093 ceph-mon[111335]: pgmap v2896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:04 smithi093 ceph-mon[118902]: pgmap v2896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:05.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:05.071099+0000) 2023-08-27T22:30:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/690543502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:30:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:30:05] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:30:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/690543502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/690543502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:06.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:06.071377+0000) 2023-08-27T22:30:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:06 smithi136 ceph-mon[113531]: pgmap v2897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:06.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:06 smithi093 ceph-mon[118902]: pgmap v2897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:06 smithi093 ceph-mon[111335]: pgmap v2897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:07.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:07.071591+0000) 2023-08-27T22:30:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:08.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:08.071879+0000) 2023-08-27T22:30:09.072 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:08 smithi136 ceph-mon[113531]: pgmap v2898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:09.072 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1803531161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:09.072 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:08 smithi093 ceph-mon[111335]: pgmap v2898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1803531161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:08 smithi093 ceph-mon[118902]: pgmap v2898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1803531161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:09.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:09.072181+0000) 2023-08-27T22:30:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:10.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:10.072486+0000) 2023-08-27T22:30:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:10 smithi093 ceph-mon[111335]: pgmap v2899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4075143977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:10 smithi093 ceph-mon[118902]: pgmap v2899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4075143977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:10.498Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:10.499Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:10 smithi136 ceph-mon[113531]: pgmap v2899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4075143977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:11.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:11.072710+0000) 2023-08-27T22:30:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:12.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:12.072971+0000) 2023-08-27T22:30:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:12.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:12.012Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:13.073 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:12 smithi136 ceph-mon[113531]: pgmap v2900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:13.073 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1224300086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:12 smithi093 ceph-mon[111335]: pgmap v2900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1224300086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:12 smithi093 ceph-mon[118902]: pgmap v2900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1224300086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:13.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:13.073252+0000) 2023-08-27T22:30:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:13.926Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:13.927Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:14.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:14.073503+0000) 2023-08-27T22:30:15.073 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:14 smithi136 ceph-mon[113531]: pgmap v2901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:14 smithi093 ceph-mon[111335]: pgmap v2901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:14 smithi093 ceph-mon[118902]: pgmap v2901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:15.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:15.073752+0000) 2023-08-27T22:30:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:30:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:30:15] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:30:16.074 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1374879090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1374879090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1374879090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:16.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:16.073940+0000) 2023-08-27T22:30:17.074 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:16 smithi136 ceph-mon[113531]: pgmap v2902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:16 smithi093 ceph-mon[111335]: pgmap v2902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:16 smithi093 ceph-mon[118902]: pgmap v2902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:17.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:17.074197+0000) 2023-08-27T22:30:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4194937194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:18.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:18.074446+0000) 2023-08-27T22:30:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4194937194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4194937194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:18 smithi136 ceph-mon[113531]: pgmap v2903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:19.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:19.074665+0000) 2023-08-27T22:30:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:18 smithi093 ceph-mon[111335]: pgmap v2903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:18 smithi093 ceph-mon[118902]: pgmap v2903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:20.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:20.074913+0000) 2023-08-27T22:30:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/603554967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:20 smithi093 ceph-mon[111335]: pgmap v2904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/603554967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:20 smithi093 ceph-mon[118902]: pgmap v2904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:20.499Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:20.500Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/603554967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:20 smithi136 ceph-mon[113531]: pgmap v2904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:21.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:21.075161+0000) 2023-08-27T22:30:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:22.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:22.075385+0000) 2023-08-27T22:30:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:22.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:22.012Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:23.075 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:22 smithi136 ceph-mon[113531]: pgmap v2905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:23.075 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3440602501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:22 smithi093 ceph-mon[111335]: pgmap v2905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3440602501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:22 smithi093 ceph-mon[118902]: pgmap v2905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3440602501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:23.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:23.075621+0000) 2023-08-27T22:30:24.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:23.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:23.928Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:24.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:24.075855+0000) 2023-08-27T22:30:25.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:24 smithi136 ceph-mon[113531]: pgmap v2906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:25.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2491355782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:24 smithi093 ceph-mon[111335]: pgmap v2906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2491355782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:24 smithi093 ceph-mon[118902]: pgmap v2906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2491355782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:25.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:25.076152+0000) 2023-08-27T22:30:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:30:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:30:25] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T22:30:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:26.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:26.076361+0000) 2023-08-27T22:30:27.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:26 smithi136 ceph-mon[113531]: pgmap v2907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:26 smithi093 ceph-mon[111335]: pgmap v2907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:26 smithi093 ceph-mon[118902]: pgmap v2907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:27.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:27.076573+0000) 2023-08-27T22:30:28.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1623286931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1623286931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1623286931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:28.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:28.076779+0000) 2023-08-27T22:30:29.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:28 smithi136 ceph-mon[113531]: pgmap v2908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:28 smithi093 ceph-mon[111335]: pgmap v2908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:28 smithi093 ceph-mon[118902]: pgmap v2908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:29.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:29.076995+0000) 2023-08-27T22:30:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/402175132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:30.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:30.077239+0000) 2023-08-27T22:30:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/402175132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/402175132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:30.829 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:30.500Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:30.829 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:30.501Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:30 smithi136 ceph-mon[113531]: pgmap v2909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:31.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:31.077495+0000) 2023-08-27T22:30:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:30 smithi093 ceph-mon[111335]: pgmap v2909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:30 smithi093 ceph-mon[118902]: pgmap v2909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2698412188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:32.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:32.077768+0000) 2023-08-27T22:30:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2698412188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2698412188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:32.012Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:32.013Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:32 smithi136 ceph-mon[113531]: pgmap v2910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:33.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:33.078015+0000) 2023-08-27T22:30:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:32 smithi093 ceph-mon[111335]: pgmap v2910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:32 smithi093 ceph-mon[118902]: pgmap v2910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:33.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:33.929Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:34.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:34.078267+0000) 2023-08-27T22:30:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:34 smithi136 ceph-mon[113531]: pgmap v2911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1444666666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:35.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:35.078551+0000) 2023-08-27T22:30:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:34 smithi093 ceph-mon[111335]: pgmap v2911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1444666666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:34 smithi093 ceph-mon[118902]: pgmap v2911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1444666666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:30:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:30:35] "GET /metrics HTTP/1.1" 200 239877 "" "Prometheus/2.43.0" 2023-08-27T22:30:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:36.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:36.078828+0000) 2023-08-27T22:30:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:36 smithi136 ceph-mon[113531]: pgmap v2912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1481226948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:37.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:37.079100+0000) 2023-08-27T22:30:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:36 smithi093 ceph-mon[111335]: pgmap v2912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1481226948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:36 smithi093 ceph-mon[118902]: pgmap v2912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1481226948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:38.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:38.079347+0000) 2023-08-27T22:30:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:38 smithi136 ceph-mon[113531]: pgmap v2913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1183496199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:39.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:39.079610+0000) 2023-08-27T22:30:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:38 smithi093 ceph-mon[111335]: pgmap v2913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1183496199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:38 smithi093 ceph-mon[118902]: pgmap v2913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1183496199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:40.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:40.079850+0000) 2023-08-27T22:30:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:40 smithi093 ceph-mon[111335]: pgmap v2914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:40 smithi093 ceph-mon[118902]: pgmap v2914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:40.501Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:40.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:40.502Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:40 smithi136 ceph-mon[113531]: pgmap v2914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:41.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:41.080114+0000) 2023-08-27T22:30:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/400992439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/400992439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/400992439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:42.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:42.080380+0000) 2023-08-27T22:30:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:42.013Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:42.014Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:42 smithi136 ceph-mon[113531]: pgmap v2915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:42 smithi093 ceph-mon[111335]: pgmap v2915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:42 smithi093 ceph-mon[118902]: pgmap v2915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:43.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:43.080641+0000) 2023-08-27T22:30:43.928 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3844226402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:43.928 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3844226402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3844226402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:43.928Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:43.929Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:44.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:44.080869+0000) 2023-08-27T22:30:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:44 smithi136 ceph-mon[113531]: pgmap v2916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:44 smithi093 ceph-mon[111335]: pgmap v2916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:44 smithi093 ceph-mon[118902]: pgmap v2916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:45.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:45.081111+0000) 2023-08-27T22:30:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:30:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:30:45] "GET /metrics HTTP/1.1" 200 239883 "" "Prometheus/2.43.0" 2023-08-27T22:30:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:46.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:46.081369+0000) 2023-08-27T22:30:47.081 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:46 smithi136 ceph-mon[113531]: pgmap v2917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:47.081 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1742657543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:46 smithi093 ceph-mon[111335]: pgmap v2917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1742657543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:46 smithi093 ceph-mon[118902]: pgmap v2917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1742657543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:47.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:47.081571+0000) 2023-08-27T22:30:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:48.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:48.081831+0000) 2023-08-27T22:30:49.082 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:48 smithi136 ceph-mon[113531]: pgmap v2918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:49.082 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2836786045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:48 smithi093 ceph-mon[111335]: pgmap v2918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2836786045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:48 smithi093 ceph-mon[118902]: pgmap v2918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2836786045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:49.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:49.082067+0000) 2023-08-27T22:30:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:50.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:50.082300+0000) 2023-08-27T22:30:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:50 smithi093 ceph-mon[111335]: pgmap v2919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:50 smithi093 ceph-mon[118902]: pgmap v2919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:50.502Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:50.503Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:50 smithi136 ceph-mon[113531]: pgmap v2919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:51.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:51.082535+0000) 2023-08-27T22:30:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2904489120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2904489120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2904489120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:52.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:52.082773+0000) 2023-08-27T22:30:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:52.014Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:52.015Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:52 smithi136 ceph-mon[113531]: pgmap v2920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:52 smithi093 ceph-mon[111335]: pgmap v2920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:52 smithi093 ceph-mon[118902]: pgmap v2920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:53.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:53.083082+0000) 2023-08-27T22:30:53.929 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/433146199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:53.930 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/433146199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/433146199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:53.929Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:30:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:30:53.930Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:30:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:54.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:54.083352+0000) 2023-08-27T22:30:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:54 smithi136 ceph-mon[113531]: pgmap v2921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:54 smithi093 ceph-mon[111335]: pgmap v2921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:30:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:54 smithi093 ceph-mon[118902]: pgmap v2921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:55.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:55.083645+0000) 2023-08-27T22:30:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3393037702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:30:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:30:55] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:30:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3393037702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3393037702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:56.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:56.083877+0000) 2023-08-27T22:30:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:56 smithi136 ceph-mon[113531]: pgmap v2922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:56 smithi093 ceph-mon[111335]: pgmap v2922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:56 smithi093 ceph-mon[118902]: pgmap v2922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:57.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:57.084155+0000) 2023-08-27T22:30:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:58.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:58.084432+0000) 2023-08-27T22:30:59.084 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:58 smithi136 ceph-mon[113531]: pgmap v2923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:59.085 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:30:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4067258106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:58 smithi093 ceph-mon[111335]: pgmap v2923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:30:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4067258106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:58 smithi093 ceph-mon[118902]: pgmap v2923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:30:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:30:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4067258106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:30:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:30:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:30:59.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:30:59.084646+0000) 2023-08-27T22:31:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:00.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:00.084946+0000) 2023-08-27T22:31:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:00 smithi093 ceph-mon[118902]: pgmap v2924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1201571703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:00 smithi093 ceph-mon[111335]: pgmap v2924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1201571703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:00.503Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:00.504Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:00 smithi136 ceph-mon[113531]: pgmap v2924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1201571703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:01.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:01.085169+0000) 2023-08-27T22:31:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:02.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:02.085412+0000) 2023-08-27T22:31:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:02.015Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:02.016Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:03.085 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:02 smithi136 ceph-mon[113531]: pgmap v2925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:03.085 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/395557954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:02 smithi093 ceph-mon[111335]: pgmap v2925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/395557954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:02 smithi093 ceph-mon[118902]: pgmap v2925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/395557954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:03.436 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:03.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:03.085582+0000) 2023-08-27T22:31:04.086 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:31:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:31:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:31:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:03.930Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:03.932Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:04.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:04.085747+0000) 2023-08-27T22:31:05.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:04 smithi136 ceph-mon[113531]: pgmap v2926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:05.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:05.037 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:04 smithi093 ceph-mon[111335]: pgmap v2926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:04 smithi093 ceph-mon[118902]: pgmap v2926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:05.317 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:05.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:05.086003+0000) 2023-08-27T22:31:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T22:31:05.317575657Z level=info msg="Completed cleanup jobs" duration=12.586849ms 2023-08-27T22:31:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:31:05] "GET /metrics HTTP/1.1" 200 239882 "" "Prometheus/2.43.0" 2023-08-27T22:31:06.086 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3737973217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:06.086 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.086 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.087 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:31:06.087 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:31:06.087 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3737973217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3737973217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:31:06.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:31:06.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:31:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:06.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:06.086232+0000) 2023-08-27T22:31:07.086 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:06 smithi136 ceph-mon[113531]: pgmap v2927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:06 smithi093 ceph-mon[111335]: pgmap v2927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:06 smithi093 ceph-mon[118902]: pgmap v2927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:07.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:07.086423+0000) 2023-08-27T22:31:08.086 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1719200088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1719200088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1719200088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:08.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:08.086639+0000) 2023-08-27T22:31:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:08 smithi136 ceph-mon[113531]: pgmap v2928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:09.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:09.086918+0000) 2023-08-27T22:31:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:08 smithi093 ceph-mon[111335]: pgmap v2928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:08 smithi093 ceph-mon[118902]: pgmap v2928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:10.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:10.087134+0000) 2023-08-27T22:31:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:10 smithi093 ceph-mon[111335]: pgmap v2929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3886330311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:10.504Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:10.504Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:10 smithi093 ceph-mon[118902]: pgmap v2929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:10.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3886330311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:10 smithi136 ceph-mon[113531]: pgmap v2929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3886330311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:11.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:11.087349+0000) 2023-08-27T22:31:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:12.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:12.087613+0000) 2023-08-27T22:31:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:12.016Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:12.016Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:13.087 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:12 smithi136 ceph-mon[113531]: pgmap v2930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:13.088 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4152544088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:13.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:12 smithi093 ceph-mon[111335]: pgmap v2930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:13.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4152544088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:13.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:12 smithi093 ceph-mon[118902]: pgmap v2930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:13.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4152544088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:13.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:13.087871+0000) 2023-08-27T22:31:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:13.932Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:13.933Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:14.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:14.088159+0000) 2023-08-27T22:31:15.088 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:14 smithi136 ceph-mon[113531]: pgmap v2931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:15.088 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1031670590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:14 smithi093 ceph-mon[111335]: pgmap v2931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1031670590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:14 smithi093 ceph-mon[118902]: pgmap v2931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1031670590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:15.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:15.088395+0000) 2023-08-27T22:31:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:31:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:31:15] "GET /metrics HTTP/1.1" 200 239878 "" "Prometheus/2.43.0" 2023-08-27T22:31:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:16.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:16.088652+0000) 2023-08-27T22:31:17.088 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:16 smithi136 ceph-mon[113531]: pgmap v2932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:16 smithi093 ceph-mon[111335]: pgmap v2932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:16 smithi093 ceph-mon[118902]: pgmap v2932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:17.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:17.088869+0000) 2023-08-27T22:31:18.089 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2019151159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2019151159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2019151159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:18.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:18.089150+0000) 2023-08-27T22:31:19.089 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:18 smithi136 ceph-mon[113531]: pgmap v2933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:18 smithi093 ceph-mon[111335]: pgmap v2933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:18 smithi093 ceph-mon[118902]: pgmap v2933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:19.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:19.089470+0000) 2023-08-27T22:31:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1165346146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:20.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:20.089743+0000) 2023-08-27T22:31:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1165346146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1165346146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:20.841 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:20.505Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:20.842 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:20.506Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:20 smithi136 ceph-mon[113531]: pgmap v2934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:21.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:21.090002+0000) 2023-08-27T22:31:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:20 smithi093 ceph-mon[111335]: pgmap v2934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:20 smithi093 ceph-mon[118902]: pgmap v2934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/199761899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:22.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:22.090297+0000) 2023-08-27T22:31:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/199761899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/199761899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:22.016Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:22.017Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:22 smithi136 ceph-mon[113531]: pgmap v2935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:23.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:23.090565+0000) 2023-08-27T22:31:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:22 smithi093 ceph-mon[111335]: pgmap v2935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:22 smithi093 ceph-mon[118902]: pgmap v2935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:24.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:24.090838+0000) 2023-08-27T22:31:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:23.933Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:23.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:24 smithi136 ceph-mon[113531]: pgmap v2936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/345816249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:25.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:25.091020+0000) 2023-08-27T22:31:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:24 smithi093 ceph-mon[111335]: pgmap v2936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/345816249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:24 smithi093 ceph-mon[118902]: pgmap v2936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/345816249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:31:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:31:25] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:31:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:26.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:26.091291+0000) 2023-08-27T22:31:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:26 smithi136 ceph-mon[113531]: pgmap v2937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/65144210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:27.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:27.091529+0000) 2023-08-27T22:31:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:26 smithi093 ceph-mon[111335]: pgmap v2937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/65144210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:26 smithi093 ceph-mon[118902]: pgmap v2937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/65144210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:28.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:28.091773+0000) 2023-08-27T22:31:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:28 smithi136 ceph-mon[113531]: pgmap v2938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/773663153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:29.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:29.092026+0000) 2023-08-27T22:31:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:28 smithi093 ceph-mon[111335]: pgmap v2938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/773663153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:28 smithi093 ceph-mon[118902]: pgmap v2938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/773663153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:30.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:30.092366+0000) 2023-08-27T22:31:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:30 smithi136 ceph-mon[113531]: pgmap v2939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:30 smithi093 ceph-mon[111335]: pgmap v2939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:30 smithi093 ceph-mon[118902]: pgmap v2939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:30.506Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:30.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:31.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:31.092616+0000) 2023-08-27T22:31:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/859290187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/859290187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/859290187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:32.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:32.092853+0000) 2023-08-27T22:31:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:32.016Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:32.017Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:32 smithi136 ceph-mon[113531]: pgmap v2940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:32 smithi093 ceph-mon[111335]: pgmap v2940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:32 smithi093 ceph-mon[118902]: pgmap v2940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:33.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:33.093137+0000) 2023-08-27T22:31:33.933 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4248449567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:33.934 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4248449567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4248449567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:33.933Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:33.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:34.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:34.093318+0000) 2023-08-27T22:31:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:34 smithi136 ceph-mon[113531]: pgmap v2941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:34 smithi093 ceph-mon[111335]: pgmap v2941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:34 smithi093 ceph-mon[118902]: pgmap v2941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:35.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:35.093551+0000) 2023-08-27T22:31:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:31:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:31:35] "GET /metrics HTTP/1.1" 200 239881 "" "Prometheus/2.43.0" 2023-08-27T22:31:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:36.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:36.093802+0000) 2023-08-27T22:31:37.094 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:36 smithi136 ceph-mon[113531]: pgmap v2942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:37.094 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4032912583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:36 smithi093 ceph-mon[118902]: pgmap v2942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4032912583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:36 smithi093 ceph-mon[111335]: pgmap v2942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4032912583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:37.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:37.094069+0000) 2023-08-27T22:31:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:38.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:38.094321+0000) 2023-08-27T22:31:39.094 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:38 smithi136 ceph-mon[113531]: pgmap v2943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:39.094 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1122317538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:39.094 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:38 smithi093 ceph-mon[111335]: pgmap v2943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1122317538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:38 smithi093 ceph-mon[118902]: pgmap v2943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1122317538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:39.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:39.094556+0000) 2023-08-27T22:31:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:40.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:40.094745+0000) 2023-08-27T22:31:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:40 smithi136 ceph-mon[113531]: pgmap v2944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:40 smithi093 ceph-mon[111335]: pgmap v2944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:40 smithi093 ceph-mon[118902]: pgmap v2944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:40.506Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:40.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:41.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:41.095023+0000) 2023-08-27T22:31:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3496094776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3496094776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3496094776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:42.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:42.095310+0000) 2023-08-27T22:31:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:42.017Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:42.018Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:42 smithi136 ceph-mon[113531]: pgmap v2945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:42 smithi093 ceph-mon[111335]: pgmap v2945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:42 smithi093 ceph-mon[118902]: pgmap v2945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:43.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:43.095524+0000) 2023-08-27T22:31:43.934 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1747033200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:43.934 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1747033200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1747033200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:43.934Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:43.935Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:44.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:44.095751+0000) 2023-08-27T22:31:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:44 smithi136 ceph-mon[113531]: pgmap v2946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:44 smithi093 ceph-mon[111335]: pgmap v2946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:44 smithi093 ceph-mon[118902]: pgmap v2946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:45.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:45.096011+0000) 2023-08-27T22:31:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3828010389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:31:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:31:45] "GET /metrics HTTP/1.1" 200 239884 "" "Prometheus/2.43.0" 2023-08-27T22:31:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3828010389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3828010389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:46.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:46.096301+0000) 2023-08-27T22:31:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:46 smithi136 ceph-mon[113531]: pgmap v2947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:46 smithi093 ceph-mon[111335]: pgmap v2947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:46 smithi093 ceph-mon[118902]: pgmap v2947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:47.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:47.096568+0000) 2023-08-27T22:31:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:48.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:48.096819+0000) 2023-08-27T22:31:49.097 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:48 smithi136 ceph-mon[113531]: pgmap v2948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:49.097 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3708194807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:48 smithi093 ceph-mon[111335]: pgmap v2948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3708194807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:48 smithi093 ceph-mon[118902]: pgmap v2948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3708194807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:49.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:49.097070+0000) 2023-08-27T22:31:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:50.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:50.097330+0000) 2023-08-27T22:31:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:50 smithi136 ceph-mon[113531]: pgmap v2949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3402391957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:50 smithi093 ceph-mon[111335]: pgmap v2949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3402391957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:50 smithi093 ceph-mon[118902]: pgmap v2949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3402391957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:50.506Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:50.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:51.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:51.097571+0000) 2023-08-27T22:31:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:52.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:52.097810+0000) 2023-08-27T22:31:52.617 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:52.017Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:52.617 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:52.018Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:53.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:52 smithi136 ceph-mon[113531]: pgmap v2950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:53.212 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/492531824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:52 smithi093 ceph-mon[111335]: pgmap v2950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:53.424 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/492531824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:53.424 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:52 smithi093 ceph-mon[118902]: pgmap v2950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:53.425 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/492531824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:53.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:53.098088+0000) 2023-08-27T22:31:54.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:31:54.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:53.934Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:54.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:31:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:31:53.935Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:31:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:54.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:54.098359+0000) 2023-08-27T22:31:55.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:54 smithi136 ceph-mon[113531]: pgmap v2951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:55.916 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:54 smithi093 ceph-mon[118902]: pgmap v2951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:55.916 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:55.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:55.098649+0000) 2023-08-27T22:31:55.916 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:54 smithi093 ceph-mon[111335]: pgmap v2951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:55.919 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:31:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:31:55] "GET /metrics HTTP/1.1" 200 239890 "" "Prometheus/2.43.0" 2023-08-27T22:31:56.099 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3451438566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3451438566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3451438566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:56.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:56.098915+0000) 2023-08-27T22:31:57.099 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:56 smithi136 ceph-mon[113531]: pgmap v2952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:57.346 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:56 smithi093 ceph-mon[111335]: pgmap v2952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:57.346 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:56 smithi093 ceph-mon[118902]: pgmap v2952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:57.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:57.099193+0000) 2023-08-27T22:31:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3648713900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:58.298 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:58.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:58.099476+0000) 2023-08-27T22:31:58.299 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3648713900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:58.299 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3648713900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:31:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:31:58 smithi136 ceph-mon[113531]: pgmap v2953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:59.571 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:31:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:31:59.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:31:59.099691+0000) 2023-08-27T22:31:59.572 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:31:58 smithi093 ceph-mon[111335]: pgmap v2953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:31:59.572 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:31:58 smithi093 ceph-mon[118902]: pgmap v2953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:00.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:00.099892+0000) 2023-08-27T22:32:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:00 smithi136 ceph-mon[113531]: pgmap v2954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/130876749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:00 smithi093 ceph-mon[111335]: pgmap v2954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/130876749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:00 smithi093 ceph-mon[118902]: pgmap v2954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/130876749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:00.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:00.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:01.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:01.100070+0000) 2023-08-27T22:32:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:02.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:02.100323+0000) 2023-08-27T22:32:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:02.018Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:02.018Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:03.100 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:02 smithi136 ceph-mon[113531]: pgmap v2955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:03.100 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2415654695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:02 smithi093 ceph-mon[111335]: pgmap v2955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2415654695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:02 smithi093 ceph-mon[118902]: pgmap v2955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2415654695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:03.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:03.100647+0000) 2023-08-27T22:32:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:03.935Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:03.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:04.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:04.100913+0000) 2023-08-27T22:32:05.101 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:04 smithi136 ceph-mon[113531]: pgmap v2956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:05.101 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1855331433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:04 smithi093 ceph-mon[118902]: pgmap v2956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1855331433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:04 smithi093 ceph-mon[111335]: pgmap v2956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1855331433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:05.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:05.101173+0000) 2023-08-27T22:32:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:32:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:32:05] "GET /metrics HTTP/1.1" 200 239890 "" "Prometheus/2.43.0" 2023-08-27T22:32:06.432 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:06.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:06.101328+0000) 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[118902]: pgmap v2957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[111335]: pgmap v2957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:07.101 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:06 smithi136 ceph-mon[113531]: pgmap v2957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:07.285 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:32:07.414 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:07.414 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:07.414 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:07.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:07.101506+0000) 2023-08-27T22:32:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3538308358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3538308358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:08.101 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3538308358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:08.101 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:08.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:08.101700+0000) 2023-08-27T22:32:09.102 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: pgmap v2958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:09.102 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.102 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.102 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.102 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:09.102 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:32:09.102 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:32:09.103 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: pgmap v2958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: pgmap v2958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:32:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:09.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:09.101906+0000) 2023-08-27T22:32:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/712553584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:10.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:10.102180+0000) 2023-08-27T22:32:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/712553584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/712553584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:10.855 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:10.508Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:10.856 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:10.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:10 smithi136 ceph-mon[113531]: pgmap v2959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:11.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:11.102382+0000) 2023-08-27T22:32:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:10 smithi093 ceph-mon[111335]: pgmap v2959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:10 smithi093 ceph-mon[118902]: pgmap v2959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/191141076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:12.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:12.102643+0000) 2023-08-27T22:32:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/191141076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/191141076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:12.018Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:12.019Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:12 smithi136 ceph-mon[113531]: pgmap v2960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:13.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:13.102901+0000) 2023-08-27T22:32:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:12 smithi093 ceph-mon[111335]: pgmap v2960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:12 smithi093 ceph-mon[118902]: pgmap v2960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:14.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:13.936Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:14.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:13.937Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:14.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:14.103145+0000) 2023-08-27T22:32:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:14 smithi136 ceph-mon[113531]: pgmap v2961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1425729658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:15.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:15.103391+0000) 2023-08-27T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:14 smithi093 ceph-mon[111335]: pgmap v2961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1425729658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:14 smithi093 ceph-mon[118902]: pgmap v2961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1425729658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:32:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:32:15] "GET /metrics HTTP/1.1" 200 239889 "" "Prometheus/2.43.0" 2023-08-27T22:32:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:16.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:16.103627+0000) 2023-08-27T22:32:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:16 smithi136 ceph-mon[113531]: pgmap v2962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/316626693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:17.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:17.103864+0000) 2023-08-27T22:32:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:16 smithi093 ceph-mon[111335]: pgmap v2962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/316626693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:16 smithi093 ceph-mon[118902]: pgmap v2962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/316626693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:18.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:18.104089+0000) 2023-08-27T22:32:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:18 smithi136 ceph-mon[113531]: pgmap v2963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:19.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:19.104316+0000) 2023-08-27T22:32:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:18 smithi093 ceph-mon[111335]: pgmap v2963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:18 smithi093 ceph-mon[118902]: pgmap v2963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3892873907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:20.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:20.104568+0000) 2023-08-27T22:32:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3892873907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3892873907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:20.875 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:20.508Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:20.875 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:20.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:20 smithi136 ceph-mon[113531]: pgmap v2964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:21.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:21.104823+0000) 2023-08-27T22:32:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:20 smithi093 ceph-mon[118902]: pgmap v2964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:20 smithi093 ceph-mon[111335]: pgmap v2964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2176053298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:22.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:22.105107+0000) 2023-08-27T22:32:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2176053298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2176053298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:22.019Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:22.020Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:22 smithi136 ceph-mon[113531]: pgmap v2965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:23.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:23.105359+0000) 2023-08-27T22:32:23.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:22 smithi093 ceph-mon[111335]: pgmap v2965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:23.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:22 smithi093 ceph-mon[118902]: pgmap v2965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2288754855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:24.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:24.105632+0000) 2023-08-27T22:32:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2288754855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2288754855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:23.937Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:23.938Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:24 smithi136 ceph-mon[113531]: pgmap v2966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:25.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:25.105919+0000) 2023-08-27T22:32:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:24 smithi093 ceph-mon[111335]: pgmap v2966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:24 smithi093 ceph-mon[118902]: pgmap v2966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:32:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:32:25] "GET /metrics HTTP/1.1" 200 239896 "" "Prometheus/2.43.0" 2023-08-27T22:32:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:26.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:26.106176+0000) 2023-08-27T22:32:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:26 smithi136 ceph-mon[113531]: pgmap v2967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2186653922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:27.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:27.106400+0000) 2023-08-27T22:32:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:26 smithi093 ceph-mon[111335]: pgmap v2967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2186653922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:26 smithi093 ceph-mon[118902]: pgmap v2967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2186653922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:28.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:28.106629+0000) 2023-08-27T22:32:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:28 smithi136 ceph-mon[113531]: pgmap v2968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3189699754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:29.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:29.106884+0000) 2023-08-27T22:32:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:28 smithi093 ceph-mon[111335]: pgmap v2968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3189699754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:28 smithi093 ceph-mon[118902]: pgmap v2968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3189699754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:30.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:30.107155+0000) 2023-08-27T22:32:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:30 smithi136 ceph-mon[113531]: pgmap v2969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:30 smithi093 ceph-mon[111335]: pgmap v2969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:30 smithi093 ceph-mon[118902]: pgmap v2969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:30.509Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:30.510Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:31.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:31.107402+0000) 2023-08-27T22:32:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3542977781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3542977781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3542977781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:32.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:32.107644+0000) 2023-08-27T22:32:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:32.019Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:32.020Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:32 smithi136 ceph-mon[113531]: pgmap v2970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:32 smithi093 ceph-mon[111335]: pgmap v2970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:32 smithi093 ceph-mon[118902]: pgmap v2970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:33.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:33.107846+0000) 2023-08-27T22:32:33.939 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1178264518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:33.939 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1178264518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1178264518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:33.938Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:33.940Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:34.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:34.108082+0000) 2023-08-27T22:32:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:34 smithi136 ceph-mon[113531]: pgmap v2971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:34 smithi093 ceph-mon[111335]: pgmap v2971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:34 smithi093 ceph-mon[118902]: pgmap v2971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:35.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:35.108285+0000) 2023-08-27T22:32:35.636 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:32:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:32:35] "GET /metrics HTTP/1.1" 200 239896 "" "Prometheus/2.43.0" 2023-08-27T22:32:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/244710485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/244710485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/244710485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:36.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:36.108567+0000) 2023-08-27T22:32:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:36 smithi136 ceph-mon[113531]: pgmap v2972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:36 smithi093 ceph-mon[111335]: pgmap v2972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:36 smithi093 ceph-mon[118902]: pgmap v2972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:37.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:37.108812+0000) 2023-08-27T22:32:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:38.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:38.109087+0000) 2023-08-27T22:32:39.109 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:38 smithi136 ceph-mon[113531]: pgmap v2973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:39.109 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2665340516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:39.109 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:38 smithi093 ceph-mon[111335]: pgmap v2973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2665340516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:38 smithi093 ceph-mon[118902]: pgmap v2973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2665340516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:39.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:39.109323+0000) 2023-08-27T22:32:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:40.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:40.109586+0000) 2023-08-27T22:32:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:40 smithi136 ceph-mon[113531]: pgmap v2974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1458119451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:40 smithi093 ceph-mon[111335]: pgmap v2974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1458119451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:40 smithi093 ceph-mon[118902]: pgmap v2974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1458119451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:40.510Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:40.511Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:41.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:41.109787+0000) 2023-08-27T22:32:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:42.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:42.110104+0000) 2023-08-27T22:32:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:42.019Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:42.020Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:43.110 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:42 smithi136 ceph-mon[113531]: pgmap v2975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:43.110 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/130194494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:42 smithi093 ceph-mon[111335]: pgmap v2975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/130194494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:42 smithi093 ceph-mon[118902]: pgmap v2975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/130194494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:43.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:43.110356+0000) 2023-08-27T22:32:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:43.939Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:43.940Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:44.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:44.110604+0000) 2023-08-27T22:32:45.110 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:44 smithi136 ceph-mon[113531]: pgmap v2976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:44 smithi093 ceph-mon[111335]: pgmap v2976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:44 smithi093 ceph-mon[118902]: pgmap v2976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:45.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:45.110834+0000) 2023-08-27T22:32:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:32:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:32:45] "GET /metrics HTTP/1.1" 200 239895 "" "Prometheus/2.43.0" 2023-08-27T22:32:46.111 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1164426779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1164426779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1164426779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:46.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:46.111059+0000) 2023-08-27T22:32:47.111 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:46 smithi136 ceph-mon[113531]: pgmap v2977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:46 smithi093 ceph-mon[111335]: pgmap v2977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:46 smithi093 ceph-mon[118902]: pgmap v2977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:47.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:47.111332+0000) 2023-08-27T22:32:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1886266769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:48.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:48.111586+0000) 2023-08-27T22:32:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1886266769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1886266769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:48 smithi136 ceph-mon[113531]: pgmap v2978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:49.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:49.111829+0000) 2023-08-27T22:32:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:48 smithi093 ceph-mon[111335]: pgmap v2978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:48 smithi093 ceph-mon[118902]: pgmap v2978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:50.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:50.112129+0000) 2023-08-27T22:32:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:50 smithi136 ceph-mon[113531]: pgmap v2979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4194688786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:50 smithi093 ceph-mon[111335]: pgmap v2979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4194688786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:50 smithi093 ceph-mon[118902]: pgmap v2979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4194688786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:50.511Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:50.512Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:51.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:51.112393+0000) 2023-08-27T22:32:52.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:52.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:52.112589+0000) 2023-08-27T22:32:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:52.020Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:52.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:53.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:52 smithi136 ceph-mon[113531]: pgmap v2980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:53.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3157752575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:52 smithi093 ceph-mon[111335]: pgmap v2980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3157752575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:52 smithi093 ceph-mon[118902]: pgmap v2980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3157752575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:53.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:53.112864+0000) 2023-08-27T22:32:54.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:32:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:53.940Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:32:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:32:53.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:32:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:54.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:54.113099+0000) 2023-08-27T22:32:55.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:54 smithi136 ceph-mon[113531]: pgmap v2981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:55.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2239423512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:54 smithi093 ceph-mon[111335]: pgmap v2981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2239423512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:54 smithi093 ceph-mon[118902]: pgmap v2981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2239423512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:55.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:55.113370+0000) 2023-08-27T22:32:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:32:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:32:55] "GET /metrics HTTP/1.1" 200 239896 "" "Prometheus/2.43.0" 2023-08-27T22:32:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:56.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:56.113570+0000) 2023-08-27T22:32:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:56 smithi136 ceph-mon[113531]: pgmap v2982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:57.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:57.113843+0000) 2023-08-27T22:32:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:56 smithi093 ceph-mon[111335]: pgmap v2982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:56 smithi093 ceph-mon[118902]: pgmap v2982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1535355670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:58.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:58.114118+0000) 2023-08-27T22:32:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1535355670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1535355670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:32:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:58 smithi136 ceph-mon[113531]: pgmap v2983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:32:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:32:59.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:32:59.114316+0000) 2023-08-27T22:32:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:58 smithi093 ceph-mon[111335]: pgmap v2983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:32:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:58 smithi093 ceph-mon[118902]: pgmap v2983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:32:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2180590181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:00.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:00.114568+0000) 2023-08-27T22:33:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:32:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2180590181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:32:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2180590181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:00.873 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:00.512Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:00.873 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:00.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:00 smithi136 ceph-mon[113531]: pgmap v2984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:01.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:01.114814+0000) 2023-08-27T22:33:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:00 smithi093 ceph-mon[118902]: pgmap v2984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:00 smithi093 ceph-mon[111335]: pgmap v2984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3785366798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:02.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:02.115102+0000) 2023-08-27T22:33:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3785366798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3785366798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:02.021Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:02.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:02 smithi136 ceph-mon[113531]: pgmap v2985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:03.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:03.115386+0000) 2023-08-27T22:33:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:02 smithi093 ceph-mon[111335]: pgmap v2985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:02 smithi093 ceph-mon[118902]: pgmap v2985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:03.941Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:03.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:04.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:04.115634+0000) 2023-08-27T22:33:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:04 smithi136 ceph-mon[113531]: pgmap v2986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1982367917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:05.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:05.115891+0000) 2023-08-27T22:33:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:04 smithi093 ceph-mon[111335]: pgmap v2986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1982367917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:04 smithi093 ceph-mon[118902]: pgmap v2986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1982367917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:33:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:33:05] "GET /metrics HTTP/1.1" 200 239896 "" "Prometheus/2.43.0" 2023-08-27T22:33:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:06.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:06.116169+0000) 2023-08-27T22:33:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:06 smithi136 ceph-mon[113531]: pgmap v2987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/74396811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:07.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:07.116409+0000) 2023-08-27T22:33:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:06 smithi093 ceph-mon[111335]: pgmap v2987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/74396811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:06 smithi093 ceph-mon[118902]: pgmap v2987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/74396811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:08.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:08.116670+0000) 2023-08-27T22:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:08 smithi136 ceph-mon[113531]: pgmap v2988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:09.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:09.116869+0000) 2023-08-27T22:33:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:08 smithi093 ceph-mon[111335]: pgmap v2988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:08 smithi093 ceph-mon[118902]: pgmap v2988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:10.117 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:33:10.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3602142854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:10.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:10.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:33:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3602142854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:33:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3602142854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:10.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:10.117001+0000) 2023-08-27T22:33:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:10.514Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:10.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:10.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:10 smithi136 ceph-mon[113531]: pgmap v2989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-mon[111335]: pgmap v2989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-mon[118902]: pgmap v2989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:11.362 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:11.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:11.117160+0000) 2023-08-27T22:33:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1172201788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:12.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:12.117300+0000) 2023-08-27T22:33:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1172201788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1172201788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:12.021Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:12.022Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:13.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:13.117551+0000) 2023-08-27T22:33:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:12 smithi136 ceph-mon[113531]: pgmap v2990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:33:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:33:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[111335]: pgmap v2990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:33:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:33:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[118902]: pgmap v2990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:33:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:33:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:33:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4206940981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:14.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:14.117813+0000) 2023-08-27T22:33:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4206940981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4206940981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:13.941Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:13.942Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:14 smithi136 ceph-mon[113531]: pgmap v2991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:15.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:15.118154+0000) 2023-08-27T22:33:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:14 smithi093 ceph-mon[111335]: pgmap v2991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:14 smithi093 ceph-mon[118902]: pgmap v2991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:33:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:33:15] "GET /metrics HTTP/1.1" 200 239895 "" "Prometheus/2.43.0" 2023-08-27T22:33:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:16.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:16.118429+0000) 2023-08-27T22:33:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:16 smithi136 ceph-mon[113531]: pgmap v2992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1448766533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:17.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:17.118665+0000) 2023-08-27T22:33:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:16 smithi093 ceph-mon[111335]: pgmap v2992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1448766533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:16 smithi093 ceph-mon[118902]: pgmap v2992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1448766533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:18.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:18.118926+0000) 2023-08-27T22:33:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:18 smithi136 ceph-mon[113531]: pgmap v2993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2964075996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:19.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:19.119196+0000) 2023-08-27T22:33:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:18 smithi093 ceph-mon[111335]: pgmap v2993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2964075996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:18 smithi093 ceph-mon[118902]: pgmap v2993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2964075996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:20.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:20.119473+0000) 2023-08-27T22:33:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:20 smithi136 ceph-mon[113531]: pgmap v2994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:20 smithi093 ceph-mon[111335]: pgmap v2994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:20 smithi093 ceph-mon[118902]: pgmap v2994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:20.515Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:20.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:21.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:21.119731+0000) 2023-08-27T22:33:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2127578825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2127578825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2127578825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:22.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:22.120006+0000) 2023-08-27T22:33:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:22.022Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:22.023Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:22 smithi136 ceph-mon[113531]: pgmap v2995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:22 smithi093 ceph-mon[111335]: pgmap v2995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:22 smithi093 ceph-mon[118902]: pgmap v2995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:23.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:23.120253+0000) 2023-08-27T22:33:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/146216674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:23.942 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/146216674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:23.942 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/146216674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:23.942Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:23.943Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:24.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:24.120506+0000) 2023-08-27T22:33:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:24 smithi136 ceph-mon[113531]: pgmap v2996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:24 smithi093 ceph-mon[111335]: pgmap v2996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:24 smithi093 ceph-mon[118902]: pgmap v2996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:25.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:25.120782+0000) 2023-08-27T22:33:25.692 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:33:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:33:25] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:33:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/997219075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/997219075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:26.121 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/997219075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:26.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:26.121071+0000) 2023-08-27T22:33:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:26 smithi093 ceph-mon[111335]: pgmap v2997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:26 smithi093 ceph-mon[118902]: pgmap v2997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:27.121 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:26 smithi136 ceph-mon[113531]: pgmap v2997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:27.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:27.121300+0000) 2023-08-27T22:33:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:28.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:28.121536+0000) 2023-08-27T22:33:29.121 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:28 smithi136 ceph-mon[113531]: pgmap v2998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:29.122 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/316450401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:28 smithi093 ceph-mon[111335]: pgmap v2998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/316450401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:28 smithi093 ceph-mon[118902]: pgmap v2998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/316450401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:29.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:29.121758+0000) 2023-08-27T22:33:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:30.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:30.122021+0000) 2023-08-27T22:33:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:30 smithi136 ceph-mon[113531]: pgmap v2999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1197775906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:30 smithi093 ceph-mon[111335]: pgmap v2999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1197775906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:30 smithi093 ceph-mon[118902]: pgmap v2999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1197775906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:30.515Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:30.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:31.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:31.122282+0000) 2023-08-27T22:33:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:32.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:32.122530+0000) 2023-08-27T22:33:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:32.023Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:32.024Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:33.122 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:32 smithi136 ceph-mon[113531]: pgmap v3000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:33.123 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3622088194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:32 smithi093 ceph-mon[111335]: pgmap v3000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3622088194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:32 smithi093 ceph-mon[118902]: pgmap v3000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3622088194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:33.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:33.122827+0000) 2023-08-27T22:33:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:33.942Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:33.943Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:34.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:34.123103+0000) 2023-08-27T22:33:35.123 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:34 smithi136 ceph-mon[113531]: pgmap v3001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:34 smithi093 ceph-mon[111335]: pgmap v3001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:34 smithi093 ceph-mon[118902]: pgmap v3001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:35.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:35.123362+0000) 2023-08-27T22:33:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:33:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:33:35] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:33:36.123 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2064125192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2064125192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2064125192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:36.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:36.123616+0000) 2023-08-27T22:33:37.123 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:36 smithi136 ceph-mon[113531]: pgmap v3002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:36 smithi093 ceph-mon[111335]: pgmap v3002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:36 smithi093 ceph-mon[118902]: pgmap v3002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:37.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:37.123892+0000) 2023-08-27T22:33:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1817728592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:38.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:38.124127+0000) 2023-08-27T22:33:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1817728592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1817728592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:38 smithi136 ceph-mon[113531]: pgmap v3003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:39.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:39.124381+0000) 2023-08-27T22:33:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:38 smithi093 ceph-mon[111335]: pgmap v3003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:38 smithi093 ceph-mon[118902]: pgmap v3003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:40.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:40.124675+0000) 2023-08-27T22:33:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:40 smithi136 ceph-mon[113531]: pgmap v3004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2635872146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:40 smithi093 ceph-mon[111335]: pgmap v3004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2635872146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:40 smithi093 ceph-mon[118902]: pgmap v3004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2635872146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:40.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:40.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:41.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:41.124917+0000) 2023-08-27T22:33:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:42.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:42.125177+0000) 2023-08-27T22:33:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:42.024Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:42.025Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:43.126 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:42 smithi136 ceph-mon[113531]: pgmap v3005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:43.126 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2966227921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:42 smithi093 ceph-mon[111335]: pgmap v3005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2966227921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:42 smithi093 ceph-mon[118902]: pgmap v3005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2966227921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:43.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:43.125435+0000) 2023-08-27T22:33:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:43.943Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:43.944Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:44.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:44.125687+0000) 2023-08-27T22:33:45.126 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:44 smithi136 ceph-mon[113531]: pgmap v3006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:45.126 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2084420469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:44 smithi093 ceph-mon[118902]: pgmap v3006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2084420469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:44 smithi093 ceph-mon[111335]: pgmap v3006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2084420469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:45.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:45.125950+0000) 2023-08-27T22:33:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:33:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:33:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:33:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:46.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:46.126192+0000) 2023-08-27T22:33:47.126 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:46 smithi136 ceph-mon[113531]: pgmap v3007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:46 smithi093 ceph-mon[111335]: pgmap v3007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:46 smithi093 ceph-mon[118902]: pgmap v3007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:47.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:47.126430+0000) 2023-08-27T22:33:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2901080029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:48.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:48.126657+0000) 2023-08-27T22:33:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2901080029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2901080029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:48 smithi136 ceph-mon[113531]: pgmap v3008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:49.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:49.126884+0000) 2023-08-27T22:33:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:48 smithi093 ceph-mon[111335]: pgmap v3008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:48 smithi093 ceph-mon[118902]: pgmap v3008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3886701340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:50.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:50.127112+0000) 2023-08-27T22:33:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3886701340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3886701340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:50.880 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:50.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:50.881 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:50.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:50 smithi136 ceph-mon[113531]: pgmap v3009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:51.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:51.127282+0000) 2023-08-27T22:33:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:50 smithi093 ceph-mon[111335]: pgmap v3009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:50 smithi093 ceph-mon[118902]: pgmap v3009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3424128447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:52.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:52.127532+0000) 2023-08-27T22:33:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3424128447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3424128447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:52.025Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:52.026Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:52 smithi136 ceph-mon[113531]: pgmap v3010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:53.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:53.127729+0000) 2023-08-27T22:33:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:52 smithi093 ceph-mon[111335]: pgmap v3010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:52 smithi093 ceph-mon[118902]: pgmap v3010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:54.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:54.127985+0000) 2023-08-27T22:33:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:33:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:53.943Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:33:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:33:53.944Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:33:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:55.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:55.128289+0000) 2023-08-27T22:33:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:54 smithi136 ceph-mon[113531]: pgmap v3011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2162317001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:54 smithi093 ceph-mon[111335]: pgmap v3011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2162317001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:54 smithi093 ceph-mon[118902]: pgmap v3011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2162317001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:33:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:33:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:33:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:56.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:56.128539+0000) 2023-08-27T22:33:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:56 smithi136 ceph-mon[113531]: pgmap v3012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2116107679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:57.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:57.128796+0000) 2023-08-27T22:33:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:56 smithi093 ceph-mon[111335]: pgmap v3012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2116107679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:56 smithi093 ceph-mon[118902]: pgmap v3012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2116107679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:33:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:58.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:58.129024+0000) 2023-08-27T22:33:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:58 smithi136 ceph-mon[113531]: pgmap v3013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:33:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:33:59.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:33:59.129295+0000) 2023-08-27T22:33:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:58 smithi093 ceph-mon[111335]: pgmap v3013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:33:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:58 smithi093 ceph-mon[118902]: pgmap v3013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:33:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1529612966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:00.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:00.129545+0000) 2023-08-27T22:34:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:33:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1529612966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:33:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1529612966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:00.907 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:00.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:00.907 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:00.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:00 smithi136 ceph-mon[113531]: pgmap v3014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:01.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:01.129798+0000) 2023-08-27T22:34:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:00 smithi093 ceph-mon[111335]: pgmap v3014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:00 smithi093 ceph-mon[118902]: pgmap v3014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2401959417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:02.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:02.130095+0000) 2023-08-27T22:34:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2401959417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2401959417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:02.026Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:02.027Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:02 smithi136 ceph-mon[113531]: pgmap v3015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:03.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:03.130360+0000) 2023-08-27T22:34:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:02 smithi093 ceph-mon[111335]: pgmap v3015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:02 smithi093 ceph-mon[118902]: pgmap v3015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1519343876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:04.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:04.130620+0000) 2023-08-27T22:34:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1519343876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1519343876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:03.943Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:03.944Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:04 smithi136 ceph-mon[113531]: pgmap v3016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:05.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:05.130870+0000) 2023-08-27T22:34:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:04 smithi093 ceph-mon[111335]: pgmap v3016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:04 smithi093 ceph-mon[118902]: pgmap v3016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:34:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:34:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:34:06.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:06.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:06.131134+0000) 2023-08-27T22:34:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:06 smithi136 ceph-mon[113531]: pgmap v3017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/436061195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:07.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:07.131389+0000) 2023-08-27T22:34:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:06 smithi093 ceph-mon[111335]: pgmap v3017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/436061195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:06 smithi093 ceph-mon[118902]: pgmap v3017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/436061195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:08.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:08.131624+0000) 2023-08-27T22:34:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:09.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:09.131844+0000) 2023-08-27T22:34:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:08 smithi136 ceph-mon[113531]: pgmap v3018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3391143984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:08 smithi093 ceph-mon[111335]: pgmap v3018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3391143984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:08 smithi093 ceph-mon[118902]: pgmap v3018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3391143984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:10.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:10.132115+0000) 2023-08-27T22:34:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:10 smithi136 ceph-mon[113531]: pgmap v3019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:10 smithi093 ceph-mon[118902]: pgmap v3019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:10 smithi093 ceph-mon[111335]: pgmap v3019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:10.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:10.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:11.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:11.132356+0000) 2023-08-27T22:34:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3478698840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3478698840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3478698840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:12.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:12.132598+0000) 2023-08-27T22:34:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:12.026Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:12.027Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:12.818 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:12 smithi093 ceph-mon[111335]: pgmap v3020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:12.819 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:34:12.819 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:12 smithi093 ceph-mon[118902]: pgmap v3020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:12.819 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:34:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:12 smithi136 ceph-mon[113531]: pgmap v3020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:34:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:13.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:13.132770+0000) 2023-08-27T22:34:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3888132932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3888132932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:13.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3888132932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:14.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:14.132954+0000) 2023-08-27T22:34:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:13.944Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:13.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: pgmap v3021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: pgmap v3021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:34:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:15.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:15.133174+0000) 2023-08-27T22:34:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: pgmap v3021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:34:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:34:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:34:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:34:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:34:15] "GET /metrics HTTP/1.1" 200 239911 "" "Prometheus/2.43.0" 2023-08-27T22:34:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/605888875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:16.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:16.133345+0000) 2023-08-27T22:34:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/605888875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/605888875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:17 smithi136 ceph-mon[113531]: pgmap v3022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:17.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:17.133585+0000) 2023-08-27T22:34:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:17 smithi093 ceph-mon[111335]: pgmap v3022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:17 smithi093 ceph-mon[118902]: pgmap v3022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:18.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:18.133815+0000) 2023-08-27T22:34:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:19 smithi136 ceph-mon[113531]: pgmap v3023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/725542679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:19.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:19.134089+0000) 2023-08-27T22:34:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:19 smithi093 ceph-mon[111335]: pgmap v3023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/725542679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:19 smithi093 ceph-mon[118902]: pgmap v3023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/725542679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:20.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:20.134331+0000) 2023-08-27T22:34:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:20 smithi136 ceph-mon[113531]: pgmap v3024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2522706599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:20 smithi093 ceph-mon[111335]: pgmap v3024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2522706599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:20 smithi093 ceph-mon[118902]: pgmap v3024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2522706599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:20.518Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:20.519Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:21.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:21.134585+0000) 2023-08-27T22:34:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:22.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:22.134835+0000) 2023-08-27T22:34:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:22.027Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:22.028Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:23.135 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:22 smithi136 ceph-mon[113531]: pgmap v3025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:23.135 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2668955792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:22 smithi093 ceph-mon[111335]: pgmap v3025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2668955792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:22 smithi093 ceph-mon[118902]: pgmap v3025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2668955792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:23.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:23.135082+0000) 2023-08-27T22:34:24.135 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:23.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:23.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:24.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:24.135321+0000) 2023-08-27T22:34:25.135 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:24 smithi136 ceph-mon[113531]: pgmap v3026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:24 smithi093 ceph-mon[111335]: pgmap v3026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:24 smithi093 ceph-mon[118902]: pgmap v3026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:25.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:25.135595+0000) 2023-08-27T22:34:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:34:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:34:25] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:34:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/52044684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:26.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:26.135786+0000) 2023-08-27T22:34:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/52044684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/52044684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:26 smithi136 ceph-mon[113531]: pgmap v3027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:27.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:27.136020+0000) 2023-08-27T22:34:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:26 smithi093 ceph-mon[111335]: pgmap v3027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:26 smithi093 ceph-mon[118902]: pgmap v3027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/921999727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:28.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:28.136293+0000) 2023-08-27T22:34:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/921999727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/921999727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:28 smithi136 ceph-mon[113531]: pgmap v3028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:29.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:29.136493+0000) 2023-08-27T22:34:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:28 smithi093 ceph-mon[111335]: pgmap v3028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:28 smithi093 ceph-mon[118902]: pgmap v3028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:30.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:30.136747+0000) 2023-08-27T22:34:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:30 smithi136 ceph-mon[113531]: pgmap v3029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/931859262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:30 smithi093 ceph-mon[111335]: pgmap v3029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/931859262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:30 smithi093 ceph-mon[118902]: pgmap v3029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/931859262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:30.519Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:30.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:31.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:31.136978+0000) 2023-08-27T22:34:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:32.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:32.137210+0000) 2023-08-27T22:34:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:32.028Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:32.029Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:33.137 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:32 smithi136 ceph-mon[113531]: pgmap v3030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:33.137 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2630948555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:32 smithi093 ceph-mon[111335]: pgmap v3030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2630948555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:32 smithi093 ceph-mon[118902]: pgmap v3030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2630948555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:33.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:33.137494+0000) 2023-08-27T22:34:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:33.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:33.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:34.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:34.137713+0000) 2023-08-27T22:34:35.138 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:34 smithi136 ceph-mon[113531]: pgmap v3031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:35.138 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1847548317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:34 smithi093 ceph-mon[111335]: pgmap v3031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1847548317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:34 smithi093 ceph-mon[118902]: pgmap v3031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1847548317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:35.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:35.137996+0000) 2023-08-27T22:34:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:34:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:34:35] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:34:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:36.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:36.138264+0000) 2023-08-27T22:34:37.138 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:36 smithi136 ceph-mon[113531]: pgmap v3032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:36 smithi093 ceph-mon[111335]: pgmap v3032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:36 smithi093 ceph-mon[118902]: pgmap v3032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:37.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:37.138522+0000) 2023-08-27T22:34:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3781797134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:38.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:38.138788+0000) 2023-08-27T22:34:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3781797134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3781797134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:38 smithi136 ceph-mon[113531]: pgmap v3033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:39.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:39.139060+0000) 2023-08-27T22:34:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:38 smithi093 ceph-mon[111335]: pgmap v3033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:38 smithi093 ceph-mon[118902]: pgmap v3033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:40.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:40.139323+0000) 2023-08-27T22:34:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3395523744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:40 smithi136 ceph-mon[113531]: pgmap v3034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3395523744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:41.072 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:40 smithi093 ceph-mon[111335]: pgmap v3034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:41.072 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3395523744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:41.072 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:40 smithi093 ceph-mon[118902]: pgmap v3034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:41.073 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:40.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:41.073 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:40.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:41.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:41.139554+0000) 2023-08-27T22:34:43.304 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:42.028Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:43.304 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:42.029Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:43.304 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:42.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:42.139833+0000) 2023-08-27T22:34:43.304 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:42 smithi093 ceph-mon[111335]: pgmap v3035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:43.304 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3500542722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:43.305 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:42 smithi136 ceph-mon[113531]: pgmap v3035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:43.305 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3500542722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:43.305 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:42 smithi093 ceph-mon[118902]: pgmap v3035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:43.305 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3500542722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:43.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:43.140107+0000) 2023-08-27T22:34:44.451 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:43.946Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:44.452 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:43.947Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:44.452 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:44.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:44.140310+0000) 2023-08-27T22:34:45.140 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:44 smithi136 ceph-mon[113531]: pgmap v3036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:45.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3212255334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:45.273 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:44 smithi093 ceph-mon[111335]: pgmap v3036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:45.273 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3212255334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:45.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:44 smithi093 ceph-mon[118902]: pgmap v3036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:45.274 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3212255334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:45.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:45.140592+0000) 2023-08-27T22:34:45.756 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:34:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:34:45] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:34:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:46.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:46.140836+0000) 2023-08-27T22:34:47.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:46 smithi136 ceph-mon[113531]: pgmap v3037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:47.175 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1274454203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:46 smithi093 ceph-mon[118902]: pgmap v3037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:47.652 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1274454203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:47.653 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:46 smithi093 ceph-mon[111335]: pgmap v3037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:47.653 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1274454203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:47.653 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:47.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:47.141113+0000) 2023-08-27T22:34:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:48.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:48.141305+0000) 2023-08-27T22:34:49.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:48 smithi136 ceph-mon[113531]: pgmap v3038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:48 smithi093 ceph-mon[111335]: pgmap v3038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:49.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:48 smithi093 ceph-mon[118902]: pgmap v3038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:49.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:49.141556+0000) 2023-08-27T22:34:50.141 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/637457210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/637457210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:50.599 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/637457210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:50.601 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:50.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:50.141820+0000) 2023-08-27T22:34:50.890 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:50.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:50.890 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:50.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:51.142 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:50 smithi136 ceph-mon[113531]: pgmap v3039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:50 smithi093 ceph-mon[111335]: pgmap v3039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:51.443 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:50 smithi093 ceph-mon[118902]: pgmap v3039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:51.443 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:51.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:51.142087+0000) 2023-08-27T22:34:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3457033873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:52.192 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:52.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:52.142328+0000) 2023-08-27T22:34:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3457033873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:52.288 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3457033873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:52.288 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:52.029Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:52.288 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:52.030Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:52 smithi136 ceph-mon[113531]: pgmap v3040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:53.319 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:53.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:53.142582+0000) 2023-08-27T22:34:53.319 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:52 smithi093 ceph-mon[111335]: pgmap v3040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:53.320 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:52 smithi093 ceph-mon[118902]: pgmap v3040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3062478453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:54.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:54.142840+0000) 2023-08-27T22:34:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3062478453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:34:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3062478453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:53.947Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:34:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:34:53.948Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:34:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:54 smithi136 ceph-mon[113531]: pgmap v3041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:55.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:55.143084+0000) 2023-08-27T22:34:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:54 smithi093 ceph-mon[111335]: pgmap v3041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:54 smithi093 ceph-mon[118902]: pgmap v3041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:34:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:34:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:34:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:56.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:56.143288+0000) 2023-08-27T22:34:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:56 smithi136 ceph-mon[113531]: pgmap v3042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/383162215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:57.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:57.143545+0000) 2023-08-27T22:34:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:56 smithi093 ceph-mon[111335]: pgmap v3042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/383162215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:56 smithi093 ceph-mon[118902]: pgmap v3042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/383162215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:58.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:58.143775+0000) 2023-08-27T22:34:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:58 smithi136 ceph-mon[113531]: pgmap v3043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:34:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3175103281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:34:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:34:59.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:34:59.143992+0000) 2023-08-27T22:34:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:58 smithi093 ceph-mon[111335]: pgmap v3043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:34:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3175103281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:34:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:58 smithi093 ceph-mon[118902]: pgmap v3043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:34:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:34:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3175103281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:00.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:00.144222+0000) 2023-08-27T22:35:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:00 smithi136 ceph-mon[113531]: pgmap v3044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:00 smithi093 ceph-mon[111335]: pgmap v3044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:00 smithi093 ceph-mon[118902]: pgmap v3044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:00.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:00.522Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:01.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:01.144476+0000) 2023-08-27T22:35:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1147701146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1147701146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1147701146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:02.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:02.144717+0000) 2023-08-27T22:35:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:02.029Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:02.030Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:02 smithi136 ceph-mon[113531]: pgmap v3045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:02 smithi093 ceph-mon[111335]: pgmap v3045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:02 smithi093 ceph-mon[118902]: pgmap v3045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:03.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:03.145049+0000) 2023-08-27T22:35:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/495597533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:03.947 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/495597533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:03.948 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/495597533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:03.947Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:03.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:04.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:04.145286+0000) 2023-08-27T22:35:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:04 smithi136 ceph-mon[113531]: pgmap v3046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:04 smithi093 ceph-mon[111335]: pgmap v3046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:04 smithi093 ceph-mon[118902]: pgmap v3046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:05.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:05.145546+0000) 2023-08-27T22:35:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:35:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:35:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:35:06.146 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/816251423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/816251423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/816251423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:06.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:06.145779+0000) 2023-08-27T22:35:07.146 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:06 smithi136 ceph-mon[113531]: pgmap v3047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:06 smithi093 ceph-mon[111335]: pgmap v3047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:06 smithi093 ceph-mon[118902]: pgmap v3047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:07.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:07.146026+0000) 2023-08-27T22:35:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:08.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:08.146298+0000) 2023-08-27T22:35:09.146 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:08 smithi136 ceph-mon[113531]: pgmap v3048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:09.146 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1152627697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:09.147 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:08 smithi093 ceph-mon[111335]: pgmap v3048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1152627697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:08 smithi093 ceph-mon[118902]: pgmap v3048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1152627697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:09.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:09.146555+0000) 2023-08-27T22:35:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:10.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:10.146746+0000) 2023-08-27T22:35:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:10 smithi136 ceph-mon[113531]: pgmap v3049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:10 smithi093 ceph-mon[111335]: pgmap v3049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:10 smithi093 ceph-mon[118902]: pgmap v3049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:10.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:10.522Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:11.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:11.147004+0000) 2023-08-27T22:35:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2916569636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2916569636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2916569636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:12.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:12.147232+0000) 2023-08-27T22:35:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:12.030Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:12.031Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:12 smithi136 ceph-mon[113531]: pgmap v3050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:12 smithi093 ceph-mon[111335]: pgmap v3050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:13.163 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:12 smithi093 ceph-mon[118902]: pgmap v3050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:13.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:13.147471+0000) 2023-08-27T22:35:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2802442828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:13.948 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2802442828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:13.948 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2802442828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:13.948Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:13.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:14.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:14.147728+0000) 2023-08-27T22:35:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:14 smithi136 ceph-mon[113531]: pgmap v3051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:14 smithi093 ceph-mon[111335]: pgmap v3051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:14 smithi093 ceph-mon[118902]: pgmap v3051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:15.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:15.148007+0000) 2023-08-27T22:35:15.651 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:35:15.651 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1019933535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:15.652 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:35:15.652 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1019933535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:15.652 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:35:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:35:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:35:15.899 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:35:15.899 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1019933535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:16.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:16.148127+0000) 2023-08-27T22:35:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:16 smithi136 ceph-mon[113531]: pgmap v3052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:16 smithi093 ceph-mon[111335]: pgmap v3052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:16 smithi093 ceph-mon[118902]: pgmap v3052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:17.148 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:17.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:17.148257+0000) 2023-08-27T22:35:18.148 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.148 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:35:18.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:35:18.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:35:18.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:35:18.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:35:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:18.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:18.148511+0000) 2023-08-27T22:35:19.148 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1341637245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:19.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:18 smithi136 ceph-mon[113531]: pgmap v3053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1341637245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:18 smithi093 ceph-mon[118902]: pgmap v3053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1341637245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:18 smithi093 ceph-mon[111335]: pgmap v3053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:19.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:19.148805+0000) 2023-08-27T22:35:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:20.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:20.149072+0000) 2023-08-27T22:35:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:20 smithi136 ceph-mon[113531]: pgmap v3054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3671977314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:20 smithi093 ceph-mon[111335]: pgmap v3054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3671977314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:20 smithi093 ceph-mon[118902]: pgmap v3054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3671977314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:20.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:20.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:21.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:21.149248+0000) 2023-08-27T22:35:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:22.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:22.149492+0000) 2023-08-27T22:35:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:22.031Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:22.032Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:23.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:22 smithi136 ceph-mon[113531]: pgmap v3055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:23.150 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2192189382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:22 smithi093 ceph-mon[111335]: pgmap v3055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2192189382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:22 smithi093 ceph-mon[118902]: pgmap v3055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2192189382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:23.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:23.149711+0000) 2023-08-27T22:35:24.150 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:23.949Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:23.951Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:24.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:24.149964+0000) 2023-08-27T22:35:25.150 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:24 smithi136 ceph-mon[113531]: pgmap v3056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:24 smithi093 ceph-mon[111335]: pgmap v3056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:24 smithi093 ceph-mon[118902]: pgmap v3056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:25.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:25.150234+0000) 2023-08-27T22:35:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:35:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:35:25] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:35:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2260357714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:26.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:26.150499+0000) 2023-08-27T22:35:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2260357714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2260357714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:26 smithi136 ceph-mon[113531]: pgmap v3057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:27.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:27.150756+0000) 2023-08-27T22:35:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:26 smithi093 ceph-mon[111335]: pgmap v3057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:26 smithi093 ceph-mon[118902]: pgmap v3057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4199330835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:28.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:28.151002+0000) 2023-08-27T22:35:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4199330835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4199330835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:28 smithi136 ceph-mon[113531]: pgmap v3058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:29.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:29.151280+0000) 2023-08-27T22:35:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:28 smithi093 ceph-mon[111335]: pgmap v3058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:28 smithi093 ceph-mon[118902]: pgmap v3058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:30.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:30.151538+0000) 2023-08-27T22:35:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3275830212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:30 smithi136 ceph-mon[113531]: pgmap v3059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3275830212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:30 smithi093 ceph-mon[118902]: pgmap v3059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3275830212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:30 smithi093 ceph-mon[111335]: pgmap v3059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:30.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:30.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:31.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:31.151793+0000) 2023-08-27T22:35:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:32.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:32.152007+0000) 2023-08-27T22:35:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:32.031Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:32.032Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:33.152 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:32 smithi136 ceph-mon[113531]: pgmap v3060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:33.152 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/140585104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:32 smithi093 ceph-mon[111335]: pgmap v3060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/140585104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:32 smithi093 ceph-mon[118902]: pgmap v3060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/140585104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:33.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:33.152308+0000) 2023-08-27T22:35:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:33.950Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:33.951Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:34.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:34.152590+0000) 2023-08-27T22:35:35.152 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:34 smithi136 ceph-mon[113531]: pgmap v3061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:35.153 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2067760838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:34 smithi093 ceph-mon[111335]: pgmap v3061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2067760838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:34 smithi093 ceph-mon[118902]: pgmap v3061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2067760838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:35.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:35.152869+0000) 2023-08-27T22:35:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:35:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:35:35] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:35:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:36.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:36.153144+0000) 2023-08-27T22:35:37.153 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:36 smithi136 ceph-mon[113531]: pgmap v3062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:37.153 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340912036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:36 smithi093 ceph-mon[118902]: pgmap v3062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340912036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:36 smithi093 ceph-mon[111335]: pgmap v3062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340912036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:37.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:37.153390+0000) 2023-08-27T22:35:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:38.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:38.153652+0000) 2023-08-27T22:35:39.153 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:38 smithi136 ceph-mon[113531]: pgmap v3063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:39.154 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:38 smithi093 ceph-mon[111335]: pgmap v3063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:38 smithi093 ceph-mon[118902]: pgmap v3063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:39.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:39.153920+0000) 2023-08-27T22:35:40.154 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/202613555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/202613555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/202613555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:40.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:40.154140+0000) 2023-08-27T22:35:40.903 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:40.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:40.903 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:40.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:41.154 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:40 smithi136 ceph-mon[113531]: pgmap v3064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:40 smithi093 ceph-mon[111335]: pgmap v3064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:40 smithi093 ceph-mon[118902]: pgmap v3064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:41.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:41.154297+0000) 2023-08-27T22:35:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/15724581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:42.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:42.154564+0000) 2023-08-27T22:35:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/15724581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/15724581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:42.032Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:42.033Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:42 smithi136 ceph-mon[113531]: pgmap v3065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:43.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:43.154849+0000) 2023-08-27T22:35:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:42 smithi093 ceph-mon[111335]: pgmap v3065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:42 smithi093 ceph-mon[118902]: pgmap v3065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:43.951Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:43.951Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:44.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:44.155100+0000) 2023-08-27T22:35:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:44 smithi136 ceph-mon[113531]: pgmap v3066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4057505998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:45.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:45.155378+0000) 2023-08-27T22:35:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:44 smithi093 ceph-mon[111335]: pgmap v3066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4057505998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:44 smithi093 ceph-mon[118902]: pgmap v3066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4057505998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:35:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:35:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:35:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:46.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:46.155656+0000) 2023-08-27T22:35:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:46 smithi136 ceph-mon[113531]: pgmap v3067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2615801507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:47.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:47.155900+0000) 2023-08-27T22:35:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:46 smithi093 ceph-mon[111335]: pgmap v3067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2615801507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:46 smithi093 ceph-mon[118902]: pgmap v3067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2615801507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:48.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:48.156176+0000) 2023-08-27T22:35:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:48 smithi136 ceph-mon[113531]: pgmap v3068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2455947011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:49.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:49.156424+0000) 2023-08-27T22:35:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:48 smithi093 ceph-mon[111335]: pgmap v3068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2455947011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:48 smithi093 ceph-mon[118902]: pgmap v3068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2455947011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:50.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:50.156654+0000) 2023-08-27T22:35:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:50 smithi136 ceph-mon[113531]: pgmap v3069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:50 smithi093 ceph-mon[111335]: pgmap v3069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:50 smithi093 ceph-mon[118902]: pgmap v3069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:50.523Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:50.525Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:51.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:51.156926+0000) 2023-08-27T22:35:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3683874430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3683874430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3683874430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:52.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:52.157182+0000) 2023-08-27T22:35:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:52.033Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:52.034Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:52 smithi136 ceph-mon[113531]: pgmap v3070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:52 smithi093 ceph-mon[111335]: pgmap v3070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:52 smithi093 ceph-mon[118902]: pgmap v3070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:53.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:53.157436+0000) 2023-08-27T22:35:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3419418617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:53.951 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3419418617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:53.951 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3419418617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:53.951Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:54.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:35:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:35:53.952Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:35:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:54.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:54.157705+0000) 2023-08-27T22:35:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:54 smithi136 ceph-mon[113531]: pgmap v3071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:54 smithi093 ceph-mon[111335]: pgmap v3071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:35:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:54 smithi093 ceph-mon[118902]: pgmap v3071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:55.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:55.157984+0000) 2023-08-27T22:35:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:35:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:35:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T22:35:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:56.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:56.158286+0000) 2023-08-27T22:35:57.158 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:56 smithi136 ceph-mon[113531]: pgmap v3072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:57.159 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/241462158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:56 smithi093 ceph-mon[111335]: pgmap v3072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/241462158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:56 smithi093 ceph-mon[118902]: pgmap v3072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/241462158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:57.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:57.158531+0000) 2023-08-27T22:35:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:58.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:58.158763+0000) 2023-08-27T22:35:59.159 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:58 smithi136 ceph-mon[113531]: pgmap v3073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:59.159 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:35:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2225806616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:58 smithi093 ceph-mon[111335]: pgmap v3073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:35:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2225806616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:58 smithi093 ceph-mon[118902]: pgmap v3073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:35:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:35:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2225806616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:35:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:35:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:35:59.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:35:59.158980+0000) 2023-08-27T22:36:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:00.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:00.159196+0000) 2023-08-27T22:36:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:00 smithi136 ceph-mon[113531]: pgmap v3074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:00 smithi093 ceph-mon[111335]: pgmap v3074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:00 smithi093 ceph-mon[118902]: pgmap v3074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:00.524Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:00.525Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:01.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:01.159433+0000) 2023-08-27T22:36:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4048482071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4048482071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4048482071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:02.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:02.159737+0000) 2023-08-27T22:36:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:02.034Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:02.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:02 smithi136 ceph-mon[113531]: pgmap v3075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:02 smithi093 ceph-mon[111335]: pgmap v3075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:02 smithi093 ceph-mon[118902]: pgmap v3075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:03.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:03.160010+0000) 2023-08-27T22:36:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4250126589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:03.952 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4250126589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:03.952 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4250126589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:03.952Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:03.953Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:04.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:04.160299+0000) 2023-08-27T22:36:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:04 smithi136 ceph-mon[113531]: pgmap v3076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:04 smithi093 ceph-mon[111335]: pgmap v3076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:04 smithi093 ceph-mon[118902]: pgmap v3076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:05.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:05.160589+0000) 2023-08-27T22:36:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1701038672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:36:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:36:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T22:36:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1701038672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1701038672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:06.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:06.160812+0000) 2023-08-27T22:36:07.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:06 smithi093 ceph-mon[111335]: pgmap v3077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:07.176 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:06 smithi136 ceph-mon[113531]: pgmap v3077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:07.176 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:06 smithi093 ceph-mon[118902]: pgmap v3077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:07.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:07.161090+0000) 2023-08-27T22:36:08.162 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1397533512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1397533512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:08.267 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1397533512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:08.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:08.161337+0000) 2023-08-27T22:36:09.161 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:08 smithi136 ceph-mon[113531]: pgmap v3078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:09.161 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:08 smithi093 ceph-mon[111335]: pgmap v3078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:08 smithi093 ceph-mon[118902]: pgmap v3078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:09.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:09.161531+0000) 2023-08-27T22:36:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:10.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:10.161751+0000) 2023-08-27T22:36:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:10 smithi136 ceph-mon[113531]: pgmap v3079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:11.003 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/378645368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:11.004 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:10 smithi093 ceph-mon[111335]: pgmap v3079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:11.004 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/378645368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:11.004 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:10 smithi093 ceph-mon[118902]: pgmap v3079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:11.004 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/378645368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:11.004 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:10.525Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:11.005 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:10.526Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:11.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:11.161985+0000) 2023-08-27T22:36:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:12.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:12.162283+0000) 2023-08-27T22:36:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:12.034Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:12.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:13.162 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:12 smithi136 ceph-mon[113531]: pgmap v3080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:13.162 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1544852700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:12 smithi093 ceph-mon[111335]: pgmap v3080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1544852700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:12 smithi093 ceph-mon[118902]: pgmap v3080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1544852700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:13.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:13.162553+0000) 2023-08-27T22:36:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:13.953Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:13.954Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:14.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:14.162824+0000) 2023-08-27T22:36:15.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:14 smithi136 ceph-mon[113531]: pgmap v3081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:14 smithi093 ceph-mon[111335]: pgmap v3081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:14 smithi093 ceph-mon[118902]: pgmap v3081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:15.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:15.163123+0000) 2023-08-27T22:36:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:36:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:36:15] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:36:16.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2393859814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2393859814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2393859814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:16.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:16.163383+0000) 2023-08-27T22:36:17.163 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:16 smithi136 ceph-mon[113531]: pgmap v3082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:16 smithi093 ceph-mon[111335]: pgmap v3082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:16 smithi093 ceph-mon[118902]: pgmap v3082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:17.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:17.163645+0000) 2023-08-27T22:36:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3075809637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:36:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:18.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:18.163813+0000) 2023-08-27T22:36:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3075809637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:36:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3075809637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:36:19.128 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:18 smithi136 ceph-mon[113531]: pgmap v3083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:18 smithi093 ceph-mon[111335]: pgmap v3083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:18 smithi093 ceph-mon[118902]: pgmap v3083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:19.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:19.164082+0000) 2023-08-27T22:36:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:20.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:20.164271+0000) 2023-08-27T22:36:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2883825730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: pgmap v3084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:36:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:36:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:20.526Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:20.527Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2883825730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: pgmap v3084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2883825730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: pgmap v3084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:36:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:36:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:21.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:21.164489+0000) 2023-08-27T22:36:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:22.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:22.164704+0000) 2023-08-27T22:36:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:22.036Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:22.036Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:23.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:22 smithi136 ceph-mon[113531]: pgmap v3085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:23.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/121327354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:22 smithi093 ceph-mon[111335]: pgmap v3085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/121327354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:22 smithi093 ceph-mon[118902]: pgmap v3085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/121327354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:23.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:23.164970+0000) 2023-08-27T22:36:24.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:23.953Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:23.954Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:24.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:24.165228+0000) 2023-08-27T22:36:25.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:24 smithi136 ceph-mon[113531]: pgmap v3086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:25.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1614697695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:24 smithi093 ceph-mon[111335]: pgmap v3086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1614697695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:24 smithi093 ceph-mon[118902]: pgmap v3086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1614697695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:25.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:25.165496+0000) 2023-08-27T22:36:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:36:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:36:25] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:36:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:26.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:26.165745+0000) 2023-08-27T22:36:27.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:26 smithi136 ceph-mon[113531]: pgmap v3087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:26 smithi093 ceph-mon[111335]: pgmap v3087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:26 smithi093 ceph-mon[118902]: pgmap v3087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:27.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:27.166008+0000) 2023-08-27T22:36:28.166 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/764161570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/764161570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/764161570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:28.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:28.166226+0000) 2023-08-27T22:36:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:28 smithi136 ceph-mon[113531]: pgmap v3088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:29.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:29.166482+0000) 2023-08-27T22:36:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:28 smithi093 ceph-mon[111335]: pgmap v3088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:28 smithi093 ceph-mon[118902]: pgmap v3088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2309953445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:30.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:30.166767+0000) 2023-08-27T22:36:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2309953445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:30.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2309953445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:30.920 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:30.527Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:30.920 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:30.528Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:30 smithi136 ceph-mon[113531]: pgmap v3089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:31.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:31.167081+0000) 2023-08-27T22:36:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:30 smithi093 ceph-mon[111335]: pgmap v3089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:30 smithi093 ceph-mon[118902]: pgmap v3089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/15227822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:32.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:32.167380+0000) 2023-08-27T22:36:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/15227822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/15227822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:32.036Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:32.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:32 smithi136 ceph-mon[113531]: pgmap v3090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:33.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:33.167636+0000) 2023-08-27T22:36:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:32 smithi093 ceph-mon[111335]: pgmap v3090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:32 smithi093 ceph-mon[118902]: pgmap v3090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:33.953Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:33.954Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:34.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:34.167923+0000) 2023-08-27T22:36:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:34 smithi136 ceph-mon[113531]: pgmap v3091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/965655499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:35.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:35.168208+0000) 2023-08-27T22:36:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:34 smithi093 ceph-mon[111335]: pgmap v3091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/965655499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:34 smithi093 ceph-mon[118902]: pgmap v3091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/965655499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:36:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:36:35] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:36:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:36.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:36.168461+0000) 2023-08-27T22:36:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:36 smithi136 ceph-mon[113531]: pgmap v3092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1373723685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:37.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:37.168723+0000) 2023-08-27T22:36:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:36 smithi093 ceph-mon[118902]: pgmap v3092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1373723685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:36 smithi093 ceph-mon[111335]: pgmap v3092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1373723685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:38.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:38.168982+0000) 2023-08-27T22:36:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:38 smithi136 ceph-mon[113531]: pgmap v3093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3274989121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:39.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:39.169287+0000) 2023-08-27T22:36:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:38 smithi093 ceph-mon[118902]: pgmap v3093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3274989121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:38 smithi093 ceph-mon[111335]: pgmap v3093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3274989121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:40.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:40.169531+0000) 2023-08-27T22:36:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:40 smithi136 ceph-mon[113531]: pgmap v3094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:40 smithi093 ceph-mon[111335]: pgmap v3094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:40 smithi093 ceph-mon[118902]: pgmap v3094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:40.527Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:40.528Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:41.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:41.169778+0000) 2023-08-27T22:36:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3265151855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3265151855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3265151855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:42.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:42.169979+0000) 2023-08-27T22:36:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:42.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:42.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:42 smithi136 ceph-mon[113531]: pgmap v3095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:42 smithi093 ceph-mon[111335]: pgmap v3095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:42 smithi093 ceph-mon[118902]: pgmap v3095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:43.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:43.170138+0000) 2023-08-27T22:36:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2488960669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:43.954 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2488960669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:43.954 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2488960669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:43.954Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:43.955Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:44.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:44.170352+0000) 2023-08-27T22:36:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:44 smithi136 ceph-mon[113531]: pgmap v3096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:44 smithi093 ceph-mon[111335]: pgmap v3096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:44 smithi093 ceph-mon[118902]: pgmap v3096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:45.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:45.170645+0000) 2023-08-27T22:36:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:36:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:36:45] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:36:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:46.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:46.170902+0000) 2023-08-27T22:36:47.171 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:46 smithi136 ceph-mon[113531]: pgmap v3097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:47.171 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/560545164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:46 smithi093 ceph-mon[111335]: pgmap v3097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/560545164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:46 smithi093 ceph-mon[118902]: pgmap v3097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/560545164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:47.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:47.171146+0000) 2023-08-27T22:36:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:48.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:48.171380+0000) 2023-08-27T22:36:49.171 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:48 smithi136 ceph-mon[113531]: pgmap v3098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:49.172 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3342608477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:48 smithi093 ceph-mon[111335]: pgmap v3098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3342608477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:48 smithi093 ceph-mon[118902]: pgmap v3098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3342608477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:49.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:49.171666+0000) 2023-08-27T22:36:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:50.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:50.171915+0000) 2023-08-27T22:36:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:50 smithi136 ceph-mon[113531]: pgmap v3099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:50 smithi093 ceph-mon[111335]: pgmap v3099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:50 smithi093 ceph-mon[118902]: pgmap v3099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:50.527Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:50.528Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:51.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:51.172173+0000) 2023-08-27T22:36:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1031206643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1031206643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1031206643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:52.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:52.172433+0000) 2023-08-27T22:36:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:52.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:52.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:52 smithi136 ceph-mon[113531]: pgmap v3100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:52 smithi093 ceph-mon[111335]: pgmap v3100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:52 smithi093 ceph-mon[118902]: pgmap v3100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:53.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:53.172700+0000) 2023-08-27T22:36:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1665835237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:53.954 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1665835237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:53.954 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1665835237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:53.954Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:36:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:36:53.955Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:36:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:54.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:54.172894+0000) 2023-08-27T22:36:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:54 smithi136 ceph-mon[113531]: pgmap v3101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:54 smithi093 ceph-mon[111335]: pgmap v3101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:36:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:54 smithi093 ceph-mon[118902]: pgmap v3101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:55.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:55.173156+0000) 2023-08-27T22:36:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2709085440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2709085440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:36:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:36:55] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:36:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2709085440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:36:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:56.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:56.173408+0000) 2023-08-27T22:36:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:56 smithi136 ceph-mon[113531]: pgmap v3102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:56 smithi093 ceph-mon[111335]: pgmap v3102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:57.102 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:56 smithi093 ceph-mon[118902]: pgmap v3102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:36:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:57.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:57.173641+0000) 2023-08-27T22:36:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:58.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:58.173903+0000) 2023-08-27T22:37:00.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:58 smithi093 ceph-mon[111335]: pgmap v3103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:00.716 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:36:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1364735622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:00.717 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:58 smithi136 ceph-mon[113531]: pgmap v3103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:00.717 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:36:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1364735622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:00.717 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:58 smithi093 ceph-mon[118902]: pgmap v3103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:00.717 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:36:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1364735622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:00.718 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:36:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:36:59.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:36:59.174116+0000) 2023-08-27T22:37:00.718 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:00.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:00.174386+0000) 2023-08-27T22:37:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:00 smithi136 ceph-mon[113531]: pgmap v3104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:01.235 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3852737934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:01.236 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:00 smithi093 ceph-mon[111335]: pgmap v3104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:01.236 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3852737934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:01.236 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:00 smithi093 ceph-mon[118902]: pgmap v3104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:01.236 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3852737934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:01.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:00.528Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:01.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:00.529Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:01.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:01.174676+0000) 2023-08-27T22:37:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:02.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:02.174841+0000) 2023-08-27T22:37:02.530 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:02.038Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:02.531 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:02.039Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:03.175 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:02 smithi136 ceph-mon[113531]: pgmap v3105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:03.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/561756210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:03.350 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:02 smithi093 ceph-mon[111335]: pgmap v3105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:03.352 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/561756210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:03.352 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:02 smithi093 ceph-mon[118902]: pgmap v3105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:03.352 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/561756210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:03.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:03.175105+0000) 2023-08-27T22:37:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:03.954Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:04.773 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:03.956Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:04.774 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:04.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:04.175307+0000) 2023-08-27T22:37:05.175 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:04 smithi136 ceph-mon[113531]: pgmap v3106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:04 smithi093 ceph-mon[118902]: pgmap v3106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:05.275 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:04 smithi093 ceph-mon[111335]: pgmap v3106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:05.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:05.175539+0000) 2023-08-27T22:37:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:37:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:37:05] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:37:06.175 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1975286346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1975286346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:06.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1975286346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:06.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:06.175798+0000) 2023-08-27T22:37:07.175 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:06 smithi136 ceph-mon[113531]: pgmap v3107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:06 smithi093 ceph-mon[111335]: pgmap v3107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:06 smithi093 ceph-mon[118902]: pgmap v3107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:07.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:07.175973+0000) 2023-08-27T22:37:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/670175454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:08.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:08.176182+0000) 2023-08-27T22:37:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/670175454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/670175454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:08 smithi136 ceph-mon[113531]: pgmap v3108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:09.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:09.176430+0000) 2023-08-27T22:37:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:08 smithi093 ceph-mon[111335]: pgmap v3108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:08 smithi093 ceph-mon[118902]: pgmap v3108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/196093843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:10 smithi136 ceph-mon[113531]: pgmap v3109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:10.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:10.176672+0000) 2023-08-27T22:37:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/196093843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:10 smithi093 ceph-mon[111335]: pgmap v3109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/196093843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:10 smithi093 ceph-mon[118902]: pgmap v3109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:10.528Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:10.530Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:11.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:11.176907+0000) 2023-08-27T22:37:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:12.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:12.177131+0000) 2023-08-27T22:37:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:12.039Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:12.040Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:13.177 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:12 smithi136 ceph-mon[113531]: pgmap v3110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:13.177 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3236244792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:12 smithi093 ceph-mon[111335]: pgmap v3110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3236244792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:12 smithi093 ceph-mon[118902]: pgmap v3110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3236244792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:13.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:13.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:13.177407+0000) 2023-08-27T22:37:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:13.955Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:14.261 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:13.956Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:14.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:14.177641+0000) 2023-08-27T22:37:15.177 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:14 smithi136 ceph-mon[113531]: pgmap v3111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:15.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3523412553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:14 smithi093 ceph-mon[111335]: pgmap v3111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3523412553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:14 smithi093 ceph-mon[118902]: pgmap v3111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3523412553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:15.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:15.177912+0000) 2023-08-27T22:37:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:37:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:37:15] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:37:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:16.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:16.178150+0000) 2023-08-27T22:37:17.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:16 smithi136 ceph-mon[113531]: pgmap v3112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:16 smithi093 ceph-mon[111335]: pgmap v3112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:16 smithi093 ceph-mon[118902]: pgmap v3112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:17.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:17.178404+0000) 2023-08-27T22:37:18.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/893345939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/893345939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/893345939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:18.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:18.178725+0000) 2023-08-27T22:37:19.179 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:18 smithi136 ceph-mon[113531]: pgmap v3113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:18 smithi093 ceph-mon[111335]: pgmap v3113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:18 smithi093 ceph-mon[118902]: pgmap v3113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:19.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:19.178944+0000) 2023-08-27T22:37:20.179 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3092385571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3092385571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3092385571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:20.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:20.179199+0000) 2023-08-27T22:37:20.933 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:20.529Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:20.933 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:20.530Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:21.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:21.179427+0000) 2023-08-27T22:37:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:20 smithi136 ceph-mon[113531]: pgmap v3114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:37:21.191 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:20 smithi093 ceph-mon[111335]: pgmap v3114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:21.192 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:37:21.192 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:20 smithi093 ceph-mon[118902]: pgmap v3114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:21.192 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:37:22.152 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2192945555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:22.153 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2192945555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:22.153 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:22.039Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:22.153 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:22.040Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2192945555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:22.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:22.179561+0000) 2023-08-27T22:37:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[111335]: pgmap v3115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[118902]: pgmap v3115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:23.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:23.179697+0000) 2023-08-27T22:37:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:22 smithi136 ceph-mon[113531]: pgmap v3115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:37:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:37:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:37:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:37:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:23.955Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:24.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:23.957Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:37:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:37:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:37:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:24.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:24.179964+0000) 2023-08-27T22:37:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:24 smithi093 ceph-mon[118902]: pgmap v3116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2845402954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:24 smithi093 ceph-mon[111335]: pgmap v3116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2845402954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:24 smithi136 ceph-mon[113531]: pgmap v3116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2845402954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:25.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:25.180261+0000) 2023-08-27T22:37:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:37:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:37:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:37:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:26.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:26.180475+0000) 2023-08-27T22:37:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:26 smithi093 ceph-mon[111335]: pgmap v3117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1586361392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:26 smithi093 ceph-mon[118902]: pgmap v3117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1586361392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:26 smithi136 ceph-mon[113531]: pgmap v3117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1586361392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:27.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:27.180749+0000) 2023-08-27T22:37:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:28.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:28.180955+0000) 2023-08-27T22:37:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:28 smithi093 ceph-mon[111335]: pgmap v3118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4221274335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:28 smithi093 ceph-mon[118902]: pgmap v3118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4221274335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:28 smithi136 ceph-mon[113531]: pgmap v3118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4221274335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:29.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:29.181232+0000) 2023-08-27T22:37:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:30.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:30.181515+0000) 2023-08-27T22:37:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:30 smithi136 ceph-mon[113531]: pgmap v3119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:30 smithi093 ceph-mon[111335]: pgmap v3119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:30 smithi093 ceph-mon[118902]: pgmap v3119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:30.530Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:30.531Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:31.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:31.181815+0000) 2023-08-27T22:37:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1555013520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1555013520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1555013520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:32.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:32.182101+0000) 2023-08-27T22:37:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:32.040Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:32.041Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:32 smithi136 ceph-mon[113531]: pgmap v3120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:32 smithi093 ceph-mon[111335]: pgmap v3120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:32 smithi093 ceph-mon[118902]: pgmap v3120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:33.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:33.182361+0000) 2023-08-27T22:37:34.182 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1464356057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1464356057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1464356057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:33.956Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:33.958Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:34.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:34.182611+0000) 2023-08-27T22:37:35.183 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:34 smithi136 ceph-mon[113531]: pgmap v3121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:34 smithi093 ceph-mon[111335]: pgmap v3121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:34 smithi093 ceph-mon[118902]: pgmap v3121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:35.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:35.182873+0000) 2023-08-27T22:37:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:37:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:37:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:37:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:36.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:36.183175+0000) 2023-08-27T22:37:37.183 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:36 smithi136 ceph-mon[113531]: pgmap v3122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:37.183 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2250420345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:36 smithi093 ceph-mon[111335]: pgmap v3122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2250420345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:36 smithi093 ceph-mon[118902]: pgmap v3122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2250420345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:37.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:37.183469+0000) 2023-08-27T22:37:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:38.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:38.183772+0000) 2023-08-27T22:37:39.184 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:38 smithi136 ceph-mon[113531]: pgmap v3123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:39.184 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3741349134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:39.184 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:38 smithi093 ceph-mon[111335]: pgmap v3123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3741349134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:38 smithi093 ceph-mon[118902]: pgmap v3123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3741349134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:39.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:39.184066+0000) 2023-08-27T22:37:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:40.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:40.184286+0000) 2023-08-27T22:37:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:40 smithi136 ceph-mon[113531]: pgmap v3124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:40 smithi093 ceph-mon[111335]: pgmap v3124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:40 smithi093 ceph-mon[118902]: pgmap v3124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:40.531Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:40.532Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:41.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:41.184538+0000) 2023-08-27T22:37:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2674994397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2674994397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2674994397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:42.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:42.184738+0000) 2023-08-27T22:37:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:42.041Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:42.042Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:42 smithi136 ceph-mon[113531]: pgmap v3125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:42 smithi093 ceph-mon[111335]: pgmap v3125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:42 smithi093 ceph-mon[118902]: pgmap v3125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:43.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:43.184997+0000) 2023-08-27T22:37:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4052835515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:43.957 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4052835515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:43.958 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4052835515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:43.957Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:43.958Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:44.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:44.185266+0000) 2023-08-27T22:37:44.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:44 smithi136 ceph-mon[113531]: pgmap v3126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:44 smithi093 ceph-mon[111335]: pgmap v3126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:44 smithi093 ceph-mon[118902]: pgmap v3126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:45.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:45.185534+0000) 2023-08-27T22:37:45.656 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:37:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:37:45] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:37:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4274110988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4274110988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4274110988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:46.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:46.185808+0000) 2023-08-27T22:37:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:46 smithi136 ceph-mon[113531]: pgmap v3127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:46 smithi093 ceph-mon[111335]: pgmap v3127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:46 smithi093 ceph-mon[118902]: pgmap v3127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:47.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:47.186078+0000) 2023-08-27T22:37:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:48.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:48.186337+0000) 2023-08-27T22:37:49.186 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:48 smithi136 ceph-mon[113531]: pgmap v3128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:49.186 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2415847509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:48 smithi093 ceph-mon[111335]: pgmap v3128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2415847509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:48 smithi093 ceph-mon[118902]: pgmap v3128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2415847509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:49.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:49.186600+0000) 2023-08-27T22:37:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:50.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:50.186813+0000) 2023-08-27T22:37:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:50 smithi136 ceph-mon[113531]: pgmap v3129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3817106813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:50 smithi093 ceph-mon[111335]: pgmap v3129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3817106813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:50 smithi093 ceph-mon[118902]: pgmap v3129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3817106813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:50.532Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:50.533Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:51.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:51.187134+0000) 2023-08-27T22:37:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:52.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:52.187358+0000) 2023-08-27T22:37:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:52.041Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:52.042Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:52 smithi136 ceph-mon[113531]: pgmap v3130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1719863767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:52 smithi093 ceph-mon[111335]: pgmap v3130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1719863767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:52 smithi093 ceph-mon[118902]: pgmap v3130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1719863767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:53.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:53.187629+0000) 2023-08-27T22:37:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:37:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:53.958Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:37:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:37:53.960Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:37:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:54.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:54.187890+0000) 2023-08-27T22:37:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:54 smithi136 ceph-mon[113531]: pgmap v3131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:54 smithi093 ceph-mon[111335]: pgmap v3131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:54 smithi093 ceph-mon[118902]: pgmap v3131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:55.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:55.188149+0000) 2023-08-27T22:37:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:37:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:37:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:37:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4276436963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4276436963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4276436963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:56.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:56.188337+0000) 2023-08-27T22:37:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:56 smithi136 ceph-mon[113531]: pgmap v3132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:57.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:57.188559+0000) 2023-08-27T22:37:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:56 smithi093 ceph-mon[111335]: pgmap v3132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:56 smithi093 ceph-mon[118902]: pgmap v3132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1667204951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1667204951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1667204951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:37:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:58.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:58.188822+0000) 2023-08-27T22:37:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:37:58 smithi093 ceph-mon[111335]: pgmap v3133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:37:58 smithi093 ceph-mon[118902]: pgmap v3133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:37:58 smithi136 ceph-mon[113531]: pgmap v3133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:37:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:37:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:37:59.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:37:59.189094+0000) 2023-08-27T22:38:00.555 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:00.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:00.189382+0000) 2023-08-27T22:38:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:00 smithi136 ceph-mon[113531]: pgmap v3134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2824470805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:00 smithi093 ceph-mon[118902]: pgmap v3134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2824470805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:00 smithi093 ceph-mon[111335]: pgmap v3134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2824470805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:00.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:00.534Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:01.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:01.189630+0000) 2023-08-27T22:38:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:02.042Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:02.043Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:02.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:02.189848+0000) 2023-08-27T22:38:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:02 smithi136 ceph-mon[113531]: pgmap v3135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2646344206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:02 smithi093 ceph-mon[111335]: pgmap v3135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2646344206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:02 smithi093 ceph-mon[118902]: pgmap v3135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2646344206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:03.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:03.190020+0000) 2023-08-27T22:38:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:03.959Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:03.960Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:04.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:04.190271+0000) 2023-08-27T22:38:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:04 smithi136 ceph-mon[113531]: pgmap v3136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3440756302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:04 smithi093 ceph-mon[111335]: pgmap v3136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3440756302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:04 smithi093 ceph-mon[118902]: pgmap v3136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3440756302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:05.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:05.190516+0000) 2023-08-27T22:38:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:38:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:38:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:38:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:06.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:06.190775+0000) 2023-08-27T22:38:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:06 smithi136 ceph-mon[113531]: pgmap v3137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:06 smithi093 ceph-mon[111335]: pgmap v3137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:06 smithi093 ceph-mon[118902]: pgmap v3137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:07.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:07.191029+0000) 2023-08-27T22:38:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/721913126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/721913126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/721913126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:08.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:08.191188+0000) 2023-08-27T22:38:09.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:08 smithi136 ceph-mon[113531]: pgmap v3138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:09.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:08 smithi093 ceph-mon[111335]: pgmap v3138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:08 smithi093 ceph-mon[118902]: pgmap v3138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:09.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:09.191364+0000) 2023-08-27T22:38:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3396545507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3396545507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3396545507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:10.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:10.191583+0000) 2023-08-27T22:38:10.943 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:10.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:10.943 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:10.534Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:10 smithi093 ceph-mon[111335]: pgmap v3139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:10 smithi093 ceph-mon[118902]: pgmap v3139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:10 smithi136 ceph-mon[113531]: pgmap v3139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:11.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:11.191831+0000) 2023-08-27T22:38:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/973753622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/973753622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:12.043Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:12.043Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/973753622' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:12.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:12.192122+0000) 2023-08-27T22:38:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:12 smithi093 ceph-mon[111335]: pgmap v3140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:12 smithi093 ceph-mon[118902]: pgmap v3140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:12 smithi136 ceph-mon[113531]: pgmap v3140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:13.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:13.192344+0000) 2023-08-27T22:38:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:13.960Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:13.961Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:14.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:14.192605+0000) 2023-08-27T22:38:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:14 smithi093 ceph-mon[111335]: pgmap v3141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3651292621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:14 smithi093 ceph-mon[118902]: pgmap v3141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3651292621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:14 smithi136 ceph-mon[113531]: pgmap v3141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3651292621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:15.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:15.192879+0000) 2023-08-27T22:38:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:38:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:38:15] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:38:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:16.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:16.193140+0000) 2023-08-27T22:38:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:16 smithi093 ceph-mon[111335]: pgmap v3142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/256904562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:16 smithi093 ceph-mon[118902]: pgmap v3142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/256904562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:16 smithi136 ceph-mon[113531]: pgmap v3142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/256904562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:17.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:17.193332+0000) 2023-08-27T22:38:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:18.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:18.193615+0000) 2023-08-27T22:38:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:18 smithi093 ceph-mon[111335]: pgmap v3143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:18 smithi093 ceph-mon[118902]: pgmap v3143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:18 smithi136 ceph-mon[113531]: pgmap v3143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:19.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:19.193850+0000) 2023-08-27T22:38:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2051954999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2051954999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2051954999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:20.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:20.194116+0000) 2023-08-27T22:38:20.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:20.534Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:20.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:20.535Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:20 smithi093 ceph-mon[111335]: pgmap v3144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:20 smithi093 ceph-mon[118902]: pgmap v3144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:20 smithi136 ceph-mon[113531]: pgmap v3144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:21.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:21.194372+0000) 2023-08-27T22:38:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3856198138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3856198138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:22.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:22.045Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3856198138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:22.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:22.194646+0000) 2023-08-27T22:38:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:22 smithi093 ceph-mon[111335]: pgmap v3145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:22 smithi093 ceph-mon[118902]: pgmap v3145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:22 smithi136 ceph-mon[113531]: pgmap v3145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:23.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:23.194848+0000) 2023-08-27T22:38:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:38:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3323648451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3323648451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:23.961Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:23.962Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3323648451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:24.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:24.195002+0000) 2023-08-27T22:38:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:24 smithi136 ceph-mon[113531]: pgmap v3146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:24 smithi093 ceph-mon[111335]: pgmap v3146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:24 smithi093 ceph-mon[118902]: pgmap v3146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:25.572 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:25.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:25.195140+0000) 2023-08-27T22:38:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:38:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:38:25] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:38:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:26.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:26.195312+0000) 2023-08-27T22:38:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:27.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:27.195500+0000) 2023-08-27T22:38:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:27 smithi136 ceph-mon[113531]: pgmap v3147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2318679220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:38:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:38:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[111335]: pgmap v3147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2318679220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[118902]: pgmap v3147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2318679220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:38:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:38:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:28.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:28.195756+0000) 2023-08-27T22:38:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:29 smithi136 ceph-mon[113531]: pgmap v3148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/226761810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:29.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:29.195999+0000) 2023-08-27T22:38:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:29 smithi093 ceph-mon[118902]: pgmap v3148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/226761810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:29 smithi093 ceph-mon[111335]: pgmap v3148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/226761810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:30.562 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:30.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:30.196223+0000) 2023-08-27T22:38:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:30 smithi136 ceph-mon[113531]: pgmap v3149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:30 smithi093 ceph-mon[111335]: pgmap v3149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:30 smithi093 ceph-mon[118902]: pgmap v3149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:30.535Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:30.536Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:31.564 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:31.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:31.196480+0000) 2023-08-27T22:38:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/448619028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/448619028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/448619028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:32.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:32.045Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:32.571 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:32.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:32.196744+0000) 2023-08-27T22:38:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:32 smithi136 ceph-mon[113531]: pgmap v3150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:32 smithi093 ceph-mon[111335]: pgmap v3150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:32 smithi093 ceph-mon[118902]: pgmap v3150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:33.569 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:33.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:33.196948+0000) 2023-08-27T22:38:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/78727808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:33.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/78727808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:33.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/78727808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:34.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:33.961Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:34.243 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:33.962Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:34.578 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:34.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:34.197207+0000) 2023-08-27T22:38:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:34 smithi136 ceph-mon[113531]: pgmap v3151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:34 smithi093 ceph-mon[111335]: pgmap v3151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:34 smithi093 ceph-mon[118902]: pgmap v3151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:35.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:35.197453+0000) 2023-08-27T22:38:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:38:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:38:35] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:38:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2462638891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2462638891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2462638891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:36.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:36.197745+0000) 2023-08-27T22:38:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:36 smithi136 ceph-mon[113531]: pgmap v3152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:36 smithi093 ceph-mon[111335]: pgmap v3152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:36 smithi093 ceph-mon[118902]: pgmap v3152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:37.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:37.197971+0000) 2023-08-27T22:38:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:38.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:38.198159+0000) 2023-08-27T22:38:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:38 smithi136 ceph-mon[113531]: pgmap v3153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3194247255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:38 smithi093 ceph-mon[111335]: pgmap v3153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3194247255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:38 smithi093 ceph-mon[118902]: pgmap v3153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3194247255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:39.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:39.198383+0000) 2023-08-27T22:38:40.563 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:40.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:40.198625+0000) 2023-08-27T22:38:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:40 smithi136 ceph-mon[113531]: pgmap v3154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/37594123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:40.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:40 smithi093 ceph-mon[111335]: pgmap v3154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:40.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/37594123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:40 smithi093 ceph-mon[118902]: pgmap v3154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/37594123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:40.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:40.536Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:40.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:40.537Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:41.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:41.198880+0000) 2023-08-27T22:38:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:42.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:42.046Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:42.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:42.199133+0000) 2023-08-27T22:38:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:42 smithi136 ceph-mon[113531]: pgmap v3155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1079525534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:42 smithi093 ceph-mon[111335]: pgmap v3155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1079525534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:42 smithi093 ceph-mon[118902]: pgmap v3155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1079525534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:43.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:43.199361+0000) 2023-08-27T22:38:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:43.962Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:43.963Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:44.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:44.199618+0000) 2023-08-27T22:38:45.200 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:44 smithi136 ceph-mon[113531]: pgmap v3156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:44 smithi093 ceph-mon[111335]: pgmap v3156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:44 smithi093 ceph-mon[118902]: pgmap v3156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:45.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:45.199889+0000) 2023-08-27T22:38:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:38:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:38:45] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T22:38:46.200 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/361940019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/361940019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/361940019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:46.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:46.200170+0000) 2023-08-27T22:38:47.200 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:46 smithi136 ceph-mon[113531]: pgmap v3157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:46 smithi093 ceph-mon[111335]: pgmap v3157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:46 smithi093 ceph-mon[118902]: pgmap v3157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:47.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:47.200400+0000) 2023-08-27T22:38:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1512525548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1512525548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1512525548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:48.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:48.200627+0000) 2023-08-27T22:38:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:48 smithi093 ceph-mon[118902]: pgmap v3158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:48 smithi093 ceph-mon[111335]: pgmap v3158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:48 smithi136 ceph-mon[113531]: pgmap v3158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:49.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:49.200852+0000) 2023-08-27T22:38:50.564 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:50.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:50.201102+0000) 2023-08-27T22:38:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:50 smithi136 ceph-mon[113531]: pgmap v3159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2739504026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:50 smithi093 ceph-mon[118902]: pgmap v3159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2739504026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:50 smithi093 ceph-mon[111335]: pgmap v3159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2739504026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:50.537Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:50.538Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:51.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:51.201340+0000) 2023-08-27T22:38:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:52.046Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:52.047Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:52.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:52.201558+0000) 2023-08-27T22:38:53.201 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:52 smithi136 ceph-mon[113531]: pgmap v3160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:53.201 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2839237389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:52 smithi093 ceph-mon[111335]: pgmap v3160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2839237389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:52 smithi093 ceph-mon[118902]: pgmap v3160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2839237389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:53.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:53.201755+0000) 2023-08-27T22:38:54.202 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:38:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:53.963Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:38:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:38:53.964Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:38:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:54.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:54.202018+0000) 2023-08-27T22:38:55.202 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:54 smithi136 ceph-mon[113531]: pgmap v3161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:55.202 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1441313616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:54 smithi093 ceph-mon[111335]: pgmap v3161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1441313616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:54 smithi093 ceph-mon[118902]: pgmap v3161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1441313616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:55.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:55.202351+0000) 2023-08-27T22:38:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:38:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:38:55] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:38:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:56.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:56.202602+0000) 2023-08-27T22:38:57.203 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:56 smithi136 ceph-mon[113531]: pgmap v3162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:56 smithi093 ceph-mon[111335]: pgmap v3162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:56 smithi093 ceph-mon[118902]: pgmap v3162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:57.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:57.202846+0000) 2023-08-27T22:38:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2142736052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2142736052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2142736052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:38:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:58.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:58.203075+0000) 2023-08-27T22:38:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:58 smithi093 ceph-mon[111335]: pgmap v3163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:58 smithi093 ceph-mon[118902]: pgmap v3163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:58 smithi136 ceph-mon[113531]: pgmap v3163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:38:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:38:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:38:59.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:38:59.203341+0000) 2023-08-27T22:39:00.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:38:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/658762825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:38:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/658762825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:38:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/658762825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:00.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:00.203612+0000) 2023-08-27T22:39:00.960 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:00.538Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:00.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:00.539Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:00 smithi093 ceph-mon[111335]: pgmap v3164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:00 smithi093 ceph-mon[118902]: pgmap v3164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:00 smithi136 ceph-mon[113531]: pgmap v3164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:01.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:01.203901+0000) 2023-08-27T22:39:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2212097273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2212097273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:02.046Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:02.047Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2212097273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:02.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:02.204186+0000) 2023-08-27T22:39:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:02 smithi093 ceph-mon[111335]: pgmap v3165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:02 smithi093 ceph-mon[118902]: pgmap v3165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:02 smithi136 ceph-mon[113531]: pgmap v3165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:03.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:03.204398+0000) 2023-08-27T22:39:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:03.963Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:03.965Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:04.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:04.204610+0000) 2023-08-27T22:39:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:04 smithi093 ceph-mon[111335]: pgmap v3166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1717472676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:04 smithi093 ceph-mon[118902]: pgmap v3166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1717472676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:04 smithi136 ceph-mon[113531]: pgmap v3166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1717472676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:05.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:05.204880+0000) 2023-08-27T22:39:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:39:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:39:05] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:39:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:06.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:06.205182+0000) 2023-08-27T22:39:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:06 smithi093 ceph-mon[111335]: pgmap v3167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1965089796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:06 smithi093 ceph-mon[118902]: pgmap v3167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1965089796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:06 smithi136 ceph-mon[113531]: pgmap v3167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1965089796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:07.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:07.205367+0000) 2023-08-27T22:39:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:08.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:08.205624+0000) 2023-08-27T22:39:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:08 smithi093 ceph-mon[111335]: pgmap v3168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:08 smithi093 ceph-mon[118902]: pgmap v3168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:08 smithi136 ceph-mon[113531]: pgmap v3168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:09.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:09.205930+0000) 2023-08-27T22:39:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2415114376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2415114376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2415114376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:10.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:10.206170+0000) 2023-08-27T22:39:10.984 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:10.539Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:10.984 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:10.540Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:10 smithi093 ceph-mon[111335]: pgmap v3169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:10 smithi093 ceph-mon[118902]: pgmap v3169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:10 smithi136 ceph-mon[113531]: pgmap v3169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:11.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:11.206384+0000) 2023-08-27T22:39:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1008403643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:12.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:12.206644+0000) 2023-08-27T22:39:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1008403643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1008403643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:12.048Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:12.049Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:12 smithi136 ceph-mon[113531]: pgmap v3170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:13.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:13.206952+0000) 2023-08-27T22:39:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:12 smithi093 ceph-mon[111335]: pgmap v3170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:12 smithi093 ceph-mon[118902]: pgmap v3170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/645048404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/645048404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:13.964Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:13.965Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/645048404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:14.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:14.207211+0000) 2023-08-27T22:39:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:15 smithi093 ceph-mon[111335]: pgmap v3171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:15 smithi093 ceph-mon[118902]: pgmap v3171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:15 smithi136 ceph-mon[113531]: pgmap v3171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:15.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:15.207456+0000) 2023-08-27T22:39:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:39:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:39:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:39:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:16.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:16.207766+0000) 2023-08-27T22:39:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:17 smithi136 ceph-mon[113531]: pgmap v3172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1480438830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:17.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:17.208103+0000) 2023-08-27T22:39:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:17 smithi093 ceph-mon[111335]: pgmap v3172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1480438830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:17 smithi093 ceph-mon[118902]: pgmap v3172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1480438830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:18.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:18.208351+0000) 2023-08-27T22:39:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:19 smithi136 ceph-mon[113531]: pgmap v3173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2226346258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:19.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:19.208590+0000) 2023-08-27T22:39:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:19 smithi093 ceph-mon[111335]: pgmap v3173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2226346258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:19 smithi093 ceph-mon[118902]: pgmap v3173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2226346258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:20.569 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:20.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:20.208839+0000) 2023-08-27T22:39:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:20 smithi136 ceph-mon[113531]: pgmap v3174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:20 smithi093 ceph-mon[111335]: pgmap v3174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:20 smithi093 ceph-mon[118902]: pgmap v3174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:20.540Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:20.541Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:21.570 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:21.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:21.209147+0000) 2023-08-27T22:39:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1602769351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1602769351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1602769351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:22.049Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:22.050Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:22.575 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:22.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:22.209347+0000) 2023-08-27T22:39:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:22 smithi136 ceph-mon[113531]: pgmap v3175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:22 smithi093 ceph-mon[111335]: pgmap v3175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:23.039 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:22 smithi093 ceph-mon[118902]: pgmap v3175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:23.577 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:23.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:23.209601+0000) 2023-08-27T22:39:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/363977665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:23.964 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/363977665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:23.964 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/363977665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:23.964Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:24.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:23.965Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:24.581 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:24.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:24.209821+0000) 2023-08-27T22:39:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:24 smithi136 ceph-mon[113531]: pgmap v3176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:24 smithi093 ceph-mon[111335]: pgmap v3176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:24 smithi093 ceph-mon[118902]: pgmap v3176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:25.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:25.210124+0000) 2023-08-27T22:39:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:39:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:39:25] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:39:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2632074212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2632074212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2632074212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:26.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:26.210405+0000) 2023-08-27T22:39:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:26 smithi136 ceph-mon[113531]: pgmap v3177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:39:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:26 smithi093 ceph-mon[111335]: pgmap v3177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:39:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:26 smithi093 ceph-mon[118902]: pgmap v3177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:39:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:27.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:27.210591+0000) 2023-08-27T22:39:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:28.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:28.210779+0000) 2023-08-27T22:39:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:29.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:29.210940+0000) 2023-08-27T22:39:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: pgmap v3178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2782702852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:39:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:39:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: pgmap v3178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2782702852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: pgmap v3178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2782702852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:29.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:29.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:39:29.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:39:29.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:39:30.570 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:30.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:30.211246+0000) 2023-08-27T22:39:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:30 smithi136 ceph-mon[113531]: pgmap v3179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3293855042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:30 smithi093 ceph-mon[118902]: pgmap v3179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3293855042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:30 smithi093 ceph-mon[111335]: pgmap v3179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3293855042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:30.541Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:30.542Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:31.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:31.211548+0000) 2023-08-27T22:39:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:32.050Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:32.051Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:32.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:32.211819+0000) 2023-08-27T22:39:33.212 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:32 smithi136 ceph-mon[113531]: pgmap v3180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:33.212 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3759612307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:32 smithi093 ceph-mon[111335]: pgmap v3180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3759612307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:32 smithi093 ceph-mon[118902]: pgmap v3180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3759612307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:33.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:33.212065+0000) 2023-08-27T22:39:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:33.965Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:33.966Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:34.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:34.212325+0000) 2023-08-27T22:39:35.212 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:34 smithi136 ceph-mon[113531]: pgmap v3181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:34 smithi093 ceph-mon[111335]: pgmap v3181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:34 smithi093 ceph-mon[118902]: pgmap v3181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:35.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:35.212625+0000) 2023-08-27T22:39:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:39:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:39:35] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:39:36.213 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3211415259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3211415259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3211415259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:36.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:36.212926+0000) 2023-08-27T22:39:37.213 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:36 smithi136 ceph-mon[113531]: pgmap v3182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:36 smithi093 ceph-mon[111335]: pgmap v3182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:36 smithi093 ceph-mon[118902]: pgmap v3182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:37.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:37.213171+0000) 2023-08-27T22:39:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4208951271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4208951271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4208951271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:38.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:38.213390+0000) 2023-08-27T22:39:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:38 smithi093 ceph-mon[111335]: pgmap v3183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:38 smithi093 ceph-mon[118902]: pgmap v3183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:39.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:39.213689+0000) 2023-08-27T22:39:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:38 smithi136 ceph-mon[113531]: pgmap v3183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:40.571 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:40.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:40.213918+0000) 2023-08-27T22:39:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:40 smithi136 ceph-mon[113531]: pgmap v3184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4027098417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:40 smithi093 ceph-mon[118902]: pgmap v3184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4027098417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:40 smithi093 ceph-mon[111335]: pgmap v3184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4027098417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:40.541Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:40.542Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:41.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:41.214129+0000) 2023-08-27T22:39:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:42.051Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:42.052Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:42.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:42.214389+0000) 2023-08-27T22:39:43.214 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:42 smithi136 ceph-mon[113531]: pgmap v3185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:43.215 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1355161283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:42 smithi093 ceph-mon[111335]: pgmap v3185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1355161283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:42 smithi093 ceph-mon[118902]: pgmap v3185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1355161283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:43.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:43.214699+0000) 2023-08-27T22:39:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:43.965Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:43.966Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:44.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:44.214970+0000) 2023-08-27T22:39:45.215 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:44 smithi136 ceph-mon[113531]: pgmap v3186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:45.215 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1161482564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:44 smithi093 ceph-mon[118902]: pgmap v3186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1161482564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:44 smithi093 ceph-mon[111335]: pgmap v3186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1161482564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:45.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:45.215196+0000) 2023-08-27T22:39:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:39:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:39:45] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:39:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:46.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:46.215455+0000) 2023-08-27T22:39:47.215 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:46 smithi136 ceph-mon[113531]: pgmap v3187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:46 smithi093 ceph-mon[111335]: pgmap v3187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:46 smithi093 ceph-mon[118902]: pgmap v3187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:47.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:47.215699+0000) 2023-08-27T22:39:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/295557367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/295557367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/295557367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:48.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:48.215896+0000) 2023-08-27T22:39:49.216 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:48 smithi136 ceph-mon[113531]: pgmap v3188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:48 smithi093 ceph-mon[111335]: pgmap v3188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:48 smithi093 ceph-mon[118902]: pgmap v3188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:49.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:49.216134+0000) 2023-08-27T22:39:50.574 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:50.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:50.216343+0000) 2023-08-27T22:39:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2044041582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:50 smithi136 ceph-mon[113531]: pgmap v3189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2044041582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:50 smithi093 ceph-mon[111335]: pgmap v3189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2044041582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:50 smithi093 ceph-mon[118902]: pgmap v3189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:50.542Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:50.543Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:51.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:51.216607+0000) 2023-08-27T22:39:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:52.052Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:52.052Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:52.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:52.216874+0000) 2023-08-27T22:39:53.217 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:52 smithi136 ceph-mon[113531]: pgmap v3190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:53.217 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1684794293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:52 smithi093 ceph-mon[111335]: pgmap v3190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:53.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1684794293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:53.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:52 smithi093 ceph-mon[118902]: pgmap v3190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:53.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1684794293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:53.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:53.217113+0000) 2023-08-27T22:39:54.217 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:39:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:53.966Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:39:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:39:53.967Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:39:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:54.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:54.217377+0000) 2023-08-27T22:39:55.217 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:54 smithi136 ceph-mon[113531]: pgmap v3191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:55.217 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1127183629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:54 smithi093 ceph-mon[111335]: pgmap v3191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1127183629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:54 smithi093 ceph-mon[118902]: pgmap v3191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1127183629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:55.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:55.217670+0000) 2023-08-27T22:39:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:39:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:39:55] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:39:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:56.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:56.217950+0000) 2023-08-27T22:39:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:56 smithi093 ceph-mon[111335]: pgmap v3192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2842218348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:56 smithi093 ceph-mon[118902]: pgmap v3192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2842218348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:56 smithi136 ceph-mon[113531]: pgmap v3192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2842218348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:39:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:57.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:57.218168+0000) 2023-08-27T22:39:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:58.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:58.218381+0000) 2023-08-27T22:39:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:58 smithi093 ceph-mon[111335]: pgmap v3193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:58 smithi093 ceph-mon[118902]: pgmap v3193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:58 smithi136 ceph-mon[113531]: pgmap v3193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:39:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:39:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:39:59.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:39:59.218666+0000) 2023-08-27T22:40:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:39:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3007435121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:39:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3007435121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:39:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3007435121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:00.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:00.218930+0000) 2023-08-27T22:40:00.977 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:00.543Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:00.978 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:00.544Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:00 smithi093 ceph-mon[118902]: pgmap v3194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:40:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:00 smithi093 ceph-mon[111335]: pgmap v3194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:40:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:00 smithi136 ceph-mon[113531]: pgmap v3194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:40:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:01.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:01.219171+0000) 2023-08-27T22:40:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/990246542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/990246542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:02.052Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:02.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:02.053Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/990246542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:02.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:02.219381+0000) 2023-08-27T22:40:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:02 smithi093 ceph-mon[111335]: pgmap v3195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:02 smithi093 ceph-mon[118902]: pgmap v3195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:02 smithi136 ceph-mon[113531]: pgmap v3195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:03.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:03.219639+0000) 2023-08-27T22:40:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3433529203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3433529203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:03.966Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:03.967Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3433529203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:04.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:04.219926+0000) 2023-08-27T22:40:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:04 smithi093 ceph-mon[111335]: pgmap v3196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:04 smithi093 ceph-mon[118902]: pgmap v3196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:04 smithi136 ceph-mon[113531]: pgmap v3196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:05.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:05.220202+0000) 2023-08-27T22:40:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:40:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:40:05] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:40:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:06.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:06.220421+0000) 2023-08-27T22:40:07.221 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:06 smithi136 ceph-mon[113531]: pgmap v3197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:07.221 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2189258099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:06 smithi093 ceph-mon[111335]: pgmap v3197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2189258099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:06 smithi093 ceph-mon[118902]: pgmap v3197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2189258099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:07.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:07.220630+0000) 2023-08-27T22:40:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:08.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:08.220904+0000) 2023-08-27T22:40:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:09.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:09.221130+0000) 2023-08-27T22:40:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:09 smithi136 ceph-mon[113531]: pgmap v3198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2154601244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:08 smithi093 ceph-mon[111335]: pgmap v3198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2154601244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:09 smithi093 ceph-mon[118902]: pgmap v3198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2154601244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:10.578 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:10.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:10.221403+0000) 2023-08-27T22:40:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:10 smithi136 ceph-mon[113531]: pgmap v3199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:10 smithi093 ceph-mon[111335]: pgmap v3199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:10 smithi093 ceph-mon[118902]: pgmap v3199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:10.544Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:10.545Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:11.579 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:11.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:11.221644+0000) 2023-08-27T22:40:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/993976268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/993976268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/993976268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:12.053Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:12.054Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:12.584 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:12.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:12.221912+0000) 2023-08-27T22:40:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:12 smithi136 ceph-mon[113531]: pgmap v3200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:12 smithi093 ceph-mon[111335]: pgmap v3200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:12 smithi093 ceph-mon[118902]: pgmap v3200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:13.585 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:13.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:13.222212+0000) 2023-08-27T22:40:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3938590362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:13.967 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3938590362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:13.967 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3938590362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:13.967Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:13.968Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:14.590 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:14.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:14.222523+0000) 2023-08-27T22:40:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:14 smithi136 ceph-mon[113531]: pgmap v3201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:14 smithi093 ceph-mon[111335]: pgmap v3201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:14 smithi093 ceph-mon[118902]: pgmap v3201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:15.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:15.222819+0000) 2023-08-27T22:40:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:40:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:40:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:40:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3923195767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3923195767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3923195767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:16.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:16.223117+0000) 2023-08-27T22:40:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:16 smithi136 ceph-mon[113531]: pgmap v3202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:16 smithi093 ceph-mon[111335]: pgmap v3202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:17.496 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:16 smithi093 ceph-mon[118902]: pgmap v3202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:17.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:17.223375+0000) 2023-08-27T22:40:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:18.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:18.223628+0000) 2023-08-27T22:40:19.223 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:18 smithi136 ceph-mon[113531]: pgmap v3203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:19.224 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1696891815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:18 smithi093 ceph-mon[111335]: pgmap v3203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1696891815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:18 smithi093 ceph-mon[118902]: pgmap v3203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1696891815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:19.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:19.223902+0000) 2023-08-27T22:40:20.582 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:20.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:20.224186+0000) 2023-08-27T22:40:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:20 smithi136 ceph-mon[113531]: pgmap v3204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/157348376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:20 smithi093 ceph-mon[111335]: pgmap v3204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/157348376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:20 smithi093 ceph-mon[118902]: pgmap v3204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/157348376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:20.545Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:20.546Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:21.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:21.224434+0000) 2023-08-27T22:40:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:22.054Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:22.055Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:22.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:22.224705+0000) 2023-08-27T22:40:23.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:22 smithi136 ceph-mon[113531]: pgmap v3205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:23.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2058785255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:22 smithi093 ceph-mon[111335]: pgmap v3205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2058785255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:22 smithi093 ceph-mon[118902]: pgmap v3205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2058785255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:23.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:23.225010+0000) 2023-08-27T22:40:24.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:23.967Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:23.968Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:24.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:24.225311+0000) 2023-08-27T22:40:25.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:24 smithi136 ceph-mon[113531]: pgmap v3206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:24 smithi093 ceph-mon[111335]: pgmap v3206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:24 smithi093 ceph-mon[118902]: pgmap v3206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:25.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:25.225574+0000) 2023-08-27T22:40:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:40:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:40:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:40:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1175859968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1175859968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1175859968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:26.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:26.225832+0000) 2023-08-27T22:40:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:26 smithi093 ceph-mon[111335]: pgmap v3207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:26 smithi093 ceph-mon[118902]: pgmap v3207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:26 smithi136 ceph-mon[113531]: pgmap v3207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:27.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:27.226133+0000) 2023-08-27T22:40:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1703783491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1703783491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1703783491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:28.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:28.226369+0000) 2023-08-27T22:40:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:28 smithi093 ceph-mon[111335]: pgmap v3208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:28 smithi093 ceph-mon[118902]: pgmap v3208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:28 smithi136 ceph-mon[113531]: pgmap v3208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:29.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:29.226543+0000) 2023-08-27T22:40:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:40:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:40:30.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:40:30.636 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:30.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:30.226694+0000) 2023-08-27T22:40:30.708 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:30.546Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:30.708 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:30.547Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[111335]: pgmap v3209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1457037015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[118902]: pgmap v3209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1457037015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:31.333 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:31.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:31.226849+0000) 2023-08-27T22:40:31.333 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:30 smithi136 ceph-mon[113531]: pgmap v3209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:31.333 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1457037015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:31.333 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:31.333 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:32.055Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:32.055Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:32.518 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:32.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:32.227089+0000) 2023-08-27T22:40:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:32 smithi136 ceph-mon[113531]: pgmap v3210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:40:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:40:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[118902]: pgmap v3210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[111335]: pgmap v3210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:40:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:40:32.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:40:33.527 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:33.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:33.227344+0000) 2023-08-27T22:40:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/766120619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:33.968 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/766120619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:33.968 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/766120619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:33.968Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:33.969Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:34.531 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:34.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:34.227598+0000) 2023-08-27T22:40:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:34 smithi136 ceph-mon[113531]: pgmap v3211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:34 smithi093 ceph-mon[111335]: pgmap v3211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:34.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:34 smithi093 ceph-mon[118902]: pgmap v3211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:35.578 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:35.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:35.227876+0000) 2023-08-27T22:40:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3854631145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3854631145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:40:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:40:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:40:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3854631145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:36.582 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:36.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:36.228112+0000) 2023-08-27T22:40:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:36 smithi136 ceph-mon[113531]: pgmap v3212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:36 smithi093 ceph-mon[111335]: pgmap v3212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:36 smithi093 ceph-mon[118902]: pgmap v3212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:37.583 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:37.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:37.228334+0000) 2023-08-27T22:40:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4067602225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4067602225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4067602225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:38.588 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:38.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:38.228564+0000) 2023-08-27T22:40:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:38 smithi136 ceph-mon[113531]: pgmap v3213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:38 smithi093 ceph-mon[111335]: pgmap v3213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:38 smithi093 ceph-mon[118902]: pgmap v3213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:39.592 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:39.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:39.228848+0000) 2023-08-27T22:40:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:40.597 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:40.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:40.229164+0000) 2023-08-27T22:40:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3716653850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:40 smithi136 ceph-mon[113531]: pgmap v3214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3716653850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:40 smithi093 ceph-mon[111335]: pgmap v3214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3716653850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:40 smithi093 ceph-mon[118902]: pgmap v3214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:40.548Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:40.548Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:41.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:41.229436+0000) 2023-08-27T22:40:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:42.055Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:42.056Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:42.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:42.229703+0000) 2023-08-27T22:40:43.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:42 smithi136 ceph-mon[113531]: pgmap v3215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:43.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3176910727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:42 smithi093 ceph-mon[118902]: pgmap v3215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3176910727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:42 smithi093 ceph-mon[111335]: pgmap v3215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3176910727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:43.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:43.229979+0000) 2023-08-27T22:40:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:43.969Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:43.970Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:44.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:44.230226+0000) 2023-08-27T22:40:45.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:44 smithi136 ceph-mon[113531]: pgmap v3216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:45.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2240485935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:44 smithi093 ceph-mon[111335]: pgmap v3216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2240485935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:44 smithi093 ceph-mon[118902]: pgmap v3216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2240485935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:45.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:45.230452+0000) 2023-08-27T22:40:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:40:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:40:45] "GET /metrics HTTP/1.1" 200 239911 "" "Prometheus/2.43.0" 2023-08-27T22:40:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:46.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:46.230709+0000) 2023-08-27T22:40:47.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:46 smithi136 ceph-mon[113531]: pgmap v3217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:47.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2541039120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:46 smithi093 ceph-mon[111335]: pgmap v3217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2541039120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:46 smithi093 ceph-mon[118902]: pgmap v3217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2541039120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:47.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:47.230953+0000) 2023-08-27T22:40:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:48.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:48.231180+0000) 2023-08-27T22:40:49.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:48 smithi136 ceph-mon[113531]: pgmap v3218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:48 smithi093 ceph-mon[111335]: pgmap v3218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:48 smithi093 ceph-mon[118902]: pgmap v3218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:49.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:49.231435+0000) 2023-08-27T22:40:50.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3648010635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3648010635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3648010635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:50.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:50.231681+0000) 2023-08-27T22:40:50.977 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:50.549Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:50.977 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:50.550Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:51.232 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:50 smithi136 ceph-mon[113531]: pgmap v3219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:50 smithi093 ceph-mon[111335]: pgmap v3219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:50 smithi093 ceph-mon[118902]: pgmap v3219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:51.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:51.231918+0000) 2023-08-27T22:40:52.232 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1333690657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1333690657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1333690657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:52.056Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:52.057Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:52.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:52.232160+0000) 2023-08-27T22:40:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:52 smithi093 ceph-mon[111335]: pgmap v3220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:52 smithi093 ceph-mon[118902]: pgmap v3220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:52 smithi136 ceph-mon[113531]: pgmap v3220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:53.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:53.232405+0000) 2023-08-27T22:40:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:53.970Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:40:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:40:53.971Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:40:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:40:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:54.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:54.232728+0000) 2023-08-27T22:40:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:54 smithi093 ceph-mon[111335]: pgmap v3221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2774196185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:54 smithi093 ceph-mon[118902]: pgmap v3221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2774196185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:55.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:55.233049+0000) 2023-08-27T22:40:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:54 smithi136 ceph-mon[113531]: pgmap v3221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2774196185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:40:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:40:55] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:40:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:56.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:56.233347+0000) 2023-08-27T22:40:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:57.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:57.233562+0000) 2023-08-27T22:40:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:56 smithi136 ceph-mon[113531]: pgmap v3222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4179665812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:56 smithi093 ceph-mon[118902]: pgmap v3222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4179665812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:56 smithi093 ceph-mon[111335]: pgmap v3222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4179665812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:58.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:58.233820+0000) 2023-08-27T22:40:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:40:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:40:59.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:40:59.234103+0000) 2023-08-27T22:40:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:58 smithi136 ceph-mon[113531]: pgmap v3223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:40:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3020313509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:58 smithi093 ceph-mon[111335]: pgmap v3223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:40:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3020313509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:40:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:59 smithi093 ceph-mon[118902]: pgmap v3223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:40:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:40:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3020313509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:00.588 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:00.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:00.234342+0000) 2023-08-27T22:41:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:00 smithi136 ceph-mon[113531]: pgmap v3224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:00 smithi093 ceph-mon[111335]: pgmap v3224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:00 smithi093 ceph-mon[118902]: pgmap v3224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:00.549Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:00.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:01.589 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:01.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:01.234627+0000) 2023-08-27T22:41:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3092717400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3092717400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3092717400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:02.057Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:02.058Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:02.594 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:02.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:02.234859+0000) 2023-08-27T22:41:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:02 smithi136 ceph-mon[113531]: pgmap v3225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:02 smithi093 ceph-mon[111335]: pgmap v3225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:02 smithi093 ceph-mon[118902]: pgmap v3225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:03.595 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:03.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:03.235149+0000) 2023-08-27T22:41:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2130982031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:03.971 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2130982031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:03.971 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2130982031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:03.971Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:03.972Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:04.605 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:04.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:04.235435+0000) 2023-08-27T22:41:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:04 smithi136 ceph-mon[113531]: pgmap v3226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:04 smithi093 ceph-mon[111335]: pgmap v3226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:04 smithi093 ceph-mon[118902]: pgmap v3226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:05.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:05.235731+0000) 2023-08-27T22:41:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 22:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T22:41:05.318241951Z level=info msg="Completed cleanup jobs" duration=13.123196ms 2023-08-27T22:41:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:41:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:41:05] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:41:06.236 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3296197944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3296197944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3296197944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:06.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:06.235978+0000) 2023-08-27T22:41:07.236 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:06 smithi136 ceph-mon[113531]: pgmap v3227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:06 smithi093 ceph-mon[111335]: pgmap v3227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:06 smithi093 ceph-mon[118902]: pgmap v3227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:07.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:07.236227+0000) 2023-08-27T22:41:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:08.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:08.236490+0000) 2023-08-27T22:41:09.237 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:08 smithi136 ceph-mon[113531]: pgmap v3228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:09.237 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/693473498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:09.237 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:08 smithi093 ceph-mon[111335]: pgmap v3228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/693473498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:08 smithi093 ceph-mon[118902]: pgmap v3228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/693473498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:09.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:09.236765+0000) 2023-08-27T22:41:10.587 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:10.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:10.237008+0000) 2023-08-27T22:41:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:10 smithi136 ceph-mon[113531]: pgmap v3229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:10 smithi093 ceph-mon[111335]: pgmap v3229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:10 smithi093 ceph-mon[118902]: pgmap v3229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:10.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:10.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:11.588 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:11.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:11.237229+0000) 2023-08-27T22:41:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2204614696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2204614696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2204614696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:12.057Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:12.058Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:12.593 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:12.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:12.237445+0000) 2023-08-27T22:41:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:12 smithi136 ceph-mon[113531]: pgmap v3230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:12 smithi093 ceph-mon[111335]: pgmap v3230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:12 smithi093 ceph-mon[118902]: pgmap v3230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:13.595 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:13.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:13.237704+0000) 2023-08-27T22:41:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3217513422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:13.972 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3217513422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:13.972 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3217513422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:13.972Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:13.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:14.604 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:14.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:14.237988+0000) 2023-08-27T22:41:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:14 smithi136 ceph-mon[113531]: pgmap v3231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:14 smithi093 ceph-mon[111335]: pgmap v3231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:14 smithi093 ceph-mon[118902]: pgmap v3231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:15.605 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:15.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:15.238267+0000) 2023-08-27T22:41:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2819593654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2819593654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:41:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:41:15] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T22:41:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2819593654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:16.610 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:16.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:16.238521+0000) 2023-08-27T22:41:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:16 smithi136 ceph-mon[113531]: pgmap v3232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:16 smithi093 ceph-mon[111335]: pgmap v3232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:16 smithi093 ceph-mon[118902]: pgmap v3232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:17.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:17.238799+0000) 2023-08-27T22:41:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3251954676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3251954676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3251954676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:18.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:18.239067+0000) 2023-08-27T22:41:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:18 smithi136 ceph-mon[113531]: pgmap v3233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:18 smithi093 ceph-mon[111335]: pgmap v3233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:18 smithi093 ceph-mon[118902]: pgmap v3233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:19.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:19.239355+0000) 2023-08-27T22:41:20.589 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:20.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:20.239657+0000) 2023-08-27T22:41:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:20 smithi136 ceph-mon[113531]: pgmap v3234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4071355942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:20 smithi093 ceph-mon[111335]: pgmap v3234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4071355942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:20 smithi093 ceph-mon[118902]: pgmap v3234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4071355942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:20.551Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:20.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:21.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:21.239943+0000) 2023-08-27T22:41:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:22.058Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:22.059Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:22.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:22.240173+0000) 2023-08-27T22:41:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:22 smithi093 ceph-mon[118902]: pgmap v3235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2268559883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:22 smithi093 ceph-mon[111335]: pgmap v3235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2268559883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:23.240 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:22 smithi136 ceph-mon[113531]: pgmap v3235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:23.240 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2268559883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:23.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:23.240438+0000) 2023-08-27T22:41:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:23.972Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:23.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:24.240 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:24.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:24.240724+0000) 2023-08-27T22:41:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:24 smithi093 ceph-mon[111335]: pgmap v3236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1235433923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:24 smithi093 ceph-mon[118902]: pgmap v3236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1235433923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:25.240 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:24 smithi136 ceph-mon[113531]: pgmap v3236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:25.240 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1235433923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:25.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:25.241002+0000) 2023-08-27T22:41:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:41:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:41:25] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:41:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:26.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:26.241203+0000) 2023-08-27T22:41:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:26 smithi093 ceph-mon[111335]: pgmap v3237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:26 smithi093 ceph-mon[118902]: pgmap v3237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:27.241 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:26 smithi136 ceph-mon[113531]: pgmap v3237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:27.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:27.241401+0000) 2023-08-27T22:41:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1265496515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:28.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:28.241688+0000) 2023-08-27T22:41:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1265496515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1265496515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:28 smithi136 ceph-mon[113531]: pgmap v3238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:29.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:29.241911+0000) 2023-08-27T22:41:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:28 smithi093 ceph-mon[111335]: pgmap v3238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:28 smithi093 ceph-mon[118902]: pgmap v3238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:30.593 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:30.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:30.242183+0000) 2023-08-27T22:41:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3013212814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:30 smithi136 ceph-mon[113531]: pgmap v3239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3013212814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:30 smithi093 ceph-mon[111335]: pgmap v3239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3013212814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:30 smithi093 ceph-mon[118902]: pgmap v3239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:30.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:30.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:31.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:31.242412+0000) 2023-08-27T22:41:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:32.058Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:32.059Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:32.670 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:32.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:32.242580+0000) 2023-08-27T22:41:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-mon[118902]: pgmap v3240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:41:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1628207011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-mon[111335]: pgmap v3240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:41:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1628207011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:33.243 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:32 smithi136 ceph-mon[113531]: pgmap v3240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:33.243 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:41:33.243 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1628207011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:33.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:33.242739+0000) 2023-08-27T22:41:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:33.973Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:33.974Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:34.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:34.242898+0000) 2023-08-27T22:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: pgmap v3241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3879735413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: pgmap v3241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3879735413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: pgmap v3241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3879735413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:41:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:35.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:35.243173+0000) 2023-08-27T22:41:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:41:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:41:35] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:41:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:36.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:36.243402+0000) 2023-08-27T22:41:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:36 smithi093 ceph-mon[118902]: pgmap v3242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2038385472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:36 smithi093 ceph-mon[111335]: pgmap v3242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2038385472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:37.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:36 smithi136 ceph-mon[113531]: pgmap v3242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:37.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2038385472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:37.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:37.243676+0000) 2023-08-27T22:41:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:38.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:38.243955+0000) 2023-08-27T22:41:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:38 smithi093 ceph-mon[111335]: pgmap v3243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:38 smithi093 ceph-mon[118902]: pgmap v3243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:39.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:38 smithi136 ceph-mon[113531]: pgmap v3243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:39.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:39.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:39.244218+0000) 2023-08-27T22:41:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1088168487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1088168487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:40.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1088168487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:40.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:40.244471+0000) 2023-08-27T22:41:40.986 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:40.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:40.986 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:40.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:40 smithi093 ceph-mon[111335]: pgmap v3244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:40 smithi093 ceph-mon[118902]: pgmap v3244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:41.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:40 smithi136 ceph-mon[113531]: pgmap v3244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:41.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:41.244682+0000) 2023-08-27T22:41:42.245 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2660979163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2660979163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2660979163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:42.059Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:42.060Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:42.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:42.244915+0000) 2023-08-27T22:41:43.245 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:42 smithi136 ceph-mon[113531]: pgmap v3245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:42 smithi093 ceph-mon[111335]: pgmap v3245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:42 smithi093 ceph-mon[118902]: pgmap v3245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:43.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:43.245090+0000) 2023-08-27T22:41:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:43.974Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:43.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:44.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:44.245349+0000) 2023-08-27T22:41:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:44 smithi093 ceph-mon[111335]: pgmap v3246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/984813285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:45 smithi093 ceph-mon[118902]: pgmap v3246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/984813285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:45.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:45.245618+0000) 2023-08-27T22:41:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:44 smithi136 ceph-mon[113531]: pgmap v3246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/984813285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:41:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:41:45] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T22:41:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:46.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:46.245816+0000) 2023-08-27T22:41:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:47.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:47.246093+0000) 2023-08-27T22:41:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:47 smithi136 ceph-mon[113531]: pgmap v3247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340889949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:47 smithi093 ceph-mon[111335]: pgmap v3247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340889949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:47 smithi093 ceph-mon[118902]: pgmap v3247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340889949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:48.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:48.246313+0000) 2023-08-27T22:41:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:49.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:49.246501+0000) 2023-08-27T22:41:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:49 smithi136 ceph-mon[113531]: pgmap v3248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1285565550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:49 smithi093 ceph-mon[111335]: pgmap v3248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1285565550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:49 smithi093 ceph-mon[118902]: pgmap v3248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1285565550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:50.596 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:50.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:50.246760+0000) 2023-08-27T22:41:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:50 smithi136 ceph-mon[113531]: pgmap v3249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:50 smithi093 ceph-mon[111335]: pgmap v3249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:50 smithi093 ceph-mon[118902]: pgmap v3249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:50.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:50.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:51.598 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:51.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:51.247097+0000) 2023-08-27T22:41:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2155388685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2155388685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2155388685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:52.060Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:52.061Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:52.602 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:52.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:52.247389+0000) 2023-08-27T22:41:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:52 smithi136 ceph-mon[113531]: pgmap v3250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:52 smithi093 ceph-mon[111335]: pgmap v3250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:52 smithi093 ceph-mon[118902]: pgmap v3250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:53.604 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:53.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:53.247597+0000) 2023-08-27T22:41:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1455665379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:53.974 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1455665379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:53.975 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1455665379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:53.974Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:41:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:41:53.976Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:41:54.608 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:54.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:54.247793+0000) 2023-08-27T22:41:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:54 smithi136 ceph-mon[113531]: pgmap v3251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:54 smithi093 ceph-mon[111335]: pgmap v3251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:41:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:54 smithi093 ceph-mon[118902]: pgmap v3251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:55.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:55.248005+0000) 2023-08-27T22:41:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:41:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:41:55] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:41:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2415339856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:56.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:56.248191+0000) 2023-08-27T22:41:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2415339856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2415339856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:57 smithi136 ceph-mon[113531]: pgmap v3252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:57.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:57.248411+0000) 2023-08-27T22:41:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:57 smithi093 ceph-mon[111335]: pgmap v3252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:57 smithi093 ceph-mon[118902]: pgmap v3252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:58.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:58.248639+0000) 2023-08-27T22:41:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:59 smithi136 ceph-mon[113531]: pgmap v3253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:41:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/600983943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:41:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:41:59.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:41:59.248909+0000) 2023-08-27T22:41:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:59 smithi093 ceph-mon[111335]: pgmap v3253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:41:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/600983943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:41:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:59 smithi093 ceph-mon[118902]: pgmap v3253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:41:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:41:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/600983943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:00.596 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:00.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:00.249154+0000) 2023-08-27T22:42:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:00 smithi136 ceph-mon[113531]: pgmap v3254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:00 smithi093 ceph-mon[111335]: pgmap v3254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:00 smithi093 ceph-mon[118902]: pgmap v3254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:00.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:00.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:01.598 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:01.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:01.249305+0000) 2023-08-27T22:42:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3137397030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3137397030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3137397030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:02.060Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:02.061Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:02.602 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:02.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:02.249524+0000) 2023-08-27T22:42:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:02 smithi136 ceph-mon[113531]: pgmap v3255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:02 smithi093 ceph-mon[111335]: pgmap v3255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:02 smithi093 ceph-mon[118902]: pgmap v3255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:03.604 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:03.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:03.249728+0000) 2023-08-27T22:42:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3812294583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:03.975 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3812294583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:03.975 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3812294583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:03.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:03.976Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:04.607 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:04.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:04.249968+0000) 2023-08-27T22:42:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:04 smithi136 ceph-mon[113531]: pgmap v3256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:04 smithi093 ceph-mon[111335]: pgmap v3256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:04 smithi093 ceph-mon[118902]: pgmap v3256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:05.609 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:05.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:05.250233+0000) 2023-08-27T22:42:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/155778404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/155778404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:42:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:42:05] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:42:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/155778404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:06.614 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:06.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:06.250486+0000) 2023-08-27T22:42:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:06 smithi136 ceph-mon[113531]: pgmap v3257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:06 smithi093 ceph-mon[111335]: pgmap v3257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:06 smithi093 ceph-mon[118902]: pgmap v3257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:07.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:07.250721+0000) 2023-08-27T22:42:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/839973220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/839973220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/839973220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:08.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:08.250972+0000) 2023-08-27T22:42:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:08 smithi136 ceph-mon[113531]: pgmap v3258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:08 smithi093 ceph-mon[111335]: pgmap v3258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:08 smithi093 ceph-mon[118902]: pgmap v3258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:09.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:09.251113+0000) 2023-08-27T22:42:10.599 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:10.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:10.251296+0000) 2023-08-27T22:42:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:10 smithi136 ceph-mon[113531]: pgmap v3259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4051083435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:10 smithi093 ceph-mon[111335]: pgmap v3259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4051083435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:10 smithi093 ceph-mon[118902]: pgmap v3259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4051083435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:10.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:10.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:11.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:11.251519+0000) 2023-08-27T22:42:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:12.061Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:12.062Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:12.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:12.251742+0000) 2023-08-27T22:42:13.252 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:12 smithi136 ceph-mon[113531]: pgmap v3260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:13.252 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3622112945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:12 smithi093 ceph-mon[111335]: pgmap v3260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3622112945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:12 smithi093 ceph-mon[118902]: pgmap v3260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3622112945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:13.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:13.252002+0000) 2023-08-27T22:42:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:13.976Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:13.976Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:14.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:14.252317+0000) 2023-08-27T22:42:15.252 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:14 smithi136 ceph-mon[113531]: pgmap v3261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:14 smithi093 ceph-mon[118902]: pgmap v3261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:14 smithi093 ceph-mon[111335]: pgmap v3261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:15.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:15.252588+0000) 2023-08-27T22:42:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:42:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:42:15] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:42:16.252 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3083190135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3083190135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3083190135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:16.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:16.252852+0000) 2023-08-27T22:42:17.253 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:17 smithi136 ceph-mon[113531]: pgmap v3262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:17 smithi093 ceph-mon[111335]: pgmap v3262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:17 smithi093 ceph-mon[118902]: pgmap v3262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:17.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:17.253043+0000) 2023-08-27T22:42:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4020195888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:18.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:18.253237+0000) 2023-08-27T22:42:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4020195888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4020195888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:19.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:19 smithi136 ceph-mon[113531]: pgmap v3263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:19.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:19.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:19.253472+0000) 2023-08-27T22:42:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:19 smithi093 ceph-mon[111335]: pgmap v3263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:19 smithi093 ceph-mon[118902]: pgmap v3263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:20.599 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:20.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:20.253751+0000) 2023-08-27T22:42:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2385755932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:20 smithi136 ceph-mon[113531]: pgmap v3264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2385755932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:20 smithi093 ceph-mon[111335]: pgmap v3264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2385755932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:20 smithi093 ceph-mon[118902]: pgmap v3264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:20.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:20.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:21.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:21.253994+0000) 2023-08-27T22:42:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:22.061Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:22.062Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:22.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:22.254223+0000) 2023-08-27T22:42:23.254 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:22 smithi136 ceph-mon[113531]: pgmap v3265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:23.254 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3000590752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:22 smithi093 ceph-mon[111335]: pgmap v3265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3000590752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:22 smithi093 ceph-mon[118902]: pgmap v3265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3000590752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:23.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:23.254493+0000) 2023-08-27T22:42:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:23.976Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:23.977Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:24.254 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:24.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:24.254717+0000) 2023-08-27T22:42:25.254 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:25 smithi136 ceph-mon[113531]: pgmap v3266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:25.255 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/65854476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:25 smithi093 ceph-mon[111335]: pgmap v3266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/65854476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:25 smithi093 ceph-mon[118902]: pgmap v3266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/65854476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:25.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:25.254924+0000) 2023-08-27T22:42:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:42:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:42:25] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:42:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:26.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:26.255161+0000) 2023-08-27T22:42:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:27 smithi136 ceph-mon[113531]: pgmap v3267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2353532866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:27.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:27.255463+0000) 2023-08-27T22:42:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:27 smithi093 ceph-mon[111335]: pgmap v3267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2353532866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:27 smithi093 ceph-mon[118902]: pgmap v3267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2353532866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:28.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:28.255687+0000) 2023-08-27T22:42:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:29 smithi136 ceph-mon[113531]: pgmap v3268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:29.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:29.255958+0000) 2023-08-27T22:42:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:29 smithi093 ceph-mon[111335]: pgmap v3268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:29 smithi093 ceph-mon[118902]: pgmap v3268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3285161446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:30.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:30.256218+0000) 2023-08-27T22:42:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3285161446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3285161446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:30.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:30.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:31 smithi136 ceph-mon[113531]: pgmap v3269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:31.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:31.256477+0000) 2023-08-27T22:42:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:31 smithi093 ceph-mon[111335]: pgmap v3269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:31 smithi093 ceph-mon[118902]: pgmap v3269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1442396118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:32.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:32.256671+0000) 2023-08-27T22:42:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1442396118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1442396118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:32.061Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:32.062Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:33 smithi136 ceph-mon[113531]: pgmap v3270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:33.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:33.256912+0000) 2023-08-27T22:42:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:33 smithi093 ceph-mon[111335]: pgmap v3270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:33 smithi093 ceph-mon[118902]: pgmap v3270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:33.977Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:33.978Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:34.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:34.257165+0000) 2023-08-27T22:42:35.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:35 smithi093 ceph-mon[111335]: pgmap v3271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3233658516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:42:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:35 smithi093 ceph-mon[118902]: pgmap v3271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3233658516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:42:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:35 smithi136 ceph-mon[113531]: pgmap v3271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3233658516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:42:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:35.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:35.257334+0000) 2023-08-27T22:42:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:42:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:42:35] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:42:36.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:36.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:36.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:36 smithi093 ceph-mon[118902]: pgmap v3272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:36 smithi093 ceph-mon[111335]: pgmap v3272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:36 smithi136 ceph-mon[113531]: pgmap v3272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:36.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:36.257478+0000) 2023-08-27T22:42:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/106969100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/106969100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/106969100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:37.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:37.257670+0000) 2023-08-27T22:42:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:42:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:42:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[111335]: pgmap v3273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:42:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:42:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:38 smithi093 ceph-mon[118902]: pgmap v3273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:38.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:38.257786+0000) 2023-08-27T22:42:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:42:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:42:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:42:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:38 smithi136 ceph-mon[113531]: pgmap v3273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:39.579 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:39.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:39.258027+0000) 2023-08-27T22:42:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3698705414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3698705414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3698705414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:40.603 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:40.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:40.258300+0000) 2023-08-27T22:42:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:40 smithi136 ceph-mon[113531]: pgmap v3274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:40 smithi093 ceph-mon[111335]: pgmap v3274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:40 smithi093 ceph-mon[118902]: pgmap v3274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:40.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:40.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:41.604 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:41.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:41.258527+0000) 2023-08-27T22:42:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4249470779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4249470779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4249470779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:42.062Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:42.063Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:42.609 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:42.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:42.258801+0000) 2023-08-27T22:42:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:42 smithi136 ceph-mon[113531]: pgmap v3275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:42 smithi093 ceph-mon[111335]: pgmap v3275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:42 smithi093 ceph-mon[118902]: pgmap v3275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:43.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:43.259069+0000) 2023-08-27T22:42:43.978 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4080623984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:43.978 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4080623984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4080623984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:43.978Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:43.979Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:44.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:44.259331+0000) 2023-08-27T22:42:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:44 smithi093 ceph-mon[111335]: pgmap v3276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:44 smithi093 ceph-mon[118902]: pgmap v3276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:44 smithi136 ceph-mon[113531]: pgmap v3276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:45.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:45.259597+0000) 2023-08-27T22:42:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:42:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:42:45] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:42:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:46.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:46.259861+0000) 2023-08-27T22:42:47.260 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:46 smithi136 ceph-mon[113531]: pgmap v3277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:47.260 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1068332187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:46 smithi093 ceph-mon[111335]: pgmap v3277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1068332187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:46 smithi093 ceph-mon[118902]: pgmap v3277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1068332187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:47.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:47.260044+0000) 2023-08-27T22:42:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:48.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:48.260280+0000) 2023-08-27T22:42:49.260 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:49 smithi136 ceph-mon[113531]: pgmap v3278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:49.260 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3931077024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:49 smithi093 ceph-mon[111335]: pgmap v3278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3931077024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:49 smithi093 ceph-mon[118902]: pgmap v3278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3931077024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:49.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:49.260540+0000) 2023-08-27T22:42:50.607 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:50.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:50.260786+0000) 2023-08-27T22:42:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:50 smithi136 ceph-mon[113531]: pgmap v3279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:50 smithi093 ceph-mon[111335]: pgmap v3279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:50 smithi093 ceph-mon[118902]: pgmap v3279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:50.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:50.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:51.608 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:51.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:51.261060+0000) 2023-08-27T22:42:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/37301792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/37301792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/37301792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:52.063Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:52.064Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:52.614 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:52.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:52.261309+0000) 2023-08-27T22:42:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:52 smithi136 ceph-mon[113531]: pgmap v3280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:52 smithi093 ceph-mon[111335]: pgmap v3280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:52 smithi093 ceph-mon[118902]: pgmap v3280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:53.617 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:53.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:53.261568+0000) 2023-08-27T22:42:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/109969393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:53.979 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/109969393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:53.979 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/109969393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:53.979Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:42:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:42:53.980Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:42:54.624 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:54.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:54.261819+0000) 2023-08-27T22:42:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:54 smithi136 ceph-mon[113531]: pgmap v3281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:54 smithi093 ceph-mon[111335]: pgmap v3281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:42:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:54 smithi093 ceph-mon[118902]: pgmap v3281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:55.625 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:42:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:42:55] "GET /metrics HTTP/1.1" 200 239907 "" "Prometheus/2.43.0" 2023-08-27T22:42:55.625 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:55.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:55.262127+0000) 2023-08-27T22:42:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1846055479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1846055479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1846055479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:56.632 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:56.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:56.262382+0000) 2023-08-27T22:42:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:56 smithi136 ceph-mon[113531]: pgmap v3282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:56 smithi093 ceph-mon[111335]: pgmap v3282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:56 smithi093 ceph-mon[118902]: pgmap v3282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:57.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:57.262610+0000) 2023-08-27T22:42:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:58.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:58.262820+0000) 2023-08-27T22:42:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3720184123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3720184123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3720184123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:42:59.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:42:59 smithi136 ceph-mon[113531]: pgmap v3283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:59.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:42:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:42:59.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:42:59.263086+0000) 2023-08-27T22:42:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:42:59 smithi093 ceph-mon[111335]: pgmap v3283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:42:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:42:59 smithi093 ceph-mon[118902]: pgmap v3283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:00.607 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:00.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:00.263377+0000) 2023-08-27T22:43:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:00 smithi136 ceph-mon[113531]: pgmap v3284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2952577158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:00 smithi093 ceph-mon[111335]: pgmap v3284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2952577158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:00 smithi093 ceph-mon[118902]: pgmap v3284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2952577158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:00.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:00.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:01.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:01.263631+0000) 2023-08-27T22:43:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:02.063Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:02.064Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:02.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:02.263862+0000) 2023-08-27T22:43:03.264 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:03 smithi136 ceph-mon[113531]: pgmap v3285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:03.264 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2981825042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:03 smithi093 ceph-mon[111335]: pgmap v3285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2981825042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:03 smithi093 ceph-mon[118902]: pgmap v3285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2981825042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:03.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:03.264145+0000) 2023-08-27T22:43:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:03.980Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:03.981Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:04.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:04.264382+0000) 2023-08-27T22:43:05.264 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:05 smithi136 ceph-mon[113531]: pgmap v3286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:05.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:05 smithi093 ceph-mon[111335]: pgmap v3286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:05 smithi093 ceph-mon[118902]: pgmap v3286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:05.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:05.264643+0000) 2023-08-27T22:43:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:43:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:43:05] "GET /metrics HTTP/1.1" 200 239907 "" "Prometheus/2.43.0" 2023-08-27T22:43:06.265 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1805143866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1805143866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1805143866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:06.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:06.264876+0000) 2023-08-27T22:43:07.265 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:07 smithi136 ceph-mon[113531]: pgmap v3287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:07 smithi093 ceph-mon[118902]: pgmap v3287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:07 smithi093 ceph-mon[111335]: pgmap v3287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:07.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:07.265145+0000) 2023-08-27T22:43:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3064034208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:08.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:08.265345+0000) 2023-08-27T22:43:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3064034208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3064034208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:09 smithi136 ceph-mon[113531]: pgmap v3288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:09.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:09.265600+0000) 2023-08-27T22:43:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:09 smithi093 ceph-mon[111335]: pgmap v3288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:09 smithi093 ceph-mon[118902]: pgmap v3288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:10.611 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:10.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:10.265843+0000) 2023-08-27T22:43:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1451194329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:10 smithi136 ceph-mon[113531]: pgmap v3289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1451194329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:10 smithi093 ceph-mon[111335]: pgmap v3289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1451194329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:10 smithi093 ceph-mon[118902]: pgmap v3289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:10.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:10.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:11.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:11.266080+0000) 2023-08-27T22:43:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:12.064Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:12.065Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:12.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:12.266333+0000) 2023-08-27T22:43:13.266 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:13 smithi136 ceph-mon[113531]: pgmap v3290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:13.266 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2080219284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:13 smithi093 ceph-mon[111335]: pgmap v3290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2080219284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:13 smithi093 ceph-mon[118902]: pgmap v3290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2080219284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:13.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:13.266553+0000) 2023-08-27T22:43:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:13.981Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:13.982Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:14.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:14.266820+0000) 2023-08-27T22:43:15.267 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:15 smithi136 ceph-mon[113531]: pgmap v3291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:15.267 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/89825487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:15 smithi093 ceph-mon[111335]: pgmap v3291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/89825487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:15 smithi093 ceph-mon[118902]: pgmap v3291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/89825487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:15.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:15.267107+0000) 2023-08-27T22:43:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:43:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:43:15] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:43:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:16.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:16.267359+0000) 2023-08-27T22:43:17.267 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:17 smithi136 ceph-mon[113531]: pgmap v3292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:17 smithi093 ceph-mon[111335]: pgmap v3292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:17 smithi093 ceph-mon[118902]: pgmap v3292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:17.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:17.267589+0000) 2023-08-27T22:43:18.268 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1508375413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1508375413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1508375413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:18.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:18.267868+0000) 2023-08-27T22:43:19.268 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:19 smithi136 ceph-mon[113531]: pgmap v3293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:19 smithi093 ceph-mon[111335]: pgmap v3293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:19 smithi093 ceph-mon[118902]: pgmap v3293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:19.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:19.268145+0000) 2023-08-27T22:43:20.268 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/300340126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/300340126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/300340126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:20.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:20.268372+0000) 2023-08-27T22:43:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:20.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:20.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:21.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:21.268669+0000) 2023-08-27T22:43:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:21 smithi136 ceph-mon[113531]: pgmap v3294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:21 smithi093 ceph-mon[111335]: pgmap v3294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:21 smithi093 ceph-mon[118902]: pgmap v3294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1495868415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:22.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:22.268913+0000) 2023-08-27T22:43:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1495868415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1495868415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:22.065Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:22.066Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:23 smithi136 ceph-mon[113531]: pgmap v3295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:23.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:23.269164+0000) 2023-08-27T22:43:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:23 smithi093 ceph-mon[111335]: pgmap v3295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:23 smithi093 ceph-mon[118902]: pgmap v3295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:23.981Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:23.982Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:24.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:24.269388+0000) 2023-08-27T22:43:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:25 smithi093 ceph-mon[118902]: pgmap v3296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3875171322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:25 smithi093 ceph-mon[111335]: pgmap v3296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3875171322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:25 smithi136 ceph-mon[113531]: pgmap v3296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3875171322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:25.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:25.269612+0000) 2023-08-27T22:43:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:43:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:43:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T22:43:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:26.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:26.269867+0000) 2023-08-27T22:43:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:27 smithi136 ceph-mon[113531]: pgmap v3297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/657302902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:27.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:27.270150+0000) 2023-08-27T22:43:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:27 smithi093 ceph-mon[111335]: pgmap v3297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/657302902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:27 smithi093 ceph-mon[118902]: pgmap v3297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/657302902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:28.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:28.270355+0000) 2023-08-27T22:43:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:29 smithi136 ceph-mon[113531]: pgmap v3298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3587226741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:29.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:29.270611+0000) 2023-08-27T22:43:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:29 smithi093 ceph-mon[111335]: pgmap v3298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3587226741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:29 smithi093 ceph-mon[118902]: pgmap v3298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3587226741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:30.611 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:30.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:30.270876+0000) 2023-08-27T22:43:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:30 smithi136 ceph-mon[113531]: pgmap v3299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:30 smithi093 ceph-mon[111335]: pgmap v3299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:30 smithi093 ceph-mon[118902]: pgmap v3299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:30.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:30.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:31.612 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:31.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:31.271152+0000) 2023-08-27T22:43:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3712694333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3712694333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3712694333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:32.066Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:32.066Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:32.617 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:32.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:32.271422+0000) 2023-08-27T22:43:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:32 smithi136 ceph-mon[113531]: pgmap v3300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:32 smithi093 ceph-mon[111335]: pgmap v3300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:32 smithi093 ceph-mon[118902]: pgmap v3300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:33.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:33.271715+0000) 2023-08-27T22:43:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4122621334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4122621334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4122621334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:33.982Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:34.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:33.983Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:34.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:34.271920+0000) 2023-08-27T22:43:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:34 smithi136 ceph-mon[113531]: pgmap v3301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:34 smithi093 ceph-mon[111335]: pgmap v3301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:34 smithi093 ceph-mon[118902]: pgmap v3301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:35.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:35.272159+0000) 2023-08-27T22:43:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:43:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:43:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T22:43:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:36.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:36.272412+0000) 2023-08-27T22:43:37.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:37 smithi136 ceph-mon[113531]: pgmap v3302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:37.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3655289103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:37 smithi093 ceph-mon[111335]: pgmap v3302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3655289103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:37 smithi093 ceph-mon[118902]: pgmap v3302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3655289103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:37.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:37.272656+0000) 2023-08-27T22:43:38.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:43:38.331 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:43:38.331 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:43:38.606 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:38.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:38.272830+0000) 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[111335]: pgmap v3303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/727401618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[118902]: pgmap v3303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/727401618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:39.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:39.272974+0000) 2023-08-27T22:43:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:39 smithi136 ceph-mon[113531]: pgmap v3303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/727401618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:40.540 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:40.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:40.273150+0000) 2023-08-27T22:43:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:40 smithi136 ceph-mon[113531]: pgmap v3304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[111335]: pgmap v3304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[118902]: pgmap v3304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:40.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:40.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:40.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:40.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:40.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:41.611 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:41.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:41.273380+0000) 2023-08-27T22:43:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:43:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:43:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3992423664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:43:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:43:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3992423664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:43:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:43:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:43:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3992423664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:42.066Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:42.067Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:42.618 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:42.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:42.273630+0000) 2023-08-27T22:43:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:42 smithi136 ceph-mon[113531]: pgmap v3305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:42 smithi093 ceph-mon[111335]: pgmap v3305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:42 smithi093 ceph-mon[118902]: pgmap v3305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:43.619 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:43.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:43.273870+0000) 2023-08-27T22:43:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3054528140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:43.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3054528140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:43.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3054528140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:43.983Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:43.984Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:44.628 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:44.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:44.274090+0000) 2023-08-27T22:43:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:44 smithi136 ceph-mon[113531]: pgmap v3306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:44 smithi093 ceph-mon[111335]: pgmap v3306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:44 smithi093 ceph-mon[118902]: pgmap v3306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:45.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:45.274358+0000) 2023-08-27T22:43:45.705 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:43:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:43:45] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:43:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2897677838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2897677838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2897677838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:46.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:46.274617+0000) 2023-08-27T22:43:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:46 smithi093 ceph-mon[111335]: pgmap v3307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:46 smithi093 ceph-mon[118902]: pgmap v3307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:46 smithi136 ceph-mon[113531]: pgmap v3307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:47.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:47.274834+0000) 2023-08-27T22:43:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:48.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:48.275108+0000) 2023-08-27T22:43:49.275 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:49 smithi136 ceph-mon[113531]: pgmap v3308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:49.275 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1888620478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:49 smithi093 ceph-mon[111335]: pgmap v3308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1888620478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:49 smithi093 ceph-mon[118902]: pgmap v3308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1888620478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:49.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:49.275378+0000) 2023-08-27T22:43:50.614 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:50.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:50.275634+0000) 2023-08-27T22:43:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:50 smithi136 ceph-mon[113531]: pgmap v3309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1385952698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:50 smithi093 ceph-mon[118902]: pgmap v3309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1385952698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:50 smithi093 ceph-mon[111335]: pgmap v3309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1385952698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:50.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:50.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:51.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:51.275944+0000) 2023-08-27T22:43:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:52.067Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:52.068Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:52.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:52.276188+0000) 2023-08-27T22:43:53.276 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:53 smithi136 ceph-mon[113531]: pgmap v3310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:53.276 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3403725362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:53 smithi093 ceph-mon[111335]: pgmap v3310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3403725362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:53 smithi093 ceph-mon[118902]: pgmap v3310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3403725362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:53.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:53.276443+0000) 2023-08-27T22:43:54.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:54.243 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:53.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:54.243 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:43:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:43:53.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:43:54.276 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:43:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:54.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:54.276697+0000) 2023-08-27T22:43:55.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:55 smithi136 ceph-mon[113531]: pgmap v3311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:55 smithi093 ceph-mon[111335]: pgmap v3311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:55 smithi093 ceph-mon[118902]: pgmap v3311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:55.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:55.276962+0000) 2023-08-27T22:43:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:43:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:43:55] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:43:56.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2302892247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2302892247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2302892247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:56.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:56.277196+0000) 2023-08-27T22:43:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:57 smithi136 ceph-mon[113531]: pgmap v3312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:57.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:57.277425+0000) 2023-08-27T22:43:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:57 smithi093 ceph-mon[111335]: pgmap v3312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:57 smithi093 ceph-mon[118902]: pgmap v3312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4030604990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:58.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:58.277712+0000) 2023-08-27T22:43:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4030604990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4030604990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:43:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:43:59 smithi136 ceph-mon[113531]: pgmap v3313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:43:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:43:59.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:43:59.277964+0000) 2023-08-27T22:43:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:43:59 smithi093 ceph-mon[111335]: pgmap v3313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:43:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:43:59 smithi093 ceph-mon[118902]: pgmap v3313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:00.619 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:00.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:00.278166+0000) 2023-08-27T22:44:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1608982094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:00 smithi136 ceph-mon[113531]: pgmap v3314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1608982094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:00 smithi093 ceph-mon[111335]: pgmap v3314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1608982094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:00 smithi093 ceph-mon[118902]: pgmap v3314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:00.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:00.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:01.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:01.278419+0000) 2023-08-27T22:44:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:02.068Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:02.069Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:02.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:02.278673+0000) 2023-08-27T22:44:03.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:03 smithi136 ceph-mon[113531]: pgmap v3315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:03.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/113084986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:03 smithi093 ceph-mon[111335]: pgmap v3315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/113084986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:03 smithi093 ceph-mon[118902]: pgmap v3315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/113084986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:03.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:03.278955+0000) 2023-08-27T22:44:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:03.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:03.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:04.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:04.279229+0000) 2023-08-27T22:44:05.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:05 smithi136 ceph-mon[113531]: pgmap v3316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:05.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2858018480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:05 smithi093 ceph-mon[118902]: pgmap v3316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2858018480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:05 smithi093 ceph-mon[111335]: pgmap v3316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2858018480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:05.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:05.279469+0000) 2023-08-27T22:44:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:44:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:44:05] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:44:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:06.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:06.279729+0000) 2023-08-27T22:44:07.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:07 smithi136 ceph-mon[113531]: pgmap v3317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:07 smithi093 ceph-mon[111335]: pgmap v3317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:07 smithi093 ceph-mon[118902]: pgmap v3317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:07.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:07.279970+0000) 2023-08-27T22:44:08.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3833936937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3833936937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3833936937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:08.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:08.280222+0000) 2023-08-27T22:44:09.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:09 smithi136 ceph-mon[113531]: pgmap v3318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:09.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:09 smithi093 ceph-mon[111335]: pgmap v3318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:09 smithi093 ceph-mon[118902]: pgmap v3318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:09.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:09.280439+0000) 2023-08-27T22:44:10.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2353147996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2353147996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2353147996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:10.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:10.280715+0000) 2023-08-27T22:44:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:10.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:10.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:11 smithi136 ceph-mon[113531]: pgmap v3319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:11.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:11.280964+0000) 2023-08-27T22:44:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:11 smithi093 ceph-mon[111335]: pgmap v3319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:11 smithi093 ceph-mon[118902]: pgmap v3319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2127633703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:12.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:12.281167+0000) 2023-08-27T22:44:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2127633703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2127633703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:12.068Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:12.069Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:13 smithi136 ceph-mon[113531]: pgmap v3320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:13.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:13.281369+0000) 2023-08-27T22:44:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:13 smithi093 ceph-mon[111335]: pgmap v3320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:13 smithi093 ceph-mon[118902]: pgmap v3320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:13.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:13.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:14.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:14.281624+0000) 2023-08-27T22:44:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:15 smithi093 ceph-mon[118902]: pgmap v3321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1456064417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:15 smithi093 ceph-mon[111335]: pgmap v3321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1456064417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:15 smithi136 ceph-mon[113531]: pgmap v3321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1456064417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:15.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:15.281903+0000) 2023-08-27T22:44:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:44:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:44:15] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:44:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:16.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:16.282158+0000) 2023-08-27T22:44:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:17 smithi136 ceph-mon[113531]: pgmap v3322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3375319186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:17.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:17.282374+0000) 2023-08-27T22:44:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:17 smithi093 ceph-mon[111335]: pgmap v3322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3375319186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:17 smithi093 ceph-mon[118902]: pgmap v3322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3375319186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:18.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:18.282578+0000) 2023-08-27T22:44:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:19 smithi136 ceph-mon[113531]: pgmap v3323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3928656655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:19.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:19.282828+0000) 2023-08-27T22:44:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:19 smithi093 ceph-mon[111335]: pgmap v3323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3928656655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:19 smithi093 ceph-mon[118902]: pgmap v3323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:19.601 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3928656655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:20.622 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:20.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:20.283093+0000) 2023-08-27T22:44:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:20 smithi136 ceph-mon[113531]: pgmap v3324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:20 smithi093 ceph-mon[111335]: pgmap v3324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:20 smithi093 ceph-mon[118902]: pgmap v3324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:20.563Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:20.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:21.623 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:21.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:21.283352+0000) 2023-08-27T22:44:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3879444304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3879444304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3879444304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:22.069Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:22.070Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:22.628 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:22.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:22.283569+0000) 2023-08-27T22:44:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:22 smithi136 ceph-mon[113531]: pgmap v3325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:22 smithi093 ceph-mon[111335]: pgmap v3325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:22 smithi093 ceph-mon[118902]: pgmap v3325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:23.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:23.283827+0000) 2023-08-27T22:44:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:23.985Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:24.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:23.986Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:24.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:24.284093+0000) 2023-08-27T22:44:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/96057437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:24 smithi136 ceph-mon[113531]: pgmap v3326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/96057437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:24 smithi093 ceph-mon[118902]: pgmap v3326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/96057437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:24 smithi093 ceph-mon[111335]: pgmap v3326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:25.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:25.284357+0000) 2023-08-27T22:44:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:44:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:44:25] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T22:44:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:26.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:26.284610+0000) 2023-08-27T22:44:27.285 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:27 smithi136 ceph-mon[113531]: pgmap v3327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:27.285 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1412171957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:27 smithi093 ceph-mon[111335]: pgmap v3327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1412171957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:27 smithi093 ceph-mon[118902]: pgmap v3327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1412171957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:27.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:27.284844+0000) 2023-08-27T22:44:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:28.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:28.285125+0000) 2023-08-27T22:44:29.285 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:29 smithi136 ceph-mon[113531]: pgmap v3328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:29.285 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2772235802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:29 smithi093 ceph-mon[111335]: pgmap v3328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2772235802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:29 smithi093 ceph-mon[118902]: pgmap v3328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2772235802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:29.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:29.285410+0000) 2023-08-27T22:44:30.621 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:30.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:30.285586+0000) 2023-08-27T22:44:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:30 smithi136 ceph-mon[113531]: pgmap v3329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:30 smithi093 ceph-mon[111335]: pgmap v3329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:30 smithi093 ceph-mon[118902]: pgmap v3329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:30.564Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:30.565Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:31.622 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:31.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:31.285840+0000) 2023-08-27T22:44:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3494874633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:31.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3494874633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3494874633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:32.069Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:32.070Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:32.628 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:32.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:32.286102+0000) 2023-08-27T22:44:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:32 smithi136 ceph-mon[113531]: pgmap v3330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:32 smithi093 ceph-mon[111335]: pgmap v3330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:32 smithi093 ceph-mon[118902]: pgmap v3330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:33.628 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:33.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:33.286357+0000) 2023-08-27T22:44:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2498030378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:33.986 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2498030378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:33.986 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2498030378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:33.986Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:34.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:33.987Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:34.636 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:34.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:34.286660+0000) 2023-08-27T22:44:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:34 smithi136 ceph-mon[113531]: pgmap v3331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:34 smithi093 ceph-mon[111335]: pgmap v3331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:34 smithi093 ceph-mon[118902]: pgmap v3331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:35.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:35.286928+0000) 2023-08-27T22:44:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:44:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:44:35] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T22:44:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1849292721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1849292721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1849292721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:36.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:36.287148+0000) 2023-08-27T22:44:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:36 smithi136 ceph-mon[113531]: pgmap v3332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:36 smithi093 ceph-mon[111335]: pgmap v3332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:36 smithi093 ceph-mon[118902]: pgmap v3332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:37.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:37.287412+0000) 2023-08-27T22:44:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:38.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:38.287658+0000) 2023-08-27T22:44:39.288 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:39 smithi136 ceph-mon[113531]: pgmap v3333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:39.288 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2762117742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:39.288 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:39 smithi093 ceph-mon[111335]: pgmap v3333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2762117742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:39 smithi093 ceph-mon[118902]: pgmap v3333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2762117742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:39.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:39.287911+0000) 2023-08-27T22:44:40.623 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:40.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:40.288120+0000) 2023-08-27T22:44:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:40 smithi136 ceph-mon[113531]: pgmap v3334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1197010742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:40 smithi093 ceph-mon[111335]: pgmap v3334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1197010742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:40 smithi093 ceph-mon[118902]: pgmap v3334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1197010742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:40.565Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:40.566Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:41.624 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:41.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:41.288279+0000) 2023-08-27T22:44:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:44:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:44:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:44:42.375 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:42.070Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:42.375 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:42.071Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:42.629 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:42.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:42.288455+0000) 2023-08-27T22:44:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:42 smithi136 ceph-mon[113531]: pgmap v3335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:42 smithi093 ceph-mon[111335]: pgmap v3335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:42 smithi093 ceph-mon[118902]: pgmap v3335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:43.650 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:43.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:43.288601+0000) 2023-08-27T22:44:43.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1536265539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:43.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1536265539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1536265539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:43.987Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:44.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:43.988Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:44.653 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:44.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:44.288843+0000) 2023-08-27T22:44:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:44:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:44:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:44 smithi136 ceph-mon[113531]: pgmap v3336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:44:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:44:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[111335]: pgmap v3336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:44:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:44:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:44:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:44 smithi093 ceph-mon[118902]: pgmap v3336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:45.682 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:44:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:44:45] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:44:45.682 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:45.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:45.289147+0000) 2023-08-27T22:44:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2860607763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2860607763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2860607763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:46.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:46.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:46.289434+0000) 2023-08-27T22:44:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:46 smithi136 ceph-mon[113531]: pgmap v3337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:46 smithi093 ceph-mon[111335]: pgmap v3337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:46 smithi093 ceph-mon[118902]: pgmap v3337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:47.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:47.289690+0000) 2023-08-27T22:44:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1202399950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1202399950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1202399950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:48.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:48.289991+0000) 2023-08-27T22:44:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:48 smithi093 ceph-mon[111335]: pgmap v3338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:48 smithi093 ceph-mon[118902]: pgmap v3338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:48 smithi136 ceph-mon[113531]: pgmap v3338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:49.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:49.290265+0000) 2023-08-27T22:44:50.625 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:50.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:50.290530+0000) 2023-08-27T22:44:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:50 smithi136 ceph-mon[113531]: pgmap v3339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2582343635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:50 smithi093 ceph-mon[111335]: pgmap v3339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2582343635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:50 smithi093 ceph-mon[118902]: pgmap v3339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2582343635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:50.565Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:50.566Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:51.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:51.290799+0000) 2023-08-27T22:44:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:52.071Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:52.072Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:52.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:52.291075+0000) 2023-08-27T22:44:53.291 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:53 smithi136 ceph-mon[113531]: pgmap v3340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:53.291 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/437920367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:53 smithi093 ceph-mon[111335]: pgmap v3340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/437920367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:53 smithi093 ceph-mon[118902]: pgmap v3340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/437920367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:53.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:53.291353+0000) 2023-08-27T22:44:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:53.988Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:44:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:44:53.989Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:44:54.291 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:44:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:54.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:54.291612+0000) 2023-08-27T22:44:55.292 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:55 smithi136 ceph-mon[113531]: pgmap v3341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:55.292 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2010645233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:55 smithi093 ceph-mon[111335]: pgmap v3341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2010645233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:55 smithi093 ceph-mon[118902]: pgmap v3341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2010645233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:55.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:55.291880+0000) 2023-08-27T22:44:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:44:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:44:55] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:44:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:56.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:56.292158+0000) 2023-08-27T22:44:57.292 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:57 smithi136 ceph-mon[113531]: pgmap v3342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:57 smithi093 ceph-mon[118902]: pgmap v3342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:57 smithi093 ceph-mon[111335]: pgmap v3342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:57.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:57.292464+0000) 2023-08-27T22:44:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/605559920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:58.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:58.292718+0000) 2023-08-27T22:44:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/605559920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/605559920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:44:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:44:59 smithi136 ceph-mon[113531]: pgmap v3343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:44:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:44:59.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:44:59.292982+0000) 2023-08-27T22:44:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:44:59 smithi093 ceph-mon[111335]: pgmap v3343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:44:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:44:59 smithi093 ceph-mon[118902]: pgmap v3343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2651834607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:00.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:00.293182+0000) 2023-08-27T22:45:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2651834607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2651834607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:00.566Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:00.567Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:01 smithi136 ceph-mon[113531]: pgmap v3344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:01.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:01.293432+0000) 2023-08-27T22:45:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:01 smithi093 ceph-mon[111335]: pgmap v3344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:01 smithi093 ceph-mon[118902]: pgmap v3344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1037078564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:02.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:02.293633+0000) 2023-08-27T22:45:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1037078564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1037078564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:02.071Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:02.072Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:03 smithi136 ceph-mon[113531]: pgmap v3345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:03.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:03.293891+0000) 2023-08-27T22:45:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:03 smithi093 ceph-mon[118902]: pgmap v3345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:03 smithi093 ceph-mon[111335]: pgmap v3345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:03.989Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:04.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:03.990Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:04.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:04.294155+0000) 2023-08-27T22:45:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:05 smithi093 ceph-mon[118902]: pgmap v3346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3185414682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:05 smithi093 ceph-mon[111335]: pgmap v3346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3185414682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:05 smithi136 ceph-mon[113531]: pgmap v3346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3185414682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:05.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:05.294376+0000) 2023-08-27T22:45:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:45:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:45:05] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:45:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:06.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:06.294626+0000) 2023-08-27T22:45:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:07 smithi136 ceph-mon[113531]: pgmap v3347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3014877502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:07.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:07.294898+0000) 2023-08-27T22:45:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:07 smithi093 ceph-mon[111335]: pgmap v3347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3014877502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:07 smithi093 ceph-mon[118902]: pgmap v3347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3014877502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:08.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:08.295129+0000) 2023-08-27T22:45:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:09 smithi136 ceph-mon[113531]: pgmap v3348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:09.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:09.295347+0000) 2023-08-27T22:45:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:09 smithi093 ceph-mon[111335]: pgmap v3348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:09 smithi093 ceph-mon[118902]: pgmap v3348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3923659522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:10.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:10.295572+0000) 2023-08-27T22:45:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3923659522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3923659522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:10.567Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:10.568Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:11 smithi136 ceph-mon[113531]: pgmap v3349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:11.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:11.295860+0000) 2023-08-27T22:45:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:11 smithi093 ceph-mon[111335]: pgmap v3349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:11 smithi093 ceph-mon[118902]: pgmap v3349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471579146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:12.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:12.296107+0000) 2023-08-27T22:45:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471579146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471579146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:12.072Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:12.073Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:13 smithi136 ceph-mon[113531]: pgmap v3350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:13.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:13.296311+0000) 2023-08-27T22:45:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:13 smithi093 ceph-mon[111335]: pgmap v3350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:13 smithi093 ceph-mon[118902]: pgmap v3350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3982477389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:14.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:14.296569+0000) 2023-08-27T22:45:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3982477389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:14.528 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3982477389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:14.528 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:13.989Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:14.528 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:13.990Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:15 smithi093 ceph-mon[111335]: pgmap v3351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:15 smithi093 ceph-mon[118902]: pgmap v3351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:15 smithi136 ceph-mon[113531]: pgmap v3351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:15.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:15.296807+0000) 2023-08-27T22:45:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:45:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:45:15] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:45:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:16.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:16.297029+0000) 2023-08-27T22:45:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:17 smithi136 ceph-mon[113531]: pgmap v3352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4162455816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:17.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:17.297295+0000) 2023-08-27T22:45:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:17 smithi093 ceph-mon[111335]: pgmap v3352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4162455816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:17 smithi093 ceph-mon[118902]: pgmap v3352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4162455816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:18.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:18.297570+0000) 2023-08-27T22:45:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:19 smithi136 ceph-mon[113531]: pgmap v3353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/191707419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:19.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:19.297825+0000) 2023-08-27T22:45:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:19 smithi093 ceph-mon[111335]: pgmap v3353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:19.710 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/191707419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:19.715 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:19 smithi093 ceph-mon[118902]: pgmap v3353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:19.715 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/191707419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:20.635 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:20.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:20.298092+0000) 2023-08-27T22:45:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:20 smithi136 ceph-mon[113531]: pgmap v3354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:20 smithi093 ceph-mon[111335]: pgmap v3354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:20 smithi093 ceph-mon[118902]: pgmap v3354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:20.568Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:20.569Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:21.637 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:21.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:21.298357+0000) 2023-08-27T22:45:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1001935506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1001935506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1001935506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:22.073Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:22.074Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:22.641 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:22.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:22.298572+0000) 2023-08-27T22:45:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:22 smithi136 ceph-mon[113531]: pgmap v3355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:22.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:22 smithi093 ceph-mon[111335]: pgmap v3355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:22 smithi093 ceph-mon[118902]: pgmap v3355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:23.642 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:23.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:23.298805+0000) 2023-08-27T22:45:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/145516187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/145516187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/145516187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:24.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:23.990Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:24.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:23.991Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:24.648 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:24.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:24.299044+0000) 2023-08-27T22:45:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:24 smithi136 ceph-mon[113531]: pgmap v3356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:24 smithi093 ceph-mon[111335]: pgmap v3356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:24 smithi093 ceph-mon[118902]: pgmap v3356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:25.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:25.299307+0000) 2023-08-27T22:45:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:45:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:45:25] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:45:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/858799372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/858799372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/858799372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:26.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:26.299602+0000) 2023-08-27T22:45:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:26 smithi136 ceph-mon[113531]: pgmap v3357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:26 smithi093 ceph-mon[111335]: pgmap v3357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:26 smithi093 ceph-mon[118902]: pgmap v3357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:27.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:27.299873+0000) 2023-08-27T22:45:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:28.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:28.300150+0000) 2023-08-27T22:45:29.300 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:29 smithi136 ceph-mon[113531]: pgmap v3358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:29.300 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2939747597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:29 smithi093 ceph-mon[111335]: pgmap v3358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2939747597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:29 smithi093 ceph-mon[118902]: pgmap v3358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2939747597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:29.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:29.300375+0000) 2023-08-27T22:45:30.631 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:30.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:30.300599+0000) 2023-08-27T22:45:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:30 smithi136 ceph-mon[113531]: pgmap v3359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3508342320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:30 smithi093 ceph-mon[111335]: pgmap v3359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3508342320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:30 smithi093 ceph-mon[118902]: pgmap v3359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3508342320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:30.568Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:30.569Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:31.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:31.300855+0000) 2023-08-27T22:45:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:32.073Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:32.074Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:32.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:32.301122+0000) 2023-08-27T22:45:33.301 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:33 smithi136 ceph-mon[113531]: pgmap v3360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:33.301 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1104595699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:33 smithi093 ceph-mon[111335]: pgmap v3360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1104595699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:33 smithi093 ceph-mon[118902]: pgmap v3360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1104595699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:33.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:33.301360+0000) 2023-08-27T22:45:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:33.991Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:33.992Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:34.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:34.301625+0000) 2023-08-27T22:45:35.302 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:35 smithi136 ceph-mon[113531]: pgmap v3361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:35 smithi093 ceph-mon[118902]: pgmap v3361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:35 smithi093 ceph-mon[111335]: pgmap v3361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:35.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:35.301906+0000) 2023-08-27T22:45:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:45:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:45:35] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:45:36.302 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3825555006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3825555006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3825555006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:36.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:36.302173+0000) 2023-08-27T22:45:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:37 smithi136 ceph-mon[113531]: pgmap v3362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:37.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:37.302411+0000) 2023-08-27T22:45:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:37 smithi093 ceph-mon[111335]: pgmap v3362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:37 smithi093 ceph-mon[118902]: pgmap v3362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2595118718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:38.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:38.302665+0000) 2023-08-27T22:45:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2595118718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2595118718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:39.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:39.302914+0000) 2023-08-27T22:45:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:39 smithi136 ceph-mon[113531]: pgmap v3363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:39 smithi093 ceph-mon[118902]: pgmap v3363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:39 smithi093 ceph-mon[111335]: pgmap v3363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:40.638 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:40.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:40.303135+0000) 2023-08-27T22:45:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:40 smithi136 ceph-mon[113531]: pgmap v3364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/162844910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:40 smithi093 ceph-mon[111335]: pgmap v3364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/162844910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:40 smithi093 ceph-mon[118902]: pgmap v3364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/162844910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:40.569Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:40.570Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:41.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:41.303388+0000) 2023-08-27T22:45:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:42.073Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:42.074Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:42.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:42.303642+0000) 2023-08-27T22:45:43.304 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:43 smithi136 ceph-mon[113531]: pgmap v3365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:43.304 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1370209980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:43 smithi093 ceph-mon[111335]: pgmap v3365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1370209980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:43 smithi093 ceph-mon[118902]: pgmap v3365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1370209980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:43.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:43.303900+0000) 2023-08-27T22:45:44.304 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:45:44.465 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:45:44.465 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:45:44.465 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:43.992Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:44.465 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:43.993Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:44.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:44.304169+0000) 2023-08-27T22:45:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:45 smithi136 ceph-mon[113531]: pgmap v3366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/311976202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:45 smithi093 ceph-mon[111335]: pgmap v3366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/311976202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:45 smithi093 ceph-mon[118902]: pgmap v3366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/311976202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:45.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:45.304390+0000) 2023-08-27T22:45:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:45:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:45:45] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:45:46.636 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:46.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:46.304604+0000) 2023-08-27T22:45:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:46 smithi136 ceph-mon[113531]: pgmap v3367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[111335]: pgmap v3367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[118902]: pgmap v3367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:47.645 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:47.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:47.304843+0000) 2023-08-27T22:45:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:45:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:45:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3985226606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:45:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:45:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3985226606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:45:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:45:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:45:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3985226606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:48.650 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:48.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:48.305121+0000) 2023-08-27T22:45:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:48 smithi136 ceph-mon[113531]: pgmap v3368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:48 smithi093 ceph-mon[111335]: pgmap v3368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:48 smithi093 ceph-mon[118902]: pgmap v3368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:49.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:49.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:49.305316+0000) 2023-08-27T22:45:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1859957067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1859957067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1859957067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:50.657 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:50.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:50.305555+0000) 2023-08-27T22:45:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:50 smithi136 ceph-mon[113531]: pgmap v3369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:50 smithi093 ceph-mon[111335]: pgmap v3369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:50 smithi093 ceph-mon[118902]: pgmap v3369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:50.570Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:50.571Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:51.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:51.305805+0000) 2023-08-27T22:45:52.306 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3834058921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3834058921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3834058921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:52.074Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:52.075Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:52.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:52.306025+0000) 2023-08-27T22:45:53.306 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:53 smithi136 ceph-mon[113531]: pgmap v3370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:53 smithi093 ceph-mon[111335]: pgmap v3370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:53 smithi093 ceph-mon[118902]: pgmap v3370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:53.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:53.306285+0000) 2023-08-27T22:45:54.306 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:45:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:53.993Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:45:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:45:53.994Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:45:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:54.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:54.306528+0000) 2023-08-27T22:45:55.306 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:55 smithi136 ceph-mon[113531]: pgmap v3371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:55.307 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1889262809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:55 smithi093 ceph-mon[111335]: pgmap v3371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1889262809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:55 smithi093 ceph-mon[118902]: pgmap v3371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1889262809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:55.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:55.306801+0000) 2023-08-27T22:45:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:45:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:45:55] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:45:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:56.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:56.307070+0000) 2023-08-27T22:45:57.307 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:57 smithi136 ceph-mon[113531]: pgmap v3372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:57.307 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2228787004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:57 smithi093 ceph-mon[111335]: pgmap v3372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2228787004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:57 smithi093 ceph-mon[118902]: pgmap v3372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2228787004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:45:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:57.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:57.307340+0000) 2023-08-27T22:45:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:58.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:58.307610+0000) 2023-08-27T22:45:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:45:59 smithi136 ceph-mon[113531]: pgmap v3373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:45:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:45:59.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:45:59.307866+0000) 2023-08-27T22:45:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:45:59 smithi093 ceph-mon[111335]: pgmap v3373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:45:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:45:59 smithi093 ceph-mon[118902]: pgmap v3373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/991662312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:00.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:00.308137+0000) 2023-08-27T22:46:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/991662312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/991662312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:00.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:00.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:01 smithi136 ceph-mon[113531]: pgmap v3374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:01.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:01.308370+0000) 2023-08-27T22:46:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:01 smithi093 ceph-mon[111335]: pgmap v3374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:01 smithi093 ceph-mon[118902]: pgmap v3374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/987113303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:02.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:02.308595+0000) 2023-08-27T22:46:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/987113303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/987113303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:02.075Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:02.076Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:03 smithi136 ceph-mon[113531]: pgmap v3375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:03.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:03.308850+0000) 2023-08-27T22:46:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:03 smithi093 ceph-mon[111335]: pgmap v3375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:03 smithi093 ceph-mon[118902]: pgmap v3375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1512547952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:04.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:04.309125+0000) 2023-08-27T22:46:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1512547952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1512547952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:03.994Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:03.994Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:05 smithi093 ceph-mon[111335]: pgmap v3376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:05 smithi093 ceph-mon[118902]: pgmap v3376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:05 smithi136 ceph-mon[113531]: pgmap v3376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:05.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:05.309393+0000) 2023-08-27T22:46:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:46:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:46:05] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:46:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:06.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:06.309669+0000) 2023-08-27T22:46:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:07 smithi136 ceph-mon[113531]: pgmap v3377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/907789832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:07.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:07.309947+0000) 2023-08-27T22:46:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:07 smithi093 ceph-mon[111335]: pgmap v3377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/907789832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:07 smithi093 ceph-mon[118902]: pgmap v3377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/907789832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:08.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:08.310205+0000) 2023-08-27T22:46:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:09 smithi136 ceph-mon[113531]: pgmap v3378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/933730259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:09.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:09.310439+0000) 2023-08-27T22:46:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:09 smithi093 ceph-mon[111335]: pgmap v3378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/933730259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:09 smithi093 ceph-mon[118902]: pgmap v3378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/933730259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:10.641 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:10.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:10.310687+0000) 2023-08-27T22:46:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:10 smithi136 ceph-mon[113531]: pgmap v3379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:10 smithi093 ceph-mon[111335]: pgmap v3379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:10 smithi093 ceph-mon[118902]: pgmap v3379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:10.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:10.573Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:11.642 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:11.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:11.310963+0000) 2023-08-27T22:46:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2579769925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2579769925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2579769925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:12.075Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:12.076Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:12.646 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:12.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:12.311216+0000) 2023-08-27T22:46:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:12 smithi136 ceph-mon[113531]: pgmap v3380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:12 smithi093 ceph-mon[111335]: pgmap v3380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:12 smithi093 ceph-mon[118902]: pgmap v3380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:13.648 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:13.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:13.311466+0000) 2023-08-27T22:46:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3271855053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3271855053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3271855053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:13.995Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:13.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:14.655 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:14.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:14.311730+0000) 2023-08-27T22:46:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:14 smithi136 ceph-mon[113531]: pgmap v3381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:14 smithi093 ceph-mon[111335]: pgmap v3381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:14 smithi093 ceph-mon[118902]: pgmap v3381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:15.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:15.311972+0000) 2023-08-27T22:46:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:46:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:46:15] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:46:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2411820438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2411820438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2411820438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:16.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:16.312229+0000) 2023-08-27T22:46:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:16 smithi136 ceph-mon[113531]: pgmap v3382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:16 smithi093 ceph-mon[111335]: pgmap v3382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:16 smithi093 ceph-mon[118902]: pgmap v3382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:17.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:17.312492+0000) 2023-08-27T22:46:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:18.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:18.312759+0000) 2023-08-27T22:46:19.313 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:19 smithi136 ceph-mon[113531]: pgmap v3383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:19.313 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3469352543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:19 smithi093 ceph-mon[111335]: pgmap v3383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3469352543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:19 smithi093 ceph-mon[118902]: pgmap v3383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3469352543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:19.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:19.312957+0000) 2023-08-27T22:46:20.647 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:20.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:20.313227+0000) 2023-08-27T22:46:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:20 smithi136 ceph-mon[113531]: pgmap v3384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1869650085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:20 smithi093 ceph-mon[111335]: pgmap v3384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1869650085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:20 smithi093 ceph-mon[118902]: pgmap v3384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1869650085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:20.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:20.573Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:21.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:21.313486+0000) 2023-08-27T22:46:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:22.076Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:22.077Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:22.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:22.313732+0000) 2023-08-27T22:46:23.314 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:23 smithi136 ceph-mon[113531]: pgmap v3385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:23.314 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1235001393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:23 smithi093 ceph-mon[111335]: pgmap v3385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1235001393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:23 smithi093 ceph-mon[118902]: pgmap v3385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1235001393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:23.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:23.313948+0000) 2023-08-27T22:46:24.314 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:23.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:23.997Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:24.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:24.314177+0000) 2023-08-27T22:46:25.314 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:25 smithi136 ceph-mon[113531]: pgmap v3386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:25.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:25 smithi093 ceph-mon[118902]: pgmap v3386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:25 smithi093 ceph-mon[111335]: pgmap v3386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:25.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:25.314460+0000) 2023-08-27T22:46:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:46:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:46:25] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:46:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/995516053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:26.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:26.314735+0000) 2023-08-27T22:46:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/995516053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/995516053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:27 smithi136 ceph-mon[113531]: pgmap v3387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:27.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:27.314992+0000) 2023-08-27T22:46:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:27 smithi093 ceph-mon[111335]: pgmap v3387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:27 smithi093 ceph-mon[118902]: pgmap v3387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/487070006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:28.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:28.315232+0000) 2023-08-27T22:46:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/487070006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/487070006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:29 smithi136 ceph-mon[113531]: pgmap v3388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:29.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:29.315485+0000) 2023-08-27T22:46:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:29 smithi093 ceph-mon[111335]: pgmap v3388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:29 smithi093 ceph-mon[118902]: pgmap v3388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:30.647 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:30.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:30.315741+0000) 2023-08-27T22:46:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:30 smithi136 ceph-mon[113531]: pgmap v3389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1034436900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:30 smithi093 ceph-mon[111335]: pgmap v3389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1034436900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:30 smithi093 ceph-mon[118902]: pgmap v3389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1034436900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:30.573Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:30.574Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:31.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:31.316018+0000) 2023-08-27T22:46:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:32.077Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:32.078Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:32.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:32.316212+0000) 2023-08-27T22:46:33.316 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:33 smithi136 ceph-mon[113531]: pgmap v3390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:33.316 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3967357271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:33 smithi093 ceph-mon[111335]: pgmap v3390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3967357271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:33 smithi093 ceph-mon[118902]: pgmap v3390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3967357271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:33.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:33.316419+0000) 2023-08-27T22:46:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:33.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:33.997Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:34.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:34.316667+0000) 2023-08-27T22:46:35.316 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:35 smithi136 ceph-mon[113531]: pgmap v3391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:35.317 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3178624388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:35 smithi093 ceph-mon[111335]: pgmap v3391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3178624388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:35 smithi093 ceph-mon[118902]: pgmap v3391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3178624388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:35.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:35.316880+0000) 2023-08-27T22:46:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:46:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:46:35] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:46:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:36.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:36.317151+0000) 2023-08-27T22:46:37.317 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:37 smithi136 ceph-mon[113531]: pgmap v3392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:38.070 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:37 smithi093 ceph-mon[111335]: pgmap v3392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:38.070 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:37.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:37.317403+0000) 2023-08-27T22:46:38.070 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:37 smithi093 ceph-mon[118902]: pgmap v3392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1498023346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:38.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:38.317630+0000) 2023-08-27T22:46:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1498023346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:38.591 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1498023346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:39 smithi136 ceph-mon[113531]: pgmap v3393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:39.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:39.317917+0000) 2023-08-27T22:46:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:39 smithi093 ceph-mon[111335]: pgmap v3393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:39 smithi093 ceph-mon[118902]: pgmap v3393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/278236557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:40.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:40.318182+0000) 2023-08-27T22:46:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/278236557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/278236557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:40.574Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:40.575Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:41 smithi136 ceph-mon[113531]: pgmap v3394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:41.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:41.318429+0000) 2023-08-27T22:46:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:41 smithi093 ceph-mon[111335]: pgmap v3394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:41 smithi093 ceph-mon[118902]: pgmap v3394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/474545241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:42.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:42.318678+0000) 2023-08-27T22:46:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/474545241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/474545241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:42.078Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:42.079Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:43 smithi136 ceph-mon[113531]: pgmap v3395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:43.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:43.318910+0000) 2023-08-27T22:46:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:43 smithi093 ceph-mon[111335]: pgmap v3395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:43 smithi093 ceph-mon[118902]: pgmap v3395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:43.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:43.998Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:44.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:44.319177+0000) 2023-08-27T22:46:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:45 smithi093 ceph-mon[111335]: pgmap v3396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/379897006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:45 smithi093 ceph-mon[118902]: pgmap v3396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/379897006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:45 smithi136 ceph-mon[113531]: pgmap v3396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/379897006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:45.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:45.319445+0000) 2023-08-27T22:46:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:46:45] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:46:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:46.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:46.319700+0000) 2023-08-27T22:46:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:47 smithi136 ceph-mon[113531]: pgmap v3397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1402730416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:46:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:47.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:47.319872+0000) 2023-08-27T22:46:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:47 smithi093 ceph-mon[111335]: pgmap v3397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1402730416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:46:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:47 smithi093 ceph-mon[118902]: pgmap v3397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1402730416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:46:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:48.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:48.320029+0000) 2023-08-27T22:46:49.320 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:49 smithi136 ceph-mon[113531]: pgmap v3398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:49.320 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:49.320 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:49 smithi093 ceph-mon[111335]: pgmap v3398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:49 smithi093 ceph-mon[118902]: pgmap v3398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:49.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:49.320179+0000) 2023-08-27T22:46:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:50.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:50.320328+0000) 2023-08-27T22:46:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/364884995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:46:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:46:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/364884995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/364884995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:46:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:46:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:46:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:50.575Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:50.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:51 smithi136 ceph-mon[113531]: pgmap v3399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:51.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:51.320563+0000) 2023-08-27T22:46:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:51 smithi093 ceph-mon[111335]: pgmap v3399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:51 smithi093 ceph-mon[118902]: pgmap v3399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2931534706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:52.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:52.320820+0000) 2023-08-27T22:46:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2931534706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2931534706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:52.079Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:52.080Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:53 smithi136 ceph-mon[113531]: pgmap v3400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:53.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:53.321016+0000) 2023-08-27T22:46:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:53 smithi093 ceph-mon[111335]: pgmap v3400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:53 smithi093 ceph-mon[118902]: pgmap v3400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/263277287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:54.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:54.321292+0000) 2023-08-27T22:46:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/263277287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:46:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/263277287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:53.997Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:46:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:46:53.998Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:46:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:55 smithi093 ceph-mon[111335]: pgmap v3401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:55 smithi093 ceph-mon[118902]: pgmap v3401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:55 smithi136 ceph-mon[113531]: pgmap v3401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:55.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:55.321558+0000) 2023-08-27T22:46:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:46:55] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:46:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:56.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:56.321835+0000) 2023-08-27T22:46:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:57.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:57.322116+0000) 2023-08-27T22:46:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:57 smithi136 ceph-mon[113531]: pgmap v3402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3720226981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:57 smithi093 ceph-mon[111335]: pgmap v3402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3720226981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:57 smithi093 ceph-mon[118902]: pgmap v3402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3720226981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:58.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:58.322348+0000) 2023-08-27T22:46:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:59 smithi136 ceph-mon[113531]: pgmap v3403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:46:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2959136199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:46:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:46:59.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:46:59.322607+0000) 2023-08-27T22:46:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:59 smithi093 ceph-mon[111335]: pgmap v3403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:46:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2959136199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:46:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:59 smithi093 ceph-mon[118902]: pgmap v3403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:46:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:46:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2959136199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:00.651 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:00.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:00.322871+0000) 2023-08-27T22:47:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:00 smithi136 ceph-mon[113531]: pgmap v3404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:00 smithi093 ceph-mon[111335]: pgmap v3404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:00 smithi093 ceph-mon[118902]: pgmap v3404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:00.577Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:00.578Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:01.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:01.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:01.323155+0000) 2023-08-27T22:47:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2042997138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2042997138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2042997138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:02.080Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:02.081Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:02.657 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:02.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:02.323351+0000) 2023-08-27T22:47:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:02 smithi136 ceph-mon[113531]: pgmap v3405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:02 smithi093 ceph-mon[111335]: pgmap v3405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:02 smithi093 ceph-mon[118902]: pgmap v3405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:03.659 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:03.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:03.323621+0000) 2023-08-27T22:47:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2671710339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2671710339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2671710339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:03.998Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:03.999Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:04.663 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:04.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:04.323887+0000) 2023-08-27T22:47:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:04 smithi136 ceph-mon[113531]: pgmap v3406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:04 smithi093 ceph-mon[111335]: pgmap v3406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:04 smithi093 ceph-mon[118902]: pgmap v3406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:05.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:05.324141+0000) 2023-08-27T22:47:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:47:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:47:05] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:47:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/837170454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/837170454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/837170454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:06.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:06.324397+0000) 2023-08-27T22:47:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:06 smithi136 ceph-mon[113531]: pgmap v3407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:06 smithi093 ceph-mon[111335]: pgmap v3407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:06 smithi093 ceph-mon[118902]: pgmap v3407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:07.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:07.324661+0000) 2023-08-27T22:47:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:08.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:08.324888+0000) 2023-08-27T22:47:09.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:09 smithi136 ceph-mon[113531]: pgmap v3408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:09.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/873301130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:09.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:09 smithi093 ceph-mon[118902]: pgmap v3408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/873301130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:09 smithi093 ceph-mon[111335]: pgmap v3408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/873301130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:09.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:09.325171+0000) 2023-08-27T22:47:10.651 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:10.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:10.325412+0000) 2023-08-27T22:47:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:10 smithi136 ceph-mon[113531]: pgmap v3409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1005120897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:10 smithi093 ceph-mon[111335]: pgmap v3409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1005120897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:10 smithi093 ceph-mon[118902]: pgmap v3409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1005120897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:10.578Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:10.580Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:11.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:11.325618+0000) 2023-08-27T22:47:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:12.081Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:12.082Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:12.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:12.325881+0000) 2023-08-27T22:47:13.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:13 smithi136 ceph-mon[113531]: pgmap v3410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:13.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3245993898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:13 smithi093 ceph-mon[111335]: pgmap v3410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3245993898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:13 smithi093 ceph-mon[118902]: pgmap v3410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3245993898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:13.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:13.326157+0000) 2023-08-27T22:47:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:13.999Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:13.999Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:14.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:14.326414+0000) 2023-08-27T22:47:15.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:15 smithi136 ceph-mon[113531]: pgmap v3411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:15 smithi093 ceph-mon[111335]: pgmap v3411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:15 smithi093 ceph-mon[118902]: pgmap v3411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:15.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:15.326664+0000) 2023-08-27T22:47:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:47:15] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:47:16.327 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/178663298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/178663298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/178663298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:16.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:16.326915+0000) 2023-08-27T22:47:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:17 smithi136 ceph-mon[113531]: pgmap v3412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:17.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:17.327158+0000) 2023-08-27T22:47:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:17 smithi093 ceph-mon[111335]: pgmap v3412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:17 smithi093 ceph-mon[118902]: pgmap v3412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4095391840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:18.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:18.327367+0000) 2023-08-27T22:47:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4095391840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4095391840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:19 smithi136 ceph-mon[113531]: pgmap v3413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:19.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:19.327585+0000) 2023-08-27T22:47:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:19 smithi093 ceph-mon[118902]: pgmap v3413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:19 smithi093 ceph-mon[111335]: pgmap v3413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:20.655 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:20.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:20.327847+0000) 2023-08-27T22:47:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:20 smithi136 ceph-mon[113531]: pgmap v3414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3035191184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:20 smithi093 ceph-mon[111335]: pgmap v3414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3035191184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:20 smithi093 ceph-mon[118902]: pgmap v3414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3035191184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:20.579Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:20.579Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:21.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:21.328082+0000) 2023-08-27T22:47:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:22.082Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:22.083Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:22.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:22.328282+0000) 2023-08-27T22:47:23.330 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:23 smithi136 ceph-mon[113531]: pgmap v3415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:23.331 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2303606130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:23 smithi093 ceph-mon[111335]: pgmap v3415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2303606130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:23 smithi093 ceph-mon[118902]: pgmap v3415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2303606130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:23.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:23.328480+0000) 2023-08-27T22:47:24.329 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:24.000Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:24.001Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:24.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:24.328750+0000) 2023-08-27T22:47:25.329 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:25 smithi136 ceph-mon[113531]: pgmap v3416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:25.329 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/763215734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:25 smithi093 ceph-mon[111335]: pgmap v3416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/763215734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:25 smithi093 ceph-mon[118902]: pgmap v3416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/763215734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:25.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:25.328991+0000) 2023-08-27T22:47:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:47:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:47:25] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:47:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:26.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:26.329264+0000) 2023-08-27T22:47:27.330 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:27 smithi136 ceph-mon[113531]: pgmap v3417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:27 smithi093 ceph-mon[111335]: pgmap v3417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:27 smithi093 ceph-mon[118902]: pgmap v3417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:27.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:27.329538+0000) 2023-08-27T22:47:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1959798878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:28.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:28.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:28.329767+0000) 2023-08-27T22:47:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1959798878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1959798878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:29 smithi136 ceph-mon[113531]: pgmap v3418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:29.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:29.329967+0000) 2023-08-27T22:47:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:29 smithi093 ceph-mon[111335]: pgmap v3418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:29 smithi093 ceph-mon[118902]: pgmap v3418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:30.654 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:30.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:30.330233+0000) 2023-08-27T22:47:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2568857907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:30 smithi136 ceph-mon[113531]: pgmap v3419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2568857907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:30 smithi093 ceph-mon[111335]: pgmap v3419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2568857907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:30 smithi093 ceph-mon[118902]: pgmap v3419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:30.579Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:30.580Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:31.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:31.330486+0000) 2023-08-27T22:47:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2826287942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:32.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:32.330737+0000) 2023-08-27T22:47:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2826287942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2826287942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:32.082Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:32.083Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:33 smithi136 ceph-mon[113531]: pgmap v3420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:33.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:33.331006+0000) 2023-08-27T22:47:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:33 smithi093 ceph-mon[111335]: pgmap v3420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:33 smithi093 ceph-mon[118902]: pgmap v3420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:34.000Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:34.001Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:34.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:34.331235+0000) 2023-08-27T22:47:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:35 smithi093 ceph-mon[111335]: pgmap v3421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1172997353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:35 smithi093 ceph-mon[118902]: pgmap v3421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1172997353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:35 smithi136 ceph-mon[113531]: pgmap v3421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1172997353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:35.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:35.331486+0000) 2023-08-27T22:47:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:47:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:47:35] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:47:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:36.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:36.331715+0000) 2023-08-27T22:47:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:37 smithi136 ceph-mon[113531]: pgmap v3422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3414985542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:37.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:37.331977+0000) 2023-08-27T22:47:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:37 smithi093 ceph-mon[111335]: pgmap v3422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3414985542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:37 smithi093 ceph-mon[118902]: pgmap v3422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3414985542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:38.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:38.332254+0000) 2023-08-27T22:47:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:39 smithi136 ceph-mon[113531]: pgmap v3423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:39.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:39.332512+0000) 2023-08-27T22:47:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:39 smithi093 ceph-mon[111335]: pgmap v3423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:39 smithi093 ceph-mon[118902]: pgmap v3423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1215969181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:40.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:40.332777+0000) 2023-08-27T22:47:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1215969181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1215969181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:40.580Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:40.581Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:41 smithi136 ceph-mon[113531]: pgmap v3424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:41.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:41.332982+0000) 2023-08-27T22:47:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:41 smithi093 ceph-mon[111335]: pgmap v3424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:41 smithi093 ceph-mon[118902]: pgmap v3424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/471358178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:42.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:42.333226+0000) 2023-08-27T22:47:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/471358178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/471358178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:42.083Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:42.084Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:43 smithi136 ceph-mon[113531]: pgmap v3425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:43.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:43.333373+0000) 2023-08-27T22:47:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:43 smithi093 ceph-mon[111335]: pgmap v3425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:43 smithi093 ceph-mon[118902]: pgmap v3425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/743961814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:44.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:44.333636+0000) 2023-08-27T22:47:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/743961814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/743961814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:44.001Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:44.002Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:45 smithi136 ceph-mon[113531]: pgmap v3426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:45.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:45.333870+0000) 2023-08-27T22:47:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:45 smithi093 ceph-mon[118902]: pgmap v3426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:45 smithi093 ceph-mon[111335]: pgmap v3426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:47:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:47:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:47:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:46.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:46.334159+0000) 2023-08-27T22:47:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:47 smithi136 ceph-mon[113531]: pgmap v3427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2174569135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:47.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:47.334354+0000) 2023-08-27T22:47:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:47 smithi093 ceph-mon[111335]: pgmap v3427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2174569135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:47 smithi093 ceph-mon[118902]: pgmap v3427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2174569135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:48.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:48.334637+0000) 2023-08-27T22:47:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:49 smithi136 ceph-mon[113531]: pgmap v3428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/673801094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:49.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:49.334864+0000) 2023-08-27T22:47:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:49 smithi093 ceph-mon[111335]: pgmap v3428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/673801094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:49 smithi093 ceph-mon[118902]: pgmap v3428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/673801094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:50.660 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:50.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:50.335125+0000) 2023-08-27T22:47:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:47:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:50 smithi093 ceph-mon[111335]: pgmap v3429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:47:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:50 smithi093 ceph-mon[118902]: pgmap v3429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:50.580Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:50.581Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:47:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:50 smithi136 ceph-mon[113531]: pgmap v3429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:51.659 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:51.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:51.335321+0000) 2023-08-27T22:47:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2810139650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2810139650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2810139650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:52.083Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:52.084Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:52.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:52.335516+0000) 2023-08-27T22:47:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:53.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:53.335751+0000) 2023-08-27T22:47:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:53 smithi136 ceph-mon[113531]: pgmap v3430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:47:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:47:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[111335]: pgmap v3430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[118902]: pgmap v3430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:47:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:47:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:47:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1118062259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:54.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:54.336056+0000) 2023-08-27T22:47:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1118062259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1118062259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:47:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:54.002Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:47:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:47:54.003Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:47:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:55 smithi136 ceph-mon[113531]: pgmap v3431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:55.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:55.336363+0000) 2023-08-27T22:47:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:55 smithi093 ceph-mon[111335]: pgmap v3431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:47:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:47:55] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:47:55.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:55 smithi093 ceph-mon[118902]: pgmap v3431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4276594385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:56.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:56.336597+0000) 2023-08-27T22:47:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4276594385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4276594385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:57 smithi136 ceph-mon[113531]: pgmap v3432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:57.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:57.336817+0000) 2023-08-27T22:47:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:57 smithi093 ceph-mon[111335]: pgmap v3432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:57 smithi093 ceph-mon[118902]: pgmap v3432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:58.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:58.337090+0000) 2023-08-27T22:47:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:59 smithi136 ceph-mon[113531]: pgmap v3433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:47:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3990514863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:47:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:47:59.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:47:59.337347+0000) 2023-08-27T22:47:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:59 smithi093 ceph-mon[111335]: pgmap v3433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:47:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3990514863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:47:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:59 smithi093 ceph-mon[118902]: pgmap v3433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:47:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:47:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3990514863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:00.659 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:00.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:00.337591+0000) 2023-08-27T22:48:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:00 smithi136 ceph-mon[113531]: pgmap v3434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:00 smithi093 ceph-mon[111335]: pgmap v3434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:01.154 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:00 smithi093 ceph-mon[118902]: pgmap v3434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:01.155 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:00.581Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:01.155 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:00.582Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:01.661 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:01.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:01.337791+0000) 2023-08-27T22:48:02.042 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1109366058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:02.042 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1109366058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:02.043 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1109366058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:02.084Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:02.505 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:02.085Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:02.665 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:02.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:02.338079+0000) 2023-08-27T22:48:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:02 smithi136 ceph-mon[113531]: pgmap v3435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:02 smithi093 ceph-mon[111335]: pgmap v3435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:02 smithi093 ceph-mon[118902]: pgmap v3435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:03.667 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:03.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:03.338389+0000) 2023-08-27T22:48:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/956157788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/956157788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/956157788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:04.002Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:04.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:04.003Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:04.675 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:04.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:04.338676+0000) 2023-08-27T22:48:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:04 smithi136 ceph-mon[113531]: pgmap v3436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:04 smithi093 ceph-mon[111335]: pgmap v3436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:04 smithi093 ceph-mon[118902]: pgmap v3436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:05.676 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:48:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:48:05] "GET /metrics HTTP/1.1" 200 239912 "" "Prometheus/2.43.0" 2023-08-27T22:48:05.677 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:05.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:05.338933+0000) 2023-08-27T22:48:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4236567827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4236567827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4236567827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:06.684 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:06.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:06.339223+0000) 2023-08-27T22:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:06 smithi136 ceph-mon[113531]: pgmap v3437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:06 smithi093 ceph-mon[111335]: pgmap v3437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:06 smithi093 ceph-mon[118902]: pgmap v3437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:07.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:07.339457+0000) 2023-08-27T22:48:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/837560886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/837560886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/837560886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:08.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:08.339705+0000) 2023-08-27T22:48:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:08 smithi136 ceph-mon[113531]: pgmap v3438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:08 smithi093 ceph-mon[111335]: pgmap v3438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:08 smithi093 ceph-mon[118902]: pgmap v3438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:09.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:09.340009+0000) 2023-08-27T22:48:10.664 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:10.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:10.340332+0000) 2023-08-27T22:48:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:10 smithi136 ceph-mon[113531]: pgmap v3439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/737342205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:10 smithi093 ceph-mon[111335]: pgmap v3439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/737342205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:10 smithi093 ceph-mon[118902]: pgmap v3439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/737342205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:10.582Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:10.583Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:11.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:11.340556+0000) 2023-08-27T22:48:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:12.084Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:12.085Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:12.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:12.340815+0000) 2023-08-27T22:48:13.341 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:13 smithi136 ceph-mon[113531]: pgmap v3440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:13.341 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3217714019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:13 smithi093 ceph-mon[118902]: pgmap v3440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3217714019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:13 smithi093 ceph-mon[111335]: pgmap v3440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3217714019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:13.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:13.341010+0000) 2023-08-27T22:48:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:14.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:14.004Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:14.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:14.341206+0000) 2023-08-27T22:48:15.341 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:15 smithi136 ceph-mon[113531]: pgmap v3441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:15.341 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3188185419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:15 smithi093 ceph-mon[111335]: pgmap v3441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3188185419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:15 smithi093 ceph-mon[118902]: pgmap v3441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3188185419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:15.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:15.341434+0000) 2023-08-27T22:48:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:48:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:48:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:48:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:16.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:16.341707+0000) 2023-08-27T22:48:17.342 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:17 smithi136 ceph-mon[113531]: pgmap v3442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:17 smithi093 ceph-mon[111335]: pgmap v3442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:17 smithi093 ceph-mon[118902]: pgmap v3442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:17.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:17.341993+0000) 2023-08-27T22:48:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1197075013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:18.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:18.342319+0000) 2023-08-27T22:48:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1197075013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1197075013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:19 smithi136 ceph-mon[113531]: pgmap v3443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:19.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:19.342540+0000) 2023-08-27T22:48:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:19 smithi093 ceph-mon[111335]: pgmap v3443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:19 smithi093 ceph-mon[118902]: pgmap v3443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:20.666 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:20.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:20.342810+0000) 2023-08-27T22:48:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2840457892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:20 smithi136 ceph-mon[113531]: pgmap v3444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2840457892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:20 smithi093 ceph-mon[111335]: pgmap v3444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2840457892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:20 smithi093 ceph-mon[118902]: pgmap v3444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:20.583Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:20.584Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:21.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:21.343089+0000) 2023-08-27T22:48:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:22.084Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:22.085Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:22.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:22.343346+0000) 2023-08-27T22:48:23.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:23 smithi136 ceph-mon[113531]: pgmap v3445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:23.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/982871840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:23 smithi093 ceph-mon[111335]: pgmap v3445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/982871840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:23 smithi093 ceph-mon[118902]: pgmap v3445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/982871840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:23.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:23.343552+0000) 2023-08-27T22:48:24.344 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:24.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:24.005Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:24.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:24.343852+0000) 2023-08-27T22:48:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:25 smithi093 ceph-mon[118902]: pgmap v3446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/414486054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:25 smithi093 ceph-mon[111335]: pgmap v3446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:25.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/414486054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:25 smithi136 ceph-mon[113531]: pgmap v3446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/414486054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:25.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:25.344157+0000) 2023-08-27T22:48:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:48:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:48:25] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:48:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:26.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:26.344439+0000) 2023-08-27T22:48:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:27 smithi136 ceph-mon[113531]: pgmap v3447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1255976275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:27.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:27.344727+0000) 2023-08-27T22:48:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:27 smithi093 ceph-mon[111335]: pgmap v3447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1255976275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:27 smithi093 ceph-mon[118902]: pgmap v3447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1255976275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:28.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:28.345024+0000) 2023-08-27T22:48:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:29 smithi136 ceph-mon[113531]: pgmap v3448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:29.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:29.345308+0000) 2023-08-27T22:48:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:29 smithi093 ceph-mon[111335]: pgmap v3448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:29 smithi093 ceph-mon[118902]: pgmap v3448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1997688676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:30.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:30.345527+0000) 2023-08-27T22:48:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1997688676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1997688676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:30.585Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:30.586Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:31 smithi136 ceph-mon[113531]: pgmap v3449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:31.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:31.345779+0000) 2023-08-27T22:48:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:31 smithi093 ceph-mon[111335]: pgmap v3449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:31 smithi093 ceph-mon[118902]: pgmap v3449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/970605049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:32.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:32.346066+0000) 2023-08-27T22:48:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/970605049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/970605049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:32.085Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:32.086Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:33 smithi136 ceph-mon[113531]: pgmap v3450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:33.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:33.346352+0000) 2023-08-27T22:48:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:33 smithi093 ceph-mon[111335]: pgmap v3450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:33 smithi093 ceph-mon[118902]: pgmap v3450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2322649023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:34.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:34.346639+0000) 2023-08-27T22:48:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2322649023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2322649023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:34.004Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:34.005Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:35 smithi136 ceph-mon[113531]: pgmap v3451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:35.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:35.346923+0000) 2023-08-27T22:48:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:35 smithi093 ceph-mon[111335]: pgmap v3451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:35 smithi093 ceph-mon[118902]: pgmap v3451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:48:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:48:35] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:48:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:36.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:36.347160+0000) 2023-08-27T22:48:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:37 smithi136 ceph-mon[113531]: pgmap v3452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3885100756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:37.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:37.347398+0000) 2023-08-27T22:48:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:37 smithi093 ceph-mon[118902]: pgmap v3452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3885100756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:37 smithi093 ceph-mon[111335]: pgmap v3452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3885100756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:38.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:38.347675+0000) 2023-08-27T22:48:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:39 smithi136 ceph-mon[113531]: pgmap v3453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2697731606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:39.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:39.347910+0000) 2023-08-27T22:48:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:39 smithi093 ceph-mon[111335]: pgmap v3453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2697731606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:39 smithi093 ceph-mon[118902]: pgmap v3453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2697731606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:40.670 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:40.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:40.348200+0000) 2023-08-27T22:48:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:40 smithi136 ceph-mon[113531]: pgmap v3454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:40 smithi093 ceph-mon[111335]: pgmap v3454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:40 smithi093 ceph-mon[118902]: pgmap v3454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:40.585Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:40.586Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:41.669 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:41.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:41.348386+0000) 2023-08-27T22:48:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3094218533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3094218533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3094218533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:42.085Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:42.086Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:42.676 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:42.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:42.348637+0000) 2023-08-27T22:48:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:42 smithi136 ceph-mon[113531]: pgmap v3455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:42 smithi093 ceph-mon[111335]: pgmap v3455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:42 smithi093 ceph-mon[118902]: pgmap v3455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:43.675 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:43.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:43.348921+0000) 2023-08-27T22:48:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3267951914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3267951914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3267951914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:44.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:44.006Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:44.684 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:44.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:44.349180+0000) 2023-08-27T22:48:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:44 smithi136 ceph-mon[113531]: pgmap v3456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:44 smithi093 ceph-mon[111335]: pgmap v3456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:44 smithi093 ceph-mon[118902]: pgmap v3456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:45.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:45.349438+0000) 2023-08-27T22:48:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:48:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:48:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:48:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1219267792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1219267792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:46.349 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1219267792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:46.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:46.349659+0000) 2023-08-27T22:48:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:46 smithi093 ceph-mon[111335]: pgmap v3457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:46 smithi093 ceph-mon[118902]: pgmap v3457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:47.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:46 smithi136 ceph-mon[113531]: pgmap v3457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:47.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:47.349899+0000) 2023-08-27T22:48:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:48.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:48.350186+0000) 2023-08-27T22:48:49.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:49 smithi136 ceph-mon[113531]: pgmap v3458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:49.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1684033022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:49 smithi093 ceph-mon[111335]: pgmap v3458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1684033022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:49 smithi093 ceph-mon[118902]: pgmap v3458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1684033022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:49.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:49.350397+0000) 2023-08-27T22:48:50.667 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:50.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:50.350594+0000) 2023-08-27T22:48:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:50 smithi136 ceph-mon[113531]: pgmap v3459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:50 smithi093 ceph-mon[111335]: pgmap v3459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:50 smithi093 ceph-mon[118902]: pgmap v3459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:50.586Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:50.587Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:51.671 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:51.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:51.350864+0000) 2023-08-27T22:48:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1558355673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1558355673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1558355673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:52.086Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:52.087Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:52.673 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:52.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:52.351082+0000) 2023-08-27T22:48:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:52 smithi136 ceph-mon[113531]: pgmap v3460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:52 smithi093 ceph-mon[111335]: pgmap v3460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:52 smithi093 ceph-mon[118902]: pgmap v3460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:53.675 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:53.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:53.351250+0000) 2023-08-27T22:48:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:48:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1844444739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:53.970 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:48:53.970 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1844444739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:53.970 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:48:53.970 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1844444739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:54.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:54.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:48:54.006Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:48:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:54.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:54.351445+0000) 2023-08-27T22:48:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:54 smithi136 ceph-mon[113531]: pgmap v3461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[111335]: pgmap v3461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:48:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[118902]: pgmap v3461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:55.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:55.351638+0000) 2023-08-27T22:48:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:48:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:48:55] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:48:56.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:48:56.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:48:56.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2609713498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:48:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:48:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2609713498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:48:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:48:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:48:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2609713498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:56.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:56.351867+0000) 2023-08-27T22:48:57.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:57 smithi136 ceph-mon[113531]: pgmap v3462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:57 smithi093 ceph-mon[111335]: pgmap v3462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:57 smithi093 ceph-mon[118902]: pgmap v3462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:57.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:57.352186+0000) 2023-08-27T22:48:58.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/906786351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/906786351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/906786351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:48:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:58.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:58.352391+0000) 2023-08-27T22:48:59.353 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:48:59 smithi136 ceph-mon[113531]: pgmap v3463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:48:59 smithi093 ceph-mon[111335]: pgmap v3463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:48:59 smithi093 ceph-mon[118902]: pgmap v3463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:48:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:48:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:48:59.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:48:59.352685+0000) 2023-08-27T22:49:00.672 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:00.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:00.352889+0000) 2023-08-27T22:49:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:00 smithi136 ceph-mon[113531]: pgmap v3464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1970703132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:00 smithi093 ceph-mon[111335]: pgmap v3464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1970703132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:00 smithi093 ceph-mon[118902]: pgmap v3464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1970703132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:00.586Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:00.587Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:01.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:01.353173+0000) 2023-08-27T22:49:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:02.086Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:02.087Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:02.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:02.353402+0000) 2023-08-27T22:49:03.353 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:03 smithi136 ceph-mon[113531]: pgmap v3465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:03.354 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/987770804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:03 smithi093 ceph-mon[111335]: pgmap v3465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/987770804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:03 smithi093 ceph-mon[118902]: pgmap v3465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/987770804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:03.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:03.353616+0000) 2023-08-27T22:49:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:04.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:04.006Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:04.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:04.353870+0000) 2023-08-27T22:49:05.354 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:05 smithi136 ceph-mon[113531]: pgmap v3466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:05.354 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/786415237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:05.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:05 smithi093 ceph-mon[111335]: pgmap v3466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/786415237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:05 smithi093 ceph-mon[118902]: pgmap v3466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/786415237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:05.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:05.354135+0000) 2023-08-27T22:49:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:49:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:49:05] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:49:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:06.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:06.354416+0000) 2023-08-27T22:49:07.354 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:07 smithi136 ceph-mon[113531]: pgmap v3467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:07 smithi093 ceph-mon[111335]: pgmap v3467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:07 smithi093 ceph-mon[118902]: pgmap v3467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:07.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:07.354642+0000) 2023-08-27T22:49:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4017780086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:08.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:08.354906+0000) 2023-08-27T22:49:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4017780086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4017780086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:09 smithi136 ceph-mon[113531]: pgmap v3468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:09.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:09.355183+0000) 2023-08-27T22:49:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:09 smithi093 ceph-mon[111335]: pgmap v3468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:09 smithi093 ceph-mon[118902]: pgmap v3468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:10.671 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:10.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:10.355465+0000) 2023-08-27T22:49:11.457 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/715796684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:11.458 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:10 smithi093 ceph-mon[111335]: pgmap v3469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:11.458 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/715796684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:11.458 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:10 smithi136 ceph-mon[113531]: pgmap v3469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:11.523 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/715796684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:11.523 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:10 smithi093 ceph-mon[118902]: pgmap v3469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:11.524 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:10.587Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:11.524 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:10.588Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:11.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:11.355688+0000) 2023-08-27T22:49:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:12.087Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:12.088Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:12.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:12.355947+0000) 2023-08-27T22:49:13.356 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:13 smithi136 ceph-mon[113531]: pgmap v3470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:13.356 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/179136319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:13 smithi093 ceph-mon[111335]: pgmap v3470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/179136319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:13 smithi093 ceph-mon[118902]: pgmap v3470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/179136319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:13.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:13.356167+0000) 2023-08-27T22:49:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:14.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:14.008Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:14.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:14.356444+0000) 2023-08-27T22:49:15.356 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:15 smithi136 ceph-mon[113531]: pgmap v3471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:15.357 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1852174246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:15 smithi093 ceph-mon[111335]: pgmap v3471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1852174246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:15 smithi093 ceph-mon[118902]: pgmap v3471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1852174246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:15.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:15.356700+0000) 2023-08-27T22:49:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:49:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:49:15] "GET /metrics HTTP/1.1" 200 239911 "" "Prometheus/2.43.0" 2023-08-27T22:49:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:16.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:16.356924+0000) 2023-08-27T22:49:17.357 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:17 smithi136 ceph-mon[113531]: pgmap v3472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:17.357 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4270133717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:17 smithi093 ceph-mon[111335]: pgmap v3472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4270133717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:17 smithi093 ceph-mon[118902]: pgmap v3472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4270133717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:17.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:17.357161+0000) 2023-08-27T22:49:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:18.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:18.357360+0000) 2023-08-27T22:49:19.357 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:19 smithi136 ceph-mon[113531]: pgmap v3473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:19 smithi093 ceph-mon[111335]: pgmap v3473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:19 smithi093 ceph-mon[118902]: pgmap v3473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:19.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:19.357577+0000) 2023-08-27T22:49:20.358 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1067513754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1067513754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1067513754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:20.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:20.357742+0000) 2023-08-27T22:49:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:20.588Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:20.589Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:21.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:21 smithi136 ceph-mon[113531]: pgmap v3474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:21 smithi093 ceph-mon[111335]: pgmap v3474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:21 smithi093 ceph-mon[118902]: pgmap v3474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:21.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:21.358000+0000) 2023-08-27T22:49:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2026534009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:22.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:22.358260+0000) 2023-08-27T22:49:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2026534009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2026534009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:22.087Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:22.088Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:23 smithi136 ceph-mon[113531]: pgmap v3475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:23.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:23.358517+0000) 2023-08-27T22:49:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:23 smithi093 ceph-mon[111335]: pgmap v3475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:23 smithi093 ceph-mon[118902]: pgmap v3475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1564910378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:24.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:24.358775+0000) 2023-08-27T22:49:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1564910378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1564910378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:24.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:24.008Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:25 smithi136 ceph-mon[113531]: pgmap v3476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:25.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:25.359013+0000) 2023-08-27T22:49:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:25 smithi093 ceph-mon[111335]: pgmap v3476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:25 smithi093 ceph-mon[118902]: pgmap v3476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:49:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:49:25] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:49:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:26.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:26.359301+0000) 2023-08-27T22:49:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:27 smithi136 ceph-mon[113531]: pgmap v3477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2359058550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:27.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:27.359606+0000) 2023-08-27T22:49:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:27 smithi093 ceph-mon[111335]: pgmap v3477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2359058550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:27 smithi093 ceph-mon[118902]: pgmap v3477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2359058550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:28.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:28.359887+0000) 2023-08-27T22:49:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:29 smithi136 ceph-mon[113531]: pgmap v3478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/606313301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:29.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:29.360175+0000) 2023-08-27T22:49:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:29 smithi093 ceph-mon[111335]: pgmap v3478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/606313301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:29 smithi093 ceph-mon[118902]: pgmap v3478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/606313301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:30.676 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:30.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:30.360364+0000) 2023-08-27T22:49:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:30 smithi136 ceph-mon[113531]: pgmap v3479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:30 smithi093 ceph-mon[111335]: pgmap v3479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:30 smithi093 ceph-mon[118902]: pgmap v3479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:30.589Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:30.590Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:31.677 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:31.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:31.360579+0000) 2023-08-27T22:49:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/235747339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/235747339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/235747339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:32.088Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:32.089Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:32.684 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:32.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:32.360823+0000) 2023-08-27T22:49:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:32 smithi136 ceph-mon[113531]: pgmap v3480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:32 smithi093 ceph-mon[111335]: pgmap v3480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:32 smithi093 ceph-mon[118902]: pgmap v3480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:33.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:33.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:33.361063+0000) 2023-08-27T22:49:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2761013068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2761013068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2761013068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:34.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:34.008Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:34.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:34.361325+0000) 2023-08-27T22:49:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:34 smithi093 ceph-mon[111335]: pgmap v3481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:34 smithi093 ceph-mon[118902]: pgmap v3481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:34 smithi136 ceph-mon[113531]: pgmap v3481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:35.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:35.361555+0000) 2023-08-27T22:49:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:49:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:49:35] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:49:36.362 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/172138380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/172138380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/172138380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:36.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:36.361830+0000) 2023-08-27T22:49:37.362 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:37 smithi136 ceph-mon[113531]: pgmap v3482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:37 smithi093 ceph-mon[111335]: pgmap v3482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:37 smithi093 ceph-mon[118902]: pgmap v3482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:37.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:37.362122+0000) 2023-08-27T22:49:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:38.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:38.362419+0000) 2023-08-27T22:49:39.363 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:39 smithi136 ceph-mon[113531]: pgmap v3483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:39.363 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3888116242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:39.363 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:39 smithi093 ceph-mon[111335]: pgmap v3483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3888116242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:39 smithi093 ceph-mon[118902]: pgmap v3483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3888116242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:39.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:39.362709+0000) 2023-08-27T22:49:40.677 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:40.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:40.363006+0000) 2023-08-27T22:49:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:40 smithi136 ceph-mon[113531]: pgmap v3484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:40 smithi093 ceph-mon[111335]: pgmap v3484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:40 smithi093 ceph-mon[118902]: pgmap v3484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:40.589Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:40.590Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:41.678 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:41.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:41.363295+0000) 2023-08-27T22:49:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1586806426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1586806426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1586806426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:42.088Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:42.089Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:42.683 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:42.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:42.363520+0000) 2023-08-27T22:49:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:42 smithi136 ceph-mon[113531]: pgmap v3485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:42 smithi093 ceph-mon[111335]: pgmap v3485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:42 smithi093 ceph-mon[118902]: pgmap v3485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:43.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:43.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:43.363765+0000) 2023-08-27T22:49:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/486641981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/486641981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/486641981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:44.008Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:44.009Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:44.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:44.364020+0000) 2023-08-27T22:49:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:44 smithi136 ceph-mon[113531]: pgmap v3486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:44 smithi093 ceph-mon[111335]: pgmap v3486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:44 smithi093 ceph-mon[118902]: pgmap v3486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:45.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:45.364258+0000) 2023-08-27T22:49:45.690 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:49:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:49:45] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:49:45.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/746760301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/746760301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/746760301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:46.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:46.364504+0000) 2023-08-27T22:49:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:46 smithi093 ceph-mon[111335]: pgmap v3487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:46 smithi093 ceph-mon[118902]: pgmap v3487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:46 smithi136 ceph-mon[113531]: pgmap v3487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:47.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:47.364745+0000) 2023-08-27T22:49:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1290450445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1290450445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:48.365 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1290450445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:48.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:48.364974+0000) 2023-08-27T22:49:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:48 smithi093 ceph-mon[111335]: pgmap v3488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:48 smithi093 ceph-mon[118902]: pgmap v3488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:49.365 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:48 smithi136 ceph-mon[113531]: pgmap v3488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:49.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:49.365257+0000) 2023-08-27T22:49:50.677 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:50.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:50.365480+0000) 2023-08-27T22:49:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:50 smithi136 ceph-mon[113531]: pgmap v3489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2240914066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:50 smithi093 ceph-mon[111335]: pgmap v3489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2240914066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:50 smithi093 ceph-mon[118902]: pgmap v3489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2240914066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:50.590Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:50.591Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:51.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:51.365721+0000) 2023-08-27T22:49:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:52.089Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:52.090Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:52.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:52.365955+0000) 2023-08-27T22:49:53.366 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:53 smithi136 ceph-mon[113531]: pgmap v3490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:53.366 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1971035128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:53 smithi093 ceph-mon[111335]: pgmap v3490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1971035128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:53 smithi093 ceph-mon[118902]: pgmap v3490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1971035128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:53.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:53.366235+0000) 2023-08-27T22:49:54.366 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:49:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:54.008Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:49:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:49:54.009Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:49:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:54.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:54.366523+0000) 2023-08-27T22:49:55.367 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:55 smithi136 ceph-mon[113531]: pgmap v3491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:55.367 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4205899965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:55 smithi093 ceph-mon[111335]: pgmap v3491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4205899965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:49:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:49:55] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:49:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:55 smithi093 ceph-mon[118902]: pgmap v3491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:55.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4205899965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:55.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:55.366829+0000) 2023-08-27T22:49:56.367 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:49:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:49:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:49:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:56.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:56.367029+0000) 2023-08-27T22:49:57.367 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:57 smithi136 ceph-mon[113531]: pgmap v3492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:57 smithi093 ceph-mon[111335]: pgmap v3492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:57 smithi093 ceph-mon[118902]: pgmap v3492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:57.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:57.367322+0000) 2023-08-27T22:49:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:58.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:58.367514+0000) 2023-08-27T22:49:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2182536517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:58 smithi136 ceph-mon[113531]: pgmap v3493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2182536517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[111335]: pgmap v3493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2182536517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:49:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[118902]: pgmap v3493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:49:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:49:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:49:59.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:49:59.367715+0000) 2023-08-27T22:49:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:49:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:49:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:49:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:49:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:49:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:50:00.679 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:00.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:00.367935+0000) 2023-08-27T22:50:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2262674824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:50:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:00 smithi136 ceph-mon[113531]: pgmap v3494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2262674824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:50:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-mon[111335]: pgmap v3494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2262674824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T22:50:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-mon[118902]: pgmap v3494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:00.591Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:00.592Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:01.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:01.368214+0000) 2023-08-27T22:50:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:02.090Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:02.090Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:02.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:02.368503+0000) 2023-08-27T22:50:03.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:03 smithi136 ceph-mon[113531]: pgmap v3495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:03.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2926290430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:03 smithi093 ceph-mon[111335]: pgmap v3495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2926290430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:03 smithi093 ceph-mon[118902]: pgmap v3495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2926290430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:03.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:03.368825+0000) 2023-08-27T22:50:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:04.009Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:04.010Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:04.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:04.369104+0000) 2023-08-27T22:50:05.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:05 smithi136 ceph-mon[113531]: pgmap v3496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:05.369 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2545089145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:50:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:50:05] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:50:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:05 smithi093 ceph-mon[111335]: pgmap v3496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2545089145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:05 smithi093 ceph-mon[118902]: pgmap v3496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2545089145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:05.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:05.369389+0000) 2023-08-27T22:50:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:06.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:06.369637+0000) 2023-08-27T22:50:07.370 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:07 smithi136 ceph-mon[113531]: pgmap v3497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:07.370 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2714423641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:07 smithi093 ceph-mon[111335]: pgmap v3497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2714423641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:07 smithi093 ceph-mon[118902]: pgmap v3497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2714423641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:07.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:07.369901+0000) 2023-08-27T22:50:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:08.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:08.370171+0000) 2023-08-27T22:50:09.370 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:09 smithi136 ceph-mon[113531]: pgmap v3498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:09.370 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:09 smithi093 ceph-mon[111335]: pgmap v3498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:09 smithi093 ceph-mon[118902]: pgmap v3498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:09.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:09.370426+0000) 2023-08-27T22:50:10.371 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1718503491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1718503491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1718503491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:10.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:10.370711+0000) 2023-08-27T22:50:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:10.592Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:10.593Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:11.371 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:11 smithi136 ceph-mon[113531]: pgmap v3499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:11 smithi093 ceph-mon[111335]: pgmap v3499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:11 smithi093 ceph-mon[118902]: pgmap v3499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:11.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:11.370971+0000) 2023-08-27T22:50:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2695546739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:12.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:12.371227+0000) 2023-08-27T22:50:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2695546739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2695546739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:12.090Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:12.091Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:13 smithi136 ceph-mon[113531]: pgmap v3500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:13.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:13.371510+0000) 2023-08-27T22:50:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:13 smithi093 ceph-mon[118902]: pgmap v3500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:13 smithi093 ceph-mon[111335]: pgmap v3500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:14.010Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:14.011Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:14.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:14.371822+0000) 2023-08-27T22:50:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:15.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:15.372107+0000) 2023-08-27T22:50:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:15 smithi136 ceph-mon[113531]: pgmap v3501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3621739267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:50:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:50:15] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:50:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:15 smithi093 ceph-mon[111335]: pgmap v3501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3621739267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:15 smithi093 ceph-mon[118902]: pgmap v3501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3621739267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:16.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:16.372332+0000) 2023-08-27T22:50:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:17.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:17.372580+0000) 2023-08-27T22:50:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:17 smithi136 ceph-mon[113531]: pgmap v3502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1294169508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:17 smithi093 ceph-mon[111335]: pgmap v3502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1294169508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:17 smithi093 ceph-mon[118902]: pgmap v3502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1294169508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:18.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:18.372810+0000) 2023-08-27T22:50:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:19.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:19.373012+0000) 2023-08-27T22:50:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:19 smithi136 ceph-mon[113531]: pgmap v3503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1615352339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:19 smithi093 ceph-mon[111335]: pgmap v3503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1615352339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:19 smithi093 ceph-mon[118902]: pgmap v3503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1615352339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:20.684 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:20.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:20.373324+0000) 2023-08-27T22:50:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:20 smithi136 ceph-mon[113531]: pgmap v3504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:20 smithi093 ceph-mon[111335]: pgmap v3504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:20 smithi093 ceph-mon[118902]: pgmap v3504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:20.593Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:20.594Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:21.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:21.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:21.373570+0000) 2023-08-27T22:50:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3825942663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3825942663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3825942663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:22.091Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:22.092Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:22.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:22.373767+0000) 2023-08-27T22:50:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:22 smithi093 ceph-mon[111335]: pgmap v3505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:22 smithi093 ceph-mon[118902]: pgmap v3505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:22 smithi136 ceph-mon[113531]: pgmap v3505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:23.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:23.374083+0000) 2023-08-27T22:50:24.011 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/17634697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:24.011 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:24.012 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/17634697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:24.012 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/17634697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:24.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:24.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:24.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:24.012Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:24.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:24.374332+0000) 2023-08-27T22:50:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:24 smithi136 ceph-mon[113531]: pgmap v3506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:24 smithi093 ceph-mon[111335]: pgmap v3506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:24 smithi093 ceph-mon[118902]: pgmap v3506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:25.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:25.374598+0000) 2023-08-27T22:50:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:50:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:50:25] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:50:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:26.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:26.374821+0000) 2023-08-27T22:50:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1871936434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1871936434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1871936434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:27 smithi136 ceph-mon[113531]: pgmap v3507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:27.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:27.375095+0000) 2023-08-27T22:50:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:27 smithi093 ceph-mon[111335]: pgmap v3507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:27 smithi093 ceph-mon[118902]: pgmap v3507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:28.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:28.375345+0000) 2023-08-27T22:50:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:29 smithi136 ceph-mon[113531]: pgmap v3508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/630159116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:29.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:29.375614+0000) 2023-08-27T22:50:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:29 smithi093 ceph-mon[111335]: pgmap v3508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/630159116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:29 smithi093 ceph-mon[118902]: pgmap v3508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/630159116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:30.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:30.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:30.375909+0000) 2023-08-27T22:50:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:30 smithi136 ceph-mon[113531]: pgmap v3509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:30 smithi093 ceph-mon[118902]: pgmap v3509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:30 smithi093 ceph-mon[111335]: pgmap v3509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:30.594Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:30.594Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:31.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:31.376176+0000) 2023-08-27T22:50:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2898748532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2898748532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:31.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2898748532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:32.092Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:32.093Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:32.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:32.376422+0000) 2023-08-27T22:50:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:32 smithi093 ceph-mon[111335]: pgmap v3510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:32 smithi093 ceph-mon[118902]: pgmap v3510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:32 smithi136 ceph-mon[113531]: pgmap v3510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:33.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:33.376701+0000) 2023-08-27T22:50:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2558436831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2558436831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2558436831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:34.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:34.012Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:34.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:34.376912+0000) 2023-08-27T22:50:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:34 smithi093 ceph-mon[111335]: pgmap v3511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:34 smithi093 ceph-mon[118902]: pgmap v3511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:34 smithi136 ceph-mon[113531]: pgmap v3511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:35.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:35.377170+0000) 2023-08-27T22:50:35.699 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:50:35] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:50:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/604338808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/604338808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/604338808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:36.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:36.377410+0000) 2023-08-27T22:50:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:36 smithi093 ceph-mon[111335]: pgmap v3512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:36 smithi093 ceph-mon[118902]: pgmap v3512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:36 smithi136 ceph-mon[113531]: pgmap v3512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:37.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:37.377657+0000) 2023-08-27T22:50:38.378 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1350216388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1350216388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1350216388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:38.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:38.377839+0000) 2023-08-27T22:50:39.378 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:39 smithi136 ceph-mon[113531]: pgmap v3513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:39.378 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:39 smithi093 ceph-mon[111335]: pgmap v3513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:39 smithi093 ceph-mon[118902]: pgmap v3513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:39.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:39.378074+0000) 2023-08-27T22:50:40.684 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:40.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:40.378269+0000) 2023-08-27T22:50:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:40 smithi136 ceph-mon[113531]: pgmap v3514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2604950689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:40 smithi093 ceph-mon[111335]: pgmap v3514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2604950689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:40 smithi093 ceph-mon[118902]: pgmap v3514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2604950689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:40.594Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:40.595Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:41.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:41.378546+0000) 2023-08-27T22:50:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:42.092Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:42.093Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:42.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:42.378787+0000) 2023-08-27T22:50:43.379 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:43 smithi136 ceph-mon[113531]: pgmap v3515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:43.379 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3592964175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:43 smithi093 ceph-mon[111335]: pgmap v3515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3592964175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:43 smithi093 ceph-mon[118902]: pgmap v3515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3592964175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:43.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:43.379063+0000) 2023-08-27T22:50:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:44.012Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:44.013Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:44.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:44.379282+0000) 2023-08-27T22:50:45.379 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:45 smithi136 ceph-mon[113531]: pgmap v3516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:45 smithi093 ceph-mon[111335]: pgmap v3516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:50:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:50:45] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:50:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:45 smithi093 ceph-mon[118902]: pgmap v3516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:45.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:45.379476+0000) 2023-08-27T22:50:46.380 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3960488844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3960488844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3960488844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:46.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:46.379747+0000) 2023-08-27T22:50:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:47 smithi136 ceph-mon[113531]: pgmap v3517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:47.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:47.380060+0000) 2023-08-27T22:50:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:47 smithi093 ceph-mon[111335]: pgmap v3517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:47 smithi093 ceph-mon[118902]: pgmap v3517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1116272962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:48.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:48.380351+0000) 2023-08-27T22:50:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1116272962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1116272962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:49 smithi136 ceph-mon[113531]: pgmap v3518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:49.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:49.380604+0000) 2023-08-27T22:50:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:49 smithi093 ceph-mon[111335]: pgmap v3518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:49 smithi093 ceph-mon[118902]: pgmap v3518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:50.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:50.380868+0000) 2023-08-27T22:50:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2436770881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:50 smithi093 ceph-mon[118902]: pgmap v3519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2436770881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:50 smithi093 ceph-mon[111335]: pgmap v3519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:50.595Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:50.596Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2436770881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:50 smithi136 ceph-mon[113531]: pgmap v3519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:51.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:51.381160+0000) 2023-08-27T22:50:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:52.093Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:52.095Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:52.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:52.381398+0000) 2023-08-27T22:50:53.382 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:53 smithi136 ceph-mon[113531]: pgmap v3520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:53.382 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002100627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:53 smithi093 ceph-mon[111335]: pgmap v3520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002100627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:53 smithi093 ceph-mon[118902]: pgmap v3520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002100627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:53.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:53.381666+0000) 2023-08-27T22:50:54.382 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:50:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:54.013Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:50:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:50:54.014Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:50:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:54.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:54.381875+0000) 2023-08-27T22:50:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:55 smithi136 ceph-mon[113531]: pgmap v3521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1241267549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:55.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:55.382167+0000) 2023-08-27T22:50:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:55 smithi093 ceph-mon[118902]: pgmap v3521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1241267549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:55 smithi093 ceph-mon[111335]: pgmap v3521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1241267549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:50:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:50:55] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:50:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:56.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:56.382448+0000) 2023-08-27T22:50:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:57 smithi136 ceph-mon[113531]: pgmap v3522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4134184230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:57.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:57.382706+0000) 2023-08-27T22:50:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:57 smithi093 ceph-mon[111335]: pgmap v3522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4134184230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:57 smithi093 ceph-mon[118902]: pgmap v3522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4134184230' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:50:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:58.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:58.382926+0000) 2023-08-27T22:50:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:59 smithi136 ceph-mon[113531]: pgmap v3523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:50:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:50:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:50:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:50:59.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:50:59.383120+0000) 2023-08-27T22:50:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:59 smithi093 ceph-mon[111335]: pgmap v3523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:50:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:50:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:59 smithi093 ceph-mon[118902]: pgmap v3523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:50:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:50:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:51:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3956278966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:00.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:00.383265+0000) 2023-08-27T22:51:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3956278966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3956278966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:00.595Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:00.597Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:01.383 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:01 smithi136 ceph-mon[113531]: pgmap v3524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:01.383 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:01.383 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:01 smithi093 ceph-mon[111335]: pgmap v3524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:01 smithi093 ceph-mon[118902]: pgmap v3524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:01.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:01.383405+0000) 2023-08-27T22:51:02.390 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:02.390 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:02.390 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1455503811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:02.390 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:02.390 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:02.391 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1455503811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:02.391 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:02.094Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:02.391 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:02.095Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1455503811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:02.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:02.383712+0000) 2023-08-27T22:51:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:03.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:03.383995+0000) 2023-08-27T22:51:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:03 smithi136 ceph-mon[113531]: pgmap v3525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:51:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:51:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[111335]: pgmap v3525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:03.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:51:03.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:51:03.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[118902]: pgmap v3525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:03.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:03.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:51:03.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:51:03.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:51:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1964842297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1964842297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:04.014Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:04.015Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1964842297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:04.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:04.384262+0000) 2023-08-27T22:51:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:05 smithi136 ceph-mon[113531]: pgmap v3526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:05.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:05.384479+0000) 2023-08-27T22:51:05.690 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 22:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T22:51:05.319930841Z level=info msg="Completed cleanup jobs" duration=15.408212ms 2023-08-27T22:51:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:05 smithi093 ceph-mon[111335]: pgmap v3526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:05 smithi093 ceph-mon[118902]: pgmap v3526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:51:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:51:05] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:51:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:06.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:06.384731+0000) 2023-08-27T22:51:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:07 smithi136 ceph-mon[113531]: pgmap v3527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2581322892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:07.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:07.384945+0000) 2023-08-27T22:51:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:07 smithi093 ceph-mon[111335]: pgmap v3527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2581322892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:07 smithi093 ceph-mon[118902]: pgmap v3527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2581322892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:08.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:08.385209+0000) 2023-08-27T22:51:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:09 smithi136 ceph-mon[113531]: pgmap v3528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4039349045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:09.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:09.385453+0000) 2023-08-27T22:51:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:09 smithi093 ceph-mon[111335]: pgmap v3528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4039349045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:09 smithi093 ceph-mon[118902]: pgmap v3528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4039349045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:10.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:10.385625+0000) 2023-08-27T22:51:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:10 smithi093 ceph-mon[111335]: pgmap v3529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:10 smithi093 ceph-mon[118902]: pgmap v3529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:10.597Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:10.598Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:10 smithi136 ceph-mon[113531]: pgmap v3529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:11.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:11.385847+0000) 2023-08-27T22:51:11.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2261587668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2261587668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2261587668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:12.094Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:12.095Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:12.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:12.386121+0000) 2023-08-27T22:51:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:12 smithi093 ceph-mon[111335]: pgmap v3530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:12 smithi093 ceph-mon[118902]: pgmap v3530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:12 smithi136 ceph-mon[113531]: pgmap v3530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:13.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:13.386351+0000) 2023-08-27T22:51:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3085578392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3085578392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3085578392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:14.015Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:14.016Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:14.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:14.386580+0000) 2023-08-27T22:51:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:14 smithi136 ceph-mon[113531]: pgmap v3531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:14 smithi093 ceph-mon[111335]: pgmap v3531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:14 smithi093 ceph-mon[118902]: pgmap v3531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:15.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:15.386835+0000) 2023-08-27T22:51:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:51:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:51:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:51:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:16.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:16.387073+0000) 2023-08-27T22:51:17.387 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:17 smithi136 ceph-mon[113531]: pgmap v3532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:17.460 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3952559690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:17 smithi093 ceph-mon[111335]: pgmap v3532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3952559690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:17 smithi093 ceph-mon[118902]: pgmap v3532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3952559690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:17.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:17.387257+0000) 2023-08-27T22:51:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:18.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:18.387454+0000) 2023-08-27T22:51:19.387 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:19 smithi136 ceph-mon[113531]: pgmap v3533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:19.388 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2162076669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:19 smithi093 ceph-mon[111335]: pgmap v3533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2162076669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:19 smithi093 ceph-mon[118902]: pgmap v3533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2162076669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:19.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:19.387710+0000) 2023-08-27T22:51:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:20.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:20.387951+0000) 2023-08-27T22:51:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:20 smithi093 ceph-mon[111335]: pgmap v3534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:20 smithi093 ceph-mon[118902]: pgmap v3534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:20.597Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:20.598Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:20 smithi136 ceph-mon[113531]: pgmap v3534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:21.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:21.388224+0000) 2023-08-27T22:51:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1438470614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1438470614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1438470614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:22.095Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:22.096Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:22.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:22.388482+0000) 2023-08-27T22:51:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:22 smithi093 ceph-mon[111335]: pgmap v3535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:22 smithi093 ceph-mon[118902]: pgmap v3535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:22 smithi136 ceph-mon[113531]: pgmap v3535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:23.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:23.388739+0000) 2023-08-27T22:51:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/135084027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/135084027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/135084027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:24.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:24.016Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:24.017Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:24.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:24.389004+0000) 2023-08-27T22:51:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:24 smithi093 ceph-mon[111335]: pgmap v3536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:24 smithi093 ceph-mon[118902]: pgmap v3536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:24 smithi136 ceph-mon[113531]: pgmap v3536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:25.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:25.389252+0000) 2023-08-27T22:51:25.709 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:51:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:51:25] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:51:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2714632303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2714632303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2714632303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:26.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:26.389468+0000) 2023-08-27T22:51:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:26 smithi093 ceph-mon[111335]: pgmap v3537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:26 smithi093 ceph-mon[118902]: pgmap v3537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:26 smithi136 ceph-mon[113531]: pgmap v3537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:27.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:27.389737+0000) 2023-08-27T22:51:28.390 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3511175062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3511175062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3511175062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:28.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:28.389956+0000) 2023-08-27T22:51:29.390 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:29 smithi136 ceph-mon[113531]: pgmap v3538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:29 smithi093 ceph-mon[111335]: pgmap v3538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:29 smithi093 ceph-mon[118902]: pgmap v3538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:29.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:29.390204+0000) 2023-08-27T22:51:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:30.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:30.390484+0000) 2023-08-27T22:51:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:30 smithi093 ceph-mon[111335]: pgmap v3539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1384029145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:30 smithi093 ceph-mon[118902]: pgmap v3539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1384029145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:30.598Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:30.599Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:30 smithi136 ceph-mon[113531]: pgmap v3539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1384029145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:31.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:31.390707+0000) 2023-08-27T22:51:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:32.096Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:32.097Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:32.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:32.390945+0000) 2023-08-27T22:51:33.391 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:33 smithi136 ceph-mon[113531]: pgmap v3540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:33.391 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1162013936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:33 smithi093 ceph-mon[111335]: pgmap v3540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1162013936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:33 smithi093 ceph-mon[118902]: pgmap v3540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1162013936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:33.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:33.391150+0000) 2023-08-27T22:51:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:34.016Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:34.017Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:34.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:34.391388+0000) 2023-08-27T22:51:35.391 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:35 smithi136 ceph-mon[113531]: pgmap v3541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:35 smithi093 ceph-mon[111335]: pgmap v3541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:35 smithi093 ceph-mon[118902]: pgmap v3541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:51:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:51:35] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:51:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:35.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:35.391641+0000) 2023-08-27T22:51:36.392 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3606682025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3606682025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3606682025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:36.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:36.391897+0000) 2023-08-27T22:51:37.392 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:37 smithi136 ceph-mon[113531]: pgmap v3542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:37 smithi093 ceph-mon[111335]: pgmap v3542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:37 smithi093 ceph-mon[118902]: pgmap v3542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:37.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:37.392138+0000) 2023-08-27T22:51:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3194271182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:38.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:38.392388+0000) 2023-08-27T22:51:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3194271182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3194271182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:39 smithi136 ceph-mon[113531]: pgmap v3543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:39.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:39.392658+0000) 2023-08-27T22:51:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:39 smithi093 ceph-mon[111335]: pgmap v3543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:39 smithi093 ceph-mon[118902]: pgmap v3543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:40.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:40.392916+0000) 2023-08-27T22:51:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/513401286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:40 smithi093 ceph-mon[111335]: pgmap v3544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/513401286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:40 smithi093 ceph-mon[118902]: pgmap v3544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:40.598Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:40.599Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/513401286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:40 smithi136 ceph-mon[113531]: pgmap v3544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:41.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:41.393188+0000) 2023-08-27T22:51:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:42.097Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:42.098Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:42.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:42.393454+0000) 2023-08-27T22:51:43.393 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:43 smithi136 ceph-mon[113531]: pgmap v3545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:43.394 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2576199779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:43 smithi093 ceph-mon[111335]: pgmap v3545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2576199779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:43 smithi093 ceph-mon[118902]: pgmap v3545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2576199779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:43.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:43.393703+0000) 2023-08-27T22:51:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:44.017Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:44.017Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:44.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:44.393962+0000) 2023-08-27T22:51:45.394 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:45 smithi136 ceph-mon[113531]: pgmap v3546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:45.394 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2115304330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:45 smithi093 ceph-mon[111335]: pgmap v3546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2115304330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:45 smithi093 ceph-mon[118902]: pgmap v3546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2115304330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:51:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:51:45] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:51:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:45.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:45.394239+0000) 2023-08-27T22:51:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:46.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:46.394496+0000) 2023-08-27T22:51:47.394 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:47 smithi136 ceph-mon[113531]: pgmap v3547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:47.395 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1598340553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:47 smithi093 ceph-mon[111335]: pgmap v3547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1598340553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:47 smithi093 ceph-mon[118902]: pgmap v3547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1598340553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:47.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:47.394749+0000) 2023-08-27T22:51:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:48.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:48.395004+0000) 2023-08-27T22:51:49.395 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:49 smithi136 ceph-mon[113531]: pgmap v3548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:49 smithi093 ceph-mon[111335]: pgmap v3548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:49 smithi093 ceph-mon[118902]: pgmap v3548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:49.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:49.395241+0000) 2023-08-27T22:51:50.395 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1796209282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1796209282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1796209282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:50.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:50.395473+0000) 2023-08-27T22:51:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:50.599Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:50.600Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:51.395 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:51 smithi136 ceph-mon[113531]: pgmap v3549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:51 smithi093 ceph-mon[111335]: pgmap v3549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:51 smithi093 ceph-mon[118902]: pgmap v3549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:51.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:51.395726+0000) 2023-08-27T22:51:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3621336061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:52.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:52.395936+0000) 2023-08-27T22:51:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3621336061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3621336061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:52.097Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:52.098Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:53 smithi136 ceph-mon[113531]: pgmap v3550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:53.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:53.396196+0000) 2023-08-27T22:51:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:53 smithi093 ceph-mon[111335]: pgmap v3550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:53 smithi093 ceph-mon[118902]: pgmap v3550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:54.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:54.396427+0000) 2023-08-27T22:51:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:51:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:54.017Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:51:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:51:54.018Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:51:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:55 smithi136 ceph-mon[113531]: pgmap v3551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2755592146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:55.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:55.396690+0000) 2023-08-27T22:51:55.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:51:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:51:55] "GET /metrics HTTP/1.1" 200 239909 "" "Prometheus/2.43.0" 2023-08-27T22:51:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:55 smithi093 ceph-mon[111335]: pgmap v3551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2755592146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:55 smithi093 ceph-mon[118902]: pgmap v3551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2755592146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:56.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:56.396941+0000) 2023-08-27T22:51:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:57 smithi136 ceph-mon[113531]: pgmap v3552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1674986312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:57.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:57.397203+0000) 2023-08-27T22:51:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:57 smithi093 ceph-mon[111335]: pgmap v3552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1674986312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:57 smithi093 ceph-mon[118902]: pgmap v3552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1674986312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:58.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:58.397376+0000) 2023-08-27T22:51:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:59 smithi136 ceph-mon[113531]: pgmap v3553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:51:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1405001810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:51:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:51:59.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:51:59.397649+0000) 2023-08-27T22:51:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:59 smithi093 ceph-mon[111335]: pgmap v3553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:51:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1405001810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:51:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:59 smithi093 ceph-mon[118902]: pgmap v3553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:51:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:51:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1405001810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:00.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:00.397937+0000) 2023-08-27T22:52:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:00 smithi093 ceph-mon[111335]: pgmap v3554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:01.092 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:00 smithi093 ceph-mon[118902]: pgmap v3554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:01.092 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:00.599Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:01.092 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:00.600Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:00 smithi136 ceph-mon[113531]: pgmap v3554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:01.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:01.398165+0000) 2023-08-27T22:52:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1530402058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:02.346 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1530402058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:02.347 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1530402058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:02.097Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:02.501 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:02.098Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:02.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:02.398420+0000) 2023-08-27T22:52:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:02 smithi093 ceph-mon[111335]: pgmap v3555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:02 smithi093 ceph-mon[118902]: pgmap v3555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:02 smithi136 ceph-mon[113531]: pgmap v3555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:03.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:03.398601+0000) 2023-08-27T22:52:03.960 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:52:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:52:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:52:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:04.018Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:04.020Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:04.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:04.398749+0000) 2023-08-27T22:52:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4130752608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:04 smithi093 ceph-mon[111335]: pgmap v3556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4130752608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:04 smithi093 ceph-mon[118902]: pgmap v3556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4130752608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:04 smithi136 ceph-mon[113531]: pgmap v3556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:05.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:05.398995+0000) 2023-08-27T22:52:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:52:05] "GET /metrics HTTP/1.1" 200 239909 "" "Prometheus/2.43.0" 2023-08-27T22:52:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:06.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:06.399236+0000) 2023-08-27T22:52:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:52:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:52:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:06 smithi136 ceph-mon[113531]: pgmap v3557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2226377487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[111335]: pgmap v3557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2226377487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:52:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[118902]: pgmap v3557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2226377487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:07.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:07.399506+0000) 2023-08-27T22:52:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:08.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:08.399729+0000) 2023-08-27T22:52:09.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:09 smithi136 ceph-mon[113531]: pgmap v3558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:09.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/415998717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:09.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:09 smithi093 ceph-mon[111335]: pgmap v3558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/415998717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:09 smithi093 ceph-mon[118902]: pgmap v3558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/415998717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:09.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:09.399932+0000) 2023-08-27T22:52:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:10.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:10.400130+0000) 2023-08-27T22:52:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:10 smithi093 ceph-mon[111335]: pgmap v3559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:10 smithi093 ceph-mon[118902]: pgmap v3559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:10.600Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:10.601Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:10 smithi136 ceph-mon[113531]: pgmap v3559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:11.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:11.400379+0000) 2023-08-27T22:52:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3622918430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3622918430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3622918430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:12.098Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:12.099Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:12.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:12.400622+0000) 2023-08-27T22:52:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:12 smithi093 ceph-mon[111335]: pgmap v3560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:12 smithi093 ceph-mon[118902]: pgmap v3560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:12 smithi136 ceph-mon[113531]: pgmap v3560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:13.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:13.400845+0000) 2023-08-27T22:52:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3185840934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3185840934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3185840934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:14.020Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:14.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:14.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:14.401083+0000) 2023-08-27T22:52:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:14 smithi093 ceph-mon[111335]: pgmap v3561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:14 smithi093 ceph-mon[118902]: pgmap v3561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:14 smithi136 ceph-mon[113531]: pgmap v3561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:15.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:15.401352+0000) 2023-08-27T22:52:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:52:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:52:15] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:52:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3933065627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3933065627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3933065627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:16.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:16.401636+0000) 2023-08-27T22:52:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:16 smithi136 ceph-mon[113531]: pgmap v3562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:16 smithi093 ceph-mon[111335]: pgmap v3562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:16 smithi093 ceph-mon[118902]: pgmap v3562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:17.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:17.401859+0000) 2023-08-27T22:52:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2204487801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:18.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:18.402096+0000) 2023-08-27T22:52:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2204487801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2204487801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:19 smithi136 ceph-mon[113531]: pgmap v3563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:19.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:19.402258+0000) 2023-08-27T22:52:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:19 smithi093 ceph-mon[111335]: pgmap v3563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:19 smithi093 ceph-mon[118902]: pgmap v3563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:20.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:20.402484+0000) 2023-08-27T22:52:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:20 smithi093 ceph-mon[111335]: pgmap v3564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2529458073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:20 smithi093 ceph-mon[118902]: pgmap v3564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2529458073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:20.601Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:20.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:20 smithi136 ceph-mon[113531]: pgmap v3564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2529458073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:21.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:21.402757+0000) 2023-08-27T22:52:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:22.099Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:22.100Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:22.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:22.403073+0000) 2023-08-27T22:52:23.403 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:23 smithi136 ceph-mon[113531]: pgmap v3565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:23.403 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/559113235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:23 smithi093 ceph-mon[111335]: pgmap v3565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/559113235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:23 smithi093 ceph-mon[118902]: pgmap v3565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/559113235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:23.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:23.403331+0000) 2023-08-27T22:52:24.403 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:24.021Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:24.022Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:24.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:24.403596+0000) 2023-08-27T22:52:25.404 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:25 smithi136 ceph-mon[113531]: pgmap v3566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:25 smithi093 ceph-mon[111335]: pgmap v3566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:25 smithi093 ceph-mon[118902]: pgmap v3566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:52:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:52:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:52:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:25.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:25.403858+0000) 2023-08-27T22:52:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/114160506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:26.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:26.404140+0000) 2023-08-27T22:52:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/114160506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/114160506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:27 smithi136 ceph-mon[113531]: pgmap v3567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:27.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:27.404330+0000) 2023-08-27T22:52:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:27 smithi093 ceph-mon[111335]: pgmap v3567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:27 smithi093 ceph-mon[118902]: pgmap v3567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3520018333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:28.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:28.404562+0000) 2023-08-27T22:52:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3520018333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3520018333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:29 smithi136 ceph-mon[113531]: pgmap v3568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:29.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:29.404822+0000) 2023-08-27T22:52:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:29 smithi093 ceph-mon[111335]: pgmap v3568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:29 smithi093 ceph-mon[118902]: pgmap v3568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:30.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:30.405097+0000) 2023-08-27T22:52:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3549358583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:30 smithi093 ceph-mon[111335]: pgmap v3569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3549358583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:30 smithi093 ceph-mon[118902]: pgmap v3569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:30.602Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:30.603Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3549358583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:30 smithi136 ceph-mon[113531]: pgmap v3569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:31.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:31.405328+0000) 2023-08-27T22:52:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:32.100Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:32.101Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:32.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:32.405553+0000) 2023-08-27T22:52:33.406 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:33 smithi136 ceph-mon[113531]: pgmap v3570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:33.406 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3858045927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:33 smithi093 ceph-mon[111335]: pgmap v3570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3858045927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:33 smithi093 ceph-mon[118902]: pgmap v3570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3858045927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:33.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:33.405838+0000) 2023-08-27T22:52:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:34.021Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:34.022Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:34.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:34.406115+0000) 2023-08-27T22:52:35.406 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:35 smithi136 ceph-mon[113531]: pgmap v3571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:35.406 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3416695548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:35 smithi093 ceph-mon[111335]: pgmap v3571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3416695548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:52:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:52:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:52:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:35 smithi093 ceph-mon[118902]: pgmap v3571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3416695548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:35.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:35.406318+0000) 2023-08-27T22:52:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:36.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:36.406583+0000) 2023-08-27T22:52:37.407 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:37 smithi136 ceph-mon[113531]: pgmap v3572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:37 smithi093 ceph-mon[111335]: pgmap v3572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:37 smithi093 ceph-mon[118902]: pgmap v3572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:37.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:37.406845+0000) 2023-08-27T22:52:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3723421784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:38.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:38.407138+0000) 2023-08-27T22:52:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3723421784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3723421784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:39 smithi136 ceph-mon[113531]: pgmap v3573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:39.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:39.407395+0000) 2023-08-27T22:52:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:39 smithi093 ceph-mon[111335]: pgmap v3573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:39 smithi093 ceph-mon[118902]: pgmap v3573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3800124877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:40.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:40.407628+0000) 2023-08-27T22:52:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3800124877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3800124877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:40.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:40.603Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:40.604Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:41 smithi136 ceph-mon[113531]: pgmap v3574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:41.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:41.407856+0000) 2023-08-27T22:52:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:41 smithi093 ceph-mon[111335]: pgmap v3574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:41 smithi093 ceph-mon[118902]: pgmap v3574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1213356082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:42.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:42.408161+0000) 2023-08-27T22:52:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1213356082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1213356082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:42.100Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:42.101Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:43 smithi136 ceph-mon[113531]: pgmap v3575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:43.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:43.408440+0000) 2023-08-27T22:52:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:43 smithi093 ceph-mon[118902]: pgmap v3575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:43 smithi093 ceph-mon[111335]: pgmap v3575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:44.022Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:44.023Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:44.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:44.408684+0000) 2023-08-27T22:52:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:45 smithi136 ceph-mon[113531]: pgmap v3576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1665980643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:45.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:45.408909+0000) 2023-08-27T22:52:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:45 smithi093 ceph-mon[111335]: pgmap v3576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1665980643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:52:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:52:45] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:52:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:45 smithi093 ceph-mon[118902]: pgmap v3576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1665980643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:46.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:46.409175+0000) 2023-08-27T22:52:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:47 smithi136 ceph-mon[113531]: pgmap v3577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3465757557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:47.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:47.409351+0000) 2023-08-27T22:52:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:47 smithi093 ceph-mon[111335]: pgmap v3577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3465757557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:47 smithi093 ceph-mon[118902]: pgmap v3577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3465757557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:48.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:48.409593+0000) 2023-08-27T22:52:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:49 smithi136 ceph-mon[113531]: pgmap v3578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3854843381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:49.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:49.409850+0000) 2023-08-27T22:52:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:49 smithi093 ceph-mon[111335]: pgmap v3578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3854843381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:49 smithi093 ceph-mon[118902]: pgmap v3578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3854843381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:50.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:50.410173+0000) 2023-08-27T22:52:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:50 smithi093 ceph-mon[111335]: pgmap v3579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:50 smithi093 ceph-mon[118902]: pgmap v3579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:50.603Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:50.604Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:50 smithi136 ceph-mon[113531]: pgmap v3579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:51.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:51.410377+0000) 2023-08-27T22:52:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2540909711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2540909711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2540909711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:52.101Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:52.102Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:52.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:52.410613+0000) 2023-08-27T22:52:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:52 smithi093 ceph-mon[111335]: pgmap v3580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:52 smithi093 ceph-mon[118902]: pgmap v3580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:52 smithi136 ceph-mon[113531]: pgmap v3580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:53.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:53.410860+0000) 2023-08-27T22:52:54.023 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:54.024 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:52:54.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:54.023Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:52:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:52:54.024Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:52:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:54.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:54.411079+0000) 2023-08-27T22:52:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1307492826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:54 smithi136 ceph-mon[113531]: pgmap v3581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1307492826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:54 smithi093 ceph-mon[111335]: pgmap v3581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1307492826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:54 smithi093 ceph-mon[118902]: pgmap v3581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:55.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:55.411346+0000) 2023-08-27T22:52:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:52:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:52:55] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:52:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:56.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:56.411598+0000) 2023-08-27T22:52:57.413 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:57 smithi136 ceph-mon[113531]: pgmap v3582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:57.413 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3875966251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:57 smithi093 ceph-mon[111335]: pgmap v3582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3875966251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:57 smithi093 ceph-mon[118902]: pgmap v3582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3875966251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:57.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:57.411850+0000) 2023-08-27T22:52:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:58.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:58.412154+0000) 2023-08-27T22:52:59.412 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:59 smithi136 ceph-mon[113531]: pgmap v3583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:59.412 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:52:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2496474813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:59 smithi093 ceph-mon[111335]: pgmap v3583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:52:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2496474813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:59 smithi093 ceph-mon[118902]: pgmap v3583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:52:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:52:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2496474813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:52:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:52:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:52:59.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:52:59.412320+0000) 2023-08-27T22:53:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:00.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:00.412551+0000) 2023-08-27T22:53:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:00 smithi093 ceph-mon[111335]: pgmap v3584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:00 smithi093 ceph-mon[118902]: pgmap v3584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:00.604Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:00.605Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:00 smithi136 ceph-mon[113531]: pgmap v3584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:01.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:01.412832+0000) 2023-08-27T22:53:01.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/145932406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/145932406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/145932406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:02.101Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:02.103Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:02.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:02.413105+0000) 2023-08-27T22:53:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:02 smithi093 ceph-mon[111335]: pgmap v3585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:02 smithi093 ceph-mon[118902]: pgmap v3585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:02 smithi136 ceph-mon[113531]: pgmap v3585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:03.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:03.413271+0000) 2023-08-27T22:53:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1210936804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1210936804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1210936804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:04.023Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:04.024Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:04.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:04.413495+0000) 2023-08-27T22:53:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:04 smithi093 ceph-mon[111335]: pgmap v3586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:04 smithi093 ceph-mon[118902]: pgmap v3586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:04 smithi136 ceph-mon[113531]: pgmap v3586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:05.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:05.413757+0000) 2023-08-27T22:53:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:53:05] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:53:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3586716549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3586716549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3586716549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:06.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:06.413938+0000) 2023-08-27T22:53:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:53:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:06 smithi136 ceph-mon[113531]: pgmap v3587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:53:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[118902]: pgmap v3587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:07.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:53:07.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[111335]: pgmap v3587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:07.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:07.243 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:07.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:07.414097+0000) 2023-08-27T22:53:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:08.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:08.414255+0000) 2023-08-27T22:53:09.414 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: pgmap v3588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3583912685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:53:09.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: pgmap v3588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3583912685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: pgmap v3588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3583912685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:53:09.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:53:09.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:53:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:09.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:09.414508+0000) 2023-08-27T22:53:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:10.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:10.414714+0000) 2023-08-27T22:53:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:10 smithi093 ceph-mon[111335]: pgmap v3589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/348643807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:10 smithi093 ceph-mon[118902]: pgmap v3589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/348643807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:10.605Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:10.606Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:10 smithi136 ceph-mon[113531]: pgmap v3589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/348643807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:11.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:11.414948+0000) 2023-08-27T22:53:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:12.102Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:12.103Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:12.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:12.415196+0000) 2023-08-27T22:53:13.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:13 smithi136 ceph-mon[113531]: pgmap v3590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:13.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/100972273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:13 smithi093 ceph-mon[111335]: pgmap v3590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/100972273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:13 smithi093 ceph-mon[118902]: pgmap v3590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/100972273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:13.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:13.415440+0000) 2023-08-27T22:53:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:14.024Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:14.025Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:14.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:14.415694+0000) 2023-08-27T22:53:15.416 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:15 smithi136 ceph-mon[113531]: pgmap v3591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:15 smithi093 ceph-mon[111335]: pgmap v3591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:53:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:53:15] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:53:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:15 smithi093 ceph-mon[118902]: pgmap v3591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:15.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:15.415957+0000) 2023-08-27T22:53:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2629844202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:16.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:16.416206+0000) 2023-08-27T22:53:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2629844202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2629844202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:17 smithi136 ceph-mon[113531]: pgmap v3592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:17.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:17.416433+0000) 2023-08-27T22:53:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:17 smithi093 ceph-mon[111335]: pgmap v3592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:17 smithi093 ceph-mon[118902]: pgmap v3592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2519423597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:18.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:18.416694+0000) 2023-08-27T22:53:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2519423597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2519423597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:19 smithi136 ceph-mon[113531]: pgmap v3593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:19.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:19.416927+0000) 2023-08-27T22:53:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:19 smithi093 ceph-mon[111335]: pgmap v3593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:19 smithi093 ceph-mon[118902]: pgmap v3593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:20.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:20.417181+0000) 2023-08-27T22:53:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2819436505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:20 smithi093 ceph-mon[111335]: pgmap v3594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2819436505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:20 smithi093 ceph-mon[118902]: pgmap v3594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:20.605Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:20.606Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2819436505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:20 smithi136 ceph-mon[113531]: pgmap v3594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:21.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:21.417434+0000) 2023-08-27T22:53:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:22.102Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:22.103Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:22.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:22.417706+0000) 2023-08-27T22:53:23.418 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:23 smithi136 ceph-mon[113531]: pgmap v3595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:23.418 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4091576558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:23 smithi093 ceph-mon[118902]: pgmap v3595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4091576558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:23 smithi093 ceph-mon[111335]: pgmap v3595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4091576558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:23.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:23.417973+0000) 2023-08-27T22:53:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:24.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:24.418225+0000) 2023-08-27T22:53:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:24.025Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:24.026Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:25 smithi136 ceph-mon[113531]: pgmap v3596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/210794886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:25.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:25.418491+0000) 2023-08-27T22:53:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:25 smithi093 ceph-mon[118902]: pgmap v3596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/210794886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:25 smithi093 ceph-mon[111335]: pgmap v3596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/210794886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:53:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:53:25] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:53:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:26.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:26.418759+0000) 2023-08-27T22:53:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:27 smithi136 ceph-mon[113531]: pgmap v3597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:27.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:27.419023+0000) 2023-08-27T22:53:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:27 smithi093 ceph-mon[111335]: pgmap v3597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:27 smithi093 ceph-mon[118902]: pgmap v3597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/462275399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:28.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:28.419266+0000) 2023-08-27T22:53:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/462275399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/462275399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:29.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:29 smithi136 ceph-mon[113531]: pgmap v3598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:29.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:29.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:29.419510+0000) 2023-08-27T22:53:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:29 smithi093 ceph-mon[111335]: pgmap v3598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:29 smithi093 ceph-mon[118902]: pgmap v3598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4251130555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:30.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:30.419734+0000) 2023-08-27T22:53:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4251130555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4251130555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:30.606Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:30.607Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:31 smithi136 ceph-mon[113531]: pgmap v3599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:31.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:31.419995+0000) 2023-08-27T22:53:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:31 smithi093 ceph-mon[111335]: pgmap v3599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:31 smithi093 ceph-mon[118902]: pgmap v3599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1058441043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1058441043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:32.103Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:32.104Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1058441043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:32.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:32.420196+0000) 2023-08-27T22:53:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:33 smithi093 ceph-mon[111335]: pgmap v3600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:33 smithi093 ceph-mon[118902]: pgmap v3600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:33 smithi136 ceph-mon[113531]: pgmap v3600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:33.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:33.420453+0000) 2023-08-27T22:53:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:34.026Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:34.027Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:34.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:34.420709+0000) 2023-08-27T22:53:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:35 smithi093 ceph-mon[118902]: pgmap v3601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/674824315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:35 smithi093 ceph-mon[111335]: pgmap v3601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/674824315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:53:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:53:35] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:53:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:35 smithi136 ceph-mon[113531]: pgmap v3601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/674824315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:35.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:35.421013+0000) 2023-08-27T22:53:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:36.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:36.421286+0000) 2023-08-27T22:53:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:37 smithi093 ceph-mon[111335]: pgmap v3602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1945243895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:37 smithi093 ceph-mon[118902]: pgmap v3602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1945243895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:37 smithi136 ceph-mon[113531]: pgmap v3602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1945243895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:37.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:37.421527+0000) 2023-08-27T22:53:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:38.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:38.421781+0000) 2023-08-27T22:53:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:39 smithi093 ceph-mon[111335]: pgmap v3603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:39.674 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:39.674 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1792023267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:39.675 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:39 smithi093 ceph-mon[118902]: pgmap v3603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:39.675 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:39.675 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1792023267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:39 smithi136 ceph-mon[113531]: pgmap v3603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:39.797 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:39.797 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1792023267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:39.797 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:39.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:39.421995+0000) 2023-08-27T22:53:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:40.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:40.422270+0000) 2023-08-27T22:53:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:40 smithi093 ceph-mon[111335]: pgmap v3604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:40 smithi093 ceph-mon[118902]: pgmap v3604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:40.607Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:40.608Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:40 smithi136 ceph-mon[113531]: pgmap v3604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:41.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:41.422560+0000) 2023-08-27T22:53:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2519678630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2519678630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2519678630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:42.103Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:42.104Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:42.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:42.422813+0000) 2023-08-27T22:53:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:42 smithi093 ceph-mon[111335]: pgmap v3605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:42 smithi093 ceph-mon[118902]: pgmap v3605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:42 smithi136 ceph-mon[113531]: pgmap v3605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:43.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:43.423066+0000) 2023-08-27T22:53:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/379333605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/379333605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:44.026Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:44.027Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:44.423 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/379333605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:44.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:44.423317+0000) 2023-08-27T22:53:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:44 smithi093 ceph-mon[111335]: pgmap v3606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:44 smithi093 ceph-mon[118902]: pgmap v3606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:45.423 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:44 smithi136 ceph-mon[113531]: pgmap v3606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:45.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:45.423578+0000) 2023-08-27T22:53:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:53:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:53:45] "GET /metrics HTTP/1.1" 200 239910 "" "Prometheus/2.43.0" 2023-08-27T22:53:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:46.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:46.423845+0000) 2023-08-27T22:53:47.424 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:47 smithi136 ceph-mon[113531]: pgmap v3607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:47.424 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2553164911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:47 smithi093 ceph-mon[111335]: pgmap v3607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2553164911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:47 smithi093 ceph-mon[118902]: pgmap v3607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2553164911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:47.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:47.424124+0000) 2023-08-27T22:53:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:48.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:48.424294+0000) 2023-08-27T22:53:49.424 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:49 smithi136 ceph-mon[113531]: pgmap v3608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:49.424 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1661207383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:49 smithi093 ceph-mon[111335]: pgmap v3608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1661207383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:49 smithi093 ceph-mon[118902]: pgmap v3608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1661207383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:49.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:49.424534+0000) 2023-08-27T22:53:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:50.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:50.424746+0000) 2023-08-27T22:53:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:50 smithi093 ceph-mon[111335]: pgmap v3609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:50 smithi093 ceph-mon[118902]: pgmap v3609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:50.608Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:50.609Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:50 smithi136 ceph-mon[113531]: pgmap v3609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:51.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:51.424986+0000) 2023-08-27T22:53:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1500528197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1500528197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1500528197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:52.104Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:52.105Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:52.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:52.425246+0000) 2023-08-27T22:53:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:52 smithi093 ceph-mon[111335]: pgmap v3610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:52 smithi093 ceph-mon[118902]: pgmap v3610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:52 smithi136 ceph-mon[113531]: pgmap v3610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:53.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:53.425493+0000) 2023-08-27T22:53:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3977831914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3977831914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3977831914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:53:54.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:54.027Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:53:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:53:54.028Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:53:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:54.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:54.425758+0000) 2023-08-27T22:53:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:54 smithi093 ceph-mon[111335]: pgmap v3611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:54 smithi093 ceph-mon[118902]: pgmap v3611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:54 smithi136 ceph-mon[113531]: pgmap v3611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:55.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:55.426002+0000) 2023-08-27T22:53:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:53:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:53:55] "GET /metrics HTTP/1.1" 200 239911 "" "Prometheus/2.43.0" 2023-08-27T22:53:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2106291277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2106291277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2106291277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:56.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:56.426274+0000) 2023-08-27T22:53:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:56 smithi136 ceph-mon[113531]: pgmap v3612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:56 smithi093 ceph-mon[111335]: pgmap v3612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:56 smithi093 ceph-mon[118902]: pgmap v3612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:57.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:57.426546+0000) 2023-08-27T22:53:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:58.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:58.426777+0000) 2023-08-27T22:53:59.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:59 smithi136 ceph-mon[113531]: pgmap v3613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:59.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:53:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3234158957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:59 smithi093 ceph-mon[111335]: pgmap v3613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:53:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3234158957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:59 smithi093 ceph-mon[118902]: pgmap v3613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:53:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:53:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3234158957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:53:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:53:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:53:59.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:53:59.427058+0000) 2023-08-27T22:54:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:00.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:00.427313+0000) 2023-08-27T22:54:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:00 smithi093 ceph-mon[111335]: pgmap v3614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3910515145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:00 smithi093 ceph-mon[118902]: pgmap v3614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3910515145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:00.608Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:00.609Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:00 smithi136 ceph-mon[113531]: pgmap v3614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3910515145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:01.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:01.427512+0000) 2023-08-27T22:54:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:02.104Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:02.105Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:02.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:02.427797+0000) 2023-08-27T22:54:03.172 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:03 smithi093 ceph-mon[111335]: pgmap v3615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:03.172 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/917388476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:03.173 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:03 smithi093 ceph-mon[118902]: pgmap v3615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:03.173 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/917388476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:03.428 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:03 smithi136 ceph-mon[113531]: pgmap v3615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:03.428 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/917388476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:03.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:03.428090+0000) 2023-08-27T22:54:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:04.028Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:04.029Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:04.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:04.428323+0000) 2023-08-27T22:54:05.428 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:05 smithi136 ceph-mon[113531]: pgmap v3616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:05 smithi093 ceph-mon[118902]: pgmap v3616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:05 smithi093 ceph-mon[111335]: pgmap v3616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:54:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:54:05] "GET /metrics HTTP/1.1" 200 239911 "" "Prometheus/2.43.0" 2023-08-27T22:54:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:05.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:05.428610+0000) 2023-08-27T22:54:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2798711748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:06.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:06.428843+0000) 2023-08-27T22:54:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2798711748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2798711748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:07 smithi136 ceph-mon[113531]: pgmap v3617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:07.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:07.429096+0000) 2023-08-27T22:54:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:07 smithi093 ceph-mon[111335]: pgmap v3617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:07 smithi093 ceph-mon[118902]: pgmap v3617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/153632950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:08.543 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:08.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:08.429320+0000) 2023-08-27T22:54:08.544 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/153632950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:08.545 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/153632950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:09 smithi136 ceph-mon[113531]: pgmap v3618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:54:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:09.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:09.429505+0000) 2023-08-27T22:54:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:09 smithi093 ceph-mon[111335]: pgmap v3618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:54:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:09 smithi093 ceph-mon[118902]: pgmap v3618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:54:10.609 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:10.610 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:10.610 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[111335]: pgmap v3619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:10.610 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3031987815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:10.610 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:10.610 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:10.610 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[118902]: pgmap v3619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:10.610 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3031987815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:10.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:10.429672+0000) 2023-08-27T22:54:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:10 smithi136 ceph-mon[113531]: pgmap v3619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3031987815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:10.609Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:10.610Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:11.462 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:11.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:11.429889+0000) 2023-08-27T22:54:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:12.105Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:12.106Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:12.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:12.430132+0000) 2023-08-27T22:54:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:12 smithi136 ceph-mon[113531]: pgmap v3620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:54:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:54:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3750247944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[111335]: pgmap v3620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:54:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:54:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3750247944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[118902]: pgmap v3620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:54:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:54:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:54:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3750247944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:13.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:13.430357+0000) 2023-08-27T22:54:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:14.029Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:14.030Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:14.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:14.430616+0000) 2023-08-27T22:54:15.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:15 smithi136 ceph-mon[113531]: pgmap v3621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:15.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/872661396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:54:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:54:15] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:54:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:15 smithi093 ceph-mon[111335]: pgmap v3621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/872661396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:15 smithi093 ceph-mon[118902]: pgmap v3621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/872661396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:15.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:15.430928+0000) 2023-08-27T22:54:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:16.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:16.431183+0000) 2023-08-27T22:54:17.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:17 smithi136 ceph-mon[113531]: pgmap v3622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:20.504 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:17 smithi093 ceph-mon[111335]: pgmap v3622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:20.504 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:17.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:17.431426+0000) 2023-08-27T22:54:20.505 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:17 smithi093 ceph-mon[118902]: pgmap v3622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:20.505 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3579625627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:20.507 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3579625627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:21.025 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:18.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:18.431694+0000) 2023-08-27T22:54:21.025 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3579625627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:21.025 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:19 smithi136 ceph-mon[113531]: pgmap v3623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:21.029 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:19 smithi093 ceph-mon[111335]: pgmap v3623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:21.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:19.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:19.431912+0000) 2023-08-27T22:54:21.191 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:19 smithi093 ceph-mon[118902]: pgmap v3623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:21.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2739730242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:21.199 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2739730242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:21.347 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:20.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:20.432165+0000) 2023-08-27T22:54:21.347 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2739730242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:21.355 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:20.609Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:21.355 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:20.610Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:21 smithi093 ceph-mon[111335]: pgmap v3624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:21.509 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:21 smithi093 ceph-mon[118902]: pgmap v3624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:21 smithi136 ceph-mon[113531]: pgmap v3624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:21.996 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:21.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:21.432392+0000) 2023-08-27T22:54:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/494357610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:22.816 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/494357610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:22.816 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:22.106Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:22.817 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:22.107Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:22.817 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/494357610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:22.935 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:22.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:22.432642+0000) 2023-08-27T22:54:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:23 smithi093 ceph-mon[118902]: pgmap v3625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:23.606 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:23 smithi093 ceph-mon[111335]: pgmap v3625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:23 smithi136 ceph-mon[113531]: pgmap v3625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:23.766 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:23.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:23.432886+0000) 2023-08-27T22:54:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:24.767 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:24.767 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:24.030Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:24.767 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:24.031Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:24.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:24.768 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:24.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:24.433097+0000) 2023-08-27T22:54:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:54:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:54:25] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:54:25.581 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:25 smithi093 ceph-mon[111335]: pgmap v3626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:25.581 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4130712182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:25.582 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:25 smithi093 ceph-mon[118902]: pgmap v3626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:25.582 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4130712182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:25 smithi136 ceph-mon[113531]: pgmap v3626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:25.926 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4130712182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:25.926 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:25.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:25.433329+0000) 2023-08-27T22:54:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:26.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:26.433583+0000) 2023-08-27T22:54:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:27 smithi093 ceph-mon[111335]: pgmap v3627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:27.545 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3943390617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:27.545 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:27 smithi093 ceph-mon[118902]: pgmap v3627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:27.546 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3943390617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:27 smithi136 ceph-mon[113531]: pgmap v3627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:27.701 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3943390617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:27.701 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:27.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:27.433804+0000) 2023-08-27T22:54:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:28.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:28.434101+0000) 2023-08-27T22:54:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:29 smithi093 ceph-mon[111335]: pgmap v3628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:29.585 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:29 smithi093 ceph-mon[118902]: pgmap v3628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:29 smithi136 ceph-mon[113531]: pgmap v3628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:29.906 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:29.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:29.434339+0000) 2023-08-27T22:54:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1107086371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:30.525 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1107086371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1107086371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:30.713 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:30.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:30.434593+0000) 2023-08-27T22:54:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:30.610Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:30.611Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:31 smithi093 ceph-mon[111335]: pgmap v3629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:31 smithi093 ceph-mon[118902]: pgmap v3629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:31 smithi136 ceph-mon[113531]: pgmap v3629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:31.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:31.434812+0000) 2023-08-27T22:54:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/85949341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/85949341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:32.107Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:32.107Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/85949341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:32.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:32.435068+0000) 2023-08-27T22:54:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:33 smithi093 ceph-mon[111335]: pgmap v3630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:33 smithi093 ceph-mon[118902]: pgmap v3630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:33 smithi136 ceph-mon[113531]: pgmap v3630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:33.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:33.435336+0000) 2023-08-27T22:54:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2234708645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2234708645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:34.030Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:34.031Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2234708645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:34.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:34.435580+0000) 2023-08-27T22:54:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:35 smithi093 ceph-mon[111335]: pgmap v3631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:35.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:35 smithi093 ceph-mon[118902]: pgmap v3631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:35.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:54:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:54:35] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:54:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:35 smithi136 ceph-mon[113531]: pgmap v3631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:35.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:35.435829+0000) 2023-08-27T22:54:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:36.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:36.436108+0000) 2023-08-27T22:54:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:37 smithi093 ceph-mon[111335]: pgmap v3632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2944470378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:37 smithi093 ceph-mon[118902]: pgmap v3632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2944470378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:37 smithi136 ceph-mon[113531]: pgmap v3632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2944470378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:37.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:37.436321+0000) 2023-08-27T22:54:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:38.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:38.436599+0000) 2023-08-27T22:54:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:39 smithi093 ceph-mon[111335]: pgmap v3633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/289565261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:39 smithi093 ceph-mon[118902]: pgmap v3633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/289565261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:39 smithi136 ceph-mon[113531]: pgmap v3633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/289565261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:39.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:39.436861+0000) 2023-08-27T22:54:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:40.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:40.437097+0000) 2023-08-27T22:54:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:40 smithi093 ceph-mon[111335]: pgmap v3634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:40 smithi093 ceph-mon[118902]: pgmap v3634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:40.611Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:40.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:40 smithi136 ceph-mon[113531]: pgmap v3634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:41.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:41.437322+0000) 2023-08-27T22:54:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3274649865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:43.028 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3274649865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:43.028 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:42.107Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:43.028 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:42.108Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:43.028 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:42 smithi093 ceph-mon[111335]: pgmap v3635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:43.029 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:42 smithi093 ceph-mon[118902]: pgmap v3635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:43.029 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3274649865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:43.029 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:42.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:42.437556+0000) 2023-08-27T22:54:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:42 smithi136 ceph-mon[113531]: pgmap v3635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:43.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:43.437787+0000) 2023-08-27T22:54:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3723462383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:44.055 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3723462383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3723462383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:44.031Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:44.032Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:44.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:44.438074+0000) 2023-08-27T22:54:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:44 smithi136 ceph-mon[113531]: pgmap v3636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:45.290 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:44 smithi093 ceph-mon[111335]: pgmap v3636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:45.290 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:44 smithi093 ceph-mon[118902]: pgmap v3636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:45.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:45.438300+0000) 2023-08-27T22:54:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:54:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:54:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:54:46.446 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1597883439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:46.446 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1597883439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:46.446 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1597883439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:46.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:46.438527+0000) 2023-08-27T22:54:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:46 smithi093 ceph-mon[111335]: pgmap v3637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:47.609 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:46 smithi093 ceph-mon[118902]: pgmap v3637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:47.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:46 smithi136 ceph-mon[113531]: pgmap v3637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:47.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:47.438779+0000) 2023-08-27T22:54:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:48.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:48.438997+0000) 2023-08-27T22:54:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:49 smithi136 ceph-mon[113531]: pgmap v3638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:49.526 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2398156664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:49.528 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:49 smithi093 ceph-mon[111335]: pgmap v3638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:49.528 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2398156664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:49.528 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:49 smithi093 ceph-mon[118902]: pgmap v3638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:49.528 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2398156664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:49.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:49.439207+0000) 2023-08-27T22:54:50.728 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:50.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:50.439465+0000) 2023-08-27T22:54:50.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:50 smithi093 ceph-mon[111335]: pgmap v3639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:50.998 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1067962809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:50.999 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:50 smithi093 ceph-mon[118902]: pgmap v3639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:50.999 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1067962809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:50.999 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:50.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:51.000 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:50.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:50 smithi136 ceph-mon[113531]: pgmap v3639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1067962809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:51.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:51.439746+0000) 2023-08-27T22:54:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:52.107Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:52.575 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:52.108Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:52.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:52.440009+0000) 2023-08-27T22:54:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:53 smithi136 ceph-mon[113531]: pgmap v3640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:53.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3913121267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:53.584 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:53 smithi093 ceph-mon[111335]: pgmap v3640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:53.584 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3913121267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:53.584 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:53 smithi093 ceph-mon[118902]: pgmap v3640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:53.584 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3913121267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:53.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:53.440203+0000) 2023-08-27T22:54:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:54.533 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:54:54.533 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:54.031Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:54.533 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:54:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:54:54.033Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:54:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:54.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:54.440434+0000) 2023-08-27T22:54:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:55 smithi093 ceph-mon[111335]: pgmap v3641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:54:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:54:55] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:54:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:55 smithi093 ceph-mon[118902]: pgmap v3641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:55 smithi136 ceph-mon[113531]: pgmap v3641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:55.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:55.440683+0000) 2023-08-27T22:54:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3187519626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:56.538 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3187519626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3187519626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:57.219 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:56.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:56.440939+0000) 2023-08-27T22:54:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:57 smithi093 ceph-mon[111335]: pgmap v3642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:57.993 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:57 smithi093 ceph-mon[118902]: pgmap v3642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:57.994 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:57 smithi136 ceph-mon[113531]: pgmap v3642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:57.994 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:57.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:57.441205+0000) 2023-08-27T22:54:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3993340770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:59.008 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3993340770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:59.009 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3993340770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:54:59.009 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:58.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:58.441446+0000) 2023-08-27T22:54:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:54:59 smithi093 ceph-mon[111335]: pgmap v3643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:59.626 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:54:59 smithi093 ceph-mon[118902]: pgmap v3643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:54:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:54:59 smithi136 ceph-mon[113531]: pgmap v3643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:00.000 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:54:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:54:59.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:54:59.441679+0000) 2023-08-27T22:55:00.731 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:00.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:00.441928+0000) 2023-08-27T22:55:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:00 smithi093 ceph-mon[111335]: pgmap v3644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:00.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2299098799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:00.993 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:00 smithi093 ceph-mon[118902]: pgmap v3644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:00.993 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2299098799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:00.993 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:00.613Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:00.993 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:00.614Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:00 smithi136 ceph-mon[113531]: pgmap v3644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:01.344 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2299098799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:01.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:01.442140+0000) 2023-08-27T22:55:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:02.107Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:02.108Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:02.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:02.442380+0000) 2023-08-27T22:55:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:03 smithi136 ceph-mon[113531]: pgmap v3645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:03.527 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1574338906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:03.531 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:03 smithi093 ceph-mon[111335]: pgmap v3645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:03.531 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1574338906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:03.534 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:03 smithi093 ceph-mon[118902]: pgmap v3645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:03.534 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1574338906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:03.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:03.442587+0000) 2023-08-27T22:55:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:04.032Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:04.674 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:04.033Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:04.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:04.442841+0000) 2023-08-27T22:55:05.664 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:05 smithi093 ceph-mon[111335]: pgmap v3646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:05.664 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/937855875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:05.664 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:05 smithi136 ceph-mon[113531]: pgmap v3646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:05.664 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/937855875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:05.665 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:55:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:55:05] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:55:05.665 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:05 smithi093 ceph-mon[118902]: pgmap v3646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:05.665 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/937855875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:05.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:05.443073+0000) 2023-08-27T22:55:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:06.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:06.443345+0000) 2023-08-27T22:55:07.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:07 smithi136 ceph-mon[113531]: pgmap v3647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:07 smithi093 ceph-mon[111335]: pgmap v3647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:07 smithi093 ceph-mon[118902]: pgmap v3647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:07.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:07.443598+0000) 2023-08-27T22:55:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3327671059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:08.598 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3327671059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3327671059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:08.755 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:08.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:08.443828+0000) 2023-08-27T22:55:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:09 smithi093 ceph-mon[111335]: pgmap v3648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:09.557 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:09.558 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:09 smithi093 ceph-mon[118902]: pgmap v3648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:09.558 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:09 smithi136 ceph-mon[113531]: pgmap v3648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:09.925 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:09.925 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:09.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:09.444076+0000) 2023-08-27T22:55:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/784540600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:10.516 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/784540600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/784540600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:10.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:10.444282+0000) 2023-08-27T22:55:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:10.613Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:11.048 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:10.614Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:11 smithi093 ceph-mon[111335]: pgmap v3649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:11 smithi093 ceph-mon[118902]: pgmap v3649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:11 smithi136 ceph-mon[113531]: pgmap v3649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:11.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:11.444506+0000) 2023-08-27T22:55:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4085477443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4085477443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:12.108Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:12.109Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4085477443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:12.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:12.444703+0000) 2023-08-27T22:55:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:13 smithi093 ceph-mon[111335]: pgmap v3650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:55:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:13 smithi093 ceph-mon[118902]: pgmap v3650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:55:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:13 smithi136 ceph-mon[113531]: pgmap v3650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:13.721 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:55:13.721 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:13.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:13.444873+0000) 2023-08-27T22:55:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:14.032Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:14.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:14.034Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:14.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:14.445058+0000) 2023-08-27T22:55:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[118902]: pgmap v3651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2091149811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[111335]: pgmap v3651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2091149811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:15 smithi136 ceph-mon[113531]: pgmap v3651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2091149811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:55:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:55:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:55:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:15.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:15.445206+0000) 2023-08-27T22:55:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:55:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:55:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:55:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:55:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:55:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:55:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:55:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:16.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:16.445387+0000) 2023-08-27T22:55:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:17 smithi136 ceph-mon[113531]: pgmap v3652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/29827487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:17 smithi093 ceph-mon[111335]: pgmap v3652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/29827487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:17 smithi093 ceph-mon[118902]: pgmap v3652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/29827487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:17.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:17.445611+0000) 2023-08-27T22:55:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:18.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:18.445888+0000) 2023-08-27T22:55:19.446 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:19 smithi136 ceph-mon[113531]: pgmap v3653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:19 smithi093 ceph-mon[111335]: pgmap v3653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:19 smithi093 ceph-mon[118902]: pgmap v3653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:19.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:19.446195+0000) 2023-08-27T22:55:20.446 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/792741141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/792741141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/792741141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:20.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:20.446448+0000) 2023-08-27T22:55:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:20.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:20.615Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:21.446 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:21 smithi136 ceph-mon[113531]: pgmap v3654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:21 smithi093 ceph-mon[111335]: pgmap v3654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:21 smithi093 ceph-mon[118902]: pgmap v3654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:21.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:21.446695+0000) 2023-08-27T22:55:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/966383360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/966383360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:22.108Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:22.110Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/966383360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:22.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:22.446934+0000) 2023-08-27T22:55:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:23 smithi093 ceph-mon[111335]: pgmap v3655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:23 smithi093 ceph-mon[118902]: pgmap v3655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:23 smithi136 ceph-mon[113531]: pgmap v3655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:23.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:23.447187+0000) 2023-08-27T22:55:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2251380225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2251380225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:24.034Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:24.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2251380225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:24.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:24.447441+0000) 2023-08-27T22:55:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:25 smithi093 ceph-mon[111335]: pgmap v3656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:25 smithi093 ceph-mon[118902]: pgmap v3656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:55:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:55:25] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:55:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:25 smithi136 ceph-mon[113531]: pgmap v3656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:25.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:25.447712+0000) 2023-08-27T22:55:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:26.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:26.447973+0000) 2023-08-27T22:55:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:27 smithi093 ceph-mon[111335]: pgmap v3657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1787790286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:27 smithi093 ceph-mon[118902]: pgmap v3657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1787790286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:27.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:27.448197+0000) 2023-08-27T22:55:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:27 smithi136 ceph-mon[113531]: pgmap v3657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1787790286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:28.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:28.448446+0000) 2023-08-27T22:55:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:29 smithi093 ceph-mon[111335]: pgmap v3658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2153855564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:29 smithi093 ceph-mon[118902]: pgmap v3658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2153855564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:29 smithi136 ceph-mon[113531]: pgmap v3658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2153855564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:29.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:29.448718+0000) 2023-08-27T22:55:30.735 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:30.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:30.448974+0000) 2023-08-27T22:55:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:30 smithi093 ceph-mon[111335]: pgmap v3659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:30 smithi093 ceph-mon[118902]: pgmap v3659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:30.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:30.615Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:30 smithi136 ceph-mon[113531]: pgmap v3659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:31.736 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:31.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:31.449216+0000) 2023-08-27T22:55:31.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/622244007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/622244007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/622244007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:32.109Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:32.110Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:32.741 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:32.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:32.449428+0000) 2023-08-27T22:55:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:32 smithi136 ceph-mon[113531]: pgmap v3660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:32 smithi093 ceph-mon[111335]: pgmap v3660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:32 smithi093 ceph-mon[118902]: pgmap v3660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:33.743 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:33.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:33.449676+0000) 2023-08-27T22:55:34.034 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3953962551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:34.035 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3953962551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3953962551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:34.034Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:34.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:34.751 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:34.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:34.450103+0000) 2023-08-27T22:55:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:34 smithi136 ceph-mon[113531]: pgmap v3661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:34 smithi093 ceph-mon[111335]: pgmap v3661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:34 smithi093 ceph-mon[118902]: pgmap v3661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:55:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:55:35] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:55:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:35.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:35.450403+0000) 2023-08-27T22:55:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3322149795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3322149795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3322149795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:36.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:36.450655+0000) 2023-08-27T22:55:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:37 smithi136 ceph-mon[113531]: pgmap v3662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:37 smithi093 ceph-mon[111335]: pgmap v3662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:37 smithi093 ceph-mon[118902]: pgmap v3662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:37.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:37.450888+0000) 2023-08-27T22:55:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:38.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:38.451089+0000) 2023-08-27T22:55:39.451 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:39 smithi136 ceph-mon[113531]: pgmap v3663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:39.451 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3909766738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:39.451 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:39 smithi093 ceph-mon[111335]: pgmap v3663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3909766738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:39 smithi093 ceph-mon[118902]: pgmap v3663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3909766738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:39.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:39.451318+0000) 2023-08-27T22:55:40.736 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:40.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:40.451558+0000) 2023-08-27T22:55:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:40 smithi093 ceph-mon[111335]: pgmap v3664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:40 smithi093 ceph-mon[118902]: pgmap v3664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:40.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:40.615Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:40 smithi136 ceph-mon[113531]: pgmap v3664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:41.738 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:41.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:41.451811+0000) 2023-08-27T22:55:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2516493898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2516493898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2516493898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:42.110Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:42.111Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:42.745 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:42.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:42.452027+0000) 2023-08-27T22:55:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:42 smithi136 ceph-mon[113531]: pgmap v3665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:42 smithi093 ceph-mon[111335]: pgmap v3665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:42 smithi093 ceph-mon[118902]: pgmap v3665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:43.745 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:43.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:43.452254+0000) 2023-08-27T22:55:44.035 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3601296731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:44.035 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3601296731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3601296731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:44.035Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:44.036Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:44.754 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:44.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:44.452457+0000) 2023-08-27T22:55:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:44 smithi136 ceph-mon[113531]: pgmap v3666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:44 smithi093 ceph-mon[111335]: pgmap v3666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:44 smithi093 ceph-mon[118902]: pgmap v3666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:55:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:55:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:55:45.755 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:45.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:45.452721+0000) 2023-08-27T22:55:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/693407407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:47.270 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/693407407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:47.270 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:46.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:46.452968+0000) 2023-08-27T22:55:47.271 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/693407407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:47.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:46 smithi136 ceph-mon[113531]: pgmap v3667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:47.592 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:46 smithi093 ceph-mon[111335]: pgmap v3667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:47.592 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:46 smithi093 ceph-mon[118902]: pgmap v3667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:47.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:47.453200+0000) 2023-08-27T22:55:48.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1825216773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1825216773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1825216773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:48.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:48.453373+0000) 2023-08-27T22:55:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:48 smithi093 ceph-mon[111335]: pgmap v3668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:48 smithi093 ceph-mon[118902]: pgmap v3668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:48 smithi136 ceph-mon[113531]: pgmap v3668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:49.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:49.453581+0000) 2023-08-27T22:55:50.738 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:50.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:50.453841+0000) 2023-08-27T22:55:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:50 smithi093 ceph-mon[111335]: pgmap v3669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2231773044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:50 smithi093 ceph-mon[118902]: pgmap v3669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2231773044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:50.615Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:50.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:50 smithi136 ceph-mon[113531]: pgmap v3669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2231773044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:51.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:51.454080+0000) 2023-08-27T22:55:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:52.110Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:52.111Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:52.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:52.454362+0000) 2023-08-27T22:55:53.454 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:53 smithi136 ceph-mon[113531]: pgmap v3670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:53.455 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2423436540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:53 smithi093 ceph-mon[111335]: pgmap v3670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2423436540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:53 smithi093 ceph-mon[118902]: pgmap v3670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2423436540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:53.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:53.454603+0000) 2023-08-27T22:55:54.454 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:55:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:54.035Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:55:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:55:54.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:55:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:54.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:54.454774+0000) 2023-08-27T22:55:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:55 smithi093 ceph-mon[118902]: pgmap v3671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3523343247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:55 smithi093 ceph-mon[111335]: pgmap v3671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3523343247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:55:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:55:55] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:55:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:55 smithi136 ceph-mon[113531]: pgmap v3671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3523343247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:55.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:55.454969+0000) 2023-08-27T22:55:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:56.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:56.455224+0000) 2023-08-27T22:55:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:57 smithi093 ceph-mon[111335]: pgmap v3672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:57 smithi093 ceph-mon[118902]: pgmap v3672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:57 smithi136 ceph-mon[113531]: pgmap v3672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:57.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:57.455508+0000) 2023-08-27T22:55:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/178993202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/178993202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/178993202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:55:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:58.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:58.455761+0000) 2023-08-27T22:55:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:55:59 smithi093 ceph-mon[111335]: pgmap v3673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:55:59 smithi093 ceph-mon[118902]: pgmap v3673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:55:59 smithi136 ceph-mon[113531]: pgmap v3673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:55:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:55:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:55:59.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:55:59.455992+0000) 2023-08-27T22:56:00.743 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:00.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:00.456213+0000) 2023-08-27T22:56:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1675154309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:00 smithi093 ceph-mon[111335]: pgmap v3674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1675154309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:00 smithi093 ceph-mon[118902]: pgmap v3674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:00.616Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:00.617Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1675154309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:00 smithi136 ceph-mon[113531]: pgmap v3674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:01.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:01.456466+0000) 2023-08-27T22:56:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3852356600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3852356600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:02.111Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:02.112Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3852356600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:02.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:02.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:02.456721+0000) 2023-08-27T22:56:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:03 smithi136 ceph-mon[113531]: pgmap v3675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:03.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:03.456996+0000) 2023-08-27T22:56:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:03 smithi093 ceph-mon[111335]: pgmap v3675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:03 smithi093 ceph-mon[118902]: pgmap v3675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:04.036Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:04.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:04.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:04.457261+0000) 2023-08-27T22:56:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:05 smithi136 ceph-mon[113531]: pgmap v3676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2909220636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:05.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:05.457537+0000) 2023-08-27T22:56:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:05 smithi093 ceph-mon[111335]: pgmap v3676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2909220636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:05 smithi093 ceph-mon[118902]: pgmap v3676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2909220636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:56:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:56:05] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:56:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:06.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:06.457802+0000) 2023-08-27T22:56:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:07 smithi136 ceph-mon[113531]: pgmap v3677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2958995852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:07.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:07.458104+0000) 2023-08-27T22:56:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:07 smithi093 ceph-mon[111335]: pgmap v3677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2958995852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:07 smithi093 ceph-mon[118902]: pgmap v3677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2958995852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:08.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:08.458333+0000) 2023-08-27T22:56:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:09 smithi136 ceph-mon[113531]: pgmap v3678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:09.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:09.458596+0000) 2023-08-27T22:56:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:09 smithi093 ceph-mon[111335]: pgmap v3678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:09 smithi093 ceph-mon[118902]: pgmap v3678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:10.617 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/798142517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:10.618 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/798142517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/798142517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:10.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:10.458839+0000) 2023-08-27T22:56:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:10.617Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:10.618Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:11 smithi136 ceph-mon[113531]: pgmap v3679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:11.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:11.459113+0000) 2023-08-27T22:56:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:11 smithi093 ceph-mon[111335]: pgmap v3679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:11 smithi093 ceph-mon[118902]: pgmap v3679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/828493038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/828493038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:12.112Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:12.113Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/828493038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:12.726 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:12.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:12.459368+0000) 2023-08-27T22:56:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:13 smithi136 ceph-mon[113531]: pgmap v3680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:13.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:13.459600+0000) 2023-08-27T22:56:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:13 smithi093 ceph-mon[111335]: pgmap v3680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:13 smithi093 ceph-mon[118902]: pgmap v3680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2860806691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2860806691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:14.036Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:14.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2860806691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:14.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:14.459837+0000) 2023-08-27T22:56:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:15 smithi136 ceph-mon[113531]: pgmap v3681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:15.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:15.460054+0000) 2023-08-27T22:56:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:15 smithi093 ceph-mon[111335]: pgmap v3681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:15 smithi093 ceph-mon[118902]: pgmap v3681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:56:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:56:15] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:56:16.665 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:56:16.666 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:16.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:16.460190+0000) 2023-08-27T22:56:16.724 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:56:16.724 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:56:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:17 smithi136 ceph-mon[113531]: pgmap v3682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/943756483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[118902]: pgmap v3682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/943756483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[111335]: pgmap v3682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/943756483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:17.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:17.460323+0000) 2023-08-27T22:56:18.799 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.799 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.799 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.799 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.799 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-mon[113531]: pgmap v3683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:18.800 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.800 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.800 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:18.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:18.460491+0000) 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[111335]: pgmap v3683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[118902]: pgmap v3683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:19.763 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1745163514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:19.763 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:19.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:19.460917+0000) 2023-08-27T22:56:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1745163514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1745163514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:20.748 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:20.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:20.461085+0000) 2023-08-27T22:56:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:20.617Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:20.618Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[111335]: pgmap v3684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[118902]: pgmap v3684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:20 smithi136 ceph-mon[113531]: pgmap v3684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:56:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:56:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:56:21.747 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:21.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:21.461259+0000) 2023-08-27T22:56:22.114 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/10949737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:22.114 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/10949737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/10949737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:22.113Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:22.114Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:22.755 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:22.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:22.461516+0000) 2023-08-27T22:56:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:22 smithi136 ceph-mon[113531]: pgmap v3685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:22 smithi093 ceph-mon[111335]: pgmap v3685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:22 smithi093 ceph-mon[118902]: pgmap v3685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:23.761 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:23.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:23.461751+0000) 2023-08-27T22:56:24.037 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2687702503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:24.038 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:24.038 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2687702503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:24.038 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2687702503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:24.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:24.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:24.767 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:24.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:24.461951+0000) 2023-08-27T22:56:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:24 smithi136 ceph-mon[113531]: pgmap v3686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:24 smithi093 ceph-mon[111335]: pgmap v3686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:24 smithi093 ceph-mon[118902]: pgmap v3686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:56:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:56:25] "GET /metrics HTTP/1.1" 200 239910 "" "Prometheus/2.43.0" 2023-08-27T22:56:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:25.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:25.462276+0000) 2023-08-27T22:56:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2439071532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2439071532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2439071532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:26.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:26.462552+0000) 2023-08-27T22:56:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:27 smithi136 ceph-mon[113531]: pgmap v3687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:27 smithi093 ceph-mon[111335]: pgmap v3687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:27 smithi093 ceph-mon[118902]: pgmap v3687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:27.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:27.462790+0000) 2023-08-27T22:56:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:28.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:28.462974+0000) 2023-08-27T22:56:29.463 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:29 smithi136 ceph-mon[113531]: pgmap v3688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:29.463 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1636897180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:29 smithi093 ceph-mon[111335]: pgmap v3688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1636897180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:29 smithi093 ceph-mon[118902]: pgmap v3688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1636897180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:29.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:29.463215+0000) 2023-08-27T22:56:30.749 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:30.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:30.463489+0000) 2023-08-27T22:56:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:30 smithi093 ceph-mon[111335]: pgmap v3689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:30 smithi093 ceph-mon[118902]: pgmap v3689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:30.617Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:30.618Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:30 smithi136 ceph-mon[113531]: pgmap v3689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:31.750 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:31.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:31.463746+0000) 2023-08-27T22:56:32.114 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/63802429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:32.115 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/63802429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/63802429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:32.114Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:32.115Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:32.754 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:32.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:32.463989+0000) 2023-08-27T22:56:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:32 smithi136 ceph-mon[113531]: pgmap v3690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:32 smithi093 ceph-mon[111335]: pgmap v3690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:32 smithi093 ceph-mon[118902]: pgmap v3690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:33.756 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:33.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:33.464191+0000) 2023-08-27T22:56:34.038 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2444122669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:34.038 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2444122669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2444122669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:34.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:34.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:34.759 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:34.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:34.464462+0000) 2023-08-27T22:56:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:34 smithi136 ceph-mon[113531]: pgmap v3691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:34 smithi093 ceph-mon[111335]: pgmap v3691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:34 smithi093 ceph-mon[118902]: pgmap v3691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:56:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:56:35] "GET /metrics HTTP/1.1" 200 239910 "" "Prometheus/2.43.0" 2023-08-27T22:56:35.761 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:35.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:35.464751+0000) 2023-08-27T22:56:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3714147606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3714147606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3714147606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:36.765 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:36.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:36.464976+0000) 2023-08-27T22:56:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:36 smithi136 ceph-mon[113531]: pgmap v3692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:36 smithi093 ceph-mon[111335]: pgmap v3692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:36 smithi093 ceph-mon[118902]: pgmap v3692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:37.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:37.465174+0000) 2023-08-27T22:56:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1602544146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1602544146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1602544146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:38.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:38.465404+0000) 2023-08-27T22:56:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:39 smithi136 ceph-mon[113531]: pgmap v3693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:39 smithi093 ceph-mon[118902]: pgmap v3693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:39 smithi093 ceph-mon[111335]: pgmap v3693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:39.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:39.465691+0000) 2023-08-27T22:56:40.750 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:40.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:40.465987+0000) 2023-08-27T22:56:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:40 smithi093 ceph-mon[111335]: pgmap v3694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3401762950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:40 smithi093 ceph-mon[118902]: pgmap v3694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3401762950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:40.618Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:40.619Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:40 smithi136 ceph-mon[113531]: pgmap v3694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3401762950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:41.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:41.466131+0000) 2023-08-27T22:56:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:42.115Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:42.116Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:42.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:42.466352+0000) 2023-08-27T22:56:43.466 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:43 smithi136 ceph-mon[113531]: pgmap v3695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:43.467 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3071554209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:43 smithi093 ceph-mon[111335]: pgmap v3695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3071554209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:43 smithi093 ceph-mon[118902]: pgmap v3695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3071554209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:43.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:43.466536+0000) 2023-08-27T22:56:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:44.038Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:44.039Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:44.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:44.466778+0000) 2023-08-27T22:56:45.467 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:45 smithi136 ceph-mon[113531]: pgmap v3696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:45.467 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1010228099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:45 smithi093 ceph-mon[118902]: pgmap v3696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1010228099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:45 smithi093 ceph-mon[111335]: pgmap v3696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1010228099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:56:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:56:45] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:56:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:45.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:45.467060+0000) 2023-08-27T22:56:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:46.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:46.467348+0000) 2023-08-27T22:56:47.467 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:47 smithi136 ceph-mon[113531]: pgmap v3697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:47 smithi093 ceph-mon[111335]: pgmap v3697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:47 smithi093 ceph-mon[118902]: pgmap v3697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:47.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:47.467607+0000) 2023-08-27T22:56:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3272205163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3272205163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3272205163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:48.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:48.467857+0000) 2023-08-27T22:56:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:49 smithi093 ceph-mon[111335]: pgmap v3698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:49 smithi093 ceph-mon[118902]: pgmap v3698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:49 smithi136 ceph-mon[113531]: pgmap v3698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:49.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:49.468110+0000) 2023-08-27T22:56:50.750 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:50.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:50.468394+0000) 2023-08-27T22:56:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/223425475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:50 smithi093 ceph-mon[111335]: pgmap v3699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/223425475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:50.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:50 smithi093 ceph-mon[118902]: pgmap v3699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:50.619Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:50.621Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/223425475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:50 smithi136 ceph-mon[113531]: pgmap v3699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:51.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:51.468586+0000) 2023-08-27T22:56:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:52.116Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:52.117Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:52.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:52.468805+0000) 2023-08-27T22:56:53.469 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:53 smithi136 ceph-mon[113531]: pgmap v3700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:53.469 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2056941187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:53 smithi093 ceph-mon[111335]: pgmap v3700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2056941187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:53 smithi093 ceph-mon[118902]: pgmap v3700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2056941187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:53.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:53.469060+0000) 2023-08-27T22:56:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:54.039Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:56:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:56:54.040Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:56:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:56:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:54.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:54.469278+0000) 2023-08-27T22:56:55.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:56:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:56:55] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:56:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:55 smithi093 ceph-mon[111335]: pgmap v3701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2136421824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:55 smithi093 ceph-mon[118902]: pgmap v3701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2136421824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:55 smithi136 ceph-mon[113531]: pgmap v3701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2136421824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:55.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:55.469483+0000) 2023-08-27T22:56:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:56.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:56.469688+0000) 2023-08-27T22:56:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:57 smithi093 ceph-mon[111335]: pgmap v3702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1349873835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:57 smithi093 ceph-mon[118902]: pgmap v3702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1349873835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:57 smithi136 ceph-mon[113531]: pgmap v3702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1349873835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:56:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:57.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:57.469951+0000) 2023-08-27T22:56:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:58.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:58.470209+0000) 2023-08-27T22:56:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:56:59 smithi093 ceph-mon[111335]: pgmap v3703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:56:59 smithi093 ceph-mon[118902]: pgmap v3703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:56:59 smithi136 ceph-mon[113531]: pgmap v3703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:56:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:56:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:56:59.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:56:59.470476+0000) 2023-08-27T22:57:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3287391772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3287391772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3287391772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:00.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:00.470704+0000) 2023-08-27T22:57:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:00.620Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:00.622Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:01 smithi093 ceph-mon[111335]: pgmap v3704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:01 smithi093 ceph-mon[118902]: pgmap v3704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:01.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:01 smithi136 ceph-mon[113531]: pgmap v3704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:01.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:01.470943+0000) 2023-08-27T22:57:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1777006941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1777006941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:02.116Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:02.117Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1777006941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:02.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:02.471205+0000) 2023-08-27T22:57:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:03 smithi136 ceph-mon[113531]: pgmap v3705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:03.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:03.471419+0000) 2023-08-27T22:57:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:03 smithi093 ceph-mon[111335]: pgmap v3705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:03 smithi093 ceph-mon[118902]: pgmap v3705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3481064103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3481064103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:04.039Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:04.040Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3481064103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:04.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:04.471637+0000) 2023-08-27T22:57:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:05 smithi136 ceph-mon[113531]: pgmap v3706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:05.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:05.471831+0000) 2023-08-27T22:57:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:05 smithi093 ceph-mon[118902]: pgmap v3706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:05 smithi093 ceph-mon[111335]: pgmap v3706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:57:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:57:05] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:57:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:06.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:06.472088+0000) 2023-08-27T22:57:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:07 smithi136 ceph-mon[113531]: pgmap v3707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2456316816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:07.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:07.472292+0000) 2023-08-27T22:57:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:07 smithi093 ceph-mon[111335]: pgmap v3707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2456316816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:07 smithi093 ceph-mon[118902]: pgmap v3707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2456316816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:08.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:08.472505+0000) 2023-08-27T22:57:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:09.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:09.472736+0000) 2023-08-27T22:57:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:09 smithi136 ceph-mon[113531]: pgmap v3708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3873834061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:09 smithi093 ceph-mon[111335]: pgmap v3708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3873834061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:09 smithi093 ceph-mon[118902]: pgmap v3708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3873834061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:10.756 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:10.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:10.472965+0000) 2023-08-27T22:57:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:10 smithi093 ceph-mon[111335]: pgmap v3709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:10 smithi093 ceph-mon[118902]: pgmap v3709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:10.621Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:10.622Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:10 smithi136 ceph-mon[113531]: pgmap v3709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:11.757 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:11.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:11.473209+0000) 2023-08-27T22:57:12.117 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2367249700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:12.118 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2367249700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2367249700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:12.117Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:12.118Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:12.762 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:12.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:12.473454+0000) 2023-08-27T22:57:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:12 smithi136 ceph-mon[113531]: pgmap v3710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:12 smithi093 ceph-mon[111335]: pgmap v3710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:12 smithi093 ceph-mon[118902]: pgmap v3710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:13.763 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:13.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:13.473711+0000) 2023-08-27T22:57:14.039 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1838413257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:14.040 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1838413257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1838413257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:14.039Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:14.040Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:14.765 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:14.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:14.473976+0000) 2023-08-27T22:57:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:14 smithi136 ceph-mon[113531]: pgmap v3711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:14 smithi093 ceph-mon[111335]: pgmap v3711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:14 smithi093 ceph-mon[118902]: pgmap v3711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:57:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:57:15] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:57:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:15.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:15.474228+0000) 2023-08-27T22:57:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/221455470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/221455470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/221455470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:16.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:16.474411+0000) 2023-08-27T22:57:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:17 smithi136 ceph-mon[113531]: pgmap v3712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:17 smithi093 ceph-mon[111335]: pgmap v3712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:17 smithi093 ceph-mon[118902]: pgmap v3712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:17.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:17.474597+0000) 2023-08-27T22:57:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:18.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:18.474851+0000) 2023-08-27T22:57:19.475 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:19 smithi136 ceph-mon[113531]: pgmap v3713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:19.475 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3618325693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:19 smithi093 ceph-mon[111335]: pgmap v3713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3618325693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:19 smithi093 ceph-mon[118902]: pgmap v3713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3618325693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:19.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:19.475114+0000) 2023-08-27T22:57:20.753 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:20.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:20.475299+0000) 2023-08-27T22:57:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:20 smithi093 ceph-mon[111335]: pgmap v3714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:57:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:20 smithi093 ceph-mon[118902]: pgmap v3714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:57:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:20.622Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:20.622Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:21.110 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:20 smithi136 ceph-mon[113531]: pgmap v3714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:21.110 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:57:21.754 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:21.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:21.475449+0000) 2023-08-27T22:57:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4005375697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4005375697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4005375697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:22.118Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:22.119Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:22.673 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:22.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:22.475674+0000) 2023-08-27T22:57:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:22 smithi136 ceph-mon[113531]: pgmap v3715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-mon[111335]: pgmap v3715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-mon[118902]: pgmap v3715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:23.936 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:23.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:23.475879+0000) 2023-08-27T22:57:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1765553797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:57:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:57:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1765553797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1765553797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:57:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:57:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:57:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:24.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:24.040Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:24.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:24.041Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:24.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:24.476135+0000) 2023-08-27T22:57:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:24 smithi093 ceph-mon[111335]: pgmap v3716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:24 smithi093 ceph-mon[118902]: pgmap v3716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:24 smithi136 ceph-mon[113531]: pgmap v3716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:57:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:57:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:57:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:25.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:25.476446+0000) 2023-08-27T22:57:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1660492192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1660492192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1660492192' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:26.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:26.476696+0000) 2023-08-27T22:57:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:26 smithi093 ceph-mon[118902]: pgmap v3717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:26 smithi093 ceph-mon[111335]: pgmap v3717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:26 smithi136 ceph-mon[113531]: pgmap v3717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:27.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:27.476951+0000) 2023-08-27T22:57:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1448178189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1448178189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1448178189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:28.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:28.477221+0000) 2023-08-27T22:57:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:29 smithi136 ceph-mon[113531]: pgmap v3718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:29 smithi093 ceph-mon[111335]: pgmap v3718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:29.527 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:29 smithi093 ceph-mon[118902]: pgmap v3718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:29.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:29.477409+0000) 2023-08-27T22:57:30.754 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:30.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:30.477642+0000) 2023-08-27T22:57:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:30 smithi093 ceph-mon[118902]: pgmap v3719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3187297151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:30 smithi093 ceph-mon[111335]: pgmap v3719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3187297151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:30.622Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:30.623Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:30 smithi136 ceph-mon[113531]: pgmap v3719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3187297151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:31.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:31.477886+0000) 2023-08-27T22:57:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:32.119Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:32.120Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:32.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:32.478177+0000) 2023-08-27T22:57:33.478 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:33 smithi136 ceph-mon[113531]: pgmap v3720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:33.478 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3578996920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:33 smithi093 ceph-mon[111335]: pgmap v3720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3578996920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:33 smithi093 ceph-mon[118902]: pgmap v3720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3578996920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:33.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:33.478368+0000) 2023-08-27T22:57:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:34.041Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:34.042Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:34.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:34.478602+0000) 2023-08-27T22:57:35.479 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:35 smithi136 ceph-mon[113531]: pgmap v3721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:35.479 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3083413840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:35 smithi093 ceph-mon[111335]: pgmap v3721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3083413840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:57:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:57:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T22:57:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:35 smithi093 ceph-mon[118902]: pgmap v3721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3083413840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:35.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:35.478891+0000) 2023-08-27T22:57:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:36.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:36.479107+0000) 2023-08-27T22:57:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:37 smithi093 ceph-mon[111335]: pgmap v3722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:37 smithi093 ceph-mon[118902]: pgmap v3722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:37 smithi136 ceph-mon[113531]: pgmap v3722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:37.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:37.479348+0000) 2023-08-27T22:57:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2451349221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2451349221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2451349221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:38.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:38.479555+0000) 2023-08-27T22:57:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:39 smithi093 ceph-mon[111335]: pgmap v3723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:39 smithi093 ceph-mon[118902]: pgmap v3723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:39 smithi136 ceph-mon[113531]: pgmap v3723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:39.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:39.479789+0000) 2023-08-27T22:57:40.760 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:40.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:40.480070+0000) 2023-08-27T22:57:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2190777255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:40 smithi093 ceph-mon[111335]: pgmap v3724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2190777255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:40 smithi093 ceph-mon[118902]: pgmap v3724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:40.623Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:40.624Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2190777255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:40 smithi136 ceph-mon[113531]: pgmap v3724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:41.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:41.480317+0000) 2023-08-27T22:57:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:42.120Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:42.544 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:42.121Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:42.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:42.480593+0000) 2023-08-27T22:57:43.481 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:43 smithi136 ceph-mon[113531]: pgmap v3725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:43.481 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1903541586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:43 smithi093 ceph-mon[111335]: pgmap v3725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1903541586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:43 smithi093 ceph-mon[118902]: pgmap v3725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1903541586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:43.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:43.480858+0000) 2023-08-27T22:57:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:44.041Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:44.042Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:44.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:44.481168+0000) 2023-08-27T22:57:45.481 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:45 smithi136 ceph-mon[113531]: pgmap v3726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:45.481 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1588374687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:45 smithi093 ceph-mon[118902]: pgmap v3726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1588374687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:45 smithi093 ceph-mon[111335]: pgmap v3726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1588374687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:57:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:57:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T22:57:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:45.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:45.481427+0000) 2023-08-27T22:57:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:46.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:46.481656+0000) 2023-08-27T22:57:47.482 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:47 smithi136 ceph-mon[113531]: pgmap v3727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:47.482 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4103502004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:47 smithi093 ceph-mon[111335]: pgmap v3727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4103502004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:47 smithi093 ceph-mon[118902]: pgmap v3727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4103502004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:47.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:47.481902+0000) 2023-08-27T22:57:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:48.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:48.482153+0000) 2023-08-27T22:57:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:49 smithi093 ceph-mon[111335]: pgmap v3728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:49 smithi093 ceph-mon[118902]: pgmap v3728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:49 smithi136 ceph-mon[113531]: pgmap v3728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:49.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:49.482422+0000) 2023-08-27T22:57:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/654719674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/654719674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/654719674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:50.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:50.482686+0000) 2023-08-27T22:57:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:50.624Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:50.625Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:51 smithi136 ceph-mon[113531]: pgmap v3729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:51.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:51.482937+0000) 2023-08-27T22:57:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:51 smithi093 ceph-mon[111335]: pgmap v3729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:51 smithi093 ceph-mon[118902]: pgmap v3729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/768320568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/768320568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:52.121Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:52.122Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/768320568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:52.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:52.483207+0000) 2023-08-27T22:57:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:53 smithi136 ceph-mon[113531]: pgmap v3730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:53.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:53.483421+0000) 2023-08-27T22:57:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:53 smithi093 ceph-mon[111335]: pgmap v3730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:53 smithi093 ceph-mon[118902]: pgmap v3730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3715654894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3715654894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:54.042Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:57:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:57:54.043Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:57:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:57:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3715654894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:54.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:54.483659+0000) 2023-08-27T22:57:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:55 smithi136 ceph-mon[113531]: pgmap v3731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:55.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:55.483917+0000) 2023-08-27T22:57:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:55 smithi093 ceph-mon[111335]: pgmap v3731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:55 smithi093 ceph-mon[118902]: pgmap v3731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:57:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:57:55] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:57:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:56.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:56.484158+0000) 2023-08-27T22:57:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:57.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:57.484388+0000) 2023-08-27T22:57:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:57 smithi136 ceph-mon[113531]: pgmap v3732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3132535088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:57 smithi093 ceph-mon[111335]: pgmap v3732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3132535088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:57 smithi093 ceph-mon[118902]: pgmap v3732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3132535088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:58.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:58.484619+0000) 2023-08-27T22:57:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:59 smithi136 ceph-mon[113531]: pgmap v3733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:57:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2461109620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:57:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:57:59.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:57:59.484877+0000) 2023-08-27T22:57:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:59 smithi093 ceph-mon[111335]: pgmap v3733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:57:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2461109620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:57:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:59 smithi093 ceph-mon[118902]: pgmap v3733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:57:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:57:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2461109620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:00.766 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:00.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:00.485126+0000) 2023-08-27T22:58:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:00 smithi093 ceph-mon[111335]: pgmap v3734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:00.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:00 smithi093 ceph-mon[118902]: pgmap v3734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:00.625Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:00.626Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:00 smithi136 ceph-mon[113531]: pgmap v3734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:01.767 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:01.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:01.485353+0000) 2023-08-27T22:58:02.122 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1609395180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:02.122 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1609395180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1609395180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:02.122Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:02.122Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:02.772 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:02.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:02.485623+0000) 2023-08-27T22:58:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:02 smithi136 ceph-mon[113531]: pgmap v3735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:02 smithi093 ceph-mon[111335]: pgmap v3735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:02 smithi093 ceph-mon[118902]: pgmap v3735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:03.773 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:03.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:03.485862+0000) 2023-08-27T22:58:04.043 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3940069320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:04.043 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3940069320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3940069320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:04.042Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:04.043Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:04.781 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:04.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:04.486140+0000) 2023-08-27T22:58:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:04 smithi136 ceph-mon[113531]: pgmap v3736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:04 smithi093 ceph-mon[111335]: pgmap v3736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:04 smithi093 ceph-mon[118902]: pgmap v3736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:58:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:58:05] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T22:58:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:05.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:05.486386+0000) 2023-08-27T22:58:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3567746094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3567746094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3567746094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:06.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:06.486682+0000) 2023-08-27T22:58:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:07 smithi136 ceph-mon[113531]: pgmap v3737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:07 smithi093 ceph-mon[111335]: pgmap v3737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:07 smithi093 ceph-mon[118902]: pgmap v3737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:07.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:07.486905+0000) 2023-08-27T22:58:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:08.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:08.487166+0000) 2023-08-27T22:58:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:09 smithi136 ceph-mon[113531]: pgmap v3738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3462921388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:09 smithi093 ceph-mon[111335]: pgmap v3738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3462921388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:09 smithi093 ceph-mon[118902]: pgmap v3738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3462921388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:09.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:09.487392+0000) 2023-08-27T22:58:10.765 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:10.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:10.487672+0000) 2023-08-27T22:58:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:10 smithi093 ceph-mon[111335]: pgmap v3739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:10 smithi093 ceph-mon[118902]: pgmap v3739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:10.625Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:10.626Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:10 smithi136 ceph-mon[113531]: pgmap v3739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:11.767 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:11.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:11.487929+0000) 2023-08-27T22:58:12.123 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4076982902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:12.123 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4076982902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4076982902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:12.122Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:12.123Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:12.773 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:12.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:12.488177+0000) 2023-08-27T22:58:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:12 smithi136 ceph-mon[113531]: pgmap v3740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:12 smithi093 ceph-mon[111335]: pgmap v3740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:12 smithi093 ceph-mon[118902]: pgmap v3740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:13.775 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:13.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:13.488417+0000) 2023-08-27T22:58:14.043 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2289930571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:14.043 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2289930571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2289930571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:14.043Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:14.044Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:14.781 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:14.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:14.488677+0000) 2023-08-27T22:58:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:14 smithi136 ceph-mon[113531]: pgmap v3741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:14 smithi093 ceph-mon[111335]: pgmap v3741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:14 smithi093 ceph-mon[118902]: pgmap v3741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:58:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:58:15] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:58:15.780 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:15.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:15.488899+0000) 2023-08-27T22:58:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2157793149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2157793149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2157793149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:16.787 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:16.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:16.489137+0000) 2023-08-27T22:58:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:16 smithi136 ceph-mon[113531]: pgmap v3742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:16 smithi093 ceph-mon[111335]: pgmap v3742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:16 smithi093 ceph-mon[118902]: pgmap v3742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:17.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:17.489360+0000) 2023-08-27T22:58:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3880749303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3880749303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3880749303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:18.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:18.489646+0000) 2023-08-27T22:58:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:19 smithi136 ceph-mon[113531]: pgmap v3743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:19 smithi093 ceph-mon[111335]: pgmap v3743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:19 smithi093 ceph-mon[118902]: pgmap v3743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:19.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:19.489900+0000) 2023-08-27T22:58:20.763 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:20.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:20.490177+0000) 2023-08-27T22:58:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:20 smithi093 ceph-mon[111335]: pgmap v3744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/872777329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:20 smithi093 ceph-mon[118902]: pgmap v3744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/872777329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:20.626Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:20.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:20 smithi136 ceph-mon[113531]: pgmap v3744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/872777329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:21.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:21.490407+0000) 2023-08-27T22:58:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:22.123Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:22.124Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:22.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:22.490644+0000) 2023-08-27T22:58:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:23 smithi093 ceph-mon[111335]: pgmap v3745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2932579080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:23 smithi093 ceph-mon[118902]: pgmap v3745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2932579080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:23.490 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:23 smithi136 ceph-mon[113531]: pgmap v3745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:23.491 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2932579080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:23.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:23.490882+0000) 2023-08-27T22:58:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:58:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:58:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:58:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:24.043Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:24.044Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:24.874 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:24.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:24.491108+0000) 2023-08-27T22:58:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:25 smithi093 ceph-mon[111335]: pgmap v3746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3342050957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:25 smithi093 ceph-mon[118902]: pgmap v3746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3342050957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:25.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:25 smithi136 ceph-mon[113531]: pgmap v3746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:25.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3342050957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:25.568 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:25.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:25.491306+0000) 2023-08-27T22:58:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:58:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:58:25] "GET /metrics HTTP/1.1" 200 239908 "" "Prometheus/2.43.0" 2023-08-27T22:58:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:26.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:26.491562+0000) 2023-08-27T22:58:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:58:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:58:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:58:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:58:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:58:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:27 smithi136 ceph-mon[113531]: pgmap v3747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:27.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:27.491774+0000) 2023-08-27T22:58:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:27 smithi093 ceph-mon[118902]: pgmap v3747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:27 smithi093 ceph-mon[111335]: pgmap v3747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2457912638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:28.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:28.491985+0000) 2023-08-27T22:58:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2457912638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2457912638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:29 smithi136 ceph-mon[113531]: pgmap v3748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:29.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:29.492230+0000) 2023-08-27T22:58:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:29 smithi093 ceph-mon[111335]: pgmap v3748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:29 smithi093 ceph-mon[118902]: pgmap v3748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:30.766 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:30.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:30.492460+0000) 2023-08-27T22:58:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2024208603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:30 smithi093 ceph-mon[118902]: pgmap v3749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2024208603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:30 smithi093 ceph-mon[111335]: pgmap v3749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:30.627Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:30.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2024208603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:30 smithi136 ceph-mon[113531]: pgmap v3749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:31.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:31.492697+0000) 2023-08-27T22:58:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:32.124Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:32.125Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:32.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:32.492957+0000) 2023-08-27T22:58:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:33 smithi093 ceph-mon[111335]: pgmap v3750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/751285276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:33 smithi093 ceph-mon[118902]: pgmap v3750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/751285276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:33.493 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:33 smithi136 ceph-mon[113531]: pgmap v3750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:33.493 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/751285276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:33.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:33.493200+0000) 2023-08-27T22:58:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:34.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:34.045Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:34.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:34.493394+0000) 2023-08-27T22:58:35.493 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:35 smithi136 ceph-mon[113531]: pgmap v3751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:35.494 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3846822199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:35 smithi093 ceph-mon[111335]: pgmap v3751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3846822199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:35 smithi093 ceph-mon[118902]: pgmap v3751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3846822199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:58:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:58:35] "GET /metrics HTTP/1.1" 200 239908 "" "Prometheus/2.43.0" 2023-08-27T22:58:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:35.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:35.493635+0000) 2023-08-27T22:58:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:36.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:36.493817+0000) 2023-08-27T22:58:37.494 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:37 smithi136 ceph-mon[113531]: pgmap v3752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:37.494 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2989157014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:37 smithi093 ceph-mon[111335]: pgmap v3752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2989157014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:37 smithi093 ceph-mon[118902]: pgmap v3752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2989157014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:37.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:37.494005+0000) 2023-08-27T22:58:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:38.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:38.494241+0000) 2023-08-27T22:58:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:39 smithi136 ceph-mon[113531]: pgmap v3753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:39.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:39.494426+0000) 2023-08-27T22:58:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:39 smithi093 ceph-mon[111335]: pgmap v3753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:39 smithi093 ceph-mon[118902]: pgmap v3753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:40.627 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2553665224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:40.628 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2553665224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2553665224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:40.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:40.494661+0000) 2023-08-27T22:58:40.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:40.627Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:40.628Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:41 smithi136 ceph-mon[113531]: pgmap v3754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:41.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:41.494905+0000) 2023-08-27T22:58:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:41 smithi093 ceph-mon[111335]: pgmap v3754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:41 smithi093 ceph-mon[118902]: pgmap v3754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2713368371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2713368371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:42.124Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:42.125Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2713368371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:42.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:42.495185+0000) 2023-08-27T22:58:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:43 smithi136 ceph-mon[113531]: pgmap v3755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:43.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:43.495448+0000) 2023-08-27T22:58:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:43 smithi093 ceph-mon[111335]: pgmap v3755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:43 smithi093 ceph-mon[118902]: pgmap v3755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:44.045Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:44.046Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:44.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:44.495693+0000) 2023-08-27T22:58:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:45 smithi136 ceph-mon[113531]: pgmap v3756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3400686103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:45.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:45.495955+0000) 2023-08-27T22:58:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:45 smithi093 ceph-mon[118902]: pgmap v3756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3400686103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:45 smithi093 ceph-mon[111335]: pgmap v3756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3400686103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:58:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:58:45] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:58:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:46.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:46.496217+0000) 2023-08-27T22:58:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:47 smithi136 ceph-mon[113531]: pgmap v3757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3978948727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:47.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:47.496541+0000) 2023-08-27T22:58:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:47 smithi093 ceph-mon[111335]: pgmap v3757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3978948727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:47 smithi093 ceph-mon[118902]: pgmap v3757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3978948727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:48.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:48.496788+0000) 2023-08-27T22:58:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:49 smithi136 ceph-mon[113531]: pgmap v3758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1129369359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:49.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:49.497060+0000) 2023-08-27T22:58:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:49 smithi093 ceph-mon[111335]: pgmap v3758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1129369359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:49 smithi093 ceph-mon[118902]: pgmap v3758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1129369359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:50.772 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:50.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:50.497354+0000) 2023-08-27T22:58:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:50 smithi093 ceph-mon[111335]: pgmap v3759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:51.522 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:50 smithi093 ceph-mon[118902]: pgmap v3759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:51.522 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:50.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:51.522 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:50.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:51.523 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:50 smithi136 ceph-mon[113531]: pgmap v3759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:51.772 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:51.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:51.497606+0000) 2023-08-27T22:58:52.125 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1124083772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:52.511 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1124083772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:52.511 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:52.125Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:52.512 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:52.126Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:52.512 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1124083772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:52.777 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:52.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:52.497853+0000) 2023-08-27T22:58:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:52 smithi136 ceph-mon[113531]: pgmap v3760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:52 smithi093 ceph-mon[111335]: pgmap v3760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:53.415 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:52 smithi093 ceph-mon[118902]: pgmap v3760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:53.778 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:53.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:53.498125+0000) 2023-08-27T22:58:54.046 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:54.431 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:54.432 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:58:54.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:54.046Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:55.044 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:58:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:58:54.047Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:58:55.046 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:54.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:54.498362+0000) 2023-08-27T22:58:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/107417746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:55.248 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:54 smithi136 ceph-mon[113531]: pgmap v3761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:55.249 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/107417746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:55.249 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:54 smithi093 ceph-mon[111335]: pgmap v3761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:55.249 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/107417746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:55.250 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:54 smithi093 ceph-mon[118902]: pgmap v3761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:58:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:58:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:58:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:55.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:55.498658+0000) 2023-08-27T22:58:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/160495854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:56.955 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:56.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:56.498895+0000) 2023-08-27T22:58:56.956 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/160495854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:56.956 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/160495854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:57 smithi136 ceph-mon[113531]: pgmap v3762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:57.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:57.499150+0000) 2023-08-27T22:58:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:57 smithi093 ceph-mon[111335]: pgmap v3762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:57 smithi093 ceph-mon[118902]: pgmap v3762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:58.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:58.499409+0000) 2023-08-27T22:58:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:59 smithi136 ceph-mon[113531]: pgmap v3763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:58:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/459738160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:58:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:58:59.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:58:59.499643+0000) 2023-08-27T22:58:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:59 smithi093 ceph-mon[118902]: pgmap v3763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:58:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/459738160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:58:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:59 smithi093 ceph-mon[111335]: pgmap v3763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:58:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:58:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/459738160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:00.771 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:00.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:00.499889+0000) 2023-08-27T22:59:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:00 smithi093 ceph-mon[111335]: pgmap v3764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:00 smithi093 ceph-mon[118902]: pgmap v3764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:00.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:00.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:00 smithi136 ceph-mon[113531]: pgmap v3764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:01.773 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:01.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:01.500153+0000) 2023-08-27T22:59:02.127 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1220183886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:02.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1220183886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1220183886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:02.126Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:02.127Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:02.779 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:02.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:02.500409+0000) 2023-08-27T22:59:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:02 smithi136 ceph-mon[113531]: pgmap v3765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:02 smithi093 ceph-mon[111335]: pgmap v3765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:02 smithi093 ceph-mon[118902]: pgmap v3765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:03.780 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:03.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:03.500663+0000) 2023-08-27T22:59:04.047 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1217762113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:04.047 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1217762113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1217762113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:04.047Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:04.048Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:04.785 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:04.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:04.500930+0000) 2023-08-27T22:59:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:04 smithi136 ceph-mon[113531]: pgmap v3766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:04 smithi093 ceph-mon[111335]: pgmap v3766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:04 smithi093 ceph-mon[118902]: pgmap v3766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:59:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:59:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T22:59:05.787 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:05.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:05.501180+0000) 2023-08-27T22:59:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2635774258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2635774258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2635774258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:06.791 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:06.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:06.501411+0000) 2023-08-27T22:59:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:06 smithi136 ceph-mon[113531]: pgmap v3767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:06 smithi093 ceph-mon[111335]: pgmap v3767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:06 smithi093 ceph-mon[118902]: pgmap v3767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:07.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:07.501636+0000) 2023-08-27T22:59:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2265130041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2265130041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2265130041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:08.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:08.501868+0000) 2023-08-27T22:59:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:09 smithi136 ceph-mon[113531]: pgmap v3768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:09 smithi093 ceph-mon[118902]: pgmap v3768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:09 smithi093 ceph-mon[111335]: pgmap v3768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:09.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:09.502140+0000) 2023-08-27T22:59:10.771 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:10.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:10.502431+0000) 2023-08-27T22:59:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:10 smithi093 ceph-mon[111335]: pgmap v3769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2210941829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:10 smithi093 ceph-mon[118902]: pgmap v3769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2210941829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:10.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:10.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:10 smithi136 ceph-mon[113531]: pgmap v3769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2210941829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:11.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:11.502686+0000) 2023-08-27T22:59:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:12.127Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:12.127Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:12.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:12.502884+0000) 2023-08-27T22:59:13.503 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:13 smithi136 ceph-mon[113531]: pgmap v3770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:13.503 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/299882989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:13 smithi093 ceph-mon[111335]: pgmap v3770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/299882989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:13 smithi093 ceph-mon[118902]: pgmap v3770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/299882989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:13.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:13.503120+0000) 2023-08-27T22:59:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:14.047Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:14.048Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:14.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:14.503383+0000) 2023-08-27T22:59:15.503 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:15 smithi136 ceph-mon[113531]: pgmap v3771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:15 smithi093 ceph-mon[111335]: pgmap v3771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:15 smithi093 ceph-mon[118902]: pgmap v3771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:59:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:59:15] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T22:59:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:15.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:15.503657+0000) 2023-08-27T22:59:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1263969576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:16.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:16.503859+0000) 2023-08-27T22:59:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1263969576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1263969576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:17 smithi136 ceph-mon[113531]: pgmap v3772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:17.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:17.504099+0000) 2023-08-27T22:59:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:17 smithi093 ceph-mon[111335]: pgmap v3772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:17 smithi093 ceph-mon[118902]: pgmap v3772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1212176980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:18.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:18.504339+0000) 2023-08-27T22:59:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1212176980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1212176980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:19 smithi136 ceph-mon[113531]: pgmap v3773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:19.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:19.504601+0000) 2023-08-27T22:59:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:19 smithi093 ceph-mon[111335]: pgmap v3773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:19 smithi093 ceph-mon[118902]: pgmap v3773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:20.775 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:20.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:20.504874+0000) 2023-08-27T22:59:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/465309016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:20 smithi093 ceph-mon[111335]: pgmap v3774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/465309016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:20 smithi093 ceph-mon[118902]: pgmap v3774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:20.629Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:20.630Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/465309016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:20 smithi136 ceph-mon[113531]: pgmap v3774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:21.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:21.505167+0000) 2023-08-27T22:59:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:22.127Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:22.128Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:22.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:22.505439+0000) 2023-08-27T22:59:23.505 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:23 smithi136 ceph-mon[113531]: pgmap v3775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:23.506 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4033630525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:23 smithi093 ceph-mon[118902]: pgmap v3775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4033630525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:23 smithi093 ceph-mon[111335]: pgmap v3775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4033630525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:23.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:23.505610+0000) 2023-08-27T22:59:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:24.048Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:24.049Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:24.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:24.505859+0000) 2023-08-27T22:59:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:25 smithi136 ceph-mon[113531]: pgmap v3776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/117449876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:25.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:25.506183+0000) 2023-08-27T22:59:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:25 smithi093 ceph-mon[111335]: pgmap v3776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/117449876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:25 smithi093 ceph-mon[118902]: pgmap v3776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/117449876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:59:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:59:25] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:59:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:26.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:26.506368+0000) 2023-08-27T22:59:27.410 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:27 smithi136 ceph-mon[113531]: pgmap v3777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:27.410 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:59:27.410 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4020573649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:27.554 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:27 smithi093 ceph-mon[111335]: pgmap v3777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:27.554 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:59:27.554 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4020573649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:27.554 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:27 smithi093 ceph-mon[118902]: pgmap v3777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:27.554 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T22:59:27.554 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4020573649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:27.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:27.506501+0000) 2023-08-27T22:59:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:28.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:28.506751+0000) 2023-08-27T22:59:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:29 smithi136 ceph-mon[113531]: pgmap v3778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[111335]: pgmap v3778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[118902]: pgmap v3778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:29.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:29.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:29.506995+0000) 2023-08-27T22:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1946775469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:59:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:59:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1946775469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T22:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T22:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T22:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1946775469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:30.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:30.507250+0000) 2023-08-27T22:59:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:30.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:30.630Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:31 smithi136 ceph-mon[113531]: pgmap v3779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:31 smithi093 ceph-mon[111335]: pgmap v3779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:31 smithi093 ceph-mon[118902]: pgmap v3779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:31.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:31.507463+0000) 2023-08-27T22:59:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3770403719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3770403719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3770403719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:32.128Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:32.130Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:32.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:32.507729+0000) 2023-08-27T22:59:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:33 smithi136 ceph-mon[113531]: pgmap v3780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:33 smithi093 ceph-mon[111335]: pgmap v3780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:33.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:33 smithi093 ceph-mon[118902]: pgmap v3780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:33.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:33.507970+0000) 2023-08-27T22:59:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:34.049Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:34.050Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:34.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:34.508206+0000) 2023-08-27T22:59:35.508 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:35 smithi136 ceph-mon[113531]: pgmap v3781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:35.509 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4189886924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:59:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:59:35] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:59:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:35 smithi093 ceph-mon[111335]: pgmap v3781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4189886924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:35 smithi093 ceph-mon[118902]: pgmap v3781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4189886924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:35.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:35.508475+0000) 2023-08-27T22:59:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:36.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:36.508701+0000) 2023-08-27T22:59:37.509 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:37 smithi136 ceph-mon[113531]: pgmap v3782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:37.509 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/552661961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:37 smithi093 ceph-mon[118902]: pgmap v3782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/552661961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:37 smithi093 ceph-mon[111335]: pgmap v3782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/552661961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:37.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:37.508964+0000) 2023-08-27T22:59:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:38.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:38.509196+0000) 2023-08-27T22:59:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:39.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:39.509393+0000) 2023-08-27T22:59:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:39 smithi136 ceph-mon[113531]: pgmap v3783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/534042122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:39 smithi093 ceph-mon[111335]: pgmap v3783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/534042122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:39 smithi093 ceph-mon[118902]: pgmap v3783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/534042122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:40.779 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:40.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:40.509641+0000) 2023-08-27T22:59:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:40 smithi093 ceph-mon[111335]: pgmap v3784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:40 smithi093 ceph-mon[118902]: pgmap v3784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:40.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:40.631Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:40 smithi136 ceph-mon[113531]: pgmap v3784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:41.781 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:41.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:41.509895+0000) 2023-08-27T22:59:42.130 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/740819690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:42.130 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/740819690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/740819690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:42.129Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:42.130Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:42.785 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:42.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:42.510181+0000) 2023-08-27T22:59:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:42 smithi136 ceph-mon[113531]: pgmap v3785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:42 smithi093 ceph-mon[111335]: pgmap v3785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:42 smithi093 ceph-mon[118902]: pgmap v3785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:43.928 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:43.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:43.510455+0000) 2023-08-27T22:59:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/804442397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/804442397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/804442397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:44.050Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:44.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:44.051Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:44.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:44.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:44.510704+0000) 2023-08-27T22:59:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:44 smithi136 ceph-mon[113531]: pgmap v3786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:44 smithi093 ceph-mon[111335]: pgmap v3786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:44 smithi093 ceph-mon[118902]: pgmap v3786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:59:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:59:45] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:59:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:45.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:45.510988+0000) 2023-08-27T22:59:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:46.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:46.511263+0000) 2023-08-27T22:59:47.511 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:47 smithi136 ceph-mon[113531]: pgmap v3787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:47.512 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/143341920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:47 smithi093 ceph-mon[111335]: pgmap v3787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/143341920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:47 smithi093 ceph-mon[118902]: pgmap v3787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/143341920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:47.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:47.511528+0000) 2023-08-27T22:59:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:48.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:48.511750+0000) 2023-08-27T22:59:49.512 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:49 smithi136 ceph-mon[113531]: pgmap v3788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:49.512 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1496246832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:49 smithi093 ceph-mon[111335]: pgmap v3788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1496246832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:49 smithi093 ceph-mon[118902]: pgmap v3788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1496246832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:49.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:49.511979+0000) 2023-08-27T22:59:50.782 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:50.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:50.512208+0000) 2023-08-27T22:59:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:50 smithi093 ceph-mon[111335]: pgmap v3789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:50 smithi093 ceph-mon[118902]: pgmap v3789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:50.631Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:50.631Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:50 smithi136 ceph-mon[113531]: pgmap v3789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:51.783 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:51.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:51.512465+0000) 2023-08-27T22:59:52.131 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2222257814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:52.131 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2222257814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2222257814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:52.130Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:52.131Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:52.789 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:52.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:52.512685+0000) 2023-08-27T22:59:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:52 smithi136 ceph-mon[113531]: pgmap v3790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:52 smithi093 ceph-mon[111335]: pgmap v3790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:52 smithi093 ceph-mon[118902]: pgmap v3790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:53.791 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:53.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:53.512953+0000) 2023-08-27T22:59:54.050 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/252498040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:54.051 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:54.051 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/252498040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:54.051 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/252498040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T22:59:54.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:54.050Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 22:59:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T22:59:54.051Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T22:59:54.795 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:54.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:54.513203+0000) 2023-08-27T22:59:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:54 smithi136 ceph-mon[113531]: pgmap v3791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:54 smithi093 ceph-mon[111335]: pgmap v3791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:54 smithi093 ceph-mon[118902]: pgmap v3791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 22:59:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:22:59:55] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T22:59:55.795 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:55.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:55.513472+0000) 2023-08-27T22:59:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2817648097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2817648097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2817648097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:56.800 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:56.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:56.513702+0000) 2023-08-27T22:59:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:56 smithi136 ceph-mon[113531]: pgmap v3792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:56 smithi093 ceph-mon[111335]: pgmap v3792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:56 smithi093 ceph-mon[118902]: pgmap v3792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:57.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:57.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:57.513919+0000) 2023-08-27T22:59:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/998237725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/998237725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:58.514 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/998237725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T22:59:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:58.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:58.514175+0000) 2023-08-27T22:59:59.515 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 22:59:59 smithi136 ceph-mon[113531]: pgmap v3793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 22:59:59 smithi093 ceph-mon[111335]: pgmap v3793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 22:59:59 smithi093 ceph-mon[118902]: pgmap v3793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T22:59:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 22:59:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T22:59:59.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T21:59:59.514381+0000) 2023-08-27T23:00:00.779 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:00.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:00.514636+0000) 2023-08-27T23:00:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:00:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-mon[111335]: pgmap v3794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2925595677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:00:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-mon[118902]: pgmap v3794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2925595677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:00.632Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:00.633Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:00:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:00 smithi136 ceph-mon[113531]: pgmap v3794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2925595677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:01.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:01.514836+0000) 2023-08-27T23:00:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:02.130Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:02.131Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:02.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:02.515117+0000) 2023-08-27T23:00:03.515 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:03 smithi136 ceph-mon[113531]: pgmap v3795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:03.515 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4292998981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:03 smithi093 ceph-mon[111335]: pgmap v3795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4292998981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:03 smithi093 ceph-mon[118902]: pgmap v3795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4292998981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:03.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:03.515322+0000) 2023-08-27T23:00:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:04.051Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:04.052Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:04.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:04.515594+0000) 2023-08-27T23:00:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:05 smithi136 ceph-mon[113531]: pgmap v3796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:05.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:05.515829+0000) 2023-08-27T23:00:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:05 smithi093 ceph-mon[111335]: pgmap v3796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:05 smithi093 ceph-mon[118902]: pgmap v3796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:00:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:00:05] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T23:00:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1754483090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:06.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:06.516125+0000) 2023-08-27T23:00:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1754483090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1754483090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:07 smithi136 ceph-mon[113531]: pgmap v3797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:07.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:07.516353+0000) 2023-08-27T23:00:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:07 smithi093 ceph-mon[111335]: pgmap v3797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:07 smithi093 ceph-mon[118902]: pgmap v3797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2071187064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:08.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:08.516551+0000) 2023-08-27T23:00:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2071187064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2071187064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:09 smithi136 ceph-mon[113531]: pgmap v3798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:09.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:09.516777+0000) 2023-08-27T23:00:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:09 smithi093 ceph-mon[111335]: pgmap v3798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:09 smithi093 ceph-mon[118902]: pgmap v3798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:10.783 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:10.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:10.517002+0000) 2023-08-27T23:00:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3583536725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:10 smithi093 ceph-mon[111335]: pgmap v3799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3583536725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:10 smithi093 ceph-mon[118902]: pgmap v3799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:10.633Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:10.633Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3583536725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:10 smithi136 ceph-mon[113531]: pgmap v3799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:11.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:11.517271+0000) 2023-08-27T23:00:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:12.131Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:12.132Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:12.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:12.517474+0000) 2023-08-27T23:00:13.517 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:13 smithi136 ceph-mon[113531]: pgmap v3800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:13.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3936477001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:13 smithi093 ceph-mon[111335]: pgmap v3800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3936477001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:13 smithi093 ceph-mon[118902]: pgmap v3800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3936477001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:13.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:13.517720+0000) 2023-08-27T23:00:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:14.052Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:14.053Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:14.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:14.517946+0000) 2023-08-27T23:00:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:15 smithi136 ceph-mon[113531]: pgmap v3801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/537179739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:15.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:15.518213+0000) 2023-08-27T23:00:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:00:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:00:15] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:00:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:15 smithi093 ceph-mon[111335]: pgmap v3801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/537179739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:15 smithi093 ceph-mon[118902]: pgmap v3801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/537179739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:16.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:16.518467+0000) 2023-08-27T23:00:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:17 smithi136 ceph-mon[113531]: pgmap v3802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2805758386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:17.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:17.518709+0000) 2023-08-27T23:00:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:17 smithi093 ceph-mon[111335]: pgmap v3802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2805758386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:17 smithi093 ceph-mon[118902]: pgmap v3802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2805758386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:18.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:18.518941+0000) 2023-08-27T23:00:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:19 smithi136 ceph-mon[113531]: pgmap v3803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:19.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:19.519188+0000) 2023-08-27T23:00:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:19 smithi093 ceph-mon[111335]: pgmap v3803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:19 smithi093 ceph-mon[118902]: pgmap v3803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:20.634 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1592405206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:20.634 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1592405206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1592405206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:20.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:20.519410+0000) 2023-08-27T23:00:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:20.633Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:20.634Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:21 smithi136 ceph-mon[113531]: pgmap v3804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:21.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:21.519643+0000) 2023-08-27T23:00:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:21 smithi093 ceph-mon[111335]: pgmap v3804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:21 smithi093 ceph-mon[118902]: pgmap v3804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:22.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/177148117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/177148117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:22.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:22.131Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:22.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:22.132Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/177148117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:22.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:22.519910+0000) 2023-08-27T23:00:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:23 smithi136 ceph-mon[113531]: pgmap v3805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:23.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:23.520132+0000) 2023-08-27T23:00:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:23 smithi093 ceph-mon[111335]: pgmap v3805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:23 smithi093 ceph-mon[118902]: pgmap v3805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:24.053Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:24.054Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:24.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:24.520404+0000) 2023-08-27T23:00:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:25 smithi136 ceph-mon[113531]: pgmap v3806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/575384272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:25.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:25.520693+0000) 2023-08-27T23:00:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:25 smithi093 ceph-mon[111335]: pgmap v3806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/575384272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:25 smithi093 ceph-mon[118902]: pgmap v3806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/575384272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:00:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:00:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:00:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:26.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:26.520926+0000) 2023-08-27T23:00:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:27 smithi136 ceph-mon[113531]: pgmap v3807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2465702970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:27.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:27.521194+0000) 2023-08-27T23:00:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:27 smithi093 ceph-mon[111335]: pgmap v3807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2465702970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:27 smithi093 ceph-mon[118902]: pgmap v3807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2465702970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:28.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:28.521356+0000) 2023-08-27T23:00:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:29 smithi136 ceph-mon[113531]: pgmap v3808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2933115910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:29.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:29.521567+0000) 2023-08-27T23:00:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:29 smithi093 ceph-mon[111335]: pgmap v3808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2933115910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:29 smithi093 ceph-mon[118902]: pgmap v3808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2933115910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:30.522 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:00:30.635 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:00:30.635 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:00:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:30.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:30.521720+0000) 2023-08-27T23:00:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:30.634Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:30.635Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:31 smithi136 ceph-mon[113531]: pgmap v3809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:31 smithi093 ceph-mon[111335]: pgmap v3809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:31 smithi093 ceph-mon[118902]: pgmap v3809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:31.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:31.521907+0000) 2023-08-27T23:00:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:32.132Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:32.133Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2048611869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:00:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:00:32.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2048611869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:00:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:00:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:32.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:32.522102+0000) 2023-08-27T23:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2048611869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:00:32.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:00:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:33 smithi136 ceph-mon[113531]: pgmap v3810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:33.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:33.522297+0000) 2023-08-27T23:00:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:33 smithi093 ceph-mon[111335]: pgmap v3810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:33 smithi093 ceph-mon[118902]: pgmap v3810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:34.304 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:34.054Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:34.305 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:34.055Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1875453484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:34.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:34.522544+0000) 2023-08-27T23:00:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1875453484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1875453484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:35 smithi136 ceph-mon[113531]: pgmap v3811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:35.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:35.522802+0000) 2023-08-27T23:00:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:35 smithi093 ceph-mon[111335]: pgmap v3811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:00:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:00:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:00:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:35 smithi093 ceph-mon[118902]: pgmap v3811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1028535149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:36.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:36.523108+0000) 2023-08-27T23:00:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1028535149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1028535149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:37 smithi136 ceph-mon[113531]: pgmap v3812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:37.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:37.523336+0000) 2023-08-27T23:00:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:37 smithi093 ceph-mon[111335]: pgmap v3812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:37 smithi093 ceph-mon[118902]: pgmap v3812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:38.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:38.523636+0000) 2023-08-27T23:00:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:39 smithi136 ceph-mon[113531]: pgmap v3813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/817324614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:39.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:39.523910+0000) 2023-08-27T23:00:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:39 smithi093 ceph-mon[111335]: pgmap v3813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/817324614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:39 smithi093 ceph-mon[118902]: pgmap v3813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/817324614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:40.792 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:40.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:40.524155+0000) 2023-08-27T23:00:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:40 smithi093 ceph-mon[111335]: pgmap v3814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:40 smithi093 ceph-mon[118902]: pgmap v3814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:40.634Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:40.635Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:40 smithi136 ceph-mon[113531]: pgmap v3814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:41.793 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:41.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:41.524402+0000) 2023-08-27T23:00:42.134 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2123840782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:42.134 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2123840782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2123840782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:42.133Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:42.134Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:42.798 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:42.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:42.524646+0000) 2023-08-27T23:00:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:42 smithi136 ceph-mon[113531]: pgmap v3815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:42 smithi093 ceph-mon[111335]: pgmap v3815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:42 smithi093 ceph-mon[118902]: pgmap v3815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:43.799 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:43.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:43.524881+0000) 2023-08-27T23:00:44.054 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3386811213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:44.055 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3386811213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3386811213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:44.054Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:44.055Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:44.808 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:44.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:44.525093+0000) 2023-08-27T23:00:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:44 smithi136 ceph-mon[113531]: pgmap v3816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:44 smithi093 ceph-mon[111335]: pgmap v3816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:44 smithi093 ceph-mon[118902]: pgmap v3816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:00:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:00:45] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:00:45.877 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:45.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:45.525420+0000) 2023-08-27T23:00:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2880933690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2880933690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2880933690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:46.884 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:46.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:46.525686+0000) 2023-08-27T23:00:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:46 smithi136 ceph-mon[113531]: pgmap v3817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:46 smithi093 ceph-mon[111335]: pgmap v3817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:46 smithi093 ceph-mon[118902]: pgmap v3817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:47.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:47.525917+0000) 2023-08-27T23:00:48.526 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2819233728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2819233728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2819233728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:48.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:48.526113+0000) 2023-08-27T23:00:49.526 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:49 smithi136 ceph-mon[113531]: pgmap v3818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:49 smithi093 ceph-mon[111335]: pgmap v3818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:49 smithi093 ceph-mon[118902]: pgmap v3818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:49.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:49.526340+0000) 2023-08-27T23:00:50.790 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:50.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:50.526547+0000) 2023-08-27T23:00:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:50 smithi093 ceph-mon[111335]: pgmap v3819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4240113540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:50 smithi093 ceph-mon[118902]: pgmap v3819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4240113540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:50.635Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:50.636Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:50 smithi136 ceph-mon[113531]: pgmap v3819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4240113540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:51.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:51.526693+0000) 2023-08-27T23:00:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:52.134Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:52.135Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:52.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:52.526930+0000) 2023-08-27T23:00:53.527 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:53 smithi136 ceph-mon[113531]: pgmap v3820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:53.527 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/344162561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:53 smithi093 ceph-mon[118902]: pgmap v3820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/344162561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:53 smithi093 ceph-mon[111335]: pgmap v3820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/344162561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:53.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:53.527131+0000) 2023-08-27T23:00:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:54.055Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:00:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:00:54.056Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:00:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:00:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:54.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:54.527376+0000) 2023-08-27T23:00:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:55 smithi136 ceph-mon[113531]: pgmap v3821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:55.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:55.527630+0000) 2023-08-27T23:00:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:55 smithi093 ceph-mon[111335]: pgmap v3821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:55 smithi093 ceph-mon[118902]: pgmap v3821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:00:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:00:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:00:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1052858703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:56.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:56.527826+0000) 2023-08-27T23:00:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1052858703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1052858703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:57 smithi136 ceph-mon[113531]: pgmap v3822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:57.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:57.528113+0000) 2023-08-27T23:00:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:57 smithi093 ceph-mon[111335]: pgmap v3822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:57 smithi093 ceph-mon[118902]: pgmap v3822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3634581352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:58.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:58.528398+0000) 2023-08-27T23:00:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3634581352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3634581352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:00:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:00:59 smithi136 ceph-mon[113531]: pgmap v3823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:00:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:00:59.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:00:59.528652+0000) 2023-08-27T23:00:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:00:59 smithi093 ceph-mon[111335]: pgmap v3823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:00:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:00:59 smithi093 ceph-mon[118902]: pgmap v3823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:00.789 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:00.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:00.528922+0000) 2023-08-27T23:01:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3018609745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:00 smithi093 ceph-mon[111335]: pgmap v3824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3018609745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:00 smithi093 ceph-mon[118902]: pgmap v3824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:00.636Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:00.637Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3018609745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:01.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:00 smithi136 ceph-mon[113531]: pgmap v3824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:01.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:01.529177+0000) 2023-08-27T23:01:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:02.135Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:02.136Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:02.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:02.529423+0000) 2023-08-27T23:01:03.530 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:03 smithi136 ceph-mon[113531]: pgmap v3825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:03.530 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2814963913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:03 smithi093 ceph-mon[111335]: pgmap v3825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2814963913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:03 smithi093 ceph-mon[118902]: pgmap v3825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2814963913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:03.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:03.529682+0000) 2023-08-27T23:01:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:04.056Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:04.057Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:04.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:04.529880+0000) 2023-08-27T23:01:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:05 smithi136 ceph-mon[113531]: pgmap v3826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:05.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2807627674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:05.691 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 23:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T23:01:05.318413916Z level=info msg="Completed cleanup jobs" duration=13.067704ms 2023-08-27T23:01:05.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:05.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:05.530108+0000) 2023-08-27T23:01:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:01:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:01:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:01:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:05 smithi093 ceph-mon[118902]: pgmap v3826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2807627674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:05 smithi093 ceph-mon[111335]: pgmap v3826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2807627674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:06.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:06.530351+0000) 2023-08-27T23:01:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:07 smithi136 ceph-mon[113531]: pgmap v3827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/680053820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:07.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:07.530640+0000) 2023-08-27T23:01:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:07 smithi093 ceph-mon[111335]: pgmap v3827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/680053820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:07 smithi093 ceph-mon[118902]: pgmap v3827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/680053820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:08.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:08.530888+0000) 2023-08-27T23:01:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:09 smithi136 ceph-mon[113531]: pgmap v3828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:09.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:09.531124+0000) 2023-08-27T23:01:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:09 smithi093 ceph-mon[111335]: pgmap v3828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:09 smithi093 ceph-mon[118902]: pgmap v3828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:10.637 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3422489318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:10.637 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3422489318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3422489318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:10.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:10.531365+0000) 2023-08-27T23:01:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:10.637Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:10.638Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:11 smithi136 ceph-mon[113531]: pgmap v3829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:11.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:11.531612+0000) 2023-08-27T23:01:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:11 smithi093 ceph-mon[111335]: pgmap v3829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:11 smithi093 ceph-mon[118902]: pgmap v3829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2932918420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2932918420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:12.136Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:12.137Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2932918420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:12.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:12.531858+0000) 2023-08-27T23:01:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:13 smithi136 ceph-mon[113531]: pgmap v3830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:13.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:13.532091+0000) 2023-08-27T23:01:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:13 smithi093 ceph-mon[111335]: pgmap v3830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:13 smithi093 ceph-mon[118902]: pgmap v3830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:14.056Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:14.057Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:14.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:14.532336+0000) 2023-08-27T23:01:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:15 smithi136 ceph-mon[113531]: pgmap v3831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1548627597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:15.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:15.532590+0000) 2023-08-27T23:01:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:15 smithi093 ceph-mon[118902]: pgmap v3831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1548627597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:15 smithi093 ceph-mon[111335]: pgmap v3831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1548627597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:01:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:01:15] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T23:01:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:16.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:16.532839+0000) 2023-08-27T23:01:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:17 smithi136 ceph-mon[113531]: pgmap v3832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2181077802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:17.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:17.533116+0000) 2023-08-27T23:01:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:17 smithi093 ceph-mon[111335]: pgmap v3832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2181077802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:17 smithi093 ceph-mon[118902]: pgmap v3832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2181077802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:18.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:18.533344+0000) 2023-08-27T23:01:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:19 smithi136 ceph-mon[113531]: pgmap v3833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/455599430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:19.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:19.533577+0000) 2023-08-27T23:01:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:19 smithi093 ceph-mon[118902]: pgmap v3833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/455599430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:19 smithi093 ceph-mon[111335]: pgmap v3833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/455599430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:20.792 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:20.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:20.533800+0000) 2023-08-27T23:01:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:20 smithi093 ceph-mon[111335]: pgmap v3834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:20 smithi093 ceph-mon[118902]: pgmap v3834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:20.638Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:20.639Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:20 smithi136 ceph-mon[113531]: pgmap v3834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:21.793 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:21.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:21.534096+0000) 2023-08-27T23:01:22.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/664217912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:22.137 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/664217912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/664217912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:22.137Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:22.138Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:22.797 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:22.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:22.534326+0000) 2023-08-27T23:01:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:22 smithi136 ceph-mon[113531]: pgmap v3835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:22 smithi093 ceph-mon[111335]: pgmap v3835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:22 smithi093 ceph-mon[118902]: pgmap v3835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:23.798 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:23.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:23.534583+0000) 2023-08-27T23:01:24.057 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:24.057 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:24.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:24.056Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:24.058Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:24.802 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:24.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:24.534836+0000) 2023-08-27T23:01:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3595863749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:24 smithi136 ceph-mon[113531]: pgmap v3836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3595863749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:24 smithi093 ceph-mon[111335]: pgmap v3836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3595863749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:24 smithi093 ceph-mon[118902]: pgmap v3836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:01:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:01:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:01:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:25.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:25.535117+0000) 2023-08-27T23:01:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:26.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:26.535394+0000) 2023-08-27T23:01:27.535 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:27 smithi136 ceph-mon[113531]: pgmap v3837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:27.536 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1509794271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:27 smithi093 ceph-mon[111335]: pgmap v3837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1509794271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:27 smithi093 ceph-mon[118902]: pgmap v3837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1509794271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:27.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:27.535694+0000) 2023-08-27T23:01:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:28.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:28.535937+0000) 2023-08-27T23:01:29.536 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:29 smithi136 ceph-mon[113531]: pgmap v3838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:29.536 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/448041006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:29 smithi093 ceph-mon[111335]: pgmap v3838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/448041006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:29 smithi093 ceph-mon[118902]: pgmap v3838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/448041006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:29.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:29.536175+0000) 2023-08-27T23:01:30.801 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:30.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:30.536434+0000) 2023-08-27T23:01:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:30 smithi093 ceph-mon[111335]: pgmap v3839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:30 smithi093 ceph-mon[118902]: pgmap v3839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:30.638Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:30.639Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:30 smithi136 ceph-mon[113531]: pgmap v3839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:31.801 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:31.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:31.536633+0000) 2023-08-27T23:01:32.137 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/403002789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:32.138 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/403002789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/403002789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:32.137Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:32.138Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:32.806 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:32.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:32.536799+0000) 2023-08-27T23:01:33.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:01:33.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:32 smithi136 ceph-mon[113531]: pgmap v3840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:01:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:32 smithi093 ceph-mon[111335]: pgmap v3840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:01:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:32 smithi093 ceph-mon[118902]: pgmap v3840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:33.807 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:33.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:33.536959+0000) 2023-08-27T23:01:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1010612946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1010612946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1010612946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:34.057Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:34.058Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:34.933 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:34.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:34.537122+0000) 2023-08-27T23:01:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: pgmap v3841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:01:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:01:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: pgmap v3841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: pgmap v3841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:01:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:01:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:01:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:01:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:01:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:01:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:35.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:35.537344+0000) 2023-08-27T23:01:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2147636520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2147636520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2147636520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:36.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:36.537563+0000) 2023-08-27T23:01:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:36 smithi093 ceph-mon[111335]: pgmap v3842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:36 smithi093 ceph-mon[118902]: pgmap v3842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:36 smithi136 ceph-mon[113531]: pgmap v3842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:37.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:37.537825+0000) 2023-08-27T23:01:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3435454763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:38.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:38.538099+0000) 2023-08-27T23:01:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3435454763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3435454763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:39.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:39.538373+0000) 2023-08-27T23:01:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:39 smithi136 ceph-mon[113531]: pgmap v3843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:39 smithi093 ceph-mon[111335]: pgmap v3843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:39 smithi093 ceph-mon[118902]: pgmap v3843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:40.801 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:40.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:40.538620+0000) 2023-08-27T23:01:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:40 smithi093 ceph-mon[118902]: pgmap v3844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2598499503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:40 smithi093 ceph-mon[111335]: pgmap v3844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2598499503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:40.639Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:40.640Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:40 smithi136 ceph-mon[113531]: pgmap v3844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2598499503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:41.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:41.538877+0000) 2023-08-27T23:01:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:42.138Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:42.139Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:42.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:42.539111+0000) 2023-08-27T23:01:43.539 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:43 smithi136 ceph-mon[113531]: pgmap v3845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:43.539 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1019604461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:43 smithi093 ceph-mon[111335]: pgmap v3845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1019604461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:43 smithi093 ceph-mon[118902]: pgmap v3845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1019604461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:43.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:43.539366+0000) 2023-08-27T23:01:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:44.058Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:44.059Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:44.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:44.539604+0000) 2023-08-27T23:01:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:45 smithi136 ceph-mon[113531]: pgmap v3846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:45.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:45.539902+0000) 2023-08-27T23:01:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:45 smithi093 ceph-mon[111335]: pgmap v3846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:45 smithi093 ceph-mon[118902]: pgmap v3846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:01:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:01:45] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:01:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1525431482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:46.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:46.540164+0000) 2023-08-27T23:01:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1525431482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1525431482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:47 smithi136 ceph-mon[113531]: pgmap v3847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:47.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:47.540358+0000) 2023-08-27T23:01:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:47 smithi093 ceph-mon[111335]: pgmap v3847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:47 smithi093 ceph-mon[118902]: pgmap v3847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1989415589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:48.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:48.540583+0000) 2023-08-27T23:01:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1989415589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1989415589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:49 smithi136 ceph-mon[113531]: pgmap v3848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:49.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:49.540807+0000) 2023-08-27T23:01:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:49 smithi093 ceph-mon[111335]: pgmap v3848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:49 smithi093 ceph-mon[118902]: pgmap v3848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:50.800 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:50.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:50.541079+0000) 2023-08-27T23:01:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1629976377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:50 smithi093 ceph-mon[111335]: pgmap v3849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1629976377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:50 smithi093 ceph-mon[118902]: pgmap v3849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:50.640Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:50.641Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1629976377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:50 smithi136 ceph-mon[113531]: pgmap v3849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:51.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:51.541378+0000) 2023-08-27T23:01:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:52.139Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:52.140Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:52.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:52.541624+0000) 2023-08-27T23:01:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:53 smithi136 ceph-mon[113531]: pgmap v3850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/448635340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:53.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:53.541854+0000) 2023-08-27T23:01:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:53 smithi093 ceph-mon[111335]: pgmap v3850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/448635340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:53 smithi093 ceph-mon[118902]: pgmap v3850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/448635340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:54.058Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:54.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:01:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:01:54.060Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:01:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:01:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:54.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:54.542123+0000) 2023-08-27T23:01:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:55 smithi136 ceph-mon[113531]: pgmap v3851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3280906885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:55.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:55.542409+0000) 2023-08-27T23:01:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:55 smithi093 ceph-mon[111335]: pgmap v3851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3280906885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:01:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:01:55] "GET /metrics HTTP/1.1" 200 239910 "" "Prometheus/2.43.0" 2023-08-27T23:01:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:55 smithi093 ceph-mon[118902]: pgmap v3851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3280906885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:56.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:56.542639+0000) 2023-08-27T23:01:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:57 smithi136 ceph-mon[113531]: pgmap v3852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:57.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:57.542907+0000) 2023-08-27T23:01:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:57 smithi093 ceph-mon[111335]: pgmap v3852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:57 smithi093 ceph-mon[118902]: pgmap v3852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/216252492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:58.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:58.543128+0000) 2023-08-27T23:01:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/216252492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/216252492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:01:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:01:59 smithi136 ceph-mon[113531]: pgmap v3853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:01:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:01:59.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:01:59.543374+0000) 2023-08-27T23:01:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:01:59 smithi093 ceph-mon[111335]: pgmap v3853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:01:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:01:59 smithi093 ceph-mon[118902]: pgmap v3853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:00.642 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2469420107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:00.642 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2469420107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2469420107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:00.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:00.543558+0000) 2023-08-27T23:02:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:00.641Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:00.642Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:01 smithi136 ceph-mon[113531]: pgmap v3854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:01.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:01.543795+0000) 2023-08-27T23:02:01.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:01 smithi093 ceph-mon[111335]: pgmap v3854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:01.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:01 smithi093 ceph-mon[118902]: pgmap v3854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3722362386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3722362386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:02.139Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:02.140Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3722362386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:02.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:02.544095+0000) 2023-08-27T23:02:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:03 smithi136 ceph-mon[113531]: pgmap v3855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:03.694 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:03.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:03.544293+0000) 2023-08-27T23:02:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:03 smithi093 ceph-mon[111335]: pgmap v3855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:03 smithi093 ceph-mon[118902]: pgmap v3855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:04.059Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:04.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:04.060Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:04.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:04.544510+0000) 2023-08-27T23:02:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:05 smithi136 ceph-mon[113531]: pgmap v3856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1940273779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:05.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:05.544776+0000) 2023-08-27T23:02:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:05 smithi093 ceph-mon[111335]: pgmap v3856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1940273779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:02:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:02:05] "GET /metrics HTTP/1.1" 200 239910 "" "Prometheus/2.43.0" 2023-08-27T23:02:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:05 smithi093 ceph-mon[118902]: pgmap v3856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1940273779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:06.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:06.545024+0000) 2023-08-27T23:02:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:07 smithi136 ceph-mon[113531]: pgmap v3857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3497076572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:07.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:07.545318+0000) 2023-08-27T23:02:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:07 smithi093 ceph-mon[111335]: pgmap v3857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:07.773 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3497076572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:07.773 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:07 smithi093 ceph-mon[118902]: pgmap v3857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:07.773 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3497076572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:08.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:08.545548+0000) 2023-08-27T23:02:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:09 smithi136 ceph-mon[113531]: pgmap v3858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:09.811 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:09.811 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3545018832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:09.811 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:09.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:09.545858+0000) 2023-08-27T23:02:09.812 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:09 smithi093 ceph-mon[111335]: pgmap v3858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:09.812 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:09.812 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3545018832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:09.812 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:09 smithi093 ceph-mon[118902]: pgmap v3858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:09.812 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:09.813 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3545018832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:10.802 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:10.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:10.546156+0000) 2023-08-27T23:02:11.249 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:10 smithi093 ceph-mon[111335]: pgmap v3859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:11.250 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:10 smithi136 ceph-mon[113531]: pgmap v3859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:11.250 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:10 smithi093 ceph-mon[118902]: pgmap v3859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:11.250 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:10.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:11.250 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:10.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:11.804 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:11.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:11.546447+0000) 2023-08-27T23:02:12.140 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2549066013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:12.284 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2549066013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:12.285 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2549066013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:12.140Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:12.608 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:12.141Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:12.809 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:12.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:12.546681+0000) 2023-08-27T23:02:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:12 smithi136 ceph-mon[113531]: pgmap v3860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:13.275 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:12 smithi093 ceph-mon[111335]: pgmap v3860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:13.275 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:12 smithi093 ceph-mon[118902]: pgmap v3860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:13.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:13.546917+0000) 2023-08-27T23:02:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1989129014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:14.552 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1989129014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:14.553 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1989129014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:14.553 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:14.060Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:14.553 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:14.061Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:14.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:14.547144+0000) 2023-08-27T23:02:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:15 smithi093 ceph-mon[111335]: pgmap v3861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:15 smithi093 ceph-mon[118902]: pgmap v3861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:15 smithi136 ceph-mon[113531]: pgmap v3861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:02:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:02:15] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:02:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:15.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:15.547381+0000) 2023-08-27T23:02:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:16.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:16.547639+0000) 2023-08-27T23:02:17.548 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:17 smithi136 ceph-mon[113531]: pgmap v3862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:17.548 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2503220150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:17 smithi093 ceph-mon[111335]: pgmap v3862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2503220150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:17 smithi093 ceph-mon[118902]: pgmap v3862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2503220150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:17.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:17.547899+0000) 2023-08-27T23:02:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:18.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:18.548139+0000) 2023-08-27T23:02:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:19 smithi136 ceph-mon[113531]: pgmap v3863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4208971047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:19.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:19.548368+0000) 2023-08-27T23:02:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:19 smithi093 ceph-mon[111335]: pgmap v3863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4208971047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:19 smithi093 ceph-mon[118902]: pgmap v3863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4208971047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:20.810 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:20.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:20.548600+0000) 2023-08-27T23:02:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:20 smithi093 ceph-mon[111335]: pgmap v3864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:20 smithi093 ceph-mon[118902]: pgmap v3864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:20.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:20.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:20 smithi136 ceph-mon[113531]: pgmap v3864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:21.812 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:21.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:21.548848+0000) 2023-08-27T23:02:22.141 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2237668529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:22.141 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2237668529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2237668529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:22.141Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:22.142Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:22.820 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:22.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:22.549136+0000) 2023-08-27T23:02:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:22 smithi136 ceph-mon[113531]: pgmap v3865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:22 smithi093 ceph-mon[111335]: pgmap v3865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:22 smithi093 ceph-mon[118902]: pgmap v3865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:23.818 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:23.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:23.549348+0000) 2023-08-27T23:02:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4117673791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4117673791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4117673791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:24.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:24.061Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:24.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:24.062Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:24.826 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:24.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:24.549587+0000) 2023-08-27T23:02:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:24 smithi136 ceph-mon[113531]: pgmap v3866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:24 smithi093 ceph-mon[111335]: pgmap v3866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:24 smithi093 ceph-mon[118902]: pgmap v3866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:02:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:02:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:02:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:25.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:25.549866+0000) 2023-08-27T23:02:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1194211046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1194211046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1194211046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:26.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:26.550079+0000) 2023-08-27T23:02:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:26 smithi093 ceph-mon[111335]: pgmap v3867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:26 smithi093 ceph-mon[118902]: pgmap v3867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:26 smithi136 ceph-mon[113531]: pgmap v3867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:27.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:27.550347+0000) 2023-08-27T23:02:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/191241314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:28.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:28.550611+0000) 2023-08-27T23:02:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/191241314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/191241314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:29 smithi136 ceph-mon[113531]: pgmap v3868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:29.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:29.550852+0000) 2023-08-27T23:02:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:29 smithi093 ceph-mon[111335]: pgmap v3868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:29 smithi093 ceph-mon[118902]: pgmap v3868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:30.805 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:30.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:30.551140+0000) 2023-08-27T23:02:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:30 smithi093 ceph-mon[118902]: pgmap v3869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3692902100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:30 smithi093 ceph-mon[111335]: pgmap v3869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:30.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3692902100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:30.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:30.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:30 smithi136 ceph-mon[113531]: pgmap v3869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3692902100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:31.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:31.551319+0000) 2023-08-27T23:02:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:32.141Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:32.142Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:32.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:32.551525+0000) 2023-08-27T23:02:33.551 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:33 smithi136 ceph-mon[113531]: pgmap v3870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:33.552 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3885566479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:33 smithi093 ceph-mon[111335]: pgmap v3870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3885566479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:33 smithi093 ceph-mon[118902]: pgmap v3870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3885566479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:33.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:33.551718+0000) 2023-08-27T23:02:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:34.062Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:34.063Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:34.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:34.551923+0000) 2023-08-27T23:02:35.689 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:35 smithi093 ceph-mon[111335]: pgmap v3871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:35.689 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:02:35.689 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:35 smithi093 ceph-mon[118902]: pgmap v3871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:35.690 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:02:35.690 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:02:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:02:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:02:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:35 smithi136 ceph-mon[113531]: pgmap v3871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:02:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:35.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:35.552137+0000) 2023-08-27T23:02:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2762852969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:36.621 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2762852969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:36.621 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2762852969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:36.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:36.552272+0000) 2023-08-27T23:02:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:37.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:37.552580+0000) 2023-08-27T23:02:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:37 smithi136 ceph-mon[113531]: pgmap v3872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:37 smithi093 ceph-mon[111335]: pgmap v3872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:37 smithi093 ceph-mon[118902]: pgmap v3872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:38.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:38.552819+0000) 2023-08-27T23:02:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/688951168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:02:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:02:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:38 smithi136 ceph-mon[113531]: pgmap v3873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/688951168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[111335]: pgmap v3873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/688951168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:02:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:02:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:02:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:38 smithi093 ceph-mon[118902]: pgmap v3873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:39.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:39.553057+0000) 2023-08-27T23:02:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:40.806 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:40.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:40.553245+0000) 2023-08-27T23:02:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:40 smithi093 ceph-mon[111335]: pgmap v3874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1861659683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:40 smithi093 ceph-mon[118902]: pgmap v3874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1861659683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:40.643Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:40.644Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:40 smithi136 ceph-mon[113531]: pgmap v3874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1861659683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:41.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:41.553432+0000) 2023-08-27T23:02:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:42.142Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:42.143Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:42.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:42.553630+0000) 2023-08-27T23:02:43.554 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:43 smithi136 ceph-mon[113531]: pgmap v3875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:43.554 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3238414303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:43 smithi093 ceph-mon[111335]: pgmap v3875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3238414303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:43 smithi093 ceph-mon[118902]: pgmap v3875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3238414303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:43.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:43.553872+0000) 2023-08-27T23:02:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:44.063Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:44.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:44.064Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:44.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:44.554091+0000) 2023-08-27T23:02:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:45 smithi136 ceph-mon[113531]: pgmap v3876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3507235719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:45.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:45.554329+0000) 2023-08-27T23:02:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:45 smithi093 ceph-mon[111335]: pgmap v3876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3507235719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:02:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:02:45] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:02:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:45 smithi093 ceph-mon[118902]: pgmap v3876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3507235719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:46.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:46.554575+0000) 2023-08-27T23:02:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:47 smithi136 ceph-mon[113531]: pgmap v3877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:47.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:47.554848+0000) 2023-08-27T23:02:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:47 smithi093 ceph-mon[111335]: pgmap v3877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:47 smithi093 ceph-mon[118902]: pgmap v3877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3317934330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:48.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:48.555129+0000) 2023-08-27T23:02:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3317934330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3317934330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:49 smithi136 ceph-mon[113531]: pgmap v3878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:49.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:49.555338+0000) 2023-08-27T23:02:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:49 smithi093 ceph-mon[111335]: pgmap v3878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:49 smithi093 ceph-mon[118902]: pgmap v3878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:50.644 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/919432215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:50.644 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/919432215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/919432215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:50.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:50.555571+0000) 2023-08-27T23:02:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:50.644Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:50.644Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:51 smithi136 ceph-mon[113531]: pgmap v3879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:51.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:51.555814+0000) 2023-08-27T23:02:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:51 smithi093 ceph-mon[111335]: pgmap v3879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:51 smithi093 ceph-mon[118902]: pgmap v3879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/903826675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/903826675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:52.142Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:52.143Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/903826675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:52.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:52.556010+0000) 2023-08-27T23:02:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:53 smithi136 ceph-mon[113531]: pgmap v3880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:53.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:53.556247+0000) 2023-08-27T23:02:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:53 smithi093 ceph-mon[111335]: pgmap v3880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:53 smithi093 ceph-mon[118902]: pgmap v3880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:54.331 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:54.063Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:54.331 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:02:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:02:54.065Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:02:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:54.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:54.556500+0000) 2023-08-27T23:02:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:02:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:55 smithi136 ceph-mon[113531]: pgmap v3881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3575487068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:55.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:55.556789+0000) 2023-08-27T23:02:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:55 smithi093 ceph-mon[111335]: pgmap v3881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3575487068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:02:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:02:55] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:02:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:55 smithi093 ceph-mon[118902]: pgmap v3881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3575487068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:56.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:56.557053+0000) 2023-08-27T23:02:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:57 smithi136 ceph-mon[113531]: pgmap v3882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3304099089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:57.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:57.557343+0000) 2023-08-27T23:02:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:57 smithi093 ceph-mon[111335]: pgmap v3882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3304099089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:57 smithi093 ceph-mon[118902]: pgmap v3882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3304099089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:58.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:58.557622+0000) 2023-08-27T23:02:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:59 smithi136 ceph-mon[113531]: pgmap v3883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:02:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1640304198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:02:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:02:59.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:02:59.557862+0000) 2023-08-27T23:02:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:59 smithi093 ceph-mon[111335]: pgmap v3883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:02:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1640304198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:02:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:59 smithi093 ceph-mon[118902]: pgmap v3883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:02:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:02:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1640304198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:00.817 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:00.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:00.558138+0000) 2023-08-27T23:03:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:00 smithi093 ceph-mon[111335]: pgmap v3884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:00 smithi093 ceph-mon[118902]: pgmap v3884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:00.644Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:00.645Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:00 smithi136 ceph-mon[113531]: pgmap v3884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:01.818 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:01.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:01.558374+0000) 2023-08-27T23:03:02.144 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2672983519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:02.144 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2672983519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2672983519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:02.143Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:02.144Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:02.822 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:02.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:02.558617+0000) 2023-08-27T23:03:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:02 smithi136 ceph-mon[113531]: pgmap v3885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:02 smithi093 ceph-mon[111335]: pgmap v3885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:02 smithi093 ceph-mon[118902]: pgmap v3885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:03.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:03.558852+0000) 2023-08-27T23:03:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1358636016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1358636016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:04.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1358636016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:04.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:04.064Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:04.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:04.065Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:04.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:04.559089+0000) 2023-08-27T23:03:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:05 smithi136 ceph-mon[113531]: pgmap v3886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:05 smithi093 ceph-mon[111335]: pgmap v3886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:03:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:03:05] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:03:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:05 smithi093 ceph-mon[118902]: pgmap v3886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:05.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:05.559332+0000) 2023-08-27T23:03:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:06.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:06.559575+0000) 2023-08-27T23:03:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:07.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:07.559820+0000) 2023-08-27T23:03:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:07 smithi136 ceph-mon[113531]: pgmap v3887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3679906212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:07 smithi093 ceph-mon[118902]: pgmap v3887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3679906212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:07 smithi093 ceph-mon[111335]: pgmap v3887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3679906212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:08.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:08.560115+0000) 2023-08-27T23:03:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:09.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:09.560366+0000) 2023-08-27T23:03:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:09 smithi136 ceph-mon[113531]: pgmap v3888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3844319404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:09 smithi093 ceph-mon[111335]: pgmap v3888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3844319404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:09 smithi093 ceph-mon[118902]: pgmap v3888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3844319404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:10.818 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:10.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:10.560643+0000) 2023-08-27T23:03:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:10 smithi093 ceph-mon[111335]: pgmap v3889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:10 smithi093 ceph-mon[118902]: pgmap v3889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:10.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:10.646Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:10 smithi136 ceph-mon[113531]: pgmap v3889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:11.819 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:11.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:11.560907+0000) 2023-08-27T23:03:12.144 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3953868260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:12.144 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3953868260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3953868260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:12.144Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:12.145Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:12.826 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:12.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:12.561147+0000) 2023-08-27T23:03:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:12 smithi136 ceph-mon[113531]: pgmap v3890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:12 smithi093 ceph-mon[111335]: pgmap v3890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:12 smithi093 ceph-mon[118902]: pgmap v3890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:13.827 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:13.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:13.561349+0000) 2023-08-27T23:03:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/857613017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/857613017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/857613017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:14.065Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:14.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:14.066Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:14.831 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:14.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:14.561563+0000) 2023-08-27T23:03:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:14 smithi136 ceph-mon[113531]: pgmap v3891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:14 smithi093 ceph-mon[111335]: pgmap v3891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:14 smithi093 ceph-mon[118902]: pgmap v3891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:03:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:03:15] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T23:03:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:15.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:15.561799+0000) 2023-08-27T23:03:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3961680361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3961680361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3961680361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:16.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:16.562092+0000) 2023-08-27T23:03:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:17 smithi136 ceph-mon[113531]: pgmap v3892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:17 smithi093 ceph-mon[111335]: pgmap v3892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:17 smithi093 ceph-mon[118902]: pgmap v3892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:17.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:17.562327+0000) 2023-08-27T23:03:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:18.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:18.562565+0000) 2023-08-27T23:03:19.563 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:19 smithi136 ceph-mon[113531]: pgmap v3893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:19.563 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3251048316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:19 smithi093 ceph-mon[111335]: pgmap v3893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3251048316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:19 smithi093 ceph-mon[118902]: pgmap v3893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3251048316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:19.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:19.562824+0000) 2023-08-27T23:03:20.820 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:20.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:20.563085+0000) 2023-08-27T23:03:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:20 smithi093 ceph-mon[111335]: pgmap v3894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/833345254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:20 smithi093 ceph-mon[118902]: pgmap v3894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/833345254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:20.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:20.646Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:20 smithi136 ceph-mon[113531]: pgmap v3894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/833345254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:21.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:21.563303+0000) 2023-08-27T23:03:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:22.144Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:22.145Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:22.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:22.563508+0000) 2023-08-27T23:03:23.564 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:23 smithi136 ceph-mon[113531]: pgmap v3895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:23.564 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2006263507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:23 smithi093 ceph-mon[111335]: pgmap v3895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2006263507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:23 smithi093 ceph-mon[118902]: pgmap v3895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2006263507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:23.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:23.563685+0000) 2023-08-27T23:03:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:24.066Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:24.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:24.067Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:24.564 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:24.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:24.563922+0000) 2023-08-27T23:03:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:25 smithi136 ceph-mon[113531]: pgmap v3896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:25.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:25.564210+0000) 2023-08-27T23:03:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:25 smithi093 ceph-mon[111335]: pgmap v3896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:25 smithi093 ceph-mon[118902]: pgmap v3896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:03:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:03:25] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:03:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3575144450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:26.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:26.564466+0000) 2023-08-27T23:03:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3575144450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3575144450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:27 smithi136 ceph-mon[113531]: pgmap v3897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:27.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:27.564736+0000) 2023-08-27T23:03:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:27 smithi093 ceph-mon[111335]: pgmap v3897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:27 smithi093 ceph-mon[118902]: pgmap v3897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1626470308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:28.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:28.565000+0000) 2023-08-27T23:03:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1626470308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1626470308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:29 smithi136 ceph-mon[113531]: pgmap v3898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:29.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:29.565206+0000) 2023-08-27T23:03:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:29 smithi093 ceph-mon[111335]: pgmap v3898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:29 smithi093 ceph-mon[118902]: pgmap v3898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:30.824 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:30.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:30.565466+0000) 2023-08-27T23:03:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:30 smithi093 ceph-mon[111335]: pgmap v3899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2535687104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:30 smithi093 ceph-mon[118902]: pgmap v3899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2535687104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:30.646Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:30.647Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:30 smithi136 ceph-mon[113531]: pgmap v3899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2535687104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:31.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:31.565690+0000) 2023-08-27T23:03:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:32.145Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:32.146Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:32.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:32.565931+0000) 2023-08-27T23:03:33.566 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:33 smithi136 ceph-mon[113531]: pgmap v3900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:33.566 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/786353101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:33 smithi093 ceph-mon[118902]: pgmap v3900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/786353101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:33 smithi093 ceph-mon[111335]: pgmap v3900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/786353101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:33.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:33.566178+0000) 2023-08-27T23:03:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:34.066Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:34.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:34.067Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:34.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:34.566363+0000) 2023-08-27T23:03:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:35 smithi136 ceph-mon[113531]: pgmap v3901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4093775348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:35.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:35.566650+0000) 2023-08-27T23:03:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:35 smithi093 ceph-mon[111335]: pgmap v3901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4093775348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:03:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:03:35] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:03:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:35 smithi093 ceph-mon[118902]: pgmap v3901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4093775348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:36.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:36.566895+0000) 2023-08-27T23:03:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:37 smithi136 ceph-mon[113531]: pgmap v3902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:37.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:37.567114+0000) 2023-08-27T23:03:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:37 smithi093 ceph-mon[111335]: pgmap v3902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:37 smithi093 ceph-mon[118902]: pgmap v3902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:38.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1625255352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:38.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:03:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:38.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:38.567341+0000) 2023-08-27T23:03:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1625255352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:03:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1625255352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:03:39.612 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[118902]: pgmap v3903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:39.612 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:39.612 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:39.612 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:39.613 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[111335]: pgmap v3903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:39.613 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:39.613 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:39.613 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:39.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:39.567506+0000) 2023-08-27T23:03:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:39 smithi136 ceph-mon[113531]: pgmap v3903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:40.820 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:40.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:40.567684+0000) 2023-08-27T23:03:40.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:40.647Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:40.648Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/107296376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: pgmap v3904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:03:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/107296376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: pgmap v3904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:03:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:03:40.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/107296376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: pgmap v3904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:03:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:03:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:03:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:41.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:41.567916+0000) 2023-08-27T23:03:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1760118068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1760118068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:42.146Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:42.147Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:42.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1760118068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:42.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:42.568199+0000) 2023-08-27T23:03:43.568 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:43 smithi136 ceph-mon[113531]: pgmap v3905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:43 smithi093 ceph-mon[111335]: pgmap v3905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:43 smithi093 ceph-mon[118902]: pgmap v3905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:43.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:43.568437+0000) 2023-08-27T23:03:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:44.066Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:44.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:44.067Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:44.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:44.568697+0000) 2023-08-27T23:03:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:45 smithi136 ceph-mon[113531]: pgmap v3906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2425062597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:45.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:45.568959+0000) 2023-08-27T23:03:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:45 smithi093 ceph-mon[111335]: pgmap v3906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2425062597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:45 smithi093 ceph-mon[118902]: pgmap v3906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2425062597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:03:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:03:45] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:03:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:46.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:46.569211+0000) 2023-08-27T23:03:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:47 smithi136 ceph-mon[113531]: pgmap v3907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/549790864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:47.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:47.569462+0000) 2023-08-27T23:03:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:47 smithi093 ceph-mon[111335]: pgmap v3907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/549790864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:47 smithi093 ceph-mon[118902]: pgmap v3907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/549790864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:48.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:48.569677+0000) 2023-08-27T23:03:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:49 smithi136 ceph-mon[113531]: pgmap v3908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:49.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:49.569899+0000) 2023-08-27T23:03:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:49 smithi093 ceph-mon[118902]: pgmap v3908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:49 smithi093 ceph-mon[111335]: pgmap v3908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:50.648 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/617231688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:50.649 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/617231688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/617231688' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:50.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:50.570155+0000) 2023-08-27T23:03:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:50.648Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:50.649Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:51 smithi136 ceph-mon[113531]: pgmap v3909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:51.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:51.570379+0000) 2023-08-27T23:03:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:51 smithi093 ceph-mon[111335]: pgmap v3909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:51 smithi093 ceph-mon[118902]: pgmap v3909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2831976821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2831976821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:52.146Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:52.147Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2831976821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:52.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:52.570619+0000) 2023-08-27T23:03:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:53 smithi136 ceph-mon[113531]: pgmap v3910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:53.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:53.570893+0000) 2023-08-27T23:03:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:53 smithi093 ceph-mon[111335]: pgmap v3910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:53 smithi093 ceph-mon[118902]: pgmap v3910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:54.345 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:54.067Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:54.345 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:03:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:03:54.068Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:03:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1982636615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:54.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:54.571161+0000) 2023-08-27T23:03:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1982636615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:03:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1982636615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:55 smithi136 ceph-mon[113531]: pgmap v3911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:55.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:55.571394+0000) 2023-08-27T23:03:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:55 smithi093 ceph-mon[111335]: pgmap v3911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:03:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:03:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:03:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:55 smithi093 ceph-mon[118902]: pgmap v3911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:56.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:56.571597+0000) 2023-08-27T23:03:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:57 smithi136 ceph-mon[113531]: pgmap v3912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2048699115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:57.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:57.571852+0000) 2023-08-27T23:03:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:57 smithi093 ceph-mon[111335]: pgmap v3912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2048699115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:57 smithi093 ceph-mon[118902]: pgmap v3912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2048699115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:58.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:58.572152+0000) 2023-08-27T23:03:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:59 smithi136 ceph-mon[113531]: pgmap v3913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:03:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3846053184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:03:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:03:59.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:03:59.572395+0000) 2023-08-27T23:03:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:59 smithi093 ceph-mon[111335]: pgmap v3913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:03:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3846053184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:03:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:59 smithi093 ceph-mon[118902]: pgmap v3913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:03:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:03:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3846053184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:00.824 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:00.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:00.572649+0000) 2023-08-27T23:04:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:00 smithi093 ceph-mon[111335]: pgmap v3914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:00 smithi093 ceph-mon[118902]: pgmap v3914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:00.649Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:00.650Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:00 smithi136 ceph-mon[113531]: pgmap v3914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:01.826 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:01.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:01.572873+0000) 2023-08-27T23:04:02.148 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2147501166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:02.148 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2147501166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2147501166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:02.148Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:02.148Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:02.831 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:02.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:02.573167+0000) 2023-08-27T23:04:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:02 smithi136 ceph-mon[113531]: pgmap v3915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:02 smithi093 ceph-mon[111335]: pgmap v3915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:02 smithi093 ceph-mon[118902]: pgmap v3915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:03.832 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:03.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:03.573371+0000) 2023-08-27T23:04:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2811223546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2811223546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2811223546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:04.067Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:04.068Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:04.837 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:04.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:04.573576+0000) 2023-08-27T23:04:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:04 smithi136 ceph-mon[113531]: pgmap v3916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:04 smithi093 ceph-mon[118902]: pgmap v3916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:04 smithi093 ceph-mon[111335]: pgmap v3916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:04:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:04:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:04:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:05.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:05.573870+0000) 2023-08-27T23:04:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3096915070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3096915070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3096915070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:06.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:06.574153+0000) 2023-08-27T23:04:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:07 smithi136 ceph-mon[113531]: pgmap v3917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:07 smithi093 ceph-mon[111335]: pgmap v3917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:07 smithi093 ceph-mon[118902]: pgmap v3917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:07.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:07.574437+0000) 2023-08-27T23:04:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:08.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:08.574700+0000) 2023-08-27T23:04:09.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:09 smithi136 ceph-mon[113531]: pgmap v3918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:09.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1805760836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:09.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:09 smithi093 ceph-mon[111335]: pgmap v3918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1805760836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:09 smithi093 ceph-mon[118902]: pgmap v3918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1805760836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:09.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:09.574953+0000) 2023-08-27T23:04:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:10 smithi136 ceph-mon[113531]: pgmap v3919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2692401629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:10.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:10.575205+0000) 2023-08-27T23:04:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:10 smithi093 ceph-mon[111335]: pgmap v3919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2692401629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:10 smithi093 ceph-mon[118902]: pgmap v3919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2692401629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:10.650Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:10.651Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:11.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:11.575470+0000) 2023-08-27T23:04:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:12.148Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:12.149Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:12.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:12.575718+0000) 2023-08-27T23:04:13.576 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:13 smithi136 ceph-mon[113531]: pgmap v3920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:13.576 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2001625391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:13 smithi093 ceph-mon[111335]: pgmap v3920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2001625391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:13 smithi093 ceph-mon[118902]: pgmap v3920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2001625391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:13.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:13.575970+0000) 2023-08-27T23:04:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:14.068Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:14.069Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:14.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:14.576195+0000) 2023-08-27T23:04:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:15 smithi136 ceph-mon[113531]: pgmap v3921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:15.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:15.576381+0000) 2023-08-27T23:04:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:15 smithi093 ceph-mon[111335]: pgmap v3921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:04:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:04:15] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:04:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:15 smithi093 ceph-mon[118902]: pgmap v3921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1521911787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:16.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:16.576613+0000) 2023-08-27T23:04:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1521911787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1521911787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:17 smithi136 ceph-mon[113531]: pgmap v3922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:17.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:17.576842+0000) 2023-08-27T23:04:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:17 smithi093 ceph-mon[111335]: pgmap v3922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:17 smithi093 ceph-mon[118902]: pgmap v3922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2263624297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:18.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:18.577147+0000) 2023-08-27T23:04:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2263624297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2263624297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:19 smithi136 ceph-mon[113531]: pgmap v3923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:19.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:19.577395+0000) 2023-08-27T23:04:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:19 smithi093 ceph-mon[111335]: pgmap v3923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:19 smithi093 ceph-mon[118902]: pgmap v3923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:20.830 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:20.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:20.577667+0000) 2023-08-27T23:04:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:20 smithi093 ceph-mon[111335]: pgmap v3924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3003968043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:20 smithi093 ceph-mon[118902]: pgmap v3924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3003968043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:20.651Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:20.652Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:20 smithi136 ceph-mon[113531]: pgmap v3924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3003968043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:21.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:21.577936+0000) 2023-08-27T23:04:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:22.149Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:22.150Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:22.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:22.578153+0000) 2023-08-27T23:04:23.578 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:23 smithi136 ceph-mon[113531]: pgmap v3925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:23.578 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1305022315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:23 smithi093 ceph-mon[111335]: pgmap v3925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1305022315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:23 smithi093 ceph-mon[118902]: pgmap v3925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1305022315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:23.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:23.578375+0000) 2023-08-27T23:04:24.335 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:24.068Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:24.336 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:24.070Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:24.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:24.578620+0000) 2023-08-27T23:04:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:25 smithi136 ceph-mon[113531]: pgmap v3926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1218854959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:25.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:25.578840+0000) 2023-08-27T23:04:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:25 smithi093 ceph-mon[111335]: pgmap v3926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1218854959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:04:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:04:25] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:04:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:25 smithi093 ceph-mon[118902]: pgmap v3926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1218854959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:26.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:26.579022+0000) 2023-08-27T23:04:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:27 smithi136 ceph-mon[113531]: pgmap v3927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:27.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:27.579285+0000) 2023-08-27T23:04:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:27 smithi093 ceph-mon[118902]: pgmap v3927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:27 smithi093 ceph-mon[111335]: pgmap v3927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1161283129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:28.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:28.579551+0000) 2023-08-27T23:04:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1161283129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1161283129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:29 smithi136 ceph-mon[113531]: pgmap v3928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:29.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:29.579800+0000) 2023-08-27T23:04:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:29 smithi093 ceph-mon[111335]: pgmap v3928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:29 smithi093 ceph-mon[118902]: pgmap v3928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:30.838 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:30.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:30.580078+0000) 2023-08-27T23:04:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3120898734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:30 smithi093 ceph-mon[111335]: pgmap v3929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3120898734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:30 smithi093 ceph-mon[118902]: pgmap v3929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:30.652Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:30.653Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3120898734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:30 smithi136 ceph-mon[113531]: pgmap v3929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:31.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:31.580344+0000) 2023-08-27T23:04:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2858292659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2858292659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:32.150Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:32.151Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:32.580 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2858292659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:32.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:32.580626+0000) 2023-08-27T23:04:33.581 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:33 smithi136 ceph-mon[113531]: pgmap v3930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:33 smithi093 ceph-mon[111335]: pgmap v3930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:33 smithi093 ceph-mon[118902]: pgmap v3930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:33.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:33.580878+0000) 2023-08-27T23:04:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:34.069Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:34.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:34.070Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:34.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:34.581116+0000) 2023-08-27T23:04:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:35 smithi136 ceph-mon[113531]: pgmap v3931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/237545729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:35.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:35.581283+0000) 2023-08-27T23:04:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:35 smithi093 ceph-mon[111335]: pgmap v3931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/237545729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:04:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:04:35] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:04:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:35 smithi093 ceph-mon[118902]: pgmap v3931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/237545729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:36.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:36.581514+0000) 2023-08-27T23:04:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:37 smithi136 ceph-mon[113531]: pgmap v3932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1598542060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:37.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:37.581786+0000) 2023-08-27T23:04:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:37 smithi093 ceph-mon[111335]: pgmap v3932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1598542060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:37 smithi093 ceph-mon[118902]: pgmap v3932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1598542060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:38.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:38.581984+0000) 2023-08-27T23:04:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:39 smithi136 ceph-mon[113531]: pgmap v3933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:39.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:39.582217+0000) 2023-08-27T23:04:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:39 smithi093 ceph-mon[111335]: pgmap v3933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:39 smithi093 ceph-mon[118902]: pgmap v3933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:40.653 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3696964140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:40.654 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3696964140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3696964140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:40.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:40.582428+0000) 2023-08-27T23:04:40.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:40.653Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:40.654Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:41.622 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[118902]: pgmap v3934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:41.622 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:04:41.622 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:41.622 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:41.622 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[111335]: pgmap v3934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:41.623 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:04:41.623 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:41.623 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:41 smithi136 ceph-mon[113531]: pgmap v3934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:04:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:41.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:41.582616+0000) 2023-08-27T23:04:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1051010195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1051010195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1051010195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:42.150Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:42.151Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:42.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:42.582749+0000) 2023-08-27T23:04:43.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:43 smithi136 ceph-mon[113531]: pgmap v3935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:43.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:43.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:43 smithi093 ceph-mon[118902]: pgmap v3935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:43 smithi093 ceph-mon[111335]: pgmap v3935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:43.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:43.582934+0000) 2023-08-27T23:04:44.441 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:44.070Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:44.441 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:44.071Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:04:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:04:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1028335503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[111335]: pgmap v3936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1028335503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:44.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:44 smithi093 ceph-mon[118902]: pgmap v3936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:44.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:44.583107+0000) 2023-08-27T23:04:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:04:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:04:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:04:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1028335503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:44 smithi136 ceph-mon[113531]: pgmap v3936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:04:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:04:45] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:04:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:45.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:45.583335+0000) 2023-08-27T23:04:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:46.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:46.583517+0000) 2023-08-27T23:04:47.584 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:47 smithi136 ceph-mon[113531]: pgmap v3937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:47.584 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3872045401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:47 smithi093 ceph-mon[118902]: pgmap v3937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3872045401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:47 smithi093 ceph-mon[111335]: pgmap v3937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3872045401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:47.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:47.583761+0000) 2023-08-27T23:04:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:48.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:48.584010+0000) 2023-08-27T23:04:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:49 smithi136 ceph-mon[113531]: pgmap v3938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2069102774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:49.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:49.584246+0000) 2023-08-27T23:04:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:49 smithi093 ceph-mon[111335]: pgmap v3938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2069102774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:49 smithi093 ceph-mon[118902]: pgmap v3938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2069102774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:50 smithi136 ceph-mon[113531]: pgmap v3939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:50.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:50.584480+0000) 2023-08-27T23:04:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:50 smithi093 ceph-mon[111335]: pgmap v3939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:50 smithi093 ceph-mon[118902]: pgmap v3939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:50.653Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:50.654Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/17516565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:51.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:51.584717+0000) 2023-08-27T23:04:52.151 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/17516565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:52.151 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/17516565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:52.151Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:52.152Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:52.837 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:52.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:52.584942+0000) 2023-08-27T23:04:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:52 smithi136 ceph-mon[113531]: pgmap v3940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:52 smithi093 ceph-mon[111335]: pgmap v3940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:52 smithi093 ceph-mon[118902]: pgmap v3940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:53.838 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:53.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:53.585197+0000) 2023-08-27T23:04:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1995680418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1995680418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1995680418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:04:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:54.070Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:04:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:04:54.071Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:04:54.843 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:54.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:54.585449+0000) 2023-08-27T23:04:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:54 smithi136 ceph-mon[113531]: pgmap v3941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:54 smithi093 ceph-mon[111335]: pgmap v3941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:54 smithi093 ceph-mon[118902]: pgmap v3941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:04:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:04:55] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:04:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:55.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:55.585761+0000) 2023-08-27T23:04:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2127204684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2127204684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2127204684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:56.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:56.586019+0000) 2023-08-27T23:04:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:57 smithi136 ceph-mon[113531]: pgmap v3942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:57 smithi093 ceph-mon[111335]: pgmap v3942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:57 smithi093 ceph-mon[118902]: pgmap v3942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:57.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:57.586276+0000) 2023-08-27T23:04:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:58.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:58.586523+0000) 2023-08-27T23:04:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:59 smithi136 ceph-mon[113531]: pgmap v3943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:04:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2901909440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:04:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:04:59.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:04:59.586774+0000) 2023-08-27T23:04:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:59 smithi093 ceph-mon[111335]: pgmap v3943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:04:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2901909440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:04:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:59 smithi093 ceph-mon[118902]: pgmap v3943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:04:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:04:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2901909440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:00 smithi136 ceph-mon[113531]: pgmap v3944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:00.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:00.586935+0000) 2023-08-27T23:05:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:00 smithi093 ceph-mon[111335]: pgmap v3944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:00 smithi093 ceph-mon[118902]: pgmap v3944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:00.653Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:00.654Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1048712968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:01.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:01.587191+0000) 2023-08-27T23:05:02.152 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1048712968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:02.152 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1048712968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:02.152Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:02.153Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:02 smithi136 ceph-mon[113531]: pgmap v3945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:02.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:02.587440+0000) 2023-08-27T23:05:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:02 smithi093 ceph-mon[111335]: pgmap v3945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:02 smithi093 ceph-mon[118902]: pgmap v3945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:03.839 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:03.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:03.587675+0000) 2023-08-27T23:05:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/470308046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/470308046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/470308046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:04.071Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:04.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:04.072Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:04.847 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:04.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:04.587884+0000) 2023-08-27T23:05:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:04 smithi136 ceph-mon[113531]: pgmap v3946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:04 smithi093 ceph-mon[111335]: pgmap v3946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:04 smithi093 ceph-mon[118902]: pgmap v3946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:05:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:05:05] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:05:05.848 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:05.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:05.588083+0000) 2023-08-27T23:05:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2638300434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2638300434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2638300434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:06.853 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:06.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:06.588327+0000) 2023-08-27T23:05:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:06 smithi136 ceph-mon[113531]: pgmap v3947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:06 smithi093 ceph-mon[111335]: pgmap v3947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:06 smithi093 ceph-mon[118902]: pgmap v3947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:07.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:07.588568+0000) 2023-08-27T23:05:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4094872231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4094872231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4094872231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:08.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:08.588791+0000) 2023-08-27T23:05:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:09 smithi136 ceph-mon[113531]: pgmap v3948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:09 smithi093 ceph-mon[111335]: pgmap v3948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:09 smithi093 ceph-mon[118902]: pgmap v3948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:09.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:09.588959+0000) 2023-08-27T23:05:10.841 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:10.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:10.589188+0000) 2023-08-27T23:05:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:10 smithi093 ceph-mon[111335]: pgmap v3949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/378636078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:10 smithi093 ceph-mon[118902]: pgmap v3949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/378636078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:10.655Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:10.656Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:10 smithi136 ceph-mon[113531]: pgmap v3949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/378636078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:11.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:11.589438+0000) 2023-08-27T23:05:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:12.152Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:12.154Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:12.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:12.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:12.589655+0000) 2023-08-27T23:05:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:13 smithi136 ceph-mon[113531]: pgmap v3950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2716502982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:13.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:13.589889+0000) 2023-08-27T23:05:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:13 smithi093 ceph-mon[111335]: pgmap v3950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2716502982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:13 smithi093 ceph-mon[118902]: pgmap v3950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2716502982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:14.072Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:14.073Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:14.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:14.590145+0000) 2023-08-27T23:05:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:15 smithi136 ceph-mon[113531]: pgmap v3951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/285295511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:15.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:15.590328+0000) 2023-08-27T23:05:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:15 smithi093 ceph-mon[111335]: pgmap v3951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/285295511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:15 smithi093 ceph-mon[118902]: pgmap v3951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/285295511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:05:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:05:15] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:05:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:16.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:16.590555+0000) 2023-08-27T23:05:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:17 smithi136 ceph-mon[113531]: pgmap v3952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:17.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:17.590802+0000) 2023-08-27T23:05:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:17 smithi093 ceph-mon[111335]: pgmap v3952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:17 smithi093 ceph-mon[118902]: pgmap v3952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2761619966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:18.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:18.591003+0000) 2023-08-27T23:05:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2761619966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2761619966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:19 smithi136 ceph-mon[113531]: pgmap v3953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:19.984 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:19.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:19.591223+0000) 2023-08-27T23:05:19.984 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:19 smithi093 ceph-mon[111335]: pgmap v3953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:19.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:19 smithi093 ceph-mon[118902]: pgmap v3953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3731877000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:20 smithi136 ceph-mon[113531]: pgmap v3954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:20.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:20.591482+0000) 2023-08-27T23:05:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3731877000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:20 smithi093 ceph-mon[111335]: pgmap v3954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3731877000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:20 smithi093 ceph-mon[118902]: pgmap v3954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:20.655Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:20.656Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:21.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:21.591662+0000) 2023-08-27T23:05:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3229643377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3229643377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:22.153Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:22.154Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3229643377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:22.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:22.591914+0000) 2023-08-27T23:05:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:23 smithi136 ceph-mon[113531]: pgmap v3955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:23.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:23.592155+0000) 2023-08-27T23:05:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:23 smithi093 ceph-mon[111335]: pgmap v3955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:23 smithi093 ceph-mon[118902]: pgmap v3955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:24.371 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:24.072Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:24.371 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:24.073Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:24.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:24.592409+0000) 2023-08-27T23:05:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:25 smithi136 ceph-mon[113531]: pgmap v3956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/46286350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:25.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:25.592698+0000) 2023-08-27T23:05:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:25 smithi093 ceph-mon[111335]: pgmap v3956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/46286350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:05:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:05:25] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:05:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:25 smithi093 ceph-mon[118902]: pgmap v3956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/46286350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:26.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:26.592935+0000) 2023-08-27T23:05:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:27.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:27.593184+0000) 2023-08-27T23:05:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:27 smithi136 ceph-mon[113531]: pgmap v3957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3180113062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:27 smithi093 ceph-mon[118902]: pgmap v3957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3180113062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:27 smithi093 ceph-mon[111335]: pgmap v3957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3180113062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:28.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:28.593457+0000) 2023-08-27T23:05:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:29 smithi136 ceph-mon[113531]: pgmap v3958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:29.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:29.593681+0000) 2023-08-27T23:05:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:29 smithi093 ceph-mon[111335]: pgmap v3958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:29 smithi093 ceph-mon[118902]: pgmap v3958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:30.656 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1054861102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:30.657 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1054861102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1054861102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:30.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:30.593926+0000) 2023-08-27T23:05:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:30.656Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:30.657Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:31 smithi136 ceph-mon[113531]: pgmap v3959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:31.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:31.594172+0000) 2023-08-27T23:05:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:31 smithi093 ceph-mon[111335]: pgmap v3959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:31 smithi093 ceph-mon[118902]: pgmap v3959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1552173629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1552173629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:32.154Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:32.155Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1552173629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:32.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:32.594385+0000) 2023-08-27T23:05:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:33 smithi136 ceph-mon[113531]: pgmap v3960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:33.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:33.594611+0000) 2023-08-27T23:05:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:33 smithi093 ceph-mon[111335]: pgmap v3960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:33 smithi093 ceph-mon[118902]: pgmap v3960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:34.391 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:34.073Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:34.391 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:34 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:34.074Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1280880024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:34.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:34.594847+0000) 2023-08-27T23:05:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1280880024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1280880024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:35 smithi136 ceph-mon[113531]: pgmap v3961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:35.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:35.595097+0000) 2023-08-27T23:05:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:35 smithi093 ceph-mon[111335]: pgmap v3961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:35 smithi093 ceph-mon[118902]: pgmap v3961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:05:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:05:35] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:05:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:36.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:36.595344+0000) 2023-08-27T23:05:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:37 smithi136 ceph-mon[113531]: pgmap v3962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/82465677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:37.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:37.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:37.595591+0000) 2023-08-27T23:05:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:37 smithi093 ceph-mon[111335]: pgmap v3962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/82465677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:37 smithi093 ceph-mon[118902]: pgmap v3962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/82465677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:38.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:38.595841+0000) 2023-08-27T23:05:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:39.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:39.596091+0000) 2023-08-27T23:05:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:39 smithi136 ceph-mon[113531]: pgmap v3963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/252038410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:39 smithi093 ceph-mon[118902]: pgmap v3963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/252038410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:39 smithi093 ceph-mon[111335]: pgmap v3963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:39.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/252038410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:40 smithi136 ceph-mon[113531]: pgmap v3964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:40.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:40.596295+0000) 2023-08-27T23:05:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:40 smithi093 ceph-mon[111335]: pgmap v3964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:40 smithi093 ceph-mon[118902]: pgmap v3964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:40.657Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:40.658Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2159375836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:41.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:41.596539+0000) 2023-08-27T23:05:42.155 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2159375836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:42.155 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2159375836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:42.155Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:42.155Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:42 smithi136 ceph-mon[113531]: pgmap v3965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:42.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:42.596793+0000) 2023-08-27T23:05:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:42 smithi093 ceph-mon[111335]: pgmap v3965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:42 smithi093 ceph-mon[118902]: pgmap v3965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2122550044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:43.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:43.597062+0000) 2023-08-27T23:05:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2122550044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2122550044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:44.074Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:44.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:44 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:44.074Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:44.851 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:44.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:44.597233+0000) 2023-08-27T23:05:45.144 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:05:45.144 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:44 smithi093 ceph-mon[118902]: pgmap v3966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:45.144 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:05:45.144 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:44 smithi093 ceph-mon[111335]: pgmap v3966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:05:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:44 smithi136 ceph-mon[113531]: pgmap v3966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:45.469 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:05:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:05:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:05:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:45.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:45.597380+0000) 2023-08-27T23:05:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:46.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:46.597525+0000) 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1413370875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: pgmap v3967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1413370875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: pgmap v3967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1413370875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: pgmap v3967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:05:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:05:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:47.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:47.597803+0000) 2023-08-27T23:05:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:48.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:48.598060+0000) 2023-08-27T23:05:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:49 smithi136 ceph-mon[113531]: pgmap v3968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1166239976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:49.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:49.598300+0000) 2023-08-27T23:05:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:49 smithi093 ceph-mon[111335]: pgmap v3968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1166239976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:49 smithi093 ceph-mon[118902]: pgmap v3968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1166239976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:50 smithi136 ceph-mon[113531]: pgmap v3969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:50.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:50.598533+0000) 2023-08-27T23:05:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:50 smithi093 ceph-mon[111335]: pgmap v3969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:50 smithi093 ceph-mon[118902]: pgmap v3969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:50.658Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:50.659Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/318158478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:51.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:51.598748+0000) 2023-08-27T23:05:52.155 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/318158478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:52.156 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/318158478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:52.155Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:52.156Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:52 smithi136 ceph-mon[113531]: pgmap v3970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:52.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:52.599055+0000) 2023-08-27T23:05:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:52 smithi093 ceph-mon[111335]: pgmap v3970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:52 smithi093 ceph-mon[118902]: pgmap v3970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/371499356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:53.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:53.599326+0000) 2023-08-27T23:05:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/371499356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/371499356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:05:54.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:54.074Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:54.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:05:54 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:05:54.076Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:05:54.857 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:54.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:54.599545+0000) 2023-08-27T23:05:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:54 smithi136 ceph-mon[113531]: pgmap v3971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:54 smithi093 ceph-mon[111335]: pgmap v3971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:54 smithi093 ceph-mon[118902]: pgmap v3971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:05:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:05:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:05:55.860 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:55.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:55.599834+0000) 2023-08-27T23:05:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/181941451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/181941451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/181941451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:56.867 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:56.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:56.600084+0000) 2023-08-27T23:05:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:56 smithi136 ceph-mon[113531]: pgmap v3972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:56 smithi093 ceph-mon[118902]: pgmap v3972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:56 smithi093 ceph-mon[111335]: pgmap v3972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:57.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:57.600273+0000) 2023-08-27T23:05:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3404977452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3404977452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3404977452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:05:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:58.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:58.600515+0000) 2023-08-27T23:05:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:05:59 smithi136 ceph-mon[113531]: pgmap v3973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:05:59 smithi093 ceph-mon[111335]: pgmap v3973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:05:59 smithi093 ceph-mon[118902]: pgmap v3973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:05:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:05:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:05:59.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:05:59.600730+0000) 2023-08-27T23:06:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:00 smithi136 ceph-mon[113531]: pgmap v3974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/347582041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:00.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:00.601003+0000) 2023-08-27T23:06:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:00 smithi093 ceph-mon[111335]: pgmap v3974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/347582041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:00 smithi093 ceph-mon[118902]: pgmap v3974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/347582041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:00.658Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:00.659Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:01.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:01.601206+0000) 2023-08-27T23:06:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:02.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:02.156Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:02.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:02.601462+0000) 2023-08-27T23:06:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:03 smithi136 ceph-mon[113531]: pgmap v3975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4151132993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:03.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:03.601743+0000) 2023-08-27T23:06:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:03 smithi093 ceph-mon[111335]: pgmap v3975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4151132993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:03 smithi093 ceph-mon[118902]: pgmap v3975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4151132993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:04.075Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:04.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:04 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:04.076Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:04.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:04.602017+0000) 2023-08-27T23:06:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:05 smithi136 ceph-mon[113531]: pgmap v3976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1492088233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:05.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:05.602285+0000) 2023-08-27T23:06:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:05 smithi093 ceph-mon[111335]: pgmap v3976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1492088233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:05 smithi093 ceph-mon[118902]: pgmap v3976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1492088233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:06:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:06:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:06:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:06.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:06.602528+0000) 2023-08-27T23:06:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:07 smithi136 ceph-mon[113531]: pgmap v3977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:07.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:07.602778+0000) 2023-08-27T23:06:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:07 smithi093 ceph-mon[111335]: pgmap v3977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:07 smithi093 ceph-mon[118902]: pgmap v3977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1524248001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:08.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:08.603020+0000) 2023-08-27T23:06:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1524248001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1524248001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:09 smithi136 ceph-mon[113531]: pgmap v3978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:09.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:09.603246+0000) 2023-08-27T23:06:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:09 smithi093 ceph-mon[111335]: pgmap v3978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:09 smithi093 ceph-mon[118902]: pgmap v3978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3553387033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:10 smithi136 ceph-mon[113531]: pgmap v3979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:10.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:10.603481+0000) 2023-08-27T23:06:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3553387033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:10 smithi093 ceph-mon[111335]: pgmap v3979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3553387033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:10 smithi093 ceph-mon[118902]: pgmap v3979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:10.659Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:10.660Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:11.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:11.603688+0000) 2023-08-27T23:06:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:12.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:12.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:12.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:12.603928+0000) 2023-08-27T23:06:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:13 smithi136 ceph-mon[113531]: pgmap v3980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/628482389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:13.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:13.604178+0000) 2023-08-27T23:06:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:13 smithi093 ceph-mon[111335]: pgmap v3980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/628482389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:13 smithi093 ceph-mon[118902]: pgmap v3980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/628482389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:14.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:14.075Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:14.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:14 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:14.076Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:14.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:14.604437+0000) 2023-08-27T23:06:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:15 smithi136 ceph-mon[113531]: pgmap v3981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1830868737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:15.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:15.604643+0000) 2023-08-27T23:06:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:06:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:06:15] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:06:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:15 smithi093 ceph-mon[111335]: pgmap v3981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1830868737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:15 smithi093 ceph-mon[118902]: pgmap v3981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1830868737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:16.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:16.604858+0000) 2023-08-27T23:06:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:17 smithi136 ceph-mon[113531]: pgmap v3982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2792802442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:17.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:17.605152+0000) 2023-08-27T23:06:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:17 smithi093 ceph-mon[111335]: pgmap v3982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2792802442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:17 smithi093 ceph-mon[118902]: pgmap v3982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2792802442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:18.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:18.605387+0000) 2023-08-27T23:06:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:19 smithi136 ceph-mon[113531]: pgmap v3983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:19.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:19.605605+0000) 2023-08-27T23:06:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:19 smithi093 ceph-mon[111335]: pgmap v3983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:19 smithi093 ceph-mon[118902]: pgmap v3983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:20.660 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3712180857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:20.660 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3712180857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3712180857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:20.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:20.605851+0000) 2023-08-27T23:06:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:20.660Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:20.661Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:21 smithi136 ceph-mon[113531]: pgmap v3984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:21.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:21.606091+0000) 2023-08-27T23:06:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:21 smithi093 ceph-mon[111335]: pgmap v3984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:21 smithi093 ceph-mon[118902]: pgmap v3984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/861035858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/861035858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:22.157Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:22.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/861035858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:22.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:22.606344+0000) 2023-08-27T23:06:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:23 smithi136 ceph-mon[113531]: pgmap v3985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:23.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:23.606640+0000) 2023-08-27T23:06:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:23 smithi093 ceph-mon[111335]: pgmap v3985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:23 smithi093 ceph-mon[118902]: pgmap v3985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:24.378 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:24 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:24.076Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=4 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4123480448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:24.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:24.606902+0000) 2023-08-27T23:06:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4123480448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4123480448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:25 smithi136 ceph-mon[113531]: pgmap v3986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:25.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:25.607119+0000) 2023-08-27T23:06:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:25 smithi093 ceph-mon[111335]: pgmap v3986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:25 smithi093 ceph-mon[118902]: pgmap v3986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:06:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:06:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:06:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:26.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:26.607350+0000) 2023-08-27T23:06:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:27.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:27.607509+0000) 2023-08-27T23:06:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:27 smithi136 ceph-mon[113531]: pgmap v3987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3189889668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:27 smithi093 ceph-mon[111335]: pgmap v3987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3189889668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:27 smithi093 ceph-mon[118902]: pgmap v3987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3189889668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:28.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:28.607767+0000) 2023-08-27T23:06:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:29 smithi136 ceph-mon[113531]: pgmap v3988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3972753522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:29.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:29.608007+0000) 2023-08-27T23:06:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:29 smithi093 ceph-mon[118902]: pgmap v3988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3972753522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:29 smithi093 ceph-mon[111335]: pgmap v3988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3972753522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:30 smithi136 ceph-mon[113531]: pgmap v3989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:30.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:30.608288+0000) 2023-08-27T23:06:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:30 smithi093 ceph-mon[111335]: pgmap v3989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:30 smithi093 ceph-mon[118902]: pgmap v3989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:30.661Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:30.662Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3972182041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:31.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:31.608538+0000) 2023-08-27T23:06:32.158 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3972182041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:32.158 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3972182041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:32.158Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:32.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:32 smithi136 ceph-mon[113531]: pgmap v3990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:32.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:32.608808+0000) 2023-08-27T23:06:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:32 smithi093 ceph-mon[111335]: pgmap v3990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:32 smithi093 ceph-mon[118902]: pgmap v3990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:33.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3973534997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:33.943 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:33.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:33.609120+0000) 2023-08-27T23:06:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3973534997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3973534997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:34 smithi136 ceph-mon[113531]: pgmap v3991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:34.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:34.609353+0000) 2023-08-27T23:06:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:34 smithi093 ceph-mon[111335]: pgmap v3991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:34 smithi093 ceph-mon[118902]: pgmap v3991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:06:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:06:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:06:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:35.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:35.609576+0000) 2023-08-27T23:06:36.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3931414397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3931414397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3931414397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:36.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:36.609785+0000) 2023-08-27T23:06:37.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:37 smithi136 ceph-mon[113531]: pgmap v3992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:37 smithi093 ceph-mon[111335]: pgmap v3992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:37 smithi093 ceph-mon[118902]: pgmap v3992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:37.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:37.610061+0000) 2023-08-27T23:06:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:38.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:38.610314+0000) 2023-08-27T23:06:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:39 smithi136 ceph-mon[113531]: pgmap v3993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2679907921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:39.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:39.610558+0000) 2023-08-27T23:06:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:39 smithi093 ceph-mon[111335]: pgmap v3993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2679907921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:39 smithi093 ceph-mon[118902]: pgmap v3993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2679907921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:40 smithi136 ceph-mon[113531]: pgmap v3994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:40.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:40.610802+0000) 2023-08-27T23:06:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:40 smithi093 ceph-mon[111335]: pgmap v3994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:40 smithi093 ceph-mon[118902]: pgmap v3994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:40.661Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:40.662Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3873758177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:41.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:41.611060+0000) 2023-08-27T23:06:42.158 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3873758177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:42.159 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3873758177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:42.158Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:42.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:42 smithi136 ceph-mon[113531]: pgmap v3995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:42.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:42.611281+0000) 2023-08-27T23:06:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:42 smithi093 ceph-mon[111335]: pgmap v3995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:42 smithi093 ceph-mon[118902]: pgmap v3995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3722565673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:43.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:43.611532+0000) 2023-08-27T23:06:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3722565673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3722565673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:44.866 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:44.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:44.611752+0000) 2023-08-27T23:06:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:44 smithi136 ceph-mon[113531]: pgmap v3996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:44 smithi093 ceph-mon[111335]: pgmap v3996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:44 smithi093 ceph-mon[118902]: pgmap v3996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:06:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:06:45] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:06:45.865 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:45.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:45.612026+0000) 2023-08-27T23:06:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/24192215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/24192215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/24192215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:46.873 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:46.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:46.612302+0000) 2023-08-27T23:06:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:46 smithi136 ceph-mon[113531]: pgmap v3997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:06:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:46 smithi093 ceph-mon[111335]: pgmap v3997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:06:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:46 smithi093 ceph-mon[118902]: pgmap v3997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:06:47.897 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:47.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:47.612495+0000) 2023-08-27T23:06:48.411 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1179286320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:48.411 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1179286320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1179286320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:48.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:48.612639+0000) 2023-08-27T23:06:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:49 smithi136 ceph-mon[113531]: pgmap v3998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:49.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:49.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:49 smithi093 ceph-mon[111335]: pgmap v3998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:49 smithi093 ceph-mon[118902]: pgmap v3998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:49.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:49.612895+0000) 2023-08-27T23:06:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:06:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:06:50.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:06:50.614 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:06:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:50.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:50.613159+0000) 2023-08-27T23:06:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:50.662Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:50.664Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:51 smithi093 ceph-mon[118902]: pgmap v3999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1646367767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:51 smithi093 ceph-mon[111335]: pgmap v3999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1646367767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:51.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:51 smithi136 ceph-mon[113531]: pgmap v3999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:51.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1646367767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:51.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:51.613387+0000) 2023-08-27T23:06:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:52.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:06:52.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:06:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:52.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:52.613632+0000) 2023-08-27T23:06:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:53.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:53.613898+0000) 2023-08-27T23:06:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:53 smithi136 ceph-mon[113531]: pgmap v4000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/953038236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:53 smithi093 ceph-mon[111335]: pgmap v4000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/953038236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:53 smithi093 ceph-mon[118902]: pgmap v4000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/953038236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:54.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:54.614152+0000) 2023-08-27T23:06:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:06:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:55 smithi136 ceph-mon[113531]: pgmap v4001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1261340563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:55.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:55.614352+0000) 2023-08-27T23:06:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:55 smithi093 ceph-mon[111335]: pgmap v4001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1261340563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:06:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:06:55] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:06:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:55 smithi093 ceph-mon[118902]: pgmap v4001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1261340563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:56.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:56.614614+0000) 2023-08-27T23:06:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:57 smithi136 ceph-mon[113531]: pgmap v4002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:57.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:57.614866+0000) 2023-08-27T23:06:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:57 smithi093 ceph-mon[111335]: pgmap v4002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:57 smithi093 ceph-mon[118902]: pgmap v4002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3634502060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:58.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:58.615135+0000) 2023-08-27T23:06:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3634502060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3634502060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:06:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:06:59 smithi136 ceph-mon[113531]: pgmap v4003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:06:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:06:59.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:06:59.615389+0000) 2023-08-27T23:06:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:06:59 smithi093 ceph-mon[111335]: pgmap v4003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:06:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:06:59 smithi093 ceph-mon[118902]: pgmap v4003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:00.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:00.615620+0000) 2023-08-27T23:07:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3633794906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:00 smithi136 ceph-mon[113531]: pgmap v4004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3633794906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:00 smithi093 ceph-mon[111335]: pgmap v4004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3633794906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:00 smithi093 ceph-mon[118902]: pgmap v4004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:00.663Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:00.664Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:01.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:01.615808+0000) 2023-08-27T23:07:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:02.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:02.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:02.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:02.616064+0000) 2023-08-27T23:07:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:03 smithi136 ceph-mon[113531]: pgmap v4005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4105812324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:03.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:03.616316+0000) 2023-08-27T23:07:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:03 smithi093 ceph-mon[111335]: pgmap v4005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4105812324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:03 smithi093 ceph-mon[118902]: pgmap v4005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4105812324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:04.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:04.616598+0000) 2023-08-27T23:07:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:05 smithi136 ceph-mon[113531]: pgmap v4006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1667629020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:05.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:05.616864+0000) 2023-08-27T23:07:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:07:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:07:05] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:07:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:05 smithi093 ceph-mon[111335]: pgmap v4006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1667629020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:05 smithi093 ceph-mon[118902]: pgmap v4006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1667629020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:06.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:06.617131+0000) 2023-08-27T23:07:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:07.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:07.617357+0000) 2023-08-27T23:07:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:07 smithi136 ceph-mon[113531]: pgmap v4007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2565521447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:07 smithi093 ceph-mon[111335]: pgmap v4007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2565521447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:07 smithi093 ceph-mon[118902]: pgmap v4007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2565521447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:08.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:08.617588+0000) 2023-08-27T23:07:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:09.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:09.617847+0000) 2023-08-27T23:07:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:09 smithi136 ceph-mon[113531]: pgmap v4008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:09 smithi093 ceph-mon[118902]: pgmap v4008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:09 smithi093 ceph-mon[111335]: pgmap v4008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:10.664 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/450902819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:10.664 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/450902819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/450902819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:10.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:10.618122+0000) 2023-08-27T23:07:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:10.664Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:10.665Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:11 smithi136 ceph-mon[113531]: pgmap v4009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:11.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:11.618322+0000) 2023-08-27T23:07:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:11 smithi093 ceph-mon[111335]: pgmap v4009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:11 smithi093 ceph-mon[118902]: pgmap v4009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/608757708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/608757708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:12.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:12.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/608757708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:12.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:12.618532+0000) 2023-08-27T23:07:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:13 smithi136 ceph-mon[113531]: pgmap v4010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:13.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:13.618772+0000) 2023-08-27T23:07:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:13 smithi093 ceph-mon[111335]: pgmap v4010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:13 smithi093 ceph-mon[118902]: pgmap v4010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:14.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4087016553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:14.692 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:14.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:14.619074+0000) 2023-08-27T23:07:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4087016553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4087016553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:15 smithi136 ceph-mon[113531]: pgmap v4011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:15.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:15.619326+0000) 2023-08-27T23:07:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:15 smithi093 ceph-mon[118902]: pgmap v4011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:15 smithi093 ceph-mon[111335]: pgmap v4011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:07:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:07:15] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:07:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:16.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:16.619613+0000) 2023-08-27T23:07:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:17.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:17.619888+0000) 2023-08-27T23:07:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:17 smithi136 ceph-mon[113531]: pgmap v4012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/108572497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:17 smithi093 ceph-mon[118902]: pgmap v4012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/108572497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:17 smithi093 ceph-mon[111335]: pgmap v4012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/108572497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:18.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:18.620162+0000) 2023-08-27T23:07:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:19 smithi136 ceph-mon[113531]: pgmap v4013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2514017738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:19.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:19.620437+0000) 2023-08-27T23:07:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:19 smithi093 ceph-mon[111335]: pgmap v4013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2514017738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:19 smithi093 ceph-mon[118902]: pgmap v4013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2514017738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:20 smithi136 ceph-mon[113531]: pgmap v4014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:20.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:20.620664+0000) 2023-08-27T23:07:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:20 smithi093 ceph-mon[111335]: pgmap v4014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:20 smithi093 ceph-mon[118902]: pgmap v4014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:20.664Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:20.666Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/711432112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:21.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:21.620922+0000) 2023-08-27T23:07:22.161 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/711432112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:22.161 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/711432112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:22.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:22.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:22 smithi136 ceph-mon[113531]: pgmap v4015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:22.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:22.621200+0000) 2023-08-27T23:07:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:22 smithi093 ceph-mon[111335]: pgmap v4015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:22 smithi093 ceph-mon[118902]: pgmap v4015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1940102164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:23.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:23.621396+0000) 2023-08-27T23:07:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1940102164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1940102164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:24 smithi136 ceph-mon[113531]: pgmap v4016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:24.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:24.621643+0000) 2023-08-27T23:07:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:24 smithi093 ceph-mon[111335]: pgmap v4016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:24 smithi093 ceph-mon[118902]: pgmap v4016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:07:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:07:25] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:07:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:25.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:25.621843+0000) 2023-08-27T23:07:26.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1097039224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1097039224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1097039224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:26.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:26.622096+0000) 2023-08-27T23:07:27.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:27 smithi136 ceph-mon[113531]: pgmap v4017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:27 smithi093 ceph-mon[111335]: pgmap v4017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:27 smithi093 ceph-mon[118902]: pgmap v4017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:27.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:27.622366+0000) 2023-08-27T23:07:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:28.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:28.622610+0000) 2023-08-27T23:07:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:29.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:29.622842+0000) 2023-08-27T23:07:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:29 smithi136 ceph-mon[113531]: pgmap v4018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/702471401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:29 smithi093 ceph-mon[111335]: pgmap v4018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/702471401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:29 smithi093 ceph-mon[118902]: pgmap v4018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/702471401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:30 smithi136 ceph-mon[113531]: pgmap v4019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:30.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:30.623079+0000) 2023-08-27T23:07:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:30 smithi093 ceph-mon[111335]: pgmap v4019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:30 smithi093 ceph-mon[118902]: pgmap v4019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:30.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:30.665Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:30.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:30.666Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3992520009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:31.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:31.623249+0000) 2023-08-27T23:07:32.162 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3992520009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:32.162 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3992520009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:32.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:32.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:32 smithi136 ceph-mon[113531]: pgmap v4020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:32.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:32.623439+0000) 2023-08-27T23:07:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:32 smithi093 ceph-mon[111335]: pgmap v4020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:32 smithi093 ceph-mon[118902]: pgmap v4020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3442599567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:33.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:33.623654+0000) 2023-08-27T23:07:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3442599567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3442599567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:34.874 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:34.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:34.623913+0000) 2023-08-27T23:07:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:34 smithi136 ceph-mon[113531]: pgmap v4021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:34 smithi093 ceph-mon[111335]: pgmap v4021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:34 smithi093 ceph-mon[118902]: pgmap v4021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:07:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:07:35] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:07:35.876 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:35.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:35.624206+0000) 2023-08-27T23:07:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1038848969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1038848969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1038848969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:36.880 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:36.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:36.624414+0000) 2023-08-27T23:07:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:36 smithi136 ceph-mon[113531]: pgmap v4022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:36 smithi093 ceph-mon[111335]: pgmap v4022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:36 smithi093 ceph-mon[118902]: pgmap v4022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:37.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:37.624676+0000) 2023-08-27T23:07:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3365979298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3365979298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:38.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3365979298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:38.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:38.624924+0000) 2023-08-27T23:07:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:39 smithi093 ceph-mon[111335]: pgmap v4023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:39 smithi093 ceph-mon[118902]: pgmap v4023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:39.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:39 smithi136 ceph-mon[113531]: pgmap v4023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:39.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:39.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:39.625204+0000) 2023-08-27T23:07:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:40.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:40.625427+0000) 2023-08-27T23:07:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:40 smithi136 ceph-mon[113531]: pgmap v4024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/705283917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:40 smithi093 ceph-mon[118902]: pgmap v4024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/705283917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:40 smithi093 ceph-mon[111335]: pgmap v4024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/705283917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:40.666Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:40.667Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:41.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:41.625645+0000) 2023-08-27T23:07:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:42.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:42.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:42.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:42.625915+0000) 2023-08-27T23:07:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:43 smithi136 ceph-mon[113531]: pgmap v4025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/366559181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:43.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:43.626202+0000) 2023-08-27T23:07:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:43 smithi093 ceph-mon[111335]: pgmap v4025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/366559181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:43 smithi093 ceph-mon[118902]: pgmap v4025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/366559181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:44.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:44.626445+0000) 2023-08-27T23:07:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:45 smithi136 ceph-mon[113531]: pgmap v4026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2157926388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:45.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:45.626700+0000) 2023-08-27T23:07:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:45 smithi093 ceph-mon[118902]: pgmap v4026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2157926388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:45 smithi093 ceph-mon[111335]: pgmap v4026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2157926388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:07:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:07:45] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:07:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:46.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:46.626914+0000) 2023-08-27T23:07:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:47 smithi136 ceph-mon[113531]: pgmap v4027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:47.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:47.627180+0000) 2023-08-27T23:07:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:47 smithi093 ceph-mon[111335]: pgmap v4027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:47 smithi093 ceph-mon[118902]: pgmap v4027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/484309362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:48.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:48.627433+0000) 2023-08-27T23:07:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/484309362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/484309362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:49 smithi136 ceph-mon[113531]: pgmap v4028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:49.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:49.627680+0000) 2023-08-27T23:07:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:49 smithi093 ceph-mon[111335]: pgmap v4028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:49 smithi093 ceph-mon[118902]: pgmap v4028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:50.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:07:50.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/279231598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:50.666 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:07:50.666 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/279231598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:50.667 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:07:50.667 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/279231598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:50.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:50.627927+0000) 2023-08-27T23:07:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:50.666Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:50.667Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:51.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:51 smithi136 ceph-mon[113531]: pgmap v4029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:51 smithi093 ceph-mon[111335]: pgmap v4029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:51 smithi093 ceph-mon[118902]: pgmap v4029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:51.926 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:51.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:51.628092+0000) 2023-08-27T23:07:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:52.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:07:52.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:07:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:52.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:52.628315+0000) 2023-08-27T23:07:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:52 smithi136 ceph-mon[113531]: pgmap v4030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1196502662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[118902]: pgmap v4030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1196502662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:52.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[111335]: pgmap v4030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:52.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1196502662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:53.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:53.628587+0000) 2023-08-27T23:07:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:07:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:07:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:07:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:07:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:07:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:07:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:07:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:54 smithi136 ceph-mon[113531]: pgmap v4031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:54.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:54.628830+0000) 2023-08-27T23:07:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:54 smithi093 ceph-mon[111335]: pgmap v4031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:07:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:54 smithi093 ceph-mon[118902]: pgmap v4031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3250345077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:07:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:07:55] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:07:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3250345077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3250345077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:55.947 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:55.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:55.629102+0000) 2023-08-27T23:07:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:56 smithi136 ceph-mon[113531]: pgmap v4032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:56.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:56.629298+0000) 2023-08-27T23:07:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:56 smithi093 ceph-mon[111335]: pgmap v4032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:56 smithi093 ceph-mon[118902]: pgmap v4032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2212632092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:57.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:57.629511+0000) 2023-08-27T23:07:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2212632092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:57.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2212632092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:58 smithi136 ceph-mon[113531]: pgmap v4033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:58.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:58.629747+0000) 2023-08-27T23:07:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:58 smithi093 ceph-mon[111335]: pgmap v4033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:58 smithi093 ceph-mon[118902]: pgmap v4033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:07:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:07:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/920610675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:07:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:07:59.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:07:59.630009+0000) 2023-08-27T23:07:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:07:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/920610675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:07:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:07:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/920610675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:00.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:00 smithi136 ceph-mon[113531]: pgmap v4034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:00.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:00.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:00.630226+0000) 2023-08-27T23:08:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:00 smithi093 ceph-mon[111335]: pgmap v4034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:00 smithi093 ceph-mon[118902]: pgmap v4034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:00.667Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:00.668Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:01.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:01.630475+0000) 2023-08-27T23:08:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4073491726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4073491726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4073491726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:02.163Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:02.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:02.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:02.630755+0000) 2023-08-27T23:08:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:03 smithi136 ceph-mon[113531]: pgmap v4035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:03 smithi093 ceph-mon[111335]: pgmap v4035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:03 smithi093 ceph-mon[118902]: pgmap v4035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:03.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:03.630994+0000) 2023-08-27T23:08:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/10358772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:04.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:04.631263+0000) 2023-08-27T23:08:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/10358772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/10358772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:05 smithi136 ceph-mon[113531]: pgmap v4036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:05.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:05.631549+0000) 2023-08-27T23:08:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:05 smithi093 ceph-mon[118902]: pgmap v4036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:05 smithi093 ceph-mon[111335]: pgmap v4036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:08:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:08:05] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:08:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:06.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:06.631803+0000) 2023-08-27T23:08:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:07 smithi136 ceph-mon[113531]: pgmap v4037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1849509227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:07.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:07.632088+0000) 2023-08-27T23:08:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:07 smithi093 ceph-mon[111335]: pgmap v4037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1849509227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:07 smithi093 ceph-mon[118902]: pgmap v4037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1849509227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:08.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:08.632301+0000) 2023-08-27T23:08:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:09 smithi136 ceph-mon[113531]: pgmap v4038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2155582302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:09.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:09.632554+0000) 2023-08-27T23:08:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:09 smithi093 ceph-mon[111335]: pgmap v4038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2155582302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:09 smithi093 ceph-mon[118902]: pgmap v4038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2155582302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:10 smithi136 ceph-mon[113531]: pgmap v4039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:10.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:10.632810+0000) 2023-08-27T23:08:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:10 smithi093 ceph-mon[111335]: pgmap v4039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:10 smithi093 ceph-mon[118902]: pgmap v4039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:10.667Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:10.668Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1538881497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:11.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:11.633082+0000) 2023-08-27T23:08:12.163 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1538881497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:12.164 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1538881497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:12.163Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:12.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:12 smithi136 ceph-mon[113531]: pgmap v4040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:12.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:12.633356+0000) 2023-08-27T23:08:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:12 smithi093 ceph-mon[111335]: pgmap v4040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:12 smithi093 ceph-mon[118902]: pgmap v4040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:13.934 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:13.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:13.633654+0000) 2023-08-27T23:08:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2576406829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2576406829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2576406829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:14.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:14.633961+0000) 2023-08-27T23:08:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:14 smithi093 ceph-mon[111335]: pgmap v4041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:14 smithi093 ceph-mon[118902]: pgmap v4041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:14 smithi136 ceph-mon[113531]: pgmap v4041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:08:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:08:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:08:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:15.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:15.634227+0000) 2023-08-27T23:08:16.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2869652546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2869652546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2869652546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:16.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:16.634425+0000) 2023-08-27T23:08:17.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:17 smithi136 ceph-mon[113531]: pgmap v4042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:17 smithi093 ceph-mon[111335]: pgmap v4042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:17 smithi093 ceph-mon[118902]: pgmap v4042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:17.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:17.634668+0000) 2023-08-27T23:08:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:18.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:18.634926+0000) 2023-08-27T23:08:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:19 smithi136 ceph-mon[113531]: pgmap v4043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1693077469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:19.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:19.635140+0000) 2023-08-27T23:08:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:19 smithi093 ceph-mon[111335]: pgmap v4043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1693077469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:19 smithi093 ceph-mon[118902]: pgmap v4043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1693077469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:20 smithi136 ceph-mon[113531]: pgmap v4044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:20.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:20.635318+0000) 2023-08-27T23:08:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:20 smithi093 ceph-mon[111335]: pgmap v4044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:20 smithi093 ceph-mon[118902]: pgmap v4044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:20.668Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:20.669Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1232496013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:21.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:21.635556+0000) 2023-08-27T23:08:22.164 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1232496013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:22.165 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1232496013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:22.164Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:22.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:22 smithi136 ceph-mon[113531]: pgmap v4045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:22.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:22.635825+0000) 2023-08-27T23:08:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:22 smithi093 ceph-mon[111335]: pgmap v4045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:22 smithi093 ceph-mon[118902]: pgmap v4045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4233432389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:23.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:23.636124+0000) 2023-08-27T23:08:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4233432389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4233432389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:24 smithi136 ceph-mon[113531]: pgmap v4046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:24.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:24.636377+0000) 2023-08-27T23:08:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:24 smithi093 ceph-mon[111335]: pgmap v4046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:24 smithi093 ceph-mon[118902]: pgmap v4046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:08:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:08:25] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:08:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2482495880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:25.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:25.636616+0000) 2023-08-27T23:08:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2482495880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2482495880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:26 smithi136 ceph-mon[113531]: pgmap v4047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:26.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:26.636863+0000) 2023-08-27T23:08:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:26 smithi093 ceph-mon[118902]: pgmap v4047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:26 smithi093 ceph-mon[111335]: pgmap v4047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:27.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:27.637123+0000) 2023-08-27T23:08:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2032367682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2032367682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:28.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2032367682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:28.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:28.637342+0000) 2023-08-27T23:08:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:29 smithi093 ceph-mon[111335]: pgmap v4048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:29 smithi093 ceph-mon[118902]: pgmap v4048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:29.638 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:29 smithi136 ceph-mon[113531]: pgmap v4048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:29.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:29.637632+0000) 2023-08-27T23:08:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:30.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:30.637915+0000) 2023-08-27T23:08:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:30 smithi136 ceph-mon[113531]: pgmap v4049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4180216161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:30 smithi093 ceph-mon[111335]: pgmap v4049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4180216161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:30 smithi093 ceph-mon[118902]: pgmap v4049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4180216161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:30.668Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:30.669Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:31.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:31.638170+0000) 2023-08-27T23:08:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:32.164Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:32.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:32.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:32.638473+0000) 2023-08-27T23:08:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:33.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:33.638774+0000) 2023-08-27T23:08:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:33 smithi136 ceph-mon[113531]: pgmap v4050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471521578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:33 smithi093 ceph-mon[111335]: pgmap v4050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471521578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:33 smithi093 ceph-mon[118902]: pgmap v4050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471521578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:34.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:34.639090+0000) 2023-08-27T23:08:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:35 smithi136 ceph-mon[113531]: pgmap v4051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2327868603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:35.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:35.639391+0000) 2023-08-27T23:08:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:35 smithi093 ceph-mon[111335]: pgmap v4051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2327868603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:35 smithi093 ceph-mon[118902]: pgmap v4051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2327868603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:35.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:08:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:08:35] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:08:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:36.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:36.639673+0000) 2023-08-27T23:08:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:37 smithi136 ceph-mon[113531]: pgmap v4052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:37.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:37.639917+0000) 2023-08-27T23:08:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:37 smithi093 ceph-mon[111335]: pgmap v4052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:37 smithi093 ceph-mon[118902]: pgmap v4052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1377889054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:38.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:38.640170+0000) 2023-08-27T23:08:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1377889054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:38.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1377889054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:39 smithi136 ceph-mon[113531]: pgmap v4053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:39.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:39.640389+0000) 2023-08-27T23:08:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:39 smithi093 ceph-mon[111335]: pgmap v4053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:39 smithi093 ceph-mon[118902]: pgmap v4053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/477795661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:40 smithi136 ceph-mon[113531]: pgmap v4054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:40.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:40.640644+0000) 2023-08-27T23:08:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/477795661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:40 smithi093 ceph-mon[118902]: pgmap v4054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/477795661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:40 smithi093 ceph-mon[111335]: pgmap v4054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:40.669Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:40.670Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:41.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:41.640943+0000) 2023-08-27T23:08:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:42.165Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:42.166Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:42.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:42.641207+0000) 2023-08-27T23:08:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:43 smithi136 ceph-mon[113531]: pgmap v4055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1409767746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:43.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:43.641477+0000) 2023-08-27T23:08:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:43 smithi093 ceph-mon[111335]: pgmap v4055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1409767746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:43.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:43 smithi093 ceph-mon[118902]: pgmap v4055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:43.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1409767746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:44.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:44.641774+0000) 2023-08-27T23:08:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:45.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:45.642011+0000) 2023-08-27T23:08:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:45 smithi136 ceph-mon[113531]: pgmap v4056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1482173543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:08:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:08:45] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:08:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:45 smithi093 ceph-mon[118902]: pgmap v4056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1482173543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:45 smithi093 ceph-mon[111335]: pgmap v4056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1482173543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:46.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:46.642298+0000) 2023-08-27T23:08:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:47 smithi136 ceph-mon[113531]: pgmap v4057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3929059437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:47.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:47.642540+0000) 2023-08-27T23:08:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:47 smithi093 ceph-mon[111335]: pgmap v4057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3929059437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:47 smithi093 ceph-mon[118902]: pgmap v4057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3929059437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:48.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:48.642802+0000) 2023-08-27T23:08:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:49 smithi136 ceph-mon[113531]: pgmap v4058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:49.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:49.643080+0000) 2023-08-27T23:08:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:49 smithi093 ceph-mon[111335]: pgmap v4058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:49 smithi093 ceph-mon[118902]: pgmap v4058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:50.670 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1071434732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:50.671 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1071434732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1071434732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:50.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:50.643379+0000) 2023-08-27T23:08:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:50.670Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:50.671Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:51 smithi136 ceph-mon[113531]: pgmap v4059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:51.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:51.643634+0000) 2023-08-27T23:08:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:51 smithi093 ceph-mon[111335]: pgmap v4059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:51 smithi093 ceph-mon[118902]: pgmap v4059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3959362445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3959362445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:52.165Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:08:52.166Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:08:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3959362445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:52.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:52.643887+0000) 2023-08-27T23:08:53.645 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:53 smithi136 ceph-mon[113531]: pgmap v4060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:53.645 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:08:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:53 smithi093 ceph-mon[111335]: pgmap v4060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:53 smithi093 ceph-mon[118902]: pgmap v4060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:08:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:53.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:53.644123+0000) 2023-08-27T23:08:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/783955492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/783955492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:08:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/783955492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:54.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:54.644270+0000) 2023-08-27T23:08:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:55.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:55.644452+0000) 2023-08-27T23:08:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:55 smithi136 ceph-mon[113531]: pgmap v4061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:08:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:08:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[111335]: pgmap v4061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[118902]: pgmap v4061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:55.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:08:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:56.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:56.644702+0000) 2023-08-27T23:08:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:08:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:08:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:08:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:08:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:08:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:57 smithi136 ceph-mon[113531]: pgmap v4062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2935482928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:57.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:57.645002+0000) 2023-08-27T23:08:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:57 smithi093 ceph-mon[111335]: pgmap v4062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2935482928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:57 smithi093 ceph-mon[118902]: pgmap v4062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2935482928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:58.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:58.645192+0000) 2023-08-27T23:08:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:08:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:08:59.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:08:59.645453+0000) 2023-08-27T23:08:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:59 smithi136 ceph-mon[113531]: pgmap v4063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:08:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3267035677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:59 smithi093 ceph-mon[111335]: pgmap v4063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:08:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3267035677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:08:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:59 smithi093 ceph-mon[118902]: pgmap v4063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:08:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:08:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3267035677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:00 smithi136 ceph-mon[113531]: pgmap v4064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:00.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:00.645714+0000) 2023-08-27T23:09:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:00 smithi093 ceph-mon[111335]: pgmap v4064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:00 smithi093 ceph-mon[118902]: pgmap v4064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:00.671Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:00.672Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1209427574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:01.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:01.645940+0000) 2023-08-27T23:09:02.166 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1209427574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:02.167 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1209427574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:02.166Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:02.167Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:02 smithi136 ceph-mon[113531]: pgmap v4065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:02.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:02.646158+0000) 2023-08-27T23:09:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:02 smithi093 ceph-mon[111335]: pgmap v4065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:02 smithi093 ceph-mon[118902]: pgmap v4065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:03.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:03.646410+0000) 2023-08-27T23:09:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3376972364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3376972364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3376972364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:04.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:04.646632+0000) 2023-08-27T23:09:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:04 smithi093 ceph-mon[111335]: pgmap v4066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:04 smithi093 ceph-mon[118902]: pgmap v4066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:04 smithi136 ceph-mon[113531]: pgmap v4066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:09:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:09:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:09:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:05.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:05.646841+0000) 2023-08-27T23:09:06.647 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4036871636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4036871636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4036871636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:06.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:06.647127+0000) 2023-08-27T23:09:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:07 smithi136 ceph-mon[113531]: pgmap v4067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:07.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:07.647321+0000) 2023-08-27T23:09:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:07 smithi093 ceph-mon[111335]: pgmap v4067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:07 smithi093 ceph-mon[118902]: pgmap v4067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:08.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:08.647512+0000) 2023-08-27T23:09:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:09 smithi136 ceph-mon[113531]: pgmap v4068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/933122519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:09.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:09.647720+0000) 2023-08-27T23:09:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:09 smithi093 ceph-mon[111335]: pgmap v4068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/933122519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:09 smithi093 ceph-mon[118902]: pgmap v4068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/933122519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:10 smithi136 ceph-mon[113531]: pgmap v4069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:10.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:10.647976+0000) 2023-08-27T23:09:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:10 smithi093 ceph-mon[111335]: pgmap v4069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:10 smithi093 ceph-mon[118902]: pgmap v4069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:10.672Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:10.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2163097988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:11.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:11.648241+0000) 2023-08-27T23:09:12.167 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2163097988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:12.168 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2163097988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:12.167Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:12.168Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:12 smithi136 ceph-mon[113531]: pgmap v4070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:12.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:12.648509+0000) 2023-08-27T23:09:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:12 smithi093 ceph-mon[111335]: pgmap v4070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:12 smithi093 ceph-mon[118902]: pgmap v4070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1428024957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:13.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:13.648784+0000) 2023-08-27T23:09:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1428024957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1428024957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:14 smithi136 ceph-mon[113531]: pgmap v4071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:14.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:14.649081+0000) 2023-08-27T23:09:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:14 smithi093 ceph-mon[111335]: pgmap v4071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:14 smithi093 ceph-mon[118902]: pgmap v4071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:09:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:09:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:09:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3226337254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:15.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:15.649342+0000) 2023-08-27T23:09:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3226337254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3226337254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:16.901 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:16.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:16.649548+0000) 2023-08-27T23:09:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:16 smithi136 ceph-mon[113531]: pgmap v4072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:16 smithi093 ceph-mon[111335]: pgmap v4072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:16 smithi093 ceph-mon[118902]: pgmap v4072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:17.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:17.649823+0000) 2023-08-27T23:09:18.650 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1969382459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1969382459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1969382459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:18.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:18.650128+0000) 2023-08-27T23:09:19.650 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:19 smithi136 ceph-mon[113531]: pgmap v4073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:19 smithi093 ceph-mon[111335]: pgmap v4073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:19 smithi093 ceph-mon[118902]: pgmap v4073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:19.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:19.650423+0000) 2023-08-27T23:09:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:20.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:20.650659+0000) 2023-08-27T23:09:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:20 smithi136 ceph-mon[113531]: pgmap v4074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1863899324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:20 smithi093 ceph-mon[111335]: pgmap v4074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1863899324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:20 smithi093 ceph-mon[118902]: pgmap v4074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1863899324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:20.672Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:20.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:21.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:21.650903+0000) 2023-08-27T23:09:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:22.168Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:22.169Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:22.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:22.651180+0000) 2023-08-27T23:09:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:23 smithi136 ceph-mon[113531]: pgmap v4075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2571007213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:23.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:23.651456+0000) 2023-08-27T23:09:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:23 smithi093 ceph-mon[111335]: pgmap v4075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2571007213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:23 smithi093 ceph-mon[118902]: pgmap v4075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2571007213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:24.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:24.651725+0000) 2023-08-27T23:09:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:25.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:25.652026+0000) 2023-08-27T23:09:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:25 smithi136 ceph-mon[113531]: pgmap v4076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2370662785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:09:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:09:25] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:09:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:25 smithi093 ceph-mon[118902]: pgmap v4076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2370662785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:25 smithi093 ceph-mon[111335]: pgmap v4076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2370662785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:26.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:26.652317+0000) 2023-08-27T23:09:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:27.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:27.652539+0000) 2023-08-27T23:09:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:27 smithi136 ceph-mon[113531]: pgmap v4077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:27 smithi093 ceph-mon[111335]: pgmap v4077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:27 smithi093 ceph-mon[118902]: pgmap v4077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2022303670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:28.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:28.652787+0000) 2023-08-27T23:09:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2022303670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2022303670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:29 smithi136 ceph-mon[113531]: pgmap v4078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:29.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:29.653042+0000) 2023-08-27T23:09:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:29 smithi093 ceph-mon[111335]: pgmap v4078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:29 smithi093 ceph-mon[118902]: pgmap v4078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:30.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:30.653242+0000) 2023-08-27T23:09:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3963947608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:30 smithi136 ceph-mon[113531]: pgmap v4079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3963947608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:30 smithi093 ceph-mon[118902]: pgmap v4079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3963947608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:30 smithi093 ceph-mon[111335]: pgmap v4079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:30.674Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:30.674Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:31.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:31.653521+0000) 2023-08-27T23:09:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:32.168Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:32.169Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:32.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:32.653741+0000) 2023-08-27T23:09:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:33.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:33.653991+0000) 2023-08-27T23:09:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:33 smithi136 ceph-mon[113531]: pgmap v4080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3000577865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:33 smithi093 ceph-mon[111335]: pgmap v4080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3000577865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:33 smithi093 ceph-mon[118902]: pgmap v4080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3000577865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:34.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:34.654141+0000) 2023-08-27T23:09:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:35.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:35.654398+0000) 2023-08-27T23:09:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:35 smithi136 ceph-mon[113531]: pgmap v4081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/472175532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:35 smithi093 ceph-mon[111335]: pgmap v4081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/472175532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:09:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:09:35] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:09:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:35 smithi093 ceph-mon[118902]: pgmap v4081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/472175532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:36.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:36.654613+0000) 2023-08-27T23:09:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:37.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:37.654827+0000) 2023-08-27T23:09:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:37 smithi136 ceph-mon[113531]: pgmap v4082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3224271325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:37 smithi093 ceph-mon[111335]: pgmap v4082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3224271325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:37 smithi093 ceph-mon[118902]: pgmap v4082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3224271325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:38.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:38.655042+0000) 2023-08-27T23:09:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:39.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:39.655317+0000) 2023-08-27T23:09:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:39 smithi136 ceph-mon[113531]: pgmap v4083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:39 smithi093 ceph-mon[111335]: pgmap v4083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:39 smithi093 ceph-mon[118902]: pgmap v4083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:40.675 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2545724604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:40.675 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2545724604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2545724604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:40.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:40.655597+0000) 2023-08-27T23:09:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:40.675Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:40.676Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:41 smithi136 ceph-mon[113531]: pgmap v4084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:41.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:41.655874+0000) 2023-08-27T23:09:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:41 smithi093 ceph-mon[111335]: pgmap v4084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:41 smithi093 ceph-mon[118902]: pgmap v4084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:42.422 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:42.169Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:42.422 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:42.170Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2935764828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:42.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:42.656170+0000) 2023-08-27T23:09:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2935764828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2935764828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:43.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:43 smithi136 ceph-mon[113531]: pgmap v4085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:43.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:43.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:43.656411+0000) 2023-08-27T23:09:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:43 smithi093 ceph-mon[111335]: pgmap v4085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:43 smithi093 ceph-mon[118902]: pgmap v4085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:44.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:44.656683+0000) 2023-08-27T23:09:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:45.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:45.656959+0000) 2023-08-27T23:09:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:45 smithi136 ceph-mon[113531]: pgmap v4086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/143975746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:45 smithi093 ceph-mon[111335]: pgmap v4086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/143975746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:09:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:09:45] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:09:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:45 smithi093 ceph-mon[118902]: pgmap v4086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/143975746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:46.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:46.657205+0000) 2023-08-27T23:09:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:47.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:47.657463+0000) 2023-08-27T23:09:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:47 smithi136 ceph-mon[113531]: pgmap v4087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2030693260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:47 smithi093 ceph-mon[111335]: pgmap v4087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2030693260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:47 smithi093 ceph-mon[118902]: pgmap v4087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2030693260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:48.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:48.657731+0000) 2023-08-27T23:09:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:49 smithi136 ceph-mon[113531]: pgmap v4088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471289288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:49.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:49.657985+0000) 2023-08-27T23:09:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:49 smithi093 ceph-mon[111335]: pgmap v4088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471289288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:49 smithi093 ceph-mon[118902]: pgmap v4088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471289288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:50 smithi136 ceph-mon[113531]: pgmap v4089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:50.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:50.658308+0000) 2023-08-27T23:09:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:50 smithi093 ceph-mon[111335]: pgmap v4089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:50 smithi093 ceph-mon[118902]: pgmap v4089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:50.675Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:50.677Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2241383936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:51.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:51.658530+0000) 2023-08-27T23:09:52.170 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2241383936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:52.170 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2241383936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:52.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:09:52.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:09:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:52 smithi136 ceph-mon[113531]: pgmap v4090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:52.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:52.658738+0000) 2023-08-27T23:09:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:52 smithi093 ceph-mon[111335]: pgmap v4090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:52 smithi093 ceph-mon[118902]: pgmap v4090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:53.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:53.658962+0000) 2023-08-27T23:09:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:09:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2384605056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:54 smithi136 ceph-mon[113531]: pgmap v4091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:54.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:54.659241+0000) 2023-08-27T23:09:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2384605056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:54 smithi093 ceph-mon[111335]: pgmap v4091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2384605056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:54 smithi093 ceph-mon[118902]: pgmap v4091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:09:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:09:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:09:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:09:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:55.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:55.659422+0000) 2023-08-27T23:09:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:09:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:09:56.900 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:56.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:56.659583+0000) 2023-08-27T23:09:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:56 smithi136 ceph-mon[113531]: pgmap v4092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3357581034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:56 smithi093 ceph-mon[111335]: pgmap v4092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3357581034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:56 smithi093 ceph-mon[118902]: pgmap v4092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3357581034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:57.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:57.659824+0000) 2023-08-27T23:09:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:58.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:58.659983+0000) 2023-08-27T23:09:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: pgmap v4093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:09:58.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:09:58.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: pgmap v4093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: pgmap v4093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:09:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:09:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/90768571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:09:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:09:59.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:09:59.660198+0000) 2023-08-27T23:09:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:09:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/90768571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:09:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:09:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/90768571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:10:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:00 smithi136 ceph-mon[113531]: pgmap v4094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:00.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:00.660479+0000) 2023-08-27T23:10:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:10:00.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:00 smithi093 ceph-mon[111335]: pgmap v4094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:10:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:00 smithi093 ceph-mon[118902]: pgmap v4094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:00.676Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:00.677Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4072510544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:01.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:01.660781+0000) 2023-08-27T23:10:02.170 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4072510544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:02.171 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4072510544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:02.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:02.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:02 smithi136 ceph-mon[113531]: pgmap v4095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:02.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:02.661089+0000) 2023-08-27T23:10:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:02 smithi093 ceph-mon[111335]: pgmap v4095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:02 smithi093 ceph-mon[118902]: pgmap v4095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3548336888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:03.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:03.661380+0000) 2023-08-27T23:10:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3548336888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3548336888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:04 smithi136 ceph-mon[113531]: pgmap v4096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:04.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:04.661663+0000) 2023-08-27T23:10:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:04 smithi093 ceph-mon[111335]: pgmap v4096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:04 smithi093 ceph-mon[118902]: pgmap v4096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:10:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:10:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:10:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3804560595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:05.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:05.661907+0000) 2023-08-27T23:10:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3804560595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3804560595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:06.913 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:06.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:06.662166+0000) 2023-08-27T23:10:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:06 smithi136 ceph-mon[113531]: pgmap v4097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:06 smithi093 ceph-mon[111335]: pgmap v4097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:06 smithi093 ceph-mon[118902]: pgmap v4097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:07.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:07.662465+0000) 2023-08-27T23:10:08.662 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2382111802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2382111802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2382111802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:08.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:08.662733+0000) 2023-08-27T23:10:09.663 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:09 smithi136 ceph-mon[113531]: pgmap v4098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:09.663 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:09 smithi093 ceph-mon[111335]: pgmap v4098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:09 smithi093 ceph-mon[118902]: pgmap v4098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:09.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:09.663054+0000) 2023-08-27T23:10:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:10 smithi136 ceph-mon[113531]: pgmap v4099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3580211582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:10.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:10.663311+0000) 2023-08-27T23:10:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:10 smithi093 ceph-mon[111335]: pgmap v4099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3580211582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:10 smithi093 ceph-mon[118902]: pgmap v4099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3580211582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:10.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:10.678Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:11.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:11.663562+0000) 2023-08-27T23:10:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:12.171Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:12.172Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:12.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:12.663814+0000) 2023-08-27T23:10:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:13 smithi136 ceph-mon[113531]: pgmap v4100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3008619763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:13.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:13.664041+0000) 2023-08-27T23:10:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:13 smithi093 ceph-mon[111335]: pgmap v4100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3008619763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:13 smithi093 ceph-mon[118902]: pgmap v4100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3008619763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:14.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:14.664258+0000) 2023-08-27T23:10:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:15 smithi136 ceph-mon[113531]: pgmap v4101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:15.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:15.664546+0000) 2023-08-27T23:10:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:15 smithi093 ceph-mon[111335]: pgmap v4101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:10:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:10:15] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:10:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:15 smithi093 ceph-mon[118902]: pgmap v4101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1603882844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:16.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:16.664822+0000) 2023-08-27T23:10:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1603882844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1603882844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:17 smithi136 ceph-mon[113531]: pgmap v4102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:17.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:17.665083+0000) 2023-08-27T23:10:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:17 smithi093 ceph-mon[111335]: pgmap v4102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:17 smithi093 ceph-mon[118902]: pgmap v4102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2930534032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:18.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:18.665297+0000) 2023-08-27T23:10:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2930534032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2930534032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:19 smithi136 ceph-mon[113531]: pgmap v4103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:19.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:19.665582+0000) 2023-08-27T23:10:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:19 smithi093 ceph-mon[111335]: pgmap v4103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:19 smithi093 ceph-mon[118902]: pgmap v4103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3950302765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:20 smithi136 ceph-mon[113531]: pgmap v4104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:20.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:20.665841+0000) 2023-08-27T23:10:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3950302765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:20 smithi093 ceph-mon[111335]: pgmap v4104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3950302765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:20 smithi093 ceph-mon[118902]: pgmap v4104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:20.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:20.678Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:21.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:21.666094+0000) 2023-08-27T23:10:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:22.172Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:22.173Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:22.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:22.666352+0000) 2023-08-27T23:10:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:23 smithi136 ceph-mon[113531]: pgmap v4105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2730508426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:23.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:23.666631+0000) 2023-08-27T23:10:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:23 smithi093 ceph-mon[111335]: pgmap v4105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2730508426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:23 smithi093 ceph-mon[118902]: pgmap v4105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2730508426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:24.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:24.666909+0000) 2023-08-27T23:10:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:25 smithi136 ceph-mon[113531]: pgmap v4106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/110962469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:25.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:25.667165+0000) 2023-08-27T23:10:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:10:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:10:25] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:10:25.759 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:25 smithi093 ceph-mon[111335]: pgmap v4106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:25.760 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/110962469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:25.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:25 smithi093 ceph-mon[118902]: pgmap v4106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:25.760 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/110962469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:26.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:26.667387+0000) 2023-08-27T23:10:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:27 smithi136 ceph-mon[113531]: pgmap v4107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3664305496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:27.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:27.667643+0000) 2023-08-27T23:10:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:27 smithi093 ceph-mon[118902]: pgmap v4107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3664305496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:27 smithi093 ceph-mon[111335]: pgmap v4107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3664305496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:28.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:28.667913+0000) 2023-08-27T23:10:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:29 smithi136 ceph-mon[113531]: pgmap v4108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:29.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:29.668218+0000) 2023-08-27T23:10:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:29 smithi093 ceph-mon[111335]: pgmap v4108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:29 smithi093 ceph-mon[118902]: pgmap v4108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1395212463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:30.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:30.668464+0000) 2023-08-27T23:10:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1395212463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1395212463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:30.678Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:30.679Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:31 smithi136 ceph-mon[113531]: pgmap v4109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:31.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:31.668661+0000) 2023-08-27T23:10:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:31 smithi093 ceph-mon[111335]: pgmap v4109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:31 smithi093 ceph-mon[118902]: pgmap v4109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:32.441 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:32.173Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:32.441 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:32.174Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/153240131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/153240131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/153240131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:32.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:32.668913+0000) 2023-08-27T23:10:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:33 smithi093 ceph-mon[111335]: pgmap v4110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:33 smithi093 ceph-mon[118902]: pgmap v4110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:33 smithi136 ceph-mon[113531]: pgmap v4110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:33.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:33.669210+0000) 2023-08-27T23:10:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:34.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:34.669499+0000) 2023-08-27T23:10:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:10:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:10:35] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:10:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:35 smithi093 ceph-mon[111335]: pgmap v4111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2336993740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:35 smithi093 ceph-mon[118902]: pgmap v4111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2336993740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:35 smithi136 ceph-mon[113531]: pgmap v4111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2336993740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:35.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:35.669782+0000) 2023-08-27T23:10:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:36.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:36.669970+0000) 2023-08-27T23:10:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:37 smithi093 ceph-mon[111335]: pgmap v4112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2818803482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:37 smithi093 ceph-mon[118902]: pgmap v4112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2818803482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:37 smithi136 ceph-mon[113531]: pgmap v4112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2818803482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:37.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:37.670201+0000) 2023-08-27T23:10:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:38.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:38.670364+0000) 2023-08-27T23:10:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:39 smithi093 ceph-mon[111335]: pgmap v4113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3322435503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:39 smithi093 ceph-mon[118902]: pgmap v4113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3322435503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:39 smithi136 ceph-mon[113531]: pgmap v4113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3322435503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:39.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:39.670635+0000) 2023-08-27T23:10:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:40 smithi136 ceph-mon[113531]: pgmap v4114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:40.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:40.670862+0000) 2023-08-27T23:10:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:40 smithi093 ceph-mon[111335]: pgmap v4114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:40.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:40 smithi093 ceph-mon[118902]: pgmap v4114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:40.679Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:40.680Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3132281935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:41.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:41.671129+0000) 2023-08-27T23:10:42.174 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3132281935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:42.174 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3132281935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:42.174Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:42.175Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:42 smithi136 ceph-mon[113531]: pgmap v4115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:42.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:42.671345+0000) 2023-08-27T23:10:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:42 smithi093 ceph-mon[111335]: pgmap v4115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:42 smithi093 ceph-mon[118902]: pgmap v4115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:43.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:43.671600+0000) 2023-08-27T23:10:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3970747549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3970747549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3970747549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:44.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:44.671873+0000) 2023-08-27T23:10:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:45 smithi093 ceph-mon[118902]: pgmap v4116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:45 smithi093 ceph-mon[111335]: pgmap v4116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:45 smithi136 ceph-mon[113531]: pgmap v4116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:10:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:10:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:10:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:45.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:45.672172+0000) 2023-08-27T23:10:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3382590375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3382590375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3382590375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:46.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:46.672417+0000) 2023-08-27T23:10:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:47 smithi093 ceph-mon[111335]: pgmap v4117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:47 smithi093 ceph-mon[118902]: pgmap v4117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:47 smithi136 ceph-mon[113531]: pgmap v4117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:47.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:47.672657+0000) 2023-08-27T23:10:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:48 smithi093 ceph-mon[111335]: pgmap v4118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:48 smithi093 ceph-mon[118902]: pgmap v4118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:48 smithi136 ceph-mon[113531]: pgmap v4118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:48.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:48.672871+0000) 2023-08-27T23:10:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/512238668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/512238668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/512238668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:49.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:49.673154+0000) 2023-08-27T23:10:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:50 smithi136 ceph-mon[113531]: pgmap v4119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:50.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:50.673398+0000) 2023-08-27T23:10:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:50 smithi093 ceph-mon[111335]: pgmap v4119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:50 smithi093 ceph-mon[118902]: pgmap v4119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:50.679Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:50.680Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3629886901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:51.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:51.673654+0000) 2023-08-27T23:10:52.175 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3629886901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:52.175 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3629886901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:52.175Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:10:52.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:10:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:52 smithi136 ceph-mon[113531]: pgmap v4120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:52.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:52.673914+0000) 2023-08-27T23:10:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:52 smithi093 ceph-mon[111335]: pgmap v4120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:52 smithi093 ceph-mon[118902]: pgmap v4120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2604157455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:54.105 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:54.105 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:53.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:53.674135+0000) 2023-08-27T23:10:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2604157455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:54.486 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2604157455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:10:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:54 smithi136 ceph-mon[113531]: pgmap v4121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:54.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:54.674391+0000) 2023-08-27T23:10:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:54 smithi093 ceph-mon[111335]: pgmap v4121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:54 smithi093 ceph-mon[118902]: pgmap v4121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:10:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:10:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:10:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:55.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:55.674716+0000) 2023-08-27T23:10:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1023688545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1023688545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1023688545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:56.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:56.674963+0000) 2023-08-27T23:10:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:56 smithi093 ceph-mon[111335]: pgmap v4122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:56 smithi093 ceph-mon[118902]: pgmap v4122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:56 smithi136 ceph-mon[113531]: pgmap v4122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:57.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:57.675265+0000) 2023-08-27T23:10:58.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2513560241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2513560241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2513560241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:10:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:58.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:58.675468+0000) 2023-08-27T23:10:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:59 smithi136 ceph-mon[113531]: pgmap v4123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:10:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:59 smithi093 ceph-mon[111335]: pgmap v4123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:10:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:59 smithi093 ceph-mon[118902]: pgmap v4123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:10:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:10:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:10:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:10:59.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:10:59.675619+0000) 2023-08-27T23:11:00.735 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:00.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:00.675783+0000) 2023-08-27T23:11:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:00.680Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:00.682Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: pgmap v4124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3495786153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:11:01.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: pgmap v4124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3495786153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:11:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:11:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: pgmap v4124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:01.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3495786153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:01.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:11:01.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:11:01.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:11:01.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:01.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:01.675925+0000) 2023-08-27T23:11:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:02.176Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:02.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:02.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:02.676176+0000) 2023-08-27T23:11:03.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:03 smithi136 ceph-mon[113531]: pgmap v4125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:03.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1453178089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:03 smithi093 ceph-mon[111335]: pgmap v4125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1453178089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:03 smithi093 ceph-mon[118902]: pgmap v4125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1453178089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:03.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:03.676416+0000) 2023-08-27T23:11:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:04.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:04.676675+0000) 2023-08-27T23:11:05.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:05 smithi136 ceph-mon[113531]: pgmap v4126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:05.677 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 23:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T23:11:05.317986542Z level=info msg="Completed cleanup jobs" duration=12.79393ms 2023-08-27T23:11:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:05 smithi093 ceph-mon[111335]: pgmap v4126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:11:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:11:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:11:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:05 smithi093 ceph-mon[118902]: pgmap v4126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:05.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:05.676919+0000) 2023-08-27T23:11:06.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4098522942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4098522942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4098522942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:06.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:06.677159+0000) 2023-08-27T23:11:07.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:07 smithi136 ceph-mon[113531]: pgmap v4127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:07 smithi093 ceph-mon[111335]: pgmap v4127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:07.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:07 smithi093 ceph-mon[118902]: pgmap v4127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:07.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:07.677377+0000) 2023-08-27T23:11:08.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3541465291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3541465291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3541465291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:08.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:08.677635+0000) 2023-08-27T23:11:09.678 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:09 smithi136 ceph-mon[113531]: pgmap v4128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:09.678 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:09 smithi093 ceph-mon[111335]: pgmap v4128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:09 smithi093 ceph-mon[118902]: pgmap v4128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:09.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:09.677899+0000) 2023-08-27T23:11:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3256190022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:10 smithi136 ceph-mon[113531]: pgmap v4129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:10.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:10.678151+0000) 2023-08-27T23:11:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3256190022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:10 smithi093 ceph-mon[111335]: pgmap v4129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3256190022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:10 smithi093 ceph-mon[118902]: pgmap v4129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:10.681Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:10.682Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:11.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:11.678403+0000) 2023-08-27T23:11:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:12.176Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:12.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:12.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:12.678633+0000) 2023-08-27T23:11:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:13 smithi136 ceph-mon[113531]: pgmap v4130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1054942698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:13.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:13.678886+0000) 2023-08-27T23:11:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:13 smithi093 ceph-mon[111335]: pgmap v4130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1054942698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:13 smithi093 ceph-mon[118902]: pgmap v4130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1054942698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:14.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:14.679180+0000) 2023-08-27T23:11:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:15 smithi136 ceph-mon[113531]: pgmap v4131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/840858060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:15.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:15.679447+0000) 2023-08-27T23:11:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:15 smithi093 ceph-mon[111335]: pgmap v4131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/840858060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:11:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:11:15] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:11:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:15 smithi093 ceph-mon[118902]: pgmap v4131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/840858060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:16.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:16.679719+0000) 2023-08-27T23:11:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:17 smithi136 ceph-mon[113531]: pgmap v4132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2167837267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:17.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:17.679977+0000) 2023-08-27T23:11:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:17 smithi093 ceph-mon[111335]: pgmap v4132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2167837267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:17 smithi093 ceph-mon[118902]: pgmap v4132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2167837267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:18.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:18.680195+0000) 2023-08-27T23:11:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:19 smithi136 ceph-mon[113531]: pgmap v4133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:19.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:19.680429+0000) 2023-08-27T23:11:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:19 smithi093 ceph-mon[111335]: pgmap v4133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:19 smithi093 ceph-mon[118902]: pgmap v4133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2163096042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:20.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:20.680645+0000) 2023-08-27T23:11:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2163096042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2163096042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:20.682Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:20.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:20.683Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:21 smithi093 ceph-mon[111335]: pgmap v4134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:21 smithi093 ceph-mon[118902]: pgmap v4134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:21 smithi136 ceph-mon[113531]: pgmap v4134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:21.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:21.680887+0000) 2023-08-27T23:11:22.447 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:22.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:22.447 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:22.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3502752404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3502752404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3502752404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:22.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:22.681105+0000) 2023-08-27T23:11:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:23 smithi093 ceph-mon[111335]: pgmap v4135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:23 smithi093 ceph-mon[118902]: pgmap v4135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:23 smithi136 ceph-mon[113531]: pgmap v4135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:23.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:23.681293+0000) 2023-08-27T23:11:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:24.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:24.681537+0000) 2023-08-27T23:11:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:11:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:11:25] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:11:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:25 smithi093 ceph-mon[111335]: pgmap v4136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3076571140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:25 smithi093 ceph-mon[118902]: pgmap v4136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3076571140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:25 smithi136 ceph-mon[113531]: pgmap v4136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3076571140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:25.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:25.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:25.681824+0000) 2023-08-27T23:11:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:26.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:26.682062+0000) 2023-08-27T23:11:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:27 smithi093 ceph-mon[111335]: pgmap v4137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2551977097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:27 smithi093 ceph-mon[118902]: pgmap v4137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2551977097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:27 smithi136 ceph-mon[113531]: pgmap v4137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2551977097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:27.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:27.682270+0000) 2023-08-27T23:11:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:28.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:28.682546+0000) 2023-08-27T23:11:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:29 smithi093 ceph-mon[111335]: pgmap v4138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/551946490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:29 smithi093 ceph-mon[118902]: pgmap v4138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/551946490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:29 smithi136 ceph-mon[113531]: pgmap v4138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/551946490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:29.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:29.682814+0000) 2023-08-27T23:11:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:30 smithi136 ceph-mon[113531]: pgmap v4139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:30.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:30.683112+0000) 2023-08-27T23:11:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:30 smithi093 ceph-mon[111335]: pgmap v4139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:30 smithi093 ceph-mon[118902]: pgmap v4139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:30.683Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:30.684Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/981449346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:31.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:31.683356+0000) 2023-08-27T23:11:32.177 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/981449346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:32.178 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/981449346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:32.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:32.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:32 smithi136 ceph-mon[113531]: pgmap v4140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:32.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:32.683604+0000) 2023-08-27T23:11:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:32 smithi093 ceph-mon[111335]: pgmap v4140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:32 smithi093 ceph-mon[118902]: pgmap v4140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:33.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:33.683870+0000) 2023-08-27T23:11:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3812388496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3812388496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3812388496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:34.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:34.684115+0000) 2023-08-27T23:11:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:35 smithi136 ceph-mon[113531]: pgmap v4141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:35 smithi093 ceph-mon[111335]: pgmap v4141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:35 smithi093 ceph-mon[118902]: pgmap v4141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:11:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:11:35] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:11:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:35.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:35.684382+0000) 2023-08-27T23:11:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:36.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:36.684634+0000) 2023-08-27T23:11:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:37 smithi136 ceph-mon[113531]: pgmap v4142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4277390667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:37.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:37.684877+0000) 2023-08-27T23:11:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:37 smithi093 ceph-mon[111335]: pgmap v4142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4277390667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:37 smithi093 ceph-mon[118902]: pgmap v4142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4277390667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:38.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:38.685084+0000) 2023-08-27T23:11:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:39 smithi093 ceph-mon[111335]: pgmap v4143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1988488966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:39 smithi093 ceph-mon[118902]: pgmap v4143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1988488966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:39.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:39.685327+0000) 2023-08-27T23:11:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:39 smithi136 ceph-mon[113531]: pgmap v4143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1988488966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:40 smithi136 ceph-mon[113531]: pgmap v4144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:40.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:40.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:40.685590+0000) 2023-08-27T23:11:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:40 smithi093 ceph-mon[111335]: pgmap v4144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:40 smithi093 ceph-mon[118902]: pgmap v4144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:40.684Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:40.685Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/528788284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:41.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:41.685858+0000) 2023-08-27T23:11:42.178 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/528788284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:42.178 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/528788284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:42.178Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:42.179Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:42 smithi136 ceph-mon[113531]: pgmap v4145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:42.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:42.686148+0000) 2023-08-27T23:11:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:42 smithi093 ceph-mon[111335]: pgmap v4145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:42 smithi093 ceph-mon[118902]: pgmap v4145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/964981854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:43.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:43.686405+0000) 2023-08-27T23:11:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/964981854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/964981854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:44 smithi136 ceph-mon[113531]: pgmap v4146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:44.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:44.686625+0000) 2023-08-27T23:11:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:44 smithi093 ceph-mon[111335]: pgmap v4146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:44 smithi093 ceph-mon[118902]: pgmap v4146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:11:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:11:45] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:11:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:45.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:45.686899+0000) 2023-08-27T23:11:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3533157129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3533157129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3533157129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:46.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:46.687182+0000) 2023-08-27T23:11:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:47 smithi136 ceph-mon[113531]: pgmap v4147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:47 smithi093 ceph-mon[111335]: pgmap v4147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:47 smithi093 ceph-mon[118902]: pgmap v4147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:47.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:47.687379+0000) 2023-08-27T23:11:48.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2467541793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2467541793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2467541793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:48.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:48.687605+0000) 2023-08-27T23:11:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:49 smithi093 ceph-mon[111335]: pgmap v4148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:49 smithi093 ceph-mon[118902]: pgmap v4148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:49 smithi136 ceph-mon[113531]: pgmap v4148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:49.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:49.687879+0000) 2023-08-27T23:11:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:50.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:50.688132+0000) 2023-08-27T23:11:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:50 smithi136 ceph-mon[113531]: pgmap v4149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2722939934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:50 smithi093 ceph-mon[111335]: pgmap v4149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2722939934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:50 smithi093 ceph-mon[118902]: pgmap v4149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2722939934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:50.685Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:50.686Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:51.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:51.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:51.688395+0000) 2023-08-27T23:11:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:52.178Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:11:52.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:11:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:52.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:52.688633+0000) 2023-08-27T23:11:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:53 smithi093 ceph-mon[111335]: pgmap v4150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/77551693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:53 smithi093 ceph-mon[118902]: pgmap v4150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/77551693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:53 smithi136 ceph-mon[113531]: pgmap v4150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/77551693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:53.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:53.688857+0000) 2023-08-27T23:11:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:11:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:54.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:54.689131+0000) 2023-08-27T23:11:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:55 smithi093 ceph-mon[111335]: pgmap v4151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:11:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:11:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:11:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:55 smithi093 ceph-mon[118902]: pgmap v4151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:55 smithi136 ceph-mon[113531]: pgmap v4151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:55.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:55.689372+0000) 2023-08-27T23:11:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3805950468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3805950468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3805950468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:56.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:56.689616+0000) 2023-08-27T23:11:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:57 smithi093 ceph-mon[111335]: pgmap v4152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:57 smithi093 ceph-mon[118902]: pgmap v4152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:57 smithi136 ceph-mon[113531]: pgmap v4152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:57.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:57.689907+0000) 2023-08-27T23:11:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1896355549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1896355549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1896355549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:11:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:58.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:58.690180+0000) 2023-08-27T23:11:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:11:59 smithi093 ceph-mon[111335]: pgmap v4153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:11:59 smithi093 ceph-mon[118902]: pgmap v4153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:11:59 smithi136 ceph-mon[113531]: pgmap v4153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:11:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:11:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:11:59.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:11:59.690426+0000) 2023-08-27T23:12:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3393162841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:00 smithi093 ceph-mon[118902]: pgmap v4154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3393162841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:00 smithi093 ceph-mon[111335]: pgmap v4154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:00.685Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:00.686Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3393162841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:00 smithi136 ceph-mon[113531]: pgmap v4154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:00.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:00.690693+0000) 2023-08-27T23:12:02.095 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:12:02.095 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:01.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:01.690878+0000) 2023-08-27T23:12:02.179 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:12:02.180 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:12:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:02.179Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:02.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:02.886 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:02.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:02.691076+0000) 2023-08-27T23:12:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:02 smithi093 ceph-mon[111335]: pgmap v4155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1130003465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:02 smithi093 ceph-mon[118902]: pgmap v4155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1130003465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:02 smithi136 ceph-mon[113531]: pgmap v4155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1130003465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:03.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:03.691233+0000) 2023-08-27T23:12:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:04.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:04.691419+0000) 2023-08-27T23:12:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:12:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[111335]: pgmap v4156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1000547306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:05.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[118902]: pgmap v4156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:05.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1000547306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:12:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:12:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:12:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:05 smithi136 ceph-mon[113531]: pgmap v4156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1000547306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:05.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:05.691711+0000) 2023-08-27T23:12:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:06.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:06.692005+0000) 2023-08-27T23:12:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:07 smithi093 ceph-mon[111335]: pgmap v4157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:08.127 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:07 smithi093 ceph-mon[118902]: pgmap v4157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:08.128 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:07.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:07.692285+0000) 2023-08-27T23:12:08.128 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:07 smithi136 ceph-mon[113531]: pgmap v4157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3940744639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:09.059 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3940744639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:09.060 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3940744639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:09.060 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:08.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:08.692485+0000) 2023-08-27T23:12:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:09 smithi093 ceph-mon[111335]: pgmap v4158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:09 smithi093 ceph-mon[118902]: pgmap v4158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:09 smithi136 ceph-mon[113531]: pgmap v4158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:09.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:09.692736+0000) 2023-08-27T23:12:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2674598511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2674598511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:10.686Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:10.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:10.687Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2674598511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:10.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:10.693017+0000) 2023-08-27T23:12:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:11 smithi093 ceph-mon[111335]: pgmap v4159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:11 smithi093 ceph-mon[118902]: pgmap v4159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:11 smithi136 ceph-mon[113531]: pgmap v4159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:11.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:11.693295+0000) 2023-08-27T23:12:12.458 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:12.179Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:12.459 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:12.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3795859887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3795859887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3795859887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:12.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:12.693542+0000) 2023-08-27T23:12:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:13 smithi093 ceph-mon[111335]: pgmap v4160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:13 smithi093 ceph-mon[118902]: pgmap v4160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:13 smithi136 ceph-mon[113531]: pgmap v4160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:13.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:13.693815+0000) 2023-08-27T23:12:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:14.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:14.694101+0000) 2023-08-27T23:12:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:15 smithi093 ceph-mon[111335]: pgmap v4161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3065970135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:12:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:12:15] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:12:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:15 smithi093 ceph-mon[118902]: pgmap v4161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3065970135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:15.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:15.694368+0000) 2023-08-27T23:12:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:15 smithi136 ceph-mon[113531]: pgmap v4161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3065970135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:16.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:16.694581+0000) 2023-08-27T23:12:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:17 smithi093 ceph-mon[111335]: pgmap v4162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4100203158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:17 smithi093 ceph-mon[118902]: pgmap v4162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4100203158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:17 smithi136 ceph-mon[113531]: pgmap v4162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4100203158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:17.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:17.694829+0000) 2023-08-27T23:12:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:18.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:18.695058+0000) 2023-08-27T23:12:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:19 smithi093 ceph-mon[111335]: pgmap v4163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3215795523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:19 smithi093 ceph-mon[118902]: pgmap v4163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3215795523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:19 smithi136 ceph-mon[113531]: pgmap v4163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3215795523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:19.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:19.695274+0000) 2023-08-27T23:12:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:20 smithi093 ceph-mon[118902]: pgmap v4164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:20 smithi093 ceph-mon[111335]: pgmap v4164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:20.686Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:20.687Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:20 smithi136 ceph-mon[113531]: pgmap v4164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:20.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:20.695512+0000) 2023-08-27T23:12:22.180 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1703318128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:22.180 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1703318128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1703318128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:21.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:21.695755+0000) 2023-08-27T23:12:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:22.180Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:22.181Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:22 smithi136 ceph-mon[113531]: pgmap v4165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:22.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:22.695988+0000) 2023-08-27T23:12:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:22 smithi093 ceph-mon[111335]: pgmap v4165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:22 smithi093 ceph-mon[118902]: pgmap v4165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:23.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:23.696240+0000) 2023-08-27T23:12:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:24.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:24.696539+0000) 2023-08-27T23:12:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2352491151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:24 smithi136 ceph-mon[113531]: pgmap v4166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2352491151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:24 smithi093 ceph-mon[111335]: pgmap v4166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2352491151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:24 smithi093 ceph-mon[118902]: pgmap v4166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:12:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:12:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:12:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:25.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:25.696770+0000) 2023-08-27T23:12:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:26.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:26.696968+0000) 2023-08-27T23:12:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:27 smithi093 ceph-mon[111335]: pgmap v4167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3645428655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:27 smithi093 ceph-mon[118902]: pgmap v4167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3645428655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:27 smithi136 ceph-mon[113531]: pgmap v4167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3645428655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:27.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:27.697246+0000) 2023-08-27T23:12:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:28.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:28.697506+0000) 2023-08-27T23:12:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:29 smithi093 ceph-mon[111335]: pgmap v4168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3383304212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:29 smithi093 ceph-mon[118902]: pgmap v4168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3383304212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:29 smithi136 ceph-mon[113531]: pgmap v4168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3383304212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:29.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:29.697779+0000) 2023-08-27T23:12:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:30 smithi093 ceph-mon[111335]: pgmap v4169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:30 smithi093 ceph-mon[118902]: pgmap v4169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:30.687Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:30.688Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:30 smithi136 ceph-mon[113531]: pgmap v4169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:30.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:30.698082+0000) 2023-08-27T23:12:32.181 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2434959457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:32.182 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2434959457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2434959457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:31.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:31.698334+0000) 2023-08-27T23:12:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:32.181Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:32.182Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:32 smithi136 ceph-mon[113531]: pgmap v4170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:32.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:32.698575+0000) 2023-08-27T23:12:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:32 smithi093 ceph-mon[111335]: pgmap v4170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:32 smithi093 ceph-mon[118902]: pgmap v4170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1542702872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:33.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:33.698800+0000) 2023-08-27T23:12:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1542702872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1542702872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:34 smithi136 ceph-mon[113531]: pgmap v4171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:34.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:34.699110+0000) 2023-08-27T23:12:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:34 smithi093 ceph-mon[111335]: pgmap v4171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:34 smithi093 ceph-mon[118902]: pgmap v4171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:12:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:12:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:12:36.088 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:35.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:35.699353+0000) 2023-08-27T23:12:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1511290917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1511290917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1511290917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:37.093 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:36.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:36.699577+0000) 2023-08-27T23:12:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:37 smithi136 ceph-mon[113531]: pgmap v4172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:37 smithi093 ceph-mon[111335]: pgmap v4172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:37 smithi093 ceph-mon[118902]: pgmap v4172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:37.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:37.699818+0000) 2023-08-27T23:12:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2168544254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2168544254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2168544254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:38.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:38.700107+0000) 2023-08-27T23:12:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:39 smithi093 ceph-mon[111335]: pgmap v4173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:39 smithi093 ceph-mon[118902]: pgmap v4173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:39 smithi136 ceph-mon[113531]: pgmap v4173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:39.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:39.700357+0000) 2023-08-27T23:12:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:40 smithi093 ceph-mon[111335]: pgmap v4174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1097759662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:40 smithi093 ceph-mon[118902]: pgmap v4174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1097759662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:40.688Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:40.689Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:40 smithi136 ceph-mon[113531]: pgmap v4174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1097759662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:40.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:40.700604+0000) 2023-08-27T23:12:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:41.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:41.700825+0000) 2023-08-27T23:12:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:42.182Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:42.183Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:42.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:42.701085+0000) 2023-08-27T23:12:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:43 smithi093 ceph-mon[111335]: pgmap v4175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3254775482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:43 smithi093 ceph-mon[118902]: pgmap v4175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3254775482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:43 smithi136 ceph-mon[113531]: pgmap v4175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3254775482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:43.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:43.701331+0000) 2023-08-27T23:12:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:44.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:44.701548+0000) 2023-08-27T23:12:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:45 smithi093 ceph-mon[111335]: pgmap v4176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:12:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:12:45] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:12:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:45 smithi093 ceph-mon[118902]: pgmap v4176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:45 smithi136 ceph-mon[113531]: pgmap v4176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:45.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:45.701777+0000) 2023-08-27T23:12:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3666613512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3666613512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3666613512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:46.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:46.702065+0000) 2023-08-27T23:12:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:47 smithi093 ceph-mon[111335]: pgmap v4177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:47 smithi093 ceph-mon[118902]: pgmap v4177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:47 smithi136 ceph-mon[113531]: pgmap v4177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:47.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:47.702268+0000) 2023-08-27T23:12:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3028162843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3028162843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3028162843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:48.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:48.702475+0000) 2023-08-27T23:12:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:49 smithi093 ceph-mon[111335]: pgmap v4178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:49 smithi093 ceph-mon[118902]: pgmap v4178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:49 smithi136 ceph-mon[113531]: pgmap v4178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:49.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:49.702670+0000) 2023-08-27T23:12:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/981721810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:50 smithi093 ceph-mon[111335]: pgmap v4179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/981721810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:50 smithi093 ceph-mon[118902]: pgmap v4179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:50.689Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:50.690Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/981721810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:50 smithi136 ceph-mon[113531]: pgmap v4179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:50.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:50.702865+0000) 2023-08-27T23:12:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:51.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:51.703134+0000) 2023-08-27T23:12:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:52.183Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:12:52.184Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:12:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:52.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:52.703375+0000) 2023-08-27T23:12:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:53 smithi093 ceph-mon[118902]: pgmap v4180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4109882004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:53 smithi093 ceph-mon[111335]: pgmap v4180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4109882004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:53 smithi136 ceph-mon[113531]: pgmap v4180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4109882004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:53.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:53.703576+0000) 2023-08-27T23:12:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:12:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:54.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:54.703849+0000) 2023-08-27T23:12:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:55 smithi093 ceph-mon[111335]: pgmap v4181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/954358253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:12:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:12:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:12:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:55 smithi093 ceph-mon[118902]: pgmap v4181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/954358253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:55 smithi136 ceph-mon[113531]: pgmap v4181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/954358253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:55.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:55.704153+0000) 2023-08-27T23:12:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:56.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:56.704390+0000) 2023-08-27T23:12:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:57 smithi093 ceph-mon[111335]: pgmap v4182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:57 smithi093 ceph-mon[118902]: pgmap v4182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:57 smithi136 ceph-mon[113531]: pgmap v4182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:57.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:57.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:57.704616+0000) 2023-08-27T23:12:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1860214110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1860214110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1860214110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:12:58.954 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:58.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:58.704877+0000) 2023-08-27T23:12:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:12:59 smithi093 ceph-mon[111335]: pgmap v4183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:12:59 smithi093 ceph-mon[118902]: pgmap v4183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:12:59 smithi136 ceph-mon[113531]: pgmap v4183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:12:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:12:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:12:59.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:12:59.705136+0000) 2023-08-27T23:13:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2768193793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:00 smithi093 ceph-mon[111335]: pgmap v4184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2768193793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:00 smithi093 ceph-mon[118902]: pgmap v4184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:00.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:00.690Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:00.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:00.691Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2768193793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:00 smithi136 ceph-mon[113531]: pgmap v4184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:00.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:00.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:00.705322+0000) 2023-08-27T23:13:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:01.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:01.705558+0000) 2023-08-27T23:13:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2616638049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:02.184Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:02.184Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2616638049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2616638049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:02.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:02.705806+0000) 2023-08-27T23:13:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:03 smithi136 ceph-mon[113531]: pgmap v4185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:03 smithi093 ceph-mon[111335]: pgmap v4185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:03 smithi093 ceph-mon[118902]: pgmap v4185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:03.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:03.706029+0000) 2023-08-27T23:13:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:04 smithi136 ceph-mon[113531]: pgmap v4186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:13:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:04.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:04.706243+0000) 2023-08-27T23:13:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:04 smithi093 ceph-mon[111335]: pgmap v4186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:13:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:04 smithi093 ceph-mon[118902]: pgmap v4186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:13:05.657 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1168990740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:05.658 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1168990740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:05.658 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:13:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:13:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:13:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1168990740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:05.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:05.706426+0000) 2023-08-27T23:13:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:06.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:06.706577+0000) 2023-08-27T23:13:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: pgmap v4187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3445338648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:13:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: pgmap v4187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3445338648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:13:07.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: pgmap v4187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3445338648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:13:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:13:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:13:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:07.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:07.706824+0000) 2023-08-27T23:13:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:08.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:08.707108+0000) 2023-08-27T23:13:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:09 smithi093 ceph-mon[118902]: pgmap v4188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/457198025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:09 smithi093 ceph-mon[111335]: pgmap v4188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/457198025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:09 smithi136 ceph-mon[113531]: pgmap v4188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/457198025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:09.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:09.707320+0000) 2023-08-27T23:13:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:10 smithi093 ceph-mon[111335]: pgmap v4189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:10 smithi093 ceph-mon[118902]: pgmap v4189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:10.690Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:10.691Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:10 smithi136 ceph-mon[113531]: pgmap v4189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:10.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:10.707578+0000) 2023-08-27T23:13:12.185 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1403492405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:12.185 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1403492405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1403492405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:11.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:11.707873+0000) 2023-08-27T23:13:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:12.185Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:12.185Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:12 smithi136 ceph-mon[113531]: pgmap v4190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:12.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:12.708115+0000) 2023-08-27T23:13:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:12 smithi093 ceph-mon[111335]: pgmap v4190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:12 smithi093 ceph-mon[118902]: pgmap v4190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:13.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:13.708318+0000) 2023-08-27T23:13:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/712793364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/712793364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/712793364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:14.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:14.708526+0000) 2023-08-27T23:13:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:15 smithi093 ceph-mon[111335]: pgmap v4191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:15 smithi093 ceph-mon[118902]: pgmap v4191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:13:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:13:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:13:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:15 smithi136 ceph-mon[113531]: pgmap v4191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:15.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:15.708801+0000) 2023-08-27T23:13:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:16.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:16.709019+0000) 2023-08-27T23:13:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:17 smithi093 ceph-mon[118902]: pgmap v4192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1784635787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:17 smithi093 ceph-mon[111335]: pgmap v4192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1784635787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:17 smithi136 ceph-mon[113531]: pgmap v4192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1784635787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:17.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:17.709242+0000) 2023-08-27T23:13:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:18.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:18.709471+0000) 2023-08-27T23:13:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:19 smithi093 ceph-mon[118902]: pgmap v4193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1237690824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:19 smithi093 ceph-mon[111335]: pgmap v4193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1237690824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:19 smithi136 ceph-mon[113531]: pgmap v4193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1237690824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:19.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:19.709701+0000) 2023-08-27T23:13:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:20 smithi093 ceph-mon[111335]: pgmap v4194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:20 smithi093 ceph-mon[118902]: pgmap v4194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:20.691Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:20.692Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:20 smithi136 ceph-mon[113531]: pgmap v4194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:20.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:20.709952+0000) 2023-08-27T23:13:22.185 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1590719450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:22.185 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1590719450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1590719450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:21.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:21.710214+0000) 2023-08-27T23:13:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:22.185Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:22.186Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:22 smithi136 ceph-mon[113531]: pgmap v4195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:22.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:22.710450+0000) 2023-08-27T23:13:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:22 smithi093 ceph-mon[111335]: pgmap v4195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:22 smithi093 ceph-mon[118902]: pgmap v4195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2603315073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:23.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:23.710640+0000) 2023-08-27T23:13:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2603315073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2603315073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:24 smithi136 ceph-mon[113531]: pgmap v4196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:24.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:24.710847+0000) 2023-08-27T23:13:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:24 smithi093 ceph-mon[111335]: pgmap v4196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:24 smithi093 ceph-mon[118902]: pgmap v4196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:13:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:13:25] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:13:26.114 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:25.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:25.711058+0000) 2023-08-27T23:13:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3823135371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3823135371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3823135371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:27.121 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:26.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:26.711240+0000) 2023-08-27T23:13:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:27 smithi136 ceph-mon[113531]: pgmap v4197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:27 smithi093 ceph-mon[111335]: pgmap v4197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:27 smithi093 ceph-mon[118902]: pgmap v4197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:27.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:27.711473+0000) 2023-08-27T23:13:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:28 smithi136 ceph-mon[113531]: pgmap v4198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/908058448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:28.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:28.711688+0000) 2023-08-27T23:13:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:28 smithi093 ceph-mon[111335]: pgmap v4198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/908058448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:28 smithi093 ceph-mon[118902]: pgmap v4198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/908058448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:29.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:29.711924+0000) 2023-08-27T23:13:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:30 smithi093 ceph-mon[111335]: pgmap v4199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3606389801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:30 smithi093 ceph-mon[118902]: pgmap v4199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3606389801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:30.692Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:30.692Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:30.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:30.712189+0000) 2023-08-27T23:13:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:30 smithi136 ceph-mon[113531]: pgmap v4199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3606389801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:31.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:31.712457+0000) 2023-08-27T23:13:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:32.185Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:32.186Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:32.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:32.712671+0000) 2023-08-27T23:13:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:33 smithi093 ceph-mon[111335]: pgmap v4200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3480746599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:33 smithi093 ceph-mon[118902]: pgmap v4200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3480746599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:33 smithi136 ceph-mon[113531]: pgmap v4200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3480746599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:33.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:33.712900+0000) 2023-08-27T23:13:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:34.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:34.713159+0000) 2023-08-27T23:13:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:35 smithi093 ceph-mon[111335]: pgmap v4201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:13:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:13:35] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:13:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:35 smithi093 ceph-mon[118902]: pgmap v4201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:35 smithi136 ceph-mon[113531]: pgmap v4201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:35.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:35.713425+0000) 2023-08-27T23:13:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/309018024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/309018024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/309018024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:36.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:36.713721+0000) 2023-08-27T23:13:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:37 smithi093 ceph-mon[111335]: pgmap v4202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:37 smithi093 ceph-mon[118902]: pgmap v4202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:37 smithi136 ceph-mon[113531]: pgmap v4202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:37.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:37.713955+0000) 2023-08-27T23:13:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/399029240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/399029240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/399029240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:38.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:38.714214+0000) 2023-08-27T23:13:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:39 smithi093 ceph-mon[111335]: pgmap v4203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:39.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:39 smithi093 ceph-mon[118902]: pgmap v4203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:39.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:39 smithi136 ceph-mon[113531]: pgmap v4203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:39.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:39.714419+0000) 2023-08-27T23:13:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/962559133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:40 smithi093 ceph-mon[111335]: pgmap v4204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/962559133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:40.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:40 smithi093 ceph-mon[118902]: pgmap v4204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:40.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:40.692Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:40.993 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:40.693Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/962559133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:40 smithi136 ceph-mon[113531]: pgmap v4204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:40.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:40.714664+0000) 2023-08-27T23:13:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:41.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:41.714827+0000) 2023-08-27T23:13:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:42.186Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:42.187Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:42.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:42.715026+0000) 2023-08-27T23:13:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:43 smithi093 ceph-mon[111335]: pgmap v4205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1545159208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:43 smithi093 ceph-mon[118902]: pgmap v4205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1545159208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:43.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:43.715314+0000) 2023-08-27T23:13:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:43 smithi136 ceph-mon[113531]: pgmap v4205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1545159208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:44.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:44.715540+0000) 2023-08-27T23:13:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:45 smithi093 ceph-mon[111335]: pgmap v4206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/913079608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:13:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:13:45] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:13:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:45 smithi093 ceph-mon[118902]: pgmap v4206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/913079608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:45 smithi136 ceph-mon[113531]: pgmap v4206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/913079608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:45.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:45.715738+0000) 2023-08-27T23:13:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:46.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:46.715952+0000) 2023-08-27T23:13:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:47 smithi093 ceph-mon[111335]: pgmap v4207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:47 smithi093 ceph-mon[118902]: pgmap v4207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:47 smithi136 ceph-mon[113531]: pgmap v4207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:47.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:47.716206+0000) 2023-08-27T23:13:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2548914731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2548914731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2548914731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:48.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:48.716428+0000) 2023-08-27T23:13:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:49 smithi093 ceph-mon[111335]: pgmap v4208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:49 smithi093 ceph-mon[118902]: pgmap v4208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:49 smithi136 ceph-mon[113531]: pgmap v4208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:49.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:49.716677+0000) 2023-08-27T23:13:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3903979100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:50 smithi093 ceph-mon[111335]: pgmap v4209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3903979100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:50 smithi093 ceph-mon[118902]: pgmap v4209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:50.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:50.693Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:50.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:50.694Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3903979100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:50 smithi136 ceph-mon[113531]: pgmap v4209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:50.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:50.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:50.716936+0000) 2023-08-27T23:13:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:51.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:51.717180+0000) 2023-08-27T23:13:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2147022301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2147022301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:52.187Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:13:52.188Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:13:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2147022301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:52.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:52.717400+0000) 2023-08-27T23:13:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:53 smithi136 ceph-mon[113531]: pgmap v4210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:53 smithi093 ceph-mon[111335]: pgmap v4210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:53 smithi093 ceph-mon[118902]: pgmap v4210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:53.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:53.717655+0000) 2023-08-27T23:13:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:13:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:54.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:54.717937+0000) 2023-08-27T23:13:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:55 smithi136 ceph-mon[113531]: pgmap v4211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2603340608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:13:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:13:55] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:13:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:55 smithi093 ceph-mon[111335]: pgmap v4211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2603340608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:55 smithi093 ceph-mon[118902]: pgmap v4211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2603340608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:55.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:55.718202+0000) 2023-08-27T23:13:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:56.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:56.718411+0000) 2023-08-27T23:13:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:57 smithi093 ceph-mon[111335]: pgmap v4212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/414732352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:57 smithi093 ceph-mon[118902]: pgmap v4212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/414732352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:57 smithi136 ceph-mon[113531]: pgmap v4212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/414732352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:57.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:57.718661+0000) 2023-08-27T23:13:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:58.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:58.718922+0000) 2023-08-27T23:13:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:59 smithi093 ceph-mon[111335]: pgmap v4213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:13:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3282009714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:59 smithi093 ceph-mon[118902]: pgmap v4213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:13:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3282009714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:59 smithi136 ceph-mon[113531]: pgmap v4213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:13:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:13:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3282009714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:13:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:13:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:13:59.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:13:59.719128+0000) 2023-08-27T23:14:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:00 smithi093 ceph-mon[111335]: pgmap v4214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:00 smithi093 ceph-mon[118902]: pgmap v4214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:00.694Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:00.695Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:00 smithi136 ceph-mon[113531]: pgmap v4214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:00.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:00.719350+0000) 2023-08-27T23:14:02.188 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2136616650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:02.188 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2136616650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2136616650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:01.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:01.719619+0000) 2023-08-27T23:14:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:02.187Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:02.189Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:02 smithi136 ceph-mon[113531]: pgmap v4215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:02.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:02.719859+0000) 2023-08-27T23:14:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:02 smithi093 ceph-mon[111335]: pgmap v4215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:02 smithi093 ceph-mon[118902]: pgmap v4215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:03.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:03.720113+0000) 2023-08-27T23:14:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1262091677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1262091677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1262091677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:04.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:04.720295+0000) 2023-08-27T23:14:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:05 smithi093 ceph-mon[111335]: pgmap v4216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:05 smithi093 ceph-mon[118902]: pgmap v4216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:14:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:14:05] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:14:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:05 smithi136 ceph-mon[113531]: pgmap v4216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:05.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:05.720538+0000) 2023-08-27T23:14:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:06.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:06.720720+0000) 2023-08-27T23:14:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:07 smithi093 ceph-mon[111335]: pgmap v4217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3079411245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:14:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:07 smithi093 ceph-mon[118902]: pgmap v4217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3079411245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:14:07.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:07 smithi136 ceph-mon[113531]: pgmap v4217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:07.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3079411245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:07.902 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:14:07.902 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:07.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:07.720900+0000) 2023-08-27T23:14:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:08 smithi136 ceph-mon[113531]: pgmap v4218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:08.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:08.721157+0000) 2023-08-27T23:14:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:08 smithi093 ceph-mon[111335]: pgmap v4218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:08 smithi093 ceph-mon[118902]: pgmap v4218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:09.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:09.721362+0000) 2023-08-27T23:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/142451071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/142451071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/142451071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:14:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:14:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[111335]: pgmap v4219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:14:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:14:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-mon[118902]: pgmap v4219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:10.695Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:10.696Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:14:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:14:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:14:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:10 smithi136 ceph-mon[113531]: pgmap v4219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:10.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:10.721556+0000) 2023-08-27T23:14:12.038 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:11.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:11.721808+0000) 2023-08-27T23:14:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1294989489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1294989489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1294989489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:12.188Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:12.189Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:13.045 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:12.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:12.722073+0000) 2023-08-27T23:14:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:13 smithi136 ceph-mon[113531]: pgmap v4220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:13 smithi093 ceph-mon[111335]: pgmap v4220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:13 smithi093 ceph-mon[118902]: pgmap v4220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:14.047 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:13.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:13.722317+0000) 2023-08-27T23:14:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/267805896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/267805896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/267805896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:15.051 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:14.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:14.722574+0000) 2023-08-27T23:14:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:15 smithi093 ceph-mon[111335]: pgmap v4221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:15 smithi093 ceph-mon[118902]: pgmap v4221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:15 smithi136 ceph-mon[113531]: pgmap v4221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:14:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:14:15] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:14:16.167 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:15.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:15.722842+0000) 2023-08-27T23:14:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4105438161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4105438161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4105438161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:17.172 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:16.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:16.723153+0000) 2023-08-27T23:14:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:17 smithi136 ceph-mon[113531]: pgmap v4222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:17 smithi093 ceph-mon[111335]: pgmap v4222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:17 smithi093 ceph-mon[118902]: pgmap v4222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:17.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:17.723379+0000) 2023-08-27T23:14:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:18.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:18.723598+0000) 2023-08-27T23:14:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:19 smithi093 ceph-mon[111335]: pgmap v4223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1644946015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:19 smithi093 ceph-mon[118902]: pgmap v4223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1644946015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:19 smithi136 ceph-mon[113531]: pgmap v4223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1644946015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:19.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:19.723876+0000) 2023-08-27T23:14:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:20 smithi093 ceph-mon[111335]: pgmap v4224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3296728659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:20 smithi093 ceph-mon[118902]: pgmap v4224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3296728659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:20.696Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:20.696Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:20 smithi136 ceph-mon[113531]: pgmap v4224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3296728659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:20.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:20.724150+0000) 2023-08-27T23:14:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:21.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:21.724362+0000) 2023-08-27T23:14:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:22.188Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:22.189Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:22.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:22.724603+0000) 2023-08-27T23:14:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:23 smithi093 ceph-mon[111335]: pgmap v4225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3479321033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:23 smithi093 ceph-mon[118902]: pgmap v4225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3479321033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:23 smithi136 ceph-mon[113531]: pgmap v4225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3479321033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:23.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:23.724861+0000) 2023-08-27T23:14:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:24.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:24.725101+0000) 2023-08-27T23:14:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:25 smithi093 ceph-mon[111335]: pgmap v4226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:25 smithi093 ceph-mon[118902]: pgmap v4226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:14:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:14:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:14:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:25 smithi136 ceph-mon[113531]: pgmap v4226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:25.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:25.725357+0000) 2023-08-27T23:14:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4190526591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:26.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:26.725605+0000) 2023-08-27T23:14:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4190526591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4190526591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:27 smithi136 ceph-mon[113531]: pgmap v4227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:27.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:27.725845+0000) 2023-08-27T23:14:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:27 smithi093 ceph-mon[111335]: pgmap v4227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:27 smithi093 ceph-mon[118902]: pgmap v4227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1243167810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:28 smithi136 ceph-mon[113531]: pgmap v4228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:28.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:28.726098+0000) 2023-08-27T23:14:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1243167810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:28 smithi093 ceph-mon[111335]: pgmap v4228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1243167810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:28 smithi093 ceph-mon[118902]: pgmap v4228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:29.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:29.726349+0000) 2023-08-27T23:14:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2366166435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:30 smithi093 ceph-mon[111335]: pgmap v4229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2366166435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:30 smithi093 ceph-mon[118902]: pgmap v4229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:30.696Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:30.697Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2366166435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:30 smithi136 ceph-mon[113531]: pgmap v4229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:30.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:30.726617+0000) 2023-08-27T23:14:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:31.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:31.726880+0000) 2023-08-27T23:14:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:32.189Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:32.190Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:32.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:32.727099+0000) 2023-08-27T23:14:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:33 smithi093 ceph-mon[111335]: pgmap v4230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4029734597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:33 smithi093 ceph-mon[118902]: pgmap v4230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4029734597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:33 smithi136 ceph-mon[113531]: pgmap v4230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4029734597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:33.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:33.727383+0000) 2023-08-27T23:14:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:34.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:34.727633+0000) 2023-08-27T23:14:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:35 smithi093 ceph-mon[111335]: pgmap v4231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/338707814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:14:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:14:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:14:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:35 smithi093 ceph-mon[118902]: pgmap v4231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/338707814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:35 smithi136 ceph-mon[113531]: pgmap v4231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/338707814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:35.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:35.727864+0000) 2023-08-27T23:14:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:36.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:36.728137+0000) 2023-08-27T23:14:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:37 smithi093 ceph-mon[111335]: pgmap v4232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:37 smithi093 ceph-mon[118902]: pgmap v4232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:37 smithi136 ceph-mon[113531]: pgmap v4232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:37.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:37.728381+0000) 2023-08-27T23:14:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3346392784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:38.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:38.728605+0000) 2023-08-27T23:14:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3346392784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3346392784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:39.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:39 smithi136 ceph-mon[113531]: pgmap v4233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:39.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:39.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:39.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:39.728863+0000) 2023-08-27T23:14:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:39 smithi093 ceph-mon[111335]: pgmap v4233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:39 smithi093 ceph-mon[118902]: pgmap v4233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3298277539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:40 smithi093 ceph-mon[111335]: pgmap v4234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3298277539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:40 smithi093 ceph-mon[118902]: pgmap v4234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:40.696Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:40.698Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3298277539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:40 smithi136 ceph-mon[113531]: pgmap v4234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:40.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:40.729134+0000) 2023-08-27T23:14:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:41.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:41.729318+0000) 2023-08-27T23:14:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/23351377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/23351377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:42.190Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:42.191Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/23351377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:42.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:42.729605+0000) 2023-08-27T23:14:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:43 smithi136 ceph-mon[113531]: pgmap v4235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:43 smithi093 ceph-mon[111335]: pgmap v4235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:43 smithi093 ceph-mon[118902]: pgmap v4235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:43.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:43.729826+0000) 2023-08-27T23:14:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:44.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:44.730121+0000) 2023-08-27T23:14:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:45 smithi093 ceph-mon[111335]: pgmap v4236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2999522446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:14:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:14:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:14:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:45 smithi093 ceph-mon[118902]: pgmap v4236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2999522446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:45.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:45.730367+0000) 2023-08-27T23:14:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:45 smithi136 ceph-mon[113531]: pgmap v4236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2999522446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:46.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:46.730640+0000) 2023-08-27T23:14:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:47 smithi093 ceph-mon[118902]: pgmap v4237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2872792997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:47 smithi093 ceph-mon[111335]: pgmap v4237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2872792997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:47 smithi136 ceph-mon[113531]: pgmap v4237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2872792997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:47.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:47.730886+0000) 2023-08-27T23:14:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:48.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:48.731147+0000) 2023-08-27T23:14:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:49 smithi136 ceph-mon[113531]: pgmap v4238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1459545332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:49.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:49.731401+0000) 2023-08-27T23:14:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:49 smithi093 ceph-mon[111335]: pgmap v4238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1459545332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:49 smithi093 ceph-mon[118902]: pgmap v4238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:49.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1459545332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:50 smithi093 ceph-mon[118902]: pgmap v4239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:50 smithi093 ceph-mon[111335]: pgmap v4239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:50.697Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:50.698Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:50 smithi136 ceph-mon[113531]: pgmap v4239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:50.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:50.731623+0000) 2023-08-27T23:14:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1429154655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:51.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:51.731834+0000) 2023-08-27T23:14:52.190 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1429154655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:52.190 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1429154655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:52.190Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:14:52.191Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:14:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:52 smithi136 ceph-mon[113531]: pgmap v4240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:52.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:52.732068+0000) 2023-08-27T23:14:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:52 smithi093 ceph-mon[111335]: pgmap v4240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:52 smithi093 ceph-mon[118902]: pgmap v4240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:53.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:53.732317+0000) 2023-08-27T23:14:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:14:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/520243201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:54 smithi136 ceph-mon[113531]: pgmap v4241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:54.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:54.732575+0000) 2023-08-27T23:14:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/520243201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:54 smithi093 ceph-mon[111335]: pgmap v4241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/520243201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:54 smithi093 ceph-mon[118902]: pgmap v4241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:14:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:14:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:14:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:55.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:55.732810+0000) 2023-08-27T23:14:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:56.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:56.733015+0000) 2023-08-27T23:14:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:57 smithi093 ceph-mon[111335]: pgmap v4242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2129773274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:57 smithi093 ceph-mon[118902]: pgmap v4242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2129773274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:57.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:57.733280+0000) 2023-08-27T23:14:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:57 smithi136 ceph-mon[113531]: pgmap v4242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2129773274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:58.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:58.733463+0000) 2023-08-27T23:14:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:59 smithi136 ceph-mon[113531]: pgmap v4243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:14:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4148515028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:14:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:14:59.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:14:59.733704+0000) 2023-08-27T23:14:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:59 smithi093 ceph-mon[111335]: pgmap v4243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:14:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4148515028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:14:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:59 smithi093 ceph-mon[118902]: pgmap v4243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:14:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:14:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4148515028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:00 smithi093 ceph-mon[111335]: pgmap v4244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:00 smithi093 ceph-mon[118902]: pgmap v4244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:00.697Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:00.698Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:00 smithi136 ceph-mon[113531]: pgmap v4244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:00.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:00.733954+0000) 2023-08-27T23:15:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/756015989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:01.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:01.734215+0000) 2023-08-27T23:15:02.191 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/756015989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:02.192 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/756015989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:02.191Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:02.192Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:02 smithi136 ceph-mon[113531]: pgmap v4245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:02.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:02.734486+0000) 2023-08-27T23:15:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:02 smithi093 ceph-mon[111335]: pgmap v4245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:02 smithi093 ceph-mon[118902]: pgmap v4245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1764697008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:03.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:03.734751+0000) 2023-08-27T23:15:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1764697008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1764697008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:04 smithi136 ceph-mon[113531]: pgmap v4246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:04.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:04.735053+0000) 2023-08-27T23:15:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:04 smithi093 ceph-mon[111335]: pgmap v4246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:04 smithi093 ceph-mon[118902]: pgmap v4246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:15:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:15:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:15:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:05.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:05.735289+0000) 2023-08-27T23:15:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3025614919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3025614919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3025614919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:06.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:06.735453+0000) 2023-08-27T23:15:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:07 smithi093 ceph-mon[111335]: pgmap v4247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:07 smithi093 ceph-mon[118902]: pgmap v4247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:07 smithi136 ceph-mon[113531]: pgmap v4247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:07.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:07.735684+0000) 2023-08-27T23:15:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:08.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:08.735903+0000) 2023-08-27T23:15:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:09 smithi136 ceph-mon[113531]: pgmap v4248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1771122514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:09.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:09.736157+0000) 2023-08-27T23:15:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:09 smithi093 ceph-mon[111335]: pgmap v4248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1771122514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:09 smithi093 ceph-mon[118902]: pgmap v4248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1771122514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[111335]: pgmap v4249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1595885422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[118902]: pgmap v4249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1595885422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:10.698Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:10.699Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:11.057 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:10.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:10.736313+0000) 2023-08-27T23:15:11.057 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:15:11.058 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:10 smithi136 ceph-mon[113531]: pgmap v4249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:11.058 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:11.058 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:11.058 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1595885422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:11.736 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:11.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:11.736483+0000) 2023-08-27T23:15:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:12.192Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:12.193Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:12 smithi136 ceph-mon[113531]: pgmap v4250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:12.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:12.736653+0000) 2023-08-27T23:15:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[118902]: pgmap v4250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[111335]: pgmap v4250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:15:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:15:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1481208309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:13.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:13.736864+0000) 2023-08-27T23:15:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1481208309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:15:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:15:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:15:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1481208309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:14 smithi136 ceph-mon[113531]: pgmap v4251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:14.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:14.737153+0000) 2023-08-27T23:15:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:14 smithi093 ceph-mon[111335]: pgmap v4251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:14 smithi093 ceph-mon[118902]: pgmap v4251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:15:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:15:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:15:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/649216183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:15.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:15.737408+0000) 2023-08-27T23:15:16.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/649216183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/649216183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:16 smithi136 ceph-mon[113531]: pgmap v4252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:16.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:16.737623+0000) 2023-08-27T23:15:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:16 smithi093 ceph-mon[111335]: pgmap v4252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:16 smithi093 ceph-mon[118902]: pgmap v4252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:18.124 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:17.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:17.737841+0000) 2023-08-27T23:15:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1148478069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1148478069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1148478069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:19.129 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:18.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:18.738062+0000) 2023-08-27T23:15:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:19 smithi136 ceph-mon[113531]: pgmap v4253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:19 smithi093 ceph-mon[111335]: pgmap v4253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:19 smithi093 ceph-mon[118902]: pgmap v4253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:19.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:19.738311+0000) 2023-08-27T23:15:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:20 smithi093 ceph-mon[111335]: pgmap v4254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1301516838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:20 smithi093 ceph-mon[118902]: pgmap v4254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1301516838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:20.699Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:20.700Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:20 smithi136 ceph-mon[113531]: pgmap v4254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1301516838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:20.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:20.738557+0000) 2023-08-27T23:15:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:21.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:21.738824+0000) 2023-08-27T23:15:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:22.193Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:22.193Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:22.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:22.739103+0000) 2023-08-27T23:15:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:23 smithi136 ceph-mon[113531]: pgmap v4255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/252896405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:23.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:23.739256+0000) 2023-08-27T23:15:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:23 smithi093 ceph-mon[111335]: pgmap v4255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/252896405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:23 smithi093 ceph-mon[118902]: pgmap v4255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/252896405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:24 smithi136 ceph-mon[113531]: pgmap v4256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:24.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:24.739416+0000) 2023-08-27T23:15:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:24 smithi093 ceph-mon[111335]: pgmap v4256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:24 smithi093 ceph-mon[118902]: pgmap v4256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:15:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:15:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:15:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3664421388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:26.197 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:25.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:25.739653+0000) 2023-08-27T23:15:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3664421388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3664421388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:26 smithi136 ceph-mon[113531]: pgmap v4257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:26.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:26.739907+0000) 2023-08-27T23:15:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:26 smithi093 ceph-mon[111335]: pgmap v4257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:26 smithi093 ceph-mon[118902]: pgmap v4257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4078585050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:27.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:27.740116+0000) 2023-08-27T23:15:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4078585050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4078585050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:28 smithi136 ceph-mon[113531]: pgmap v4258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:28.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:28.740393+0000) 2023-08-27T23:15:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:28 smithi093 ceph-mon[111335]: pgmap v4258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:28 smithi093 ceph-mon[118902]: pgmap v4258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:29.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:29.740611+0000) 2023-08-27T23:15:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2615793974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:30 smithi093 ceph-mon[111335]: pgmap v4259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2615793974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:30 smithi093 ceph-mon[118902]: pgmap v4259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:30.700Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:30.700Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2615793974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:30 smithi136 ceph-mon[113531]: pgmap v4259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:30.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:30.740864+0000) 2023-08-27T23:15:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:31.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:31.741156+0000) 2023-08-27T23:15:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1793236660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1793236660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:32.194Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:32.194Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1793236660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:32.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:32.741406+0000) 2023-08-27T23:15:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:33 smithi136 ceph-mon[113531]: pgmap v4260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:33 smithi093 ceph-mon[111335]: pgmap v4260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:33 smithi093 ceph-mon[118902]: pgmap v4260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:33.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:33.741681+0000) 2023-08-27T23:15:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:34 smithi136 ceph-mon[113531]: pgmap v4261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:34.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:34.741888+0000) 2023-08-27T23:15:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:34 smithi093 ceph-mon[111335]: pgmap v4261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:34 smithi093 ceph-mon[118902]: pgmap v4261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2487685867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:15:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:15:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:15:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2487685867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2487685867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:35.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:35.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:35.742137+0000) 2023-08-27T23:15:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:36 smithi136 ceph-mon[113531]: pgmap v4262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:36.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:36.742392+0000) 2023-08-27T23:15:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:36 smithi093 ceph-mon[111335]: pgmap v4262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:36 smithi093 ceph-mon[118902]: pgmap v4262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2888877063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:37.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:37.742638+0000) 2023-08-27T23:15:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2888877063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2888877063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:38 smithi136 ceph-mon[113531]: pgmap v4263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:39.375 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:38.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:38.742892+0000) 2023-08-27T23:15:39.376 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:38 smithi093 ceph-mon[111335]: pgmap v4263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:39.376 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:38 smithi093 ceph-mon[118902]: pgmap v4263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1530752882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:39.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:39.743153+0000) 2023-08-27T23:15:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1530752882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1530752882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:40 smithi093 ceph-mon[118902]: pgmap v4264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:40 smithi093 ceph-mon[111335]: pgmap v4264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:40.700Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:40.702Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:40 smithi136 ceph-mon[113531]: pgmap v4264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:40.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:40.743406+0000) 2023-08-27T23:15:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/121997868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:41.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:41.743682+0000) 2023-08-27T23:15:42.196 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/121997868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:42.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/121997868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:42.197 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:42.195Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:42.197 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:42.196Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:42 smithi136 ceph-mon[113531]: pgmap v4265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:42.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:42.743979+0000) 2023-08-27T23:15:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:42 smithi093 ceph-mon[111335]: pgmap v4265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:42 smithi093 ceph-mon[118902]: pgmap v4265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:43.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:43.744289+0000) 2023-08-27T23:15:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2712310685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2712310685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2712310685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:44.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:44.744533+0000) 2023-08-27T23:15:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:45 smithi136 ceph-mon[113531]: pgmap v4266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:45 smithi093 ceph-mon[111335]: pgmap v4266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:45 smithi093 ceph-mon[118902]: pgmap v4266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:15:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:15:45] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:15:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:45.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:45.744791+0000) 2023-08-27T23:15:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:46 smithi136 ceph-mon[113531]: pgmap v4267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:46.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:46.745076+0000) 2023-08-27T23:15:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:46 smithi093 ceph-mon[111335]: pgmap v4267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:46 smithi093 ceph-mon[118902]: pgmap v4267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2447114080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:47.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:47.745315+0000) 2023-08-27T23:15:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2447114080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2447114080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:48 smithi136 ceph-mon[113531]: pgmap v4268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:48.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:48.745569+0000) 2023-08-27T23:15:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:48 smithi093 ceph-mon[111335]: pgmap v4268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:48.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:48 smithi093 ceph-mon[118902]: pgmap v4268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:49.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:49.745831+0000) 2023-08-27T23:15:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/336007717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/336007717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/336007717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:50 smithi093 ceph-mon[111335]: pgmap v4269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:50 smithi093 ceph-mon[118902]: pgmap v4269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:50.701Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:50.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:50 smithi136 ceph-mon[113531]: pgmap v4269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:50.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:50.746125+0000) 2023-08-27T23:15:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3571916115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:51.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:51.746392+0000) 2023-08-27T23:15:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3571916115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3571916115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:52.195Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:15:52.196Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:15:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:52 smithi136 ceph-mon[113531]: pgmap v4270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:52.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:52.746635+0000) 2023-08-27T23:15:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:52 smithi093 ceph-mon[111335]: pgmap v4270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:52 smithi093 ceph-mon[118902]: pgmap v4270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1918019074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:53.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:53.746884+0000) 2023-08-27T23:15:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1918019074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:15:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1918019074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:54 smithi136 ceph-mon[113531]: pgmap v4271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:54.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:54.747150+0000) 2023-08-27T23:15:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:54 smithi093 ceph-mon[111335]: pgmap v4271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:54 smithi093 ceph-mon[118902]: pgmap v4271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:15:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:15:55] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:15:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:55.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:55.747425+0000) 2023-08-27T23:15:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2725469279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2725469279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2725469279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:56.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:56.747649+0000) 2023-08-27T23:15:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:57 smithi136 ceph-mon[113531]: pgmap v4272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:57 smithi093 ceph-mon[111335]: pgmap v4272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:57 smithi093 ceph-mon[118902]: pgmap v4272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:57.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:57.747881+0000) 2023-08-27T23:15:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:58 smithi136 ceph-mon[113531]: pgmap v4273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:58.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:58.748153+0000) 2023-08-27T23:15:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:58 smithi093 ceph-mon[111335]: pgmap v4273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:58 smithi093 ceph-mon[118902]: pgmap v4273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:15:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:15:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1186612281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:15:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:15:59.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:15:59.748383+0000) 2023-08-27T23:15:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:15:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1186612281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:15:59.994 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:15:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1186612281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:00.954 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:00.702Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:00.954 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:00.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:00 smithi136 ceph-mon[113531]: pgmap v4274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:00.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:00.748640+0000) 2023-08-27T23:16:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:00 smithi093 ceph-mon[118902]: pgmap v4274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:00 smithi093 ceph-mon[111335]: pgmap v4274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3203424603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:01.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:01.748873+0000) 2023-08-27T23:16:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3203424603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3203424603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:02.195Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:02.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:02.197Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:02 smithi136 ceph-mon[113531]: pgmap v4275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:02.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:02.749167+0000) 2023-08-27T23:16:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:02 smithi093 ceph-mon[111335]: pgmap v4275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:02 smithi093 ceph-mon[118902]: pgmap v4275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/671437374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:03.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:03.749411+0000) 2023-08-27T23:16:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/671437374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/671437374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:04 smithi136 ceph-mon[113531]: pgmap v4276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:04.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:04.749660+0000) 2023-08-27T23:16:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:04 smithi093 ceph-mon[111335]: pgmap v4276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:04 smithi093 ceph-mon[118902]: pgmap v4276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:16:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:16:05] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:16:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3365206518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:05.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:05.749892+0000) 2023-08-27T23:16:06.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3365206518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:06.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3365206518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:06 smithi136 ceph-mon[113531]: pgmap v4277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:06.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:06.750153+0000) 2023-08-27T23:16:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:06 smithi093 ceph-mon[111335]: pgmap v4277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:06 smithi093 ceph-mon[118902]: pgmap v4277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:08.157 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:07.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:07.750387+0000) 2023-08-27T23:16:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3206997967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3206997967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3206997967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:09.161 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:08.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:08.750680+0000) 2023-08-27T23:16:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:09 smithi136 ceph-mon[113531]: pgmap v4278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:09 smithi093 ceph-mon[111335]: pgmap v4278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:09 smithi093 ceph-mon[118902]: pgmap v4278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:09.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:09.750980+0000) 2023-08-27T23:16:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:10 smithi093 ceph-mon[118902]: pgmap v4279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1181530400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:10 smithi093 ceph-mon[111335]: pgmap v4279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1181530400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:10.703Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:10.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:10 smithi136 ceph-mon[113531]: pgmap v4279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1181530400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:10.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:10.751266+0000) 2023-08-27T23:16:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:11.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:11.751516+0000) 2023-08-27T23:16:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:12.196Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:12.197Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:12 smithi136 ceph-mon[113531]: pgmap v4280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:12.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:12.751763+0000) 2023-08-27T23:16:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:12 smithi093 ceph-mon[111335]: pgmap v4280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:12 smithi093 ceph-mon[118902]: pgmap v4280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1158455596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:16:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:13.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:13.751933+0000) 2023-08-27T23:16:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1158455596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:16:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1158455596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:16:14.752 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:14 smithi136 ceph-mon[113531]: pgmap v4281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:14.943 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:14 smithi093 ceph-mon[111335]: pgmap v4281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:14.943 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:14 smithi093 ceph-mon[118902]: pgmap v4281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:14.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:14.752115+0000) 2023-08-27T23:16:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:16:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:16:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:16:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:15.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:15.752347+0000) 2023-08-27T23:16:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2836846149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2836846149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:16.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2836846149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:16.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:16.752590+0000) 2023-08-27T23:16:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:16:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:16:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[111335]: pgmap v4282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:16:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:16:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:17 smithi093 ceph-mon[118902]: pgmap v4282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:17 smithi136 ceph-mon[113531]: pgmap v4282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:17.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:17.752829+0000) 2023-08-27T23:16:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1985763572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1985763572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1985763572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:18.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:18.753114+0000) 2023-08-27T23:16:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:19 smithi136 ceph-mon[113531]: pgmap v4283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:19 smithi093 ceph-mon[111335]: pgmap v4283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:19 smithi093 ceph-mon[118902]: pgmap v4283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:19.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:19.753367+0000) 2023-08-27T23:16:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/179692251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:20 smithi093 ceph-mon[111335]: pgmap v4284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/179692251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:20 smithi093 ceph-mon[118902]: pgmap v4284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:20.703Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:20.704Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/179692251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:20 smithi136 ceph-mon[113531]: pgmap v4284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:20.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:20.753617+0000) 2023-08-27T23:16:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:21.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:21.753866+0000) 2023-08-27T23:16:22.472 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:22.197Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:22.472 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:22.198Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3159974953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3159974953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:22.754 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3159974953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:22.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:22.754149+0000) 2023-08-27T23:16:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:23 smithi093 ceph-mon[111335]: pgmap v4285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:23 smithi093 ceph-mon[118902]: pgmap v4285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:23.754 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:23 smithi136 ceph-mon[113531]: pgmap v4285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:23.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:23.754428+0000) 2023-08-27T23:16:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:24.754 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:24.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:24.754673+0000) 2023-08-27T23:16:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:25 smithi093 ceph-mon[111335]: pgmap v4286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1520300310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:16:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:16:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:16:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:25 smithi093 ceph-mon[118902]: pgmap v4286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1520300310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:25.755 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:25 smithi136 ceph-mon[113531]: pgmap v4286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:25.755 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1520300310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:25.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:25.754959+0000) 2023-08-27T23:16:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:26.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:26.755196+0000) 2023-08-27T23:16:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:27 smithi093 ceph-mon[118902]: pgmap v4287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1771397943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:27 smithi093 ceph-mon[111335]: pgmap v4287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1771397943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:27.755 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:27 smithi136 ceph-mon[113531]: pgmap v4287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:27.755 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1771397943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:27.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:27.755404+0000) 2023-08-27T23:16:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:28.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:28.755642+0000) 2023-08-27T23:16:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:29 smithi093 ceph-mon[111335]: pgmap v4288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:29 smithi093 ceph-mon[118902]: pgmap v4288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:29.756 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:29 smithi136 ceph-mon[113531]: pgmap v4288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:29.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:29.755917+0000) 2023-08-27T23:16:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/963657945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/963657945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:30.703Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:30.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:30.704Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:30.756 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/963657945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:30.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:30.756173+0000) 2023-08-27T23:16:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:31 smithi093 ceph-mon[111335]: pgmap v4289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:31 smithi093 ceph-mon[118902]: pgmap v4289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:31.756 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:31 smithi136 ceph-mon[113531]: pgmap v4289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:31.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:31.756438+0000) 2023-08-27T23:16:32.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:32.197Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:32.198Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:32.756 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1399950457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1399950457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1399950457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:32.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:32.756684+0000) 2023-08-27T23:16:33.757 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:33 smithi136 ceph-mon[113531]: pgmap v4290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:33 smithi093 ceph-mon[111335]: pgmap v4290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:33 smithi093 ceph-mon[118902]: pgmap v4290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:33.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:33.756947+0000) 2023-08-27T23:16:34.757 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3520973623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3520973623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3520973623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:34.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:34.757199+0000) 2023-08-27T23:16:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:35 smithi093 ceph-mon[111335]: pgmap v4291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:35 smithi093 ceph-mon[118902]: pgmap v4291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:16:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:16:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:16:35.757 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:35 smithi136 ceph-mon[113531]: pgmap v4291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:35.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:35.757400+0000) 2023-08-27T23:16:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:36.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:36.757667+0000) 2023-08-27T23:16:37.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:37 smithi136 ceph-mon[113531]: pgmap v4292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:37.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/238005042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:37 smithi093 ceph-mon[111335]: pgmap v4292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/238005042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:37 smithi093 ceph-mon[118902]: pgmap v4292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/238005042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:37.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:37.757882+0000) 2023-08-27T23:16:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:38.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:38.758172+0000) 2023-08-27T23:16:39.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:39 smithi136 ceph-mon[113531]: pgmap v4293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:39.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:39.759 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3121238816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:39 smithi093 ceph-mon[111335]: pgmap v4293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3121238816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:39 smithi093 ceph-mon[118902]: pgmap v4293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3121238816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:39.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:39.758448+0000) 2023-08-27T23:16:40.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:40.704Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:40.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:40.705Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:40 smithi136 ceph-mon[113531]: pgmap v4294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:40.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:40.758644+0000) 2023-08-27T23:16:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:40 smithi093 ceph-mon[111335]: pgmap v4294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:40 smithi093 ceph-mon[118902]: pgmap v4294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1490205197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:41.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:41.758841+0000) 2023-08-27T23:16:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1490205197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1490205197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:42.198Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:42.199Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:42 smithi136 ceph-mon[113531]: pgmap v4295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:42.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:42.759060+0000) 2023-08-27T23:16:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:42 smithi093 ceph-mon[111335]: pgmap v4295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:42 smithi093 ceph-mon[118902]: pgmap v4295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2743977608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:43.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:43.759326+0000) 2023-08-27T23:16:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2743977608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2743977608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:44 smithi136 ceph-mon[113531]: pgmap v4296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:44.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:44.759569+0000) 2023-08-27T23:16:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:44 smithi093 ceph-mon[111335]: pgmap v4296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:44 smithi093 ceph-mon[118902]: pgmap v4296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:16:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:16:45] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:16:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:45.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:45.759810+0000) 2023-08-27T23:16:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3432547361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3432547361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3432547361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:46.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:46.760105+0000) 2023-08-27T23:16:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:47 smithi136 ceph-mon[113531]: pgmap v4297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:47 smithi093 ceph-mon[111335]: pgmap v4297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:47 smithi093 ceph-mon[118902]: pgmap v4297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:47.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:47.760350+0000) 2023-08-27T23:16:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:48 smithi136 ceph-mon[113531]: pgmap v4298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:48.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:48.760599+0000) 2023-08-27T23:16:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:48 smithi093 ceph-mon[111335]: pgmap v4298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:48 smithi093 ceph-mon[118902]: pgmap v4298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/335196349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:49.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:49.760868+0000) 2023-08-27T23:16:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/335196349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/335196349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:50.955 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:50.704Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:50.955 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:50.705Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:50 smithi136 ceph-mon[113531]: pgmap v4299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:50.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:50.761153+0000) 2023-08-27T23:16:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:50 smithi093 ceph-mon[111335]: pgmap v4299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:50 smithi093 ceph-mon[118902]: pgmap v4299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3670560678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:51.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:51.761373+0000) 2023-08-27T23:16:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3670560678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3670560678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:52.199Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:52.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:16:52.200Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:16:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:52 smithi136 ceph-mon[113531]: pgmap v4300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:52.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:52.761578+0000) 2023-08-27T23:16:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:52 smithi093 ceph-mon[111335]: pgmap v4300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:52 smithi093 ceph-mon[118902]: pgmap v4300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2264813336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:53.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:53.761796+0000) 2023-08-27T23:16:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2264813336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2264813336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:16:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:54 smithi136 ceph-mon[113531]: pgmap v4301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:54.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:54.762073+0000) 2023-08-27T23:16:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:54 smithi093 ceph-mon[111335]: pgmap v4301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:54 smithi093 ceph-mon[118902]: pgmap v4301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:16:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:16:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:16:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2359146388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:55.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:55.762293+0000) 2023-08-27T23:16:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2359146388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2359146388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:56 smithi136 ceph-mon[113531]: pgmap v4302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:56.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:56.762524+0000) 2023-08-27T23:16:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:56 smithi093 ceph-mon[111335]: pgmap v4302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:56 smithi093 ceph-mon[118902]: pgmap v4302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:57.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:57.762782+0000) 2023-08-27T23:16:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1428913563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1428913563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1428913563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:16:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:58.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:58.763080+0000) 2023-08-27T23:16:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:16:59 smithi093 ceph-mon[111335]: pgmap v4303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:16:59 smithi093 ceph-mon[118902]: pgmap v4303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:16:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:16:59 smithi136 ceph-mon[113531]: pgmap v4303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:16:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:16:59.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:16:59.763284+0000) 2023-08-27T23:17:00.956 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:00.705Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:00.956 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:00.706Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:00 smithi136 ceph-mon[113531]: pgmap v4304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/205744912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:00.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:00.763552+0000) 2023-08-27T23:17:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:00 smithi093 ceph-mon[118902]: pgmap v4304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/205744912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:00 smithi093 ceph-mon[111335]: pgmap v4304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/205744912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:01.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:01.763780+0000) 2023-08-27T23:17:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:02.199Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:02.200Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:02 smithi136 ceph-mon[113531]: pgmap v4305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:02.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:02.764056+0000) 2023-08-27T23:17:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:02 smithi093 ceph-mon[118902]: pgmap v4305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:02 smithi093 ceph-mon[111335]: pgmap v4305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3431995724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:03.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:03.764279+0000) 2023-08-27T23:17:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3431995724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3431995724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:04 smithi136 ceph-mon[113531]: pgmap v4306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:04.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:04.764527+0000) 2023-08-27T23:17:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:04 smithi093 ceph-mon[111335]: pgmap v4306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:04 smithi093 ceph-mon[118902]: pgmap v4306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:17:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:17:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:17:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3727158980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:05.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:05.764755+0000) 2023-08-27T23:17:06.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3727158980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3727158980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:06 smithi136 ceph-mon[113531]: pgmap v4307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:06.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:06.765060+0000) 2023-08-27T23:17:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:06 smithi093 ceph-mon[118902]: pgmap v4307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:06 smithi093 ceph-mon[111335]: pgmap v4307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3112817427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:07.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:07.765300+0000) 2023-08-27T23:17:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3112817427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3112817427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:08 smithi136 ceph-mon[113531]: pgmap v4308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:08.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:08.765548+0000) 2023-08-27T23:17:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:08 smithi093 ceph-mon[111335]: pgmap v4308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:08 smithi093 ceph-mon[118902]: pgmap v4308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:09.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:09.765780+0000) 2023-08-27T23:17:10.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:10.706Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:10.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:10.707Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2849596036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:10 smithi136 ceph-mon[113531]: pgmap v4309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:10.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:10.766059+0000) 2023-08-27T23:17:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2849596036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:10 smithi093 ceph-mon[111335]: pgmap v4309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2849596036' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:10 smithi093 ceph-mon[118902]: pgmap v4309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:11.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:11.766279+0000) 2023-08-27T23:17:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:12.200Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:12.201Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:12.766 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4147911917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4147911917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4147911917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:12.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:12.766531+0000) 2023-08-27T23:17:13.766 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:13 smithi136 ceph-mon[113531]: pgmap v4310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:13 smithi093 ceph-mon[118902]: pgmap v4310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:13 smithi093 ceph-mon[111335]: pgmap v4310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:13.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:13.766823+0000) 2023-08-27T23:17:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:14.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:14.767102+0000) 2023-08-27T23:17:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:15 smithi093 ceph-mon[111335]: pgmap v4311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3076342445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:15 smithi093 ceph-mon[118902]: pgmap v4311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3076342445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:17:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:17:15] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:17:15.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:15 smithi136 ceph-mon[113531]: pgmap v4311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:15.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3076342445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:15.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:15.767363+0000) 2023-08-27T23:17:16.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:17:16.768 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:16.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:16.767539+0000) 2023-08-27T23:17:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:17:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:17:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:17 smithi136 ceph-mon[113531]: pgmap v4312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3124506477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:17.859 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:17 smithi093 ceph-mon[111335]: pgmap v4312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:17.859 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3124506477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:17.859 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:17 smithi093 ceph-mon[118902]: pgmap v4312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:17.859 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3124506477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:17.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:17.767778+0000) 2023-08-27T23:17:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:18.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:18.767978+0000) 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: pgmap v4313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:17:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: pgmap v4313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:17:19.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: pgmap v4313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:19.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:19.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:17:19.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:17:19.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:17:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:19.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:19.768189+0000) 2023-08-27T23:17:20.708 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/475794637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:20.708 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/475794637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:20.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/475794637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:20.707Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:20.708Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:20.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:20.768448+0000) 2023-08-27T23:17:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:21 smithi093 ceph-mon[111335]: pgmap v4314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:21 smithi093 ceph-mon[118902]: pgmap v4314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:21.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:21 smithi136 ceph-mon[113531]: pgmap v4314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:21.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:21.768666+0000) 2023-08-27T23:17:22.455 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:22.201Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:22.584 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:22.202Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3060832594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3060832594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:22.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3060832594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:22.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:22.768889+0000) 2023-08-27T23:17:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:23 smithi093 ceph-mon[111335]: pgmap v4315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:23 smithi093 ceph-mon[118902]: pgmap v4315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:23.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:23 smithi136 ceph-mon[113531]: pgmap v4315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:23.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:23.769149+0000) 2023-08-27T23:17:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/538338462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/538338462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:24.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:24.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/538338462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:24.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:24.769414+0000) 2023-08-27T23:17:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:25 smithi093 ceph-mon[111335]: pgmap v4316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:25 smithi093 ceph-mon[118902]: pgmap v4316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:17:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:17:25] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:17:25.769 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:25 smithi136 ceph-mon[113531]: pgmap v4316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:25.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:25.769684+0000) 2023-08-27T23:17:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:26 smithi136 ceph-mon[113531]: pgmap v4317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:26.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:26.769881+0000) 2023-08-27T23:17:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:26 smithi093 ceph-mon[111335]: pgmap v4317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:26 smithi093 ceph-mon[118902]: pgmap v4317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2680186501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:27.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:27.770090+0000) 2023-08-27T23:17:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2680186501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2680186501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:28 smithi136 ceph-mon[113531]: pgmap v4318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:28.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:28.770289+0000) 2023-08-27T23:17:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:28 smithi093 ceph-mon[111335]: pgmap v4318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:28 smithi093 ceph-mon[118902]: pgmap v4318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:29.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:29.770534+0000) 2023-08-27T23:17:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3086589220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3086589220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3086589220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:30.963 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:30.708Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:30.963 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:30.709Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:30 smithi136 ceph-mon[113531]: pgmap v4319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:30.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:30.770778+0000) 2023-08-27T23:17:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:30 smithi093 ceph-mon[111335]: pgmap v4319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:30 smithi093 ceph-mon[118902]: pgmap v4319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2137030944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:31.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:31.771028+0000) 2023-08-27T23:17:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2137030944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2137030944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:32.201Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:32.202Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:32 smithi136 ceph-mon[113531]: pgmap v4320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:32.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:32.771269+0000) 2023-08-27T23:17:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:32 smithi093 ceph-mon[111335]: pgmap v4320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:32 smithi093 ceph-mon[118902]: pgmap v4320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1167749067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:33.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:33.771496+0000) 2023-08-27T23:17:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1167749067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1167749067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:34 smithi136 ceph-mon[113531]: pgmap v4321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:34.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:34.771714+0000) 2023-08-27T23:17:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:34 smithi093 ceph-mon[111335]: pgmap v4321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:34 smithi093 ceph-mon[118902]: pgmap v4321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:17:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:17:35] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:17:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:35.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:35.771939+0000) 2023-08-27T23:17:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4096455196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4096455196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4096455196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:36.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:36.772144+0000) 2023-08-27T23:17:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:37 smithi136 ceph-mon[113531]: pgmap v4322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:37 smithi093 ceph-mon[111335]: pgmap v4322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:37 smithi093 ceph-mon[118902]: pgmap v4322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:37.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:37.772393+0000) 2023-08-27T23:17:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:38 smithi136 ceph-mon[113531]: pgmap v4323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:38.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:38.772650+0000) 2023-08-27T23:17:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:38 smithi093 ceph-mon[111335]: pgmap v4323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:38 smithi093 ceph-mon[118902]: pgmap v4323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2582372453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:39.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:39.772895+0000) 2023-08-27T23:17:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2582372453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2582372453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:40.963 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:40.708Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:40.963 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:40.709Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:40 smithi136 ceph-mon[113531]: pgmap v4324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:40.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:40.773168+0000) 2023-08-27T23:17:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:40 smithi093 ceph-mon[111335]: pgmap v4324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:40 smithi093 ceph-mon[118902]: pgmap v4324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2774142788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:41.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:41.773403+0000) 2023-08-27T23:17:42.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2774142788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2774142788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:42.202Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:42.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:42.203Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:42 smithi136 ceph-mon[113531]: pgmap v4325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:42.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:42.773658+0000) 2023-08-27T23:17:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:42 smithi093 ceph-mon[111335]: pgmap v4325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:42 smithi093 ceph-mon[118902]: pgmap v4325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2992954730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:43.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:43.773894+0000) 2023-08-27T23:17:44.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2992954730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2992954730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:44 smithi136 ceph-mon[113531]: pgmap v4326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:44.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:44.774170+0000) 2023-08-27T23:17:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:44 smithi093 ceph-mon[111335]: pgmap v4326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:44 smithi093 ceph-mon[118902]: pgmap v4326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:17:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:17:45] "GET /metrics HTTP/1.1" 200 239913 "" "Prometheus/2.43.0" 2023-08-27T23:17:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1725391486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:45.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:45.774443+0000) 2023-08-27T23:17:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1725391486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1725391486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:46 smithi136 ceph-mon[113531]: pgmap v4327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:46.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:46.774665+0000) 2023-08-27T23:17:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:46 smithi093 ceph-mon[111335]: pgmap v4327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:46 smithi093 ceph-mon[118902]: pgmap v4327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:47.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:47.774911+0000) 2023-08-27T23:17:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3130825485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3130825485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3130825485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:48.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:48.775129+0000) 2023-08-27T23:17:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:49 smithi136 ceph-mon[113531]: pgmap v4328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:49 smithi093 ceph-mon[111335]: pgmap v4328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:49 smithi093 ceph-mon[118902]: pgmap v4328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:49.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:49.775370+0000) 2023-08-27T23:17:50.967 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:50.709Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:50.968 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:50.710Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:50 smithi136 ceph-mon[113531]: pgmap v4329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2131885850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:50.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:50.775631+0000) 2023-08-27T23:17:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:50 smithi093 ceph-mon[111335]: pgmap v4329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2131885850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:50 smithi093 ceph-mon[118902]: pgmap v4329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2131885850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:51.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:51.775881+0000) 2023-08-27T23:17:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:52.203Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:17:52.204Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:17:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:52 smithi136 ceph-mon[113531]: pgmap v4330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:52.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:52.776108+0000) 2023-08-27T23:17:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:52 smithi093 ceph-mon[111335]: pgmap v4330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:52 smithi093 ceph-mon[118902]: pgmap v4330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/777279738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:53.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:53.776359+0000) 2023-08-27T23:17:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/777279738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:53.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/777279738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:54 smithi136 ceph-mon[113531]: pgmap v4331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:54.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:54.776581+0000) 2023-08-27T23:17:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:54 smithi093 ceph-mon[111335]: pgmap v4331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:17:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:54 smithi093 ceph-mon[118902]: pgmap v4331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:17:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:17:55] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:17:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1602308473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:55.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:55.776845+0000) 2023-08-27T23:17:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1602308473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1602308473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:56 smithi136 ceph-mon[113531]: pgmap v4332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:56.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:56.777127+0000) 2023-08-27T23:17:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:56 smithi093 ceph-mon[111335]: pgmap v4332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:56 smithi093 ceph-mon[118902]: pgmap v4332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1043955173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:57.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:57.777362+0000) 2023-08-27T23:17:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1043955173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1043955173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:17:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:17:58 smithi136 ceph-mon[113531]: pgmap v4333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:58.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:58.777652+0000) 2023-08-27T23:17:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:17:58 smithi093 ceph-mon[111335]: pgmap v4333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:17:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:17:58 smithi093 ceph-mon[118902]: pgmap v4333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:17:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:17:59.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:17:59.777908+0000) 2023-08-27T23:18:00.969 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:00.710Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:00.969 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:00.711Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1026439328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:00 smithi136 ceph-mon[113531]: pgmap v4334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:00.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:00.778130+0000) 2023-08-27T23:18:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1026439328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:00 smithi093 ceph-mon[111335]: pgmap v4334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1026439328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:00 smithi093 ceph-mon[118902]: pgmap v4334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:01.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:01.778342+0000) 2023-08-27T23:18:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:02.203Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:02.204Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:02.778 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1691619009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:02.779 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:02 smithi136 ceph-mon[113531]: pgmap v4335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1691619009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:02 smithi093 ceph-mon[111335]: pgmap v4335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1691619009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:02 smithi093 ceph-mon[118902]: pgmap v4335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:02.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:02.778559+0000) 2023-08-27T23:18:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:03.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:03.778834+0000) 2023-08-27T23:18:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:04 smithi136 ceph-mon[113531]: pgmap v4336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:04.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:04.779114+0000) 2023-08-27T23:18:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:04 smithi093 ceph-mon[111335]: pgmap v4336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:04 smithi093 ceph-mon[118902]: pgmap v4336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3344584029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3344584029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:18:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:18:05] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:18:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3344584029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:05.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:05.779328+0000) 2023-08-27T23:18:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:06 smithi136 ceph-mon[113531]: pgmap v4337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:06.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:06.779512+0000) 2023-08-27T23:18:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:06 smithi093 ceph-mon[111335]: pgmap v4337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:06.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:06 smithi093 ceph-mon[118902]: pgmap v4337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1110478682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:07.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:07.779710+0000) 2023-08-27T23:18:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1110478682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1110478682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:08 smithi136 ceph-mon[113531]: pgmap v4338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:08.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:08.779993+0000) 2023-08-27T23:18:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:08 smithi093 ceph-mon[111335]: pgmap v4338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:08 smithi093 ceph-mon[118902]: pgmap v4338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:09.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:09.780236+0000) 2023-08-27T23:18:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:10.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:10.780489+0000) 2023-08-27T23:18:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/960724501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:10 smithi136 ceph-mon[113531]: pgmap v4339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/960724501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:10.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:10 smithi093 ceph-mon[111335]: pgmap v4339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/960724501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:10 smithi093 ceph-mon[118902]: pgmap v4339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:10.711Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:10.712Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:12.056 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:11.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:11.780737+0000) 2023-08-27T23:18:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1897936691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1897936691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1897936691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:12.204Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:12.205Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:13.063 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:12.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:12.780968+0000) 2023-08-27T23:18:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:13 smithi136 ceph-mon[113531]: pgmap v4340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:13 smithi093 ceph-mon[118902]: pgmap v4340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:13 smithi093 ceph-mon[111335]: pgmap v4340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:13.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:13.781199+0000) 2023-08-27T23:18:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2609703515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2609703515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2609703515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:14.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:14.781410+0000) 2023-08-27T23:18:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:15 smithi093 ceph-mon[111335]: pgmap v4341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:18:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:18:15] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:18:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:15 smithi093 ceph-mon[118902]: pgmap v4341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:15.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:15 smithi136 ceph-mon[113531]: pgmap v4341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:15.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:15.781654+0000) 2023-08-27T23:18:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:16 smithi136 ceph-mon[113531]: pgmap v4342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:16.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:16.781847+0000) 2023-08-27T23:18:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:16 smithi093 ceph-mon[111335]: pgmap v4342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:16 smithi093 ceph-mon[118902]: pgmap v4342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2483224974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:17.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:17.782092+0000) 2023-08-27T23:18:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2483224974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2483224974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:18 smithi136 ceph-mon[113531]: pgmap v4343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:18.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:18.782343+0000) 2023-08-27T23:18:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:18 smithi093 ceph-mon[111335]: pgmap v4343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:18 smithi093 ceph-mon[118902]: pgmap v4343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2324673909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:18:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:19.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:19.782534+0000) 2023-08-27T23:18:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2324673909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:18:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2324673909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:18:20.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:20.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:20.782697+0000) 2023-08-27T23:18:20.971 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:20.712Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:20.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:20.713Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:20 smithi093 ceph-mon[111335]: pgmap v4344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:20 smithi093 ceph-mon[118902]: pgmap v4344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:21.309 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:20 smithi136 ceph-mon[113531]: pgmap v4344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:21.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:21.782846+0000) 2023-08-27T23:18:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1516517712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:18:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:18:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:22.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:22.205Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1516517712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1516517712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:18:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:18:22.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:18:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:22 smithi136 ceph-mon[113531]: pgmap v4345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:22.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:22.783111+0000) 2023-08-27T23:18:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:22 smithi093 ceph-mon[111335]: pgmap v4345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:22 smithi093 ceph-mon[118902]: pgmap v4345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3186201662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:23.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:23.783323+0000) 2023-08-27T23:18:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3186201662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3186201662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:24 smithi136 ceph-mon[113531]: pgmap v4346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:24.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:24.783573+0000) 2023-08-27T23:18:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:24 smithi093 ceph-mon[111335]: pgmap v4346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:24 smithi093 ceph-mon[118902]: pgmap v4346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:18:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:18:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:18:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:25.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:25.783839+0000) 2023-08-27T23:18:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1680841458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1680841458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1680841458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:26.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:26.784128+0000) 2023-08-27T23:18:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:27 smithi136 ceph-mon[113531]: pgmap v4347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:27 smithi093 ceph-mon[111335]: pgmap v4347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:27 smithi093 ceph-mon[118902]: pgmap v4347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:27.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:27.784386+0000) 2023-08-27T23:18:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:28 smithi136 ceph-mon[113531]: pgmap v4348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:28.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:28.784656+0000) 2023-08-27T23:18:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:28 smithi093 ceph-mon[111335]: pgmap v4348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:28 smithi093 ceph-mon[118902]: pgmap v4348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2206351356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:29.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:29.784911+0000) 2023-08-27T23:18:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2206351356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2206351356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:30.977 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:30.713Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:30.977 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:30.714Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:30 smithi136 ceph-mon[113531]: pgmap v4349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:30.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:30.785161+0000) 2023-08-27T23:18:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:30 smithi093 ceph-mon[111335]: pgmap v4349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:30 smithi093 ceph-mon[118902]: pgmap v4349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/654707954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:31.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:31.785370+0000) 2023-08-27T23:18:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/654707954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/654707954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:32.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:32.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:32.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:32 smithi136 ceph-mon[113531]: pgmap v4350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:32.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:32.785618+0000) 2023-08-27T23:18:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:32 smithi093 ceph-mon[111335]: pgmap v4350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:32 smithi093 ceph-mon[118902]: pgmap v4350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2760046756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:33.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:33.785906+0000) 2023-08-27T23:18:34.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2760046756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2760046756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:34 smithi136 ceph-mon[113531]: pgmap v4351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:34.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:34.786163+0000) 2023-08-27T23:18:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:34 smithi093 ceph-mon[118902]: pgmap v4351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:34 smithi093 ceph-mon[111335]: pgmap v4351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:18:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:18:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:18:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1529345284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:35.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:35.786372+0000) 2023-08-27T23:18:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1529345284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1529345284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:37 smithi136 ceph-mon[113531]: pgmap v4352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:36.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:36.786625+0000) 2023-08-27T23:18:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:37 smithi093 ceph-mon[118902]: pgmap v4352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:37 smithi093 ceph-mon[111335]: pgmap v4352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:37.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:37.786888+0000) 2023-08-27T23:18:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1813604225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1813604225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1813604225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:38.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:38.787171+0000) 2023-08-27T23:18:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:39 smithi136 ceph-mon[113531]: pgmap v4353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:39 smithi093 ceph-mon[111335]: pgmap v4353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:39 smithi093 ceph-mon[118902]: pgmap v4353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:39.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:39.787426+0000) 2023-08-27T23:18:40.979 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:40.714Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:40.979 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:40.715Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:40 smithi136 ceph-mon[113531]: pgmap v4354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2577340540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:40.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:40.787671+0000) 2023-08-27T23:18:41.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:40 smithi093 ceph-mon[111335]: pgmap v4354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2577340540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:40 smithi093 ceph-mon[118902]: pgmap v4354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2577340540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:41.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:41.787922+0000) 2023-08-27T23:18:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:42.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:42.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:42 smithi136 ceph-mon[113531]: pgmap v4355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:42.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:42.788193+0000) 2023-08-27T23:18:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:42 smithi093 ceph-mon[111335]: pgmap v4355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:42 smithi093 ceph-mon[118902]: pgmap v4355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/486126416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:43.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:43.788443+0000) 2023-08-27T23:18:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/486126416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/486126416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:44 smithi136 ceph-mon[113531]: pgmap v4356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:44.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:44.788687+0000) 2023-08-27T23:18:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:44 smithi093 ceph-mon[111335]: pgmap v4356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:44 smithi093 ceph-mon[118902]: pgmap v4356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:18:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:18:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:18:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1690854312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:45.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:45.788943+0000) 2023-08-27T23:18:46.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1690854312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1690854312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:46 smithi136 ceph-mon[113531]: pgmap v4357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:46.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:46.789164+0000) 2023-08-27T23:18:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:46 smithi093 ceph-mon[111335]: pgmap v4357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:46 smithi093 ceph-mon[118902]: pgmap v4357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3174560911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:47.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:47.789412+0000) 2023-08-27T23:18:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3174560911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3174560911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:48 smithi136 ceph-mon[113531]: pgmap v4358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:48.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:48.789707+0000) 2023-08-27T23:18:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:48 smithi093 ceph-mon[111335]: pgmap v4358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:48 smithi093 ceph-mon[118902]: pgmap v4358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:49.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:49.789902+0000) 2023-08-27T23:18:50.980 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:50.715Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:50.980 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:50.716Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3590057073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:50 smithi136 ceph-mon[113531]: pgmap v4359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:50.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:50.790137+0000) 2023-08-27T23:18:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3590057073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:50 smithi093 ceph-mon[111335]: pgmap v4359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3590057073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:50 smithi093 ceph-mon[118902]: pgmap v4359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:51.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:51.790388+0000) 2023-08-27T23:18:52.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:52.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:18:52.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:18:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:52 smithi136 ceph-mon[113531]: pgmap v4360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2742696963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:52.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:52.790635+0000) 2023-08-27T23:18:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:52 smithi093 ceph-mon[111335]: pgmap v4360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2742696963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:52 smithi093 ceph-mon[118902]: pgmap v4360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2742696963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:53.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:53.790880+0000) 2023-08-27T23:18:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:18:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:54 smithi136 ceph-mon[113531]: pgmap v4361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:54.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:54.791151+0000) 2023-08-27T23:18:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:54 smithi093 ceph-mon[111335]: pgmap v4361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:54 smithi093 ceph-mon[118902]: pgmap v4361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:18:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:18:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:18:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2642870129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:55.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:55.791422+0000) 2023-08-27T23:18:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2642870129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2642870129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:56 smithi136 ceph-mon[113531]: pgmap v4362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:56.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:56.791665+0000) 2023-08-27T23:18:57.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:56 smithi093 ceph-mon[111335]: pgmap v4362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:56 smithi093 ceph-mon[118902]: pgmap v4362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3385349198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:57.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:57.791893+0000) 2023-08-27T23:18:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3385349198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3385349198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:18:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:58 smithi136 ceph-mon[113531]: pgmap v4363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:58.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:58.792163+0000) 2023-08-27T23:18:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:58 smithi093 ceph-mon[111335]: pgmap v4363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:18:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:58 smithi093 ceph-mon[118902]: pgmap v4363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:18:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3255429049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:18:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:18:59.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:18:59.792407+0000) 2023-08-27T23:19:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:18:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3255429049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:18:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3255429049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:00.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:00.715Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:00.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:00.716Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:00 smithi136 ceph-mon[113531]: pgmap v4364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:00.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:00.792661+0000) 2023-08-27T23:19:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:00 smithi093 ceph-mon[111335]: pgmap v4364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:00 smithi093 ceph-mon[118902]: pgmap v4364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:02.091 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:01.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:01.792928+0000) 2023-08-27T23:19:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1277910891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1277910891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1277910891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:02.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:02.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:03.096 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:02.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:02.793203+0000) 2023-08-27T23:19:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:03 smithi136 ceph-mon[113531]: pgmap v4365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:03 smithi093 ceph-mon[118902]: pgmap v4365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:03 smithi093 ceph-mon[111335]: pgmap v4365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:03.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:03.793461+0000) 2023-08-27T23:19:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1716088195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1716088195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:04.793 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1716088195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:04.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:04.793718+0000) 2023-08-27T23:19:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:05 smithi093 ceph-mon[111335]: pgmap v4366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:05 smithi093 ceph-mon[118902]: pgmap v4366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:19:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:19:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:19:05.794 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:05 smithi136 ceph-mon[113531]: pgmap v4366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:05.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:05.793992+0000) 2023-08-27T23:19:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:06 smithi136 ceph-mon[113531]: pgmap v4367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:06.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:06.794228+0000) 2023-08-27T23:19:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:06 smithi093 ceph-mon[111335]: pgmap v4367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:06.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:06 smithi093 ceph-mon[118902]: pgmap v4367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/552109095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:07.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:07.794447+0000) 2023-08-27T23:19:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/552109095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/552109095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:08 smithi136 ceph-mon[113531]: pgmap v4368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:08.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:08.794697+0000) 2023-08-27T23:19:08.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:08 smithi093 ceph-mon[111335]: pgmap v4368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:08 smithi093 ceph-mon[118902]: pgmap v4368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4020853475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:09.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:09.794989+0000) 2023-08-27T23:19:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4020853475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4020853475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:10.984 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:10.715Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:10.985 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:10.716Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:10 smithi136 ceph-mon[113531]: pgmap v4369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:10.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:10.795235+0000) 2023-08-27T23:19:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:10 smithi093 ceph-mon[111335]: pgmap v4369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:10 smithi093 ceph-mon[118902]: pgmap v4369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3795621205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:11.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:11.795454+0000) 2023-08-27T23:19:12.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3795621205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3795621205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:12.207Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:12.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:12.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:12 smithi136 ceph-mon[113531]: pgmap v4370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:12.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:12.795740+0000) 2023-08-27T23:19:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:12 smithi093 ceph-mon[111335]: pgmap v4370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:12 smithi093 ceph-mon[118902]: pgmap v4370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3100044843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:13.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:13.796013+0000) 2023-08-27T23:19:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3100044843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3100044843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:14 smithi136 ceph-mon[113531]: pgmap v4371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:15.271 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:14.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:14.796290+0000) 2023-08-27T23:19:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:14 smithi093 ceph-mon[111335]: pgmap v4371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:14 smithi093 ceph-mon[118902]: pgmap v4371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:19:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:19:15] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:19:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:15.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:15.796621+0000) 2023-08-27T23:19:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2814234249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2814234249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2814234249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:16.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:16.796859+0000) 2023-08-27T23:19:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:17 smithi136 ceph-mon[113531]: pgmap v4372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:17 smithi093 ceph-mon[111335]: pgmap v4372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:17 smithi093 ceph-mon[118902]: pgmap v4372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:17.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:17.797152+0000) 2023-08-27T23:19:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:18 smithi136 ceph-mon[113531]: pgmap v4373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:19.010 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:18.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:18.797340+0000) 2023-08-27T23:19:19.011 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:18 smithi093 ceph-mon[111335]: pgmap v4373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:19.011 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:18 smithi093 ceph-mon[118902]: pgmap v4373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2586657116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:19.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:19.797581+0000) 2023-08-27T23:19:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2586657116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2586657116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:20.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:20.717Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:20.982 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:20.718Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:20 smithi136 ceph-mon[113531]: pgmap v4374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:20.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:20.797826+0000) 2023-08-27T23:19:21.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:20 smithi093 ceph-mon[111335]: pgmap v4374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:20 smithi093 ceph-mon[118902]: pgmap v4374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/715418460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:21.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:21.797989+0000) 2023-08-27T23:19:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/715418460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/715418460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:22.207Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:22.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:22.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:22.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:22.798119+0000) 2023-08-27T23:19:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:19:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:22 smithi093 ceph-mon[111335]: pgmap v4375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:19:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:22 smithi093 ceph-mon[118902]: pgmap v4375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:19:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:22 smithi136 ceph-mon[113531]: pgmap v4375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:24.110 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:23.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:23.798285+0000) 2023-08-27T23:19:24.110 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2351516209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:24.110 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:24.111 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:24.111 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2351516209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2351516209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:24.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:24.798462+0000) 2023-08-27T23:19:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:25 smithi136 ceph-mon[113531]: pgmap v4376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:19:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:19:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:19:25] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:19:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[111335]: pgmap v4376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:19:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:19:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[118902]: pgmap v4376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:19:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:19:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:19:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:25.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:25.798735+0000) 2023-08-27T23:19:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2907466396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2907466396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2907466396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:26.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:26.798975+0000) 2023-08-27T23:19:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:27 smithi136 ceph-mon[113531]: pgmap v4377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:27 smithi093 ceph-mon[111335]: pgmap v4377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:27 smithi093 ceph-mon[118902]: pgmap v4377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:27.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:27.799173+0000) 2023-08-27T23:19:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1652063591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1652063591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1652063591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:28.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:28.799375+0000) 2023-08-27T23:19:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:29 smithi136 ceph-mon[113531]: pgmap v4378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:29 smithi093 ceph-mon[111335]: pgmap v4378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:29 smithi093 ceph-mon[118902]: pgmap v4378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:29.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:29.799624+0000) 2023-08-27T23:19:30.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:30.717Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:30.984 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:30.718Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:30 smithi136 ceph-mon[113531]: pgmap v4379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2479295004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:30.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:30.799851+0000) 2023-08-27T23:19:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:30 smithi093 ceph-mon[111335]: pgmap v4379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2479295004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:30 smithi093 ceph-mon[118902]: pgmap v4379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2479295004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:31.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:31.800115+0000) 2023-08-27T23:19:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:32.208Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:32.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:32 smithi136 ceph-mon[113531]: pgmap v4380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:32.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:32.800375+0000) 2023-08-27T23:19:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:32 smithi093 ceph-mon[111335]: pgmap v4380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:32 smithi093 ceph-mon[118902]: pgmap v4380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/127606864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:33.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:33.800657+0000) 2023-08-27T23:19:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/127606864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/127606864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:34 smithi136 ceph-mon[113531]: pgmap v4381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:34.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:34.800894+0000) 2023-08-27T23:19:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:34 smithi093 ceph-mon[111335]: pgmap v4381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:34 smithi093 ceph-mon[118902]: pgmap v4381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:19:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:19:35] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:19:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1683152876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:35.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:35.801247+0000) 2023-08-27T23:19:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1683152876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1683152876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:36 smithi136 ceph-mon[113531]: pgmap v4382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:36.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:36.801518+0000) 2023-08-27T23:19:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:36 smithi093 ceph-mon[111335]: pgmap v4382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:36 smithi093 ceph-mon[118902]: pgmap v4382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4105215283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:37.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:37.801713+0000) 2023-08-27T23:19:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4105215283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4105215283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:39 smithi136 ceph-mon[113531]: pgmap v4383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:38.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:38.801961+0000) 2023-08-27T23:19:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:38 smithi093 ceph-mon[111335]: pgmap v4383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:38 smithi093 ceph-mon[118902]: pgmap v4383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:39.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:39.802241+0000) 2023-08-27T23:19:40.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:40.718Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:40.719Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2514719159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:40 smithi136 ceph-mon[113531]: pgmap v4384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:40.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:40.802487+0000) 2023-08-27T23:19:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2514719159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:40 smithi093 ceph-mon[111335]: pgmap v4384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2514719159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:40 smithi093 ceph-mon[118902]: pgmap v4384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:41.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:41.802705+0000) 2023-08-27T23:19:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:42.208Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:42.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:42 smithi136 ceph-mon[113531]: pgmap v4385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2239847199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:42.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:42.802924+0000) 2023-08-27T23:19:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:42 smithi093 ceph-mon[111335]: pgmap v4385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2239847199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:42 smithi093 ceph-mon[118902]: pgmap v4385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2239847199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:43.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:43.803172+0000) 2023-08-27T23:19:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:44 smithi136 ceph-mon[113531]: pgmap v4386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:44.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:44.803419+0000) 2023-08-27T23:19:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:44 smithi093 ceph-mon[111335]: pgmap v4386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:44 smithi093 ceph-mon[118902]: pgmap v4386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3067218921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:19:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:19:45] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:19:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3067218921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:45.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3067218921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:45.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:45.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:45.803687+0000) 2023-08-27T23:19:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:46 smithi136 ceph-mon[113531]: pgmap v4387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:46.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:46.803889+0000) 2023-08-27T23:19:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:46 smithi093 ceph-mon[111335]: pgmap v4387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:46 smithi093 ceph-mon[118902]: pgmap v4387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2601086161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:47.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:47.804160+0000) 2023-08-27T23:19:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2601086161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2601086161' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:48 smithi136 ceph-mon[113531]: pgmap v4388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:48.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:48.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:48.804387+0000) 2023-08-27T23:19:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:48 smithi093 ceph-mon[111335]: pgmap v4388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:48 smithi093 ceph-mon[118902]: pgmap v4388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1541991379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:49.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:49.804593+0000) 2023-08-27T23:19:50.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1541991379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1541991379' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:50.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:50.718Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:50.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:50.719Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:50 smithi136 ceph-mon[113531]: pgmap v4389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:50.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:50.804844+0000) 2023-08-27T23:19:51.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:50 smithi093 ceph-mon[111335]: pgmap v4389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:50 smithi093 ceph-mon[118902]: pgmap v4389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:52.144 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:51.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:51.805120+0000) 2023-08-27T23:19:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4200412042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4200412042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4200412042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:52.209Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:19:52.210Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:19:53.148 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:52.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:52.805305+0000) 2023-08-27T23:19:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:53 smithi136 ceph-mon[113531]: pgmap v4390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:53 smithi093 ceph-mon[111335]: pgmap v4390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:53 smithi093 ceph-mon[118902]: pgmap v4390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:54.150 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:53.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:53.805485+0000) 2023-08-27T23:19:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:19:55.155 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:54.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:54.805723+0000) 2023-08-27T23:19:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1559338128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:55 smithi136 ceph-mon[113531]: pgmap v4391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1559338128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:55 smithi093 ceph-mon[111335]: pgmap v4391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1559338128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:55 smithi093 ceph-mon[118902]: pgmap v4391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:55.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:19:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:19:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:19:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:55.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:55.805999+0000) 2023-08-27T23:19:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:56 smithi136 ceph-mon[113531]: pgmap v4392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:56.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:56.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:56.806269+0000) 2023-08-27T23:19:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:56 smithi093 ceph-mon[111335]: pgmap v4392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:56 smithi093 ceph-mon[118902]: pgmap v4392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/143693464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:57.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:57.806532+0000) 2023-08-27T23:19:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/143693464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/143693464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:58 smithi136 ceph-mon[113531]: pgmap v4393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:58.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:58.806730+0000) 2023-08-27T23:19:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:58 smithi093 ceph-mon[111335]: pgmap v4393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:58 smithi093 ceph-mon[118902]: pgmap v4393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:19:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:19:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2347917732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:19:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:19:59.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:19:59.806930+0000) 2023-08-27T23:19:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:19:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2347917732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:19:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:19:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2347917732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:00.719Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:00.720Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:20:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:00 smithi136 ceph-mon[113531]: pgmap v4394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:00.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:00.807175+0000) 2023-08-27T23:20:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:20:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:00 smithi093 ceph-mon[111335]: pgmap v4394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:20:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:00 smithi093 ceph-mon[118902]: pgmap v4394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2053667519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:01.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:01.807407+0000) 2023-08-27T23:20:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2053667519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2053667519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:02.210Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:02.211Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:02 smithi136 ceph-mon[113531]: pgmap v4395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:02.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:02.807656+0000) 2023-08-27T23:20:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:02 smithi093 ceph-mon[111335]: pgmap v4395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:02 smithi093 ceph-mon[118902]: pgmap v4395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1794979749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:03.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:03.807980+0000) 2023-08-27T23:20:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1794979749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1794979749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:05 smithi136 ceph-mon[113531]: pgmap v4396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:04.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:04.808184+0000) 2023-08-27T23:20:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:05 smithi093 ceph-mon[111335]: pgmap v4396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:05 smithi093 ceph-mon[118902]: pgmap v4396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:20:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:20:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:20:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:05.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:05.808423+0000) 2023-08-27T23:20:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4048154581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4048154581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4048154581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:06.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:06.808624+0000) 2023-08-27T23:20:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:07 smithi136 ceph-mon[113531]: pgmap v4397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:07 smithi093 ceph-mon[111335]: pgmap v4397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:07 smithi093 ceph-mon[118902]: pgmap v4397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:07.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:07.808893+0000) 2023-08-27T23:20:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:08 smithi136 ceph-mon[113531]: pgmap v4398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:08.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:08.809146+0000) 2023-08-27T23:20:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:08 smithi093 ceph-mon[111335]: pgmap v4398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:08 smithi093 ceph-mon[118902]: pgmap v4398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3107666623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:09.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:09.809405+0000) 2023-08-27T23:20:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3107666623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3107666623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:10.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:10.719Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:10.720Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:10 smithi136 ceph-mon[113531]: pgmap v4399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:10.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:10.809650+0000) 2023-08-27T23:20:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:10 smithi093 ceph-mon[111335]: pgmap v4399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:10 smithi093 ceph-mon[118902]: pgmap v4399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/197159144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:11.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:11.809891+0000) 2023-08-27T23:20:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/197159144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:12.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/197159144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:12.210Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:12.212Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:12 smithi136 ceph-mon[113531]: pgmap v4400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:12.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:12.810143+0000) 2023-08-27T23:20:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:12 smithi093 ceph-mon[111335]: pgmap v4400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:12 smithi093 ceph-mon[118902]: pgmap v4400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2876328789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:13.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:13.810391+0000) 2023-08-27T23:20:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2876328789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2876328789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:15 smithi136 ceph-mon[113531]: pgmap v4401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:14.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:14.810668+0000) 2023-08-27T23:20:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:15 smithi093 ceph-mon[111335]: pgmap v4401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:15 smithi093 ceph-mon[118902]: pgmap v4401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:20:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:20:15] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:20:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/521438335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:15.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:15.810955+0000) 2023-08-27T23:20:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/521438335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/521438335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:17 smithi136 ceph-mon[113531]: pgmap v4402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:16.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:16.811178+0000) 2023-08-27T23:20:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:17 smithi093 ceph-mon[111335]: pgmap v4402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:17 smithi093 ceph-mon[118902]: pgmap v4402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:17.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:17.811454+0000) 2023-08-27T23:20:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1398681301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1398681301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1398681301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:18.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:18.811696+0000) 2023-08-27T23:20:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:19 smithi136 ceph-mon[113531]: pgmap v4403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:19 smithi093 ceph-mon[111335]: pgmap v4403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:19 smithi093 ceph-mon[118902]: pgmap v4403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:19.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:19.811900+0000) 2023-08-27T23:20:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:20.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:20.721Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:20 smithi136 ceph-mon[113531]: pgmap v4404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/537824650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:20.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:20.812171+0000) 2023-08-27T23:20:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:20 smithi093 ceph-mon[111335]: pgmap v4404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/537824650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:20 smithi093 ceph-mon[118902]: pgmap v4404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/537824650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:21.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:21.812416+0000) 2023-08-27T23:20:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:22.210Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:22.211Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:22 smithi136 ceph-mon[113531]: pgmap v4405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:22.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:22.812671+0000) 2023-08-27T23:20:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:22 smithi093 ceph-mon[111335]: pgmap v4405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:22 smithi093 ceph-mon[118902]: pgmap v4405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002934439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:23.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:23.812941+0000) 2023-08-27T23:20:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002934439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002934439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:24 smithi136 ceph-mon[113531]: pgmap v4406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:24.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:24.813147+0000) 2023-08-27T23:20:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:24 smithi093 ceph-mon[111335]: pgmap v4406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:24 smithi093 ceph-mon[118902]: pgmap v4406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:25.520 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:20:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:20:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:20:25.814 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:20:25.814 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3059599866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:20:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3059599866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:20:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3059599866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:25.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:25.813342+0000) 2023-08-27T23:20:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:26 smithi093 ceph-mon[111335]: pgmap v4407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:26 smithi093 ceph-mon[118902]: pgmap v4407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:26.891 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:26 smithi136 ceph-mon[113531]: pgmap v4407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:26.892 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:26.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:26.813531+0000) 2023-08-27T23:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:27.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:27.813744+0000) 2023-08-27T23:20:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:20:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:20:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:20:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1461762800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:28 smithi093 ceph-mon[111335]: pgmap v4408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1461762800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:28 smithi093 ceph-mon[118902]: pgmap v4408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:28.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:28.814008+0000) 2023-08-27T23:20:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1461762800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:28 smithi136 ceph-mon[113531]: pgmap v4408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:29.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:29.814290+0000) 2023-08-27T23:20:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:30.721Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:30.722Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:30.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:30.814520+0000) 2023-08-27T23:20:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/589776422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:30 smithi136 ceph-mon[113531]: pgmap v4409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/589776422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:30 smithi093 ceph-mon[111335]: pgmap v4409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/589776422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:31 smithi093 ceph-mon[118902]: pgmap v4409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:31.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:31.814723+0000) 2023-08-27T23:20:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:32.211Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:32.212Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:32 smithi136 ceph-mon[113531]: pgmap v4410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:32.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:32.814869+0000) 2023-08-27T23:20:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:32 smithi093 ceph-mon[111335]: pgmap v4410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:32 smithi093 ceph-mon[118902]: pgmap v4410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/187605202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:33.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:33.815140+0000) 2023-08-27T23:20:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/187605202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/187605202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:34 smithi136 ceph-mon[113531]: pgmap v4411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:34.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:34.815417+0000) 2023-08-27T23:20:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:34 smithi093 ceph-mon[111335]: pgmap v4411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:34 smithi093 ceph-mon[118902]: pgmap v4411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:20:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:20:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:20:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1564219746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:35.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:35.815691+0000) 2023-08-27T23:20:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1564219746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1564219746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:37 smithi136 ceph-mon[113531]: pgmap v4412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:36.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:36.815935+0000) 2023-08-27T23:20:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:36 smithi093 ceph-mon[111335]: pgmap v4412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:37 smithi093 ceph-mon[118902]: pgmap v4412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2158725419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:37.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:37.816176+0000) 2023-08-27T23:20:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2158725419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2158725419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:39 smithi136 ceph-mon[113531]: pgmap v4413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:38.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:38.816394+0000) 2023-08-27T23:20:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:39 smithi093 ceph-mon[111335]: pgmap v4413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:39 smithi093 ceph-mon[118902]: pgmap v4413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3480287051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:39.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:39.816632+0000) 2023-08-27T23:20:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3480287051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3480287051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:40.722Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:40.723Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:41 smithi136 ceph-mon[113531]: pgmap v4414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:40.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:40.816834+0000) 2023-08-27T23:20:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:41 smithi093 ceph-mon[111335]: pgmap v4414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:41 smithi093 ceph-mon[118902]: pgmap v4414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:41.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:41.817093+0000) 2023-08-27T23:20:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2849208714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2849208714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:42.212Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:42.213Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2849208714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:42.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:42.817335+0000) 2023-08-27T23:20:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:43 smithi093 ceph-mon[111335]: pgmap v4415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:43 smithi093 ceph-mon[118902]: pgmap v4415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:43 smithi136 ceph-mon[113531]: pgmap v4415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:43.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:43.817555+0000) 2023-08-27T23:20:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1275190274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:44 smithi136 ceph-mon[113531]: pgmap v4416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:44.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:44.817799+0000) 2023-08-27T23:20:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1275190274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:44 smithi093 ceph-mon[111335]: pgmap v4416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1275190274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:44 smithi093 ceph-mon[118902]: pgmap v4416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:20:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:20:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:20:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:45.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:45.818117+0000) 2023-08-27T23:20:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:46 smithi136 ceph-mon[113531]: pgmap v4417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:46.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:46.818330+0000) 2023-08-27T23:20:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:46 smithi093 ceph-mon[111335]: pgmap v4417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:46 smithi093 ceph-mon[118902]: pgmap v4417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2871048258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:47.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:47.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:47.818567+0000) 2023-08-27T23:20:47.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2871048258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2871048258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:48 smithi136 ceph-mon[113531]: pgmap v4418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:48.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:48.818825+0000) 2023-08-27T23:20:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:48 smithi093 ceph-mon[111335]: pgmap v4418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:48 smithi093 ceph-mon[118902]: pgmap v4418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:49.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2112460208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:49.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:49.819093+0000) 2023-08-27T23:20:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2112460208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2112460208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:50.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:50.723Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:50.724Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:51 smithi136 ceph-mon[113531]: pgmap v4419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:50.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:50.819363+0000) 2023-08-27T23:20:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:51 smithi093 ceph-mon[111335]: pgmap v4419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:51 smithi093 ceph-mon[118902]: pgmap v4419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2221753525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:51.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:51.819596+0000) 2023-08-27T23:20:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2221753525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2221753525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:52.212Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:20:52.213Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:20:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:53 smithi136 ceph-mon[113531]: pgmap v4420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:52.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:52.819809+0000) 2023-08-27T23:20:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:53 smithi093 ceph-mon[111335]: pgmap v4420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:53 smithi093 ceph-mon[118902]: pgmap v4420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:53.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:53.819993+0000) 2023-08-27T23:20:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:20:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/767309805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:55 smithi136 ceph-mon[113531]: pgmap v4421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:54.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:54.820253+0000) 2023-08-27T23:20:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/767309805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:55.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:55 smithi093 ceph-mon[111335]: pgmap v4421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/767309805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:55 smithi093 ceph-mon[118902]: pgmap v4421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:20:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:20:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:20:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:55.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:55.820479+0000) 2023-08-27T23:20:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3814069159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3814069159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:56.820 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3814069159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:56.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:56.820696+0000) 2023-08-27T23:20:57.821 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:57 smithi136 ceph-mon[113531]: pgmap v4422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:57 smithi093 ceph-mon[111335]: pgmap v4422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:57 smithi093 ceph-mon[118902]: pgmap v4422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:57.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:57.820932+0000) 2023-08-27T23:20:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:58 smithi136 ceph-mon[113531]: pgmap v4423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:58.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:58.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:58.821163+0000) 2023-08-27T23:20:58.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:58 smithi093 ceph-mon[111335]: pgmap v4423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:58 smithi093 ceph-mon[118902]: pgmap v4423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:20:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:20:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/934240877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:59.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:20:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:20:59.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:20:59.821302+0000) 2023-08-27T23:20:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:20:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/934240877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:20:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:20:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/934240877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:00.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:00.724Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:00.725Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:01 smithi136 ceph-mon[113531]: pgmap v4424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:00.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:00.821549+0000) 2023-08-27T23:21:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:01 smithi093 ceph-mon[111335]: pgmap v4424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:01 smithi093 ceph-mon[118902]: pgmap v4424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1655327319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:01.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:01.821782+0000) 2023-08-27T23:21:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1655327319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1655327319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:02.213Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:02.214Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:03 smithi136 ceph-mon[113531]: pgmap v4425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:02.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:02.822074+0000) 2023-08-27T23:21:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:03 smithi093 ceph-mon[111335]: pgmap v4425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:03 smithi093 ceph-mon[118902]: pgmap v4425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1569845922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:03.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:03.822371+0000) 2023-08-27T23:21:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1569845922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1569845922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:05 smithi136 ceph-mon[113531]: pgmap v4426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:04.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:04.822565+0000) 2023-08-27T23:21:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:05 smithi093 ceph-mon[111335]: pgmap v4426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:05 smithi093 ceph-mon[118902]: pgmap v4426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 23:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T23:21:05.31779616Z level=info msg="Completed cleanup jobs" duration=12.851248ms 2023-08-27T23:21:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:21:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:21:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:21:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2308720314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:05.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:05.822799+0000) 2023-08-27T23:21:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2308720314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2308720314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:07 smithi136 ceph-mon[113531]: pgmap v4427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:06.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:06.823067+0000) 2023-08-27T23:21:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:07 smithi093 ceph-mon[111335]: pgmap v4427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:07 smithi093 ceph-mon[118902]: pgmap v4427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:07.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:07.823324+0000) 2023-08-27T23:21:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/224885013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/224885013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/224885013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:08.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:08.823588+0000) 2023-08-27T23:21:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:09 smithi136 ceph-mon[113531]: pgmap v4428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:09 smithi093 ceph-mon[111335]: pgmap v4428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:09 smithi093 ceph-mon[118902]: pgmap v4428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:09.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:09.823804+0000) 2023-08-27T23:21:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:10.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:10.725Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:11 smithi136 ceph-mon[113531]: pgmap v4429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3488943710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:10.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:10.824090+0000) 2023-08-27T23:21:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:11 smithi093 ceph-mon[111335]: pgmap v4429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3488943710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:11 smithi093 ceph-mon[118902]: pgmap v4429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:11.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3488943710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:11.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:11.824269+0000) 2023-08-27T23:21:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:12.214Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:12.215Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:12 smithi136 ceph-mon[113531]: pgmap v4430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:12.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:12.824465+0000) 2023-08-27T23:21:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:12 smithi093 ceph-mon[111335]: pgmap v4430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:12 smithi093 ceph-mon[118902]: pgmap v4430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3610701164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:13.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:13.824622+0000) 2023-08-27T23:21:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3610701164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3610701164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:14 smithi136 ceph-mon[113531]: pgmap v4431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:14.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:14.824832+0000) 2023-08-27T23:21:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:14 smithi093 ceph-mon[111335]: pgmap v4431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:14 smithi093 ceph-mon[118902]: pgmap v4431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:21:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:21:15] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:21:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3653656020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:15.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:15.825104+0000) 2023-08-27T23:21:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3653656020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3653656020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:17 smithi136 ceph-mon[113531]: pgmap v4432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:16.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:16.825332+0000) 2023-08-27T23:21:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:17 smithi093 ceph-mon[111335]: pgmap v4432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:17 smithi093 ceph-mon[118902]: pgmap v4432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/793519529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:17.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:17.825517+0000) 2023-08-27T23:21:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/793519529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/793519529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:19 smithi136 ceph-mon[113531]: pgmap v4433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:18.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:18.825736+0000) 2023-08-27T23:21:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:19 smithi093 ceph-mon[111335]: pgmap v4433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:19 smithi093 ceph-mon[118902]: pgmap v4433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:19.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:19.825979+0000) 2023-08-27T23:21:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:20.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:20.727Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3226481851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:21 smithi136 ceph-mon[113531]: pgmap v4434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:20.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:20.826173+0000) 2023-08-27T23:21:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3226481851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:21 smithi093 ceph-mon[111335]: pgmap v4434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3226481851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:21 smithi093 ceph-mon[118902]: pgmap v4434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:21.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:21.826401+0000) 2023-08-27T23:21:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:22.214Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:22.215Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:22 smithi136 ceph-mon[113531]: pgmap v4435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:22.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:22.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:22.826654+0000) 2023-08-27T23:21:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:22 smithi093 ceph-mon[111335]: pgmap v4435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:22 smithi093 ceph-mon[118902]: pgmap v4435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3429426837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:23.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:23.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:23.826896+0000) 2023-08-27T23:21:23.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3429426837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3429426837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:24 smithi136 ceph-mon[113531]: pgmap v4436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:24.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:24.827149+0000) 2023-08-27T23:21:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:24 smithi093 ceph-mon[111335]: pgmap v4436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:24 smithi093 ceph-mon[118902]: pgmap v4436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:21:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:21:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:21:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1178524417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:25.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:25.827401+0000) 2023-08-27T23:21:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1178524417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1178524417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:27 smithi136 ceph-mon[113531]: pgmap v4437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:26.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:26.827647+0000) 2023-08-27T23:21:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:27 smithi093 ceph-mon[111335]: pgmap v4437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:27 smithi093 ceph-mon[118902]: pgmap v4437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/360915635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:21:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:27.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:27.827852+0000) 2023-08-27T23:21:28.266 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/360915635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:28.266 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:21:28.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/360915635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:28.266 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:21:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:28.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:28.828107+0000) 2023-08-27T23:21:29.136 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:29 smithi093 ceph-mon[111335]: pgmap v4438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:29.136 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:29 smithi093 ceph-mon[118902]: pgmap v4438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:29 smithi136 ceph-mon[113531]: pgmap v4438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:29.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:29.828312+0000) 2023-08-27T23:21:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1035332932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.695 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.695 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.695 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1035332932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:30.695 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.695 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1035332932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:30.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:30.726Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:30.728Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:30.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:30.828560+0000) 2023-08-27T23:21:31.566 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:31 smithi093 ceph-mon[111335]: pgmap v4439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:31.566 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:31 smithi093 ceph-mon[118902]: pgmap v4439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:31 smithi136 ceph-mon[113531]: pgmap v4439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:31.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:31.828755+0000) 2023-08-27T23:21:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:32.215Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:32.216Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:32.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:32.829018+0000) 2023-08-27T23:21:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:21:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:21:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2825602219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:32 smithi136 ceph-mon[113531]: pgmap v4440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2825602219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[118902]: pgmap v4440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:21:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:21:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:21:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2825602219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:32 smithi093 ceph-mon[111335]: pgmap v4440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:33.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:33.829282+0000) 2023-08-27T23:21:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:34 smithi136 ceph-mon[113531]: pgmap v4441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1264265585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:34.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:34.829517+0000) 2023-08-27T23:21:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:34 smithi093 ceph-mon[111335]: pgmap v4441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1264265585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:34 smithi093 ceph-mon[118902]: pgmap v4441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1264265585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:21:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:21:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:21:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:35.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:35.829778+0000) 2023-08-27T23:21:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:36 smithi136 ceph-mon[113531]: pgmap v4442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:36.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:36.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:36.830026+0000) 2023-08-27T23:21:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:36 smithi093 ceph-mon[111335]: pgmap v4442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:36 smithi093 ceph-mon[118902]: pgmap v4442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/122595857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:37.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:37.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:37.830278+0000) 2023-08-27T23:21:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/122595857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/122595857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:38 smithi136 ceph-mon[113531]: pgmap v4443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:38.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:38.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:38.830467+0000) 2023-08-27T23:21:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:38 smithi093 ceph-mon[111335]: pgmap v4443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:38 smithi093 ceph-mon[118902]: pgmap v4443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1836862857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:39.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:39.830690+0000) 2023-08-27T23:21:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1836862857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1836862857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:40.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:40.728Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:40.728Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:41 smithi136 ceph-mon[113531]: pgmap v4444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:40.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:40.830929+0000) 2023-08-27T23:21:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:41 smithi093 ceph-mon[111335]: pgmap v4444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:41 smithi093 ceph-mon[118902]: pgmap v4444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3039466666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:41.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:41.831125+0000) 2023-08-27T23:21:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3039466666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3039466666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:42.217Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:42.218Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:43 smithi136 ceph-mon[113531]: pgmap v4445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:42.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:42.831311+0000) 2023-08-27T23:21:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:43 smithi093 ceph-mon[111335]: pgmap v4445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:43 smithi093 ceph-mon[118902]: pgmap v4445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:44.162 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:43.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:43.831577+0000) 2023-08-27T23:21:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/169401684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/169401684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/169401684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:45.166 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:44.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:44.831795+0000) 2023-08-27T23:21:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:45 smithi136 ceph-mon[113531]: pgmap v4446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:45 smithi093 ceph-mon[111335]: pgmap v4446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:21:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:21:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:21:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:45 smithi093 ceph-mon[118902]: pgmap v4446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:45.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:45.832090+0000) 2023-08-27T23:21:46.832 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3770636430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3770636430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3770636430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:46.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:46.832329+0000) 2023-08-27T23:21:47.832 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:47 smithi136 ceph-mon[113531]: pgmap v4447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:47 smithi093 ceph-mon[111335]: pgmap v4447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:47 smithi093 ceph-mon[118902]: pgmap v4447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:47.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:47.832534+0000) 2023-08-27T23:21:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:48.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:48.832745+0000) 2023-08-27T23:21:49.833 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:49 smithi136 ceph-mon[113531]: pgmap v4448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:49.833 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/556639336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:49 smithi093 ceph-mon[111335]: pgmap v4448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/556639336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:49.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:49 smithi093 ceph-mon[118902]: pgmap v4448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:49.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/556639336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:49.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:49.833001+0000) 2023-08-27T23:21:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:50.729Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:50.730Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:51 smithi136 ceph-mon[113531]: pgmap v4449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:50.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:50.833239+0000) 2023-08-27T23:21:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:51 smithi093 ceph-mon[111335]: pgmap v4449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:51 smithi093 ceph-mon[118902]: pgmap v4449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/806997409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:51.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:51.833498+0000) 2023-08-27T23:21:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/806997409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/806997409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:52.217Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:21:52.218Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:21:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:53 smithi136 ceph-mon[113531]: pgmap v4450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:52.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:52.833689+0000) 2023-08-27T23:21:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:53 smithi093 ceph-mon[111335]: pgmap v4450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:53 smithi093 ceph-mon[118902]: pgmap v4450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:53.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:53.833904+0000) 2023-08-27T23:21:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2029215019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2029215019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2029215019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:21:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:55 smithi136 ceph-mon[113531]: pgmap v4451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:54.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:54.834160+0000) 2023-08-27T23:21:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:55 smithi093 ceph-mon[111335]: pgmap v4451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:55 smithi093 ceph-mon[118902]: pgmap v4451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:21:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:21:55] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:21:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/377632363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:55.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:55.834427+0000) 2023-08-27T23:21:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/377632363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/377632363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:57 smithi136 ceph-mon[113531]: pgmap v4452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:56.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:56.834641+0000) 2023-08-27T23:21:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:57 smithi093 ceph-mon[118902]: pgmap v4452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:57 smithi093 ceph-mon[111335]: pgmap v4452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:57.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:57.834867+0000) 2023-08-27T23:21:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/755379392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/755379392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:58.835 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/755379392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:21:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:58.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:58.835140+0000) 2023-08-27T23:21:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:21:59 smithi093 ceph-mon[111335]: pgmap v4453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:21:59 smithi093 ceph-mon[118902]: pgmap v4453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:21:59.835 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:21:59 smithi136 ceph-mon[113531]: pgmap v4453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:21:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:21:59.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:21:59.835361+0000) 2023-08-27T23:22:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:00.730Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:00.731Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:01 smithi136 ceph-mon[113531]: pgmap v4454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/156666989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:00.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:00.835618+0000) 2023-08-27T23:22:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:01 smithi093 ceph-mon[111335]: pgmap v4454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/156666989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:01 smithi093 ceph-mon[118902]: pgmap v4454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/156666989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:01.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:01.835857+0000) 2023-08-27T23:22:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:02.218Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:02.219Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:02 smithi136 ceph-mon[113531]: pgmap v4455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:02.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:02.836084+0000) 2023-08-27T23:22:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:02 smithi093 ceph-mon[111335]: pgmap v4455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:02 smithi093 ceph-mon[118902]: pgmap v4455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/830952659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:03.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:03.836283+0000) 2023-08-27T23:22:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/830952659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/830952659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:04 smithi136 ceph-mon[113531]: pgmap v4456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:04.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:04.836489+0000) 2023-08-27T23:22:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:04 smithi093 ceph-mon[111335]: pgmap v4456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:04 smithi093 ceph-mon[118902]: pgmap v4456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:22:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:22:05] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:22:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1009530995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:05.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:05.836719+0000) 2023-08-27T23:22:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1009530995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1009530995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:07 smithi136 ceph-mon[113531]: pgmap v4457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:06.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:06.836956+0000) 2023-08-27T23:22:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:07 smithi093 ceph-mon[111335]: pgmap v4457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:07 smithi093 ceph-mon[118902]: pgmap v4457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1034242311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:07.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:07.837238+0000) 2023-08-27T23:22:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1034242311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1034242311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:09 smithi136 ceph-mon[113531]: pgmap v4458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:08.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:08.837462+0000) 2023-08-27T23:22:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:09 smithi093 ceph-mon[111335]: pgmap v4458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:09 smithi093 ceph-mon[118902]: pgmap v4458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:09.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:09.837604+0000) 2023-08-27T23:22:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:10.731Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:10.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1596593885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:11 smithi136 ceph-mon[113531]: pgmap v4459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:10.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:10.837827+0000) 2023-08-27T23:22:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1596593885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:11 smithi093 ceph-mon[111335]: pgmap v4459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1596593885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:11 smithi093 ceph-mon[118902]: pgmap v4459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:11.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:11.841701+0000) 2023-08-27T23:22:12.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:12.219Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:12.220Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:12 smithi136 ceph-mon[113531]: pgmap v4460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:12.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:12.841904+0000) 2023-08-27T23:22:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:12 smithi093 ceph-mon[111335]: pgmap v4460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:12 smithi093 ceph-mon[118902]: pgmap v4460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4032117228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:13.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:13.842140+0000) 2023-08-27T23:22:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4032117228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4032117228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:14 smithi136 ceph-mon[113531]: pgmap v4461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:14.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:14.842395+0000) 2023-08-27T23:22:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:14 smithi093 ceph-mon[111335]: pgmap v4461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:14.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:14 smithi093 ceph-mon[118902]: pgmap v4461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:22:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:22:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:22:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1638766847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:15.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:15.842668+0000) 2023-08-27T23:22:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1638766847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1638766847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:17 smithi136 ceph-mon[113531]: pgmap v4462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:16.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:16.842928+0000) 2023-08-27T23:22:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:17 smithi093 ceph-mon[111335]: pgmap v4462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:17 smithi093 ceph-mon[118902]: pgmap v4462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1337350292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:17.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:17.843113+0000) 2023-08-27T23:22:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1337350292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1337350292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:19 smithi136 ceph-mon[113531]: pgmap v4463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:18.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:18.843345+0000) 2023-08-27T23:22:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:19 smithi093 ceph-mon[111335]: pgmap v4463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:19 smithi093 ceph-mon[118902]: pgmap v4463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/967407009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:19.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:19.843585+0000) 2023-08-27T23:22:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/967407009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/967407009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:20.732Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:20.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:21 smithi136 ceph-mon[113531]: pgmap v4464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:20.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:20.843867+0000) 2023-08-27T23:22:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:21 smithi093 ceph-mon[111335]: pgmap v4464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:21 smithi093 ceph-mon[118902]: pgmap v4464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:21.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:21.844156+0000) 2023-08-27T23:22:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1849385550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1849385550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:22.220Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:22.221Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1849385550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:22.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:22.844379+0000) 2023-08-27T23:22:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:23 smithi136 ceph-mon[113531]: pgmap v4465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:23 smithi093 ceph-mon[118902]: pgmap v4465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:23 smithi093 ceph-mon[111335]: pgmap v4465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:23.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:23.844637+0000) 2023-08-27T23:22:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:24.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:24.844906+0000) 2023-08-27T23:22:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:25 smithi136 ceph-mon[113531]: pgmap v4466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1112165015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:25 smithi093 ceph-mon[111335]: pgmap v4466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1112165015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:25 smithi093 ceph-mon[118902]: pgmap v4466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1112165015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:22:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:22:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:22:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:25.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:25.845196+0000) 2023-08-27T23:22:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:26 smithi136 ceph-mon[113531]: pgmap v4467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:26.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:26.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:26.845454+0000) 2023-08-27T23:22:26.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:26 smithi093 ceph-mon[111335]: pgmap v4467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:26 smithi093 ceph-mon[118902]: pgmap v4467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3687526726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:27.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:27.845664+0000) 2023-08-27T23:22:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3687526726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3687526726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:28 smithi136 ceph-mon[113531]: pgmap v4468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:28.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:28.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:28.845916+0000) 2023-08-27T23:22:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:28 smithi093 ceph-mon[111335]: pgmap v4468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:28 smithi093 ceph-mon[118902]: pgmap v4468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1015556780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:29.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:29.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:29.846187+0000) 2023-08-27T23:22:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1015556780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1015556780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:30.732Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:30.733Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:31 smithi136 ceph-mon[113531]: pgmap v4469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:30.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:30.846443+0000) 2023-08-27T23:22:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:31 smithi093 ceph-mon[111335]: pgmap v4469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:31 smithi093 ceph-mon[118902]: pgmap v4469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4180800969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:22:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:31.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:31.846698+0000) 2023-08-27T23:22:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4180800969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:22:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4180800969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:22:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:32.221Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:32.222Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:33 smithi136 ceph-mon[113531]: pgmap v4470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:32.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:32.846877+0000) 2023-08-27T23:22:33.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:33 smithi093 ceph-mon[111335]: pgmap v4470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:33.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:33 smithi093 ceph-mon[118902]: pgmap v4470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:34.059 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:33.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:33.847022+0000) 2023-08-27T23:22:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:34.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:34.847223+0000) 2023-08-27T23:22:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3693805050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:34 smithi136 ceph-mon[113531]: pgmap v4471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3693805050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[111335]: pgmap v4471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3693805050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:34 smithi093 ceph-mon[118902]: pgmap v4471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:35.622 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:22:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:22:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:22:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:22:35.953 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:22:35.954 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:35.954 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:35.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:35.847384+0000) 2023-08-27T23:22:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:22:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:22:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:22:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:22:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:22:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2422426414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:36 smithi136 ceph-mon[113531]: pgmap v4472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:36.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:36.847564+0000) 2023-08-27T23:22:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2422426414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:36 smithi093 ceph-mon[111335]: pgmap v4472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2422426414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:36 smithi093 ceph-mon[118902]: pgmap v4472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:37.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:37.847771+0000) 2023-08-27T23:22:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:38 smithi136 ceph-mon[113531]: pgmap v4473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:38.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:38.848022+0000) 2023-08-27T23:22:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:38 smithi093 ceph-mon[111335]: pgmap v4473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:38 smithi093 ceph-mon[118902]: pgmap v4473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1120125308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:39.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:39.848246+0000) 2023-08-27T23:22:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1120125308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1120125308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:40.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:40.733Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:40.734Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:41 smithi136 ceph-mon[113531]: pgmap v4474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:40.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:40.848477+0000) 2023-08-27T23:22:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:41 smithi093 ceph-mon[111335]: pgmap v4474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:41 smithi093 ceph-mon[118902]: pgmap v4474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2281648623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:41.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:41.848733+0000) 2023-08-27T23:22:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2281648623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2281648623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:42.222Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:42.223Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:43 smithi136 ceph-mon[113531]: pgmap v4475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:42.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:42.849026+0000) 2023-08-27T23:22:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:43 smithi093 ceph-mon[111335]: pgmap v4475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:43 smithi093 ceph-mon[118902]: pgmap v4475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3635961686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:43.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:43.849269+0000) 2023-08-27T23:22:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3635961686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3635961686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:45 smithi136 ceph-mon[113531]: pgmap v4476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:44.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:44.849469+0000) 2023-08-27T23:22:45.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:45 smithi093 ceph-mon[111335]: pgmap v4476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:45 smithi093 ceph-mon[118902]: pgmap v4476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:22:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:22:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:22:46.110 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:45.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:45.849712+0000) 2023-08-27T23:22:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1127182906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1127182906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1127182906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:47.114 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:46.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:46.849936+0000) 2023-08-27T23:22:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:47 smithi136 ceph-mon[113531]: pgmap v4477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:47 smithi093 ceph-mon[118902]: pgmap v4477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:47 smithi093 ceph-mon[111335]: pgmap v4477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:47.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:47.850197+0000) 2023-08-27T23:22:48.851 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2774360460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2774360460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2774360460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:48.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:48.850412+0000) 2023-08-27T23:22:49.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:49 smithi136 ceph-mon[113531]: pgmap v4478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:49 smithi093 ceph-mon[111335]: pgmap v4478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:49 smithi093 ceph-mon[118902]: pgmap v4478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:49.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:49.850626+0000) 2023-08-27T23:22:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:50.734Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:50.735Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:51 smithi136 ceph-mon[113531]: pgmap v4479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4289154250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:50.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:50.850819+0000) 2023-08-27T23:22:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:51 smithi093 ceph-mon[111335]: pgmap v4479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4289154250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:51 smithi093 ceph-mon[118902]: pgmap v4479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4289154250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:51.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:51.851040+0000) 2023-08-27T23:22:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:52.223Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:22:52.224Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:22:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:52 smithi136 ceph-mon[113531]: pgmap v4480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:52.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:52.851289+0000) 2023-08-27T23:22:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:52 smithi093 ceph-mon[111335]: pgmap v4480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:52 smithi093 ceph-mon[118902]: pgmap v4480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3023335257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:53.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:53.851520+0000) 2023-08-27T23:22:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3023335257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3023335257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:54 smithi136 ceph-mon[113531]: pgmap v4481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:54.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:54.851707+0000) 2023-08-27T23:22:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:54 smithi093 ceph-mon[111335]: pgmap v4481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:22:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:54 smithi093 ceph-mon[118902]: pgmap v4481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:22:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:22:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:22:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2465413959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:55.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:55.851939+0000) 2023-08-27T23:22:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2465413959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2465413959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:57 smithi136 ceph-mon[113531]: pgmap v4482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:56.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:56.852197+0000) 2023-08-27T23:22:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:57 smithi093 ceph-mon[111335]: pgmap v4482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:57 smithi093 ceph-mon[118902]: pgmap v4482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1553446574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:57.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:57.852391+0000) 2023-08-27T23:22:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1553446574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1553446574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:22:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:22:59 smithi136 ceph-mon[113531]: pgmap v4483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:58.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:58.852642+0000) 2023-08-27T23:22:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:22:59 smithi093 ceph-mon[111335]: pgmap v4483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:22:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:22:59 smithi093 ceph-mon[118902]: pgmap v4483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:22:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:22:59.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:22:59.852897+0000) 2023-08-27T23:23:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:00.735Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:00.736Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:01.024 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:00.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:00.853088+0000) 2023-08-27T23:23:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2895872992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:01 smithi136 ceph-mon[113531]: pgmap v4484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2895872992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:01 smithi093 ceph-mon[111335]: pgmap v4484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2895872992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:01 smithi093 ceph-mon[118902]: pgmap v4484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:01.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:01.853281+0000) 2023-08-27T23:23:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:02.223Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:02.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:02.224Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:02 smithi136 ceph-mon[113531]: pgmap v4485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:02.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:02.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:02.853494+0000) 2023-08-27T23:23:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:02 smithi093 ceph-mon[111335]: pgmap v4485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:02 smithi093 ceph-mon[118902]: pgmap v4485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2004713907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:03.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:03.853766+0000) 2023-08-27T23:23:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2004713907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2004713907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:04 smithi136 ceph-mon[113531]: pgmap v4486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:04.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:04.853959+0000) 2023-08-27T23:23:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:04 smithi093 ceph-mon[111335]: pgmap v4486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:04 smithi093 ceph-mon[118902]: pgmap v4486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:23:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:23:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:23:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3810539478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:05.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:05.854152+0000) 2023-08-27T23:23:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3810539478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3810539478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:07 smithi136 ceph-mon[113531]: pgmap v4487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:06.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:06.854352+0000) 2023-08-27T23:23:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:07 smithi093 ceph-mon[111335]: pgmap v4487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:07 smithi093 ceph-mon[118902]: pgmap v4487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1668378210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:07.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:07.854576+0000) 2023-08-27T23:23:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1668378210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1668378210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:09 smithi136 ceph-mon[113531]: pgmap v4488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:08.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:08.854806+0000) 2023-08-27T23:23:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:09 smithi093 ceph-mon[111335]: pgmap v4488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:09 smithi093 ceph-mon[118902]: pgmap v4488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2032079877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:09.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:09.855106+0000) 2023-08-27T23:23:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2032079877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2032079877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:10.737Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:10.738Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:11 smithi136 ceph-mon[113531]: pgmap v4489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:10.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:10.855363+0000) 2023-08-27T23:23:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:11 smithi093 ceph-mon[111335]: pgmap v4489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:11 smithi093 ceph-mon[118902]: pgmap v4489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:11.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:11.855629+0000) 2023-08-27T23:23:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1315937360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1315937360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:12.224Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:12.224Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1315937360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:12.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:12.855872+0000) 2023-08-27T23:23:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:13 smithi136 ceph-mon[113531]: pgmap v4490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:13 smithi093 ceph-mon[111335]: pgmap v4490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:13 smithi093 ceph-mon[118902]: pgmap v4490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:13.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:13.856123+0000) 2023-08-27T23:23:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:14 smithi136 ceph-mon[113531]: pgmap v4491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:14.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:14.856363+0000) 2023-08-27T23:23:14.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:14 smithi093 ceph-mon[111335]: pgmap v4491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:14 smithi093 ceph-mon[118902]: pgmap v4491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:15.621 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:23:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:23:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:23:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2050511556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:15.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:15.856647+0000) 2023-08-27T23:23:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2050511556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2050511556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:16 smithi136 ceph-mon[113531]: pgmap v4492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:16.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:16.856907+0000) 2023-08-27T23:23:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:16 smithi093 ceph-mon[111335]: pgmap v4492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:16 smithi093 ceph-mon[118902]: pgmap v4492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3694240756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:17.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:17.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:17.857154+0000) 2023-08-27T23:23:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3694240756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3694240756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:18 smithi136 ceph-mon[113531]: pgmap v4493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:18.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:18.857414+0000) 2023-08-27T23:23:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:18 smithi093 ceph-mon[111335]: pgmap v4493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:18 smithi093 ceph-mon[118902]: pgmap v4493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2985805993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:19.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:19.857688+0000) 2023-08-27T23:23:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2985805993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2985805993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:20.738Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:20.739Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:21 smithi136 ceph-mon[113531]: pgmap v4494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:20.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:20.857888+0000) 2023-08-27T23:23:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:21 smithi093 ceph-mon[111335]: pgmap v4494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:21 smithi093 ceph-mon[118902]: pgmap v4494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1582878576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:21.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:21.858090+0000) 2023-08-27T23:23:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1582878576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1582878576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:22.224Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:22.225Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:23 smithi136 ceph-mon[113531]: pgmap v4495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:22.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:22.858278+0000) 2023-08-27T23:23:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:23 smithi093 ceph-mon[111335]: pgmap v4495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:23 smithi093 ceph-mon[118902]: pgmap v4495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:23.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:23.858533+0000) 2023-08-27T23:23:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/251519329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:25 smithi136 ceph-mon[113531]: pgmap v4496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:24.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:24.858777+0000) 2023-08-27T23:23:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/251519329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:25 smithi093 ceph-mon[111335]: pgmap v4496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:25.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/251519329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:25.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:25 smithi093 ceph-mon[118902]: pgmap v4496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:23:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:23:25] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:23:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:25.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:25.859058+0000) 2023-08-27T23:23:26.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3698963110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:26.859 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:26 smithi136 ceph-mon[113531]: pgmap v4497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3698963110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:26 smithi093 ceph-mon[111335]: pgmap v4497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3698963110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:26 smithi093 ceph-mon[118902]: pgmap v4497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:26.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:26.859286+0000) 2023-08-27T23:23:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:27.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:27.859530+0000) 2023-08-27T23:23:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:28 smithi136 ceph-mon[113531]: pgmap v4498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:28.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:28.859801+0000) 2023-08-27T23:23:28.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:28 smithi093 ceph-mon[111335]: pgmap v4498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:28 smithi093 ceph-mon[118902]: pgmap v4498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/547765808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:29.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:29.860021+0000) 2023-08-27T23:23:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/547765808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/547765808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:30.738Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:30.739Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:31 smithi136 ceph-mon[113531]: pgmap v4499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:30.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:30.860258+0000) 2023-08-27T23:23:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:31 smithi093 ceph-mon[111335]: pgmap v4499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:31 smithi093 ceph-mon[118902]: pgmap v4499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1346608498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:31.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:31.860497+0000) 2023-08-27T23:23:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1346608498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1346608498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:32.225Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:32.226Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:33 smithi136 ceph-mon[113531]: pgmap v4500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:32.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:32.860706+0000) 2023-08-27T23:23:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:33 smithi093 ceph-mon[111335]: pgmap v4500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:33 smithi093 ceph-mon[118902]: pgmap v4500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4073747778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:33.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:33.860944+0000) 2023-08-27T23:23:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4073747778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4073747778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:35 smithi136 ceph-mon[113531]: pgmap v4501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:23:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:34.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:34.861139+0000) 2023-08-27T23:23:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:35 smithi093 ceph-mon[111335]: pgmap v4501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:23:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:35 smithi093 ceph-mon[118902]: pgmap v4501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:23:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:23:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:23:35] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:23:36.166 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:35.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:35.861301+0000) 2023-08-27T23:23:36.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2700807889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2700807889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2700807889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:36.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:36.861539+0000) 2023-08-27T23:23:37.861 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:37 smithi136 ceph-mon[113531]: pgmap v4502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:37.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[111335]: pgmap v4502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[118902]: pgmap v4502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:37.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:37.861723+0000) 2023-08-27T23:23:38.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:23:38.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:23:38.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:38.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3388226791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:23:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:23:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3388226791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:23:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:23:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:23:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3388226791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:38.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:38.861918+0000) 2023-08-27T23:23:39.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:39 smithi136 ceph-mon[113531]: pgmap v4503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:39.862 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:39 smithi093 ceph-mon[111335]: pgmap v4503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:39 smithi093 ceph-mon[118902]: pgmap v4503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:39.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:39.862163+0000) 2023-08-27T23:23:41.026 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:40.739Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:41.027 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:40.740Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:40.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:40.862414+0000) 2023-08-27T23:23:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:41 smithi136 ceph-mon[113531]: pgmap v4504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2438000433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:41 smithi093 ceph-mon[111335]: pgmap v4504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2438000433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:41 smithi093 ceph-mon[118902]: pgmap v4504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2438000433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:41.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:41.862729+0000) 2023-08-27T23:23:42.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:42.226Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:42.227Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:42 smithi136 ceph-mon[113531]: pgmap v4505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:42.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:42.863010+0000) 2023-08-27T23:23:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:42 smithi093 ceph-mon[111335]: pgmap v4505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:42 smithi093 ceph-mon[118902]: pgmap v4505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3874692245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:43.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:43.863260+0000) 2023-08-27T23:23:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3874692245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3874692245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:44 smithi136 ceph-mon[113531]: pgmap v4506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:44.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:44.863511+0000) 2023-08-27T23:23:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:44 smithi093 ceph-mon[111335]: pgmap v4506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:44 smithi093 ceph-mon[118902]: pgmap v4506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:23:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:23:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:23:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/822481409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:45.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:45.863776+0000) 2023-08-27T23:23:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/822481409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/822481409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:47 smithi136 ceph-mon[113531]: pgmap v4507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:46.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:46.864003+0000) 2023-08-27T23:23:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:47 smithi093 ceph-mon[111335]: pgmap v4507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:47 smithi093 ceph-mon[118902]: pgmap v4507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/40148096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:47.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:47.864200+0000) 2023-08-27T23:23:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/40148096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/40148096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:49 smithi136 ceph-mon[113531]: pgmap v4508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:48.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:48.864452+0000) 2023-08-27T23:23:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:49 smithi093 ceph-mon[111335]: pgmap v4508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:49 smithi093 ceph-mon[118902]: pgmap v4508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:49.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:49.864695+0000) 2023-08-27T23:23:51.028 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:50.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:51.028 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:50.740Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2384124754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:51 smithi136 ceph-mon[113531]: pgmap v4509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:50.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:50.864960+0000) 2023-08-27T23:23:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2384124754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:51 smithi093 ceph-mon[111335]: pgmap v4509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2384124754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:51 smithi093 ceph-mon[118902]: pgmap v4509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:51.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:51.865215+0000) 2023-08-27T23:23:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:52.227Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:23:52.228Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:23:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:52 smithi136 ceph-mon[113531]: pgmap v4510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:52.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:52.865435+0000) 2023-08-27T23:23:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:52 smithi093 ceph-mon[118902]: pgmap v4510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:52 smithi093 ceph-mon[111335]: pgmap v4510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1676496645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:53.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:53.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:53.865693+0000) 2023-08-27T23:23:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1676496645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1676496645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:54 smithi136 ceph-mon[113531]: pgmap v4511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:54.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:54.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:54.865935+0000) 2023-08-27T23:23:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:54 smithi093 ceph-mon[111335]: pgmap v4511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:23:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:54 smithi093 ceph-mon[118902]: pgmap v4511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:23:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:23:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:23:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1952771764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:55.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:55.866202+0000) 2023-08-27T23:23:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1952771764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1952771764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:57 smithi136 ceph-mon[113531]: pgmap v4512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:56.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:56.866482+0000) 2023-08-27T23:23:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:57 smithi093 ceph-mon[118902]: pgmap v4512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:57 smithi093 ceph-mon[111335]: pgmap v4512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3479976293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:57.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:57.866747+0000) 2023-08-27T23:23:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3479976293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3479976293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:23:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:23:59 smithi136 ceph-mon[113531]: pgmap v4513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:58.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:58.866995+0000) 2023-08-27T23:23:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:23:59 smithi093 ceph-mon[111335]: pgmap v4513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:23:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:23:59 smithi093 ceph-mon[118902]: pgmap v4513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3777487611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:23:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:23:59.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:23:59.867256+0000) 2023-08-27T23:24:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3777487611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3777487611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:01.043 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:00.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:01.043 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:00.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:01 smithi136 ceph-mon[113531]: pgmap v4514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:00.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:00.867516+0000) 2023-08-27T23:24:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:01 smithi093 ceph-mon[111335]: pgmap v4514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:01 smithi093 ceph-mon[118902]: pgmap v4514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:01.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:01.867790+0000) 2023-08-27T23:24:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/309691247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/309691247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:02.227Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:02.228Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/309691247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:02.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:02.868063+0000) 2023-08-27T23:24:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:03 smithi136 ceph-mon[113531]: pgmap v4515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:03 smithi093 ceph-mon[111335]: pgmap v4515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:03 smithi093 ceph-mon[118902]: pgmap v4515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:03.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:03.868270+0000) 2023-08-27T23:24:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:04 smithi136 ceph-mon[113531]: pgmap v4516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:04.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:04.868535+0000) 2023-08-27T23:24:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:04 smithi093 ceph-mon[111335]: pgmap v4516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:04 smithi093 ceph-mon[118902]: pgmap v4516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:05.631 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:24:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:24:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:24:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/85448921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:05.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:05.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:05.868790+0000) 2023-08-27T23:24:05.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/85448921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/85448921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:06 smithi136 ceph-mon[113531]: pgmap v4517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:06.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:06.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:06.868987+0000) 2023-08-27T23:24:06.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:06 smithi093 ceph-mon[111335]: pgmap v4517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:06 smithi093 ceph-mon[118902]: pgmap v4517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/389421817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:07.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:07.869246+0000) 2023-08-27T23:24:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/389421817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/389421817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:08 smithi136 ceph-mon[113531]: pgmap v4518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:08.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:08.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:08.869521+0000) 2023-08-27T23:24:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:08 smithi093 ceph-mon[111335]: pgmap v4518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:08 smithi093 ceph-mon[118902]: pgmap v4518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/320138556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:09.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:09.869721+0000) 2023-08-27T23:24:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/320138556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/320138556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:11.031 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:10.741Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:11.031 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:10.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:11 smithi136 ceph-mon[113531]: pgmap v4519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:10.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:10.869978+0000) 2023-08-27T23:24:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:11 smithi093 ceph-mon[111335]: pgmap v4519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:11 smithi093 ceph-mon[118902]: pgmap v4519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1090459448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:11.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:11.870243+0000) 2023-08-27T23:24:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1090459448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1090459448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:12.228Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:12.229Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:13 smithi136 ceph-mon[113531]: pgmap v4520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:12.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:12.870479+0000) 2023-08-27T23:24:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:13 smithi093 ceph-mon[111335]: pgmap v4520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:13 smithi093 ceph-mon[118902]: pgmap v4520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:13.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:13.870733+0000) 2023-08-27T23:24:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3288715154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3288715154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3288715154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:14.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:14.870989+0000) 2023-08-27T23:24:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:15 smithi136 ceph-mon[113531]: pgmap v4521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:15 smithi093 ceph-mon[111335]: pgmap v4521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:24:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:24:15] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:24:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:15 smithi093 ceph-mon[118902]: pgmap v4521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:15.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:15.871291+0000) 2023-08-27T23:24:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:16 smithi136 ceph-mon[113531]: pgmap v4522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3751079156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:16.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:16.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:16.871535+0000) 2023-08-27T23:24:16.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:16 smithi093 ceph-mon[111335]: pgmap v4522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3751079156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:16 smithi093 ceph-mon[118902]: pgmap v4522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3751079156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:17.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:17.871773+0000) 2023-08-27T23:24:18.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:18.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:18.872025+0000) 2023-08-27T23:24:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:18 smithi136 ceph-mon[113531]: pgmap v4523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:18 smithi093 ceph-mon[111335]: pgmap v4523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:18 smithi093 ceph-mon[118902]: pgmap v4523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3772816684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:19.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:19.872292+0000) 2023-08-27T23:24:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3772816684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3772816684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:21.033 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:20.742Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:21.033 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:20.743Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:21 smithi136 ceph-mon[113531]: pgmap v4524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:20.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:20.872537+0000) 2023-08-27T23:24:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:21 smithi093 ceph-mon[111335]: pgmap v4524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:21 smithi093 ceph-mon[118902]: pgmap v4524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/654087497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:21.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:21.872777+0000) 2023-08-27T23:24:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/654087497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/654087497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:22.228Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:22.230Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:23 smithi136 ceph-mon[113531]: pgmap v4525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:22.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:22.873025+0000) 2023-08-27T23:24:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:23 smithi093 ceph-mon[111335]: pgmap v4525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:23 smithi093 ceph-mon[118902]: pgmap v4525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/110156303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:23.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:23.873289+0000) 2023-08-27T23:24:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/110156303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/110156303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:25 smithi136 ceph-mon[113531]: pgmap v4526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:24.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:24.873561+0000) 2023-08-27T23:24:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:25 smithi093 ceph-mon[111335]: pgmap v4526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:25.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:25 smithi093 ceph-mon[118902]: pgmap v4526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:24:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:24:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:24:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:25.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:25.873803+0000) 2023-08-27T23:24:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2625534193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2625534193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2625534193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:26.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:26.874014+0000) 2023-08-27T23:24:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:27 smithi093 ceph-mon[111335]: pgmap v4527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:27 smithi093 ceph-mon[118902]: pgmap v4527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:27 smithi136 ceph-mon[113531]: pgmap v4527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:27.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:27.874249+0000) 2023-08-27T23:24:28.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/401603331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:28.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:28 smithi136 ceph-mon[113531]: pgmap v4528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/401603331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:28 smithi093 ceph-mon[111335]: pgmap v4528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/401603331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:28 smithi093 ceph-mon[118902]: pgmap v4528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:28.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:28.874499+0000) 2023-08-27T23:24:30.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:29.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:29.874753+0000) 2023-08-27T23:24:31.035 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:30.742Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:31.035 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:30.743Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:31 smithi136 ceph-mon[113531]: pgmap v4529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1082420371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:30.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:30.875003+0000) 2023-08-27T23:24:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:31 smithi093 ceph-mon[111335]: pgmap v4529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1082420371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:31 smithi093 ceph-mon[118902]: pgmap v4529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1082420371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:31.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:31.875309+0000) 2023-08-27T23:24:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:32.229Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:32.230Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:32 smithi136 ceph-mon[113531]: pgmap v4530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:32.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:32.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:32.875583+0000) 2023-08-27T23:24:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:32 smithi093 ceph-mon[111335]: pgmap v4530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:32 smithi093 ceph-mon[118902]: pgmap v4530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4054750794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:33.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:33.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:33.875783+0000) 2023-08-27T23:24:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4054750794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4054750794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:34 smithi136 ceph-mon[113531]: pgmap v4531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:34.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:34.876059+0000) 2023-08-27T23:24:34.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:34 smithi093 ceph-mon[111335]: pgmap v4531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:34 smithi093 ceph-mon[118902]: pgmap v4531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:24:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:24:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:24:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/764727065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:35.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:35.876357+0000) 2023-08-27T23:24:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/764727065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/764727065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:37 smithi136 ceph-mon[113531]: pgmap v4532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:36.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:36.876592+0000) 2023-08-27T23:24:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:37 smithi093 ceph-mon[111335]: pgmap v4532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:37 smithi093 ceph-mon[118902]: pgmap v4532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:24:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:37.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:37.876797+0000) 2023-08-27T23:24:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:24:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:24:39.104 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:38.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:38.876936+0000) 2023-08-27T23:24:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1217574634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:39 smithi136 ceph-mon[113531]: pgmap v4533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1217574634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[118902]: pgmap v4533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1217574634' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[111335]: pgmap v4533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:39.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:39.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:39.877178+0000) 2023-08-27T23:24:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:40.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:40.877391+0000) 2023-08-27T23:24:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:24:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:24:41.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/160914971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:41.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:40 smithi136 ceph-mon[113531]: pgmap v4534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/160914971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[118902]: pgmap v4534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/160914971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:41.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-mon[111335]: pgmap v4534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:40.743Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:40.744Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:41.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:41.877643+0000) 2023-08-27T23:24:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:42.229Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:42.230Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:42 smithi136 ceph-mon[113531]: pgmap v4535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:42.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:42.877918+0000) 2023-08-27T23:24:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:42 smithi093 ceph-mon[118902]: pgmap v4535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:42 smithi093 ceph-mon[111335]: pgmap v4535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/765478835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:43.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:43.878227+0000) 2023-08-27T23:24:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/765478835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/765478835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:44 smithi136 ceph-mon[113531]: pgmap v4536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:44.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:44.878492+0000) 2023-08-27T23:24:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:44 smithi093 ceph-mon[111335]: pgmap v4536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:44 smithi093 ceph-mon[118902]: pgmap v4536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:24:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:24:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:24:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/975409697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:45.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:45.878790+0000) 2023-08-27T23:24:46.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/975409697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:46.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/975409697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:47 smithi136 ceph-mon[113531]: pgmap v4537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:46.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:46.879009+0000) 2023-08-27T23:24:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:47 smithi093 ceph-mon[111335]: pgmap v4537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:47 smithi093 ceph-mon[118902]: pgmap v4537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1579709732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:47.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:47.879250+0000) 2023-08-27T23:24:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1579709732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1579709732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:49 smithi136 ceph-mon[113531]: pgmap v4538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:48.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:48.879446+0000) 2023-08-27T23:24:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:49 smithi093 ceph-mon[111335]: pgmap v4538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:49 smithi093 ceph-mon[118902]: pgmap v4538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3202372950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:49.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:49.879698+0000) 2023-08-27T23:24:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3202372950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3202372950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:50.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:50.879969+0000) 2023-08-27T23:24:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:50.744Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:50.745Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:52 smithi136 ceph-mon[113531]: pgmap v4539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:51.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:51.880222+0000) 2023-08-27T23:24:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:52 smithi093 ceph-mon[111335]: pgmap v4539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:52 smithi093 ceph-mon[118902]: pgmap v4539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:52.231Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:24:52.231Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:24:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:52.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:52.880516+0000) 2023-08-27T23:24:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1685313813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1685313813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1685313813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:54 smithi136 ceph-mon[113531]: pgmap v4540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:53.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:53.880818+0000) 2023-08-27T23:24:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:54 smithi093 ceph-mon[111335]: pgmap v4540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:54 smithi093 ceph-mon[118902]: pgmap v4540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:24:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2367874727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:54.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:54.881129+0000) 2023-08-27T23:24:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2367874727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2367874727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:24:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:24:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:24:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:56 smithi136 ceph-mon[113531]: pgmap v4541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:55.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:55.881416+0000) 2023-08-27T23:24:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:56 smithi093 ceph-mon[111335]: pgmap v4541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:56 smithi093 ceph-mon[118902]: pgmap v4541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:56.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:56.881719+0000) 2023-08-27T23:24:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:58 smithi136 ceph-mon[113531]: pgmap v4542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:24:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1480559586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:57.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:57.882000+0000) 2023-08-27T23:24:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:58 smithi093 ceph-mon[111335]: pgmap v4542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:24:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1480559586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:58 smithi093 ceph-mon[118902]: pgmap v4542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:24:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:24:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1480559586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:24:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:58.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:58.882278+0000) 2023-08-27T23:25:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:00 smithi136 ceph-mon[113531]: pgmap v4543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3577039524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:24:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:24:59.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:24:59.882531+0000) 2023-08-27T23:25:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:00 smithi093 ceph-mon[111335]: pgmap v4543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3577039524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:00 smithi093 ceph-mon[118902]: pgmap v4543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3577039524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:00.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:00.882715+0000) 2023-08-27T23:25:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:00.744Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:00.746Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:02 smithi136 ceph-mon[113531]: pgmap v4544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/611185571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:01.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:01.882980+0000) 2023-08-27T23:25:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:02 smithi093 ceph-mon[111335]: pgmap v4544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/611185571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:02 smithi093 ceph-mon[118902]: pgmap v4544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/611185571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:02.231Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:02.232Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:02.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:02.883241+0000) 2023-08-27T23:25:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:04 smithi136 ceph-mon[113531]: pgmap v4545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:03.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:03.883528+0000) 2023-08-27T23:25:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:04 smithi093 ceph-mon[111335]: pgmap v4545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:04 smithi093 ceph-mon[118902]: pgmap v4545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2936976360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:04.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:04.883802+0000) 2023-08-27T23:25:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2936976360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2936976360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:25:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:25:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:25:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:06 smithi136 ceph-mon[113531]: pgmap v4546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:05.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:05.884144+0000) 2023-08-27T23:25:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:06 smithi093 ceph-mon[111335]: pgmap v4546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:06 smithi093 ceph-mon[118902]: pgmap v4546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2392332065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:06.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:06.884424+0000) 2023-08-27T23:25:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2392332065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2392332065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:08 smithi136 ceph-mon[113531]: pgmap v4547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:07.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:07.884651+0000) 2023-08-27T23:25:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:08 smithi093 ceph-mon[111335]: pgmap v4547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:08 smithi093 ceph-mon[118902]: pgmap v4547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/905875467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:08.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:08.884924+0000) 2023-08-27T23:25:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/905875467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/905875467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:10 smithi136 ceph-mon[113531]: pgmap v4548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:09.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:09.885219+0000) 2023-08-27T23:25:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:10 smithi093 ceph-mon[111335]: pgmap v4548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:10 smithi093 ceph-mon[118902]: pgmap v4548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:10.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:10.885518+0000) 2023-08-27T23:25:11.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:10.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:10.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:12 smithi136 ceph-mon[113531]: pgmap v4549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/324460283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:11.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:11.885739+0000) 2023-08-27T23:25:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:12 smithi093 ceph-mon[111335]: pgmap v4549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/324460283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:12 smithi093 ceph-mon[118902]: pgmap v4549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/324460283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:12.232Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:12.233Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:12.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:12.886003+0000) 2023-08-27T23:25:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:14 smithi136 ceph-mon[113531]: pgmap v4550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/282013713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:13.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:13.886240+0000) 2023-08-27T23:25:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:14 smithi093 ceph-mon[111335]: pgmap v4550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/282013713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:14 smithi093 ceph-mon[118902]: pgmap v4550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/282013713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:14.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:14.886458+0000) 2023-08-27T23:25:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:25:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:25:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:25:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:16 smithi136 ceph-mon[113531]: pgmap v4551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:15.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:15.886721+0000) 2023-08-27T23:25:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:16 smithi093 ceph-mon[111335]: pgmap v4551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:16 smithi093 ceph-mon[118902]: pgmap v4551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1630479081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:16.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:16.886996+0000) 2023-08-27T23:25:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1630479081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1630479081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:18 smithi136 ceph-mon[113531]: pgmap v4552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:17.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:17.887176+0000) 2023-08-27T23:25:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:18 smithi093 ceph-mon[111335]: pgmap v4552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:18 smithi093 ceph-mon[118902]: pgmap v4552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2772969615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:18.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:18.887430+0000) 2023-08-27T23:25:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2772969615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2772969615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:20 smithi136 ceph-mon[113531]: pgmap v4553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:19.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:19.887720+0000) 2023-08-27T23:25:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:20 smithi093 ceph-mon[111335]: pgmap v4553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:20 smithi093 ceph-mon[118902]: pgmap v4553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:21.142 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:20.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:21.142 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:20.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:21.142 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:20.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:20.888005+0000) 2023-08-27T23:25:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3983620839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3983620839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3983620839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:22.143 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:21.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:21.888251+0000) 2023-08-27T23:25:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:22 smithi136 ceph-mon[113531]: pgmap v4554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:22 smithi093 ceph-mon[111335]: pgmap v4554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:22 smithi093 ceph-mon[118902]: pgmap v4554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:22.233Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:22.234Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:22.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:22.888496+0000) 2023-08-27T23:25:24.146 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:23.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:23.888792+0000) 2023-08-27T23:25:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:24 smithi136 ceph-mon[113531]: pgmap v4555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1284950655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:24 smithi093 ceph-mon[118902]: pgmap v4555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1284950655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:24 smithi093 ceph-mon[111335]: pgmap v4555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1284950655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:24.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:24.889113+0000) 2023-08-27T23:25:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:25:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:25:25] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:25:26.151 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:25.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:25.889407+0000) 2023-08-27T23:25:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:26 smithi136 ceph-mon[113531]: pgmap v4556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1843041450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:26 smithi093 ceph-mon[111335]: pgmap v4556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1843041450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:26 smithi093 ceph-mon[118902]: pgmap v4556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1843041450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:26.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:26.889690+0000) 2023-08-27T23:25:28.153 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:27.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:27.889929+0000) 2023-08-27T23:25:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:28 smithi136 ceph-mon[113531]: pgmap v4557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/259976809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:28 smithi093 ceph-mon[111335]: pgmap v4557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/259976809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:28 smithi093 ceph-mon[118902]: pgmap v4557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/259976809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:28.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:28.890184+0000) 2023-08-27T23:25:30.157 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:29.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:29.890417+0000) 2023-08-27T23:25:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:30 smithi136 ceph-mon[113531]: pgmap v4558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:30 smithi093 ceph-mon[111335]: pgmap v4558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:30 smithi093 ceph-mon[118902]: pgmap v4558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:31.160 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:30.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:31.160 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:30.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:31.161 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:30.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:30.890638+0000) 2023-08-27T23:25:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1529732370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1529732370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1529732370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:32.162 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:31.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:31.890892+0000) 2023-08-27T23:25:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:32 smithi136 ceph-mon[113531]: pgmap v4559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:32 smithi093 ceph-mon[111335]: pgmap v4559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:32.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:32 smithi093 ceph-mon[118902]: pgmap v4559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:32.234Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:32.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:32.235Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:33.167 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:32.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:32.891147+0000) 2023-08-27T23:25:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3924852839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3924852839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3924852839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:34.168 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:33.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:33.891384+0000) 2023-08-27T23:25:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:34 smithi136 ceph-mon[113531]: pgmap v4560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:34 smithi093 ceph-mon[111335]: pgmap v4560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:34 smithi093 ceph-mon[118902]: pgmap v4560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:34.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:34.891589+0000) 2023-08-27T23:25:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:25:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:25:35] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:25:36.170 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:35.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:35.891872+0000) 2023-08-27T23:25:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:36 smithi136 ceph-mon[113531]: pgmap v4561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2650326771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:36 smithi093 ceph-mon[118902]: pgmap v4561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2650326771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:36 smithi093 ceph-mon[111335]: pgmap v4561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2650326771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:36.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:36.892093+0000) 2023-08-27T23:25:38.177 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:37.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:37.892311+0000) 2023-08-27T23:25:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:38 smithi136 ceph-mon[113531]: pgmap v4562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4078941402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:38 smithi093 ceph-mon[118902]: pgmap v4562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4078941402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:38 smithi093 ceph-mon[111335]: pgmap v4562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4078941402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:39.184 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:38.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:38.892584+0000) 2023-08-27T23:25:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:40.185 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:39.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:39.892832+0000) 2023-08-27T23:25:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:40 smithi136 ceph-mon[113531]: pgmap v4563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1938838800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:40 smithi093 ceph-mon[118902]: pgmap v4563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1938838800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:40 smithi093 ceph-mon[111335]: pgmap v4563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1938838800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:40.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:40.893199+0000) 2023-08-27T23:25:41.193 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:40.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:41.193 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:40.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:25:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:25:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:41.502 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:25:41.502 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:41.502 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:41.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:41.893359+0000) 2023-08-27T23:25:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:42 smithi093 ceph-mon[111335]: pgmap v4564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:42 smithi093 ceph-mon[118902]: pgmap v4564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:42.234Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:42.235Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:42 smithi136 ceph-mon[113531]: pgmap v4564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:42.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:42.893531+0000) 2023-08-27T23:25:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2327285360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:43.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2327285360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2327285360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:43.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:43.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:43.893725+0000) 2023-08-27T23:25:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:44 smithi136 ceph-mon[113531]: pgmap v4565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:25:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:25:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[111335]: pgmap v4565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:25:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:25:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[118902]: pgmap v4565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:25:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:25:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:25:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:44.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:44.893927+0000) 2023-08-27T23:25:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4018185538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4018185538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:25:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:25:45] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:25:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4018185538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:45.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:45.894183+0000) 2023-08-27T23:25:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:46 smithi136 ceph-mon[113531]: pgmap v4566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:46 smithi093 ceph-mon[111335]: pgmap v4566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:46 smithi093 ceph-mon[118902]: pgmap v4566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:46.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:46.894423+0000) 2023-08-27T23:25:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1657996844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1657996844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1657996844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:47.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:47.894671+0000) 2023-08-27T23:25:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:48 smithi136 ceph-mon[113531]: pgmap v4567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:48 smithi093 ceph-mon[111335]: pgmap v4567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:48 smithi093 ceph-mon[118902]: pgmap v4567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:48.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:48.894909+0000) 2023-08-27T23:25:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:49.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:49.895157+0000) 2023-08-27T23:25:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:50 smithi136 ceph-mon[113531]: pgmap v4568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/560170960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:50 smithi093 ceph-mon[118902]: pgmap v4568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/560170960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:50 smithi093 ceph-mon[111335]: pgmap v4568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/560170960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:50.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:50.895411+0000) 2023-08-27T23:25:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:50.747Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:50.748Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:51.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:51.895673+0000) 2023-08-27T23:25:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:52 smithi093 ceph-mon[111335]: pgmap v4569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/899672794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:52 smithi093 ceph-mon[118902]: pgmap v4569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/899672794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:52.235Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:25:52.236Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:25:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:52 smithi136 ceph-mon[113531]: pgmap v4569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/899672794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:52.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:52.895862+0000) 2023-08-27T23:25:54.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:53.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:53.896137+0000) 2023-08-27T23:25:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:54 smithi136 ceph-mon[113531]: pgmap v4570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3881311561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:54 smithi093 ceph-mon[111335]: pgmap v4570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3881311561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:54 smithi093 ceph-mon[118902]: pgmap v4570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:25:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3881311561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:54.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:54.896407+0000) 2023-08-27T23:25:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:25:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:25:55] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:25:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:55.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:55.896688+0000) 2023-08-27T23:25:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:56 smithi093 ceph-mon[111335]: pgmap v4571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:56 smithi093 ceph-mon[118902]: pgmap v4571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:56.897 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:56 smithi136 ceph-mon[113531]: pgmap v4571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:56.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:56.896972+0000) 2023-08-27T23:25:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1808121908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1808121908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:57.897 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1808121908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:57.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:57.897178+0000) 2023-08-27T23:25:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:58 smithi093 ceph-mon[111335]: pgmap v4572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:58 smithi093 ceph-mon[118902]: pgmap v4572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:58.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:58 smithi136 ceph-mon[113531]: pgmap v4572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:25:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:58.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:58.897387+0000) 2023-08-27T23:25:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:25:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/835499482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:25:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/835499482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:25:59.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:25:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/835499482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:25:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:25:59.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:25:59.897646+0000) 2023-08-27T23:26:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:00 smithi093 ceph-mon[111335]: pgmap v4573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:00 smithi093 ceph-mon[118902]: pgmap v4573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:00.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:00 smithi136 ceph-mon[113531]: pgmap v4573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:00.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:00.897898+0000) 2023-08-27T23:26:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:00.748Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:00.749Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:01.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:01.898154+0000) 2023-08-27T23:26:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:02 smithi093 ceph-mon[111335]: pgmap v4574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3089776319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:02 smithi093 ceph-mon[118902]: pgmap v4574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3089776319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:02.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:02.236Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:02.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:02.237Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:02.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:02 smithi136 ceph-mon[113531]: pgmap v4574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:02.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3089776319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:02.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:02.898376+0000) 2023-08-27T23:26:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:03.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:03.898618+0000) 2023-08-27T23:26:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:04 smithi093 ceph-mon[118902]: pgmap v4575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2279770517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:04 smithi093 ceph-mon[111335]: pgmap v4575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2279770517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:04.899 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:04 smithi136 ceph-mon[113531]: pgmap v4575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:04.899 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2279770517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:04.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:04.898872+0000) 2023-08-27T23:26:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:26:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:26:05] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:26:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:05.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:05.899119+0000) 2023-08-27T23:26:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:06 smithi093 ceph-mon[111335]: pgmap v4576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/230378058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:06 smithi093 ceph-mon[118902]: pgmap v4576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/230378058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:06.899 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:06 smithi136 ceph-mon[113531]: pgmap v4576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:06.899 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/230378058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:06.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:06.899334+0000) 2023-08-27T23:26:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:07.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:07.899612+0000) 2023-08-27T23:26:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:08 smithi093 ceph-mon[111335]: pgmap v4577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:08 smithi093 ceph-mon[118902]: pgmap v4577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:08.900 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:08 smithi136 ceph-mon[113531]: pgmap v4577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:08.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:08.899833+0000) 2023-08-27T23:26:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1552178663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1552178663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:09.900 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1552178663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:09.900 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:09.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:09.900099+0000) 2023-08-27T23:26:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:10 smithi093 ceph-mon[111335]: pgmap v4578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:10 smithi093 ceph-mon[118902]: pgmap v4578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:10.900 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:10 smithi136 ceph-mon[113531]: pgmap v4578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:10.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:10.900332+0000) 2023-08-27T23:26:11.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:10.749Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:10.750Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/29059046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/29059046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:11.900 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/29059046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:11.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:11.900559+0000) 2023-08-27T23:26:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:12 smithi093 ceph-mon[111335]: pgmap v4579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:12 smithi093 ceph-mon[118902]: pgmap v4579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:12.236Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:12.237Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:12.900 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:12 smithi136 ceph-mon[113531]: pgmap v4579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:12.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:12.900766+0000) 2023-08-27T23:26:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778446075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778446075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:13.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778446075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:13.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:13.901021+0000) 2023-08-27T23:26:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:14 smithi093 ceph-mon[111335]: pgmap v4580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:14 smithi093 ceph-mon[118902]: pgmap v4580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:14.901 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:14 smithi136 ceph-mon[113531]: pgmap v4580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:14.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:14.901242+0000) 2023-08-27T23:26:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:26:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:26:15] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:26:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:15.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:15.901515+0000) 2023-08-27T23:26:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:16 smithi093 ceph-mon[118902]: pgmap v4581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/674812699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:16 smithi093 ceph-mon[111335]: pgmap v4581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/674812699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:16.902 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:16 smithi136 ceph-mon[113531]: pgmap v4581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:16.902 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/674812699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:16.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:16.901758+0000) 2023-08-27T23:26:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:17.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:17.902005+0000) 2023-08-27T23:26:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:18 smithi093 ceph-mon[111335]: pgmap v4582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3845894770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:18 smithi093 ceph-mon[118902]: pgmap v4582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3845894770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:18.902 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:18 smithi136 ceph-mon[113531]: pgmap v4582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:18.902 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3845894770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:18.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:18.902289+0000) 2023-08-27T23:26:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:19.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:19.902529+0000) 2023-08-27T23:26:20.750 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:20 smithi093 ceph-mon[111335]: pgmap v4583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:20.750 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:20 smithi093 ceph-mon[118902]: pgmap v4583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:20.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:20 smithi136 ceph-mon[113531]: pgmap v4583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:21.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:20.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:20.902782+0000) 2023-08-27T23:26:21.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:20.750Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:21.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:20.751Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:21.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2908757982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:21.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2908757982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2908757982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:21.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:21.902982+0000) 2023-08-27T23:26:22.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:22.237Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:22.239Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:22.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:22 smithi136 ceph-mon[113531]: pgmap v4584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:22 smithi093 ceph-mon[111335]: pgmap v4584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:22 smithi093 ceph-mon[118902]: pgmap v4584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:22.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:22.903204+0000) 2023-08-27T23:26:23.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3689180578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3689180578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3689180578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:23.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:23.903457+0000) 2023-08-27T23:26:24.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:24 smithi136 ceph-mon[113531]: pgmap v4585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:24.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:24 smithi093 ceph-mon[118902]: pgmap v4585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:24 smithi093 ceph-mon[111335]: pgmap v4585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:24.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:24.903729+0000) 2023-08-27T23:26:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:26:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:26:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:26:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:25.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:25.903978+0000) 2023-08-27T23:26:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:26 smithi136 ceph-mon[113531]: pgmap v4586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/361456041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:26 smithi093 ceph-mon[118902]: pgmap v4586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/361456041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:26 smithi093 ceph-mon[111335]: pgmap v4586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/361456041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:26.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:26.904222+0000) 2023-08-27T23:26:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:27.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:27.904450+0000) 2023-08-27T23:26:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:28 smithi136 ceph-mon[113531]: pgmap v4587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:28.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/414046229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:28 smithi093 ceph-mon[111335]: pgmap v4587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/414046229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:28 smithi093 ceph-mon[118902]: pgmap v4587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/414046229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:28.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:28.904692+0000) 2023-08-27T23:26:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:30 smithi136 ceph-mon[113531]: pgmap v4588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:29.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:29.904940+0000) 2023-08-27T23:26:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:30 smithi093 ceph-mon[111335]: pgmap v4588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:30 smithi093 ceph-mon[118902]: pgmap v4588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:31.062 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:30.750Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:31.062 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:30.751Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3851225612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:30.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:30.905196+0000) 2023-08-27T23:26:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3851225612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3851225612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:32 smithi136 ceph-mon[113531]: pgmap v4589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:31.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:31.905453+0000) 2023-08-27T23:26:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:32 smithi093 ceph-mon[111335]: pgmap v4589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:32 smithi093 ceph-mon[118902]: pgmap v4589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:32.238Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:32.238Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3881872085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:32.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:32.905673+0000) 2023-08-27T23:26:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3881872085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3881872085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:34 smithi136 ceph-mon[113531]: pgmap v4590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:34.827 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:33.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:33.905939+0000) 2023-08-27T23:26:34.831 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:34 smithi093 ceph-mon[111335]: pgmap v4590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:34.832 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:34 smithi093 ceph-mon[118902]: pgmap v4590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1542976815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:34.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:34.906151+0000) 2023-08-27T23:26:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1542976815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:35.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1542976815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:26:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:26:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:26:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:36 smithi136 ceph-mon[113531]: pgmap v4591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:35.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:35.906409+0000) 2023-08-27T23:26:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:36 smithi093 ceph-mon[111335]: pgmap v4591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:36 smithi093 ceph-mon[118902]: pgmap v4591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:36.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:36.906634+0000) 2023-08-27T23:26:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:38 smithi136 ceph-mon[113531]: pgmap v4592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1685146262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:37.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:37.906826+0000) 2023-08-27T23:26:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:38 smithi093 ceph-mon[111335]: pgmap v4592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1685146262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:38 smithi093 ceph-mon[118902]: pgmap v4592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1685146262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:38.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:38.907085+0000) 2023-08-27T23:26:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:40 smithi136 ceph-mon[113531]: pgmap v4593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1514483200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:39.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:39.907303+0000) 2023-08-27T23:26:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:40 smithi093 ceph-mon[118902]: pgmap v4593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1514483200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:40 smithi093 ceph-mon[111335]: pgmap v4593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1514483200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:40.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:40.907511+0000) 2023-08-27T23:26:41.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:40.751Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:40.752Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:42 smithi136 ceph-mon[113531]: pgmap v4594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2616069336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:41.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:41.907773+0000) 2023-08-27T23:26:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:42 smithi093 ceph-mon[111335]: pgmap v4594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2616069336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:42 smithi093 ceph-mon[118902]: pgmap v4594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2616069336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:42.239Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:42.240Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:42.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:42.908070+0000) 2023-08-27T23:26:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:44 smithi136 ceph-mon[113531]: pgmap v4595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:26:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:43.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:43.908282+0000) 2023-08-27T23:26:44.386 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:44 smithi093 ceph-mon[111335]: pgmap v4595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:44.386 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:26:44.386 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:44 smithi093 ceph-mon[118902]: pgmap v4595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:44.387 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:26:45.102 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:44.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:44.908466+0000) 2023-08-27T23:26:45.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2708756827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:45.231 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2708756827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2708756827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:26:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:26:45] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:26:46.135 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:45.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:45.908642+0000) 2023-08-27T23:26:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[118902]: pgmap v4596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[111335]: pgmap v4596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.909 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-mon[113531]: pgmap v4596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:46.909 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.909 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.910 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.910 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.910 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:46.910 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:46.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:46.908835+0000) 2023-08-27T23:26:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2837846483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2837846483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:47.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:47.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:47.883 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2837846483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:47.883 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:47.883 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:47.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:47.909022+0000) 2023-08-27T23:26:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:48 smithi093 ceph-mon[111335]: pgmap v4597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:48 smithi093 ceph-mon[118902]: pgmap v4597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:48.830 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:48 smithi136 ceph-mon[113531]: pgmap v4597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:48.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:48.909178+0000) 2023-08-27T23:26:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:49.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:49.909359+0000) 2023-08-27T23:26:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-mon[113531]: pgmap v4598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:26:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:26:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1751765670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[118902]: pgmap v4598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1751765670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[111335]: pgmap v4598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:26:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:26:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:26:50.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1751765670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:50.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:50.909584+0000) 2023-08-27T23:26:51.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:50.752Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:50.753Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:52 smithi136 ceph-mon[113531]: pgmap v4599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1196089641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:51.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:51.909791+0000) 2023-08-27T23:26:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:52 smithi093 ceph-mon[111335]: pgmap v4599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1196089641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:52 smithi093 ceph-mon[118902]: pgmap v4599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1196089641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:52.239Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:26:52.240Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:26:53.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:52.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:52.910061+0000) 2023-08-27T23:26:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:54 smithi136 ceph-mon[113531]: pgmap v4600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1107038771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:53.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:53.910297+0000) 2023-08-27T23:26:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:54 smithi093 ceph-mon[111335]: pgmap v4600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1107038771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:54 smithi093 ceph-mon[118902]: pgmap v4600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:26:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1107038771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:54.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:54.910539+0000) 2023-08-27T23:26:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:26:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:26:55] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:26:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:56 smithi136 ceph-mon[113531]: pgmap v4601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:55.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:55.910754+0000) 2023-08-27T23:26:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:56 smithi093 ceph-mon[111335]: pgmap v4601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:56 smithi093 ceph-mon[118902]: pgmap v4601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2170540898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:56.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:56.910979+0000) 2023-08-27T23:26:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2170540898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2170540898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:58 smithi136 ceph-mon[113531]: pgmap v4602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:57.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:57.911244+0000) 2023-08-27T23:26:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:58 smithi093 ceph-mon[111335]: pgmap v4602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:58 smithi093 ceph-mon[118902]: pgmap v4602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:26:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:26:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1433512520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:59.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:58.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:58.911438+0000) 2023-08-27T23:26:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:26:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1433512520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:26:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:26:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1433512520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:00 smithi136 ceph-mon[113531]: pgmap v4603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:26:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:26:59.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:26:59.911688+0000) 2023-08-27T23:27:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:00 smithi093 ceph-mon[111335]: pgmap v4603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:00 smithi093 ceph-mon[118902]: pgmap v4603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:00.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:00.911915+0000) 2023-08-27T23:27:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:00.752Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:00.753Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:02 smithi136 ceph-mon[113531]: pgmap v4604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3713432010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:01.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:01.912123+0000) 2023-08-27T23:27:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:02 smithi093 ceph-mon[111335]: pgmap v4604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3713432010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:02.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:02 smithi093 ceph-mon[118902]: pgmap v4604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:02.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3713432010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:02.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:02.240Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:02.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:02.241Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:02.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:02.912337+0000) 2023-08-27T23:27:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:04 smithi136 ceph-mon[113531]: pgmap v4605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3153489980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:03.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:03.912600+0000) 2023-08-27T23:27:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:04 smithi093 ceph-mon[111335]: pgmap v4605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3153489980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:04 smithi093 ceph-mon[118902]: pgmap v4605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3153489980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:04.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:04.912815+0000) 2023-08-27T23:27:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:27:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:27:05] "GET /metrics HTTP/1.1" 200 239915 "" "Prometheus/2.43.0" 2023-08-27T23:27:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:06 smithi136 ceph-mon[113531]: pgmap v4606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1063698832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:05.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:05.913105+0000) 2023-08-27T23:27:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:06 smithi093 ceph-mon[111335]: pgmap v4606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1063698832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:06 smithi093 ceph-mon[118902]: pgmap v4606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1063698832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:06.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:06.913362+0000) 2023-08-27T23:27:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:08 smithi136 ceph-mon[113531]: pgmap v4607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:07.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:07.913576+0000) 2023-08-27T23:27:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:08 smithi093 ceph-mon[111335]: pgmap v4607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:08 smithi093 ceph-mon[118902]: pgmap v4607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/257197486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:08.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:08.913832+0000) 2023-08-27T23:27:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/257197486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/257197486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:10 smithi136 ceph-mon[113531]: pgmap v4608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:09.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:09.914074+0000) 2023-08-27T23:27:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:10 smithi093 ceph-mon[111335]: pgmap v4608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:10 smithi093 ceph-mon[118902]: pgmap v4608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:11.120 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:10.753Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:11.120 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:10.754Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3640812394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:10.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:10.914293+0000) 2023-08-27T23:27:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3640812394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3640812394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:12 smithi136 ceph-mon[113531]: pgmap v4609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:11.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:11.914510+0000) 2023-08-27T23:27:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:12 smithi093 ceph-mon[111335]: pgmap v4609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:12 smithi093 ceph-mon[118902]: pgmap v4609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:12.241Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:12.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2010581450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:12.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:12.914764+0000) 2023-08-27T23:27:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2010581450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2010581450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:14 smithi136 ceph-mon[113531]: pgmap v4610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:13.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:13.915011+0000) 2023-08-27T23:27:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:14 smithi093 ceph-mon[111335]: pgmap v4610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:14 smithi093 ceph-mon[118902]: pgmap v4610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:14.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:14.915328+0000) 2023-08-27T23:27:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:27:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:27:15] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:27:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:16 smithi136 ceph-mon[113531]: pgmap v4611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1651212199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:15.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:15.915596+0000) 2023-08-27T23:27:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:16 smithi093 ceph-mon[111335]: pgmap v4611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1651212199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:16 smithi093 ceph-mon[118902]: pgmap v4611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1651212199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:16.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:16.915841+0000) 2023-08-27T23:27:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:18 smithi136 ceph-mon[113531]: pgmap v4612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3590379696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:17.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:17.916111+0000) 2023-08-27T23:27:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:18 smithi093 ceph-mon[118902]: pgmap v4612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3590379696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:18 smithi093 ceph-mon[111335]: pgmap v4612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3590379696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:19.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:18.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:18.916300+0000) 2023-08-27T23:27:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:20 smithi136 ceph-mon[113531]: pgmap v4613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:19.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:19.916553+0000) 2023-08-27T23:27:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:20 smithi093 ceph-mon[111335]: pgmap v4613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:20 smithi093 ceph-mon[118902]: pgmap v4613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:21.143 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:20.754Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:21.143 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:20.755Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/717129879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:20.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:20.916857+0000) 2023-08-27T23:27:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/717129879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/717129879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:22 smithi136 ceph-mon[113531]: pgmap v4614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:21.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:21.917125+0000) 2023-08-27T23:27:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:22 smithi093 ceph-mon[111335]: pgmap v4614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:22 smithi093 ceph-mon[118902]: pgmap v4614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:22.242Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:22.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3985751799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:22.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:22.917346+0000) 2023-08-27T23:27:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3985751799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3985751799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:24 smithi136 ceph-mon[113531]: pgmap v4615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:23.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:23.917560+0000) 2023-08-27T23:27:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:24 smithi093 ceph-mon[111335]: pgmap v4615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:24 smithi093 ceph-mon[118902]: pgmap v4615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2833387135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:24.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:24.917825+0000) 2023-08-27T23:27:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2833387135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2833387135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:27:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:27:25] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:27:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:26 smithi136 ceph-mon[113531]: pgmap v4616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:25.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:25.918072+0000) 2023-08-27T23:27:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:26 smithi093 ceph-mon[111335]: pgmap v4616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:26 smithi093 ceph-mon[118902]: pgmap v4616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:27.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:26.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:26.918235+0000) 2023-08-27T23:27:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:28 smithi136 ceph-mon[113531]: pgmap v4617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2179925825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:27.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:27.918432+0000) 2023-08-27T23:27:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:28 smithi093 ceph-mon[118902]: pgmap v4617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2179925825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:28 smithi093 ceph-mon[111335]: pgmap v4617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2179925825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:28.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:28.918684+0000) 2023-08-27T23:27:30.169 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:29.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:29.918963+0000) 2023-08-27T23:27:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:30 smithi136 ceph-mon[113531]: pgmap v4618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/639857446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:30 smithi093 ceph-mon[111335]: pgmap v4618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/639857446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:30 smithi093 ceph-mon[118902]: pgmap v4618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/639857446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:30.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:30.919205+0000) 2023-08-27T23:27:31.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:30.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:30.755Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:32.171 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:31.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:31.919446+0000) 2023-08-27T23:27:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:32 smithi136 ceph-mon[113531]: pgmap v4619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2771413668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:32 smithi093 ceph-mon[118902]: pgmap v4619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2771413668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:32 smithi093 ceph-mon[111335]: pgmap v4619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:32.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2771413668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:32.242Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:32.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:32.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:32.919745+0000) 2023-08-27T23:27:34.175 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:33.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:33.920012+0000) 2023-08-27T23:27:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:34 smithi136 ceph-mon[113531]: pgmap v4620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:34 smithi093 ceph-mon[111335]: pgmap v4620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:34 smithi093 ceph-mon[118902]: pgmap v4620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:35.184 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:34.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:34.920211+0000) 2023-08-27T23:27:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1750923350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1750923350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1750923350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:27:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:27:35] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:27:36.185 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:35.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:35.920500+0000) 2023-08-27T23:27:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:36 smithi136 ceph-mon[113531]: pgmap v4621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:36 smithi093 ceph-mon[111335]: pgmap v4621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:36 smithi093 ceph-mon[118902]: pgmap v4621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:36.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:36.920780+0000) 2023-08-27T23:27:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/357600564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/357600564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/357600564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:37.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:37.920998+0000) 2023-08-27T23:27:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:38 smithi093 ceph-mon[111335]: pgmap v4622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:38 smithi093 ceph-mon[118902]: pgmap v4622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:38 smithi136 ceph-mon[113531]: pgmap v4622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:38.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:38.921252+0000) 2023-08-27T23:27:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:39.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:39.921498+0000) 2023-08-27T23:27:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:40 smithi093 ceph-mon[111335]: pgmap v4623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1831138583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:40 smithi093 ceph-mon[118902]: pgmap v4623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1831138583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:40 smithi136 ceph-mon[113531]: pgmap v4623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1831138583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:40.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:40.921767+0000) 2023-08-27T23:27:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:40.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:40.756Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:41.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:41.922052+0000) 2023-08-27T23:27:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:42 smithi093 ceph-mon[111335]: pgmap v4624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3936235381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:42 smithi093 ceph-mon[118902]: pgmap v4624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3936235381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:42.243Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:42.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:42 smithi136 ceph-mon[113531]: pgmap v4624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3936235381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:42.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:42.922273+0000) 2023-08-27T23:27:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:43.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:43.922518+0000) 2023-08-27T23:27:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:44 smithi093 ceph-mon[111335]: pgmap v4625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/175517745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:44 smithi093 ceph-mon[118902]: pgmap v4625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/175517745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:44 smithi136 ceph-mon[113531]: pgmap v4625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/175517745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:44.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:44.922788+0000) 2023-08-27T23:27:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:27:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:27:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:27:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:45.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:45.923030+0000) 2023-08-27T23:27:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:46 smithi093 ceph-mon[118902]: pgmap v4626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:46 smithi093 ceph-mon[111335]: pgmap v4626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:46 smithi136 ceph-mon[113531]: pgmap v4626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:46.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:46.923305+0000) 2023-08-27T23:27:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/499221396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/499221396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/499221396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:47.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:47.923552+0000) 2023-08-27T23:27:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:48 smithi093 ceph-mon[111335]: pgmap v4627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:48 smithi093 ceph-mon[118902]: pgmap v4627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:48 smithi136 ceph-mon[113531]: pgmap v4627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:49.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:48.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:48.923737+0000) 2023-08-27T23:27:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2806231728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:27:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2806231728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:27:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2806231728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:27:50.129 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:49.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:49.923934+0000) 2023-08-27T23:27:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:50 smithi136 ceph-mon[113531]: pgmap v4628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:50 smithi093 ceph-mon[111335]: pgmap v4628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:50 smithi093 ceph-mon[118902]: pgmap v4628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:50.924 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:50.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:50.924117+0000) 2023-08-27T23:27:51.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:50.756Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:50.757Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:51.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:51.924262+0000) 2023-08-27T23:27:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:52 smithi136 ceph-mon[113531]: pgmap v4629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3412685483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[111335]: pgmap v4629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3412685483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[118902]: pgmap v4629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:52.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3412685483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:52.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:52.245Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:27:52.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:27:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:27:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:27:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:52.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:52.924458+0000) 2023-08-27T23:27:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:27:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:27:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:27:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:27:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:27:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:54 smithi136 ceph-mon[113531]: pgmap v4630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2032449808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:53.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:53.924759+0000) 2023-08-27T23:27:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:54 smithi093 ceph-mon[111335]: pgmap v4630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2032449808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:54 smithi093 ceph-mon[118902]: pgmap v4630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2032449808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:27:55.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:54.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:54.924998+0000) 2023-08-27T23:27:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:27:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:27:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:27:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:56 smithi136 ceph-mon[113531]: pgmap v4631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2455369772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:55.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:55.925263+0000) 2023-08-27T23:27:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:56 smithi093 ceph-mon[111335]: pgmap v4631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2455369772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:56 smithi093 ceph-mon[118902]: pgmap v4631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2455369772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:57.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:56.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:56.925717+0000) 2023-08-27T23:27:58.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:58 smithi136 ceph-mon[113531]: pgmap v4632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:57.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:57.925971+0000) 2023-08-27T23:27:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:58 smithi093 ceph-mon[111335]: pgmap v4632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:58 smithi093 ceph-mon[118902]: pgmap v4632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:27:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:27:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3405315816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:58.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:58.926241+0000) 2023-08-27T23:27:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:27:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3405315816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:27:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:27:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3405315816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:00 smithi136 ceph-mon[113531]: pgmap v4633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:27:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:27:59.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:27:59.926526+0000) 2023-08-27T23:28:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:00 smithi093 ceph-mon[111335]: pgmap v4633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:00 smithi093 ceph-mon[118902]: pgmap v4633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:01.085 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:00.757Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:01.085 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:00.758Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2726242145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:00.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:00.926781+0000) 2023-08-27T23:28:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2726242145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2726242145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:02 smithi136 ceph-mon[113531]: pgmap v4634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:01.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:01.926987+0000) 2023-08-27T23:28:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:02 smithi093 ceph-mon[111335]: pgmap v4634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:02 smithi093 ceph-mon[118902]: pgmap v4634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:02.246Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:02.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:02.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:02.927251+0000) 2023-08-27T23:28:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:03.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:03.927504+0000) 2023-08-27T23:28:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:04 smithi136 ceph-mon[113531]: pgmap v4635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2010320028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:04 smithi093 ceph-mon[111335]: pgmap v4635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2010320028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:04 smithi093 ceph-mon[118902]: pgmap v4635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2010320028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:04.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:04.927778+0000) 2023-08-27T23:28:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:28:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:28:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:28:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:06 smithi136 ceph-mon[113531]: pgmap v4636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/350346005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:05.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:05.927987+0000) 2023-08-27T23:28:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:06 smithi093 ceph-mon[111335]: pgmap v4636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/350346005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:06 smithi093 ceph-mon[118902]: pgmap v4636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/350346005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:07.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:06.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:06.928193+0000) 2023-08-27T23:28:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:08 smithi136 ceph-mon[113531]: pgmap v4637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:08.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2283782102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:07.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:07.928469+0000) 2023-08-27T23:28:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:08 smithi093 ceph-mon[111335]: pgmap v4637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2283782102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:08 smithi093 ceph-mon[118902]: pgmap v4637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2283782102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:08.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:08.928800+0000) 2023-08-27T23:28:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:10 smithi136 ceph-mon[113531]: pgmap v4638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:09.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:09.929015+0000) 2023-08-27T23:28:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:10 smithi093 ceph-mon[111335]: pgmap v4638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:10 smithi093 ceph-mon[118902]: pgmap v4638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:11.108 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:10.757Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:11.108 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:10.758Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2394596880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:10.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:10.929299+0000) 2023-08-27T23:28:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2394596880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2394596880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:12 smithi136 ceph-mon[113531]: pgmap v4639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:11.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:11.929561+0000) 2023-08-27T23:28:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:12 smithi093 ceph-mon[111335]: pgmap v4639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:12 smithi093 ceph-mon[118902]: pgmap v4639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:12.246Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:12.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/741006644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:12.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:12.929845+0000) 2023-08-27T23:28:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/741006644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/741006644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:14 smithi136 ceph-mon[113531]: pgmap v4640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:13.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:13.930080+0000) 2023-08-27T23:28:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:14 smithi093 ceph-mon[111335]: pgmap v4640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:14 smithi093 ceph-mon[118902]: pgmap v4640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3270220811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:15.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:14.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:14.930332+0000) 2023-08-27T23:28:15.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3270220811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:15.666 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3270220811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:15.666 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:28:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:28:15] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:28:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:16 smithi136 ceph-mon[113531]: pgmap v4641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:15.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:15.930632+0000) 2023-08-27T23:28:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:16 smithi093 ceph-mon[111335]: pgmap v4641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:16 smithi093 ceph-mon[118902]: pgmap v4641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:17.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:16.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:16.930914+0000) 2023-08-27T23:28:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:18 smithi136 ceph-mon[113531]: pgmap v4642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3195679293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:17.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:17.931203+0000) 2023-08-27T23:28:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:18 smithi093 ceph-mon[111335]: pgmap v4642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3195679293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:18 smithi093 ceph-mon[118902]: pgmap v4642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3195679293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:18.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:18.931481+0000) 2023-08-27T23:28:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:20 smithi136 ceph-mon[113531]: pgmap v4643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2459459828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:19.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:19.931770+0000) 2023-08-27T23:28:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:20 smithi093 ceph-mon[111335]: pgmap v4643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2459459828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:20 smithi093 ceph-mon[118902]: pgmap v4643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:20.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2459459828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:20.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:20.932029+0000) 2023-08-27T23:28:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:20.758Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:20.759Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:22 smithi136 ceph-mon[113531]: pgmap v4644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:21.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:21.932318+0000) 2023-08-27T23:28:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:22 smithi093 ceph-mon[111335]: pgmap v4644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:22 smithi093 ceph-mon[118902]: pgmap v4644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:22.247Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:22.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1500396945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:22.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:22.932519+0000) 2023-08-27T23:28:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1500396945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1500396945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:24 smithi136 ceph-mon[113531]: pgmap v4645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:23.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:23.932740+0000) 2023-08-27T23:28:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:24 smithi093 ceph-mon[118902]: pgmap v4645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:24 smithi093 ceph-mon[111335]: pgmap v4645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/793762250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:25.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:24.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:24.932995+0000) 2023-08-27T23:28:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/793762250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/793762250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:28:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:28:25] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:28:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:26 smithi136 ceph-mon[113531]: pgmap v4646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:25.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:25.933217+0000) 2023-08-27T23:28:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:26 smithi093 ceph-mon[111335]: pgmap v4646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:26 smithi093 ceph-mon[118902]: pgmap v4646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3689562147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:26.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:26.933436+0000) 2023-08-27T23:28:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3689562147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3689562147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:28 smithi136 ceph-mon[113531]: pgmap v4647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:27.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:27.933696+0000) 2023-08-27T23:28:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:28 smithi093 ceph-mon[111335]: pgmap v4647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:28 smithi093 ceph-mon[118902]: pgmap v4647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:28.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:28.933959+0000) 2023-08-27T23:28:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:30 smithi136 ceph-mon[113531]: pgmap v4648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3544551668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:29.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:29.934222+0000) 2023-08-27T23:28:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:30 smithi093 ceph-mon[111335]: pgmap v4648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3544551668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:30 smithi093 ceph-mon[118902]: pgmap v4648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3544551668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:31.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:30.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:30.934482+0000) 2023-08-27T23:28:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:30.759Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:30.760Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:32 smithi136 ceph-mon[113531]: pgmap v4649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1559000502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:31.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:31.934748+0000) 2023-08-27T23:28:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:32 smithi093 ceph-mon[111335]: pgmap v4649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1559000502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:32 smithi093 ceph-mon[118902]: pgmap v4649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1559000502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:32.248Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:32.249Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:32.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:32.935079+0000) 2023-08-27T23:28:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:34 smithi136 ceph-mon[113531]: pgmap v4650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2394973811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:33.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:33.935359+0000) 2023-08-27T23:28:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:34 smithi093 ceph-mon[111335]: pgmap v4650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2394973811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:34 smithi093 ceph-mon[118902]: pgmap v4650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2394973811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:35.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:34.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:34.935640+0000) 2023-08-27T23:28:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:28:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:28:35] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:28:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:36 smithi136 ceph-mon[113531]: pgmap v4651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:35.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:35.935947+0000) 2023-08-27T23:28:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:36 smithi093 ceph-mon[111335]: pgmap v4651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:36.755 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:36 smithi093 ceph-mon[118902]: pgmap v4651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/320789182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:36.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:36.936151+0000) 2023-08-27T23:28:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/320789182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/320789182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:38 smithi136 ceph-mon[113531]: pgmap v4652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:37.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:37.936403+0000) 2023-08-27T23:28:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:38 smithi093 ceph-mon[111335]: pgmap v4652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:38 smithi093 ceph-mon[118902]: pgmap v4652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:39.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:38.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:38.936647+0000) 2023-08-27T23:28:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/449556547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/449556547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/449556547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:40 smithi136 ceph-mon[113531]: pgmap v4653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:39.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:39.936903+0000) 2023-08-27T23:28:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:40 smithi093 ceph-mon[111335]: pgmap v4653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:40 smithi093 ceph-mon[118902]: pgmap v4653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:41.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:40.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:40.937178+0000) 2023-08-27T23:28:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:40.759Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:40.760Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:42.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:41.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:41.937407+0000) 2023-08-27T23:28:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:42 smithi136 ceph-mon[113531]: pgmap v4654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3183563211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:42 smithi093 ceph-mon[111335]: pgmap v4654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3183563211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:42 smithi093 ceph-mon[118902]: pgmap v4654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3183563211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:42.249Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:42.250Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:43.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:42.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:42.937691+0000) 2023-08-27T23:28:44.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:43.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:43.937961+0000) 2023-08-27T23:28:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:44 smithi093 ceph-mon[118902]: pgmap v4655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3627393159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:44 smithi093 ceph-mon[111335]: pgmap v4655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3627393159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:44 smithi136 ceph-mon[113531]: pgmap v4655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3627393159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:45.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:44.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:44.938238+0000) 2023-08-27T23:28:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:28:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:28:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:28:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:45.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:45.938538+0000) 2023-08-27T23:28:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:46 smithi093 ceph-mon[111335]: pgmap v4656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1934493547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:46 smithi093 ceph-mon[118902]: pgmap v4656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1934493547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:46 smithi136 ceph-mon[113531]: pgmap v4656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1934493547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:47.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:46.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:46.938740+0000) 2023-08-27T23:28:48.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:47.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:47.938916+0000) 2023-08-27T23:28:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:48 smithi093 ceph-mon[111335]: pgmap v4657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:48 smithi093 ceph-mon[118902]: pgmap v4657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:48 smithi136 ceph-mon[113531]: pgmap v4657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:49.202 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:48.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:48.939175+0000) 2023-08-27T23:28:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2910140534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2910140534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2910140534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:50.205 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:49.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:49.939458+0000) 2023-08-27T23:28:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:50 smithi093 ceph-mon[111335]: pgmap v4658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:50 smithi093 ceph-mon[118902]: pgmap v4658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:50 smithi136 ceph-mon[113531]: pgmap v4658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:51.211 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:50.760Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:51.211 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:50.761Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:51.211 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:50.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:50.939707+0000) 2023-08-27T23:28:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4133521591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4133521591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4133521591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:52.212 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:51.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:51.939948+0000) 2023-08-27T23:28:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:52 smithi093 ceph-mon[111335]: pgmap v4659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:28:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:52 smithi093 ceph-mon[118902]: pgmap v4659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:28:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:52.250Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:28:52.250Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:28:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:52 smithi136 ceph-mon[113531]: pgmap v4659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:28:53.133 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:52.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:52.940102+0000) 2023-08-27T23:28:53.219 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2151566593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2151566593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2151566593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:54.222 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:53.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:53.940249+0000) 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[111335]: pgmap v4660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[118902]: pgmap v4660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:54 smithi136 ceph-mon[113531]: pgmap v4660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:28:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:54.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:54.940408+0000) 2023-08-27T23:28:55.660 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:28:55] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:28:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-mon[113531]: pgmap v4661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:28:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:28:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4067359506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[111335]: pgmap v4661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4067359506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[118902]: pgmap v4661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:55.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:28:55.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:28:55.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:28:55.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4067359506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:55.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:55.940576+0000) 2023-08-27T23:28:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:56.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:56.940868+0000) 2023-08-27T23:28:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:57 smithi136 ceph-mon[113531]: pgmap v4662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:57.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:57 smithi093 ceph-mon[111335]: pgmap v4662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:57 smithi093 ceph-mon[118902]: pgmap v4662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:57.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:57.941162+0000) 2023-08-27T23:28:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1073859565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1073859565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1073859565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:28:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:58.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:58.941373+0000) 2023-08-27T23:28:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:28:59 smithi136 ceph-mon[113531]: pgmap v4663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:59.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:28:59 smithi093 ceph-mon[111335]: pgmap v4663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:28:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:28:59 smithi093 ceph-mon[118902]: pgmap v4663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:28:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:28:59.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:28:59.941619+0000) 2023-08-27T23:29:01.081 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:00.761Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:01.082 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:00.762Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/317559473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:00.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:00.941915+0000) 2023-08-27T23:29:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/317559473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/317559473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:02 smithi136 ceph-mon[113531]: pgmap v4664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:01.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:01.942209+0000) 2023-08-27T23:29:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:02 smithi093 ceph-mon[111335]: pgmap v4664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:02 smithi093 ceph-mon[118902]: pgmap v4664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:02.250Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:02.251Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/89168261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:02.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:02.942435+0000) 2023-08-27T23:29:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/89168261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/89168261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:04 smithi136 ceph-mon[113531]: pgmap v4665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:03.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:03.942622+0000) 2023-08-27T23:29:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:04 smithi093 ceph-mon[111335]: pgmap v4665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:04 smithi093 ceph-mon[118902]: pgmap v4665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:04.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:04.942863+0000) 2023-08-27T23:29:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:29:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:29:05] "GET /metrics HTTP/1.1" 200 239917 "" "Prometheus/2.43.0" 2023-08-27T23:29:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:06 smithi136 ceph-mon[113531]: pgmap v4666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1575604110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:05.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:05.943164+0000) 2023-08-27T23:29:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:06 smithi093 ceph-mon[111335]: pgmap v4666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1575604110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:06 smithi093 ceph-mon[118902]: pgmap v4666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1575604110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:06.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:06.943413+0000) 2023-08-27T23:29:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:08 smithi136 ceph-mon[113531]: pgmap v4667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4276639001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:07.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:07.943684+0000) 2023-08-27T23:29:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:08 smithi093 ceph-mon[111335]: pgmap v4667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4276639001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:08 smithi093 ceph-mon[118902]: pgmap v4667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4276639001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:08.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:08.943944+0000) 2023-08-27T23:29:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:10 smithi136 ceph-mon[113531]: pgmap v4668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2519782294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:09.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:09.944181+0000) 2023-08-27T23:29:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:10 smithi093 ceph-mon[111335]: pgmap v4668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2519782294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:10 smithi093 ceph-mon[118902]: pgmap v4668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2519782294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:10.761Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:10.763Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:10.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:10.944394+0000) 2023-08-27T23:29:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:12 smithi136 ceph-mon[113531]: pgmap v4669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:11.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:11.944629+0000) 2023-08-27T23:29:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:12 smithi093 ceph-mon[111335]: pgmap v4669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:12 smithi093 ceph-mon[118902]: pgmap v4669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:12.251Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:12.252Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/262729687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:12.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:12.944892+0000) 2023-08-27T23:29:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/262729687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/262729687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:14 smithi136 ceph-mon[113531]: pgmap v4670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:13.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:13.945131+0000) 2023-08-27T23:29:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:14 smithi093 ceph-mon[111335]: pgmap v4670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:14 smithi093 ceph-mon[118902]: pgmap v4670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3645152255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:14.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:14.945322+0000) 2023-08-27T23:29:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3645152255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:29:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:29:15] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:29:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3645152255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:16 smithi136 ceph-mon[113531]: pgmap v4671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:15.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:15.945562+0000) 2023-08-27T23:29:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:16 smithi093 ceph-mon[111335]: pgmap v4671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:16 smithi093 ceph-mon[118902]: pgmap v4671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2618985052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:16.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:16.945817+0000) 2023-08-27T23:29:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2618985052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2618985052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:18 smithi136 ceph-mon[113531]: pgmap v4672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:17.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:17.946131+0000) 2023-08-27T23:29:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:18 smithi093 ceph-mon[111335]: pgmap v4672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:18 smithi093 ceph-mon[118902]: pgmap v4672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:18.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:18.946415+0000) 2023-08-27T23:29:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:20 smithi136 ceph-mon[113531]: pgmap v4673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:20.764 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3272276669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:20.764 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:19.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:19.946655+0000) 2023-08-27T23:29:20.765 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:20 smithi093 ceph-mon[111335]: pgmap v4673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:20.765 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3272276669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:20.765 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:20 smithi093 ceph-mon[118902]: pgmap v4673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:20.765 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3272276669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:21.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:20.763Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:20.764Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:20.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:20.946915+0000) 2023-08-27T23:29:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:21.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:21.947186+0000) 2023-08-27T23:29:22.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:22 smithi136 ceph-mon[113531]: pgmap v4674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:22.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3736686305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:22 smithi093 ceph-mon[111335]: pgmap v4674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3736686305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:22 smithi093 ceph-mon[118902]: pgmap v4674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3736686305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:22.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:22.251Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:22.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:22.252Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:22.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:22.947407+0000) 2023-08-27T23:29:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:24 smithi136 ceph-mon[113531]: pgmap v4675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:23.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:23.947664+0000) 2023-08-27T23:29:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:24 smithi093 ceph-mon[111335]: pgmap v4675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:24 smithi093 ceph-mon[118902]: pgmap v4675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/51730158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:24.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:24.947973+0000) 2023-08-27T23:29:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/51730158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:29:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:29:25] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:29:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/51730158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:26 smithi136 ceph-mon[113531]: pgmap v4676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:25.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:25.948314+0000) 2023-08-27T23:29:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:26 smithi093 ceph-mon[111335]: pgmap v4676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:26 smithi093 ceph-mon[118902]: pgmap v4676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3089133355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:26.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:26.948527+0000) 2023-08-27T23:29:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3089133355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3089133355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:28 smithi136 ceph-mon[113531]: pgmap v4677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:27.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:27.948835+0000) 2023-08-27T23:29:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:28 smithi093 ceph-mon[111335]: pgmap v4677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:28 smithi093 ceph-mon[118902]: pgmap v4677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/708525095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:28.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:28.949123+0000) 2023-08-27T23:29:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/708525095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/708525095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:30 smithi136 ceph-mon[113531]: pgmap v4678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:29.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:29.949360+0000) 2023-08-27T23:29:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:30 smithi093 ceph-mon[111335]: pgmap v4678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:30 smithi093 ceph-mon[118902]: pgmap v4678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:31.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:30.764Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:30.764Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:30.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:30.949553+0000) 2023-08-27T23:29:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:32 smithi136 ceph-mon[113531]: pgmap v4679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3258996302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:31.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:31.949792+0000) 2023-08-27T23:29:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:32 smithi093 ceph-mon[111335]: pgmap v4679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3258996302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:32 smithi093 ceph-mon[118902]: pgmap v4679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3258996302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:32.252Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:32.253Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:32.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:32.950061+0000) 2023-08-27T23:29:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:34 smithi136 ceph-mon[113531]: pgmap v4680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3919017516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:33.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:33.950329+0000) 2023-08-27T23:29:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:34 smithi093 ceph-mon[111335]: pgmap v4680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3919017516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:34 smithi093 ceph-mon[118902]: pgmap v4680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3919017516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:34.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:34.950619+0000) 2023-08-27T23:29:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:29:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:29:35] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:29:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:36 smithi136 ceph-mon[113531]: pgmap v4681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/444551359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:35.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:35.950958+0000) 2023-08-27T23:29:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:36 smithi093 ceph-mon[111335]: pgmap v4681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/444551359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:36 smithi093 ceph-mon[118902]: pgmap v4681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/444551359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:36.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:36.951218+0000) 2023-08-27T23:29:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:38 smithi136 ceph-mon[113531]: pgmap v4682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:37.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:37.951467+0000) 2023-08-27T23:29:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:38 smithi093 ceph-mon[111335]: pgmap v4682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:38 smithi093 ceph-mon[118902]: pgmap v4682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1233254957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:38.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:38.951758+0000) 2023-08-27T23:29:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1233254957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1233254957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:40 smithi136 ceph-mon[113531]: pgmap v4683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:39.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:39.952019+0000) 2023-08-27T23:29:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:40 smithi093 ceph-mon[111335]: pgmap v4683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:40 smithi093 ceph-mon[118902]: pgmap v4683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:41.187 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:40.764Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:41.188 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:40.765Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2892458530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:40.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:40.952311+0000) 2023-08-27T23:29:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2892458530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2892458530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:42 smithi136 ceph-mon[113531]: pgmap v4684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:41.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:41.952553+0000) 2023-08-27T23:29:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:42 smithi093 ceph-mon[111335]: pgmap v4684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:42 smithi093 ceph-mon[118902]: pgmap v4684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:42.253Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:42.254Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:42.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:42.952871+0000) 2023-08-27T23:29:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:44 smithi136 ceph-mon[113531]: pgmap v4685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4079961809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:43.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:43.953168+0000) 2023-08-27T23:29:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:44 smithi093 ceph-mon[111335]: pgmap v4685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4079961809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:44 smithi093 ceph-mon[118902]: pgmap v4685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4079961809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:44.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:44.953453+0000) 2023-08-27T23:29:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:29:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:29:45] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:29:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:46 smithi136 ceph-mon[113531]: pgmap v4686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3164113029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:45.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:45.953717+0000) 2023-08-27T23:29:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:46 smithi093 ceph-mon[111335]: pgmap v4686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3164113029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:46 smithi093 ceph-mon[118902]: pgmap v4686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3164113029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:46.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:46.954012+0000) 2023-08-27T23:29:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:48 smithi136 ceph-mon[113531]: pgmap v4687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2698066925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:47.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:47.954333+0000) 2023-08-27T23:29:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:48 smithi093 ceph-mon[111335]: pgmap v4687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2698066925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:48 smithi093 ceph-mon[118902]: pgmap v4687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2698066925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:48.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:48.954605+0000) 2023-08-27T23:29:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:50 smithi136 ceph-mon[113531]: pgmap v4688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:49.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:49.954884+0000) 2023-08-27T23:29:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:50 smithi093 ceph-mon[111335]: pgmap v4688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:50 smithi093 ceph-mon[118902]: pgmap v4688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:51.208 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:50.765Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:51.208 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:50.766Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:51.209 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:50.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:50.955121+0000) 2023-08-27T23:29:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1769234653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1769234653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1769234653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:52.209 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:51.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:51.955320+0000) 2023-08-27T23:29:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:52 smithi093 ceph-mon[111335]: pgmap v4689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:52 smithi093 ceph-mon[118902]: pgmap v4689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:52.253Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:29:52.254Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:29:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:52 smithi136 ceph-mon[113531]: pgmap v4689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:53.215 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:52.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:52.955532+0000) 2023-08-27T23:29:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/746835239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:53.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/746835239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/746835239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:54.216 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:53.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:53.955726+0000) 2023-08-27T23:29:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:54 smithi093 ceph-mon[111335]: pgmap v4690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:54 smithi093 ceph-mon[118902]: pgmap v4690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:54 smithi136 ceph-mon[113531]: pgmap v4690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:29:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:54.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:54.955943+0000) 2023-08-27T23:29:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:29:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:29:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:29:56.086 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:55.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:55.956121+0000) 2023-08-27T23:29:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:56 smithi093 ceph-mon[111335]: pgmap v4691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1114729521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:29:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:56 smithi093 ceph-mon[118902]: pgmap v4691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:56.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1114729521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:56.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:29:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:56 smithi136 ceph-mon[113531]: pgmap v4691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1114729521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:29:57.404 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:56.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:56.956269+0000) 2023-08-27T23:29:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:57.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:57.956466+0000) 2023-08-27T23:29:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:58 smithi136 ceph-mon[113531]: pgmap v4692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3538198772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[111335]: pgmap v4692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3538198772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[118902]: pgmap v4692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:29:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3538198772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:29:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:58.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:58.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:58.956727+0000) 2023-08-27T23:29:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:29:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:29:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:29:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:29:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:29:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:29:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:29:59.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:30:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:29:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:29:59.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:29:59.956956+0000) 2023-08-27T23:30:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:00 smithi136 ceph-mon[113531]: pgmap v4693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2721488265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:30:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-mon[111335]: pgmap v4693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2721488265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:30:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-mon[118902]: pgmap v4693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2721488265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:30:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:00.766Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:00.768Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:00.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:00.957207+0000) 2023-08-27T23:30:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:02 smithi136 ceph-mon[113531]: pgmap v4694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:01.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:01.957441+0000) 2023-08-27T23:30:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:02 smithi093 ceph-mon[111335]: pgmap v4694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:02 smithi093 ceph-mon[118902]: pgmap v4694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:02.253Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:02.254Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2058508347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:02.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:02.957722+0000) 2023-08-27T23:30:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2058508347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2058508347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:04.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:04 smithi136 ceph-mon[113531]: pgmap v4695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:04.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:03.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:03.957967+0000) 2023-08-27T23:30:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:04 smithi093 ceph-mon[111335]: pgmap v4695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:04 smithi093 ceph-mon[118902]: pgmap v4695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2913258158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2913258158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2913258158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:04.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:04.958213+0000) 2023-08-27T23:30:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:30:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:30:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:30:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:06 smithi136 ceph-mon[113531]: pgmap v4696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:05.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:05.958429+0000) 2023-08-27T23:30:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:06 smithi093 ceph-mon[111335]: pgmap v4696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:06 smithi093 ceph-mon[118902]: pgmap v4696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3006993556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:06.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:06.958653+0000) 2023-08-27T23:30:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3006993556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3006993556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:08 smithi136 ceph-mon[113531]: pgmap v4697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:07.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:07.958905+0000) 2023-08-27T23:30:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:08 smithi093 ceph-mon[111335]: pgmap v4697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:08 smithi093 ceph-mon[118902]: pgmap v4697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:08.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:08.959186+0000) 2023-08-27T23:30:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:10 smithi136 ceph-mon[113531]: pgmap v4698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3270520799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:09.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:09.959426+0000) 2023-08-27T23:30:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:10 smithi093 ceph-mon[111335]: pgmap v4698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3270520799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:10 smithi093 ceph-mon[118902]: pgmap v4698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3270520799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:10.767Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:10.768Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:10.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:10.959630+0000) 2023-08-27T23:30:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:12 smithi136 ceph-mon[113531]: pgmap v4699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3305196186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:11.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:11.959864+0000) 2023-08-27T23:30:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:12 smithi093 ceph-mon[111335]: pgmap v4699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3305196186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:12 smithi093 ceph-mon[118902]: pgmap v4699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3305196186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:12.254Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:12.254Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:12.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:12.960126+0000) 2023-08-27T23:30:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:14 smithi136 ceph-mon[113531]: pgmap v4700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:13.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:13.960344+0000) 2023-08-27T23:30:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:14 smithi093 ceph-mon[111335]: pgmap v4700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:14 smithi093 ceph-mon[118902]: pgmap v4700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3350283340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:14.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:14.960575+0000) 2023-08-27T23:30:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3350283340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3350283340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:30:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:30:15] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:30:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:16 smithi136 ceph-mon[113531]: pgmap v4701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:15.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:15.960792+0000) 2023-08-27T23:30:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:16 smithi093 ceph-mon[111335]: pgmap v4701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:16 smithi093 ceph-mon[118902]: pgmap v4701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2300264632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:16.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:16.960986+0000) 2023-08-27T23:30:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2300264632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2300264632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:18 smithi136 ceph-mon[113531]: pgmap v4702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:17.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:17.961225+0000) 2023-08-27T23:30:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:18 smithi093 ceph-mon[111335]: pgmap v4702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:18 smithi093 ceph-mon[118902]: pgmap v4702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4175005252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:18.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:18.961444+0000) 2023-08-27T23:30:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4175005252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4175005252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:20 smithi136 ceph-mon[113531]: pgmap v4703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:20.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:19.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:19.961684+0000) 2023-08-27T23:30:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:20 smithi093 ceph-mon[111335]: pgmap v4703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:20 smithi093 ceph-mon[118902]: pgmap v4703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:20.768Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:20.769Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:20.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:20.961925+0000) 2023-08-27T23:30:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:22 smithi136 ceph-mon[113531]: pgmap v4704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/740553146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:21.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:21.962163+0000) 2023-08-27T23:30:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:22 smithi093 ceph-mon[118902]: pgmap v4704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/740553146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:22 smithi093 ceph-mon[111335]: pgmap v4704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/740553146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:22.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:22.255Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:22.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:22.256Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:22.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:22.962427+0000) 2023-08-27T23:30:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:24 smithi136 ceph-mon[113531]: pgmap v4705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3043734555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:23.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:23.962703+0000) 2023-08-27T23:30:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:24 smithi093 ceph-mon[111335]: pgmap v4705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3043734555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:24 smithi093 ceph-mon[118902]: pgmap v4705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3043734555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:24.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:24.962970+0000) 2023-08-27T23:30:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:30:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:30:25] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:30:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:26 smithi136 ceph-mon[113531]: pgmap v4706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2078456790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:25.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:25.963231+0000) 2023-08-27T23:30:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:26 smithi093 ceph-mon[118902]: pgmap v4706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2078456790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:26 smithi093 ceph-mon[111335]: pgmap v4706: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2078456790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:26.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:26.963488+0000) 2023-08-27T23:30:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:28 smithi136 ceph-mon[113531]: pgmap v4707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:27.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:27.963684+0000) 2023-08-27T23:30:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:28 smithi093 ceph-mon[111335]: pgmap v4707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:28 smithi093 ceph-mon[118902]: pgmap v4707: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3668289134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:28.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:28.963911+0000) 2023-08-27T23:30:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3668289134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3668289134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:30 smithi136 ceph-mon[113531]: pgmap v4708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:29.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:29.964161+0000) 2023-08-27T23:30:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:30 smithi093 ceph-mon[111335]: pgmap v4708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:30 smithi093 ceph-mon[118902]: pgmap v4708: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:31.171 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:30.769Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:31.172 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:30.770Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1906136619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:30.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:30.964379+0000) 2023-08-27T23:30:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1906136619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1906136619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:32 smithi136 ceph-mon[113531]: pgmap v4709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:31.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:31.964652+0000) 2023-08-27T23:30:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:32 smithi093 ceph-mon[111335]: pgmap v4709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:32 smithi093 ceph-mon[118902]: pgmap v4709: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:32.256Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:32.257Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:32.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:32.964900+0000) 2023-08-27T23:30:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:34 smithi136 ceph-mon[113531]: pgmap v4710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/536215837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:33.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:33.965163+0000) 2023-08-27T23:30:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:34 smithi093 ceph-mon[111335]: pgmap v4710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/536215837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:34 smithi093 ceph-mon[118902]: pgmap v4710: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/536215837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:34.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:34.965394+0000) 2023-08-27T23:30:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:30:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:30:35] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:30:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:36 smithi136 ceph-mon[113531]: pgmap v4711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:36.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2845450513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:35.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:35.965649+0000) 2023-08-27T23:30:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:36 smithi093 ceph-mon[111335]: pgmap v4711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2845450513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:36 smithi093 ceph-mon[118902]: pgmap v4711: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2845450513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:36.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:36.965908+0000) 2023-08-27T23:30:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:38 smithi136 ceph-mon[113531]: pgmap v4712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3037512038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:37.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:37.966162+0000) 2023-08-27T23:30:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:38 smithi093 ceph-mon[111335]: pgmap v4712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3037512038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:38 smithi093 ceph-mon[118902]: pgmap v4712: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3037512038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:38.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:38.966422+0000) 2023-08-27T23:30:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:40 smithi136 ceph-mon[113531]: pgmap v4713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:39.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:39.966678+0000) 2023-08-27T23:30:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:40 smithi093 ceph-mon[111335]: pgmap v4713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:40 smithi093 ceph-mon[118902]: pgmap v4713: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:41.196 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:40.769Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:41.197 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:40.770Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/122909873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:40.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:40.966926+0000) 2023-08-27T23:30:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/122909873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/122909873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:42 smithi136 ceph-mon[113531]: pgmap v4714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:41.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:41.967178+0000) 2023-08-27T23:30:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:42 smithi093 ceph-mon[111335]: pgmap v4714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:42 smithi093 ceph-mon[118902]: pgmap v4714: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:42.256Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:42.257Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1908839023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:42.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:42.967421+0000) 2023-08-27T23:30:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1908839023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1908839023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:44 smithi136 ceph-mon[113531]: pgmap v4715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:43.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:43.967640+0000) 2023-08-27T23:30:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:44 smithi093 ceph-mon[111335]: pgmap v4715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:44 smithi093 ceph-mon[118902]: pgmap v4715: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:44.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:44.967899+0000) 2023-08-27T23:30:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:30:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:30:45] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:30:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:46 smithi136 ceph-mon[113531]: pgmap v4716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/389810492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:45.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:45.968189+0000) 2023-08-27T23:30:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:46 smithi093 ceph-mon[111335]: pgmap v4716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/389810492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:46 smithi093 ceph-mon[118902]: pgmap v4716: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/389810492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:46.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:46.968440+0000) 2023-08-27T23:30:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:48 smithi136 ceph-mon[113531]: pgmap v4717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1542406561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:47.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:47.968719+0000) 2023-08-27T23:30:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:48 smithi093 ceph-mon[111335]: pgmap v4717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1542406561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:48 smithi093 ceph-mon[118902]: pgmap v4717: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1542406561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:49.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:48.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:48.968974+0000) 2023-08-27T23:30:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:50 smithi136 ceph-mon[113531]: pgmap v4718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1180364357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:49.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:49.969218+0000) 2023-08-27T23:30:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:50 smithi093 ceph-mon[111335]: pgmap v4718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1180364357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:50 smithi093 ceph-mon[118902]: pgmap v4718: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1180364357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:50.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:50.771Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:50.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:50.969445+0000) 2023-08-27T23:30:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:52 smithi136 ceph-mon[113531]: pgmap v4719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:51.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:51.969682+0000) 2023-08-27T23:30:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:52 smithi093 ceph-mon[111335]: pgmap v4719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:52 smithi093 ceph-mon[118902]: pgmap v4719: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:52.257Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:30:52.258Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:30:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/615987302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:52.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:52.969948+0000) 2023-08-27T23:30:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/615987302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/615987302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:54 smithi136 ceph-mon[113531]: pgmap v4720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:53.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:53.970224+0000) 2023-08-27T23:30:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:54 smithi093 ceph-mon[111335]: pgmap v4720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:54 smithi093 ceph-mon[118902]: pgmap v4720: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:30:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/960411120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:54.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:54.970469+0000) 2023-08-27T23:30:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/960411120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/960411120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:30:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:30:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:30:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:56 smithi136 ceph-mon[113531]: pgmap v4721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:55.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:55.970731+0000) 2023-08-27T23:30:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:56 smithi093 ceph-mon[111335]: pgmap v4721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:56 smithi093 ceph-mon[118902]: pgmap v4721: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/366299727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:56.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:56.970919+0000) 2023-08-27T23:30:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/366299727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/366299727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:30:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:58 smithi136 ceph-mon[113531]: pgmap v4722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:57.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:57.971128+0000) 2023-08-27T23:30:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:58 smithi093 ceph-mon[111335]: pgmap v4722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:58 smithi093 ceph-mon[118902]: pgmap v4722: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:30:59.145 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:58.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:58.971282+0000) 2023-08-27T23:30:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:30:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:30:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:30:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:30:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:30:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:31:00.420 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:00 smithi136 ceph-mon[113531]: pgmap v4723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:00.420 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3526332370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:00.420 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:00.420 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:00.421 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:30:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:30:59.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:30:59.971434+0000) 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[111335]: pgmap v4723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3526332370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[118902]: pgmap v4723: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3526332370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:00.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:00.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:00.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:00.971611+0000) 2023-08-27T23:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:01 smithi136 ceph-mon[113531]: pgmap v4724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:31:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:31:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[111335]: pgmap v4724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:31:01.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:31:01.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[118902]: pgmap v4724: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:01.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:01.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:31:01.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:31:01.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:31:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:01.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:01.971825+0000) 2023-08-27T23:31:02.693 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:02.258Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:02.693 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:02.259Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:02.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2998352741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2998352741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2998352741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:02.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:02.972113+0000) 2023-08-27T23:31:03.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:03 smithi136 ceph-mon[113531]: pgmap v4725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:03 smithi093 ceph-mon[111335]: pgmap v4725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:03 smithi093 ceph-mon[118902]: pgmap v4725: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:03.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:03.972375+0000) 2023-08-27T23:31:04.972 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/78986816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/78986816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/78986816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:05.318 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:04.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:04.972623+0000) 2023-08-27T23:31:05.689 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 23:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T23:31:05.318210658Z level=info msg="Completed cleanup jobs" duration=13.122778ms 2023-08-27T23:31:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:31:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:31:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:31:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:06 smithi136 ceph-mon[113531]: pgmap v4726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:05.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:05.972857+0000) 2023-08-27T23:31:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:06 smithi093 ceph-mon[111335]: pgmap v4726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:06 smithi093 ceph-mon[118902]: pgmap v4726: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/627498664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:06.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:06.973114+0000) 2023-08-27T23:31:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/627498664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/627498664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:08 smithi136 ceph-mon[113531]: pgmap v4727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:07.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:07.973380+0000) 2023-08-27T23:31:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:08 smithi093 ceph-mon[111335]: pgmap v4727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:08 smithi093 ceph-mon[118902]: pgmap v4727: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2055438870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:08.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:08.973618+0000) 2023-08-27T23:31:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2055438870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2055438870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:10 smithi136 ceph-mon[113531]: pgmap v4728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:09.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:09.973868+0000) 2023-08-27T23:31:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:10 smithi093 ceph-mon[111335]: pgmap v4728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:10 smithi093 ceph-mon[118902]: pgmap v4728: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:11.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:10.772Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:10.773Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:10.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:10.974164+0000) 2023-08-27T23:31:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:12 smithi136 ceph-mon[113531]: pgmap v4729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3652444940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:11.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:11.974396+0000) 2023-08-27T23:31:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:12 smithi093 ceph-mon[111335]: pgmap v4729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3652444940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:12 smithi093 ceph-mon[118902]: pgmap v4729: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3652444940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:12.258Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:12.259Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:12.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:12.974593+0000) 2023-08-27T23:31:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:14 smithi136 ceph-mon[113531]: pgmap v4730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3847966896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:13.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:13.974849+0000) 2023-08-27T23:31:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:14 smithi093 ceph-mon[111335]: pgmap v4730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3847966896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:14 smithi093 ceph-mon[118902]: pgmap v4730: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3847966896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:14.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:14.975142+0000) 2023-08-27T23:31:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:31:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:31:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:31:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:16 smithi136 ceph-mon[113531]: pgmap v4731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:15.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:15.975403+0000) 2023-08-27T23:31:16.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:16 smithi093 ceph-mon[111335]: pgmap v4731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:16.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:16 smithi093 ceph-mon[118902]: pgmap v4731: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/639580641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:16.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:16.975653+0000) 2023-08-27T23:31:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/639580641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/639580641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:18 smithi136 ceph-mon[113531]: pgmap v4732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:17.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:17.975875+0000) 2023-08-27T23:31:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:18 smithi093 ceph-mon[111335]: pgmap v4732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:18 smithi093 ceph-mon[118902]: pgmap v4732: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2138850680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:18.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:18.976110+0000) 2023-08-27T23:31:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2138850680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2138850680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:20 smithi136 ceph-mon[113531]: pgmap v4733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:19.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:19.976381+0000) 2023-08-27T23:31:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:20 smithi093 ceph-mon[111335]: pgmap v4733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:20 smithi093 ceph-mon[118902]: pgmap v4733: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:21.151 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:20.772Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:21.703 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:20.773Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:21.704 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3264229465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:21.704 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3264229465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:21.704 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:20.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:20.976666+0000) 2023-08-27T23:31:21.705 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3264229465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:22 smithi136 ceph-mon[113531]: pgmap v4734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:21.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:21.976853+0000) 2023-08-27T23:31:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:22 smithi093 ceph-mon[111335]: pgmap v4734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:22 smithi093 ceph-mon[118902]: pgmap v4734: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:22.258Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:22.259Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:22.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:22.977146+0000) 2023-08-27T23:31:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:23.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:23.977419+0000) 2023-08-27T23:31:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:24 smithi136 ceph-mon[113531]: pgmap v4735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/97441116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:24 smithi093 ceph-mon[111335]: pgmap v4735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/97441116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:24 smithi093 ceph-mon[118902]: pgmap v4735: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/97441116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:24.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:24.977689+0000) 2023-08-27T23:31:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:31:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:31:25] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:31:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:26 smithi136 ceph-mon[113531]: pgmap v4736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/733247518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:25.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:25.977976+0000) 2023-08-27T23:31:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:26 smithi093 ceph-mon[111335]: pgmap v4736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/733247518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:26.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:26 smithi093 ceph-mon[118902]: pgmap v4736: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:26.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/733247518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:26.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:26.978175+0000) 2023-08-27T23:31:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:28 smithi136 ceph-mon[113531]: pgmap v4737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1747910914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:27.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:27.978452+0000) 2023-08-27T23:31:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:28 smithi093 ceph-mon[111335]: pgmap v4737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1747910914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:28 smithi093 ceph-mon[118902]: pgmap v4737: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1747910914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:28.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:28.978682+0000) 2023-08-27T23:31:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:30 smithi136 ceph-mon[113531]: pgmap v4738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:29.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:29.978941+0000) 2023-08-27T23:31:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:30 smithi093 ceph-mon[111335]: pgmap v4738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:30 smithi093 ceph-mon[118902]: pgmap v4738: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:31.172 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:30.773Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:31.172 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:30.774Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/327071588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:30.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:30.979192+0000) 2023-08-27T23:31:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/327071588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/327071588' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:32 smithi136 ceph-mon[113531]: pgmap v4739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:31.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:31.979396+0000) 2023-08-27T23:31:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:32 smithi093 ceph-mon[111335]: pgmap v4739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:32 smithi093 ceph-mon[118902]: pgmap v4739: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:32.259Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:32.260Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1643965021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:32.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:32.979668+0000) 2023-08-27T23:31:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1643965021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1643965021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:34 smithi136 ceph-mon[113531]: pgmap v4740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:33.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:33.979931+0000) 2023-08-27T23:31:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:34 smithi093 ceph-mon[111335]: pgmap v4740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:34 smithi093 ceph-mon[118902]: pgmap v4740: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:34.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:34.980163+0000) 2023-08-27T23:31:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:31:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:31:35] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:31:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:36 smithi136 ceph-mon[113531]: pgmap v4741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/210727380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:35.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:35.980437+0000) 2023-08-27T23:31:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:36 smithi093 ceph-mon[111335]: pgmap v4741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/210727380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:36 smithi093 ceph-mon[118902]: pgmap v4741: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/210727380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:36.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:36.980656+0000) 2023-08-27T23:31:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:38 smithi136 ceph-mon[113531]: pgmap v4742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/76144873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:37.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:37.980912+0000) 2023-08-27T23:31:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:38 smithi093 ceph-mon[111335]: pgmap v4742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/76144873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:38 smithi093 ceph-mon[118902]: pgmap v4742: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/76144873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:39.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:38.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:38.981122+0000) 2023-08-27T23:31:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:40 smithi136 ceph-mon[113531]: pgmap v4743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1179348731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:39.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:39.981319+0000) 2023-08-27T23:31:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:40 smithi093 ceph-mon[111335]: pgmap v4743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1179348731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:40 smithi093 ceph-mon[118902]: pgmap v4743: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1179348731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:40.774Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:40.775Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:40.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:40.981545+0000) 2023-08-27T23:31:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:42 smithi136 ceph-mon[113531]: pgmap v4744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:41.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:41.981782+0000) 2023-08-27T23:31:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:42 smithi093 ceph-mon[111335]: pgmap v4744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:42 smithi093 ceph-mon[118902]: pgmap v4744: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:42.260Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:42.261Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3416930269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:42.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:42.982012+0000) 2023-08-27T23:31:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3416930269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3416930269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:44 smithi136 ceph-mon[113531]: pgmap v4745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:43.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:43.982261+0000) 2023-08-27T23:31:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:44 smithi093 ceph-mon[111335]: pgmap v4745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:44 smithi093 ceph-mon[118902]: pgmap v4745: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/912095951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:44.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:44.982481+0000) 2023-08-27T23:31:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/912095951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:31:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:31:45] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:31:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/912095951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:46 smithi136 ceph-mon[113531]: pgmap v4746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:45.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:45.982765+0000) 2023-08-27T23:31:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:46 smithi093 ceph-mon[111335]: pgmap v4746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:46 smithi093 ceph-mon[118902]: pgmap v4746: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:47.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3037527968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:47.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:46.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:46.982999+0000) 2023-08-27T23:31:47.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3037527968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:47.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3037527968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:48 smithi136 ceph-mon[113531]: pgmap v4747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:47.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:47.983290+0000) 2023-08-27T23:31:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:48 smithi093 ceph-mon[111335]: pgmap v4747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:48 smithi093 ceph-mon[118902]: pgmap v4747: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:48.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:48.983546+0000) 2023-08-27T23:31:50.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:50 smithi136 ceph-mon[113531]: pgmap v4748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:50.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1342476158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:50.443 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:49.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:49.983744+0000) 2023-08-27T23:31:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:50 smithi093 ceph-mon[111335]: pgmap v4748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1342476158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:50 smithi093 ceph-mon[118902]: pgmap v4748: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1342476158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:50.775Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:50.775Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:50.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:50.984002+0000) 2023-08-27T23:31:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:52 smithi136 ceph-mon[113531]: pgmap v4749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4109788262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:51.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:51.984265+0000) 2023-08-27T23:31:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:52 smithi093 ceph-mon[111335]: pgmap v4749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4109788262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:52 smithi093 ceph-mon[118902]: pgmap v4749: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4109788262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:52.261Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:31:52.262Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:31:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:52.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:52.984508+0000) 2023-08-27T23:31:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:54 smithi136 ceph-mon[113531]: pgmap v4750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:53.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:53.984789+0000) 2023-08-27T23:31:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:54 smithi093 ceph-mon[111335]: pgmap v4750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:54 smithi093 ceph-mon[118902]: pgmap v4750: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:31:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2367597584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:54.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:54.985001+0000) 2023-08-27T23:31:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2367597584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:31:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:31:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:31:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2367597584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:56 smithi136 ceph-mon[113531]: pgmap v4751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:55.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:55.985247+0000) 2023-08-27T23:31:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:56 smithi093 ceph-mon[111335]: pgmap v4751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:56 smithi093 ceph-mon[118902]: pgmap v4751: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:57.236 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:56.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:56.985481+0000) 2023-08-27T23:31:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2517948550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2517948550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2517948550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:58.238 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:57.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:57.985736+0000) 2023-08-27T23:31:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:58 smithi093 ceph-mon[111335]: pgmap v4752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:58 smithi093 ceph-mon[118902]: pgmap v4752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:58 smithi136 ceph-mon[113531]: pgmap v4752: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:31:59.243 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:58.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:58.985938+0000) 2023-08-27T23:31:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:31:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1198797095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:31:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1198797095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:31:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:31:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1198797095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:00.243 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:31:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:31:59.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:31:59.986168+0000) 2023-08-27T23:32:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:00 smithi136 ceph-mon[113531]: pgmap v4753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:00 smithi093 ceph-mon[111335]: pgmap v4753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:00 smithi093 ceph-mon[118902]: pgmap v4753: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:01.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:00.776Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:00.777Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:00.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:00.986407+0000) 2023-08-27T23:32:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:01.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:01.986632+0000) 2023-08-27T23:32:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:02 smithi136 ceph-mon[113531]: pgmap v4754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:32:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4288574010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:02.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-mon[111335]: pgmap v4754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:02.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:32:02.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4288574010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:02.702 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-mon[118902]: pgmap v4754: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:02.702 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:32:02.702 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4288574010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:02.703 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:02.261Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:02.703 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:02.262Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:02.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:02.986847+0000) 2023-08-27T23:32:04.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[111335]: pgmap v4755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/242043452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[118902]: pgmap v4755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/242043452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:03 smithi136 ceph-mon[113531]: pgmap v4755: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/242043452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:03.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:03.986989+0000) 2023-08-27T23:32:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:32:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:32:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:32:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:32:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:32:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:32:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:32:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:04.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:04.987194+0000) 2023-08-27T23:32:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:32:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:32:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:32:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:06 smithi136 ceph-mon[113531]: pgmap v4756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:05.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:05.987430+0000) 2023-08-27T23:32:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:06 smithi093 ceph-mon[111335]: pgmap v4756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:06 smithi093 ceph-mon[118902]: pgmap v4756: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/15324168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:06.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:06.987643+0000) 2023-08-27T23:32:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/15324168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/15324168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:08 smithi136 ceph-mon[113531]: pgmap v4757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:07.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:07.987918+0000) 2023-08-27T23:32:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:08 smithi093 ceph-mon[111335]: pgmap v4757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:08 smithi093 ceph-mon[118902]: pgmap v4757: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4137126283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:08.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:08.988160+0000) 2023-08-27T23:32:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4137126283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4137126283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:10 smithi136 ceph-mon[113531]: pgmap v4758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:09.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:09.988400+0000) 2023-08-27T23:32:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:10 smithi093 ceph-mon[111335]: pgmap v4758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:10.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:10 smithi093 ceph-mon[118902]: pgmap v4758: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:11.186 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:10.777Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:11.186 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:10.778Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3157935598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:10.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:10.988675+0000) 2023-08-27T23:32:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3157935598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3157935598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:12 smithi136 ceph-mon[113531]: pgmap v4759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:11.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:11.988894+0000) 2023-08-27T23:32:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:12 smithi093 ceph-mon[111335]: pgmap v4759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:12 smithi093 ceph-mon[118902]: pgmap v4759: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:12.262Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:12.263Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:12.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:12.989154+0000) 2023-08-27T23:32:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:14 smithi136 ceph-mon[113531]: pgmap v4760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1355788407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:13.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:13.989348+0000) 2023-08-27T23:32:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:14 smithi093 ceph-mon[111335]: pgmap v4760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1355788407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:14 smithi093 ceph-mon[118902]: pgmap v4760: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1355788407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:14.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:14.989560+0000) 2023-08-27T23:32:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:32:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:32:15] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:32:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:16 smithi136 ceph-mon[113531]: pgmap v4761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2267722706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:15.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:15.989824+0000) 2023-08-27T23:32:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:16 smithi093 ceph-mon[111335]: pgmap v4761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2267722706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:16 smithi093 ceph-mon[118902]: pgmap v4761: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2267722706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:16.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:16.990094+0000) 2023-08-27T23:32:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:18 smithi136 ceph-mon[113531]: pgmap v4762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2047912101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:17.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:17.990355+0000) 2023-08-27T23:32:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:18 smithi093 ceph-mon[111335]: pgmap v4762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2047912101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:18 smithi093 ceph-mon[118902]: pgmap v4762: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2047912101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:18.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:18.990551+0000) 2023-08-27T23:32:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:20 smithi136 ceph-mon[113531]: pgmap v4763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:19.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:19.990770+0000) 2023-08-27T23:32:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:20 smithi093 ceph-mon[111335]: pgmap v4763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:20 smithi093 ceph-mon[118902]: pgmap v4763: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:21.207 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:20.778Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:21.207 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:20.779Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/907728038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:20.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:20.991027+0000) 2023-08-27T23:32:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/907728038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/907728038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:22 smithi136 ceph-mon[113531]: pgmap v4764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:21.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:21.991267+0000) 2023-08-27T23:32:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:22 smithi093 ceph-mon[111335]: pgmap v4764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:22 smithi093 ceph-mon[118902]: pgmap v4764: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:22.263Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:22.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:22.264Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/787108682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:22.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:22.991531+0000) 2023-08-27T23:32:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/787108682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/787108682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:24 smithi136 ceph-mon[113531]: pgmap v4765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:23.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:23.991789+0000) 2023-08-27T23:32:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:24 smithi093 ceph-mon[111335]: pgmap v4765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:24 smithi093 ceph-mon[118902]: pgmap v4765: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:24.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:24.991971+0000) 2023-08-27T23:32:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:32:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:32:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:32:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:26 smithi136 ceph-mon[113531]: pgmap v4766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3120863598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:25.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:25.992269+0000) 2023-08-27T23:32:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:26 smithi093 ceph-mon[111335]: pgmap v4766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3120863598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:26 smithi093 ceph-mon[118902]: pgmap v4766: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3120863598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:26.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:26.992425+0000) 2023-08-27T23:32:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:28 smithi136 ceph-mon[113531]: pgmap v4767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3771388752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:27.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:27.992654+0000) 2023-08-27T23:32:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:28 smithi093 ceph-mon[111335]: pgmap v4767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3771388752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:28 smithi093 ceph-mon[118902]: pgmap v4767: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3771388752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:28.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:28.992916+0000) 2023-08-27T23:32:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:30 smithi136 ceph-mon[113531]: pgmap v4768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/634689671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:29.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:29.993171+0000) 2023-08-27T23:32:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:30 smithi093 ceph-mon[111335]: pgmap v4768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/634689671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:30 smithi093 ceph-mon[118902]: pgmap v4768: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/634689671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:31.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:30.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:30.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:30.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:30.993408+0000) 2023-08-27T23:32:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:32 smithi136 ceph-mon[113531]: pgmap v4769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:31.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:31.993626+0000) 2023-08-27T23:32:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:32 smithi093 ceph-mon[111335]: pgmap v4769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:32 smithi093 ceph-mon[118902]: pgmap v4769: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:32.264Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:32.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:32.265Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1626769261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:32.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:32.993847+0000) 2023-08-27T23:32:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1626769261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1626769261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:34 smithi136 ceph-mon[113531]: pgmap v4770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:33.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:33.994143+0000) 2023-08-27T23:32:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:34 smithi093 ceph-mon[111335]: pgmap v4770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:34 smithi093 ceph-mon[118902]: pgmap v4770: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1074528090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:34.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:34.994368+0000) 2023-08-27T23:32:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1074528090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1074528090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:32:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:32:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:32:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:36 smithi136 ceph-mon[113531]: pgmap v4771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:35.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:35.994577+0000) 2023-08-27T23:32:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:36 smithi093 ceph-mon[111335]: pgmap v4771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:36 smithi093 ceph-mon[118902]: pgmap v4771: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:37.246 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:36.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:36.994820+0000) 2023-08-27T23:32:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2434054282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2434054282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2434054282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:38.248 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:37.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:37.995087+0000) 2023-08-27T23:32:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:38 smithi136 ceph-mon[113531]: pgmap v4772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:38 smithi093 ceph-mon[111335]: pgmap v4772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:38 smithi093 ceph-mon[118902]: pgmap v4772: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:39.252 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:38.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:38.995353+0000) 2023-08-27T23:32:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:40.254 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:39.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:39.995607+0000) 2023-08-27T23:32:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:40 smithi136 ceph-mon[113531]: pgmap v4773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4212465546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:40 smithi093 ceph-mon[111335]: pgmap v4773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4212465546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:40 smithi093 ceph-mon[118902]: pgmap v4773: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4212465546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:41.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:40.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:40.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:40.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:40.995868+0000) 2023-08-27T23:32:42.256 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:41.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:41.996127+0000) 2023-08-27T23:32:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:42 smithi136 ceph-mon[113531]: pgmap v4774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3635400211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:42 smithi093 ceph-mon[111335]: pgmap v4774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3635400211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:42 smithi093 ceph-mon[118902]: pgmap v4774: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3635400211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:42.265Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:42.266Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:42.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:42.996333+0000) 2023-08-27T23:32:44.259 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:43.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:43.996583+0000) 2023-08-27T23:32:44.692 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:44 smithi136 ceph-mon[113531]: pgmap v4775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:44 smithi093 ceph-mon[111335]: pgmap v4775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:44 smithi093 ceph-mon[118902]: pgmap v4775: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:45.268 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:44.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:44.996801+0000) 2023-08-27T23:32:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1744802297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1744802297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1744802297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:45.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:32:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:32:45] "GET /metrics HTTP/1.1" 200 239916 "" "Prometheus/2.43.0" 2023-08-27T23:32:46.266 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:45.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:45.997085+0000) 2023-08-27T23:32:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:46 smithi136 ceph-mon[113531]: pgmap v4776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:46 smithi093 ceph-mon[111335]: pgmap v4776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:46 smithi093 ceph-mon[118902]: pgmap v4776: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:47.274 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:46.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:46.997332+0000) 2023-08-27T23:32:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1090521562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1090521562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1090521562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:48.275 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:47.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:47.997574+0000) 2023-08-27T23:32:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:48 smithi136 ceph-mon[113531]: pgmap v4777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:48 smithi093 ceph-mon[111335]: pgmap v4777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:48 smithi093 ceph-mon[118902]: pgmap v4777: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:49.280 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:48.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:48.997835+0000) 2023-08-27T23:32:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/575906462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/575906462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/575906462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:50.281 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:49.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:49.998111+0000) 2023-08-27T23:32:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:50 smithi136 ceph-mon[113531]: pgmap v4778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:50 smithi093 ceph-mon[111335]: pgmap v4778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:50 smithi093 ceph-mon[118902]: pgmap v4778: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:50.780Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:50.781Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:50.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:50.998368+0000) 2023-08-27T23:32:52.287 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:51.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:51.998579+0000) 2023-08-27T23:32:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:52 smithi136 ceph-mon[113531]: pgmap v4779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4161746209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:52 smithi093 ceph-mon[111335]: pgmap v4779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4161746209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:52 smithi093 ceph-mon[118902]: pgmap v4779: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4161746209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:52.266Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:32:52.267Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:32:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:52.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:52.998868+0000) 2023-08-27T23:32:54.290 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:53.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:53.999145+0000) 2023-08-27T23:32:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:54 smithi136 ceph-mon[113531]: pgmap v4780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2998588820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:54 smithi093 ceph-mon[111335]: pgmap v4780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2998588820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:54 smithi093 ceph-mon[118902]: pgmap v4780: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:32:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2998588820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:54.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:54.999414+0000) 2023-08-27T23:32:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:32:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:32:55] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:32:56.294 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:55.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:55.999675+0000) 2023-08-27T23:32:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:56 smithi136 ceph-mon[113531]: pgmap v4781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/479312614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:56 smithi093 ceph-mon[111335]: pgmap v4781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/479312614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:56 smithi093 ceph-mon[118902]: pgmap v4781: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/479312614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:56.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:56.999916+0000) 2023-08-27T23:32:58.297 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:57.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:58.000155+0000) 2023-08-27T23:32:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:58 smithi136 ceph-mon[113531]: pgmap v4782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:58 smithi093 ceph-mon[111335]: pgmap v4782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:58 smithi093 ceph-mon[118902]: pgmap v4782: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:32:59.302 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:32:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:32:59.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:32:59.000407+0000) 2023-08-27T23:32:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:32:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2575836618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:32:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2575836618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:32:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:32:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2575836618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:00.303 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:00.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:00.000595+0000) 2023-08-27T23:33:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:00 smithi136 ceph-mon[113531]: pgmap v4783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:00 smithi093 ceph-mon[111335]: pgmap v4783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:00 smithi093 ceph-mon[118902]: pgmap v4783: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:01.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:00.781Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:00.782Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:01.308 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:01.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:01.000857+0000) 2023-08-27T23:33:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/840545060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/840545060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/840545060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:02.309 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:02.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:02.001100+0000) 2023-08-27T23:33:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:02 smithi136 ceph-mon[113531]: pgmap v4784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:02 smithi093 ceph-mon[111335]: pgmap v4784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:02 smithi093 ceph-mon[118902]: pgmap v4784: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:02.267Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:02.268Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:03.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:03.001365+0000) 2023-08-27T23:33:04.312 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:04.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:04.001563+0000) 2023-08-27T23:33:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:04 smithi136 ceph-mon[113531]: pgmap v4785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3891700627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:33:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:04 smithi093 ceph-mon[111335]: pgmap v4785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3891700627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:33:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:04 smithi093 ceph-mon[118902]: pgmap v4785: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3891700627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:33:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:05.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:05.001796+0000) 2023-08-27T23:33:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:33:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:33:05] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:33:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:06.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:06.001979+0000) 2023-08-27T23:33:06.581 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:06 smithi136 ceph-mon[113531]: pgmap v4786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:06.581 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/798489955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:06.581 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:06.581 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[111335]: pgmap v4786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/798489955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[118902]: pgmap v4786: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/798489955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:07.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:07.002149+0000) 2023-08-27T23:33:08.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:08.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:08.002361+0000) 2023-08-27T23:33:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:07 smithi136 ceph-mon[113531]: pgmap v4787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:33:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:33:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[111335]: pgmap v4787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:33:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:33:08.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[118902]: pgmap v4787: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:08.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:08.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:33:08.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:33:08.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1929199118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:09.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:09.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:09.002583+0000) 2023-08-27T23:33:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1929199118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1929199118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:09 smithi136 ceph-mon[113531]: pgmap v4788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:10.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:10.002810+0000) 2023-08-27T23:33:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:09 smithi093 ceph-mon[111335]: pgmap v4788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:09 smithi093 ceph-mon[118902]: pgmap v4788: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:11.129 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:10.782Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:11.130 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:10.783Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4131476902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:11.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:11.003085+0000) 2023-08-27T23:33:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4131476902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4131476902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:12 smithi136 ceph-mon[113531]: pgmap v4789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:12.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:12.003342+0000) 2023-08-27T23:33:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:12 smithi093 ceph-mon[111335]: pgmap v4789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:12 smithi093 ceph-mon[118902]: pgmap v4789: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:12.267Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:12.268Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/942389374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:13.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:13.003615+0000) 2023-08-27T23:33:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/942389374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/942389374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:14 smithi136 ceph-mon[113531]: pgmap v4790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:14.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:14.003870+0000) 2023-08-27T23:33:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:14 smithi093 ceph-mon[111335]: pgmap v4790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:14 smithi093 ceph-mon[118902]: pgmap v4790: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:15.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:15.004152+0000) 2023-08-27T23:33:16.014 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:33:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:33:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:33:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:16 smithi136 ceph-mon[113531]: pgmap v4791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/696256885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:16.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:16.004399+0000) 2023-08-27T23:33:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:16 smithi093 ceph-mon[111335]: pgmap v4791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/696256885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:16 smithi093 ceph-mon[118902]: pgmap v4791: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/696256885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:17.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:17.004653+0000) 2023-08-27T23:33:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:18 smithi136 ceph-mon[113531]: pgmap v4792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1149266291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:18.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:18.004900+0000) 2023-08-27T23:33:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:18 smithi093 ceph-mon[111335]: pgmap v4792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1149266291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:18 smithi093 ceph-mon[118902]: pgmap v4792: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1149266291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:19.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:19.005199+0000) 2023-08-27T23:33:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:20.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:20.005457+0000) 2023-08-27T23:33:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:20 smithi136 ceph-mon[113531]: pgmap v4793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4013666549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:20 smithi093 ceph-mon[111335]: pgmap v4793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4013666549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:20 smithi093 ceph-mon[118902]: pgmap v4793: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4013666549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:20.783Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:20.784Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:21.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:21.005741+0000) 2023-08-27T23:33:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:22.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:22.005964+0000) 2023-08-27T23:33:22.598 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:22.268Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:22.598 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:22.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:22 smithi136 ceph-mon[113531]: pgmap v4794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/573825405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:22 smithi093 ceph-mon[111335]: pgmap v4794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/573825405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:22 smithi093 ceph-mon[118902]: pgmap v4794: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/573825405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:23.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:23.006215+0000) 2023-08-27T23:33:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:24.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:24.006478+0000) 2023-08-27T23:33:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:24 smithi136 ceph-mon[113531]: pgmap v4795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:24 smithi093 ceph-mon[111335]: pgmap v4795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:24 smithi093 ceph-mon[118902]: pgmap v4795: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:25.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:25.006704+0000) 2023-08-27T23:33:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3287628824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:33:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:33:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:33:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3287628824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3287628824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:26.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:26.006982+0000) 2023-08-27T23:33:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:26 smithi136 ceph-mon[113531]: pgmap v4796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:26 smithi093 ceph-mon[111335]: pgmap v4796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:26 smithi093 ceph-mon[118902]: pgmap v4796: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:27.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:27.007251+0000) 2023-08-27T23:33:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/997362552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/997362552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/997362552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:28.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:28.007453+0000) 2023-08-27T23:33:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:28 smithi136 ceph-mon[113531]: pgmap v4797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:28 smithi093 ceph-mon[111335]: pgmap v4797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:28 smithi093 ceph-mon[118902]: pgmap v4797: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:29.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:29.007679+0000) 2023-08-27T23:33:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/318403229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/318403229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/318403229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:30.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:30.007921+0000) 2023-08-27T23:33:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:30 smithi136 ceph-mon[113531]: pgmap v4798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:30 smithi093 ceph-mon[111335]: pgmap v4798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:30 smithi093 ceph-mon[118902]: pgmap v4798: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:30.784Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:30.785Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:31.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:31.008144+0000) 2023-08-27T23:33:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:32.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:32.008401+0000) 2023-08-27T23:33:32.622 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:32.269Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:32.622 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:32.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:32 smithi136 ceph-mon[113531]: pgmap v4799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3946095853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:32 smithi093 ceph-mon[118902]: pgmap v4799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3946095853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:32 smithi093 ceph-mon[111335]: pgmap v4799: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3946095853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:33.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:33.008570+0000) 2023-08-27T23:33:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:34.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:34.008783+0000) 2023-08-27T23:33:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:34 smithi136 ceph-mon[113531]: pgmap v4800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3795998691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:34 smithi093 ceph-mon[118902]: pgmap v4800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3795998691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:34 smithi093 ceph-mon[111335]: pgmap v4800: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3795998691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:35.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:35.008966+0000) 2023-08-27T23:33:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:33:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:33:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:33:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:36.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:36.009227+0000) 2023-08-27T23:33:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:36 smithi136 ceph-mon[113531]: pgmap v4801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:36 smithi093 ceph-mon[111335]: pgmap v4801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:36 smithi093 ceph-mon[118902]: pgmap v4801: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:37.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:37.009449+0000) 2023-08-27T23:33:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3326034401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3326034401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3326034401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:38.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:38.009687+0000) 2023-08-27T23:33:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:38 smithi136 ceph-mon[113531]: pgmap v4802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:38 smithi093 ceph-mon[111335]: pgmap v4802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:38 smithi093 ceph-mon[118902]: pgmap v4802: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:39.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:39.009918+0000) 2023-08-27T23:33:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1786156677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1786156677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1786156677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:40.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:40.010183+0000) 2023-08-27T23:33:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:40 smithi136 ceph-mon[113531]: pgmap v4803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:40 smithi093 ceph-mon[111335]: pgmap v4803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:40 smithi093 ceph-mon[118902]: pgmap v4803: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:40.785Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:40.786Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:41.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:41.010384+0000) 2023-08-27T23:33:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1366812083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1366812083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1366812083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:42.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:42.010573+0000) 2023-08-27T23:33:42.661 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:42.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:42.661 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:42.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:42 smithi136 ceph-mon[113531]: pgmap v4804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:42 smithi093 ceph-mon[111335]: pgmap v4804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:42 smithi093 ceph-mon[118902]: pgmap v4804: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:43.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:43.010834+0000) 2023-08-27T23:33:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:44.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:44.011108+0000) 2023-08-27T23:33:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:44 smithi136 ceph-mon[113531]: pgmap v4805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1466775436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:44 smithi093 ceph-mon[111335]: pgmap v4805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1466775436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:44 smithi093 ceph-mon[118902]: pgmap v4805: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1466775436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:45.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:45.011306+0000) 2023-08-27T23:33:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:33:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:33:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:33:46.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:46.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:46.011584+0000) 2023-08-27T23:33:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:46 smithi136 ceph-mon[113531]: pgmap v4806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:46.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1148515430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:46.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:46 smithi093 ceph-mon[111335]: pgmap v4806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1148515430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:46 smithi093 ceph-mon[118902]: pgmap v4806: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1148515430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:47.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:47.011822+0000) 2023-08-27T23:33:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:48.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:48.012098+0000) 2023-08-27T23:33:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:48 smithi136 ceph-mon[113531]: pgmap v4807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3132352131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:48.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:48 smithi093 ceph-mon[111335]: pgmap v4807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3132352131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:48 smithi093 ceph-mon[118902]: pgmap v4807: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3132352131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:49.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:49.012331+0000) 2023-08-27T23:33:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:50.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:50.012580+0000) 2023-08-27T23:33:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:50 smithi136 ceph-mon[113531]: pgmap v4808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:50.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:50 smithi093 ceph-mon[111335]: pgmap v4808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:50 smithi093 ceph-mon[118902]: pgmap v4808: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:50.785Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:50.786Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:51.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:51.012838+0000) 2023-08-27T23:33:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:51 smithi136 ceph-mon[113531]: pgmap v4809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3641072229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:51 smithi093 ceph-mon[111335]: pgmap v4809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3641072229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:51 smithi093 ceph-mon[118902]: pgmap v4809: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3641072229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:52.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:52.013098+0000) 2023-08-27T23:33:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:52.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:33:52.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:33:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:53.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:53.013295+0000) 2023-08-27T23:33:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:53 smithi136 ceph-mon[113531]: pgmap v4810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/281211169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:54.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:54.013526+0000) 2023-08-27T23:33:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:53 smithi093 ceph-mon[111335]: pgmap v4810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/281211169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:53 smithi093 ceph-mon[118902]: pgmap v4810: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/281211169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:33:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:55.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:55.013722+0000) 2023-08-27T23:33:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:33:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:33:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:33:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:56 smithi136 ceph-mon[113531]: pgmap v4811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/274932213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:56.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:56.013979+0000) 2023-08-27T23:33:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:56 smithi093 ceph-mon[118902]: pgmap v4811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/274932213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:56 smithi093 ceph-mon[111335]: pgmap v4811: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/274932213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:57.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:57.014193+0000) 2023-08-27T23:33:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:58 smithi136 ceph-mon[113531]: pgmap v4812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:58.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:58.014382+0000) 2023-08-27T23:33:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:58 smithi093 ceph-mon[118902]: pgmap v4812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:58 smithi093 ceph-mon[111335]: pgmap v4812: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:33:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:33:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3319036421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:33:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:33:59.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:33:59.014619+0000) 2023-08-27T23:33:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:33:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3319036421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:33:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:33:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3319036421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:00 smithi136 ceph-mon[113531]: pgmap v4813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:00.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:00.014863+0000) 2023-08-27T23:34:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:00 smithi093 ceph-mon[111335]: pgmap v4813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:00 smithi093 ceph-mon[118902]: pgmap v4813: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:01.146 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:00.786Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:01.146 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:00.787Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2627768836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:01.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:01.015092+0000) 2023-08-27T23:34:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2627768836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2627768836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:02 smithi136 ceph-mon[113531]: pgmap v4814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:02.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:02.015296+0000) 2023-08-27T23:34:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:02 smithi093 ceph-mon[111335]: pgmap v4814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:02 smithi093 ceph-mon[118902]: pgmap v4814: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:02.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:02.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2754995342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:03.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:03.015549+0000) 2023-08-27T23:34:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2754995342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2754995342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:04 smithi136 ceph-mon[113531]: pgmap v4815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:04.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:04.015804+0000) 2023-08-27T23:34:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:04 smithi093 ceph-mon[111335]: pgmap v4815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:04 smithi093 ceph-mon[118902]: pgmap v4815: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:05.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:05.016076+0000) 2023-08-27T23:34:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:34:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:34:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:34:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:06 smithi136 ceph-mon[113531]: pgmap v4816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3451125142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:06.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:06.016276+0000) 2023-08-27T23:34:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:06 smithi093 ceph-mon[111335]: pgmap v4816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:06.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3451125142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:06.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:06 smithi093 ceph-mon[118902]: pgmap v4816: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:06.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3451125142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:07.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:07.016495+0000) 2023-08-27T23:34:08.397 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:08 smithi136 ceph-mon[113531]: pgmap v4817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:08.397 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:34:08.397 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3118010334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:08.398 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:08.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:08.016663+0000) 2023-08-27T23:34:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:08 smithi093 ceph-mon[111335]: pgmap v4817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:34:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3118010334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:08 smithi093 ceph-mon[118902]: pgmap v4817: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:34:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3118010334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:09.178 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:09.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:09.016905+0000) 2023-08-27T23:34:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:10 smithi136 ceph-mon[113531]: pgmap v4818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1702900895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:10.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:10.017106+0000) 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[111335]: pgmap v4818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1702900895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[118902]: pgmap v4818: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1702900895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:11.181 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:10.787Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:11.181 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:10.789Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:34:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:34:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:11.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:11.017332+0000) 2023-08-27T23:34:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:34:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:34:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:34:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:34:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:34:12.342 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:12.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:12.017582+0000) 2023-08-27T23:34:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:12 smithi136 ceph-mon[113531]: pgmap v4819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:12 smithi093 ceph-mon[111335]: pgmap v4819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:12 smithi093 ceph-mon[118902]: pgmap v4819: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:12.272Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:12.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:13.347 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:13.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:13.017798+0000) 2023-08-27T23:34:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2085166653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2085166653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2085166653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:14.348 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:14.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:14.018029+0000) 2023-08-27T23:34:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:14 smithi136 ceph-mon[113531]: pgmap v4820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:14 smithi093 ceph-mon[111335]: pgmap v4820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:14 smithi093 ceph-mon[118902]: pgmap v4820: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:15.353 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:15.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:15.018293+0000) 2023-08-27T23:34:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2708069625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2708069625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2708069625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:34:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:34:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:34:16.355 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:16.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:16.018575+0000) 2023-08-27T23:34:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:16 smithi136 ceph-mon[113531]: pgmap v4821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:16 smithi093 ceph-mon[111335]: pgmap v4821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:16 smithi093 ceph-mon[118902]: pgmap v4821: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:17.360 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:17.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:17.018827+0000) 2023-08-27T23:34:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/590702706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/590702706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:17.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/590702706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:18.361 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:18.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:18.019029+0000) 2023-08-27T23:34:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:18 smithi136 ceph-mon[113531]: pgmap v4822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:18 smithi093 ceph-mon[111335]: pgmap v4822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:18 smithi093 ceph-mon[118902]: pgmap v4822: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:19.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:19.019239+0000) 2023-08-27T23:34:20.363 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:20.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:20.019491+0000) 2023-08-27T23:34:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:20 smithi136 ceph-mon[113531]: pgmap v4823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/88805585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:20 smithi093 ceph-mon[111335]: pgmap v4823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/88805585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:20 smithi093 ceph-mon[118902]: pgmap v4823: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/88805585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:20.789Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:20.790Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:21.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:21.019722+0000) 2023-08-27T23:34:22.366 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:22.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:22.019935+0000) 2023-08-27T23:34:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:22 smithi136 ceph-mon[113531]: pgmap v4824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2434264995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:22 smithi093 ceph-mon[111335]: pgmap v4824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2434264995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:22 smithi093 ceph-mon[118902]: pgmap v4824: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2434264995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:22.273Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:22.273Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:23.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:23.020186+0000) 2023-08-27T23:34:24.369 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:24.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:24.020395+0000) 2023-08-27T23:34:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:24 smithi136 ceph-mon[113531]: pgmap v4825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:24 smithi093 ceph-mon[111335]: pgmap v4825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:24 smithi093 ceph-mon[118902]: pgmap v4825: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:25.378 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:25.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:25.020593+0000) 2023-08-27T23:34:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/829441856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/829441856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/829441856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:34:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:34:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:34:26.380 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:26.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:26.020784+0000) 2023-08-27T23:34:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:26 smithi136 ceph-mon[113531]: pgmap v4826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:26 smithi093 ceph-mon[111335]: pgmap v4826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:26 smithi093 ceph-mon[118902]: pgmap v4826: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:27.387 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:27.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:27.020982+0000) 2023-08-27T23:34:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3131397411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3131397411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3131397411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:28.388 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:28.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:28.021212+0000) 2023-08-27T23:34:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:28 smithi136 ceph-mon[113531]: pgmap v4827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:28 smithi093 ceph-mon[111335]: pgmap v4827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:28 smithi093 ceph-mon[118902]: pgmap v4827: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:29.393 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:29.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:29.021482+0000) 2023-08-27T23:34:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1411706255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1411706255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1411706255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:30.396 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:30.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:30.021670+0000) 2023-08-27T23:34:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:30 smithi136 ceph-mon[113531]: pgmap v4828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:30 smithi093 ceph-mon[111335]: pgmap v4828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:30 smithi093 ceph-mon[118902]: pgmap v4828: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:30.789Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:30.790Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:31.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:31.021877+0000) 2023-08-27T23:34:32.399 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:32.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:32.022121+0000) 2023-08-27T23:34:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:32 smithi136 ceph-mon[113531]: pgmap v4829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2703297276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:32 smithi093 ceph-mon[111335]: pgmap v4829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2703297276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:32 smithi093 ceph-mon[118902]: pgmap v4829: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2703297276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:32.273Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:32.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:33.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:33.022376+0000) 2023-08-27T23:34:34.402 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:34.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:34.022632+0000) 2023-08-27T23:34:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:34 smithi136 ceph-mon[113531]: pgmap v4830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1388255673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:34 smithi093 ceph-mon[111335]: pgmap v4830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1388255673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:34 smithi093 ceph-mon[118902]: pgmap v4830: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1388255673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:35.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:35.022848+0000) 2023-08-27T23:34:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:34:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:34:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:34:36.405 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:36.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:36.023132+0000) 2023-08-27T23:34:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:36 smithi136 ceph-mon[113531]: pgmap v4831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3407304528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:36 smithi093 ceph-mon[111335]: pgmap v4831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3407304528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:36 smithi093 ceph-mon[118902]: pgmap v4831: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3407304528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:37.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:37.023382+0000) 2023-08-27T23:34:38.408 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:38.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:38.023639+0000) 2023-08-27T23:34:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:38 smithi136 ceph-mon[113531]: pgmap v4832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:38 smithi093 ceph-mon[111335]: pgmap v4832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:38 smithi093 ceph-mon[118902]: pgmap v4832: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:39.413 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:39.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:39.023907+0000) 2023-08-27T23:34:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2414286929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2414286929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2414286929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:40.415 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:40.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:40.024198+0000) 2023-08-27T23:34:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:40 smithi136 ceph-mon[113531]: pgmap v4833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:40 smithi093 ceph-mon[111335]: pgmap v4833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:40 smithi093 ceph-mon[118902]: pgmap v4833: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:40.790Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:40.791Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:41.419 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:41.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:41.024384+0000) 2023-08-27T23:34:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1085909954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1085909954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1085909954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:42.420 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:42.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:42.024630+0000) 2023-08-27T23:34:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:42 smithi136 ceph-mon[113531]: pgmap v4834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:42 smithi093 ceph-mon[111335]: pgmap v4834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:42 smithi093 ceph-mon[118902]: pgmap v4834: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:42.274Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:42.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:43.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:43.024939+0000) 2023-08-27T23:34:44.423 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:44.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:44.025196+0000) 2023-08-27T23:34:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:44 smithi136 ceph-mon[113531]: pgmap v4835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3299256100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:44 smithi093 ceph-mon[111335]: pgmap v4835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3299256100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:44 smithi093 ceph-mon[118902]: pgmap v4835: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3299256100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:45.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:45.025470+0000) 2023-08-27T23:34:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:34:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:34:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:34:46.428 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:46.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:46.025746+0000) 2023-08-27T23:34:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:46 smithi136 ceph-mon[113531]: pgmap v4836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3093352730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:46 smithi093 ceph-mon[111335]: pgmap v4836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3093352730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:46 smithi093 ceph-mon[118902]: pgmap v4836: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3093352730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:47.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:47.025999+0000) 2023-08-27T23:34:48.430 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:48.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:48.026295+0000) 2023-08-27T23:34:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:48 smithi136 ceph-mon[113531]: pgmap v4837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3395679158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:48 smithi093 ceph-mon[118902]: pgmap v4837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3395679158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:48 smithi093 ceph-mon[111335]: pgmap v4837: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3395679158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:49.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:49.026530+0000) 2023-08-27T23:34:50.433 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:50.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:50.026740+0000) 2023-08-27T23:34:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:50 smithi136 ceph-mon[113531]: pgmap v4838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:50 smithi093 ceph-mon[118902]: pgmap v4838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:50 smithi093 ceph-mon[111335]: pgmap v4838: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:50.791Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:50.792Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:51.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:51.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:51.026985+0000) 2023-08-27T23:34:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3414360856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:51.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3414360856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3414360856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:52.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:52.027190+0000) 2023-08-27T23:34:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:52 smithi136 ceph-mon[113531]: pgmap v4839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:52 smithi093 ceph-mon[111335]: pgmap v4839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:52 smithi093 ceph-mon[118902]: pgmap v4839: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:52.275Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:34:52.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:34:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:53.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:53.027434+0000) 2023-08-27T23:34:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1849964796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1849964796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1849964796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:54.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:54.027707+0000) 2023-08-27T23:34:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:54 smithi093 ceph-mon[111335]: pgmap v4840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:54 smithi093 ceph-mon[118902]: pgmap v4840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:54 smithi136 ceph-mon[113531]: pgmap v4840: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:34:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:55.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:55.027965+0000) 2023-08-27T23:34:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:34:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:34:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:34:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:56 smithi136 ceph-mon[113531]: pgmap v4841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2749569300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:56.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:56.028240+0000) 2023-08-27T23:34:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:56 smithi093 ceph-mon[111335]: pgmap v4841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2749569300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:56 smithi093 ceph-mon[118902]: pgmap v4841: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2749569300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:57.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:57.028481+0000) 2023-08-27T23:34:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:58 smithi136 ceph-mon[113531]: pgmap v4842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:34:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2529400823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:58.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:58.028758+0000) 2023-08-27T23:34:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:58 smithi093 ceph-mon[111335]: pgmap v4842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:34:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2529400823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:58 smithi093 ceph-mon[118902]: pgmap v4842: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:34:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:34:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2529400823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:34:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:34:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:34:59.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:34:59.029069+0000) 2023-08-27T23:35:00.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:00 smithi136 ceph-mon[113531]: pgmap v4843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:00.443 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:00.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:00.029296+0000) 2023-08-27T23:35:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:00 smithi093 ceph-mon[111335]: pgmap v4843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:00 smithi093 ceph-mon[118902]: pgmap v4843: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:01.151 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:00.792Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:01.151 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:00.793Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3416877168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:01.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:01.029549+0000) 2023-08-27T23:35:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3416877168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3416877168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:02 smithi136 ceph-mon[113531]: pgmap v4844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:02.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:02.029767+0000) 2023-08-27T23:35:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:02.276Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:02.277Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:02 smithi093 ceph-mon[111335]: pgmap v4844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:02.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:02 smithi093 ceph-mon[118902]: pgmap v4844: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3892157169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:03.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:03.030054+0000) 2023-08-27T23:35:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3892157169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3892157169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:04 smithi136 ceph-mon[113531]: pgmap v4845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:04.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:04.030356+0000) 2023-08-27T23:35:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:04 smithi093 ceph-mon[111335]: pgmap v4845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:04 smithi093 ceph-mon[118902]: pgmap v4845: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2241630569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:05.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:05.030632+0000) 2023-08-27T23:35:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2241630569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2241630569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:35:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:35:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:35:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:06 smithi136 ceph-mon[113531]: pgmap v4846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:06.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:06.030931+0000) 2023-08-27T23:35:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:06 smithi093 ceph-mon[111335]: pgmap v4846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:06 smithi093 ceph-mon[118902]: pgmap v4846: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:07.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:07.031156+0000) 2023-08-27T23:35:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:08 smithi136 ceph-mon[113531]: pgmap v4847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/29802544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:08.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:08.031410+0000) 2023-08-27T23:35:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:08 smithi093 ceph-mon[111335]: pgmap v4847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/29802544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:08 smithi093 ceph-mon[118902]: pgmap v4847: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/29802544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:09.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:09.031672+0000) 2023-08-27T23:35:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:10 smithi136 ceph-mon[113531]: pgmap v4848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1267868143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:10.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:10.031948+0000) 2023-08-27T23:35:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:10 smithi093 ceph-mon[111335]: pgmap v4848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1267868143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:10.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:10 smithi093 ceph-mon[118902]: pgmap v4848: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:10.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1267868143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:11.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:10.793Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:11.179 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:10.793Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:35:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:11.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:11.032140+0000) 2023-08-27T23:35:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:35:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:35:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:12.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:12.032341+0000) 2023-08-27T23:35:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:12 smithi136 ceph-mon[113531]: pgmap v4849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/596404769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:12 smithi093 ceph-mon[111335]: pgmap v4849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/596404769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:12 smithi093 ceph-mon[118902]: pgmap v4849: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/596404769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:12.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:12.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:13.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:13.032512+0000) 2023-08-27T23:35:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:35:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:35:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:35:13.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:35:13.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:35:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:35:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:35:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:14.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:14.032737+0000) 2023-08-27T23:35:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:14 smithi093 ceph-mon[111335]: pgmap v4850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3704402103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:14 smithi093 ceph-mon[118902]: pgmap v4850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3704402103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:14 smithi136 ceph-mon[113531]: pgmap v4850: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3704402103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:15.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:15.032988+0000) 2023-08-27T23:35:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:35:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:35:15] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:35:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:16.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:16.033190+0000) 2023-08-27T23:35:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:16 smithi093 ceph-mon[111335]: pgmap v4851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:16 smithi093 ceph-mon[118902]: pgmap v4851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:16 smithi136 ceph-mon[113531]: pgmap v4851: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:17.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:17.033396+0000) 2023-08-27T23:35:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/84903314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/84903314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/84903314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:18.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:18.033649+0000) 2023-08-27T23:35:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:18 smithi136 ceph-mon[113531]: pgmap v4852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:18 smithi093 ceph-mon[111335]: pgmap v4852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:18 smithi093 ceph-mon[118902]: pgmap v4852: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:19.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:19.033867+0000) 2023-08-27T23:35:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/737210129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/737210129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/737210129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:20.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:20.034100+0000) 2023-08-27T23:35:20.794 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:20 smithi093 ceph-mon[111335]: pgmap v4853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:20.795 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:20 smithi093 ceph-mon[118902]: pgmap v4853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:20 smithi136 ceph-mon[113531]: pgmap v4853: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:20.793Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:20.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:21.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:21.034349+0000) 2023-08-27T23:35:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:22.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:22.034578+0000) 2023-08-27T23:35:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:22 smithi093 ceph-mon[111335]: pgmap v4854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1233122521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:22 smithi093 ceph-mon[118902]: pgmap v4854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1233122521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:22.278Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:22.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:22 smithi136 ceph-mon[113531]: pgmap v4854: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1233122521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:23.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:23.034841+0000) 2023-08-27T23:35:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:24.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:24.035117+0000) 2023-08-27T23:35:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:24 smithi136 ceph-mon[113531]: pgmap v4855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1314510121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:24 smithi093 ceph-mon[111335]: pgmap v4855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1314510121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:24 smithi093 ceph-mon[118902]: pgmap v4855: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1314510121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:25.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:25.035379+0000) 2023-08-27T23:35:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:35:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:35:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:35:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:26.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:26.035619+0000) 2023-08-27T23:35:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:26 smithi136 ceph-mon[113531]: pgmap v4856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/197933975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:26 smithi093 ceph-mon[111335]: pgmap v4856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/197933975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:26 smithi093 ceph-mon[118902]: pgmap v4856: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/197933975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:27.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:27.035851+0000) 2023-08-27T23:35:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:28.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:28.036093+0000) 2023-08-27T23:35:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:28 smithi136 ceph-mon[113531]: pgmap v4857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:28 smithi093 ceph-mon[111335]: pgmap v4857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:28 smithi093 ceph-mon[118902]: pgmap v4857: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:29.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:29.036317+0000) 2023-08-27T23:35:29.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3292038015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3292038015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3292038015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:30.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:30.036540+0000) 2023-08-27T23:35:30.795 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:30 smithi093 ceph-mon[111335]: pgmap v4858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:30.795 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:30 smithi093 ceph-mon[118902]: pgmap v4858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:30 smithi136 ceph-mon[113531]: pgmap v4858: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:30.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:30.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:31.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:31.036793+0000) 2023-08-27T23:35:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1397905558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1397905558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1397905558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:32.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:32.037072+0000) 2023-08-27T23:35:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:32 smithi093 ceph-mon[111335]: pgmap v4859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:32 smithi093 ceph-mon[118902]: pgmap v4859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:32.278Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:32.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:32 smithi136 ceph-mon[113531]: pgmap v4859: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:33.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:33.037337+0000) 2023-08-27T23:35:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/181371537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/181371537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/181371537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:34.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:34.037536+0000) 2023-08-27T23:35:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:34 smithi136 ceph-mon[113531]: pgmap v4860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:34 smithi093 ceph-mon[111335]: pgmap v4860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:34 smithi093 ceph-mon[118902]: pgmap v4860: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:35.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:35.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:35.037806+0000) 2023-08-27T23:35:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:35:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:35:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:35:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:36.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:36.038079+0000) 2023-08-27T23:35:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:36 smithi136 ceph-mon[113531]: pgmap v4861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2076596773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:36 smithi093 ceph-mon[111335]: pgmap v4861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2076596773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:36 smithi093 ceph-mon[118902]: pgmap v4861: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2076596773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:37.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:37.038317+0000) 2023-08-27T23:35:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:38.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:38.038507+0000) 2023-08-27T23:35:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:38 smithi136 ceph-mon[113531]: pgmap v4862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2589409888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:38.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:38 smithi093 ceph-mon[111335]: pgmap v4862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2589409888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:38 smithi093 ceph-mon[118902]: pgmap v4862: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2589409888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:39.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:39.038760+0000) 2023-08-27T23:35:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:40.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:40.038967+0000) 2023-08-27T23:35:40.796 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:40 smithi093 ceph-mon[111335]: pgmap v4863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:40.796 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:40 smithi093 ceph-mon[118902]: pgmap v4863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:40.797 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:40.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:40.797 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:40.796Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:40 smithi136 ceph-mon[113531]: pgmap v4863: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:41.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:41.039240+0000) 2023-08-27T23:35:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2474590313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2474590313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2474590313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:42.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:42.039453+0000) 2023-08-27T23:35:42.553 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:42.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:42.553 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:42.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:42 smithi136 ceph-mon[113531]: pgmap v4864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:42 smithi093 ceph-mon[111335]: pgmap v4864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:42 smithi093 ceph-mon[118902]: pgmap v4864: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:43.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:43.039716+0000) 2023-08-27T23:35:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4086952237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4086952237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4086952237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:44.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:44.039940+0000) 2023-08-27T23:35:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:44 smithi136 ceph-mon[113531]: pgmap v4865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:44 smithi093 ceph-mon[111335]: pgmap v4865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:44 smithi093 ceph-mon[118902]: pgmap v4865: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:45.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:45.040190+0000) 2023-08-27T23:35:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:35:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:35:45] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:35:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:46 smithi136 ceph-mon[113531]: pgmap v4866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/816024442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:46.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:46.040449+0000) 2023-08-27T23:35:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:46 smithi093 ceph-mon[118902]: pgmap v4866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/816024442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:46.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:46 smithi093 ceph-mon[111335]: pgmap v4866: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:46.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/816024442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:47.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:47.040658+0000) 2023-08-27T23:35:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:48 smithi136 ceph-mon[113531]: pgmap v4867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3370166181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:48.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:48.040893+0000) 2023-08-27T23:35:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:48 smithi093 ceph-mon[111335]: pgmap v4867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3370166181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:48 smithi093 ceph-mon[118902]: pgmap v4867: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3370166181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:49.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:49.041164+0000) 2023-08-27T23:35:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:50 smithi136 ceph-mon[113531]: pgmap v4868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:50.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:50.041378+0000) 2023-08-27T23:35:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:50 smithi093 ceph-mon[111335]: pgmap v4868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:50 smithi093 ceph-mon[118902]: pgmap v4868: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:51.157 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:50.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:51.157 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:50.796Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/500268838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:51.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:51.041595+0000) 2023-08-27T23:35:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/500268838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/500268838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:52 smithi136 ceph-mon[113531]: pgmap v4869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:52.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:52.041813+0000) 2023-08-27T23:35:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:52 smithi093 ceph-mon[111335]: pgmap v4869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:52 smithi093 ceph-mon[118902]: pgmap v4869: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:52.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:35:52.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:35:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1469646323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:53.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:53.042078+0000) 2023-08-27T23:35:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1469646323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1469646323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:54 smithi136 ceph-mon[113531]: pgmap v4870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:54.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:54.042280+0000) 2023-08-27T23:35:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:54 smithi093 ceph-mon[111335]: pgmap v4870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:54 smithi093 ceph-mon[118902]: pgmap v4870: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:35:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/146323914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:55.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:55.042493+0000) 2023-08-27T23:35:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/146323914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/146323914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:35:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:35:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:35:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:56 smithi136 ceph-mon[113531]: pgmap v4871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:56.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:56.042772+0000) 2023-08-27T23:35:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:56 smithi093 ceph-mon[111335]: pgmap v4871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:56 smithi093 ceph-mon[118902]: pgmap v4871: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:57.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:57.043011+0000) 2023-08-27T23:35:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:58 smithi136 ceph-mon[113531]: pgmap v4872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:35:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3542006581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:58.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:58.043266+0000) 2023-08-27T23:35:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:58 smithi093 ceph-mon[111335]: pgmap v4872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:35:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3542006581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:58 smithi093 ceph-mon[118902]: pgmap v4872: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:35:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:35:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3542006581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:35:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:35:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:35:59.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:35:59.043540+0000) 2023-08-27T23:36:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:00 smithi136 ceph-mon[113531]: pgmap v4873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4206078647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:00.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:00.043835+0000) 2023-08-27T23:36:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:00 smithi093 ceph-mon[111335]: pgmap v4873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4206078647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:00 smithi093 ceph-mon[118902]: pgmap v4873: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4206078647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:00.796Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:00.797Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:01.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:01.044084+0000) 2023-08-27T23:36:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:02 smithi136 ceph-mon[113531]: pgmap v4874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/949376735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:02.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:02.044267+0000) 2023-08-27T23:36:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:02 smithi093 ceph-mon[111335]: pgmap v4874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/949376735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:02 smithi093 ceph-mon[118902]: pgmap v4874: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/949376735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:02.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:02.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:02.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:02.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:03.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:03.044530+0000) 2023-08-27T23:36:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:04 smithi136 ceph-mon[113531]: pgmap v4875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:04.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:04.044756+0000) 2023-08-27T23:36:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:04 smithi093 ceph-mon[111335]: pgmap v4875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:04 smithi093 ceph-mon[118902]: pgmap v4875: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1123626046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:05.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:05.044976+0000) 2023-08-27T23:36:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1123626046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1123626046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:36:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:36:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:36:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:06 smithi136 ceph-mon[113531]: pgmap v4876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:06.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:06.045238+0000) 2023-08-27T23:36:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:06 smithi093 ceph-mon[111335]: pgmap v4876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:06 smithi093 ceph-mon[118902]: pgmap v4876: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/875677963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:07.517 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:07.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:07.045473+0000) 2023-08-27T23:36:07.517 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/875677963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:07.519 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/875677963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:08 smithi136 ceph-mon[113531]: pgmap v4877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:08.626 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:08.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:08.045729+0000) 2023-08-27T23:36:08.626 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:08 smithi093 ceph-mon[111335]: pgmap v4877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:08.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:08 smithi093 ceph-mon[118902]: pgmap v4877: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:09.634 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:09.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:09.045987+0000) 2023-08-27T23:36:09.635 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:09.635 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:10 smithi136 ceph-mon[113531]: pgmap v4878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:10.456 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4243083625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:10.457 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:10.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:10.046201+0000) 2023-08-27T23:36:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:10 smithi093 ceph-mon[111335]: pgmap v4878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:10.664 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4243083625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:10.664 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:10 smithi093 ceph-mon[118902]: pgmap v4878: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:10.664 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4243083625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:10.796Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:11.272 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:10.797Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:11.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:11.046461+0000) 2023-08-27T23:36:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:12 smithi136 ceph-mon[113531]: pgmap v4879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1300848060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:12.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:12.046672+0000) 2023-08-27T23:36:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:12 smithi093 ceph-mon[111335]: pgmap v4879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1300848060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:12 smithi093 ceph-mon[118902]: pgmap v4879: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1300848060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:12.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:12.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:13.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:13.046944+0000) 2023-08-27T23:36:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:14.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:14.047149+0000) 2023-08-27T23:36:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:14 smithi136 ceph-mon[113531]: pgmap v4880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:36:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1405804820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[111335]: pgmap v4880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1405804820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[118902]: pgmap v4880: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:36:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1405804820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:15.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:15.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:15.047288+0000) 2023-08-27T23:36:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:36:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:36:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:36:16.086 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:16.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:16.047473+0000) 2023-08-27T23:36:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:16 smithi136 ceph-mon[113531]: pgmap v4881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[111335]: pgmap v4881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[118902]: pgmap v4881: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:17.343 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:17.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:17.047664+0000) 2023-08-27T23:36:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1731509045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:36:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:36:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1731509045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:36:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:36:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1731509045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:17.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:36:17.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:36:17.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:36:18.344 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:18.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:18.047913+0000) 2023-08-27T23:36:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:18 smithi136 ceph-mon[113531]: pgmap v4882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:18 smithi093 ceph-mon[111335]: pgmap v4882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:18 smithi093 ceph-mon[118902]: pgmap v4882: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:19.348 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:19.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:19.048169+0000) 2023-08-27T23:36:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/185277003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/185277003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/185277003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:20.354 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:20.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:20.048409+0000) 2023-08-27T23:36:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:20 smithi136 ceph-mon[113531]: pgmap v4883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:20 smithi093 ceph-mon[111335]: pgmap v4883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:20 smithi093 ceph-mon[118902]: pgmap v4883: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:20.796Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:20.797Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:21.355 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:21.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:21.048645+0000) 2023-08-27T23:36:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/113449162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:21.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/113449162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/113449162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:22.360 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:22.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:22.048869+0000) 2023-08-27T23:36:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:22 smithi136 ceph-mon[113531]: pgmap v4884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:22 smithi093 ceph-mon[111335]: pgmap v4884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:22 smithi093 ceph-mon[118902]: pgmap v4884: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:22.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:22.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:23.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:23.049152+0000) 2023-08-27T23:36:24.363 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:24.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:24.049385+0000) 2023-08-27T23:36:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:24 smithi136 ceph-mon[113531]: pgmap v4885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4014358747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:24 smithi093 ceph-mon[111335]: pgmap v4885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4014358747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:24 smithi093 ceph-mon[118902]: pgmap v4885: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4014358747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:25.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:25.049646+0000) 2023-08-27T23:36:25.745 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:36:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:36:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:36:26.367 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:26.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:26.049906+0000) 2023-08-27T23:36:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:26 smithi136 ceph-mon[113531]: pgmap v4886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3603072440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:26 smithi093 ceph-mon[111335]: pgmap v4886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3603072440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:26 smithi093 ceph-mon[118902]: pgmap v4886: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3603072440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:27.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:27.050141+0000) 2023-08-27T23:36:28.369 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:28.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:28.050383+0000) 2023-08-27T23:36:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:28 smithi136 ceph-mon[113531]: pgmap v4887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3397220377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:28 smithi093 ceph-mon[111335]: pgmap v4887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3397220377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:28 smithi093 ceph-mon[118902]: pgmap v4887: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3397220377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:29.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:29.050629+0000) 2023-08-27T23:36:30.372 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:30.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:30.050876+0000) 2023-08-27T23:36:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:30 smithi136 ceph-mon[113531]: pgmap v4888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:30 smithi093 ceph-mon[111335]: pgmap v4888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:30 smithi093 ceph-mon[118902]: pgmap v4888: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:30.798Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:30.798Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:31.379 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:31.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:31.051167+0000) 2023-08-27T23:36:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2401945224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2401945224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2401945224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:32.381 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:32.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:32.051453+0000) 2023-08-27T23:36:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:32 smithi136 ceph-mon[113531]: pgmap v4889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:32 smithi093 ceph-mon[111335]: pgmap v4889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:32 smithi093 ceph-mon[118902]: pgmap v4889: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:32.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:32.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:33.387 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:33.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:33.051701+0000) 2023-08-27T23:36:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1636031021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1636031021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1636031021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:34.389 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:34.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:34.051947+0000) 2023-08-27T23:36:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:34 smithi136 ceph-mon[113531]: pgmap v4890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:34 smithi093 ceph-mon[111335]: pgmap v4890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:34 smithi093 ceph-mon[118902]: pgmap v4890: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:35.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:35.052217+0000) 2023-08-27T23:36:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:36:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:36:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:36:36.391 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:36.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:36.052477+0000) 2023-08-27T23:36:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:36 smithi136 ceph-mon[113531]: pgmap v4891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1266787294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:36 smithi093 ceph-mon[111335]: pgmap v4891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1266787294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:36 smithi093 ceph-mon[118902]: pgmap v4891: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1266787294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:37.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:37.052659+0000) 2023-08-27T23:36:38.394 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:38.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:38.052888+0000) 2023-08-27T23:36:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:38 smithi136 ceph-mon[113531]: pgmap v4892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1908889562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:38 smithi093 ceph-mon[111335]: pgmap v4892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1908889562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:38 smithi093 ceph-mon[118902]: pgmap v4892: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1908889562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:39.401 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:39.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:39.053132+0000) 2023-08-27T23:36:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:40.402 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:40.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:40.053364+0000) 2023-08-27T23:36:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:40 smithi136 ceph-mon[113531]: pgmap v4893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1820683570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:40 smithi093 ceph-mon[111335]: pgmap v4893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1820683570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:40 smithi093 ceph-mon[118902]: pgmap v4893: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1820683570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:40.799Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:40.800Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:41.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:41.053625+0000) 2023-08-27T23:36:42.404 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:42.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:42.053871+0000) 2023-08-27T23:36:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:42 smithi136 ceph-mon[113531]: pgmap v4894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:42.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:42 smithi093 ceph-mon[111335]: pgmap v4894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:42 smithi093 ceph-mon[118902]: pgmap v4894: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:42.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:42.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:43.409 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:43.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:43.054137+0000) 2023-08-27T23:36:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/672380543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/672380543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/672380543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:44.410 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:44.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:44.054384+0000) 2023-08-27T23:36:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:44 smithi136 ceph-mon[113531]: pgmap v4895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:44 smithi093 ceph-mon[111335]: pgmap v4895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:44 smithi093 ceph-mon[118902]: pgmap v4895: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:45.414 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:45.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:45.054650+0000) 2023-08-27T23:36:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3957757567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3957757567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3957757567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:36:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:36:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:36:46.416 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:46.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:46.054951+0000) 2023-08-27T23:36:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:46 smithi136 ceph-mon[113531]: pgmap v4896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:46 smithi093 ceph-mon[111335]: pgmap v4896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:46 smithi093 ceph-mon[118902]: pgmap v4896: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:47.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:47.055219+0000) 2023-08-27T23:36:48.418 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:48.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:48.055469+0000) 2023-08-27T23:36:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:48 smithi136 ceph-mon[113531]: pgmap v4897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1626482112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:48 smithi093 ceph-mon[111335]: pgmap v4897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1626482112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:48 smithi093 ceph-mon[118902]: pgmap v4897: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1626482112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:49.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:49.055741+0000) 2023-08-27T23:36:50.421 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:50.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:50.055986+0000) 2023-08-27T23:36:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:50 smithi136 ceph-mon[113531]: pgmap v4898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:50.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3531854989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:50 smithi093 ceph-mon[111335]: pgmap v4898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3531854989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:50 smithi093 ceph-mon[118902]: pgmap v4898: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3531854989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:50.799Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:50.800Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:51.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:51.056241+0000) 2023-08-27T23:36:52.424 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:52.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:52.056446+0000) 2023-08-27T23:36:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:52 smithi136 ceph-mon[113531]: pgmap v4899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4086777088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:52 smithi093 ceph-mon[118902]: pgmap v4899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4086777088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:52 smithi093 ceph-mon[111335]: pgmap v4899: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4086777088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:52.284Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:36:52.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:36:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:53.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:53.056733+0000) 2023-08-27T23:36:54.427 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:54.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:54.057006+0000) 2023-08-27T23:36:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:54 smithi136 ceph-mon[113531]: pgmap v4900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:54 smithi093 ceph-mon[111335]: pgmap v4900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:54 smithi093 ceph-mon[118902]: pgmap v4900: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:36:55.436 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:55.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:55.057302+0000) 2023-08-27T23:36:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/244616126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/244616126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/244616126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:36:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:36:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:36:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:56.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:56.057565+0000) 2023-08-27T23:36:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:56 smithi136 ceph-mon[113531]: pgmap v4901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:56 smithi093 ceph-mon[111335]: pgmap v4901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:56 smithi093 ceph-mon[118902]: pgmap v4901: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:57.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:57.057796+0000) 2023-08-27T23:36:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1935228267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1935228267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1935228267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:58.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:58.057997+0000) 2023-08-27T23:36:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:58 smithi093 ceph-mon[111335]: pgmap v4902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:58 smithi093 ceph-mon[118902]: pgmap v4902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:58 smithi136 ceph-mon[113531]: pgmap v4902: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:36:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:36:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:36:59.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:36:59.058199+0000) 2023-08-27T23:36:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:36:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4107974976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:36:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4107974976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:36:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:36:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4107974976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:00.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:00.058421+0000) 2023-08-27T23:37:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:00 smithi093 ceph-mon[111335]: pgmap v4903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:00 smithi093 ceph-mon[118902]: pgmap v4903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:00 smithi136 ceph-mon[113531]: pgmap v4903: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:00.800Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:00.801Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:01.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:01.058689+0000) 2023-08-27T23:37:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:02.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:02.058935+0000) 2023-08-27T23:37:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:02 smithi093 ceph-mon[118902]: pgmap v4904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/761255458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:02 smithi093 ceph-mon[111335]: pgmap v4904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/761255458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:02.284Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:02.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:02 smithi136 ceph-mon[113531]: pgmap v4904: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/761255458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:03.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:03.059153+0000) 2023-08-27T23:37:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:04.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:04.059399+0000) 2023-08-27T23:37:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:04 smithi093 ceph-mon[111335]: pgmap v4905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3685009037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:04 smithi093 ceph-mon[118902]: pgmap v4905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3685009037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:04 smithi136 ceph-mon[113531]: pgmap v4905: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3685009037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:05.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:05.059655+0000) 2023-08-27T23:37:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:37:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:37:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:37:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:06.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:06.059901+0000) 2023-08-27T23:37:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:06 smithi093 ceph-mon[111335]: pgmap v4906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:06 smithi093 ceph-mon[118902]: pgmap v4906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:06.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:06 smithi136 ceph-mon[113531]: pgmap v4906: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:07.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:07.060136+0000) 2023-08-27T23:37:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3184603357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3184603357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3184603357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:08.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:08.060377+0000) 2023-08-27T23:37:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:08 smithi093 ceph-mon[111335]: pgmap v4907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:08 smithi093 ceph-mon[118902]: pgmap v4907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:08 smithi136 ceph-mon[113531]: pgmap v4907: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:09.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:09.060617+0000) 2023-08-27T23:37:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/485965040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/485965040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/485965040' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:10.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:10.060889+0000) 2023-08-27T23:37:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:10 smithi093 ceph-mon[111335]: pgmap v4908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:10 smithi093 ceph-mon[118902]: pgmap v4908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:10 smithi136 ceph-mon[113531]: pgmap v4908: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:11.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:10.801Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:10.802Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:11.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:11.061161+0000) 2023-08-27T23:37:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1165548245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1165548245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1165548245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:12.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:12.061409+0000) 2023-08-27T23:37:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:12 smithi093 ceph-mon[111335]: pgmap v4909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:12 smithi093 ceph-mon[118902]: pgmap v4909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:12.285Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:12.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:12 smithi136 ceph-mon[113531]: pgmap v4909: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:13.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:13.061684+0000) 2023-08-27T23:37:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:14.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:14.061912+0000) 2023-08-27T23:37:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:14 smithi093 ceph-mon[111335]: pgmap v4910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3643616089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:14.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:14 smithi093 ceph-mon[118902]: pgmap v4910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:14.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3643616089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:14 smithi136 ceph-mon[113531]: pgmap v4910: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3643616089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:15.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:15.062178+0000) 2023-08-27T23:37:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:37:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:37:15] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:37:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:16.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:16.062388+0000) 2023-08-27T23:37:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:16 smithi093 ceph-mon[111335]: pgmap v4911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/287667392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:16 smithi093 ceph-mon[118902]: pgmap v4911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/287667392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:16 smithi136 ceph-mon[113531]: pgmap v4911: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/287667392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:17.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:17.062567+0000) 2023-08-27T23:37:17.860 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:37:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:37:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:37:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:18.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:18.062771+0000) 2023-08-27T23:37:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:18 smithi136 ceph-mon[113531]: pgmap v4912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3788179399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:18 smithi093 ceph-mon[111335]: pgmap v4912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3788179399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:18 smithi093 ceph-mon[118902]: pgmap v4912: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3788179399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:19.129 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:19.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:19.062951+0000) 2023-08-27T23:37:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:20.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:20.063187+0000) 2023-08-27T23:37:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:20 smithi136 ceph-mon[113531]: pgmap v4913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:37:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:37:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:20.802Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:20.803Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[111335]: pgmap v4913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:37:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:37:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[118902]: pgmap v4913: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:37:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:37:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:37:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:21.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:21.063466+0000) 2023-08-27T23:37:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1623486006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1623486006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1623486006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:22.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:22.063740+0000) 2023-08-27T23:37:22.620 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:22.285Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:23.185 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:22.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:23.186 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:22 smithi136 ceph-mon[113531]: pgmap v4914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:23.186 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:22 smithi093 ceph-mon[111335]: pgmap v4914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:23.186 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:22 smithi093 ceph-mon[118902]: pgmap v4914: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:23.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:23.064026+0000) 2023-08-27T23:37:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3972593097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3972593097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3972593097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:24.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:24.064335+0000) 2023-08-27T23:37:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:24 smithi136 ceph-mon[113531]: pgmap v4915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:24 smithi093 ceph-mon[111335]: pgmap v4915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:24 smithi093 ceph-mon[118902]: pgmap v4915: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:25.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:25.064605+0000) 2023-08-27T23:37:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:37:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:37:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:37:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:26 smithi136 ceph-mon[113531]: pgmap v4916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3158791876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:26.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:26.064811+0000) 2023-08-27T23:37:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:26 smithi093 ceph-mon[111335]: pgmap v4916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3158791876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:26 smithi093 ceph-mon[118902]: pgmap v4916: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3158791876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:27.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:27.065016+0000) 2023-08-27T23:37:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:28 smithi136 ceph-mon[113531]: pgmap v4917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2630599969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:28.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:28.065320+0000) 2023-08-27T23:37:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:28 smithi093 ceph-mon[118902]: pgmap v4917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2630599969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:28 smithi093 ceph-mon[111335]: pgmap v4917: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2630599969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:29.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:29.065632+0000) 2023-08-27T23:37:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:30 smithi136 ceph-mon[113531]: pgmap v4918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:30.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:30.065930+0000) 2023-08-27T23:37:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:30 smithi093 ceph-mon[111335]: pgmap v4918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:30 smithi093 ceph-mon[118902]: pgmap v4918: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:31.177 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:30.803Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:31.177 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:30.803Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2924351354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:31.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:31.066182+0000) 2023-08-27T23:37:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2924351354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2924351354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:32 smithi136 ceph-mon[113531]: pgmap v4919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:32.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:32.066394+0000) 2023-08-27T23:37:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:32 smithi093 ceph-mon[111335]: pgmap v4919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:32 smithi093 ceph-mon[118902]: pgmap v4919: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:32.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:32.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:32.287Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3483050021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:33.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:33.066666+0000) 2023-08-27T23:37:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3483050021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3483050021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:34 smithi136 ceph-mon[113531]: pgmap v4920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:34.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:34.066910+0000) 2023-08-27T23:37:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:34 smithi093 ceph-mon[111335]: pgmap v4920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:34 smithi093 ceph-mon[118902]: pgmap v4920: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4251707519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:35.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:35.067119+0000) 2023-08-27T23:37:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4251707519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4251707519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:37:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:37:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:37:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:36 smithi136 ceph-mon[113531]: pgmap v4921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:36.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:36.067381+0000) 2023-08-27T23:37:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:36 smithi093 ceph-mon[111335]: pgmap v4921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:36 smithi093 ceph-mon[118902]: pgmap v4921: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:37.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:37.067590+0000) 2023-08-27T23:37:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:38 smithi136 ceph-mon[113531]: pgmap v4922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2274856466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:38.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:38.067814+0000) 2023-08-27T23:37:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:38 smithi093 ceph-mon[111335]: pgmap v4922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2274856466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:38 smithi093 ceph-mon[118902]: pgmap v4922: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2274856466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:39.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:39.068050+0000) 2023-08-27T23:37:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:40 smithi136 ceph-mon[113531]: pgmap v4923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2529612446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:40.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:40.068242+0000) 2023-08-27T23:37:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:40 smithi093 ceph-mon[111335]: pgmap v4923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2529612446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:40 smithi093 ceph-mon[118902]: pgmap v4923: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2529612446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:40.804Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:40.805Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:41.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:41.068486+0000) 2023-08-27T23:37:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:42 smithi136 ceph-mon[113531]: pgmap v4924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:42.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:42.068739+0000) 2023-08-27T23:37:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:42 smithi093 ceph-mon[111335]: pgmap v4924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:42 smithi093 ceph-mon[118902]: pgmap v4924: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:42.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:42.287Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3972268780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:43.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:43.068988+0000) 2023-08-27T23:37:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3972268780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3972268780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:44 smithi136 ceph-mon[113531]: pgmap v4925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:44.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:44.069235+0000) 2023-08-27T23:37:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:44 smithi093 ceph-mon[111335]: pgmap v4925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:44 smithi093 ceph-mon[118902]: pgmap v4925: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2466081911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:45.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:45.069452+0000) 2023-08-27T23:37:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2466081911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2466081911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:37:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:37:45] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:37:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:46 smithi136 ceph-mon[113531]: pgmap v4926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:46.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:46.069729+0000) 2023-08-27T23:37:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:46 smithi093 ceph-mon[111335]: pgmap v4926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:46 smithi093 ceph-mon[118902]: pgmap v4926: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/466399046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:47.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:47.069982+0000) 2023-08-27T23:37:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/466399046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/466399046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:48 smithi136 ceph-mon[113531]: pgmap v4927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:48.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:48.070209+0000) 2023-08-27T23:37:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:48 smithi093 ceph-mon[111335]: pgmap v4927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:48 smithi093 ceph-mon[118902]: pgmap v4927: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:49.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:49.070469+0000) 2023-08-27T23:37:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:50 smithi136 ceph-mon[113531]: pgmap v4928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/186003113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:50.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:50.070737+0000) 2023-08-27T23:37:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:50 smithi093 ceph-mon[111335]: pgmap v4928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/186003113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:50 smithi093 ceph-mon[118902]: pgmap v4928: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/186003113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:50.804Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:50.805Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:51.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:51.071006+0000) 2023-08-27T23:37:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:52 smithi136 ceph-mon[113531]: pgmap v4929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1927054638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:52.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:52.071264+0000) 2023-08-27T23:37:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:52 smithi093 ceph-mon[111335]: pgmap v4929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1927054638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:52 smithi093 ceph-mon[118902]: pgmap v4929: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1927054638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:52.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:37:52.287Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:37:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:53.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:53.071479+0000) 2023-08-27T23:37:54.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:54 smithi093 ceph-mon[111335]: pgmap v4930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:54.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:54.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/687302581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:54.231 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:54 smithi093 ceph-mon[118902]: pgmap v4930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:54.232 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:54.232 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/687302581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:54.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:54.071730+0000) 2023-08-27T23:37:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:54 smithi136 ceph-mon[113531]: pgmap v4930: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:37:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/687302581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:55.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:55.071958+0000) 2023-08-27T23:37:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:37:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:37:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:37:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:56 smithi136 ceph-mon[113531]: pgmap v4931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:56.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:56.072212+0000) 2023-08-27T23:37:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:56 smithi093 ceph-mon[111335]: pgmap v4931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:56 smithi093 ceph-mon[118902]: pgmap v4931: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1010847158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:57.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:57.072445+0000) 2023-08-27T23:37:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1010847158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1010847158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:58 smithi136 ceph-mon[113531]: pgmap v4932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:58.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:58.072691+0000) 2023-08-27T23:37:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:58 smithi093 ceph-mon[111335]: pgmap v4932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:58 smithi093 ceph-mon[118902]: pgmap v4932: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:37:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:37:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2484228858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:37:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:37:59.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:37:59.072938+0000) 2023-08-27T23:37:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:37:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2484228858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:37:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:37:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2484228858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:00 smithi136 ceph-mon[113531]: pgmap v4933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:00.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:00.073188+0000) 2023-08-27T23:38:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:00 smithi093 ceph-mon[111335]: pgmap v4933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:00 smithi093 ceph-mon[118902]: pgmap v4933: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:01.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:00.805Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:00.806Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:01.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:01.073419+0000) 2023-08-27T23:38:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:02 smithi136 ceph-mon[113531]: pgmap v4934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/740645533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:02.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:02.073655+0000) 2023-08-27T23:38:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:02 smithi093 ceph-mon[111335]: pgmap v4934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/740645533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:02 smithi093 ceph-mon[118902]: pgmap v4934: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/740645533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:02.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:02.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:03.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:03.073904+0000) 2023-08-27T23:38:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:04 smithi136 ceph-mon[113531]: pgmap v4935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1415733240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:04.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:04.074153+0000) 2023-08-27T23:38:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:04 smithi093 ceph-mon[111335]: pgmap v4935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1415733240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:04 smithi093 ceph-mon[118902]: pgmap v4935: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1415733240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:05.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:05.074409+0000) 2023-08-27T23:38:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:38:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:38:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:38:06.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:06 smithi136 ceph-mon[113531]: pgmap v4936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:06.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1154085966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:06.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:06.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:06.074667+0000) 2023-08-27T23:38:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:06 smithi093 ceph-mon[111335]: pgmap v4936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1154085966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:06 smithi093 ceph-mon[118902]: pgmap v4936: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1154085966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:07.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:07.074873+0000) 2023-08-27T23:38:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:08 smithi136 ceph-mon[113531]: pgmap v4937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:08.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:08.075160+0000) 2023-08-27T23:38:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:08 smithi093 ceph-mon[111335]: pgmap v4937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:08 smithi093 ceph-mon[118902]: pgmap v4937: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1172855865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:09.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:09.075414+0000) 2023-08-27T23:38:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1172855865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1172855865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:10 smithi136 ceph-mon[113531]: pgmap v4938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:10.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:10.075651+0000) 2023-08-27T23:38:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:10 smithi093 ceph-mon[111335]: pgmap v4938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:10 smithi093 ceph-mon[118902]: pgmap v4938: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:10.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:10.806Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3300451361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:11.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:11.075916+0000) 2023-08-27T23:38:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3300451361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3300451361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:12 smithi136 ceph-mon[113531]: pgmap v4939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:12.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:12.076137+0000) 2023-08-27T23:38:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:12 smithi093 ceph-mon[111335]: pgmap v4939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:12 smithi093 ceph-mon[118902]: pgmap v4939: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:12.288Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:12.289Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/972240624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:13.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:13.076414+0000) 2023-08-27T23:38:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/972240624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/972240624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:14 smithi136 ceph-mon[113531]: pgmap v4940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:14.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:14.076664+0000) 2023-08-27T23:38:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:14 smithi093 ceph-mon[111335]: pgmap v4940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:14 smithi093 ceph-mon[118902]: pgmap v4940: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:15.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:15.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:15.076895+0000) 2023-08-27T23:38:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:38:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:38:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:38:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:16 smithi136 ceph-mon[113531]: pgmap v4941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1894461342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:16.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:16.077189+0000) 2023-08-27T23:38:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:16 smithi093 ceph-mon[111335]: pgmap v4941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1894461342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:16 smithi093 ceph-mon[118902]: pgmap v4941: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1894461342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:17.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:17.077358+0000) 2023-08-27T23:38:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:18 smithi136 ceph-mon[113531]: pgmap v4942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/103008792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:18.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:18.077603+0000) 2023-08-27T23:38:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:18 smithi093 ceph-mon[118902]: pgmap v4942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/103008792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:18 smithi093 ceph-mon[111335]: pgmap v4942: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/103008792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:19.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:19.077867+0000) 2023-08-27T23:38:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:20 smithi136 ceph-mon[113531]: pgmap v4943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:38:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:20.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:20.078026+0000) 2023-08-27T23:38:20.582 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:20 smithi093 ceph-mon[118902]: pgmap v4943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:20.582 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:38:20.582 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:20 smithi093 ceph-mon[111335]: pgmap v4943: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:20.582 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:38:20.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:20.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:20.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:21.293 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:21.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:21.078219+0000) 2023-08-27T23:38:21.430 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/279277741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:21.430 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/279277741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/279277741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:22.311 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:22.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:22.078398+0000) 2023-08-27T23:38:22.688 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:22.288Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:22.688 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:22.289Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:22 smithi136 ceph-mon[113531]: pgmap v4944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[111335]: pgmap v4944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[118902]: pgmap v4944: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:23.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:23.078574+0000) 2023-08-27T23:38:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2077408651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:38:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:38:23.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2077408651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:38:23.993 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:38:23.993 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:24.079 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2077408651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:24.079 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:38:24.079 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:38:24.079 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:38:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:24.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:24.078821+0000) 2023-08-27T23:38:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:24 smithi093 ceph-mon[111335]: pgmap v4945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:24 smithi093 ceph-mon[118902]: pgmap v4945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:25.080 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:24 smithi136 ceph-mon[113531]: pgmap v4945: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:25.080 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:25.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:25.079087+0000) 2023-08-27T23:38:25.699 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:38:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:38:25] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:38:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2329943991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:25.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2329943991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:26.079 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2329943991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:26.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:26.079376+0000) 2023-08-27T23:38:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:26 smithi093 ceph-mon[111335]: pgmap v4946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:26 smithi093 ceph-mon[118902]: pgmap v4946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:27.080 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:26 smithi136 ceph-mon[113531]: pgmap v4946: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:27.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:27.079633+0000) 2023-08-27T23:38:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3436410496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3436410496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:28.080 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3436410496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:28.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:28.079882+0000) 2023-08-27T23:38:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:28 smithi093 ceph-mon[111335]: pgmap v4947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:28 smithi093 ceph-mon[118902]: pgmap v4947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:29.080 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:28 smithi136 ceph-mon[113531]: pgmap v4947: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:29.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:29.080150+0000) 2023-08-27T23:38:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:30.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:30.080447+0000) 2023-08-27T23:38:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:30 smithi093 ceph-mon[111335]: pgmap v4948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3612994476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:30 smithi093 ceph-mon[118902]: pgmap v4948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3612994476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:30.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:30.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:31.081 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:30 smithi136 ceph-mon[113531]: pgmap v4948: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:31.081 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3612994476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:31.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:31.080730+0000) 2023-08-27T23:38:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:32.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:32.081003+0000) 2023-08-27T23:38:32.712 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:32.289Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:32.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:32.290Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:32 smithi093 ceph-mon[111335]: pgmap v4949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/147694601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:32 smithi093 ceph-mon[118902]: pgmap v4949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/147694601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:33.081 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:32 smithi136 ceph-mon[113531]: pgmap v4949: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:33.081 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/147694601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:33.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:33.081299+0000) 2023-08-27T23:38:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:34.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:34.081581+0000) 2023-08-27T23:38:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:34 smithi093 ceph-mon[111335]: pgmap v4950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1230569651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:34 smithi093 ceph-mon[118902]: pgmap v4950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1230569651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:35.082 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:34 smithi136 ceph-mon[113531]: pgmap v4950: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:35.082 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1230569651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:35.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:35.081803+0000) 2023-08-27T23:38:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:38:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:38:35] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:38:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:36.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:36.082104+0000) 2023-08-27T23:38:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:36 smithi093 ceph-mon[111335]: pgmap v4951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:36 smithi093 ceph-mon[118902]: pgmap v4951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:37.082 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:36 smithi136 ceph-mon[113531]: pgmap v4951: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:37.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:37.082340+0000) 2023-08-27T23:38:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:37 smithi093 ceph-mon[118902]: pgmap v4952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3814928655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:37 smithi093 ceph-mon[111335]: pgmap v4952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3814928655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:38.083 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:37 smithi136 ceph-mon[113531]: pgmap v4952: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:38.083 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3814928655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:38.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:38.082590+0000) 2023-08-27T23:38:39.083 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:39.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:39.082863+0000) 2023-08-27T23:38:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:39 smithi136 ceph-mon[113531]: pgmap v4953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3161715060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:40.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:40.083099+0000) 2023-08-27T23:38:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:39 smithi093 ceph-mon[111335]: pgmap v4953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3161715060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:39 smithi093 ceph-mon[118902]: pgmap v4953: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3161715060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:40.807Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:40.808Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:41.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:41.083350+0000) 2023-08-27T23:38:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:42 smithi136 ceph-mon[113531]: pgmap v4954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3314898428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:42.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:42.083589+0000) 2023-08-27T23:38:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:42 smithi093 ceph-mon[111335]: pgmap v4954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3314898428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:42 smithi093 ceph-mon[118902]: pgmap v4954: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3314898428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:42.290Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:42.290Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:43.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:43.083804+0000) 2023-08-27T23:38:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:44 smithi136 ceph-mon[113531]: pgmap v4955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:44.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:44.084070+0000) 2023-08-27T23:38:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:44 smithi093 ceph-mon[111335]: pgmap v4955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:44 smithi093 ceph-mon[118902]: pgmap v4955: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1360803730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:45.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:45.084358+0000) 2023-08-27T23:38:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1360803730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:38:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:38:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:38:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1360803730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:46 smithi136 ceph-mon[113531]: pgmap v4956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:46.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:46.084646+0000) 2023-08-27T23:38:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:46 smithi093 ceph-mon[118902]: pgmap v4956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:46 smithi093 ceph-mon[111335]: pgmap v4956: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2152246601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:47.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:47.084853+0000) 2023-08-27T23:38:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2152246601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2152246601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:48 smithi136 ceph-mon[113531]: pgmap v4957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:48.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:48.085146+0000) 2023-08-27T23:38:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:48 smithi093 ceph-mon[111335]: pgmap v4957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:48 smithi093 ceph-mon[118902]: pgmap v4957: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/465464066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:49.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:49.085373+0000) 2023-08-27T23:38:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/465464066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/465464066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:50 smithi136 ceph-mon[113531]: pgmap v4958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:50.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:50.085618+0000) 2023-08-27T23:38:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:50 smithi093 ceph-mon[111335]: pgmap v4958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:50 smithi093 ceph-mon[118902]: pgmap v4958: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:50.808Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:50.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:51.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:51.085885+0000) 2023-08-27T23:38:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:52 smithi136 ceph-mon[113531]: pgmap v4959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3301247010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:52.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:52.086125+0000) 2023-08-27T23:38:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:52 smithi093 ceph-mon[111335]: pgmap v4959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3301247010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:52 smithi093 ceph-mon[118902]: pgmap v4959: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3301247010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:52.290Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:52.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:38:52.291Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:38:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:53.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:53.086334+0000) 2023-08-27T23:38:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:54 smithi136 ceph-mon[113531]: pgmap v4960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/950739847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:54.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:54.086607+0000) 2023-08-27T23:38:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:54 smithi093 ceph-mon[118902]: pgmap v4960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/950739847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:54 smithi093 ceph-mon[111335]: pgmap v4960: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/950739847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:54.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:38:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:55.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:55.086898+0000) 2023-08-27T23:38:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:38:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:38:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:38:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:56 smithi136 ceph-mon[113531]: pgmap v4961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3354390673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:56.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:56.087189+0000) 2023-08-27T23:38:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:56 smithi093 ceph-mon[111335]: pgmap v4961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3354390673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:56 smithi093 ceph-mon[118902]: pgmap v4961: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3354390673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:57.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:57.087433+0000) 2023-08-27T23:38:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:58 smithi136 ceph-mon[113531]: pgmap v4962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:58.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:58.087710+0000) 2023-08-27T23:38:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:58 smithi093 ceph-mon[118902]: pgmap v4962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:58 smithi093 ceph-mon[111335]: pgmap v4962: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:38:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:38:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2559752955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:38:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:38:59.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:38:59.087944+0000) 2023-08-27T23:38:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:38:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2559752955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:38:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:38:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2559752955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:00 smithi136 ceph-mon[113531]: pgmap v4963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:00.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:00.088180+0000) 2023-08-27T23:39:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:00 smithi093 ceph-mon[111335]: pgmap v4963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:00 smithi093 ceph-mon[118902]: pgmap v4963: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:01.226 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:00.809Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:01.227 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:00.810Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2089942111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:01.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:01.088449+0000) 2023-08-27T23:39:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2089942111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2089942111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:02 smithi136 ceph-mon[113531]: pgmap v4964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:02.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:02.088691+0000) 2023-08-27T23:39:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:02 smithi093 ceph-mon[118902]: pgmap v4964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:02 smithi093 ceph-mon[111335]: pgmap v4964: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:02.290Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:02.291Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:03.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:03.088883+0000) 2023-08-27T23:39:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:04 smithi136 ceph-mon[113531]: pgmap v4965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/876854585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:04.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:04.089128+0000) 2023-08-27T23:39:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:04 smithi093 ceph-mon[111335]: pgmap v4965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/876854585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:04 smithi093 ceph-mon[118902]: pgmap v4965: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/876854585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:05.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:05.089383+0000) 2023-08-27T23:39:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:39:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:39:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:39:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:06 smithi136 ceph-mon[113531]: pgmap v4966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:06.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2576857810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:06.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:06.089700+0000) 2023-08-27T23:39:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:06 smithi093 ceph-mon[111335]: pgmap v4966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2576857810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:06 smithi093 ceph-mon[118902]: pgmap v4966: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2576857810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:07.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:07.089967+0000) 2023-08-27T23:39:08.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:08 smithi136 ceph-mon[113531]: pgmap v4967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:08.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1181053021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:08.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:08.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:08.090218+0000) 2023-08-27T23:39:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:08 smithi093 ceph-mon[111335]: pgmap v4967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1181053021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:08 smithi093 ceph-mon[118902]: pgmap v4967: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1181053021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:09.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:09.090469+0000) 2023-08-27T23:39:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:10 smithi136 ceph-mon[113531]: pgmap v4968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:10.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:10.090734+0000) 2023-08-27T23:39:10.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:10 smithi093 ceph-mon[111335]: pgmap v4968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:10 smithi093 ceph-mon[118902]: pgmap v4968: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:10.810Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:10.811Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:11.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/190591261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:11.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:11.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:11.091010+0000) 2023-08-27T23:39:11.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/190591261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:11.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/190591261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:12 smithi136 ceph-mon[113531]: pgmap v4969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:12.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:12.091257+0000) 2023-08-27T23:39:12.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:12 smithi093 ceph-mon[111335]: pgmap v4969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:12.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:12 smithi093 ceph-mon[118902]: pgmap v4969: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:12.743 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:12.291Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:12.743 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:12.292Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/579457632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:13.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:13.091451+0000) 2023-08-27T23:39:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/579457632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/579457632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:14 smithi136 ceph-mon[113531]: pgmap v4970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:14.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:14.091718+0000) 2023-08-27T23:39:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:14 smithi093 ceph-mon[111335]: pgmap v4970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:14 smithi093 ceph-mon[118902]: pgmap v4970: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1837696291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:15.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:15.091957+0000) 2023-08-27T23:39:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1837696291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1837696291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:39:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:39:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:39:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:16 smithi136 ceph-mon[113531]: pgmap v4971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:16.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:16.092214+0000) 2023-08-27T23:39:16.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:16 smithi093 ceph-mon[111335]: pgmap v4971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:16 smithi093 ceph-mon[118902]: pgmap v4971: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:17.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:17.092456+0000) 2023-08-27T23:39:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:18 smithi136 ceph-mon[113531]: pgmap v4972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2235019043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:18.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:18.092724+0000) 2023-08-27T23:39:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:18 smithi093 ceph-mon[111335]: pgmap v4972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2235019043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:18 smithi093 ceph-mon[118902]: pgmap v4972: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2235019043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:19.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:19.092899+0000) 2023-08-27T23:39:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:20 smithi136 ceph-mon[113531]: pgmap v4973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1586965519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:20.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:20.093142+0000) 2023-08-27T23:39:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:20 smithi093 ceph-mon[111335]: pgmap v4973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:20.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1586965519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:20.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:20 smithi093 ceph-mon[118902]: pgmap v4973: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:20.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1586965519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:20.811Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:20.812Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:21.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:21.093365+0000) 2023-08-27T23:39:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:22 smithi136 ceph-mon[113531]: pgmap v4974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:22.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:22.093600+0000) 2023-08-27T23:39:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:22 smithi093 ceph-mon[111335]: pgmap v4974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:22 smithi093 ceph-mon[118902]: pgmap v4974: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:22.292Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:22.292Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1048697059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:39:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:23.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:23.093858+0000) 2023-08-27T23:39:23.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1048697059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:23.715 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:39:23.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1048697059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:23.716 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:39:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:24 smithi136 ceph-mon[113531]: pgmap v4975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:24.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:24.094067+0000) 2023-08-27T23:39:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:24 smithi093 ceph-mon[111335]: pgmap v4975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:24 smithi093 ceph-mon[118902]: pgmap v4975: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1225419947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:25.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:25.094217+0000) 2023-08-27T23:39:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:39:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:39:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:39:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1225419947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:25.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1225419947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:26.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:26.094395+0000) 2023-08-27T23:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:26 smithi136 ceph-mon[113531]: pgmap v4976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:39:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:39:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[111335]: pgmap v4976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[118902]: pgmap v4976: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:39:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:39:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:27.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:27.094627+0000) 2023-08-27T23:39:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/32558493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/32558493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/32558493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:28.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:28.094901+0000) 2023-08-27T23:39:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:28 smithi093 ceph-mon[111335]: pgmap v4977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:28 smithi093 ceph-mon[118902]: pgmap v4977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:29.095 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:28 smithi136 ceph-mon[113531]: pgmap v4977: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:29.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:29.095146+0000) 2023-08-27T23:39:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/501015938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/501015938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:30.095 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/501015938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:30.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:30.095365+0000) 2023-08-27T23:39:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:30 smithi093 ceph-mon[118902]: pgmap v4978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:30 smithi093 ceph-mon[111335]: pgmap v4978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:30.812Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:30.813Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:31.095 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:30 smithi136 ceph-mon[113531]: pgmap v4978: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:31.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:31.095617+0000) 2023-08-27T23:39:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:32.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:32.095857+0000) 2023-08-27T23:39:32.700 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:32.292Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:32.700 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:32.293Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:32 smithi093 ceph-mon[111335]: pgmap v4979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/127924156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:32 smithi093 ceph-mon[118902]: pgmap v4979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/127924156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:33.096 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:32 smithi136 ceph-mon[113531]: pgmap v4979: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:33.096 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/127924156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:33.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:33.096133+0000) 2023-08-27T23:39:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:34.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:34.096377+0000) 2023-08-27T23:39:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:34 smithi093 ceph-mon[111335]: pgmap v4980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/172642335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:34 smithi093 ceph-mon[118902]: pgmap v4980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/172642335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:35.100 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:34 smithi136 ceph-mon[113531]: pgmap v4980: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:35.100 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/172642335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:35.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:35.096643+0000) 2023-08-27T23:39:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:39:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:39:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:39:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:36.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:36.096926+0000) 2023-08-27T23:39:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:36 smithi093 ceph-mon[111335]: pgmap v4981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/958083094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:36 smithi093 ceph-mon[118902]: pgmap v4981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/958083094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:37.097 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:36 smithi136 ceph-mon[113531]: pgmap v4981: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:37.097 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/958083094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:37.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:37.097174+0000) 2023-08-27T23:39:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:38.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:38.097413+0000) 2023-08-27T23:39:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:38 smithi093 ceph-mon[111335]: pgmap v4982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:38 smithi093 ceph-mon[118902]: pgmap v4982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:39.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:38 smithi136 ceph-mon[113531]: pgmap v4982: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:39.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:39.097674+0000) 2023-08-27T23:39:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3383796928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3383796928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:40.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:40.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3383796928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:40.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:40.097920+0000) 2023-08-27T23:39:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:40 smithi093 ceph-mon[111335]: pgmap v4983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:40 smithi093 ceph-mon[118902]: pgmap v4983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:40.812Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:40.813Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:41.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:40 smithi136 ceph-mon[113531]: pgmap v4983: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:41.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:41.098165+0000) 2023-08-27T23:39:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4186215205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4186215205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:42.098 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4186215205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:42.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:42.098426+0000) 2023-08-27T23:39:42.725 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:42.292Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:42.725 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:42.293Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:42 smithi093 ceph-mon[111335]: pgmap v4984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:42 smithi093 ceph-mon[118902]: pgmap v4984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:43.099 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:42 smithi136 ceph-mon[113531]: pgmap v4984: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:43.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:43.098694+0000) 2023-08-27T23:39:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:44.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:44.098896+0000) 2023-08-27T23:39:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:44 smithi093 ceph-mon[111335]: pgmap v4985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1531471530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:44 smithi093 ceph-mon[118902]: pgmap v4985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1531471530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:45.099 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:44 smithi136 ceph-mon[113531]: pgmap v4985: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:45.099 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1531471530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:45.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:45.099117+0000) 2023-08-27T23:39:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:39:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:39:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:39:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:46 smithi136 ceph-mon[113531]: pgmap v4986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1947583450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:46.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:46.099376+0000) 2023-08-27T23:39:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:46 smithi093 ceph-mon[111335]: pgmap v4986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1947583450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:46 smithi093 ceph-mon[118902]: pgmap v4986: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1947583450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:47.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:47.099622+0000) 2023-08-27T23:39:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:48 smithi136 ceph-mon[113531]: pgmap v4987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:48.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:48.099880+0000) 2023-08-27T23:39:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:48 smithi093 ceph-mon[111335]: pgmap v4987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:48 smithi093 ceph-mon[118902]: pgmap v4987: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1411868665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:49.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:49.100072+0000) 2023-08-27T23:39:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1411868665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1411868665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:50 smithi136 ceph-mon[113531]: pgmap v4988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:50.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:50.100302+0000) 2023-08-27T23:39:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:50 smithi093 ceph-mon[111335]: pgmap v4988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:50 smithi093 ceph-mon[118902]: pgmap v4988: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:51.204 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:50.813Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:51.204 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:50.814Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2944098397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:51.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:51.100583+0000) 2023-08-27T23:39:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2944098397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2944098397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:52 smithi136 ceph-mon[113531]: pgmap v4989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:52.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:52.100828+0000) 2023-08-27T23:39:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:52 smithi093 ceph-mon[118902]: pgmap v4989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:52 smithi093 ceph-mon[111335]: pgmap v4989: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:52.293Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:52.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:39:52.294Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:39:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:53.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:53.101095+0000) 2023-08-27T23:39:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:54.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:54.101357+0000) 2023-08-27T23:39:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:54 smithi136 ceph-mon[113531]: pgmap v4990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/536599244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:54 smithi093 ceph-mon[111335]: pgmap v4990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/536599244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:54 smithi093 ceph-mon[118902]: pgmap v4990: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/536599244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:39:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:55.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:55.101635+0000) 2023-08-27T23:39:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:39:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:39:55] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-27T23:39:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:56 smithi136 ceph-mon[113531]: pgmap v4991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/814189133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:56.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:56.101905+0000) 2023-08-27T23:39:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:56 smithi093 ceph-mon[111335]: pgmap v4991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/814189133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:56 smithi093 ceph-mon[118902]: pgmap v4991: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/814189133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:57.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:57.102146+0000) 2023-08-27T23:39:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:58 smithi136 ceph-mon[113531]: pgmap v4992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:39:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2313049152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:58.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:58.102417+0000) 2023-08-27T23:39:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:58 smithi093 ceph-mon[111335]: pgmap v4992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:58.502 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:39:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2313049152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:58.502 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:58 smithi093 ceph-mon[118902]: pgmap v4992: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:39:58.503 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:39:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2313049152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:39:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:39:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:39:59.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:39:59.102694+0000) 2023-08-27T23:40:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:00 smithi136 ceph-mon[113531]: pgmap v4993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:40:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:00.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:00.102951+0000) 2023-08-27T23:40:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:00 smithi093 ceph-mon[111335]: pgmap v4993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:40:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:00 smithi093 ceph-mon[118902]: pgmap v4993: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:40:01.231 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:00.814Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:01.231 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:00.814Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3123446149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:01.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:01.103205+0000) 2023-08-27T23:40:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3123446149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3123446149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:02 smithi136 ceph-mon[113531]: pgmap v4994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:02.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:02.103461+0000) 2023-08-27T23:40:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:02 smithi093 ceph-mon[111335]: pgmap v4994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:02 smithi093 ceph-mon[118902]: pgmap v4994: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:02.294Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:02.295Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4105284446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:03.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:03.103745+0000) 2023-08-27T23:40:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4105284446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4105284446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:04 smithi136 ceph-mon[113531]: pgmap v4995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:04.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:04.103949+0000) 2023-08-27T23:40:04.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:04 smithi093 ceph-mon[111335]: pgmap v4995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:04.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:04 smithi093 ceph-mon[118902]: pgmap v4995: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:05.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:05.104193+0000) 2023-08-27T23:40:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:40:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:40:05] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-27T23:40:06.444 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:06 smithi136 ceph-mon[113531]: pgmap v4996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:06.444 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3139869363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:06.444 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:06.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:06.104380+0000) 2023-08-27T23:40:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:06 smithi093 ceph-mon[111335]: pgmap v4996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3139869363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:06 smithi093 ceph-mon[118902]: pgmap v4996: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3139869363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:07.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:07.104622+0000) 2023-08-27T23:40:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:08 smithi136 ceph-mon[113531]: pgmap v4997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2290612825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:08.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:08.104832+0000) 2023-08-27T23:40:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:08 smithi093 ceph-mon[111335]: pgmap v4997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2290612825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:08 smithi093 ceph-mon[118902]: pgmap v4997: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2290612825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:09.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:09.105087+0000) 2023-08-27T23:40:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:10 smithi136 ceph-mon[113531]: pgmap v4998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/457587568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:10.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:10.105340+0000) 2023-08-27T23:40:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:10 smithi093 ceph-mon[111335]: pgmap v4998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/457587568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:10 smithi093 ceph-mon[118902]: pgmap v4998: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/457587568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:10.814Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:10.815Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:11.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:11.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:11.105600+0000) 2023-08-27T23:40:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:12 smithi136 ceph-mon[113531]: pgmap v4999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:12.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:12.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:12.105840+0000) 2023-08-27T23:40:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:12 smithi093 ceph-mon[111335]: pgmap v4999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:12 smithi093 ceph-mon[118902]: pgmap v4999: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:12.294Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:12.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:12.295Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3589678317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:13.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:13.106110+0000) 2023-08-27T23:40:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3589678317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:13.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3589678317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:14 smithi136 ceph-mon[113531]: pgmap v5000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:14.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:14.106343+0000) 2023-08-27T23:40:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:14 smithi093 ceph-mon[111335]: pgmap v5000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:14 smithi093 ceph-mon[118902]: pgmap v5000: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3228973077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:15.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:15.106641+0000) 2023-08-27T23:40:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3228973077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3228973077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:40:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:40:15] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:40:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:16 smithi136 ceph-mon[113531]: pgmap v5001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:16.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:16.106903+0000) 2023-08-27T23:40:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:16 smithi093 ceph-mon[111335]: pgmap v5001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:16 smithi093 ceph-mon[118902]: pgmap v5001: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1129550857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:17.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:17.107149+0000) 2023-08-27T23:40:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1129550857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1129550857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:18 smithi136 ceph-mon[113531]: pgmap v5002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:18.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:18.107402+0000) 2023-08-27T23:40:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:18 smithi093 ceph-mon[111335]: pgmap v5002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:18 smithi093 ceph-mon[118902]: pgmap v5002: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:19.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:19.107643+0000) 2023-08-27T23:40:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:20 smithi136 ceph-mon[113531]: pgmap v5003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/308979669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:20.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:20.107889+0000) 2023-08-27T23:40:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:20 smithi093 ceph-mon[111335]: pgmap v5003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/308979669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:20 smithi093 ceph-mon[118902]: pgmap v5003: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/308979669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:20.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:20.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:21.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:21.108156+0000) 2023-08-27T23:40:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:22 smithi136 ceph-mon[113531]: pgmap v5004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4226623395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:22.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:22.108341+0000) 2023-08-27T23:40:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:22 smithi093 ceph-mon[111335]: pgmap v5004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4226623395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:22.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:22 smithi093 ceph-mon[118902]: pgmap v5004: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:22.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4226623395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:22.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:22.295Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:22.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:22.296Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:23.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:23.108588+0000) 2023-08-27T23:40:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:24 smithi136 ceph-mon[113531]: pgmap v5005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:24.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:24.108877+0000) 2023-08-27T23:40:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:24 smithi093 ceph-mon[111335]: pgmap v5005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:24 smithi093 ceph-mon[118902]: pgmap v5005: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2174455548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:25.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:25.109139+0000) 2023-08-27T23:40:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2174455548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:40:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:40:25] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:40:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2174455548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:26 smithi136 ceph-mon[113531]: pgmap v5006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:26.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:26.109396+0000) 2023-08-27T23:40:26.598 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:26 smithi093 ceph-mon[111335]: pgmap v5006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:26.598 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:26 smithi093 ceph-mon[118902]: pgmap v5006: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:40:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/169861401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:27.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:27.109613+0000) 2023-08-27T23:40:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:40:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/169861401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:40:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/169861401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:28 smithi136 ceph-mon[113531]: pgmap v5007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:28.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:28.109821+0000) 2023-08-27T23:40:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:28 smithi093 ceph-mon[111335]: pgmap v5007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:28 smithi093 ceph-mon[118902]: pgmap v5007: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:29.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:29.110067+0000) 2023-08-27T23:40:29.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1975930987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:29.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:40:29.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:40:29.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1975930987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:40:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:40:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:29.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1975930987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:29.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:40:29.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:40:29.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:40:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:30.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:30.110283+0000) 2023-08-27T23:40:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:30 smithi136 ceph-mon[113531]: pgmap v5008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:30 smithi093 ceph-mon[111335]: pgmap v5008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:30 smithi093 ceph-mon[118902]: pgmap v5008: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:30.816Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:30.818Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:31.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:31.110509+0000) 2023-08-27T23:40:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2881775057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2881775057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2881775057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:32.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:32.110740+0000) 2023-08-27T23:40:32.685 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:32.296Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:32.685 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:32.297Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:32 smithi136 ceph-mon[113531]: pgmap v5009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:32 smithi093 ceph-mon[111335]: pgmap v5009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:32 smithi093 ceph-mon[118902]: pgmap v5009: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:33.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:33.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:33.111002+0000) 2023-08-27T23:40:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:34.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:34.111265+0000) 2023-08-27T23:40:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:34 smithi136 ceph-mon[113531]: pgmap v5010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/715213017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:34 smithi093 ceph-mon[111335]: pgmap v5010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:34.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/715213017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:34 smithi093 ceph-mon[118902]: pgmap v5010: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/715213017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:35.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:35.111498+0000) 2023-08-27T23:40:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:40:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:40:35] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:40:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:36.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:36.111706+0000) 2023-08-27T23:40:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:36 smithi093 ceph-mon[111335]: pgmap v5011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1265993803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:36 smithi093 ceph-mon[118902]: pgmap v5011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1265993803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:37.112 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:36 smithi136 ceph-mon[113531]: pgmap v5011: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:37.112 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1265993803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:37.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:37.111939+0000) 2023-08-27T23:40:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:38.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:38.112195+0000) 2023-08-27T23:40:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:38 smithi093 ceph-mon[111335]: pgmap v5012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3139411508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:38 smithi093 ceph-mon[118902]: pgmap v5012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3139411508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:39.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:38 smithi136 ceph-mon[113531]: pgmap v5012: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:39.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3139411508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:39.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:39.112444+0000) 2023-08-27T23:40:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:40.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:40.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:40.112690+0000) 2023-08-27T23:40:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:40 smithi093 ceph-mon[111335]: pgmap v5013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:40 smithi093 ceph-mon[118902]: pgmap v5013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:40.817Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:40.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:40.818Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:41.114 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:40 smithi136 ceph-mon[113531]: pgmap v5013: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:41.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:41.112940+0000) 2023-08-27T23:40:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2278860074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2278860074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:42.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2278860074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:42.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:42.113134+0000) 2023-08-27T23:40:42.708 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:42.297Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:42.708 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:42.298Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:42 smithi093 ceph-mon[111335]: pgmap v5014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:42 smithi093 ceph-mon[118902]: pgmap v5014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:43.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:42 smithi136 ceph-mon[113531]: pgmap v5014: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:43.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:43.113391+0000) 2023-08-27T23:40:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2388772392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2388772392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:44.113 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2388772392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:44.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:44.113629+0000) 2023-08-27T23:40:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:44 smithi093 ceph-mon[111335]: pgmap v5015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:44 smithi093 ceph-mon[118902]: pgmap v5015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:45.114 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:44 smithi136 ceph-mon[113531]: pgmap v5015: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:45.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:45.113873+0000) 2023-08-27T23:40:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:40:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:40:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:40:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:46.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:46.114115+0000) 2023-08-27T23:40:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:46 smithi093 ceph-mon[111335]: pgmap v5016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3308934715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:46 smithi093 ceph-mon[118902]: pgmap v5016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3308934715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:47.114 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:46 smithi136 ceph-mon[113531]: pgmap v5016: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:47.114 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3308934715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:47.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:47.114342+0000) 2023-08-27T23:40:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:48.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:48.114601+0000) 2023-08-27T23:40:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:48 smithi093 ceph-mon[111335]: pgmap v5017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1447064885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:48 smithi093 ceph-mon[118902]: pgmap v5017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1447064885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:49.115 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:48 smithi136 ceph-mon[113531]: pgmap v5017: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:49.115 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1447064885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:49.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:49.114809+0000) 2023-08-27T23:40:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:50.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:50.115088+0000) 2023-08-27T23:40:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:50 smithi093 ceph-mon[111335]: pgmap v5018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/288020822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:50 smithi093 ceph-mon[118902]: pgmap v5018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:50.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/288020822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:50.817Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:50.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:50.818Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:51.115 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:50 smithi136 ceph-mon[113531]: pgmap v5018: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:51.116 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/288020822' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:51.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:51.115362+0000) 2023-08-27T23:40:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:52.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:52.115560+0000) 2023-08-27T23:40:52.725 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:52.297Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:52.725 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:40:52.298Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:40:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:52 smithi093 ceph-mon[111335]: pgmap v5019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:52 smithi093 ceph-mon[118902]: pgmap v5019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:53.116 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:52 smithi136 ceph-mon[113531]: pgmap v5019: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:53.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:53.115819+0000) 2023-08-27T23:40:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2741687683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2741687683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:54.116 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2741687683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:54.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:54.116020+0000) 2023-08-27T23:40:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:54 smithi093 ceph-mon[111335]: pgmap v5020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:54 smithi093 ceph-mon[118902]: pgmap v5020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:55.116 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:54 smithi136 ceph-mon[113531]: pgmap v5020: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:55.116 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:40:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:55.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:55.116287+0000) 2023-08-27T23:40:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:40:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:40:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:40:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:56 smithi136 ceph-mon[113531]: pgmap v5021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3317647628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:56.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:56.116507+0000) 2023-08-27T23:40:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:56 smithi093 ceph-mon[111335]: pgmap v5021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3317647628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:56 smithi093 ceph-mon[118902]: pgmap v5021: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3317647628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:57.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:57.116747+0000) 2023-08-27T23:40:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:58 smithi136 ceph-mon[113531]: pgmap v5022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:40:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/697298448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:58.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:58.116936+0000) 2023-08-27T23:40:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:58 smithi093 ceph-mon[111335]: pgmap v5022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:40:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/697298448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:58 smithi093 ceph-mon[118902]: pgmap v5022: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:40:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:40:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/697298448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:40:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:40:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:40:59.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:40:59.117200+0000) 2023-08-27T23:41:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:00 smithi136 ceph-mon[113531]: pgmap v5023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2759404637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:00.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:00.117474+0000) 2023-08-27T23:41:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:00 smithi093 ceph-mon[111335]: pgmap v5023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2759404637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:00 smithi093 ceph-mon[118902]: pgmap v5023: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:00.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2759404637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:00.817Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:00.818Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:01.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:01.117723+0000) 2023-08-27T23:41:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:02 smithi136 ceph-mon[113531]: pgmap v5024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:02.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:02.117962+0000) 2023-08-27T23:41:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:02 smithi093 ceph-mon[111335]: pgmap v5024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:02 smithi093 ceph-mon[118902]: pgmap v5024: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:02.298Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:02.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:02.299Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1094035755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:03.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:03.118261+0000) 2023-08-27T23:41:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1094035755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1094035755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:04 smithi136 ceph-mon[113531]: pgmap v5025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:04.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:04.118542+0000) 2023-08-27T23:41:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:04 smithi093 ceph-mon[111335]: pgmap v5025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:04 smithi093 ceph-mon[118902]: pgmap v5025: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2297369293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:05.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:05.118778+0000) 2023-08-27T23:41:05.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 23:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T23:41:05.318553761Z level=info msg="Completed cleanup jobs" duration=13.204506ms 2023-08-27T23:41:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2297369293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:41:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:41:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:41:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2297369293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:06 smithi136 ceph-mon[113531]: pgmap v5026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:06.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:06.119018+0000) 2023-08-27T23:41:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:06 smithi093 ceph-mon[111335]: pgmap v5026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:06 smithi093 ceph-mon[118902]: pgmap v5026: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/239850783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:07.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:07.119291+0000) 2023-08-27T23:41:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/239850783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/239850783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:08 smithi136 ceph-mon[113531]: pgmap v5027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:08.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:08.119538+0000) 2023-08-27T23:41:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:08 smithi093 ceph-mon[111335]: pgmap v5027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:08 smithi093 ceph-mon[118902]: pgmap v5027: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:09.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:09.119800+0000) 2023-08-27T23:41:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:10 smithi136 ceph-mon[113531]: pgmap v5028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2061788810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:10.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:10.120103+0000) 2023-08-27T23:41:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:10 smithi093 ceph-mon[111335]: pgmap v5028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2061788810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:10 smithi093 ceph-mon[118902]: pgmap v5028: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2061788810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:10.819Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:10.820Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:11.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:11.120369+0000) 2023-08-27T23:41:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:12 smithi136 ceph-mon[113531]: pgmap v5029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/231266135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:12.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:12.120638+0000) 2023-08-27T23:41:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:12 smithi093 ceph-mon[111335]: pgmap v5029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/231266135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:12 smithi093 ceph-mon[118902]: pgmap v5029: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/231266135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:12.298Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:12.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:12.299Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:13.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:13.120902+0000) 2023-08-27T23:41:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:14 smithi136 ceph-mon[113531]: pgmap v5030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:14.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:14.121141+0000) 2023-08-27T23:41:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:14 smithi093 ceph-mon[111335]: pgmap v5030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:14 smithi093 ceph-mon[118902]: pgmap v5030: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/213081236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:15.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:15.121398+0000) 2023-08-27T23:41:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/213081236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:41:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:41:15] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:41:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/213081236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:16 smithi136 ceph-mon[113531]: pgmap v5031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:16.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:16.121705+0000) 2023-08-27T23:41:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:16 smithi093 ceph-mon[111335]: pgmap v5031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:16 smithi093 ceph-mon[118902]: pgmap v5031: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2489596317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:17.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:17.121955+0000) 2023-08-27T23:41:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2489596317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2489596317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:18 smithi136 ceph-mon[113531]: pgmap v5032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:18.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:18.122231+0000) 2023-08-27T23:41:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:18 smithi093 ceph-mon[111335]: pgmap v5032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:18 smithi093 ceph-mon[118902]: pgmap v5032: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/841078737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:19.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:19.122495+0000) 2023-08-27T23:41:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/841078737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/841078737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:20 smithi136 ceph-mon[113531]: pgmap v5033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:20.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:20.122706+0000) 2023-08-27T23:41:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:20 smithi093 ceph-mon[111335]: pgmap v5033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:20 smithi093 ceph-mon[118902]: pgmap v5033: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:20.819Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:20.820Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:21.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:21.122962+0000) 2023-08-27T23:41:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:22 smithi136 ceph-mon[113531]: pgmap v5034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3129231458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:22.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:22.123206+0000) 2023-08-27T23:41:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:22 smithi093 ceph-mon[111335]: pgmap v5034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3129231458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:22 smithi093 ceph-mon[118902]: pgmap v5034: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3129231458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:22.298Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:22.300Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:23.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:23.123453+0000) 2023-08-27T23:41:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:24.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:24.123676+0000) 2023-08-27T23:41:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:24 smithi136 ceph-mon[113531]: pgmap v5035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/659629788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:24 smithi093 ceph-mon[111335]: pgmap v5035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/659629788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:24 smithi093 ceph-mon[118902]: pgmap v5035: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/659629788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:25.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:25.123864+0000) 2023-08-27T23:41:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:41:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:41:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:41:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:26 smithi136 ceph-mon[113531]: pgmap v5036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1215207909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:26.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:26.124108+0000) 2023-08-27T23:41:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:26 smithi093 ceph-mon[111335]: pgmap v5036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1215207909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:26 smithi093 ceph-mon[118902]: pgmap v5036: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1215207909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:27.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:27.124352+0000) 2023-08-27T23:41:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:28 smithi136 ceph-mon[113531]: pgmap v5037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:28.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:28.124672+0000) 2023-08-27T23:41:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:28 smithi093 ceph-mon[111335]: pgmap v5037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:28.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:28 smithi093 ceph-mon[118902]: pgmap v5037: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2588261341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:41:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:29.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:29.124983+0000) 2023-08-27T23:41:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2588261341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:41:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2588261341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:29.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:41:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:30 smithi136 ceph-mon[113531]: pgmap v5038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:30.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:30.125136+0000) 2023-08-27T23:41:30.689 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:30 smithi093 ceph-mon[111335]: pgmap v5038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:30.689 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:30 smithi093 ceph-mon[118902]: pgmap v5038: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:30.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:30.819Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:30.820Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:31.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:31.125274+0000) 2023-08-27T23:41:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:32.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:32.125472+0000) 2023-08-27T23:41:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:31 smithi136 ceph-mon[113531]: pgmap v5039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1520486269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:32.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[118902]: pgmap v5039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1520486269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[111335]: pgmap v5039: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1520486269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:32.299Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:32.300Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:41:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:41:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:33.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:33.125744+0000) 2023-08-27T23:41:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:41:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:41:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:41:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:41:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:41:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:34.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:34.126054+0000) 2023-08-27T23:41:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:34 smithi136 ceph-mon[113531]: pgmap v5040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2625045855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:34 smithi093 ceph-mon[111335]: pgmap v5040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2625045855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:34 smithi093 ceph-mon[118902]: pgmap v5040: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2625045855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:35.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:35.126355+0000) 2023-08-27T23:41:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:41:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:41:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:41:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:36 smithi136 ceph-mon[113531]: pgmap v5041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4033500500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:36.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:36.126625+0000) 2023-08-27T23:41:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:36 smithi093 ceph-mon[111335]: pgmap v5041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4033500500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:36 smithi093 ceph-mon[118902]: pgmap v5041: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4033500500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:37.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:37.126893+0000) 2023-08-27T23:41:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:38 smithi136 ceph-mon[113531]: pgmap v5042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4174504493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:38.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:38.127176+0000) 2023-08-27T23:41:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:38 smithi093 ceph-mon[111335]: pgmap v5042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:38.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4174504493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:38.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:38 smithi093 ceph-mon[118902]: pgmap v5042: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:38.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4174504493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:39.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:39.127452+0000) 2023-08-27T23:41:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:40 smithi136 ceph-mon[113531]: pgmap v5043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:40.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:40.127645+0000) 2023-08-27T23:41:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:40 smithi093 ceph-mon[111335]: pgmap v5043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:40 smithi093 ceph-mon[118902]: pgmap v5043: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:41.219 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:40.820Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:41.219 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:40.821Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/20078721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:41.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:41.127918+0000) 2023-08-27T23:41:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/20078721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/20078721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:42 smithi136 ceph-mon[113531]: pgmap v5044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:42.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:42.128194+0000) 2023-08-27T23:41:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:42 smithi093 ceph-mon[111335]: pgmap v5044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:42 smithi093 ceph-mon[118902]: pgmap v5044: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:42.300Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:42.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:42.301Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2826720219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:43.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:43.128439+0000) 2023-08-27T23:41:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2826720219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2826720219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:44 smithi136 ceph-mon[113531]: pgmap v5045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:44.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:44.128699+0000) 2023-08-27T23:41:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:44 smithi093 ceph-mon[111335]: pgmap v5045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:44 smithi093 ceph-mon[118902]: pgmap v5045: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:45.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:45.128945+0000) 2023-08-27T23:41:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:41:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:41:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:41:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:46 smithi136 ceph-mon[113531]: pgmap v5046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2443969799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:46.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:46.129196+0000) 2023-08-27T23:41:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:46 smithi093 ceph-mon[111335]: pgmap v5046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2443969799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:46 smithi093 ceph-mon[118902]: pgmap v5046: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2443969799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:47.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:47.129399+0000) 2023-08-27T23:41:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:48 smithi136 ceph-mon[113531]: pgmap v5047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/671408499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:48.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:48.129644+0000) 2023-08-27T23:41:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:48 smithi093 ceph-mon[111335]: pgmap v5047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/671408499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:48 smithi093 ceph-mon[118902]: pgmap v5047: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/671408499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:49.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:49.129895+0000) 2023-08-27T23:41:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:50 smithi136 ceph-mon[113531]: pgmap v5048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/284050111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:50.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:50.130208+0000) 2023-08-27T23:41:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:50 smithi093 ceph-mon[118902]: pgmap v5048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/284050111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:50 smithi093 ceph-mon[111335]: pgmap v5048: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/284050111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:50.821Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:50.822Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:51.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:51.130490+0000) 2023-08-27T23:41:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:52 smithi136 ceph-mon[113531]: pgmap v5049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:52.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:52.130759+0000) 2023-08-27T23:41:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:52 smithi093 ceph-mon[111335]: pgmap v5049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:52 smithi093 ceph-mon[118902]: pgmap v5049: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:52.300Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:41:52.301Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:41:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/967197355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:53.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:53.131082+0000) 2023-08-27T23:41:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/967197355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/967197355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:54 smithi136 ceph-mon[113531]: pgmap v5050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:54.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:54.131356+0000) 2023-08-27T23:41:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:54 smithi093 ceph-mon[111335]: pgmap v5050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:54 smithi093 ceph-mon[118902]: pgmap v5050: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:41:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/780497346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:55.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:55.131612+0000) 2023-08-27T23:41:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/780497346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:41:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:41:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:41:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/780497346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:56 smithi136 ceph-mon[113531]: pgmap v5051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:56.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:56.131929+0000) 2023-08-27T23:41:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:56 smithi093 ceph-mon[111335]: pgmap v5051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:56 smithi093 ceph-mon[118902]: pgmap v5051: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/767423562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:57.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:57.132197+0000) 2023-08-27T23:41:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/767423562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/767423562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:41:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:41:58 smithi136 ceph-mon[113531]: pgmap v5052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:58.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:58.132451+0000) 2023-08-27T23:41:58.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:41:58 smithi093 ceph-mon[111335]: pgmap v5052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:41:58 smithi093 ceph-mon[118902]: pgmap v5052: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:41:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:41:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:41:59.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:41:59.132736+0000) 2023-08-27T23:42:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:00 smithi136 ceph-mon[113531]: pgmap v5053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3894545218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:00.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:00.133025+0000) 2023-08-27T23:42:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:00 smithi093 ceph-mon[118902]: pgmap v5053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3894545218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:00 smithi093 ceph-mon[111335]: pgmap v5053: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3894545218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:00.822Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:00.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:01.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:01.133233+0000) 2023-08-27T23:42:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:02 smithi136 ceph-mon[113531]: pgmap v5054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1934196308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:02.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:02.133480+0000) 2023-08-27T23:42:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:02 smithi093 ceph-mon[111335]: pgmap v5054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1934196308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:02 smithi093 ceph-mon[118902]: pgmap v5054: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1934196308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:02.300Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:02.301Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:03.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:03.133763+0000) 2023-08-27T23:42:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:04 smithi136 ceph-mon[113531]: pgmap v5055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:04.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:04.133945+0000) 2023-08-27T23:42:04.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:04 smithi093 ceph-mon[111335]: pgmap v5055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:04 smithi093 ceph-mon[118902]: pgmap v5055: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3016831093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:05.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:05.134201+0000) 2023-08-27T23:42:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3016831093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3016831093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:42:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:42:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:42:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:06 smithi136 ceph-mon[113531]: pgmap v5056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:06.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:06.134466+0000) 2023-08-27T23:42:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:06 smithi093 ceph-mon[111335]: pgmap v5056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:06 smithi093 ceph-mon[118902]: pgmap v5056: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2219613615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:07.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:07.134712+0000) 2023-08-27T23:42:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2219613615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2219613615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:08 smithi136 ceph-mon[113531]: pgmap v5057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:08.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:08.135012+0000) 2023-08-27T23:42:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:08 smithi093 ceph-mon[111335]: pgmap v5057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:08.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:08 smithi093 ceph-mon[118902]: pgmap v5057: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:09.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/879294927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:09.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:09.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:09.135328+0000) 2023-08-27T23:42:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/879294927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/879294927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:10 smithi136 ceph-mon[113531]: pgmap v5058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:10.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:10.135547+0000) 2023-08-27T23:42:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:10 smithi093 ceph-mon[118902]: pgmap v5058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:10 smithi093 ceph-mon[111335]: pgmap v5058: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:10.822Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:10.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:11.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:11.135808+0000) 2023-08-27T23:42:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:12 smithi136 ceph-mon[113531]: pgmap v5059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3108541499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:12.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:12.136076+0000) 2023-08-27T23:42:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:12 smithi093 ceph-mon[118902]: pgmap v5059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3108541499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:12 smithi093 ceph-mon[111335]: pgmap v5059: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3108541499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:12.300Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:12.301Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:13.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:13.136317+0000) 2023-08-27T23:42:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:14 smithi136 ceph-mon[113531]: pgmap v5060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/32592699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:14.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:14.136507+0000) 2023-08-27T23:42:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:14 smithi093 ceph-mon[111335]: pgmap v5060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/32592699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:14 smithi093 ceph-mon[118902]: pgmap v5060: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/32592699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:15.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:15.136760+0000) 2023-08-27T23:42:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:42:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:42:15] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:42:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:16 smithi136 ceph-mon[113531]: pgmap v5061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/209741517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:16.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:16.137081+0000) 2023-08-27T23:42:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:16 smithi093 ceph-mon[118902]: pgmap v5061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/209741517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:16 smithi093 ceph-mon[111335]: pgmap v5061: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/209741517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:17.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:17.137353+0000) 2023-08-27T23:42:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:18 smithi136 ceph-mon[113531]: pgmap v5062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:18.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:18.137642+0000) 2023-08-27T23:42:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:18 smithi093 ceph-mon[111335]: pgmap v5062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:18.772 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:18 smithi093 ceph-mon[118902]: pgmap v5062: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1775330976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:19.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:19.137898+0000) 2023-08-27T23:42:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1775330976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1775330976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:20 smithi136 ceph-mon[113531]: pgmap v5063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:20.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:20.138189+0000) 2023-08-27T23:42:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:20 smithi093 ceph-mon[111335]: pgmap v5063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:20 smithi093 ceph-mon[118902]: pgmap v5063: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:20.823Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:20.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1089084303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:21.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:21.138467+0000) 2023-08-27T23:42:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1089084303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1089084303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:22.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:22 smithi136 ceph-mon[113531]: pgmap v5064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:22.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:22.138746+0000) 2023-08-27T23:42:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:22 smithi093 ceph-mon[111335]: pgmap v5064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:22 smithi093 ceph-mon[118902]: pgmap v5064: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:22.301Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:22.302Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:23.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:23.139051+0000) 2023-08-27T23:42:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:24.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:24.139338+0000) 2023-08-27T23:42:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:24 smithi136 ceph-mon[113531]: pgmap v5065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3201532554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:24.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:24 smithi093 ceph-mon[111335]: pgmap v5065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3201532554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:24 smithi093 ceph-mon[118902]: pgmap v5065: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3201532554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:25.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:25.139586+0000) 2023-08-27T23:42:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:42:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:42:25] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:42:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:26 smithi136 ceph-mon[113531]: pgmap v5066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1960872344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:26.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:26.139849+0000) 2023-08-27T23:42:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:26 smithi093 ceph-mon[111335]: pgmap v5066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1960872344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:26 smithi093 ceph-mon[118902]: pgmap v5066: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1960872344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:27.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:27.140122+0000) 2023-08-27T23:42:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:28.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:28.140354+0000) 2023-08-27T23:42:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:28 smithi136 ceph-mon[113531]: pgmap v5067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3645186995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:28 smithi093 ceph-mon[111335]: pgmap v5067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3645186995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:28 smithi093 ceph-mon[118902]: pgmap v5067: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3645186995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:29.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:29.140598+0000) 2023-08-27T23:42:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:30 smithi136 ceph-mon[113531]: pgmap v5068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:30.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:30.140820+0000) 2023-08-27T23:42:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:30 smithi093 ceph-mon[111335]: pgmap v5068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:30 smithi093 ceph-mon[118902]: pgmap v5068: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:30.823Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:30.824Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1868009263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:31.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:31.141062+0000) 2023-08-27T23:42:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1868009263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1868009263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:32 smithi136 ceph-mon[113531]: pgmap v5069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:42:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:32.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:32.141222+0000) 2023-08-27T23:42:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:32 smithi093 ceph-mon[111335]: pgmap v5069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:42:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:32 smithi093 ceph-mon[118902]: pgmap v5069: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:42:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:32.302Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:32.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:33.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:33.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:33.141406+0000) 2023-08-27T23:42:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/626268793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/626268793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/626268793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:34.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:34.141611+0000) 2023-08-27T23:42:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:34 smithi136 ceph-mon[113531]: pgmap v5070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:34 smithi093 ceph-mon[111335]: pgmap v5070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:34 smithi093 ceph-mon[118902]: pgmap v5070: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:35.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:35.141783+0000) 2023-08-27T23:42:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:42:35] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:42:36.142 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.142 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-mon[113531]: pgmap v5071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:36.142 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.142 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:42:36.142 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:42:36.143 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.143 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1505739993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[111335]: pgmap v5071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:42:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:42:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1505739993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[118902]: pgmap v5071: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:42:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:42:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:42:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1505739993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:36.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:36.141931+0000) 2023-08-27T23:42:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:37.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:37.142098+0000) 2023-08-27T23:42:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:37 smithi136 ceph-mon[113531]: pgmap v5072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1445213119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:38.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:38.142404+0000) 2023-08-27T23:42:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:37 smithi093 ceph-mon[111335]: pgmap v5072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1445213119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:37 smithi093 ceph-mon[118902]: pgmap v5072: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1445213119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:39.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:39.142669+0000) 2023-08-27T23:42:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:39 smithi136 ceph-mon[113531]: pgmap v5073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:40.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:40.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:40.142945+0000) 2023-08-27T23:42:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:39 smithi093 ceph-mon[111335]: pgmap v5073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:39 smithi093 ceph-mon[118902]: pgmap v5073: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2413965172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:41.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:41.143197+0000) 2023-08-27T23:42:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2413965172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2413965172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:40.825Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:40.825Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:42 smithi136 ceph-mon[113531]: pgmap v5074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:42.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:42.143436+0000) 2023-08-27T23:42:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:42 smithi093 ceph-mon[111335]: pgmap v5074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:42 smithi093 ceph-mon[118902]: pgmap v5074: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:42.303Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:42.304Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1143046252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:43.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:43.143626+0000) 2023-08-27T23:42:43.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1143046252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:43.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1143046252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:44 smithi136 ceph-mon[113531]: pgmap v5075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:44.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:44.143803+0000) 2023-08-27T23:42:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:44 smithi093 ceph-mon[111335]: pgmap v5075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:44 smithi093 ceph-mon[118902]: pgmap v5075: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1682082558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:45.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:45.144076+0000) 2023-08-27T23:42:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1682082558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:45.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1682082558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:45.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:42:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:42:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:42:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:46 smithi136 ceph-mon[113531]: pgmap v5076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:46.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:46.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:46.144274+0000) 2023-08-27T23:42:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:46 smithi093 ceph-mon[111335]: pgmap v5076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:46 smithi093 ceph-mon[118902]: pgmap v5076: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:47.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:47.144510+0000) 2023-08-27T23:42:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:48 smithi136 ceph-mon[113531]: pgmap v5077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1565414329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:48.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:48.144767+0000) 2023-08-27T23:42:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:48 smithi093 ceph-mon[111335]: pgmap v5077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1565414329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:48 smithi093 ceph-mon[118902]: pgmap v5077: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1565414329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:49.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:49.144988+0000) 2023-08-27T23:42:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:50 smithi136 ceph-mon[113531]: pgmap v5078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1999334620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:50.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:50.145239+0000) 2023-08-27T23:42:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:50 smithi093 ceph-mon[111335]: pgmap v5078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1999334620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:50 smithi093 ceph-mon[118902]: pgmap v5078: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1999334620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:50.825Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:50.826Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:51.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:51.145499+0000) 2023-08-27T23:42:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:52 smithi136 ceph-mon[113531]: pgmap v5079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1183471744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:52.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:52.145743+0000) 2023-08-27T23:42:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:52 smithi093 ceph-mon[111335]: pgmap v5079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1183471744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:52 smithi093 ceph-mon[118902]: pgmap v5079: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1183471744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:52.304Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:42:52.305Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:42:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:53.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:53.145987+0000) 2023-08-27T23:42:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:54 smithi136 ceph-mon[113531]: pgmap v5080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:54.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:54.146246+0000) 2023-08-27T23:42:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:54 smithi093 ceph-mon[111335]: pgmap v5080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:54 smithi093 ceph-mon[118902]: pgmap v5080: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:54.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:42:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340486444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:55.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:55.146513+0000) 2023-08-27T23:42:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340486444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340486444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:42:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:42:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:42:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:56 smithi136 ceph-mon[113531]: pgmap v5081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:56.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:56.146807+0000) 2023-08-27T23:42:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:56 smithi093 ceph-mon[111335]: pgmap v5081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:56 smithi093 ceph-mon[118902]: pgmap v5081: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/282602207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:57.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:57.147085+0000) 2023-08-27T23:42:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/282602207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/282602207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:58 smithi136 ceph-mon[113531]: pgmap v5082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:58.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:58.147363+0000) 2023-08-27T23:42:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:58 smithi093 ceph-mon[111335]: pgmap v5082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:58 smithi093 ceph-mon[118902]: pgmap v5082: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:42:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:42:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2297795709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:42:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:42:59.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:42:59.147619+0000) 2023-08-27T23:42:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:42:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2297795709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:42:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:42:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2297795709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:00 smithi136 ceph-mon[113531]: pgmap v5083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:00.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:00.147851+0000) 2023-08-27T23:43:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:00 smithi093 ceph-mon[111335]: pgmap v5083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:00 smithi093 ceph-mon[118902]: pgmap v5083: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:00.826Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:00.827Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:01.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:01.148065+0000) 2023-08-27T23:43:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:02 smithi136 ceph-mon[113531]: pgmap v5084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2744031030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:02.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:02.148262+0000) 2023-08-27T23:43:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:02 smithi093 ceph-mon[111335]: pgmap v5084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2744031030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:02 smithi093 ceph-mon[118902]: pgmap v5084: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2744031030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:02.305Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:02.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:03.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:03.148450+0000) 2023-08-27T23:43:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:04 smithi136 ceph-mon[113531]: pgmap v5085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4046234332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:04.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:04.148704+0000) 2023-08-27T23:43:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:04 smithi093 ceph-mon[111335]: pgmap v5085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4046234332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:04 smithi093 ceph-mon[118902]: pgmap v5085: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4046234332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:05.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:05.148922+0000) 2023-08-27T23:43:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:43:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:43:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:43:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:06 smithi136 ceph-mon[113531]: pgmap v5086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:06.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:06.149198+0000) 2023-08-27T23:43:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:06 smithi093 ceph-mon[111335]: pgmap v5086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:06 smithi093 ceph-mon[118902]: pgmap v5086: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3002975816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:07.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:07.149419+0000) 2023-08-27T23:43:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3002975816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3002975816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:08 smithi136 ceph-mon[113531]: pgmap v5087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:08.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:08.149708+0000) 2023-08-27T23:43:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:08 smithi093 ceph-mon[111335]: pgmap v5087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:08 smithi093 ceph-mon[118902]: pgmap v5087: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3238081943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:09.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:09.149962+0000) 2023-08-27T23:43:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3238081943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3238081943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:10 smithi136 ceph-mon[113531]: pgmap v5088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:10.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:10.150181+0000) 2023-08-27T23:43:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:10 smithi093 ceph-mon[111335]: pgmap v5088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:10 smithi093 ceph-mon[118902]: pgmap v5088: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:10.827Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:10.828Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1106000017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:11.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:11.150354+0000) 2023-08-27T23:43:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1106000017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1106000017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:12 smithi136 ceph-mon[113531]: pgmap v5089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:12.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:12.150591+0000) 2023-08-27T23:43:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:12 smithi093 ceph-mon[111335]: pgmap v5089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:12 smithi093 ceph-mon[118902]: pgmap v5089: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:12.305Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:12.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:13.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:13.150845+0000) 2023-08-27T23:43:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:14.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:14.151125+0000) 2023-08-27T23:43:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:14 smithi136 ceph-mon[113531]: pgmap v5090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2314884269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:14 smithi093 ceph-mon[111335]: pgmap v5090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2314884269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:14 smithi093 ceph-mon[118902]: pgmap v5090: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2314884269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:15.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:15.151408+0000) 2023-08-27T23:43:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:43:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:43:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:43:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:16.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:16.151613+0000) 2023-08-27T23:43:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:16 smithi136 ceph-mon[113531]: pgmap v5091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2307306091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:16 smithi093 ceph-mon[111335]: pgmap v5091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2307306091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:16 smithi093 ceph-mon[118902]: pgmap v5091: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2307306091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:17.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:17.151851+0000) 2023-08-27T23:43:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:18.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:18.152151+0000) 2023-08-27T23:43:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:18 smithi136 ceph-mon[113531]: pgmap v5092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1552067593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:18 smithi093 ceph-mon[111335]: pgmap v5092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1552067593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:18 smithi093 ceph-mon[118902]: pgmap v5092: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1552067593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:19.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:19.152418+0000) 2023-08-27T23:43:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:20 smithi136 ceph-mon[113531]: pgmap v5093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:20.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:20.152678+0000) 2023-08-27T23:43:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:20 smithi093 ceph-mon[111335]: pgmap v5093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:20 smithi093 ceph-mon[118902]: pgmap v5093: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:20.828Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:20.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4039840493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:21.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:21.152931+0000) 2023-08-27T23:43:21.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4039840493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4039840493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:22 smithi136 ceph-mon[113531]: pgmap v5094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:22.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:22.153172+0000) 2023-08-27T23:43:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:22 smithi093 ceph-mon[111335]: pgmap v5094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:22 smithi093 ceph-mon[118902]: pgmap v5094: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:22.307Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:22.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3872679905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:23.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:23.153423+0000) 2023-08-27T23:43:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3872679905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3872679905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:24 smithi136 ceph-mon[113531]: pgmap v5095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:24.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:24.153677+0000) 2023-08-27T23:43:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:24 smithi093 ceph-mon[111335]: pgmap v5095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:24 smithi093 ceph-mon[118902]: pgmap v5095: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:25.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:25.153953+0000) 2023-08-27T23:43:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:43:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:43:25] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:43:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:26 smithi136 ceph-mon[113531]: pgmap v5096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2123822802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:26.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:26.154191+0000) 2023-08-27T23:43:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:26 smithi093 ceph-mon[111335]: pgmap v5096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2123822802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:26 smithi093 ceph-mon[118902]: pgmap v5096: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2123822802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:27.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:27.154450+0000) 2023-08-27T23:43:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:28 smithi136 ceph-mon[113531]: pgmap v5097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/390366742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:28.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:28.154688+0000) 2023-08-27T23:43:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:28 smithi093 ceph-mon[118902]: pgmap v5097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/390366742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:28 smithi093 ceph-mon[111335]: pgmap v5097: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/390366742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:29.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:29.154906+0000) 2023-08-27T23:43:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:30 smithi136 ceph-mon[113531]: pgmap v5098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2790688217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:30.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:30.155161+0000) 2023-08-27T23:43:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:30 smithi093 ceph-mon[111335]: pgmap v5098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2790688217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:30 smithi093 ceph-mon[118902]: pgmap v5098: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2790688217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:30.829Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:30.830Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:31.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:31.155415+0000) 2023-08-27T23:43:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:32 smithi136 ceph-mon[113531]: pgmap v5099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:32.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:32.155611+0000) 2023-08-27T23:43:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:32 smithi093 ceph-mon[111335]: pgmap v5099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:32 smithi093 ceph-mon[118902]: pgmap v5099: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:32.307Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:32.308Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3282202807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:33.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:33.155845+0000) 2023-08-27T23:43:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3282202807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3282202807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:34 smithi136 ceph-mon[113531]: pgmap v5100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:34.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:34.156078+0000) 2023-08-27T23:43:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:34 smithi093 ceph-mon[111335]: pgmap v5100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:34 smithi093 ceph-mon[118902]: pgmap v5100: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3135412739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:35.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:35.156286+0000) 2023-08-27T23:43:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3135412739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3135412739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:43:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:43:35] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:43:36.352 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:36.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:36.156508+0000) 2023-08-27T23:43:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:36 smithi136 ceph-mon[113531]: pgmap v5101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:43:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:36 smithi093 ceph-mon[111335]: pgmap v5101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:43:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:36 smithi093 ceph-mon[118902]: pgmap v5101: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:43:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:37.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:37.156666+0000) 2023-08-27T23:43:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:38 smithi136 ceph-mon[113531]: pgmap v5102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/408095775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:38.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:38.156825+0000) 2023-08-27T23:43:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[111335]: pgmap v5102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/408095775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:38.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[118902]: pgmap v5102: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:38.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/408095775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:38.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:38.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:39.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:39.157014+0000) 2023-08-27T23:43:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:43:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:43:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:43:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:43:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:39.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:39.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:43:39.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:43:39.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:43:39.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:40.421 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:40.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:40.157275+0000) 2023-08-27T23:43:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:40 smithi136 ceph-mon[113531]: pgmap v5103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3652820431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:40 smithi093 ceph-mon[111335]: pgmap v5103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3652820431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:40 smithi093 ceph-mon[118902]: pgmap v5103: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3652820431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:40.829Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:40.831Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:41.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:41.157493+0000) 2023-08-27T23:43:42.424 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:42.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:42.157678+0000) 2023-08-27T23:43:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:42 smithi136 ceph-mon[113531]: pgmap v5104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/751249187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:42 smithi093 ceph-mon[111335]: pgmap v5104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/751249187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:42 smithi093 ceph-mon[118902]: pgmap v5104: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/751249187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:42.308Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:42.309Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:43.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:43.157890+0000) 2023-08-27T23:43:44.426 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:44.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:44.158159+0000) 2023-08-27T23:43:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:44 smithi136 ceph-mon[113531]: pgmap v5105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:44 smithi093 ceph-mon[111335]: pgmap v5105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:44 smithi093 ceph-mon[118902]: pgmap v5105: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:45.435 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:45.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:45.158424+0000) 2023-08-27T23:43:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1154940915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1154940915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1154940915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:43:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:43:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:43:46.434 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:46.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:46.158679+0000) 2023-08-27T23:43:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:46 smithi136 ceph-mon[113531]: pgmap v5106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:46 smithi093 ceph-mon[111335]: pgmap v5106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:46 smithi093 ceph-mon[118902]: pgmap v5106: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:47.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:47.158922+0000) 2023-08-27T23:43:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/440413982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/440413982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/440413982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:48.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:48.159179+0000) 2023-08-27T23:43:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:48 smithi093 ceph-mon[111335]: pgmap v5107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:48 smithi093 ceph-mon[118902]: pgmap v5107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:48 smithi136 ceph-mon[113531]: pgmap v5107: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:49.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:49.159424+0000) 2023-08-27T23:43:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/234100296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/234100296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:49.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/234100296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:50.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:50.159655+0000) 2023-08-27T23:43:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:50 smithi093 ceph-mon[111335]: pgmap v5108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:50 smithi093 ceph-mon[118902]: pgmap v5108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:50 smithi136 ceph-mon[113531]: pgmap v5108: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:50.831Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:50.832Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:51.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:51.159858+0000) 2023-08-27T23:43:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:52.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:52.160136+0000) 2023-08-27T23:43:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:52 smithi093 ceph-mon[111335]: pgmap v5109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3503592372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:52 smithi093 ceph-mon[118902]: pgmap v5109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3503592372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:52.309Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:43:52.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:43:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:52 smithi136 ceph-mon[113531]: pgmap v5109: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3503592372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:53.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:53.160394+0000) 2023-08-27T23:43:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:54.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:54.160648+0000) 2023-08-27T23:43:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:54 smithi093 ceph-mon[111335]: pgmap v5110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1521836263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:54 smithi093 ceph-mon[118902]: pgmap v5110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1521836263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:54 smithi136 ceph-mon[113531]: pgmap v5110: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:43:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1521836263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:55.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:55.160892+0000) 2023-08-27T23:43:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:43:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:43:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:43:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:56.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:56.161175+0000) 2023-08-27T23:43:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:56 smithi093 ceph-mon[111335]: pgmap v5111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2276772716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:56 smithi093 ceph-mon[118902]: pgmap v5111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2276772716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:56 smithi136 ceph-mon[113531]: pgmap v5111: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2276772716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:57.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:57.161365+0000) 2023-08-27T23:43:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:58.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:58.161607+0000) 2023-08-27T23:43:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:58 smithi093 ceph-mon[111335]: pgmap v5112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:58 smithi093 ceph-mon[118902]: pgmap v5112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:58 smithi136 ceph-mon[113531]: pgmap v5112: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:43:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:43:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:43:59.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:43:59.161848+0000) 2023-08-27T23:43:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:43:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1888669665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:43:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1888669665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:43:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:43:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1888669665' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:00.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:00.162121+0000) 2023-08-27T23:44:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:00 smithi093 ceph-mon[111335]: pgmap v5113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:00 smithi093 ceph-mon[118902]: pgmap v5113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:00 smithi136 ceph-mon[113531]: pgmap v5113: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:00.832Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:00.833Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:01.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:01.162375+0000) 2023-08-27T23:44:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3596891415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3596891415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3596891415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:02.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:02.162580+0000) 2023-08-27T23:44:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:02 smithi093 ceph-mon[111335]: pgmap v5114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:02 smithi093 ceph-mon[118902]: pgmap v5114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:02.309Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:02.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:02 smithi136 ceph-mon[113531]: pgmap v5114: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:03.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:03.162842+0000) 2023-08-27T23:44:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:04.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:04.163096+0000) 2023-08-27T23:44:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:04 smithi093 ceph-mon[111335]: pgmap v5115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2139713477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:04 smithi093 ceph-mon[118902]: pgmap v5115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2139713477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:04 smithi136 ceph-mon[113531]: pgmap v5115: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2139713477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:05.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:05.163345+0000) 2023-08-27T23:44:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:44:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:44:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:44:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:06.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:06.163611+0000) 2023-08-27T23:44:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:06 smithi093 ceph-mon[111335]: pgmap v5116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/239744687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:06 smithi093 ceph-mon[118902]: pgmap v5116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/239744687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:06 smithi136 ceph-mon[113531]: pgmap v5116: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/239744687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:07.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:07.163825+0000) 2023-08-27T23:44:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:08.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:08.164051+0000) 2023-08-27T23:44:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:08 smithi093 ceph-mon[111335]: pgmap v5117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1335160775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:08 smithi093 ceph-mon[118902]: pgmap v5117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1335160775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:08 smithi136 ceph-mon[113531]: pgmap v5117: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1335160775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:09.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:09.164289+0000) 2023-08-27T23:44:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:10.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:10.164526+0000) 2023-08-27T23:44:10.833 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:10 smithi093 ceph-mon[111335]: pgmap v5118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:10.834 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:10 smithi093 ceph-mon[118902]: pgmap v5118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:10 smithi136 ceph-mon[113531]: pgmap v5118: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:10.833Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:10.834Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:11.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:11.164786+0000) 2023-08-27T23:44:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/673023553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/673023553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/673023553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:12.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:12.165049+0000) 2023-08-27T23:44:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:12 smithi093 ceph-mon[111335]: pgmap v5119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:12 smithi093 ceph-mon[118902]: pgmap v5119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:12.310Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:12.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:12 smithi136 ceph-mon[113531]: pgmap v5119: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:13.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:13.165224+0000) 2023-08-27T23:44:13.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1779457120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1779457120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1779457120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:14.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:14.165430+0000) 2023-08-27T23:44:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:14 smithi136 ceph-mon[113531]: pgmap v5120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:14 smithi093 ceph-mon[111335]: pgmap v5120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:14 smithi093 ceph-mon[118902]: pgmap v5120: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:15.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:15.165685+0000) 2023-08-27T23:44:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:44:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:44:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:44:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:16.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:16.165990+0000) 2023-08-27T23:44:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:16 smithi136 ceph-mon[113531]: pgmap v5121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1979390803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:16 smithi093 ceph-mon[111335]: pgmap v5121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1979390803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:16 smithi093 ceph-mon[118902]: pgmap v5121: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1979390803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:17.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:17.166193+0000) 2023-08-27T23:44:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:18.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:18.166450+0000) 2023-08-27T23:44:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:18 smithi136 ceph-mon[113531]: pgmap v5122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2618405212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:18 smithi093 ceph-mon[111335]: pgmap v5122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2618405212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:18 smithi093 ceph-mon[118902]: pgmap v5122: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2618405212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:19.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:19.166707+0000) 2023-08-27T23:44:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:20.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:20.166948+0000) 2023-08-27T23:44:20.834 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:20 smithi093 ceph-mon[111335]: pgmap v5123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:20.834 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/249440582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:20.835 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:20 smithi093 ceph-mon[118902]: pgmap v5123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:20.835 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/249440582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:20 smithi136 ceph-mon[113531]: pgmap v5123: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/249440582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:20.834Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:20.835Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:21.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:21.167161+0000) 2023-08-27T23:44:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:22.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:22.167400+0000) 2023-08-27T23:44:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:22 smithi093 ceph-mon[111335]: pgmap v5124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:22 smithi093 ceph-mon[118902]: pgmap v5124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:22.311Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:22.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:22 smithi136 ceph-mon[113531]: pgmap v5124: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:23.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:23.167690+0000) 2023-08-27T23:44:23.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2963729086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2963729086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2963729086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:24.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:24.167968+0000) 2023-08-27T23:44:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:24 smithi136 ceph-mon[113531]: pgmap v5125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:24 smithi093 ceph-mon[111335]: pgmap v5125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:24 smithi093 ceph-mon[118902]: pgmap v5125: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:25.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:25.168186+0000) 2023-08-27T23:44:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1503543175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1503543175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:44:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:44:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:44:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1503543175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:26.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:26.168457+0000) 2023-08-27T23:44:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:26 smithi136 ceph-mon[113531]: pgmap v5126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:26 smithi093 ceph-mon[111335]: pgmap v5126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:26 smithi093 ceph-mon[118902]: pgmap v5126: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:27.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:27.168697+0000) 2023-08-27T23:44:27.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3877214567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3877214567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3877214567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:28.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:28.168971+0000) 2023-08-27T23:44:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:28 smithi136 ceph-mon[113531]: pgmap v5127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:28 smithi093 ceph-mon[111335]: pgmap v5127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:28 smithi093 ceph-mon[118902]: pgmap v5127: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:29.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:29.169266+0000) 2023-08-27T23:44:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:30.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:30.169526+0000) 2023-08-27T23:44:30.835 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:30 smithi093 ceph-mon[111335]: pgmap v5128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:30.835 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2392109316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:30.835 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:30 smithi093 ceph-mon[118902]: pgmap v5128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:30.836 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2392109316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:30 smithi136 ceph-mon[113531]: pgmap v5128: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2392109316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:31.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:30.834Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:30.835Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:31.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:31.169777+0000) 2023-08-27T23:44:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:32.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:32.169967+0000) 2023-08-27T23:44:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:32 smithi093 ceph-mon[111335]: pgmap v5129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2304078538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:32 smithi093 ceph-mon[118902]: pgmap v5129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2304078538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:32.312Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:32.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:32 smithi136 ceph-mon[113531]: pgmap v5129: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2304078538' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:33.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:33.170230+0000) 2023-08-27T23:44:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:34.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:34.170477+0000) 2023-08-27T23:44:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:34 smithi136 ceph-mon[113531]: pgmap v5130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:34 smithi093 ceph-mon[111335]: pgmap v5130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:34 smithi093 ceph-mon[118902]: pgmap v5130: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:35.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:35.170759+0000) 2023-08-27T23:44:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2083225968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:44:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:44:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:44:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2083225968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:35.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2083225968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:36.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:36.171054+0000) 2023-08-27T23:44:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:36 smithi136 ceph-mon[113531]: pgmap v5131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:36.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:36 smithi093 ceph-mon[111335]: pgmap v5131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:36 smithi093 ceph-mon[118902]: pgmap v5131: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:37.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:37.171297+0000) 2023-08-27T23:44:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1285745923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1285745923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1285745923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:38.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:38.171516+0000) 2023-08-27T23:44:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:38 smithi136 ceph-mon[113531]: pgmap v5132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:44:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:38 smithi093 ceph-mon[111335]: pgmap v5132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:44:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:38 smithi093 ceph-mon[118902]: pgmap v5132: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:44:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:39.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:39.171708+0000) 2023-08-27T23:44:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4003803225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:39.979 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:39.979 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4003803225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:39.979 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:39.979 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4003803225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:40.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:40.171880+0000) 2023-08-27T23:44:40.836 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-mon[111335]: pgmap v5133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:40.836 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:40.836 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:40.836 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-mon[118902]: pgmap v5133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:40.837 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:40.837 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:40 smithi136 ceph-mon[113531]: pgmap v5133: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:41.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:40.835Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:40.836Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:41.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:41.172095+0000) 2023-08-27T23:44:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:42.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:42.172304+0000) 2023-08-27T23:44:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-mon[113531]: pgmap v5134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:44:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:44:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2414869073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:42.313Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:42.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:42.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[111335]: pgmap v5134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:44:42.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:44:42.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2414869073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[118902]: pgmap v5134: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:44:42.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:44:42.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:44:42.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2414869073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:43.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:43.172559+0000) 2023-08-27T23:44:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:44 smithi136 ceph-mon[113531]: pgmap v5135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3228511015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:44.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:44.172864+0000) 2023-08-27T23:44:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:44 smithi093 ceph-mon[111335]: pgmap v5135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3228511015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:44 smithi093 ceph-mon[118902]: pgmap v5135: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3228511015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:45.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:45.173151+0000) 2023-08-27T23:44:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:44:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:44:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:44:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:46 smithi136 ceph-mon[113531]: pgmap v5136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:46.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:46.173341+0000) 2023-08-27T23:44:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:46 smithi093 ceph-mon[111335]: pgmap v5136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:46 smithi093 ceph-mon[118902]: pgmap v5136: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/718474331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:47.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:47.173564+0000) 2023-08-27T23:44:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/718474331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/718474331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:48 smithi136 ceph-mon[113531]: pgmap v5137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:48.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:48.173812+0000) 2023-08-27T23:44:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:48 smithi093 ceph-mon[111335]: pgmap v5137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:48 smithi093 ceph-mon[118902]: pgmap v5137: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3613294570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:49.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:49.174112+0000) 2023-08-27T23:44:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3613294570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3613294570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:50 smithi136 ceph-mon[113531]: pgmap v5138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:50.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:50.174348+0000) 2023-08-27T23:44:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:50 smithi093 ceph-mon[111335]: pgmap v5138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:50 smithi093 ceph-mon[118902]: pgmap v5138: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:50.836Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:50.837Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:51.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:51.174613+0000) 2023-08-27T23:44:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:52 smithi136 ceph-mon[113531]: pgmap v5139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1223742140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:52.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:52.174807+0000) 2023-08-27T23:44:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:52 smithi093 ceph-mon[111335]: pgmap v5139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1223742140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:52 smithi093 ceph-mon[118902]: pgmap v5139: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1223742140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:52.314Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:44:52.315Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:44:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:53.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:53.175072+0000) 2023-08-27T23:44:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:54 smithi136 ceph-mon[113531]: pgmap v5140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/823416047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:54.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:54.175313+0000) 2023-08-27T23:44:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:54 smithi093 ceph-mon[111335]: pgmap v5140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/823416047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:54 smithi093 ceph-mon[118902]: pgmap v5140: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/823416047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:44:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:55.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:55.175532+0000) 2023-08-27T23:44:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:44:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:44:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:44:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:56 smithi136 ceph-mon[113531]: pgmap v5141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4018812954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:56.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:56.175806+0000) 2023-08-27T23:44:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:56 smithi093 ceph-mon[111335]: pgmap v5141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4018812954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:56 smithi093 ceph-mon[118902]: pgmap v5141: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4018812954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:57.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:57.176069+0000) 2023-08-27T23:44:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:58 smithi136 ceph-mon[113531]: pgmap v5142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:58.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:58.176323+0000) 2023-08-27T23:44:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:58 smithi093 ceph-mon[111335]: pgmap v5142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:58 smithi093 ceph-mon[118902]: pgmap v5142: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:44:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:44:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/518226355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:44:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:44:59.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:44:59.176583+0000) 2023-08-27T23:44:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:44:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/518226355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:44:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:44:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/518226355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:00 smithi136 ceph-mon[113531]: pgmap v5143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:00.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:00.176796+0000) 2023-08-27T23:45:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:00 smithi093 ceph-mon[111335]: pgmap v5143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:00 smithi093 ceph-mon[118902]: pgmap v5143: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:00.837Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:00.838Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/55427433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:01.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:01.177001+0000) 2023-08-27T23:45:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/55427433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/55427433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:02 smithi136 ceph-mon[113531]: pgmap v5144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:02.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:02.177203+0000) 2023-08-27T23:45:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:02 smithi093 ceph-mon[111335]: pgmap v5144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:02 smithi093 ceph-mon[118902]: pgmap v5144: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:02.314Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:02.315Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/477390208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:03.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:03.177412+0000) 2023-08-27T23:45:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/477390208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/477390208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:04 smithi136 ceph-mon[113531]: pgmap v5145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:04.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:04.177602+0000) 2023-08-27T23:45:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:04 smithi093 ceph-mon[111335]: pgmap v5145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:04 smithi093 ceph-mon[118902]: pgmap v5145: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:05.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:05.177855+0000) 2023-08-27T23:45:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:45:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:45:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:45:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:06 smithi136 ceph-mon[113531]: pgmap v5146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3900509463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:06.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:06.178133+0000) 2023-08-27T23:45:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:06 smithi093 ceph-mon[111335]: pgmap v5146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3900509463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:06 smithi093 ceph-mon[118902]: pgmap v5146: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3900509463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:07.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:07.178362+0000) 2023-08-27T23:45:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:08 smithi136 ceph-mon[113531]: pgmap v5147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/412934742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:08.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:08.178620+0000) 2023-08-27T23:45:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:08 smithi093 ceph-mon[111335]: pgmap v5147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/412934742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:08 smithi093 ceph-mon[118902]: pgmap v5147: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/412934742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:09.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:09.178888+0000) 2023-08-27T23:45:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:10 smithi136 ceph-mon[113531]: pgmap v5148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:10.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:10.179134+0000) 2023-08-27T23:45:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:10 smithi093 ceph-mon[118902]: pgmap v5148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:10 smithi093 ceph-mon[111335]: pgmap v5148: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:10.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:10.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4248562179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:11.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:11.179388+0000) 2023-08-27T23:45:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4248562179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4248562179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:12 smithi136 ceph-mon[113531]: pgmap v5149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:12.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:12.179666+0000) 2023-08-27T23:45:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:12 smithi093 ceph-mon[111335]: pgmap v5149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:12 smithi093 ceph-mon[118902]: pgmap v5149: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:12.315Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:12.316Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1371623907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:13.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:13.179928+0000) 2023-08-27T23:45:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1371623907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1371623907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:14 smithi136 ceph-mon[113531]: pgmap v5150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:14.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:14.180201+0000) 2023-08-27T23:45:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:14 smithi093 ceph-mon[111335]: pgmap v5150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:14 smithi093 ceph-mon[118902]: pgmap v5150: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/350510700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:15.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:15.180475+0000) 2023-08-27T23:45:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:45:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:45:15] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:45:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/350510700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/350510700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:16 smithi136 ceph-mon[113531]: pgmap v5151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:16.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:16.180795+0000) 2023-08-27T23:45:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:16 smithi093 ceph-mon[111335]: pgmap v5151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:16 smithi093 ceph-mon[118902]: pgmap v5151: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:17.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:17.181023+0000) 2023-08-27T23:45:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:18 smithi136 ceph-mon[113531]: pgmap v5152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3867465191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:18.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:18.181327+0000) 2023-08-27T23:45:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:18 smithi093 ceph-mon[111335]: pgmap v5152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3867465191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:18 smithi093 ceph-mon[118902]: pgmap v5152: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3867465191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:19.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:19.181595+0000) 2023-08-27T23:45:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:20 smithi136 ceph-mon[113531]: pgmap v5153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/927976304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:20.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:20.181829+0000) 2023-08-27T23:45:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:20 smithi093 ceph-mon[111335]: pgmap v5153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/927976304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:20 smithi093 ceph-mon[118902]: pgmap v5153: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/927976304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:20.839Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:20.840Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:21.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:21.182111+0000) 2023-08-27T23:45:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:22 smithi136 ceph-mon[113531]: pgmap v5154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/91425885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:22.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:22.182364+0000) 2023-08-27T23:45:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:22 smithi093 ceph-mon[111335]: pgmap v5154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/91425885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:22 smithi093 ceph-mon[118902]: pgmap v5154: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/91425885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:22.316Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:22.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:23.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:23.182576+0000) 2023-08-27T23:45:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:24 smithi136 ceph-mon[113531]: pgmap v5155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:24.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:24.182817+0000) 2023-08-27T23:45:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:24 smithi093 ceph-mon[111335]: pgmap v5155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:24 smithi093 ceph-mon[118902]: pgmap v5155: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1929611719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:25.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:25.183007+0000) 2023-08-27T23:45:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1929611719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1929611719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:45:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:45:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:45:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:26 smithi136 ceph-mon[113531]: pgmap v5156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:26.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:26.183287+0000) 2023-08-27T23:45:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:26 smithi093 ceph-mon[111335]: pgmap v5156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:26 smithi093 ceph-mon[118902]: pgmap v5156: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3760420672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:27.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:27.183573+0000) 2023-08-27T23:45:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3760420672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3760420672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:28 smithi136 ceph-mon[113531]: pgmap v5157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:28.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:28.183835+0000) 2023-08-27T23:45:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:28 smithi093 ceph-mon[111335]: pgmap v5157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:28 smithi093 ceph-mon[118902]: pgmap v5157: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:29.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:29.184109+0000) 2023-08-27T23:45:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:30 smithi136 ceph-mon[113531]: pgmap v5158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/87078811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:30.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:30.184367+0000) 2023-08-27T23:45:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:30 smithi093 ceph-mon[118902]: pgmap v5158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/87078811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:30 smithi093 ceph-mon[111335]: pgmap v5158: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/87078811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:30.839Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:30.840Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:31.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:31.184641+0000) 2023-08-27T23:45:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:32 smithi136 ceph-mon[113531]: pgmap v5159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3469825375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:32.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:32.184853+0000) 2023-08-27T23:45:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:32 smithi093 ceph-mon[111335]: pgmap v5159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3469825375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:32 smithi093 ceph-mon[118902]: pgmap v5159: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3469825375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:32.317Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:32.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:33.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:33.185113+0000) 2023-08-27T23:45:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:34 smithi136 ceph-mon[113531]: pgmap v5160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/51684211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:34.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:34.185335+0000) 2023-08-27T23:45:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:34 smithi093 ceph-mon[111335]: pgmap v5160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/51684211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:34 smithi093 ceph-mon[118902]: pgmap v5160: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/51684211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:35.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:35.185573+0000) 2023-08-27T23:45:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:45:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:45:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:45:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:36 smithi136 ceph-mon[113531]: pgmap v5161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:36.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:36.185844+0000) 2023-08-27T23:45:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:36 smithi093 ceph-mon[111335]: pgmap v5161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:36 smithi093 ceph-mon[118902]: pgmap v5161: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3610351077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:37.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:37.186118+0000) 2023-08-27T23:45:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3610351077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:37.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3610351077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:38 smithi136 ceph-mon[113531]: pgmap v5162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:38.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:38.186290+0000) 2023-08-27T23:45:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:38 smithi093 ceph-mon[111335]: pgmap v5162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:38 smithi093 ceph-mon[118902]: pgmap v5162: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:39.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4244887880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:39.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:39.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:39.186513+0000) 2023-08-27T23:45:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4244887880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4244887880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:40 smithi136 ceph-mon[113531]: pgmap v5163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:40.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:40.186812+0000) 2023-08-27T23:45:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:40 smithi093 ceph-mon[111335]: pgmap v5163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:40 smithi093 ceph-mon[118902]: pgmap v5163: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:40.840Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:40.841Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2571759216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:41.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:41.187133+0000) 2023-08-27T23:45:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2571759216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2571759216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:42 smithi136 ceph-mon[113531]: pgmap v5164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:45:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:42.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:42.187278+0000) 2023-08-27T23:45:42.678 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[111335]: pgmap v5164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:42.678 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:45:42.678 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:42.678 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:42.679 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[118902]: pgmap v5164: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:42.679 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:45:42.679 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:42.679 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:42.679 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:42.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:42.679 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:42.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:43.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:43.187432+0000) 2023-08-27T23:45:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:44.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:44.187579+0000) 2023-08-27T23:45:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: pgmap v5165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/771505404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:45:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: pgmap v5165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/771505404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: pgmap v5165: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/771505404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:44.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:45:44.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:45:44.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:45:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:45.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:45.187784+0000) 2023-08-27T23:45:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:45:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:45:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:45:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:46.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:46.187995+0000) 2023-08-27T23:45:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:46 smithi093 ceph-mon[111335]: pgmap v5166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/209616358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:46 smithi093 ceph-mon[118902]: pgmap v5166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/209616358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:46 smithi136 ceph-mon[113531]: pgmap v5166: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/209616358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:47.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:47.188188+0000) 2023-08-27T23:45:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:48.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:48.188371+0000) 2023-08-27T23:45:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:48 smithi093 ceph-mon[111335]: pgmap v5167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1467520564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:48 smithi093 ceph-mon[118902]: pgmap v5167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1467520564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:48 smithi136 ceph-mon[113531]: pgmap v5167: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1467520564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:49.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:49.188599+0000) 2023-08-27T23:45:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:50.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:50.188857+0000) 2023-08-27T23:45:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:50 smithi093 ceph-mon[111335]: pgmap v5168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:50 smithi093 ceph-mon[118902]: pgmap v5168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:50.840Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:50.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:50.841Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:50 smithi136 ceph-mon[113531]: pgmap v5168: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:51.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:51.189136+0000) 2023-08-27T23:45:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2955340784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2955340784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2955340784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:52.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:52.189367+0000) 2023-08-27T23:45:52.701 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:52.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:52.701 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:45:52.319Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:45:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:52 smithi093 ceph-mon[111335]: pgmap v5169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:52 smithi093 ceph-mon[118902]: pgmap v5169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:52 smithi136 ceph-mon[113531]: pgmap v5169: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:53.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:53.189604+0000) 2023-08-27T23:45:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4090195592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4090195592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4090195592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:54.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:54.189776+0000) 2023-08-27T23:45:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:54 smithi093 ceph-mon[111335]: pgmap v5170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:54 smithi093 ceph-mon[118902]: pgmap v5170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:54 smithi136 ceph-mon[113531]: pgmap v5170: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:45:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:55.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:55.189965+0000) 2023-08-27T23:45:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:45:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:45:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:45:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:56 smithi136 ceph-mon[113531]: pgmap v5171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/411472071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:56.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:56.190229+0000) 2023-08-27T23:45:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:56 smithi093 ceph-mon[111335]: pgmap v5171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/411472071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:56 smithi093 ceph-mon[118902]: pgmap v5171: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/411472071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:57.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:57.190449+0000) 2023-08-27T23:45:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:58 smithi136 ceph-mon[113531]: pgmap v5172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:45:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1862686628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:58.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:58.190708+0000) 2023-08-27T23:45:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:58 smithi093 ceph-mon[118902]: pgmap v5172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:45:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1862686628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:58 smithi093 ceph-mon[111335]: pgmap v5172: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:45:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:45:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1862686628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:45:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:45:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:45:59.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:45:59.191019+0000) 2023-08-27T23:46:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:00 smithi136 ceph-mon[113531]: pgmap v5173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:00.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:00.191305+0000) 2023-08-27T23:46:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:00 smithi093 ceph-mon[111335]: pgmap v5173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:00 smithi093 ceph-mon[118902]: pgmap v5173: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:01.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:00.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:00.841Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2477320511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:01.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:01.191588+0000) 2023-08-27T23:46:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2477320511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2477320511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:02 smithi136 ceph-mon[113531]: pgmap v5174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:02.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:02.191839+0000) 2023-08-27T23:46:02.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:02 smithi093 ceph-mon[111335]: pgmap v5174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:02 smithi093 ceph-mon[118902]: pgmap v5174: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:02.319Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:02.320Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/421666974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:03.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:03.192109+0000) 2023-08-27T23:46:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/421666974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/421666974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:04 smithi136 ceph-mon[113531]: pgmap v5175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:04.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:04.192375+0000) 2023-08-27T23:46:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:04 smithi093 ceph-mon[111335]: pgmap v5175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:04 smithi093 ceph-mon[118902]: pgmap v5175: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3181662259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:05.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:05.192644+0000) 2023-08-27T23:46:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3181662259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:46:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:46:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:46:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3181662259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:06 smithi136 ceph-mon[113531]: pgmap v5176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:06.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:06.192928+0000) 2023-08-27T23:46:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:06 smithi093 ceph-mon[111335]: pgmap v5176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:06 smithi093 ceph-mon[118902]: pgmap v5176: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:07.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:07.193185+0000) 2023-08-27T23:46:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:08 smithi136 ceph-mon[113531]: pgmap v5177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/871520085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:08.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:08.193438+0000) 2023-08-27T23:46:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:08 smithi093 ceph-mon[111335]: pgmap v5177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/871520085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:08 smithi093 ceph-mon[118902]: pgmap v5177: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/871520085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:09.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:09.193705+0000) 2023-08-27T23:46:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:10 smithi136 ceph-mon[113531]: pgmap v5178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1601804033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:10.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:10.193931+0000) 2023-08-27T23:46:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:10 smithi093 ceph-mon[111335]: pgmap v5178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1601804033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:10 smithi093 ceph-mon[118902]: pgmap v5178: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1601804033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:10.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:10.842Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:11.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:11.194143+0000) 2023-08-27T23:46:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:12 smithi136 ceph-mon[113531]: pgmap v5179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:12.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:12.194395+0000) 2023-08-27T23:46:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:12 smithi093 ceph-mon[111335]: pgmap v5179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:12 smithi093 ceph-mon[118902]: pgmap v5179: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:12.320Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:12.321Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3359623947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:13.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:13.194655+0000) 2023-08-27T23:46:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3359623947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3359623947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:14 smithi136 ceph-mon[113531]: pgmap v5180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:14.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:14.194867+0000) 2023-08-27T23:46:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:14 smithi093 ceph-mon[111335]: pgmap v5180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:14 smithi093 ceph-mon[118902]: pgmap v5180: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2745739337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:15.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:15.195085+0000) 2023-08-27T23:46:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2745739337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:46:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:46:15] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:46:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2745739337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:16 smithi136 ceph-mon[113531]: pgmap v5181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:16.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:16.195344+0000) 2023-08-27T23:46:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:16 smithi093 ceph-mon[111335]: pgmap v5181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:16 smithi093 ceph-mon[118902]: pgmap v5181: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2056792422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:17.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:17.195596+0000) 2023-08-27T23:46:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2056792422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2056792422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:18 smithi136 ceph-mon[113531]: pgmap v5182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:18.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:18.195904+0000) 2023-08-27T23:46:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:18 smithi093 ceph-mon[111335]: pgmap v5182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:18 smithi093 ceph-mon[118902]: pgmap v5182: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:19.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:19.196193+0000) 2023-08-27T23:46:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:20 smithi136 ceph-mon[113531]: pgmap v5183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3346371460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:20.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:20.196453+0000) 2023-08-27T23:46:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:20 smithi093 ceph-mon[111335]: pgmap v5183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3346371460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:20 smithi093 ceph-mon[118902]: pgmap v5183: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3346371460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:20.842Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:20.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:21.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:21.196691+0000) 2023-08-27T23:46:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:22 smithi136 ceph-mon[113531]: pgmap v5184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1899106142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:22.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:22.196927+0000) 2023-08-27T23:46:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:22 smithi093 ceph-mon[111335]: pgmap v5184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1899106142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:22 smithi093 ceph-mon[118902]: pgmap v5184: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1899106142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:22.321Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:22.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:23.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:23.197121+0000) 2023-08-27T23:46:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:24 smithi136 ceph-mon[113531]: pgmap v5185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1692960732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:24.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:24.197312+0000) 2023-08-27T23:46:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:24 smithi093 ceph-mon[118902]: pgmap v5185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1692960732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:24 smithi093 ceph-mon[111335]: pgmap v5185: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1692960732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:25.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:25.197531+0000) 2023-08-27T23:46:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:46:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:46:25] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:46:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:26 smithi136 ceph-mon[113531]: pgmap v5186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:26.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:26.197804+0000) 2023-08-27T23:46:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:26 smithi093 ceph-mon[111335]: pgmap v5186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:26 smithi093 ceph-mon[118902]: pgmap v5186: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3296509293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:27.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:27.198090+0000) 2023-08-27T23:46:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3296509293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3296509293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:28 smithi136 ceph-mon[113531]: pgmap v5187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:28.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:28.198330+0000) 2023-08-27T23:46:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:28 smithi093 ceph-mon[111335]: pgmap v5187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:28 smithi093 ceph-mon[118902]: pgmap v5187: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3256454239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:29.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:29.198612+0000) 2023-08-27T23:46:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3256454239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3256454239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:30 smithi136 ceph-mon[113531]: pgmap v5188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:30.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:30.198930+0000) 2023-08-27T23:46:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:30 smithi093 ceph-mon[111335]: pgmap v5188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:30 smithi093 ceph-mon[118902]: pgmap v5188: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:30.842Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:30.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:31.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:31.199169+0000) 2023-08-27T23:46:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:32 smithi136 ceph-mon[113531]: pgmap v5189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/427432693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:32.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:32.199393+0000) 2023-08-27T23:46:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:32 smithi093 ceph-mon[111335]: pgmap v5189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/427432693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:32 smithi093 ceph-mon[118902]: pgmap v5189: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/427432693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:32.322Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:32.323Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:33.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:33.199650+0000) 2023-08-27T23:46:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:34 smithi136 ceph-mon[113531]: pgmap v5190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2612577510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:34.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:34.199934+0000) 2023-08-27T23:46:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:34 smithi093 ceph-mon[111335]: pgmap v5190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2612577510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:34 smithi093 ceph-mon[118902]: pgmap v5190: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2612577510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:35.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:35.200195+0000) 2023-08-27T23:46:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:46:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:46:35] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:46:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:36 smithi136 ceph-mon[113531]: pgmap v5191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2932567529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:36.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:36.200447+0000) 2023-08-27T23:46:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:36 smithi093 ceph-mon[111335]: pgmap v5191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2932567529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:36 smithi093 ceph-mon[118902]: pgmap v5191: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2932567529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:37.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:37.200625+0000) 2023-08-27T23:46:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:38 smithi136 ceph-mon[113531]: pgmap v5192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:38.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:38.200837+0000) 2023-08-27T23:46:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:38 smithi093 ceph-mon[111335]: pgmap v5192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:38 smithi093 ceph-mon[118902]: pgmap v5192: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/422498037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:39.726 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:39.727 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:39.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:39.201157+0000) 2023-08-27T23:46:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/422498037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/422498037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:40 smithi136 ceph-mon[113531]: pgmap v5193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:40.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:40.201435+0000) 2023-08-27T23:46:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:40 smithi093 ceph-mon[111335]: pgmap v5193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:40 smithi093 ceph-mon[118902]: pgmap v5193: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:40.843Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:40.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:41.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/985440219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:41.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:41.201697+0000) 2023-08-27T23:46:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/985440219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/985440219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:42 smithi136 ceph-mon[113531]: pgmap v5194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:42.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:42.201946+0000) 2023-08-27T23:46:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:42 smithi093 ceph-mon[111335]: pgmap v5194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:42 smithi093 ceph-mon[118902]: pgmap v5194: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:42.323Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:42.324Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2456226922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:43.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:43.202187+0000) 2023-08-27T23:46:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2456226922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:43.780 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2456226922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:44 smithi136 ceph-mon[113531]: pgmap v5195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:44.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:44.202472+0000) 2023-08-27T23:46:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:44 smithi093 ceph-mon[111335]: pgmap v5195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:44 smithi093 ceph-mon[118902]: pgmap v5195: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:45.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:46:45.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:45.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:45.677 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:45.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:45.202686+0000) 2023-08-27T23:46:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:46:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:46:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:46:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:46:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:46.392 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:46.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:46.202920+0000) 2023-08-27T23:46:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:46 smithi136 ceph-mon[113531]: pgmap v5196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4155979686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:46 smithi093 ceph-mon[111335]: pgmap v5196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4155979686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:46 smithi093 ceph-mon[118902]: pgmap v5196: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4155979686' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:47.263 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:47.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:47.203165+0000) 2023-08-27T23:46:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:48.557 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:48.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:48.203403+0000) 2023-08-27T23:46:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:48 smithi136 ceph-mon[113531]: pgmap v5197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:46:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:46:48.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:48.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/357230971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[118902]: pgmap v5197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:46:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:46:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/357230971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[111335]: pgmap v5197: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:46:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:46:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:46:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/357230971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:49.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:49.203677+0000) 2023-08-27T23:46:50.562 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:50.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:50.203914+0000) 2023-08-27T23:46:50.845 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:50 smithi093 ceph-mon[111335]: pgmap v5198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:50.846 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1907159120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:50.846 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:50 smithi093 ceph-mon[118902]: pgmap v5198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:50.846 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1907159120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:50 smithi136 ceph-mon[113531]: pgmap v5198: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1907159120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:50.844Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:50.845Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:51.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:51.204128+0000) 2023-08-27T23:46:52.565 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:52.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:52.204327+0000) 2023-08-27T23:46:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:52 smithi093 ceph-mon[111335]: pgmap v5199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:52 smithi093 ceph-mon[118902]: pgmap v5199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:52.324Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:46:52.325Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:46:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:52 smithi136 ceph-mon[113531]: pgmap v5199: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:53.572 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:53.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:53.204530+0000) 2023-08-27T23:46:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/970393678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/970393678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/970393678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:54.574 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:54.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:54.204788+0000) 2023-08-27T23:46:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:54 smithi136 ceph-mon[113531]: pgmap v5200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:54 smithi093 ceph-mon[111335]: pgmap v5200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:54 smithi093 ceph-mon[118902]: pgmap v5200: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:46:55.582 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:55.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:55.205023+0000) 2023-08-27T23:46:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1220324694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1220324694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:46:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:46:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1220324694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:56.584 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:56.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:56.205262+0000) 2023-08-27T23:46:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:56 smithi136 ceph-mon[113531]: pgmap v5201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:56 smithi093 ceph-mon[111335]: pgmap v5201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:56 smithi093 ceph-mon[118902]: pgmap v5201: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:57.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:57.205526+0000) 2023-08-27T23:46:58.587 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:58.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:58.205721+0000) 2023-08-27T23:46:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:58 smithi136 ceph-mon[113531]: pgmap v5202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:46:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4039120820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:58 smithi093 ceph-mon[111335]: pgmap v5202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:46:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4039120820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:58 smithi093 ceph-mon[118902]: pgmap v5202: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:46:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:46:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4039120820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:46:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:46:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:46:59.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:46:59.205997+0000) 2023-08-27T23:47:00.591 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:00.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:00.206215+0000) 2023-08-27T23:47:00.846 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:00 smithi093 ceph-mon[111335]: pgmap v5203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:00.846 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/814411311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:00.846 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:00 smithi093 ceph-mon[118902]: pgmap v5203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:00.846 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/814411311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:00 smithi136 ceph-mon[113531]: pgmap v5203: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/814411311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:00.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:00.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:01.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:01.206434+0000) 2023-08-27T23:47:02.593 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:02.324Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:02.593 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:02.325Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:02.594 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:02.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:02.206674+0000) 2023-08-27T23:47:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:02 smithi136 ceph-mon[113531]: pgmap v5204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3672161051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:02 smithi093 ceph-mon[111335]: pgmap v5204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3672161051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:02 smithi093 ceph-mon[118902]: pgmap v5204: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3672161051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:03.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:03.206932+0000) 2023-08-27T23:47:04.596 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:04.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:04.207198+0000) 2023-08-27T23:47:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:04 smithi136 ceph-mon[113531]: pgmap v5205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:04 smithi093 ceph-mon[111335]: pgmap v5205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:04 smithi093 ceph-mon[118902]: pgmap v5205: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:05.606 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:05.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:05.207427+0000) 2023-08-27T23:47:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3469989186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:47:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:47:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:47:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3469989186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3469989186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:06.606 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:06.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:06.207670+0000) 2023-08-27T23:47:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:06 smithi136 ceph-mon[113531]: pgmap v5206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:06 smithi093 ceph-mon[111335]: pgmap v5206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:06 smithi093 ceph-mon[118902]: pgmap v5206: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:07.610 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:07.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:07.207928+0000) 2023-08-27T23:47:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3973677291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3973677291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3973677291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:08.612 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:08.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:08.208194+0000) 2023-08-27T23:47:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:08 smithi136 ceph-mon[113531]: pgmap v5207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:08 smithi093 ceph-mon[111335]: pgmap v5207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:08 smithi093 ceph-mon[118902]: pgmap v5207: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:09.616 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:09.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:09.208393+0000) 2023-08-27T23:47:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/279078558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/279078558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/279078558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:10.618 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:10.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:10.208646+0000) 2023-08-27T23:47:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:10 smithi136 ceph-mon[113531]: pgmap v5208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:10 smithi093 ceph-mon[111335]: pgmap v5208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:10 smithi093 ceph-mon[118902]: pgmap v5208: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:10.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:10.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:11.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:11.208899+0000) 2023-08-27T23:47:12.620 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:12.325Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:12.620 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:12.326Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:12.621 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:12.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:12.209189+0000) 2023-08-27T23:47:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:12 smithi136 ceph-mon[113531]: pgmap v5209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4121136618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:12 smithi093 ceph-mon[118902]: pgmap v5209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4121136618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:12 smithi093 ceph-mon[111335]: pgmap v5209: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4121136618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:13.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:13.209435+0000) 2023-08-27T23:47:14.624 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:14.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:14.209678+0000) 2023-08-27T23:47:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:14 smithi136 ceph-mon[113531]: pgmap v5210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2423295389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:14 smithi093 ceph-mon[111335]: pgmap v5210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2423295389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:14 smithi093 ceph-mon[118902]: pgmap v5210: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2423295389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:15.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:15.209926+0000) 2023-08-27T23:47:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:47:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:47:16.626 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:16.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:16.210137+0000) 2023-08-27T23:47:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:16 smithi136 ceph-mon[113531]: pgmap v5211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:16 smithi093 ceph-mon[111335]: pgmap v5211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:16 smithi093 ceph-mon[118902]: pgmap v5211: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:17.631 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:17.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:17.210402+0000) 2023-08-27T23:47:17.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1693071818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1693071818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1693071818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:18.633 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:18.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:18.210659+0000) 2023-08-27T23:47:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:18 smithi136 ceph-mon[113531]: pgmap v5212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:18 smithi093 ceph-mon[111335]: pgmap v5212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:18 smithi093 ceph-mon[118902]: pgmap v5212: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:19.644 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:19.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:19.210897+0000) 2023-08-27T23:47:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2634884868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2634884868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2634884868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:20.645 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:20.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:20.211169+0000) 2023-08-27T23:47:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:20 smithi136 ceph-mon[113531]: pgmap v5213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:20 smithi093 ceph-mon[118902]: pgmap v5213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:20 smithi093 ceph-mon[111335]: pgmap v5213: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:20.846Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:20.847Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:21.650 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:21.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:21.211455+0000) 2023-08-27T23:47:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/683106985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/683106985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/683106985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:22.651 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:22.326Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:22.651 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:22.327Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:22.651 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:22.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:22.211731+0000) 2023-08-27T23:47:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:22 smithi136 ceph-mon[113531]: pgmap v5214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:22 smithi093 ceph-mon[111335]: pgmap v5214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:22 smithi093 ceph-mon[118902]: pgmap v5214: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:23.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:23.211975+0000) 2023-08-27T23:47:24.654 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:24.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:24.212194+0000) 2023-08-27T23:47:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:24 smithi136 ceph-mon[113531]: pgmap v5215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1604160809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:24 smithi093 ceph-mon[111335]: pgmap v5215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1604160809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:24 smithi093 ceph-mon[118902]: pgmap v5215: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1604160809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:25.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:25.212380+0000) 2023-08-27T23:47:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:47:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:47:25] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:47:26.657 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:26.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:26.212607+0000) 2023-08-27T23:47:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:26 smithi136 ceph-mon[113531]: pgmap v5216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2196246430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:26 smithi093 ceph-mon[111335]: pgmap v5216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2196246430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:26 smithi093 ceph-mon[118902]: pgmap v5216: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2196246430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:27.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:27.212811+0000) 2023-08-27T23:47:28.660 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:28.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:28.213066+0000) 2023-08-27T23:47:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:28 smithi136 ceph-mon[113531]: pgmap v5217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1990010493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:28 smithi093 ceph-mon[111335]: pgmap v5217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1990010493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:28 smithi093 ceph-mon[118902]: pgmap v5217: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1990010493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:29.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:29.213276+0000) 2023-08-27T23:47:30.663 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:30.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:30.213488+0000) 2023-08-27T23:47:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:30 smithi136 ceph-mon[113531]: pgmap v5218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:30 smithi093 ceph-mon[111335]: pgmap v5218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:30 smithi093 ceph-mon[118902]: pgmap v5218: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:30.848Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:30.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:30.848Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:31.669 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:31.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:31.213729+0000) 2023-08-27T23:47:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1086427513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1086427513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1086427513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:32.669 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:32.326Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:32.669 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:32.328Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:32.669 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:32.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:32.213992+0000) 2023-08-27T23:47:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:32 smithi136 ceph-mon[113531]: pgmap v5219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:32 smithi093 ceph-mon[111335]: pgmap v5219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:32 smithi093 ceph-mon[118902]: pgmap v5219: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:33.674 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:33.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:33.214322+0000) 2023-08-27T23:47:33.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/851099166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/851099166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/851099166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:34.675 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:34.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:34.214576+0000) 2023-08-27T23:47:34.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:34 smithi136 ceph-mon[113531]: pgmap v5220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:34 smithi093 ceph-mon[111335]: pgmap v5220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:34 smithi093 ceph-mon[118902]: pgmap v5220: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:35.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:35.214817+0000) 2023-08-27T23:47:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:47:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:47:35] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:47:36.678 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:36.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:36.215123+0000) 2023-08-27T23:47:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:36 smithi136 ceph-mon[113531]: pgmap v5221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3979432053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:36 smithi093 ceph-mon[111335]: pgmap v5221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3979432053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:36 smithi093 ceph-mon[118902]: pgmap v5221: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3979432053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:37.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:37.215440+0000) 2023-08-27T23:47:38.682 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:38.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:38.215719+0000) 2023-08-27T23:47:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:38 smithi136 ceph-mon[113531]: pgmap v5222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:38.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2214574680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:38 smithi093 ceph-mon[111335]: pgmap v5222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2214574680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:38 smithi093 ceph-mon[118902]: pgmap v5222: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2214574680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:39.684 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:39.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:39.215998+0000) 2023-08-27T23:47:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:40.686 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:40.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:40.216310+0000) 2023-08-27T23:47:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:40 smithi136 ceph-mon[113531]: pgmap v5223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1576067886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:40 smithi093 ceph-mon[111335]: pgmap v5223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1576067886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:40 smithi093 ceph-mon[118902]: pgmap v5223: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1576067886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:40.849Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:40.850Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:41.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:41.216566+0000) 2023-08-27T23:47:42.688 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:42.328Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:42.688 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:42.329Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:42.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:42.216837+0000) 2023-08-27T23:47:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:42 smithi136 ceph-mon[113531]: pgmap v5224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:42 smithi093 ceph-mon[111335]: pgmap v5224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:42 smithi093 ceph-mon[118902]: pgmap v5224: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:43.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:43.217141+0000) 2023-08-27T23:47:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3395584608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3395584608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3395584608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:44.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:44.217378+0000) 2023-08-27T23:47:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:44 smithi093 ceph-mon[111335]: pgmap v5225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:44 smithi093 ceph-mon[118902]: pgmap v5225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:44 smithi136 ceph-mon[113531]: pgmap v5225: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:45.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:45.217654+0000) 2023-08-27T23:47:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:47:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:47:45] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:47:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:46 smithi136 ceph-mon[113531]: pgmap v5226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/379572455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:46.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:46.217934+0000) 2023-08-27T23:47:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:46 smithi093 ceph-mon[118902]: pgmap v5226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/379572455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:46 smithi093 ceph-mon[111335]: pgmap v5226: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/379572455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:47.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:47.218150+0000) 2023-08-27T23:47:48.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:48 smithi136 ceph-mon[113531]: pgmap v5227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:48.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3556992637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:48.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:47:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:48.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:48.218326+0000) 2023-08-27T23:47:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:48 smithi093 ceph-mon[111335]: pgmap v5227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3556992637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:47:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:48 smithi093 ceph-mon[118902]: pgmap v5227: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3556992637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:47:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:49.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:49.218484+0000) 2023-08-27T23:47:50.320 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:50 smithi136 ceph-mon[113531]: pgmap v5228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:50.321 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:50.321 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:50.321 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2346517748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:50.321 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:50.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:50.218730+0000) 2023-08-27T23:47:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[111335]: pgmap v5228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2346517748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[118902]: pgmap v5228: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2346517748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:50.849Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:50.850Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:51.578 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:51.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:51.218957+0000) 2023-08-27T23:47:51.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:47:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:47:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:47:51.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:47:51.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:51.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:47:51.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:47:51.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:47:52.583 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:52.328Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:52.584 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:47:52.329Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:47:52.584 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:52.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:52.219211+0000) 2023-08-27T23:47:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:52 smithi136 ceph-mon[113531]: pgmap v5229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2456668087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:52 smithi093 ceph-mon[111335]: pgmap v5229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2456668087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:52 smithi093 ceph-mon[118902]: pgmap v5229: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2456668087' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:53.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:53.219488+0000) 2023-08-27T23:47:54.587 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:54.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:54.219775+0000) 2023-08-27T23:47:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:54 smithi136 ceph-mon[113531]: pgmap v5230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:54 smithi093 ceph-mon[118902]: pgmap v5230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:54 smithi093 ceph-mon[111335]: pgmap v5230: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:47:55.596 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:55.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:55.220081+0000) 2023-08-27T23:47:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3941749812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:47:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:47:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:47:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3941749812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:55.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3941749812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:56.598 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:56.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:56.220388+0000) 2023-08-27T23:47:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:56 smithi136 ceph-mon[113531]: pgmap v5231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:56 smithi093 ceph-mon[111335]: pgmap v5231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:56 smithi093 ceph-mon[118902]: pgmap v5231: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:57.603 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:57.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:57.220654+0000) 2023-08-27T23:47:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/741797746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/741797746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/741797746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:58.604 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:58.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:58.220872+0000) 2023-08-27T23:47:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:58 smithi136 ceph-mon[113531]: pgmap v5232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:58 smithi093 ceph-mon[111335]: pgmap v5232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:58 smithi093 ceph-mon[118902]: pgmap v5232: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:47:59.611 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:47:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:47:59.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:47:59.221079+0000) 2023-08-27T23:47:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:47:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1939656138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:47:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1939656138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:47:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:47:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1939656138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:00.614 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:00.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:00.221274+0000) 2023-08-27T23:48:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:00 smithi136 ceph-mon[113531]: pgmap v5233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:00 smithi093 ceph-mon[111335]: pgmap v5233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:00 smithi093 ceph-mon[118902]: pgmap v5233: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:00.850Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:00.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:00.852Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:01.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:01.221506+0000) 2023-08-27T23:48:02.616 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:02.328Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:02.617 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:02.329Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:02.617 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:02.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:02.221770+0000) 2023-08-27T23:48:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:02 smithi136 ceph-mon[113531]: pgmap v5234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2131247432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:02 smithi093 ceph-mon[111335]: pgmap v5234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2131247432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:02 smithi093 ceph-mon[118902]: pgmap v5234: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:02.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2131247432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:03.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:03.222090+0000) 2023-08-27T23:48:04.620 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:04.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:04.222305+0000) 2023-08-27T23:48:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:04 smithi136 ceph-mon[113531]: pgmap v5235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3174426083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:04 smithi093 ceph-mon[118902]: pgmap v5235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3174426083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:04 smithi093 ceph-mon[111335]: pgmap v5235: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3174426083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:05.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:05.222541+0000) 2023-08-27T23:48:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:48:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:48:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:48:06.625 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:06.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:06.222815+0000) 2023-08-27T23:48:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:06 smithi136 ceph-mon[113531]: pgmap v5236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:06 smithi093 ceph-mon[111335]: pgmap v5236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:06 smithi093 ceph-mon[118902]: pgmap v5236: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:07.632 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:07.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:07.223096+0000) 2023-08-27T23:48:07.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2297554724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2297554724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2297554724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:08.633 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:08.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:08.223325+0000) 2023-08-27T23:48:08.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:08 smithi136 ceph-mon[113531]: pgmap v5237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:08 smithi093 ceph-mon[111335]: pgmap v5237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:08 smithi093 ceph-mon[118902]: pgmap v5237: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:09.639 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:09.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:09.223613+0000) 2023-08-27T23:48:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3368114976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3368114976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3368114976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:10.642 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:10.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:10.223893+0000) 2023-08-27T23:48:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:10 smithi136 ceph-mon[113531]: pgmap v5238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:10 smithi093 ceph-mon[111335]: pgmap v5238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:10 smithi093 ceph-mon[118902]: pgmap v5238: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:10.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:10.851Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:10.852Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:11.645 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:11.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:11.224147+0000) 2023-08-27T23:48:11.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4162651023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4162651023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4162651023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:12.647 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:12.329Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:12.647 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:12.330Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:12.647 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:12.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:12.224336+0000) 2023-08-27T23:48:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:12 smithi136 ceph-mon[113531]: pgmap v5239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:12 smithi093 ceph-mon[111335]: pgmap v5239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:12 smithi093 ceph-mon[118902]: pgmap v5239: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:13.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:13.224594+0000) 2023-08-27T23:48:14.649 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:14.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:14.224905+0000) 2023-08-27T23:48:14.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:14 smithi136 ceph-mon[113531]: pgmap v5240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/305042976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:14 smithi093 ceph-mon[118902]: pgmap v5240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/305042976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:14 smithi093 ceph-mon[111335]: pgmap v5240: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/305042976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:15.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:15.225126+0000) 2023-08-27T23:48:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:48:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:48:15] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:48:16.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:16.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:16.225396+0000) 2023-08-27T23:48:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:16 smithi136 ceph-mon[113531]: pgmap v5241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1152904349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:16 smithi093 ceph-mon[111335]: pgmap v5241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1152904349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:16 smithi093 ceph-mon[118902]: pgmap v5241: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1152904349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:17.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:17.225598+0000) 2023-08-27T23:48:18.655 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:18.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:18.225857+0000) 2023-08-27T23:48:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:18 smithi136 ceph-mon[113531]: pgmap v5242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:18 smithi093 ceph-mon[111335]: pgmap v5242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:18 smithi093 ceph-mon[118902]: pgmap v5242: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:19.659 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:19.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:19.226100+0000) 2023-08-27T23:48:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471475607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471475607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471475607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:20.663 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:20.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:20.226364+0000) 2023-08-27T23:48:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:20 smithi136 ceph-mon[113531]: pgmap v5243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:20 smithi093 ceph-mon[111335]: pgmap v5243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:20 smithi093 ceph-mon[118902]: pgmap v5243: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:20.852Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:20.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:20.853Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:21.665 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:21.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:21.226584+0000) 2023-08-27T23:48:21.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/294439708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/294439708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/294439708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:22.667 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:22.329Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:22.667 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:22.330Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:22.667 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:22.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:22.226822+0000) 2023-08-27T23:48:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:22 smithi136 ceph-mon[113531]: pgmap v5244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:22 smithi093 ceph-mon[111335]: pgmap v5244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:22 smithi093 ceph-mon[118902]: pgmap v5244: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:23.674 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:23.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:23.227143+0000) 2023-08-27T23:48:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3427300593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3427300593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3427300593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:24.673 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:24.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:24.227446+0000) 2023-08-27T23:48:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:24 smithi136 ceph-mon[113531]: pgmap v5245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:24 smithi093 ceph-mon[111335]: pgmap v5245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:24 smithi093 ceph-mon[118902]: pgmap v5245: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:25.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:25.227659+0000) 2023-08-27T23:48:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:48:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:48:25] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:48:26.678 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:26.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:26.227937+0000) 2023-08-27T23:48:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:26 smithi136 ceph-mon[113531]: pgmap v5246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1249682527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:26 smithi093 ceph-mon[111335]: pgmap v5246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1249682527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:26 smithi093 ceph-mon[118902]: pgmap v5246: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1249682527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:27.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:27.228212+0000) 2023-08-27T23:48:28.681 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:28.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:28.228456+0000) 2023-08-27T23:48:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:28 smithi136 ceph-mon[113531]: pgmap v5247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:28.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/532701729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:28 smithi093 ceph-mon[111335]: pgmap v5247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/532701729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:28 smithi093 ceph-mon[118902]: pgmap v5247: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/532701729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:29.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:29.228694+0000) 2023-08-27T23:48:30.684 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:30.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:30.228923+0000) 2023-08-27T23:48:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:30 smithi136 ceph-mon[113531]: pgmap v5248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3603883874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:30 smithi093 ceph-mon[111335]: pgmap v5248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3603883874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:30 smithi093 ceph-mon[118902]: pgmap v5248: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3603883874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:30.853Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:30.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:31.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:31.229172+0000) 2023-08-27T23:48:32.686 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:32.330Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:32.686 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:32.331Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:32.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:32.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:32.229426+0000) 2023-08-27T23:48:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:32 smithi136 ceph-mon[113531]: pgmap v5249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:32.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:32 smithi093 ceph-mon[111335]: pgmap v5249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:32 smithi093 ceph-mon[118902]: pgmap v5249: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:33.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:33.229667+0000) 2023-08-27T23:48:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1269299252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1269299252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1269299252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:34.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:34.229918+0000) 2023-08-27T23:48:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:34 smithi093 ceph-mon[111335]: pgmap v5250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:34 smithi093 ceph-mon[118902]: pgmap v5250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:34 smithi136 ceph-mon[113531]: pgmap v5250: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:35.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:35.230207+0000) 2023-08-27T23:48:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:48:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:48:35] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:48:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:36 smithi136 ceph-mon[113531]: pgmap v5251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/907472139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:36.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:36.230464+0000) 2023-08-27T23:48:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:36 smithi093 ceph-mon[111335]: pgmap v5251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/907472139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:36 smithi093 ceph-mon[118902]: pgmap v5251: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/907472139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:37.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:37.230702+0000) 2023-08-27T23:48:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:38 smithi136 ceph-mon[113531]: pgmap v5252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2390700763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:38.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:38.230955+0000) 2023-08-27T23:48:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:38 smithi093 ceph-mon[111335]: pgmap v5252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2390700763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:38 smithi093 ceph-mon[118902]: pgmap v5252: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2390700763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:39.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:39.231201+0000) 2023-08-27T23:48:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:40 smithi136 ceph-mon[113531]: pgmap v5253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/705871245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:40.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:40.231416+0000) 2023-08-27T23:48:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:40 smithi093 ceph-mon[118902]: pgmap v5253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/705871245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:40 smithi093 ceph-mon[111335]: pgmap v5253: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:40.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/705871245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:40.854Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:40.855Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:41.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:41.231672+0000) 2023-08-27T23:48:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:42 smithi136 ceph-mon[113531]: pgmap v5254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:42.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:42.231884+0000) 2023-08-27T23:48:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:42 smithi093 ceph-mon[111335]: pgmap v5254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:42 smithi093 ceph-mon[118902]: pgmap v5254: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:42.331Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:42.332Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/931952770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:43.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:43.232089+0000) 2023-08-27T23:48:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/931952770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/931952770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:44 smithi136 ceph-mon[113531]: pgmap v5255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:44.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:44.232316+0000) 2023-08-27T23:48:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:44 smithi093 ceph-mon[111335]: pgmap v5255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:44 smithi093 ceph-mon[118902]: pgmap v5255: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/745445681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:45.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:45.232607+0000) 2023-08-27T23:48:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/745445681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/745445681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:48:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:48:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:48:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:46 smithi136 ceph-mon[113531]: pgmap v5256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:46.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:46.232932+0000) 2023-08-27T23:48:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:46 smithi093 ceph-mon[111335]: pgmap v5256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:46 smithi093 ceph-mon[118902]: pgmap v5256: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1468000106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:47.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:47.233198+0000) 2023-08-27T23:48:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1468000106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1468000106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:48 smithi136 ceph-mon[113531]: pgmap v5257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:48.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:48.233435+0000) 2023-08-27T23:48:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:48 smithi093 ceph-mon[111335]: pgmap v5257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:48 smithi093 ceph-mon[118902]: pgmap v5257: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:49.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:49.233687+0000) 2023-08-27T23:48:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:50 smithi136 ceph-mon[113531]: pgmap v5258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1204930038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:50.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:50.233928+0000) 2023-08-27T23:48:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:50 smithi093 ceph-mon[111335]: pgmap v5258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1204930038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:50 smithi093 ceph-mon[118902]: pgmap v5258: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1204930038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:50.855Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:50.857Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:48:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:51.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:51.234129+0000) 2023-08-27T23:48:51.727 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:48:51.727 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:48:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:52 smithi136 ceph-mon[113531]: pgmap v5259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/585625884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:52.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:52.234288+0000) 2023-08-27T23:48:52.625 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:52 smithi093 ceph-mon[111335]: pgmap v5259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:52.625 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/585625884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:52.625 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:52 smithi093 ceph-mon[118902]: pgmap v5259: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:52.625 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/585625884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:52.626 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:52.332Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:52.626 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:48:52.333Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:48:53.328 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:53.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:53.234451+0000) 2023-08-27T23:48:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:53 smithi136 ceph-mon[113531]: pgmap v5260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[111335]: pgmap v5260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[118902]: pgmap v5260: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:48:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:54.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:54.234655+0000) 2023-08-27T23:48:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:48:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:48:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/656781395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:48:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:48:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/656781395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:48:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:48:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:48:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/656781395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:55.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:55.234913+0000) 2023-08-27T23:48:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:48:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:48:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:48:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:56 smithi136 ceph-mon[113531]: pgmap v5261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:56.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:56.235219+0000) 2023-08-27T23:48:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:56 smithi093 ceph-mon[111335]: pgmap v5261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:56 smithi093 ceph-mon[118902]: pgmap v5261: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4064485065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:57.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:57.235473+0000) 2023-08-27T23:48:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4064485065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4064485065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:58 smithi136 ceph-mon[113531]: pgmap v5262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:58.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:58.235707+0000) 2023-08-27T23:48:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:58 smithi093 ceph-mon[111335]: pgmap v5262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:58 smithi093 ceph-mon[118902]: pgmap v5262: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:48:59.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:48:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1195077016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:48:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:48:59.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:48:59.235981+0000) 2023-08-27T23:48:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:48:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1195077016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:48:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:48:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1195077016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:00 smithi136 ceph-mon[113531]: pgmap v5263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:00.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:00.236260+0000) 2023-08-27T23:49:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:00 smithi093 ceph-mon[111335]: pgmap v5263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:00 smithi093 ceph-mon[118902]: pgmap v5263: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:00.856Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:00.857Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:01.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:01.236482+0000) 2023-08-27T23:49:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:02 smithi136 ceph-mon[113531]: pgmap v5264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3979125609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:02.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:02.236740+0000) 2023-08-27T23:49:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:02 smithi093 ceph-mon[118902]: pgmap v5264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3979125609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:02 smithi093 ceph-mon[111335]: pgmap v5264: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3979125609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:02.334Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:02.334Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:03.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:03.236970+0000) 2023-08-27T23:49:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:04 smithi136 ceph-mon[113531]: pgmap v5265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/922652486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:04.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:04.237229+0000) 2023-08-27T23:49:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:04 smithi093 ceph-mon[111335]: pgmap v5265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/922652486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:04 smithi093 ceph-mon[118902]: pgmap v5265: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/922652486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:05.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:05.237491+0000) 2023-08-27T23:49:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:49:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:49:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:49:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:06 smithi136 ceph-mon[113531]: pgmap v5266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:06.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:06.237745+0000) 2023-08-27T23:49:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:06 smithi093 ceph-mon[111335]: pgmap v5266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:06 smithi093 ceph-mon[118902]: pgmap v5266: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4063252783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:07.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:07.237972+0000) 2023-08-27T23:49:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4063252783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4063252783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:08 smithi136 ceph-mon[113531]: pgmap v5267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:08.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:08.238276+0000) 2023-08-27T23:49:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:08 smithi093 ceph-mon[111335]: pgmap v5267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:08 smithi093 ceph-mon[118902]: pgmap v5267: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1005444494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:09.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:09.238549+0000) 2023-08-27T23:49:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1005444494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1005444494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:10 smithi136 ceph-mon[113531]: pgmap v5268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:10.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:10.238813+0000) 2023-08-27T23:49:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:10 smithi093 ceph-mon[111335]: pgmap v5268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:10 smithi093 ceph-mon[118902]: pgmap v5268: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:10.857Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:10.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/613198356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:11.705 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:11.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:11.239099+0000) 2023-08-27T23:49:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/613198356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/613198356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:12 smithi136 ceph-mon[113531]: pgmap v5269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:12.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:12.239384+0000) 2023-08-27T23:49:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:12 smithi093 ceph-mon[111335]: pgmap v5269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:12 smithi093 ceph-mon[118902]: pgmap v5269: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:12.334Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:12.335Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:13.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:13.239599+0000) 2023-08-27T23:49:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:14 smithi136 ceph-mon[113531]: pgmap v5270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1969102292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:14.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:14.239788+0000) 2023-08-27T23:49:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:14 smithi093 ceph-mon[111335]: pgmap v5270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1969102292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:14 smithi093 ceph-mon[118902]: pgmap v5270: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1969102292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:15.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:15.240058+0000) 2023-08-27T23:49:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:49:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:49:15] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:49:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:16 smithi136 ceph-mon[113531]: pgmap v5271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/526987059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:16.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:16.240366+0000) 2023-08-27T23:49:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:16 smithi093 ceph-mon[111335]: pgmap v5271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/526987059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:16 smithi093 ceph-mon[118902]: pgmap v5271: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/526987059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:17.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:17.240574+0000) 2023-08-27T23:49:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:18 smithi136 ceph-mon[113531]: pgmap v5272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1989678030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:18.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:18.240812+0000) 2023-08-27T23:49:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:18 smithi093 ceph-mon[111335]: pgmap v5272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1989678030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:18 smithi093 ceph-mon[118902]: pgmap v5272: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1989678030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:19.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:19.241103+0000) 2023-08-27T23:49:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:20 smithi136 ceph-mon[113531]: pgmap v5273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:20.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:20.241368+0000) 2023-08-27T23:49:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:20 smithi093 ceph-mon[111335]: pgmap v5273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:20 smithi093 ceph-mon[118902]: pgmap v5273: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:21.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:20.857Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:21.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:20.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3176439610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:21.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:21.241614+0000) 2023-08-27T23:49:21.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3176439610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3176439610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:22 smithi136 ceph-mon[113531]: pgmap v5274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:22.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:22.241835+0000) 2023-08-27T23:49:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:22 smithi093 ceph-mon[111335]: pgmap v5274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:22 smithi093 ceph-mon[118902]: pgmap v5274: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:22.335Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:22.335Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1135246507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:23.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:23.242070+0000) 2023-08-27T23:49:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1135246507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1135246507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:24 smithi136 ceph-mon[113531]: pgmap v5275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:24.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:24.242256+0000) 2023-08-27T23:49:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:24 smithi093 ceph-mon[111335]: pgmap v5275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:24 smithi093 ceph-mon[118902]: pgmap v5275: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:25.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:25.242461+0000) 2023-08-27T23:49:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:49:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:49:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:49:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:26 smithi136 ceph-mon[113531]: pgmap v5276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4269065480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:26.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:26.242768+0000) 2023-08-27T23:49:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:26 smithi093 ceph-mon[111335]: pgmap v5276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4269065480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:26 smithi093 ceph-mon[118902]: pgmap v5276: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4269065480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:27.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:27.243025+0000) 2023-08-27T23:49:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:28 smithi136 ceph-mon[113531]: pgmap v5277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2602156517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:28.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:28.243342+0000) 2023-08-27T23:49:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:28 smithi093 ceph-mon[111335]: pgmap v5277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2602156517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:28 smithi093 ceph-mon[118902]: pgmap v5277: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2602156517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:29.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:29.243636+0000) 2023-08-27T23:49:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:30 smithi136 ceph-mon[113531]: pgmap v5278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4203168878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:30.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:30.243923+0000) 2023-08-27T23:49:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:30 smithi093 ceph-mon[111335]: pgmap v5278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4203168878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:30 smithi093 ceph-mon[118902]: pgmap v5278: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4203168878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:30.858Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:30.859Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:31.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:31.244190+0000) 2023-08-27T23:49:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:32 smithi136 ceph-mon[113531]: pgmap v5279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:32.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:32.244433+0000) 2023-08-27T23:49:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:32 smithi093 ceph-mon[111335]: pgmap v5279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:32 smithi093 ceph-mon[118902]: pgmap v5279: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:32.335Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:32.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3046730299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:33.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:33.244671+0000) 2023-08-27T23:49:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3046730299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3046730299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:34 smithi136 ceph-mon[113531]: pgmap v5280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:34.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:34.244916+0000) 2023-08-27T23:49:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:34 smithi093 ceph-mon[111335]: pgmap v5280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:34 smithi093 ceph-mon[118902]: pgmap v5280: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3349235258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:35.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:35.245238+0000) 2023-08-27T23:49:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3349235258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3349235258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:49:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:49:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:49:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:36 smithi136 ceph-mon[113531]: pgmap v5281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:36.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:36.245486+0000) 2023-08-27T23:49:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:36 smithi093 ceph-mon[118902]: pgmap v5281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:36 smithi093 ceph-mon[111335]: pgmap v5281: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3803633717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:37.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:37.245677+0000) 2023-08-27T23:49:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3803633717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3803633717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:38 smithi136 ceph-mon[113531]: pgmap v5282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:38.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:38.245932+0000) 2023-08-27T23:49:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:38 smithi093 ceph-mon[111335]: pgmap v5282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:38 smithi093 ceph-mon[118902]: pgmap v5282: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:39.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:39.246176+0000) 2023-08-27T23:49:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:40 smithi136 ceph-mon[113531]: pgmap v5283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3963800536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:40.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:40.246422+0000) 2023-08-27T23:49:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:40 smithi093 ceph-mon[111335]: pgmap v5283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3963800536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:40 smithi093 ceph-mon[118902]: pgmap v5283: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3963800536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:40.858Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:40.859Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:41.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:41.246683+0000) 2023-08-27T23:49:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:42 smithi136 ceph-mon[113531]: pgmap v5284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1005815399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:42.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:42.246935+0000) 2023-08-27T23:49:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:42 smithi093 ceph-mon[111335]: pgmap v5284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1005815399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:42 smithi093 ceph-mon[118902]: pgmap v5284: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1005815399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:42.335Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:42.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:43.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:43.247130+0000) 2023-08-27T23:49:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:44 smithi136 ceph-mon[113531]: pgmap v5285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:44.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:44.247372+0000) 2023-08-27T23:49:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:44 smithi093 ceph-mon[111335]: pgmap v5285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:44 smithi093 ceph-mon[118902]: pgmap v5285: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1541467879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:45.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:45.247612+0000) 2023-08-27T23:49:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1541467879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:49:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:49:45] "GET /metrics HTTP/1.1" 200 239918 "" "Prometheus/2.43.0" 2023-08-27T23:49:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1541467879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:46 smithi136 ceph-mon[113531]: pgmap v5286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:46.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:46.247916+0000) 2023-08-27T23:49:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:46 smithi093 ceph-mon[111335]: pgmap v5286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:46 smithi093 ceph-mon[118902]: pgmap v5286: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/605438743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:47.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:47.248166+0000) 2023-08-27T23:49:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/605438743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/605438743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:48 smithi136 ceph-mon[113531]: pgmap v5287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:48.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:48.248401+0000) 2023-08-27T23:49:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:48 smithi093 ceph-mon[111335]: pgmap v5287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:48 smithi093 ceph-mon[118902]: pgmap v5287: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/395011307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:49.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:49.248617+0000) 2023-08-27T23:49:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/395011307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/395011307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:50 smithi136 ceph-mon[113531]: pgmap v5288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:50.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:50.248858+0000) 2023-08-27T23:49:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:50 smithi093 ceph-mon[111335]: pgmap v5288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:50 smithi093 ceph-mon[118902]: pgmap v5288: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:50.859Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:50.860Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:51.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:51.249068+0000) 2023-08-27T23:49:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:52 smithi136 ceph-mon[113531]: pgmap v5289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1753158038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:52.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:52.249288+0000) 2023-08-27T23:49:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:52 smithi093 ceph-mon[111335]: pgmap v5289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1753158038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:52 smithi093 ceph-mon[118902]: pgmap v5289: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1753158038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:52.337Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:49:52.337Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:49:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:53.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:53.249537+0000) 2023-08-27T23:49:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:54 smithi136 ceph-mon[113531]: pgmap v5290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2587139890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:49:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:54.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:54.249711+0000) 2023-08-27T23:49:54.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[111335]: pgmap v5290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:54.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:54.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2587139890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:54.696 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:49:54.696 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[118902]: pgmap v5290: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:54.697 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:49:54.697 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2587139890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:54.697 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:49:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:55.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:55.249981+0000) 2023-08-27T23:49:55.647 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:49:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:49:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:49:56.432 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:56.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:56.250248+0000) 2023-08-27T23:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:56 smithi136 ceph-mon[113531]: pgmap v5291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4231636619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[118902]: pgmap v5291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4231636619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[111335]: pgmap v5291: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:56.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:56.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4231636619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:57.659 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:57.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:57.250424+0000) 2023-08-27T23:49:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:49:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:49:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:49:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:49:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:57.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:49:57.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:49:57.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:49:58.665 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:58.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:58.250568+0000) 2023-08-27T23:49:58.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:58 smithi136 ceph-mon[113531]: pgmap v5292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:58 smithi093 ceph-mon[111335]: pgmap v5292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:58 smithi093 ceph-mon[118902]: pgmap v5292: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:49:59.672 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:49:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:49:59.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:49:59.250786+0000) 2023-08-27T23:49:59.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:49:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3562622200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:49:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3562622200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:49:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:49:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3562622200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:00.674 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:00.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:00.250992+0000) 2023-08-27T23:50:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:00 smithi136 ceph-mon[113531]: pgmap v5293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:50:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:00 smithi093 ceph-mon[111335]: pgmap v5293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:50:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:00 smithi093 ceph-mon[118902]: pgmap v5293: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-27T23:50:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:00.860Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:00.861Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:01.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:01.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:01.251250+0000) 2023-08-27T23:50:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3534715593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3534715593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3534715593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:02.684 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:02.337Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:02.684 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:02.338Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:02.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:02.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:02.251440+0000) 2023-08-27T23:50:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:02 smithi136 ceph-mon[113531]: pgmap v5294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:02 smithi093 ceph-mon[111335]: pgmap v5294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:02 smithi093 ceph-mon[118902]: pgmap v5294: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:03.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:03.251711+0000) 2023-08-27T23:50:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3523303281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3523303281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3523303281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:04.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:04.251941+0000) 2023-08-27T23:50:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:04 smithi093 ceph-mon[111335]: pgmap v5295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:04 smithi093 ceph-mon[118902]: pgmap v5295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:04 smithi136 ceph-mon[113531]: pgmap v5295: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:05.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:05.252221+0000) 2023-08-27T23:50:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:50:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:50:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:50:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:06.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:06.252486+0000) 2023-08-27T23:50:06.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:06 smithi136 ceph-mon[113531]: pgmap v5296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:06.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1454260865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:06 smithi093 ceph-mon[111335]: pgmap v5296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1454260865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:06 smithi093 ceph-mon[118902]: pgmap v5296: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1454260865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:07.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:07.252706+0000) 2023-08-27T23:50:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:08.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:08.252967+0000) 2023-08-27T23:50:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:08 smithi093 ceph-mon[111335]: pgmap v5297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/305440136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:08 smithi093 ceph-mon[118902]: pgmap v5297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/305440136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:08 smithi136 ceph-mon[113531]: pgmap v5297: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/305440136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:09.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:09.253215+0000) 2023-08-27T23:50:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:10.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:10.253404+0000) 2023-08-27T23:50:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:10 smithi093 ceph-mon[111335]: pgmap v5298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:10 smithi093 ceph-mon[118902]: pgmap v5298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:10.861Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:10.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:10.862Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:10 smithi136 ceph-mon[113531]: pgmap v5298: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:11.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:11.253631+0000) 2023-08-27T23:50:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/489655647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/489655647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/489655647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:12.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:12.253876+0000) 2023-08-27T23:50:12.708 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:12.338Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:12.708 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:12.339Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:12 smithi093 ceph-mon[111335]: pgmap v5299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:12 smithi093 ceph-mon[118902]: pgmap v5299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:12 smithi136 ceph-mon[113531]: pgmap v5299: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:13.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:13.254142+0000) 2023-08-27T23:50:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2833569832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2833569832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:14.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2833569832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:14.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:14.254395+0000) 2023-08-27T23:50:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:14 smithi093 ceph-mon[111335]: pgmap v5300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:14 smithi093 ceph-mon[118902]: pgmap v5300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:14 smithi136 ceph-mon[113531]: pgmap v5300: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:15.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:15.254597+0000) 2023-08-27T23:50:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:50:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:50:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:50:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:16 smithi136 ceph-mon[113531]: pgmap v5301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2886093869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:16.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:16.254882+0000) 2023-08-27T23:50:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:16 smithi093 ceph-mon[111335]: pgmap v5301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2886093869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:16 smithi093 ceph-mon[118902]: pgmap v5301: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2886093869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:17.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:17.255128+0000) 2023-08-27T23:50:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:18 smithi136 ceph-mon[113531]: pgmap v5302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1976550723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:18.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:18.255336+0000) 2023-08-27T23:50:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:18 smithi093 ceph-mon[111335]: pgmap v5302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1976550723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:18 smithi093 ceph-mon[118902]: pgmap v5302: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:18.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1976550723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:19.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:19.255548+0000) 2023-08-27T23:50:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:20 smithi136 ceph-mon[113531]: pgmap v5303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1947020961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:20.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:20.255767+0000) 2023-08-27T23:50:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:20 smithi093 ceph-mon[111335]: pgmap v5303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1947020961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:20 smithi093 ceph-mon[118902]: pgmap v5303: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1947020961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:20.861Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:20.862Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:21.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:21.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:21.255973+0000) 2023-08-27T23:50:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:22 smithi136 ceph-mon[113531]: pgmap v5304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:22.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:22.256187+0000) 2023-08-27T23:50:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:22 smithi093 ceph-mon[111335]: pgmap v5304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:22 smithi093 ceph-mon[118902]: pgmap v5304: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:22.339Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:22.340Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2766124694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:23.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:23.256405+0000) 2023-08-27T23:50:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2766124694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2766124694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:24 smithi136 ceph-mon[113531]: pgmap v5305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:24.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:24.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:24.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:24.256602+0000) 2023-08-27T23:50:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:24 smithi093 ceph-mon[111335]: pgmap v5305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:24 smithi093 ceph-mon[118902]: pgmap v5305: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/547853860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:25.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:25.256857+0000) 2023-08-27T23:50:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/547853860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:50:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:50:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:50:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/547853860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:26 smithi136 ceph-mon[113531]: pgmap v5306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:26.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:26.257141+0000) 2023-08-27T23:50:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:26 smithi093 ceph-mon[111335]: pgmap v5306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:26 smithi093 ceph-mon[118902]: pgmap v5306: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:27.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:27.257400+0000) 2023-08-27T23:50:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:28 smithi136 ceph-mon[113531]: pgmap v5307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3706933222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:28.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:28.257641+0000) 2023-08-27T23:50:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:28 smithi093 ceph-mon[118902]: pgmap v5307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3706933222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:28 smithi093 ceph-mon[111335]: pgmap v5307: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3706933222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:29.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:29.257883+0000) 2023-08-27T23:50:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:30 smithi136 ceph-mon[113531]: pgmap v5308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1301073900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:30.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:30.258112+0000) 2023-08-27T23:50:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:30 smithi093 ceph-mon[111335]: pgmap v5308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1301073900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:30 smithi093 ceph-mon[118902]: pgmap v5308: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1301073900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:30.861Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:30.862Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:31.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:31.258385+0000) 2023-08-27T23:50:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:32 smithi136 ceph-mon[113531]: pgmap v5309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2594456352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:32.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:32.258661+0000) 2023-08-27T23:50:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:32 smithi093 ceph-mon[118902]: pgmap v5309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2594456352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:32 smithi093 ceph-mon[111335]: pgmap v5309: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2594456352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:32.340Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:32.340Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:33.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:33.258915+0000) 2023-08-27T23:50:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:34 smithi136 ceph-mon[113531]: pgmap v5310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:34.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:34.259244+0000) 2023-08-27T23:50:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:34 smithi093 ceph-mon[111335]: pgmap v5310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:34 smithi093 ceph-mon[118902]: pgmap v5310: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/109910391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:35.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:35.259521+0000) 2023-08-27T23:50:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/109910391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:50:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:50:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/109910391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:36 smithi136 ceph-mon[113531]: pgmap v5311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:36.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:36.259789+0000) 2023-08-27T23:50:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:36 smithi093 ceph-mon[111335]: pgmap v5311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:36 smithi093 ceph-mon[118902]: pgmap v5311: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3004422386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:37.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:37.260051+0000) 2023-08-27T23:50:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3004422386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3004422386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:38 smithi136 ceph-mon[113531]: pgmap v5312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:38.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:38.260282+0000) 2023-08-27T23:50:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:38 smithi093 ceph-mon[111335]: pgmap v5312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:38 smithi093 ceph-mon[118902]: pgmap v5312: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2597754520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:39.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:39.260541+0000) 2023-08-27T23:50:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2597754520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2597754520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:40.531 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:40 smithi136 ceph-mon[113531]: pgmap v5313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:40.532 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:40.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:40.260792+0000) 2023-08-27T23:50:40.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:40 smithi093 ceph-mon[111335]: pgmap v5313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:40 smithi093 ceph-mon[118902]: pgmap v5313: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:40.939 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 23:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T23:50:40.531Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1693169440357 maxt=1693173600000 ulid=01H8WPG8K971SAMSAKB255GZ4X duration=41.101265ms 2023-08-27T23:50:40.939 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 27 23:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-27T23:50:40.535Z caller=head.go:1269 level=info component=tsdb msg="Head GC completed" caller=truncateMemory duration=3.667278ms 2023-08-27T23:50:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:40.862Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:40.862Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:41.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:41.261063+0000) 2023-08-27T23:50:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:42 smithi136 ceph-mon[113531]: pgmap v5314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4223276143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:42.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:42.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:42.261303+0000) 2023-08-27T23:50:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:42 smithi093 ceph-mon[111335]: pgmap v5314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4223276143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:42 smithi093 ceph-mon[118902]: pgmap v5314: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4223276143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:42.340Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:42.341Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:43.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:43.261544+0000) 2023-08-27T23:50:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:44 smithi136 ceph-mon[113531]: pgmap v5315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1005278465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:44.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:44.261790+0000) 2023-08-27T23:50:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:44 smithi093 ceph-mon[111335]: pgmap v5315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1005278465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:44 smithi093 ceph-mon[118902]: pgmap v5315: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1005278465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:45.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:45.262061+0000) 2023-08-27T23:50:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:50:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:50:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:50:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:46 smithi136 ceph-mon[113531]: pgmap v5316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:46.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:46.262281+0000) 2023-08-27T23:50:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:46 smithi093 ceph-mon[111335]: pgmap v5316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:46 smithi093 ceph-mon[118902]: pgmap v5316: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2802123504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:47.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:47.262533+0000) 2023-08-27T23:50:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2802123504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2802123504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:48 smithi136 ceph-mon[113531]: pgmap v5317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:48.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:48.262791+0000) 2023-08-27T23:50:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:48 smithi093 ceph-mon[111335]: pgmap v5317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:48 smithi093 ceph-mon[118902]: pgmap v5317: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/569947673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:49.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:49.263025+0000) 2023-08-27T23:50:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/569947673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/569947673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:50 smithi136 ceph-mon[113531]: pgmap v5318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:50.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:50.263279+0000) 2023-08-27T23:50:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:50 smithi093 ceph-mon[111335]: pgmap v5318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:50 smithi093 ceph-mon[118902]: pgmap v5318: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:50.863Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:50.864Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1884295366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:51.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:51.263535+0000) 2023-08-27T23:50:51.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1884295366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1884295366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:52 smithi136 ceph-mon[113531]: pgmap v5319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:52.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:52.263788+0000) 2023-08-27T23:50:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:52 smithi093 ceph-mon[111335]: pgmap v5319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:52 smithi093 ceph-mon[118902]: pgmap v5319: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:52.341Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:50:52.342Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:50:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:53.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:53.264078+0000) 2023-08-27T23:50:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:54.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:54.264327+0000) 2023-08-27T23:50:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:54 smithi136 ceph-mon[113531]: pgmap v5320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3472056763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:54 smithi093 ceph-mon[111335]: pgmap v5320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3472056763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:54 smithi093 ceph-mon[118902]: pgmap v5320: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3472056763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:50:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:55.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:55.264599+0000) 2023-08-27T23:50:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:50:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:50:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:50:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:56 smithi136 ceph-mon[113531]: pgmap v5321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2908679853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:56.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:56.264876+0000) 2023-08-27T23:50:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:56 smithi093 ceph-mon[111335]: pgmap v5321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2908679853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:56 smithi093 ceph-mon[118902]: pgmap v5321: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2908679853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:57.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:50:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:57.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:57.265049+0000) 2023-08-27T23:50:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:50:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:50:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:58 smithi136 ceph-mon[113531]: pgmap v5322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:58.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:58.265276+0000) 2023-08-27T23:50:58.704 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:58 smithi093 ceph-mon[111335]: pgmap v5322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:58.704 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:58 smithi093 ceph-mon[118902]: pgmap v5322: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:50:59.499 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/240360381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:59.500 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:50:59.500 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:50:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:50:59.500 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:50:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:50:59.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:50:59.265523+0000) 2023-08-27T23:50:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/240360381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:50:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:50:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:50:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/240360381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:50:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:50:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:50:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:00.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:00.265720+0000) 2023-08-27T23:51:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:00 smithi136 ceph-mon[113531]: pgmap v5323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:51:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:51:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:00.863Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:00.865Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[111335]: pgmap v5323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[118902]: pgmap v5323: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:51:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:51:01.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:51:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:01.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:01.265970+0000) 2023-08-27T23:51:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3230721139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3230721139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3230721139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:02.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:02.266191+0000) 2023-08-27T23:51:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:02.342Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:02.343Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:02 smithi136 ceph-mon[113531]: pgmap v5324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:02 smithi093 ceph-mon[111335]: pgmap v5324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:02 smithi093 ceph-mon[118902]: pgmap v5324: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:03.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:03.266442+0000) 2023-08-27T23:51:04.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1237897978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1237897978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1237897978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:04.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:04.266703+0000) 2023-08-27T23:51:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:04 smithi136 ceph-mon[113531]: pgmap v5325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:04 smithi093 ceph-mon[111335]: pgmap v5325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:04 smithi093 ceph-mon[118902]: pgmap v5325: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:05.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:05.266941+0000) 2023-08-27T23:51:05.690 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 27 23:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-27T23:51:05.318373017Z level=info msg="Completed cleanup jobs" duration=13.020956ms 2023-08-27T23:51:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:51:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:51:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:51:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:06.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:06.267174+0000) 2023-08-27T23:51:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:06 smithi136 ceph-mon[113531]: pgmap v5326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3427494741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:06 smithi093 ceph-mon[111335]: pgmap v5326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3427494741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:06 smithi093 ceph-mon[118902]: pgmap v5326: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3427494741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:07.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:07.267396+0000) 2023-08-27T23:51:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:08 smithi136 ceph-mon[113531]: pgmap v5327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/643445259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:08.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:08.267658+0000) 2023-08-27T23:51:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:08 smithi093 ceph-mon[111335]: pgmap v5327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/643445259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:08 smithi093 ceph-mon[118902]: pgmap v5327: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:08.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/643445259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:09.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:09.267877+0000) 2023-08-27T23:51:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:10 smithi136 ceph-mon[113531]: pgmap v5328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:10.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:10.268101+0000) 2023-08-27T23:51:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:10 smithi093 ceph-mon[111335]: pgmap v5328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:10 smithi093 ceph-mon[118902]: pgmap v5328: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:10.865Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:10.866Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/168198556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:11.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:11.268301+0000) 2023-08-27T23:51:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/168198556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:11.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/168198556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:12 smithi136 ceph-mon[113531]: pgmap v5329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:12.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:12.268498+0000) 2023-08-27T23:51:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:12 smithi093 ceph-mon[111335]: pgmap v5329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:12 smithi093 ceph-mon[118902]: pgmap v5329: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:12.343Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:12.344Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3636585060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:13.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:13.268759+0000) 2023-08-27T23:51:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3636585060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3636585060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:14 smithi136 ceph-mon[113531]: pgmap v5330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:14.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:14.269019+0000) 2023-08-27T23:51:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:14 smithi093 ceph-mon[111335]: pgmap v5330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:14 smithi093 ceph-mon[118902]: pgmap v5330: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2913965226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:15.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:15.269323+0000) 2023-08-27T23:51:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2913965226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2913965226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:51:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:51:15] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:51:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:16 smithi136 ceph-mon[113531]: pgmap v5331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:16.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:16.269609+0000) 2023-08-27T23:51:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:16 smithi093 ceph-mon[118902]: pgmap v5331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:16 smithi093 ceph-mon[111335]: pgmap v5331: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:17.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:17.269882+0000) 2023-08-27T23:51:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:18 smithi136 ceph-mon[113531]: pgmap v5332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/38316552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:18.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:18.270104+0000) 2023-08-27T23:51:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:18 smithi093 ceph-mon[111335]: pgmap v5332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/38316552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:18 smithi093 ceph-mon[118902]: pgmap v5332: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/38316552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:19.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:19.270348+0000) 2023-08-27T23:51:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:20 smithi136 ceph-mon[113531]: pgmap v5333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1251023320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:20.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:20.270649+0000) 2023-08-27T23:51:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:20 smithi093 ceph-mon[111335]: pgmap v5333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1251023320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:20 smithi093 ceph-mon[118902]: pgmap v5333: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1251023320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:20.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:20.866Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:21.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:21.270939+0000) 2023-08-27T23:51:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:22 smithi136 ceph-mon[113531]: pgmap v5334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2075098309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:22.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:22.271185+0000) 2023-08-27T23:51:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:22 smithi093 ceph-mon[111335]: pgmap v5334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2075098309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:22 smithi093 ceph-mon[118902]: pgmap v5334: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2075098309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:22.344Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:22.345Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:23.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:23.271399+0000) 2023-08-27T23:51:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:24 smithi136 ceph-mon[113531]: pgmap v5335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:24.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:24.271626+0000) 2023-08-27T23:51:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:24 smithi093 ceph-mon[111335]: pgmap v5335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:24 smithi093 ceph-mon[118902]: pgmap v5335: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1945537852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:25.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:25.271870+0000) 2023-08-27T23:51:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1945537852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:51:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:51:25] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:51:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1945537852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:26 smithi136 ceph-mon[113531]: pgmap v5336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:26.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:26.272152+0000) 2023-08-27T23:51:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:26 smithi093 ceph-mon[111335]: pgmap v5336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:26 smithi093 ceph-mon[118902]: pgmap v5336: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/841680642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:27.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:27.272394+0000) 2023-08-27T23:51:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/841680642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/841680642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:28 smithi136 ceph-mon[113531]: pgmap v5337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:28.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:28.272637+0000) 2023-08-27T23:51:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:28 smithi093 ceph-mon[111335]: pgmap v5337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:28 smithi093 ceph-mon[118902]: pgmap v5337: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:29.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:29.272891+0000) 2023-08-27T23:51:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:30 smithi136 ceph-mon[113531]: pgmap v5338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1129885097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:30.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:30.273157+0000) 2023-08-27T23:51:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:30 smithi093 ceph-mon[111335]: pgmap v5338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1129885097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:30 smithi093 ceph-mon[118902]: pgmap v5338: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1129885097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:30.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:30.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:31.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:31.273409+0000) 2023-08-27T23:51:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:32 smithi136 ceph-mon[113531]: pgmap v5339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3055465005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:32.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:32.273639+0000) 2023-08-27T23:51:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:32 smithi093 ceph-mon[111335]: pgmap v5339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3055465005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:32 smithi093 ceph-mon[118902]: pgmap v5339: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3055465005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:32.344Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:32.345Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:33.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:33.273891+0000) 2023-08-27T23:51:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:34 smithi136 ceph-mon[113531]: pgmap v5340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/948661417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:34.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:34.274175+0000) 2023-08-27T23:51:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:34 smithi093 ceph-mon[111335]: pgmap v5340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/948661417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:34 smithi093 ceph-mon[118902]: pgmap v5340: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/948661417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:35.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:35.274439+0000) 2023-08-27T23:51:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:51:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:51:35] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:51:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:36 smithi136 ceph-mon[113531]: pgmap v5341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:36.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:36.274683+0000) 2023-08-27T23:51:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:36 smithi093 ceph-mon[111335]: pgmap v5341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:36 smithi093 ceph-mon[118902]: pgmap v5341: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/32229223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:37.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:37.274915+0000) 2023-08-27T23:51:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/32229223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/32229223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:38 smithi136 ceph-mon[113531]: pgmap v5342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:38.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:38.275101+0000) 2023-08-27T23:51:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:38 smithi093 ceph-mon[111335]: pgmap v5342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:38 smithi093 ceph-mon[118902]: pgmap v5342: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/242958569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:39.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:39.275291+0000) 2023-08-27T23:51:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/242958569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/242958569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:40 smithi136 ceph-mon[113531]: pgmap v5343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:40.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:40.275534+0000) 2023-08-27T23:51:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:40 smithi093 ceph-mon[111335]: pgmap v5343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:40 smithi093 ceph-mon[118902]: pgmap v5343: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:40.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:40.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:41.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1646949725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:41.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:41.275785+0000) 2023-08-27T23:51:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1646949725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1646949725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:42 smithi136 ceph-mon[113531]: pgmap v5344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:42.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:42.276091+0000) 2023-08-27T23:51:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:42 smithi093 ceph-mon[111335]: pgmap v5344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:42 smithi093 ceph-mon[118902]: pgmap v5344: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:42.344Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:42.345Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:43.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:43.276318+0000) 2023-08-27T23:51:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:44 smithi136 ceph-mon[113531]: pgmap v5345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1522737098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:44.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:44.276581+0000) 2023-08-27T23:51:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:44 smithi093 ceph-mon[111335]: pgmap v5345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1522737098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:44 smithi093 ceph-mon[118902]: pgmap v5345: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1522737098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:45.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:45.276842+0000) 2023-08-27T23:51:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:51:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:51:45] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:51:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:46 smithi136 ceph-mon[113531]: pgmap v5346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1346086311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:46.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:46.277140+0000) 2023-08-27T23:51:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:46 smithi093 ceph-mon[111335]: pgmap v5346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1346086311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:46 smithi093 ceph-mon[118902]: pgmap v5346: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1346086311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:47.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:47.277412+0000) 2023-08-27T23:51:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:48 smithi136 ceph-mon[113531]: pgmap v5347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:48.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:48.277633+0000) 2023-08-27T23:51:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:48 smithi093 ceph-mon[111335]: pgmap v5347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:48 smithi093 ceph-mon[118902]: pgmap v5347: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:49.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3141530111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:49.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:49.277877+0000) 2023-08-27T23:51:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3141530111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3141530111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:50 smithi136 ceph-mon[113531]: pgmap v5348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:50.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:50.278153+0000) 2023-08-27T23:51:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:50 smithi093 ceph-mon[111335]: pgmap v5348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:50 smithi093 ceph-mon[118902]: pgmap v5348: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:50.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:50.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4186959022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:51.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:51.278450+0000) 2023-08-27T23:51:51.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4186959022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4186959022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:52 smithi136 ceph-mon[113531]: pgmap v5349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:52.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:52.278685+0000) 2023-08-27T23:51:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:52 smithi093 ceph-mon[111335]: pgmap v5349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:52 smithi093 ceph-mon[118902]: pgmap v5349: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:52.346Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:51:52.346Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:51:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/301278703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:53.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:53.278876+0000) 2023-08-27T23:51:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/301278703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/301278703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:54 smithi136 ceph-mon[113531]: pgmap v5350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:54.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:54.279098+0000) 2023-08-27T23:51:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:54 smithi093 ceph-mon[118902]: pgmap v5350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:54 smithi093 ceph-mon[111335]: pgmap v5350: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:51:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:55.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:55.279322+0000) 2023-08-27T23:51:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:51:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:51:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:51:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:56 smithi136 ceph-mon[113531]: pgmap v5351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1937656385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:56.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:56.279604+0000) 2023-08-27T23:51:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:56 smithi093 ceph-mon[111335]: pgmap v5351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1937656385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:56 smithi093 ceph-mon[118902]: pgmap v5351: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1937656385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:57.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:57.279873+0000) 2023-08-27T23:51:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:58 smithi136 ceph-mon[113531]: pgmap v5352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:51:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2483447259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:58.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:58.280112+0000) 2023-08-27T23:51:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:58 smithi093 ceph-mon[111335]: pgmap v5352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:51:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2483447259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:58 smithi093 ceph-mon[118902]: pgmap v5352: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:51:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:51:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2483447259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:51:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:51:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:51:59.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:51:59.280364+0000) 2023-08-27T23:52:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:00 smithi136 ceph-mon[113531]: pgmap v5353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:52:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1606094133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:00.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:00.280574+0000) 2023-08-27T23:52:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-mon[111335]: pgmap v5353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:52:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1606094133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-mon[118902]: pgmap v5353: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:52:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1606094133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:00.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:00.868Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:01.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:01.280746+0000) 2023-08-27T23:52:02.586 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:02 smithi136 ceph-mon[113531]: pgmap v5354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:02.586 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:02.587 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:02.587 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:02.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:02.280906+0000) 2023-08-27T23:52:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-mon[118902]: pgmap v5354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-mon[111335]: pgmap v5354: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:02.346Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:02.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:02.347Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/587300134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:03.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/587300134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:03.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:03.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/587300134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:03.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:03.281142+0000) 2023-08-27T23:52:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:04.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:04.281344+0000) 2023-08-27T23:52:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:04 smithi136 ceph-mon[113531]: pgmap v5355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:52:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:52:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[111335]: pgmap v5355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[118902]: pgmap v5355: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:52:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1258951097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:05.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:05.281523+0000) 2023-08-27T23:52:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1258951097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:52:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:52:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:52:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1258951097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:06 smithi136 ceph-mon[113531]: pgmap v5356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:06.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:06.281775+0000) 2023-08-27T23:52:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:06 smithi093 ceph-mon[111335]: pgmap v5356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:06 smithi093 ceph-mon[118902]: pgmap v5356: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:07.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:07.282013+0000) 2023-08-27T23:52:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:08 smithi136 ceph-mon[113531]: pgmap v5357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/407871607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:08.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:08.282236+0000) 2023-08-27T23:52:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:08 smithi093 ceph-mon[111335]: pgmap v5357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/407871607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:08 smithi093 ceph-mon[118902]: pgmap v5357: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/407871607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:09.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:09.282511+0000) 2023-08-27T23:52:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:10 smithi136 ceph-mon[113531]: pgmap v5358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3217752711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:10.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:10.282762+0000) 2023-08-27T23:52:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:10 smithi093 ceph-mon[111335]: pgmap v5358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3217752711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:10 smithi093 ceph-mon[118902]: pgmap v5358: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3217752711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:10.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:10.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:11.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:11.282994+0000) 2023-08-27T23:52:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:12 smithi136 ceph-mon[113531]: pgmap v5359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1706384895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:12.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:12.283200+0000) 2023-08-27T23:52:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:12 smithi093 ceph-mon[118902]: pgmap v5359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1706384895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:12 smithi093 ceph-mon[111335]: pgmap v5359: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1706384895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:12.347Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:12.348Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:13.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:13.283452+0000) 2023-08-27T23:52:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:14 smithi136 ceph-mon[113531]: pgmap v5360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:14.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:14.283751+0000) 2023-08-27T23:52:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:14 smithi093 ceph-mon[111335]: pgmap v5360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:14 smithi093 ceph-mon[118902]: pgmap v5360: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:15.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3365800193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:15.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:15.283974+0000) 2023-08-27T23:52:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3365800193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3365800193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:52:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:52:15] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:52:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:16 smithi136 ceph-mon[113531]: pgmap v5361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:16.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:16.284231+0000) 2023-08-27T23:52:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:16 smithi093 ceph-mon[111335]: pgmap v5361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:16 smithi093 ceph-mon[118902]: pgmap v5361: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3396494098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:17.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:17.284461+0000) 2023-08-27T23:52:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3396494098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3396494098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:18 smithi136 ceph-mon[113531]: pgmap v5362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:18.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:18.284706+0000) 2023-08-27T23:52:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:18 smithi093 ceph-mon[111335]: pgmap v5362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:18.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:18 smithi093 ceph-mon[118902]: pgmap v5362: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:19.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:19.284975+0000) 2023-08-27T23:52:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:20 smithi136 ceph-mon[113531]: pgmap v5363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2936013661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:20.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:20.285214+0000) 2023-08-27T23:52:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:20 smithi093 ceph-mon[111335]: pgmap v5363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2936013661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:20 smithi093 ceph-mon[118902]: pgmap v5363: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2936013661' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:20.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:20.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:21.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:21.285450+0000) 2023-08-27T23:52:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:22 smithi136 ceph-mon[113531]: pgmap v5364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1725155660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:22.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:22.285601+0000) 2023-08-27T23:52:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:22 smithi093 ceph-mon[111335]: pgmap v5364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1725155660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:22 smithi093 ceph-mon[118902]: pgmap v5364: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1725155660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:22.348Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:22.349Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:23.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:23.285883+0000) 2023-08-27T23:52:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:24 smithi136 ceph-mon[113531]: pgmap v5365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3893516301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:24.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:24.286156+0000) 2023-08-27T23:52:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:24 smithi093 ceph-mon[111335]: pgmap v5365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3893516301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:24 smithi093 ceph-mon[118902]: pgmap v5365: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3893516301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:25.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:25.286540+0000) 2023-08-27T23:52:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:52:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:52:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:52:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:26 smithi136 ceph-mon[113531]: pgmap v5366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:26.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:26.286834+0000) 2023-08-27T23:52:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:26 smithi093 ceph-mon[111335]: pgmap v5366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:26 smithi093 ceph-mon[118902]: pgmap v5366: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2848584492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:27.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:27.287098+0000) 2023-08-27T23:52:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2848584492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2848584492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:28 smithi136 ceph-mon[113531]: pgmap v5367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:28.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:28.287330+0000) 2023-08-27T23:52:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:28 smithi093 ceph-mon[111335]: pgmap v5367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:28 smithi093 ceph-mon[118902]: pgmap v5367: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/151728164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:29.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:29.287598+0000) 2023-08-27T23:52:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/151728164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/151728164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:30 smithi136 ceph-mon[113531]: pgmap v5368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:30.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:30.287852+0000) 2023-08-27T23:52:30.869 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:30 smithi093 ceph-mon[111335]: pgmap v5368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:30.869 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:30 smithi093 ceph-mon[118902]: pgmap v5368: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:30.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:30.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/826737913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:31.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:31.288117+0000) 2023-08-27T23:52:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/826737913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/826737913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:32 smithi136 ceph-mon[113531]: pgmap v5369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:32.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:32.288350+0000) 2023-08-27T23:52:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:32 smithi093 ceph-mon[111335]: pgmap v5369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:32 smithi093 ceph-mon[118902]: pgmap v5369: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:32.348Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:32.349Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:33.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:33.288615+0000) 2023-08-27T23:52:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:34 smithi136 ceph-mon[113531]: pgmap v5370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3836879076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:34.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:34.288870+0000) 2023-08-27T23:52:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:34 smithi093 ceph-mon[118902]: pgmap v5370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3836879076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:34 smithi093 ceph-mon[111335]: pgmap v5370: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3836879076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:35.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:35.289118+0000) 2023-08-27T23:52:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:52:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:52:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:52:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:36 smithi136 ceph-mon[113531]: pgmap v5371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3099108801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:36.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:36.289334+0000) 2023-08-27T23:52:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:36 smithi093 ceph-mon[111335]: pgmap v5371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3099108801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:36 smithi093 ceph-mon[118902]: pgmap v5371: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3099108801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:37.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:37.289564+0000) 2023-08-27T23:52:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:38 smithi136 ceph-mon[113531]: pgmap v5372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:38.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:38.289799+0000) 2023-08-27T23:52:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:38 smithi093 ceph-mon[111335]: pgmap v5372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:38 smithi093 ceph-mon[118902]: pgmap v5372: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:39.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:39.290089+0000) 2023-08-27T23:52:39.816 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/540667823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:39.816 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/540667823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/540667823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:40 smithi136 ceph-mon[113531]: pgmap v5373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:40.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:40.290340+0000) 2023-08-27T23:52:40.870 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:40 smithi093 ceph-mon[111335]: pgmap v5373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:40.871 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:40 smithi093 ceph-mon[118902]: pgmap v5373: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:40.869Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:40.870Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/30953655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:41.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:41.290587+0000) 2023-08-27T23:52:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/30953655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/30953655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:42 smithi136 ceph-mon[113531]: pgmap v5374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:42.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:42.290809+0000) 2023-08-27T23:52:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:42 smithi093 ceph-mon[111335]: pgmap v5374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:42 smithi093 ceph-mon[118902]: pgmap v5374: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:42.349Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:42.350Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4091481738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:43.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:43.291094+0000) 2023-08-27T23:52:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4091481738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4091481738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:44 smithi136 ceph-mon[113531]: pgmap v5375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:44.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:44.291344+0000) 2023-08-27T23:52:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:44 smithi093 ceph-mon[111335]: pgmap v5375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:44 smithi093 ceph-mon[118902]: pgmap v5375: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:45.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:45.291577+0000) 2023-08-27T23:52:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:52:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:52:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:52:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:46.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:46.291817+0000) 2023-08-27T23:52:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:46 smithi136 ceph-mon[113531]: pgmap v5376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3973560361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:46 smithi093 ceph-mon[118902]: pgmap v5376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3973560361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:46 smithi093 ceph-mon[111335]: pgmap v5376: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:46.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3973560361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:47.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:47.292075+0000) 2023-08-27T23:52:48.543 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:48 smithi136 ceph-mon[113531]: pgmap v5377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:48.543 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:48.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:48.292300+0000) 2023-08-27T23:52:48.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2749065645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:48 smithi093 ceph-mon[111335]: pgmap v5377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2749065645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:48 smithi093 ceph-mon[118902]: pgmap v5377: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2749065645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:49.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:49.292552+0000) 2023-08-27T23:52:50.548 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:50.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:50.292807+0000) 2023-08-27T23:52:50.870 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:50 smithi093 ceph-mon[111335]: pgmap v5378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:50.870 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3586627502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:50.870 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:50 smithi093 ceph-mon[118902]: pgmap v5378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:50.871 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3586627502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:50.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:50 smithi136 ceph-mon[113531]: pgmap v5378: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3586627502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:50.869Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:50.870Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:51.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:51.293115+0000) 2023-08-27T23:52:52.550 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:52.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:52.293302+0000) 2023-08-27T23:52:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:52 smithi093 ceph-mon[111335]: pgmap v5379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:52 smithi093 ceph-mon[118902]: pgmap v5379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:52.350Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:52:52.350Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:52:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:52 smithi136 ceph-mon[113531]: pgmap v5379: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:53.557 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:53.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:53.293572+0000) 2023-08-27T23:52:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3945313678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3945313678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3945313678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:54.557 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:54.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:54.293820+0000) 2023-08-27T23:52:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:54 smithi136 ceph-mon[113531]: pgmap v5380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:54 smithi093 ceph-mon[118902]: pgmap v5380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:54 smithi093 ceph-mon[111335]: pgmap v5380: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:52:55.563 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:55.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:55.294124+0000) 2023-08-27T23:52:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2263855126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:52:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:52:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:52:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2263855126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2263855126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:56.564 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:56.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:56.294409+0000) 2023-08-27T23:52:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:56 smithi136 ceph-mon[113531]: pgmap v5381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:56 smithi093 ceph-mon[111335]: pgmap v5381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:56 smithi093 ceph-mon[118902]: pgmap v5381: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:57.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:57.294622+0000) 2023-08-27T23:52:58.567 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:58.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:58.294837+0000) 2023-08-27T23:52:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:58 smithi136 ceph-mon[113531]: pgmap v5382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:52:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3159190029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:58 smithi093 ceph-mon[111335]: pgmap v5382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:52:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3159190029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:58 smithi093 ceph-mon[118902]: pgmap v5382: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:52:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:52:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3159190029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:52:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:52:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:52:59.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:52:59.295115+0000) 2023-08-27T23:53:00.570 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:00.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:00.295363+0000) 2023-08-27T23:53:00.872 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:00 smithi093 ceph-mon[111335]: pgmap v5383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:00.872 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/123135985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:00.872 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:00 smithi093 ceph-mon[118902]: pgmap v5383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:00.872 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/123135985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:00 smithi136 ceph-mon[113531]: pgmap v5383: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/123135985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:00.870Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:00.872Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:01.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:01.295622+0000) 2023-08-27T23:53:02.572 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:02.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:02.295826+0000) 2023-08-27T23:53:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:02 smithi093 ceph-mon[111335]: pgmap v5384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3909392111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:02 smithi093 ceph-mon[118902]: pgmap v5384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3909392111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:02.350Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:02.351Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:02 smithi136 ceph-mon[113531]: pgmap v5384: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3909392111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:03.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:03.296123+0000) 2023-08-27T23:53:04.576 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:04.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:04.296399+0000) 2023-08-27T23:53:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:04 smithi136 ceph-mon[113531]: pgmap v5385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:53:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:04 smithi093 ceph-mon[111335]: pgmap v5385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:53:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:04 smithi093 ceph-mon[118902]: pgmap v5385: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:53:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:53:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:53:05.574 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:05.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:05.296572+0000) 2023-08-27T23:53:05.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1037387863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:05.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1037387863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:05.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1037387863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:06.582 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:06.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:06.296785+0000) 2023-08-27T23:53:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:06 smithi136 ceph-mon[113531]: pgmap v5386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:06.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:06 smithi093 ceph-mon[111335]: pgmap v5386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:06 smithi093 ceph-mon[118902]: pgmap v5386: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:07.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:07.296951+0000) 2023-08-27T23:53:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:08.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:08.297144+0000) 2023-08-27T23:53:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-mon[113531]: pgmap v5387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1346556644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:53:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:53:08.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[118902]: pgmap v5387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1346556644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:53:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:53:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[111335]: pgmap v5387: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:08.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:08.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1346556644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:08.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:53:08.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:53:08.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:53:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:09.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:09.297390+0000) 2023-08-27T23:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:10 smithi136 ceph-mon[113531]: pgmap v5388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4271258509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:10.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:10.297674+0000) 2023-08-27T23:53:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:10 smithi093 ceph-mon[111335]: pgmap v5388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4271258509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:10 smithi093 ceph-mon[118902]: pgmap v5388: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4271258509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:10.872Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:10.873Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:11.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:11.297958+0000) 2023-08-27T23:53:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:12 smithi136 ceph-mon[113531]: pgmap v5389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2944612587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:12.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:12.298200+0000) 2023-08-27T23:53:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:12 smithi093 ceph-mon[111335]: pgmap v5389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2944612587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:12 smithi093 ceph-mon[118902]: pgmap v5389: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2944612587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:12.351Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:12.352Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:13.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:13.298457+0000) 2023-08-27T23:53:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:14 smithi136 ceph-mon[113531]: pgmap v5390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3767561561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:14.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:14.298740+0000) 2023-08-27T23:53:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:14 smithi093 ceph-mon[111335]: pgmap v5390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3767561561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:14 smithi093 ceph-mon[118902]: pgmap v5390: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3767561561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:15.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:15.299044+0000) 2023-08-27T23:53:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:53:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:53:15] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:53:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:16 smithi136 ceph-mon[113531]: pgmap v5391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:16.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:16.299357+0000) 2023-08-27T23:53:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:16 smithi093 ceph-mon[111335]: pgmap v5391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:16 smithi093 ceph-mon[118902]: pgmap v5391: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/34054228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:17.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:17.299582+0000) 2023-08-27T23:53:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/34054228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/34054228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:18 smithi136 ceph-mon[113531]: pgmap v5392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:18.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:18.299807+0000) 2023-08-27T23:53:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:18 smithi093 ceph-mon[111335]: pgmap v5392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:18 smithi093 ceph-mon[118902]: pgmap v5392: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:19.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3053020770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:19.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:19.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:19.300011+0000) 2023-08-27T23:53:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3053020770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3053020770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:20 smithi136 ceph-mon[113531]: pgmap v5393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:20.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:20.300301+0000) 2023-08-27T23:53:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:20 smithi093 ceph-mon[111335]: pgmap v5393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:20 smithi093 ceph-mon[118902]: pgmap v5393: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:20.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:20.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:21.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:21.300549+0000) 2023-08-27T23:53:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:22 smithi136 ceph-mon[113531]: pgmap v5394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2535792373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:22.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:22.300807+0000) 2023-08-27T23:53:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:22 smithi093 ceph-mon[118902]: pgmap v5394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2535792373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:22 smithi093 ceph-mon[111335]: pgmap v5394: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2535792373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:22.352Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:22.353Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:23.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:23.301136+0000) 2023-08-27T23:53:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:24 smithi136 ceph-mon[113531]: pgmap v5395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2455911681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:24.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:24.301355+0000) 2023-08-27T23:53:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:24 smithi093 ceph-mon[118902]: pgmap v5395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2455911681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:24 smithi093 ceph-mon[111335]: pgmap v5395: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:24.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:24.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2455911681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:25.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:25.301598+0000) 2023-08-27T23:53:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:53:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:53:25] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:53:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:26 smithi136 ceph-mon[113531]: pgmap v5396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3553061486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:26.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:26.301852+0000) 2023-08-27T23:53:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:26 smithi093 ceph-mon[111335]: pgmap v5396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3553061486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:26 smithi093 ceph-mon[118902]: pgmap v5396: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3553061486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:27.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:27.302112+0000) 2023-08-27T23:53:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:28 smithi136 ceph-mon[113531]: pgmap v5397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:28.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:28.302328+0000) 2023-08-27T23:53:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:28 smithi093 ceph-mon[111335]: pgmap v5397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:28 smithi093 ceph-mon[118902]: pgmap v5397: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3014946324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:29.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:29.302553+0000) 2023-08-27T23:53:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3014946324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3014946324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:30 smithi136 ceph-mon[113531]: pgmap v5398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:30.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:30.302792+0000) 2023-08-27T23:53:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:30 smithi093 ceph-mon[111335]: pgmap v5398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:30 smithi093 ceph-mon[118902]: pgmap v5398: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:30.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:30.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3901505074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:31.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:31.303096+0000) 2023-08-27T23:53:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3901505074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3901505074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:32 smithi136 ceph-mon[113531]: pgmap v5399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:32.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:32.303353+0000) 2023-08-27T23:53:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:32 smithi093 ceph-mon[111335]: pgmap v5399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:32 smithi093 ceph-mon[118902]: pgmap v5399: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:32.352Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:32.353Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2079391120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:33.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:33.303621+0000) 2023-08-27T23:53:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2079391120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2079391120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:34 smithi136 ceph-mon[113531]: pgmap v5400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:34.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:34.303901+0000) 2023-08-27T23:53:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:34 smithi093 ceph-mon[111335]: pgmap v5400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:34 smithi093 ceph-mon[118902]: pgmap v5400: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:35.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:35.304154+0000) 2023-08-27T23:53:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:53:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:53:35] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-27T23:53:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:36 smithi136 ceph-mon[113531]: pgmap v5401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2798317570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:36.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:36.304420+0000) 2023-08-27T23:53:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:36 smithi093 ceph-mon[111335]: pgmap v5401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2798317570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:36 smithi093 ceph-mon[118902]: pgmap v5401: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2798317570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:37.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:37.304665+0000) 2023-08-27T23:53:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:38 smithi136 ceph-mon[113531]: pgmap v5402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1450364536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:38.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:38.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:38.304912+0000) 2023-08-27T23:53:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:38 smithi093 ceph-mon[111335]: pgmap v5402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1450364536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:38 smithi093 ceph-mon[118902]: pgmap v5402: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1450364536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:39.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:39.305181+0000) 2023-08-27T23:53:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:40 smithi136 ceph-mon[113531]: pgmap v5403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:40.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:40.305408+0000) 2023-08-27T23:53:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:40 smithi093 ceph-mon[111335]: pgmap v5403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:40 smithi093 ceph-mon[118902]: pgmap v5403: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:40.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:40.875Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:41.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2918135038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:41.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:41.305611+0000) 2023-08-27T23:53:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2918135038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2918135038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:42 smithi136 ceph-mon[113531]: pgmap v5404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:42.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:42.305850+0000) 2023-08-27T23:53:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:42 smithi093 ceph-mon[111335]: pgmap v5404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:42 smithi093 ceph-mon[118902]: pgmap v5404: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:42.353Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:42.354Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/172260406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:43.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:43.306121+0000) 2023-08-27T23:53:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/172260406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/172260406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:44 smithi136 ceph-mon[113531]: pgmap v5405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:44.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:44.306355+0000) 2023-08-27T23:53:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:44 smithi093 ceph-mon[111335]: pgmap v5405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:44 smithi093 ceph-mon[118902]: pgmap v5405: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1718692030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:45.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:45.306590+0000) 2023-08-27T23:53:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1718692030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:53:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:53:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:53:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1718692030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:46 smithi136 ceph-mon[113531]: pgmap v5406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:46.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:46.306866+0000) 2023-08-27T23:53:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:46 smithi093 ceph-mon[111335]: pgmap v5406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:46 smithi093 ceph-mon[118902]: pgmap v5406: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:47.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:47.307143+0000) 2023-08-27T23:53:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:48 smithi136 ceph-mon[113531]: pgmap v5407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1260881903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:48.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:48.307390+0000) 2023-08-27T23:53:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:48 smithi093 ceph-mon[111335]: pgmap v5407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:48.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1260881903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:48 smithi093 ceph-mon[118902]: pgmap v5407: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1260881903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:49.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:49.307641+0000) 2023-08-27T23:53:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:50 smithi136 ceph-mon[113531]: pgmap v5408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/60612617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:50.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:50.307866+0000) 2023-08-27T23:53:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:50 smithi093 ceph-mon[111335]: pgmap v5408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/60612617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:50 smithi093 ceph-mon[118902]: pgmap v5408: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/60612617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:50.875Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:50.876Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:51.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:51.308139+0000) 2023-08-27T23:53:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:52.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:52.308406+0000) 2023-08-27T23:53:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:52 smithi136 ceph-mon[113531]: pgmap v5409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/57635656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:52 smithi093 ceph-mon[111335]: pgmap v5409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/57635656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:52 smithi093 ceph-mon[118902]: pgmap v5409: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/57635656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:52.354Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:53:52.355Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:53:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:53.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:53.308705+0000) 2023-08-27T23:53:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:54.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:54.309021+0000) 2023-08-27T23:53:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:54 smithi136 ceph-mon[113531]: pgmap v5410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:54 smithi093 ceph-mon[111335]: pgmap v5410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:54 smithi093 ceph-mon[118902]: pgmap v5410: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:53:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/87496532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:55.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:55.309298+0000) 2023-08-27T23:53:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/87496532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/87496532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:53:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:53:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:53:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:56 smithi136 ceph-mon[113531]: pgmap v5411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:56.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:56.309558+0000) 2023-08-27T23:53:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:56 smithi093 ceph-mon[111335]: pgmap v5411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:56 smithi093 ceph-mon[118902]: pgmap v5411: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:57.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3503710085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:57.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:57.309817+0000) 2023-08-27T23:53:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3503710085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3503710085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:53:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:53:58 smithi136 ceph-mon[113531]: pgmap v5412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:58.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:58.310148+0000) 2023-08-27T23:53:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:53:58 smithi093 ceph-mon[111335]: pgmap v5412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:53:58 smithi093 ceph-mon[118902]: pgmap v5412: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:53:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:53:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:53:59.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:53:59.310451+0000) 2023-08-27T23:54:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:00 smithi136 ceph-mon[113531]: pgmap v5413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2315071753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:00.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:00.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:00.310699+0000) 2023-08-27T23:54:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:00 smithi093 ceph-mon[111335]: pgmap v5413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2315071753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:00 smithi093 ceph-mon[118902]: pgmap v5413: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2315071753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:00.876Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:00.877Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:01.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:01.310964+0000) 2023-08-27T23:54:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:02 smithi136 ceph-mon[113531]: pgmap v5414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3852499487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:02.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:02.311218+0000) 2023-08-27T23:54:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:02 smithi093 ceph-mon[111335]: pgmap v5414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3852499487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:02 smithi093 ceph-mon[118902]: pgmap v5414: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3852499487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:02.354Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:02.355Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:03.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:03.311464+0000) 2023-08-27T23:54:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:04 smithi136 ceph-mon[113531]: pgmap v5415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/204591369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:04.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:04.311759+0000) 2023-08-27T23:54:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:04 smithi093 ceph-mon[111335]: pgmap v5415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/204591369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:04 smithi093 ceph-mon[118902]: pgmap v5415: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/204591369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:05.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:05.312058+0000) 2023-08-27T23:54:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:54:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:54:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:54:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:06 smithi136 ceph-mon[113531]: pgmap v5416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:06.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:06.312362+0000) 2023-08-27T23:54:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:06 smithi093 ceph-mon[111335]: pgmap v5416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:06 smithi093 ceph-mon[118902]: pgmap v5416: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1476893009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:07.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:07.312634+0000) 2023-08-27T23:54:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1476893009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1476893009' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:08.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:08 smithi136 ceph-mon[113531]: pgmap v5417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:08.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:54:08.637 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:08.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:08.312831+0000) 2023-08-27T23:54:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:08 smithi093 ceph-mon[111335]: pgmap v5417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:54:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:08 smithi093 ceph-mon[118902]: pgmap v5417: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:54:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:09.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:09.313086+0000) 2023-08-27T23:54:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2829701263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2829701263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2829701263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:10.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:10.313256+0000) 2023-08-27T23:54:10.792 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:10 smithi136 ceph-mon[113531]: pgmap v5418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:10.792 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.792 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.792 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.792 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.878 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[111335]: pgmap v5418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:10.878 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.878 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.879 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.879 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.879 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[118902]: pgmap v5418: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:10.879 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.879 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.879 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:10.879 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:10.877Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:10.878Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:11.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:11.313436+0000) 2023-08-27T23:54:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:54:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:54:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3690235344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:54:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:54:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:11.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3690235344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:11.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:54:11.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:54:11.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:54:11.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3690235344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:12.679 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:12.355Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:12.679 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:12.356Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:12.680 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:12.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:12.313695+0000) 2023-08-27T23:54:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:12 smithi136 ceph-mon[113531]: pgmap v5419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:12 smithi093 ceph-mon[111335]: pgmap v5419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:12 smithi093 ceph-mon[118902]: pgmap v5419: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:13.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:13.313947+0000) 2023-08-27T23:54:14.682 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:14.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:14.314161+0000) 2023-08-27T23:54:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:14 smithi136 ceph-mon[113531]: pgmap v5420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2400796334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:14 smithi093 ceph-mon[111335]: pgmap v5420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2400796334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:14 smithi093 ceph-mon[118902]: pgmap v5420: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2400796334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:15.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:15.314432+0000) 2023-08-27T23:54:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:54:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:54:15] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:54:16.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:16.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:16.314729+0000) 2023-08-27T23:54:16.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:16 smithi136 ceph-mon[113531]: pgmap v5421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/716888006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:16 smithi093 ceph-mon[118902]: pgmap v5421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/716888006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:16 smithi093 ceph-mon[111335]: pgmap v5421: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/716888006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:17.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:17.314939+0000) 2023-08-27T23:54:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:18.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:18.315159+0000) 2023-08-27T23:54:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:18 smithi136 ceph-mon[113531]: pgmap v5422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1917850507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:18 smithi093 ceph-mon[111335]: pgmap v5422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1917850507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:18 smithi093 ceph-mon[118902]: pgmap v5422: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1917850507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:19.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:19.315415+0000) 2023-08-27T23:54:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:20.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:20.315667+0000) 2023-08-27T23:54:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:20 smithi093 ceph-mon[111335]: pgmap v5423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:21.019 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:20 smithi093 ceph-mon[118902]: pgmap v5423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:21.020 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:20.877Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:21.020 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:20.878Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:20 smithi136 ceph-mon[113531]: pgmap v5423: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:21.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:21.315952+0000) 2023-08-27T23:54:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2924541563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2924541563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2924541563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:22.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:22.316235+0000) 2023-08-27T23:54:22.697 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:22.356Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:22.698 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:22.357Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:22 smithi093 ceph-mon[111335]: pgmap v5424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:22 smithi093 ceph-mon[118902]: pgmap v5424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:22 smithi136 ceph-mon[113531]: pgmap v5424: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:23.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:23.316487+0000) 2023-08-27T23:54:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2039979558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2039979558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2039979558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:24.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:24.316721+0000) 2023-08-27T23:54:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:24 smithi093 ceph-mon[111335]: pgmap v5425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:24 smithi093 ceph-mon[118902]: pgmap v5425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:24 smithi136 ceph-mon[113531]: pgmap v5425: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:25.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:25.316905+0000) 2023-08-27T23:54:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:54:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:54:25] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:54:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:26.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:26.317168+0000) 2023-08-27T23:54:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:26 smithi093 ceph-mon[111335]: pgmap v5426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2244410398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:26 smithi093 ceph-mon[118902]: pgmap v5426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2244410398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:26 smithi136 ceph-mon[113531]: pgmap v5426: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2244410398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:27.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:27.317411+0000) 2023-08-27T23:54:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:28.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:28.317662+0000) 2023-08-27T23:54:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:28 smithi093 ceph-mon[111335]: pgmap v5427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2665332583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:28 smithi093 ceph-mon[118902]: pgmap v5427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2665332583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:28 smithi136 ceph-mon[113531]: pgmap v5427: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2665332583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:29.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:29.317881+0000) 2023-08-27T23:54:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:30.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:30.318044+0000) 2023-08-27T23:54:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:30 smithi093 ceph-mon[111335]: pgmap v5428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1475384306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:30 smithi093 ceph-mon[118902]: pgmap v5428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1475384306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:30.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:30.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:30 smithi136 ceph-mon[113531]: pgmap v5428: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1475384306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:31.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:31.318305+0000) 2023-08-27T23:54:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:32.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:32.318590+0000) 2023-08-27T23:54:32.719 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:32.357Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:32.719 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:32.357Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:32 smithi093 ceph-mon[111335]: pgmap v5429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:32 smithi093 ceph-mon[118902]: pgmap v5429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:32 smithi136 ceph-mon[113531]: pgmap v5429: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:33.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:33.318851+0000) 2023-08-27T23:54:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2651092778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2651092778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2651092778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:34.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:34.319104+0000) 2023-08-27T23:54:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:34 smithi093 ceph-mon[111335]: pgmap v5430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:34 smithi093 ceph-mon[118902]: pgmap v5430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:34 smithi136 ceph-mon[113531]: pgmap v5430: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:35.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:35.319318+0000) 2023-08-27T23:54:35.731 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:54:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:54:35] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:54:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/773465173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/773465173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/773465173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:36.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:36.319595+0000) 2023-08-27T23:54:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:36 smithi093 ceph-mon[111335]: pgmap v5431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:36 smithi093 ceph-mon[118902]: pgmap v5431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:36 smithi136 ceph-mon[113531]: pgmap v5431: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:37.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:37.319844+0000) 2023-08-27T23:54:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3981751703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3981751703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3981751703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:38.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:38.320095+0000) 2023-08-27T23:54:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:38 smithi136 ceph-mon[113531]: pgmap v5432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:38 smithi093 ceph-mon[111335]: pgmap v5432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:38 smithi093 ceph-mon[118902]: pgmap v5432: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:39.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:39.320392+0000) 2023-08-27T23:54:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:40.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:40.320674+0000) 2023-08-27T23:54:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:40 smithi136 ceph-mon[113531]: pgmap v5433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1300557779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:40 smithi093 ceph-mon[111335]: pgmap v5433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1300557779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:40 smithi093 ceph-mon[118902]: pgmap v5433: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1300557779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:40.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:40.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:41.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:41.320906+0000) 2023-08-27T23:54:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:42.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:42.321165+0000) 2023-08-27T23:54:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:42.357Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:42.358Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:42 smithi136 ceph-mon[113531]: pgmap v5434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1560989357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:42 smithi093 ceph-mon[111335]: pgmap v5434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1560989357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:42 smithi093 ceph-mon[118902]: pgmap v5434: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1560989357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:43.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:43.321443+0000) 2023-08-27T23:54:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:44.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:44.321693+0000) 2023-08-27T23:54:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:44 smithi136 ceph-mon[113531]: pgmap v5435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:44 smithi093 ceph-mon[111335]: pgmap v5435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:44 smithi093 ceph-mon[118902]: pgmap v5435: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:45.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:45.321907+0000) 2023-08-27T23:54:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:54:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:54:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:54:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1135139373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1135139373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1135139373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:46.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:46.322174+0000) 2023-08-27T23:54:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:46 smithi136 ceph-mon[113531]: pgmap v5436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:46 smithi093 ceph-mon[111335]: pgmap v5436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:46 smithi093 ceph-mon[118902]: pgmap v5436: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:47.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:47.322375+0000) 2023-08-27T23:54:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1186516870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1186516870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1186516870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:48.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:48.322590+0000) 2023-08-27T23:54:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:48 smithi136 ceph-mon[113531]: pgmap v5437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:48 smithi093 ceph-mon[111335]: pgmap v5437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:48 smithi093 ceph-mon[118902]: pgmap v5437: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:49.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:49.322839+0000) 2023-08-27T23:54:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3527587374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3527587374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3527587374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:50.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:50.323111+0000) 2023-08-27T23:54:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:50 smithi136 ceph-mon[113531]: pgmap v5438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:50 smithi093 ceph-mon[111335]: pgmap v5438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:50 smithi093 ceph-mon[118902]: pgmap v5438: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:50.879Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:50.880Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:51.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:51.323395+0000) 2023-08-27T23:54:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:52.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:52.323672+0000) 2023-08-27T23:54:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:52.358Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:54:52.359Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:54:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:52 smithi136 ceph-mon[113531]: pgmap v5439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2214355296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:52 smithi093 ceph-mon[111335]: pgmap v5439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2214355296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:52 smithi093 ceph-mon[118902]: pgmap v5439: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2214355296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:53.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:53.323977+0000) 2023-08-27T23:54:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:54:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:54.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:54.324246+0000) 2023-08-27T23:54:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:54 smithi136 ceph-mon[113531]: pgmap v5440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4050809424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:54 smithi093 ceph-mon[111335]: pgmap v5440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4050809424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:54 smithi093 ceph-mon[118902]: pgmap v5440: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4050809424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:55.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:55.324544+0000) 2023-08-27T23:54:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:54:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:54:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:54:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:56.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:56.324846+0000) 2023-08-27T23:54:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:56 smithi136 ceph-mon[113531]: pgmap v5441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2128601707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:56 smithi093 ceph-mon[111335]: pgmap v5441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2128601707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:56 smithi093 ceph-mon[118902]: pgmap v5441: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2128601707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:54:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:57.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:57.325136+0000) 2023-08-27T23:54:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:58.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:58.325364+0000) 2023-08-27T23:54:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:58 smithi136 ceph-mon[113531]: pgmap v5442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:58 smithi093 ceph-mon[111335]: pgmap v5442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:58 smithi093 ceph-mon[118902]: pgmap v5442: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:54:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:54:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:54:59.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:54:59.325667+0000) 2023-08-27T23:55:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:54:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2944846930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:54:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2944846930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:54:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2944846930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:00.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:00.326014+0000) 2023-08-27T23:55:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:00 smithi136 ceph-mon[113531]: pgmap v5443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:00 smithi093 ceph-mon[111335]: pgmap v5443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:00 smithi093 ceph-mon[118902]: pgmap v5443: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:00.880Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:00.881Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:01.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:01.326234+0000) 2023-08-27T23:55:02.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/998710094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/998710094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/998710094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:02.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:02.326493+0000) 2023-08-27T23:55:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:02.359Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:02.359Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:02 smithi136 ceph-mon[113531]: pgmap v5444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:03.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:02 smithi093 ceph-mon[111335]: pgmap v5444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:03.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:02 smithi093 ceph-mon[118902]: pgmap v5444: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:03.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:03.326758+0000) 2023-08-27T23:55:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:04.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:04.326999+0000) 2023-08-27T23:55:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:04 smithi136 ceph-mon[113531]: pgmap v5445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3195203413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:04 smithi093 ceph-mon[111335]: pgmap v5445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3195203413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:04 smithi093 ceph-mon[118902]: pgmap v5445: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:05.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3195203413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:05.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:05.327267+0000) 2023-08-27T23:55:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:55:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:55:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:55:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:06.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:06.327549+0000) 2023-08-27T23:55:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:06 smithi136 ceph-mon[113531]: pgmap v5446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2817105477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:06 smithi093 ceph-mon[111335]: pgmap v5446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2817105477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:06 smithi093 ceph-mon[118902]: pgmap v5446: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2817105477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:07.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:07.327819+0000) 2023-08-27T23:55:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:08.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:08.328104+0000) 2023-08-27T23:55:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:08 smithi136 ceph-mon[113531]: pgmap v5447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4079528857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:08 smithi093 ceph-mon[111335]: pgmap v5447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4079528857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:08 smithi093 ceph-mon[118902]: pgmap v5447: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4079528857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:09.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:09.328369+0000) 2023-08-27T23:55:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:10.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:10.328660+0000) 2023-08-27T23:55:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:10 smithi136 ceph-mon[113531]: pgmap v5448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:55:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:10 smithi093 ceph-mon[111335]: pgmap v5448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:55:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:10 smithi093 ceph-mon[118902]: pgmap v5448: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:55:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:10.881Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:10.882Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:11.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:11.328830+0000) 2023-08-27T23:55:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3416077734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:12.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3416077734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:12.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3416077734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:12.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:12.329029+0000) 2023-08-27T23:55:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:12.359Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:12.360Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:12.993 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:12 smithi136 ceph-mon[113531]: pgmap v5449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:12.993 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:12.993 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-mon[111335]: pgmap v5449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-mon[118902]: pgmap v5449: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:13.330 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:13.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:13.329220+0000) 2023-08-27T23:55:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:14.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:14.329412+0000) 2023-08-27T23:55:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-mon[113531]: pgmap v5450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1485451679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:14.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:55:14.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:55:14.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[118902]: pgmap v5450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1485451679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[111335]: pgmap v5450: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1485451679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:55:14.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:55:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:15.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:15.329663+0000) 2023-08-27T23:55:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:55:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:55:15] "GET /metrics HTTP/1.1" 200 239914 "" "Prometheus/2.43.0" 2023-08-27T23:55:16.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:16 smithi136 ceph-mon[113531]: pgmap v5451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/224401310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:16.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:16.329929+0000) 2023-08-27T23:55:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:16 smithi093 ceph-mon[111335]: pgmap v5451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/224401310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:16 smithi093 ceph-mon[118902]: pgmap v5451: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/224401310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:17.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:17.330139+0000) 2023-08-27T23:55:18.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:18 smithi136 ceph-mon[113531]: pgmap v5452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2658396166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:18.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:18.330365+0000) 2023-08-27T23:55:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:18 smithi093 ceph-mon[111335]: pgmap v5452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2658396166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:18 smithi093 ceph-mon[118902]: pgmap v5452: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2658396166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:19.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:19.330602+0000) 2023-08-27T23:55:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:20 smithi136 ceph-mon[113531]: pgmap v5453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:20.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:20.330878+0000) 2023-08-27T23:55:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:20 smithi093 ceph-mon[111335]: pgmap v5453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:20 smithi093 ceph-mon[118902]: pgmap v5453: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:20.882Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:20.883Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2378968979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:21.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:21.331183+0000) 2023-08-27T23:55:21.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2378968979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:21.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2378968979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:22 smithi136 ceph-mon[113531]: pgmap v5454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:22.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:22.331434+0000) 2023-08-27T23:55:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:22 smithi093 ceph-mon[111335]: pgmap v5454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:22 smithi093 ceph-mon[118902]: pgmap v5454: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:22.360Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:22.360Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:23.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2190112285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:23.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:23.331687+0000) 2023-08-27T23:55:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2190112285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2190112285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:24 smithi136 ceph-mon[113531]: pgmap v5455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:24.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:24.331935+0000) 2023-08-27T23:55:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:24 smithi093 ceph-mon[111335]: pgmap v5455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:24 smithi093 ceph-mon[118902]: pgmap v5455: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:25.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:25.332170+0000) 2023-08-27T23:55:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:55:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:55:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:55:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:26 smithi136 ceph-mon[113531]: pgmap v5456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1138095413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:26.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:26.332406+0000) 2023-08-27T23:55:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:26 smithi093 ceph-mon[111335]: pgmap v5456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1138095413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:26 smithi093 ceph-mon[118902]: pgmap v5456: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1138095413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:27.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:27.332652+0000) 2023-08-27T23:55:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:28 smithi136 ceph-mon[113531]: pgmap v5457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778079952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:28.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:28.332918+0000) 2023-08-27T23:55:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:28 smithi093 ceph-mon[111335]: pgmap v5457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778079952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:28 smithi093 ceph-mon[118902]: pgmap v5457: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778079952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:29.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:29.333135+0000) 2023-08-27T23:55:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:30 smithi136 ceph-mon[113531]: pgmap v5458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2714127525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:30.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:30.333387+0000) 2023-08-27T23:55:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:30 smithi093 ceph-mon[118902]: pgmap v5458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2714127525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:30 smithi093 ceph-mon[111335]: pgmap v5458: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2714127525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:30.884Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:30.885Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:31.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:31.333654+0000) 2023-08-27T23:55:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:32 smithi136 ceph-mon[113531]: pgmap v5459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:32.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:32.333872+0000) 2023-08-27T23:55:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:32 smithi093 ceph-mon[111335]: pgmap v5459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:32 smithi093 ceph-mon[118902]: pgmap v5459: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:32.361Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:32.362Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1398597606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:33.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:33.334100+0000) 2023-08-27T23:55:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1398597606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1398597606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:34 smithi136 ceph-mon[113531]: pgmap v5460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:34.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:34.334354+0000) 2023-08-27T23:55:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:34 smithi093 ceph-mon[111335]: pgmap v5460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:34 smithi093 ceph-mon[118902]: pgmap v5460: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3858414776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:35.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:35.334577+0000) 2023-08-27T23:55:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3858414776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3858414776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:55:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:55:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:55:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:36 smithi136 ceph-mon[113531]: pgmap v5461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:36.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:36.334802+0000) 2023-08-27T23:55:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:36 smithi093 ceph-mon[111335]: pgmap v5461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:36 smithi093 ceph-mon[118902]: pgmap v5461: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3377190396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:37.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:37.335070+0000) 2023-08-27T23:55:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3377190396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:37.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3377190396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:38 smithi136 ceph-mon[113531]: pgmap v5462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:38.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:38.335329+0000) 2023-08-27T23:55:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:38 smithi093 ceph-mon[111335]: pgmap v5462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:38 smithi093 ceph-mon[118902]: pgmap v5462: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:39.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:39.335605+0000) 2023-08-27T23:55:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:40 smithi136 ceph-mon[113531]: pgmap v5463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1026851612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:40.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:40.335846+0000) 2023-08-27T23:55:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:40 smithi093 ceph-mon[111335]: pgmap v5463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1026851612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:40 smithi093 ceph-mon[118902]: pgmap v5463: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1026851612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:40.885Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:40.886Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:41.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:41.336126+0000) 2023-08-27T23:55:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:42 smithi136 ceph-mon[113531]: pgmap v5464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12616435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:42.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:42.336282+0000) 2023-08-27T23:55:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:42 smithi093 ceph-mon[118902]: pgmap v5464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12616435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:42 smithi093 ceph-mon[111335]: pgmap v5464: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12616435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:42.362Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:42.363Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:43.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:43.336525+0000) 2023-08-27T23:55:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:44 smithi136 ceph-mon[113531]: pgmap v5465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:44.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:44.336761+0000) 2023-08-27T23:55:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:44 smithi093 ceph-mon[118902]: pgmap v5465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:44 smithi093 ceph-mon[111335]: pgmap v5465: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1008484668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:45.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:45.336971+0000) 2023-08-27T23:55:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1008484668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:55:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:55:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:55:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1008484668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:46 smithi136 ceph-mon[113531]: pgmap v5466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:46.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:46.337241+0000) 2023-08-27T23:55:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:46 smithi093 ceph-mon[111335]: pgmap v5466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:46 smithi093 ceph-mon[118902]: pgmap v5466: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:47.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1465638542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:47.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:47.337457+0000) 2023-08-27T23:55:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1465638542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1465638542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:48 smithi136 ceph-mon[113531]: pgmap v5467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:48.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:48.337696+0000) 2023-08-27T23:55:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:48 smithi093 ceph-mon[118902]: pgmap v5467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:48 smithi093 ceph-mon[111335]: pgmap v5467: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:49.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3547237462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:49.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:49.337953+0000) 2023-08-27T23:55:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3547237462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3547237462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:50 smithi136 ceph-mon[113531]: pgmap v5468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:50.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:50.338225+0000) 2023-08-27T23:55:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:50 smithi093 ceph-mon[111335]: pgmap v5468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:50 smithi093 ceph-mon[118902]: pgmap v5468: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:50.885Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:50.887Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:51.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:51.338498+0000) 2023-08-27T23:55:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:52 smithi136 ceph-mon[113531]: pgmap v5469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4060596163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:52.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:52.338738+0000) 2023-08-27T23:55:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:52 smithi093 ceph-mon[111335]: pgmap v5469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4060596163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:52 smithi093 ceph-mon[118902]: pgmap v5469: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4060596163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:52.363Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:55:52.364Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:55:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:53.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:53.338946+0000) 2023-08-27T23:55:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:54 smithi136 ceph-mon[113531]: pgmap v5470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3682858486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:54.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:54.339223+0000) 2023-08-27T23:55:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:54 smithi093 ceph-mon[111335]: pgmap v5470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3682858486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:54 smithi093 ceph-mon[118902]: pgmap v5470: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:55:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3682858486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:55.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:55.339416+0000) 2023-08-27T23:55:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:55:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:55:55] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:55:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:56 smithi136 ceph-mon[113531]: pgmap v5471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3404697937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:56.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:56.339697+0000) 2023-08-27T23:55:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:56 smithi093 ceph-mon[111335]: pgmap v5471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3404697937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:56 smithi093 ceph-mon[118902]: pgmap v5471: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3404697937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:57.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:57.339954+0000) 2023-08-27T23:55:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:58 smithi136 ceph-mon[113531]: pgmap v5472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:58.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:58.340181+0000) 2023-08-27T23:55:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:58 smithi093 ceph-mon[111335]: pgmap v5472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:58 smithi093 ceph-mon[118902]: pgmap v5472: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:55:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:55:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/167150474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:59.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:55:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:55:59.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:55:59.340404+0000) 2023-08-27T23:55:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:55:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/167150474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:55:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:55:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/167150474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:00 smithi136 ceph-mon[113531]: pgmap v5473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:00.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:00.340656+0000) 2023-08-27T23:56:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:00 smithi093 ceph-mon[111335]: pgmap v5473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:00 smithi093 ceph-mon[118902]: pgmap v5473: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:00.886Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:00.887Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:01.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3277226620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:01.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:01.340893+0000) 2023-08-27T23:56:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3277226620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3277226620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:02 smithi136 ceph-mon[113531]: pgmap v5474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:02.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:02.341118+0000) 2023-08-27T23:56:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:02 smithi093 ceph-mon[111335]: pgmap v5474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:02 smithi093 ceph-mon[118902]: pgmap v5474: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:02.363Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:02.365Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:03.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:03.341343+0000) 2023-08-27T23:56:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:04 smithi136 ceph-mon[113531]: pgmap v5475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2886493499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:04.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:04.341578+0000) 2023-08-27T23:56:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:04 smithi093 ceph-mon[111335]: pgmap v5475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2886493499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:04 smithi093 ceph-mon[118902]: pgmap v5475: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2886493499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:05.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:05.341810+0000) 2023-08-27T23:56:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:56:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:56:05] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:56:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:06 smithi136 ceph-mon[113531]: pgmap v5476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1687234656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:06.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:06.342070+0000) 2023-08-27T23:56:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:06 smithi093 ceph-mon[111335]: pgmap v5476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1687234656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:06 smithi093 ceph-mon[118902]: pgmap v5476: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1687234656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:07.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:07.342295+0000) 2023-08-27T23:56:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:08 smithi136 ceph-mon[113531]: pgmap v5477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3936014685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:08.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:08.342532+0000) 2023-08-27T23:56:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:08 smithi093 ceph-mon[111335]: pgmap v5477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3936014685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:08 smithi093 ceph-mon[118902]: pgmap v5477: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3936014685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:09.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:09.342757+0000) 2023-08-27T23:56:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:10 smithi136 ceph-mon[113531]: pgmap v5478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:10.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:10.342982+0000) 2023-08-27T23:56:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:10 smithi093 ceph-mon[111335]: pgmap v5478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:10 smithi093 ceph-mon[118902]: pgmap v5478: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:10.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:10.887Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3655051248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:11.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:11.343258+0000) 2023-08-27T23:56:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3655051248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3655051248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:12 smithi136 ceph-mon[113531]: pgmap v5479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:12.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:12.343497+0000) 2023-08-27T23:56:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:12 smithi093 ceph-mon[111335]: pgmap v5479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:12 smithi093 ceph-mon[118902]: pgmap v5479: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:12.364Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:12.365Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1627639719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:13.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:13.343730+0000) 2023-08-27T23:56:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1627639719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1627639719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:14 smithi136 ceph-mon[113531]: pgmap v5480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:56:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:14.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:14.343928+0000) 2023-08-27T23:56:14.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[111335]: pgmap v5480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:14.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:56:14.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:14.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:14.756 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[118902]: pgmap v5480: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:14.757 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:56:14.757 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:14.757 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:15.681 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:15.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:15.344104+0000) 2023-08-27T23:56:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:56:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:56:15] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:56:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:16.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:16.344247+0000) 2023-08-27T23:56:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:16 smithi136 ceph-mon[113531]: pgmap v5481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/355697810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[111335]: pgmap v5481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/355697810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[118902]: pgmap v5481: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/355697810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:56:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:56:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:17.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:17.344402+0000) 2023-08-27T23:56:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:56:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:56:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:56:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:56:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:56:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:18.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:18.344635+0000) 2023-08-27T23:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:18 smithi093 ceph-mon[111335]: pgmap v5482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1945956568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:18 smithi093 ceph-mon[118902]: pgmap v5482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1945956568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:18 smithi136 ceph-mon[113531]: pgmap v5482: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1945956568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:19.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:19.344898+0000) 2023-08-27T23:56:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:20.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:20.345162+0000) 2023-08-27T23:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:20 smithi093 ceph-mon[111335]: pgmap v5483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2364265418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:20 smithi093 ceph-mon[118902]: pgmap v5483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2364265418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:20.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:20.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:20.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:20.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:20 smithi136 ceph-mon[113531]: pgmap v5483: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2364265418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:21.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:21.345429+0000) 2023-08-27T23:56:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:22.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:22.345691+0000) 2023-08-27T23:56:22.715 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:22.365Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:22.715 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:22.366Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:22 smithi093 ceph-mon[111335]: pgmap v5484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1615408696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:22 smithi093 ceph-mon[118902]: pgmap v5484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1615408696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:22 smithi136 ceph-mon[113531]: pgmap v5484: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1615408696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:23.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:23.345941+0000) 2023-08-27T23:56:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:24.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:24.346187+0000) 2023-08-27T23:56:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:24 smithi093 ceph-mon[111335]: pgmap v5485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:24.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:24 smithi093 ceph-mon[118902]: pgmap v5485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:24.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:25.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:24 smithi136 ceph-mon[113531]: pgmap v5485: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:25.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:25.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:25.346429+0000) 2023-08-27T23:56:25.724 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:56:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:56:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:56:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1174939417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1174939417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1174939417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:26.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:26.346692+0000) 2023-08-27T23:56:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:26 smithi093 ceph-mon[111335]: pgmap v5486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:26 smithi093 ceph-mon[118902]: pgmap v5486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:26 smithi136 ceph-mon[113531]: pgmap v5486: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:27.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:27.346980+0000) 2023-08-27T23:56:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3103930443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3103930443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3103930443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:28.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:28.347217+0000) 2023-08-27T23:56:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:28 smithi093 ceph-mon[111335]: pgmap v5487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:28 smithi093 ceph-mon[118902]: pgmap v5487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:28 smithi136 ceph-mon[113531]: pgmap v5487: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:29.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:29.347459+0000) 2023-08-27T23:56:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:30.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:30.347728+0000) 2023-08-27T23:56:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:30 smithi093 ceph-mon[111335]: pgmap v5488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002486635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:30 smithi093 ceph-mon[118902]: pgmap v5488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002486635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:30.888Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:30.889Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:30 smithi136 ceph-mon[113531]: pgmap v5488: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002486635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:31.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:31.347956+0000) 2023-08-27T23:56:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:32.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:32.348173+0000) 2023-08-27T23:56:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:32.365Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:32.366Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:32 smithi136 ceph-mon[113531]: pgmap v5489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2270857021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:32 smithi093 ceph-mon[111335]: pgmap v5489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2270857021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:32 smithi093 ceph-mon[118902]: pgmap v5489: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2270857021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:33.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:33.348404+0000) 2023-08-27T23:56:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:34.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:34.348655+0000) 2023-08-27T23:56:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:34 smithi136 ceph-mon[113531]: pgmap v5490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3161172712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:34 smithi093 ceph-mon[111335]: pgmap v5490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3161172712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:34 smithi093 ceph-mon[118902]: pgmap v5490: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3161172712' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:35.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:35.348907+0000) 2023-08-27T23:56:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:56:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:56:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:56:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:36.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:36.349181+0000) 2023-08-27T23:56:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:36 smithi136 ceph-mon[113531]: pgmap v5491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:36 smithi093 ceph-mon[111335]: pgmap v5491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:36 smithi093 ceph-mon[118902]: pgmap v5491: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:37.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:37.349403+0000) 2023-08-27T23:56:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1882291973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1882291973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1882291973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:38.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:38.349630+0000) 2023-08-27T23:56:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:38 smithi136 ceph-mon[113531]: pgmap v5492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:38 smithi093 ceph-mon[111335]: pgmap v5492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:38 smithi093 ceph-mon[118902]: pgmap v5492: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:39.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:39.349877+0000) 2023-08-27T23:56:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/296200908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/296200908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/296200908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:40.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:40.350151+0000) 2023-08-27T23:56:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:40 smithi136 ceph-mon[113531]: pgmap v5493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:40 smithi093 ceph-mon[111335]: pgmap v5493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:40 smithi093 ceph-mon[118902]: pgmap v5493: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:40.889Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:40.890Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:41.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:41.350404+0000) 2023-08-27T23:56:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4201045469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4201045469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4201045469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:42.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:42.350592+0000) 2023-08-27T23:56:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:42.365Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:42.366Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:42 smithi136 ceph-mon[113531]: pgmap v5494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:42 smithi093 ceph-mon[111335]: pgmap v5494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:42 smithi093 ceph-mon[118902]: pgmap v5494: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:43.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:43.350842+0000) 2023-08-27T23:56:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:44.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:44.351131+0000) 2023-08-27T23:56:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:44 smithi136 ceph-mon[113531]: pgmap v5495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1620666616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:44 smithi093 ceph-mon[111335]: pgmap v5495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1620666616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:44 smithi093 ceph-mon[118902]: pgmap v5495: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1620666616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:45.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:45.351410+0000) 2023-08-27T23:56:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:56:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:56:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-27T23:56:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:46.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:46.351675+0000) 2023-08-27T23:56:47.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:46 smithi136 ceph-mon[113531]: pgmap v5496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2497828825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:46 smithi093 ceph-mon[111335]: pgmap v5496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2497828825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:46 smithi093 ceph-mon[118902]: pgmap v5496: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2497828825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:47.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:47.351918+0000) 2023-08-27T23:56:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:48.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:48.352170+0000) 2023-08-27T23:56:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:48 smithi136 ceph-mon[113531]: pgmap v5497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2708974438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:48 smithi093 ceph-mon[111335]: pgmap v5497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2708974438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:48 smithi093 ceph-mon[118902]: pgmap v5497: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2708974438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:49.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:49.352412+0000) 2023-08-27T23:56:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:50.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:50.352610+0000) 2023-08-27T23:56:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:50 smithi136 ceph-mon[113531]: pgmap v5498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:50 smithi093 ceph-mon[118902]: pgmap v5498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:50 smithi093 ceph-mon[111335]: pgmap v5498: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:50.890Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:50.891Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:51.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:51.352892+0000) 2023-08-27T23:56:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1620747852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1620747852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1620747852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:52.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:52.353149+0000) 2023-08-27T23:56:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:52.366Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:56:52.367Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:56:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:52 smithi136 ceph-mon[113531]: pgmap v5499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:52 smithi093 ceph-mon[111335]: pgmap v5499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:52 smithi093 ceph-mon[118902]: pgmap v5499: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:53.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:53.353345+0000) 2023-08-27T23:56:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2237494018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2237494018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2237494018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:56:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:54.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:54.353604+0000) 2023-08-27T23:56:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:54 smithi136 ceph-mon[113531]: pgmap v5500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:54 smithi093 ceph-mon[118902]: pgmap v5500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:54 smithi093 ceph-mon[111335]: pgmap v5500: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:55.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:55.353835+0000) 2023-08-27T23:56:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:56:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:56:55] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:56:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:56.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:56.354118+0000) 2023-08-27T23:56:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:56 smithi136 ceph-mon[113531]: pgmap v5501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/180982121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:56 smithi093 ceph-mon[111335]: pgmap v5501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/180982121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:57.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:56 smithi093 ceph-mon[118902]: pgmap v5501: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:57.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/180982121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:57.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:57.354359+0000) 2023-08-27T23:56:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:58.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:58.354627+0000) 2023-08-27T23:56:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:58 smithi136 ceph-mon[113531]: pgmap v5502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:56:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1939775434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:58 smithi093 ceph-mon[111335]: pgmap v5502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:56:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1939775434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:58 smithi093 ceph-mon[118902]: pgmap v5502: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:56:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:56:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1939775434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:56:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:56:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:56:59.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:56:59.354878+0000) 2023-08-27T23:57:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:00.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:00.355127+0000) 2023-08-27T23:57:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:00 smithi136 ceph-mon[113531]: pgmap v5503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/158310901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:00 smithi093 ceph-mon[111335]: pgmap v5503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/158310901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:00 smithi093 ceph-mon[118902]: pgmap v5503: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/158310901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:00.890Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:00.891Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:01.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:01.355348+0000) 2023-08-27T23:57:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:02.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:02.355595+0000) 2023-08-27T23:57:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:02.366Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:02.367Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:02 smithi136 ceph-mon[113531]: pgmap v5504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:02 smithi093 ceph-mon[111335]: pgmap v5504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:02 smithi093 ceph-mon[118902]: pgmap v5504: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:03.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:03.355847+0000) 2023-08-27T23:57:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3827936281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:04.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3827936281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:04.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3827936281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:04.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:04.356110+0000) 2023-08-27T23:57:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:04 smithi136 ceph-mon[113531]: pgmap v5505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:04 smithi093 ceph-mon[111335]: pgmap v5505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:04 smithi093 ceph-mon[118902]: pgmap v5505: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:05.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:05.356375+0000) 2023-08-27T23:57:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:57:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:57:05] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:57:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:06 smithi136 ceph-mon[113531]: pgmap v5506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2212547997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:06.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:06.356619+0000) 2023-08-27T23:57:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:06 smithi093 ceph-mon[111335]: pgmap v5506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:06.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2212547997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:06 smithi093 ceph-mon[118902]: pgmap v5506: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2212547997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:07.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:07.356923+0000) 2023-08-27T23:57:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:08.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:08.357188+0000) 2023-08-27T23:57:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:08 smithi136 ceph-mon[113531]: pgmap v5507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3959811289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:08 smithi093 ceph-mon[111335]: pgmap v5507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3959811289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:08.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:08 smithi093 ceph-mon[118902]: pgmap v5507: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:08.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3959811289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:09.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:09.357475+0000) 2023-08-27T23:57:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:10 smithi136 ceph-mon[113531]: pgmap v5508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2430838982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:10.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:10.357665+0000) 2023-08-27T23:57:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:10 smithi093 ceph-mon[111335]: pgmap v5508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2430838982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:10 smithi093 ceph-mon[118902]: pgmap v5508: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2430838982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:10.891Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:10.892Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:11.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:11.357949+0000) 2023-08-27T23:57:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:12 smithi136 ceph-mon[113531]: pgmap v5509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:12.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:12.358223+0000) 2023-08-27T23:57:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:12 smithi093 ceph-mon[111335]: pgmap v5509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:12 smithi093 ceph-mon[118902]: pgmap v5509: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:12.367Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:12.368Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/776603979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:13.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:13.358512+0000) 2023-08-27T23:57:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/776603979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/776603979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:14.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:14 smithi136 ceph-mon[113531]: pgmap v5510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:14.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:14.358733+0000) 2023-08-27T23:57:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:14 smithi093 ceph-mon[111335]: pgmap v5510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:14 smithi093 ceph-mon[118902]: pgmap v5510: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/773350045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:15.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:15.358931+0000) 2023-08-27T23:57:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/773350045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/773350045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:57:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:57:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-27T23:57:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:16 smithi136 ceph-mon[113531]: pgmap v5511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:16.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:16.359204+0000) 2023-08-27T23:57:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:16 smithi093 ceph-mon[111335]: pgmap v5511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:16 smithi093 ceph-mon[118902]: pgmap v5511: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:17.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:17.359390+0000) 2023-08-27T23:57:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:57:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/660546007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:57:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/660546007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:57:17.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:17.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:17.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/660546007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:18.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:18 smithi136 ceph-mon[113531]: pgmap v5512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:18.633 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:18.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:18.359564+0000) 2023-08-27T23:57:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:18 smithi093 ceph-mon[111335]: pgmap v5512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:18 smithi093 ceph-mon[118902]: pgmap v5512: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:19.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:19.359720+0000) 2023-08-27T23:57:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:19.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:20.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:20.359871+0000) 2023-08-27T23:57:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[111335]: pgmap v5513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4027618193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:20.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:20.892Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:20.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:20.893Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:20.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[118902]: pgmap v5513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:20.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4027618193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:20.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:20.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:20 smithi136 ceph-mon[113531]: pgmap v5513: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4027618193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:21.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:21.360015+0000) 2023-08-27T23:57:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:22.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:22.360250+0000) 2023-08-27T23:57:22.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:22.367Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:22.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:22.368Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[111335]: pgmap v5514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3623684549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:57:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[118902]: pgmap v5514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3623684549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:57:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-mon[113531]: pgmap v5514: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3623684549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:57:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:57:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:57:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:23.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:23.360512+0000) 2023-08-27T23:57:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:24.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:24.360789+0000) 2023-08-27T23:57:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:24 smithi136 ceph-mon[113531]: pgmap v5515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/412117007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:24 smithi093 ceph-mon[111335]: pgmap v5515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/412117007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:24 smithi093 ceph-mon[118902]: pgmap v5515: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/412117007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:25.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:25.360993+0000) 2023-08-27T23:57:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:57:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:57:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:57:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:26.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:26.361275+0000) 2023-08-27T23:57:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:26 smithi136 ceph-mon[113531]: pgmap v5516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:26 smithi093 ceph-mon[111335]: pgmap v5516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:26 smithi093 ceph-mon[118902]: pgmap v5516: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:27.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:27.361535+0000) 2023-08-27T23:57:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1554350877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1554350877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1554350877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:28.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:28.361830+0000) 2023-08-27T23:57:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:28 smithi136 ceph-mon[113531]: pgmap v5517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:28 smithi093 ceph-mon[111335]: pgmap v5517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:28 smithi093 ceph-mon[118902]: pgmap v5517: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:29.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:29.362128+0000) 2023-08-27T23:57:30.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1814808412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1814808412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1814808412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:30.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:30.362294+0000) 2023-08-27T23:57:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:30 smithi136 ceph-mon[113531]: pgmap v5518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:30 smithi093 ceph-mon[111335]: pgmap v5518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:30 smithi093 ceph-mon[118902]: pgmap v5518: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:30.893Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:30.894Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:31.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:31.362512+0000) 2023-08-27T23:57:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/364563171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/364563171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/364563171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:32.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:32.362706+0000) 2023-08-27T23:57:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:32.368Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:32.369Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:32 smithi136 ceph-mon[113531]: pgmap v5519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:32 smithi093 ceph-mon[111335]: pgmap v5519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:32 smithi093 ceph-mon[118902]: pgmap v5519: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:33.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:33.362962+0000) 2023-08-27T23:57:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:34.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:34.363244+0000) 2023-08-27T23:57:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:34 smithi136 ceph-mon[113531]: pgmap v5520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/461872082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:34 smithi093 ceph-mon[111335]: pgmap v5520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/461872082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:34 smithi093 ceph-mon[118902]: pgmap v5520: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/461872082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:35.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:35.363443+0000) 2023-08-27T23:57:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:57:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:57:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:57:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:36.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:36.363686+0000) 2023-08-27T23:57:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:36 smithi136 ceph-mon[113531]: pgmap v5521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3715966478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:36 smithi093 ceph-mon[118902]: pgmap v5521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3715966478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:36 smithi093 ceph-mon[111335]: pgmap v5521: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3715966478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:37.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:37.363943+0000) 2023-08-27T23:57:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:38.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:38.364179+0000) 2023-08-27T23:57:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:38 smithi136 ceph-mon[113531]: pgmap v5522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:38 smithi093 ceph-mon[111335]: pgmap v5522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:38 smithi093 ceph-mon[118902]: pgmap v5522: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:39.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:39.364473+0000) 2023-08-27T23:57:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1841983831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1841983831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1841983831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:40.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:40.364713+0000) 2023-08-27T23:57:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:40 smithi136 ceph-mon[113531]: pgmap v5523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:40 smithi093 ceph-mon[111335]: pgmap v5523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:40 smithi093 ceph-mon[118902]: pgmap v5523: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:40.894Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:40.895Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:41.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:41.364958+0000) 2023-08-27T23:57:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3445287750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3445287750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3445287750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:42.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:42.365191+0000) 2023-08-27T23:57:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:42.368Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:42.369Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:42 smithi136 ceph-mon[113531]: pgmap v5524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:42 smithi093 ceph-mon[111335]: pgmap v5524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:42 smithi093 ceph-mon[118902]: pgmap v5524: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:43.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:43.365470+0000) 2023-08-27T23:57:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2225883900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2225883900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2225883900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:44.692 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:44.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:44.365695+0000) 2023-08-27T23:57:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:44 smithi136 ceph-mon[113531]: pgmap v5525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:44 smithi093 ceph-mon[111335]: pgmap v5525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:44 smithi093 ceph-mon[118902]: pgmap v5525: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:45.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:45.365889+0000) 2023-08-27T23:57:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:57:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:57:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:57:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:46.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:46.366156+0000) 2023-08-27T23:57:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:46 smithi136 ceph-mon[113531]: pgmap v5526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/906205477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:46 smithi093 ceph-mon[111335]: pgmap v5526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/906205477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:46 smithi093 ceph-mon[118902]: pgmap v5526: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/906205477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:47.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:47.366352+0000) 2023-08-27T23:57:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:48.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:48.366619+0000) 2023-08-27T23:57:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:48 smithi136 ceph-mon[113531]: pgmap v5527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2550417237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:48 smithi093 ceph-mon[111335]: pgmap v5527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2550417237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:48 smithi093 ceph-mon[118902]: pgmap v5527: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2550417237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:49.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:49.366848+0000) 2023-08-27T23:57:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:50.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:50.367075+0000) 2023-08-27T23:57:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:50 smithi136 ceph-mon[113531]: pgmap v5528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/867781789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:50 smithi093 ceph-mon[111335]: pgmap v5528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/867781789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:50 smithi093 ceph-mon[118902]: pgmap v5528: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:51.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/867781789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:50.894Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:50.895Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:51.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:51.367321+0000) 2023-08-27T23:57:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:52.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:52.367578+0000) 2023-08-27T23:57:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:52.369Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:57:52.370Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:57:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:52 smithi136 ceph-mon[113531]: pgmap v5529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:52 smithi093 ceph-mon[111335]: pgmap v5529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:52 smithi093 ceph-mon[118902]: pgmap v5529: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:53.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:53.367849+0000) 2023-08-27T23:57:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4090195126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4090195126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4090195126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:57:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:54.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:54.368134+0000) 2023-08-27T23:57:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:54 smithi136 ceph-mon[113531]: pgmap v5530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:54 smithi093 ceph-mon[111335]: pgmap v5530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:54 smithi093 ceph-mon[118902]: pgmap v5530: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:55.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:55.368307+0000) 2023-08-27T23:57:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:57:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:57:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:57:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:56 smithi136 ceph-mon[113531]: pgmap v5531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2797190157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:56.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:56.368493+0000) 2023-08-27T23:57:56.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:56 smithi093 ceph-mon[111335]: pgmap v5531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:56.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2797190157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:56 smithi093 ceph-mon[118902]: pgmap v5531: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2797190157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:57.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:57.368672+0000) 2023-08-27T23:57:58.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:58 smithi136 ceph-mon[113531]: pgmap v5532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:58.702 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:57:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1101291098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:58.702 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:58.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:58.368911+0000) 2023-08-27T23:57:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:58 smithi093 ceph-mon[111335]: pgmap v5532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:57:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1101291098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:58 smithi093 ceph-mon[118902]: pgmap v5532: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:57:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:57:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1101291098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:57:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:57:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:57:59.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:57:59.369179+0000) 2023-08-27T23:58:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:00 smithi136 ceph-mon[113531]: pgmap v5533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1710937817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:00.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:00.369371+0000) 2023-08-27T23:58:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:00 smithi093 ceph-mon[111335]: pgmap v5533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1710937817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:00 smithi093 ceph-mon[118902]: pgmap v5533: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1710937817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:00.895Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:00.896Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:01.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:01.369598+0000) 2023-08-27T23:58:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:02 smithi136 ceph-mon[113531]: pgmap v5534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:02.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:02.369843+0000) 2023-08-27T23:58:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:02 smithi093 ceph-mon[111335]: pgmap v5534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:02 smithi093 ceph-mon[118902]: pgmap v5534: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:02.370Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:02.371Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/180816345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:03.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:03.370127+0000) 2023-08-27T23:58:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/180816345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/180816345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:04 smithi136 ceph-mon[113531]: pgmap v5535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:04.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:04.370397+0000) 2023-08-27T23:58:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:04 smithi093 ceph-mon[111335]: pgmap v5535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:04 smithi093 ceph-mon[118902]: pgmap v5535: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2943199231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:05.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:05.370652+0000) 2023-08-27T23:58:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2943199231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2943199231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:58:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:58:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:58:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:06 smithi136 ceph-mon[113531]: pgmap v5536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:06.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:06.370938+0000) 2023-08-27T23:58:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:06 smithi093 ceph-mon[111335]: pgmap v5536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:06 smithi093 ceph-mon[118902]: pgmap v5536: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:07.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:07.371186+0000) 2023-08-27T23:58:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:08 smithi136 ceph-mon[113531]: pgmap v5537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:08.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2680026498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:08.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:08.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:08.371445+0000) 2023-08-27T23:58:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:08 smithi093 ceph-mon[111335]: pgmap v5537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2680026498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:08 smithi093 ceph-mon[118902]: pgmap v5537: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:08.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2680026498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:09.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:09.371676+0000) 2023-08-27T23:58:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:10 smithi136 ceph-mon[113531]: pgmap v5538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3093525490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:10.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:10.371889+0000) 2023-08-27T23:58:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:10 smithi093 ceph-mon[111335]: pgmap v5538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3093525490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:10 smithi093 ceph-mon[118902]: pgmap v5538: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3093525490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:10.896Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:10.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:11.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:11.372146+0000) 2023-08-27T23:58:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:12 smithi136 ceph-mon[113531]: pgmap v5539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/997048982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:12.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:12.372506+0000) 2023-08-27T23:58:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:12 smithi093 ceph-mon[111335]: pgmap v5539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:12.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/997048982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:12 smithi093 ceph-mon[118902]: pgmap v5539: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/997048982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:12.370Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:12.371Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:13.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:13.372731+0000) 2023-08-27T23:58:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:14 smithi136 ceph-mon[113531]: pgmap v5540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:14.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:14.372976+0000) 2023-08-27T23:58:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:14 smithi093 ceph-mon[111335]: pgmap v5540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:14 smithi093 ceph-mon[118902]: pgmap v5540: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1306641199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:15.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:15.373122+0000) 2023-08-27T23:58:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1306641199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:58:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:58:15] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-27T23:58:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1306641199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:16 smithi136 ceph-mon[113531]: pgmap v5541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:16.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:16.373419+0000) 2023-08-27T23:58:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:16 smithi093 ceph-mon[111335]: pgmap v5541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:16 smithi093 ceph-mon[118902]: pgmap v5541: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2009888407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:17.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:17.373665+0000) 2023-08-27T23:58:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2009888407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2009888407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:18 smithi136 ceph-mon[113531]: pgmap v5542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:18.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:18.373928+0000) 2023-08-27T23:58:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:18 smithi093 ceph-mon[111335]: pgmap v5542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:18 smithi093 ceph-mon[118902]: pgmap v5542: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:19.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/759093146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:19.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:19.374143+0000) 2023-08-27T23:58:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/759093146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/759093146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:20 smithi136 ceph-mon[113531]: pgmap v5543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:20.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:20.374367+0000) 2023-08-27T23:58:20.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:20 smithi093 ceph-mon[111335]: pgmap v5543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:20.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:20 smithi093 ceph-mon[118902]: pgmap v5543: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:20.896Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:20.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:21.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:21.374623+0000) 2023-08-27T23:58:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:22 smithi136 ceph-mon[113531]: pgmap v5544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1349160565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:22.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:22.374881+0000) 2023-08-27T23:58:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:22 smithi093 ceph-mon[111335]: pgmap v5544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1349160565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:22 smithi093 ceph-mon[118902]: pgmap v5544: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1349160565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:22.371Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:22.372Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:23.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:58:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:23.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:23.375091+0000) 2023-08-27T23:58:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:58:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:58:24.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:24 smithi136 ceph-mon[113531]: pgmap v5545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:24.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:24.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1301552505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:24.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:24.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:24.671 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:24.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:24.375288+0000) 2023-08-27T23:58:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[111335]: pgmap v5545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1301552505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[118902]: pgmap v5545: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1301552505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:25.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:25.375484+0000) 2023-08-27T23:58:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:58:25] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:58:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:25 smithi136 ceph-mon[113531]: pgmap v5546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:58:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:58:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[111335]: pgmap v5546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:58:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:58:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[118902]: pgmap v5546: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:58:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:58:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:58:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:26.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:26.375761+0000) 2023-08-27T23:58:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4117935680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4117935680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4117935680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:27.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:27.376006+0000) 2023-08-27T23:58:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:27 smithi136 ceph-mon[113531]: pgmap v5547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:27 smithi093 ceph-mon[111335]: pgmap v5547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:27 smithi093 ceph-mon[118902]: pgmap v5547: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:28.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:28.376186+0000) 2023-08-27T23:58:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2428500184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2428500184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2428500184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:29.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:29.376449+0000) 2023-08-27T23:58:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:29 smithi093 ceph-mon[111335]: pgmap v5548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:29 smithi093 ceph-mon[118902]: pgmap v5548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:30.377 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:29 smithi136 ceph-mon[113531]: pgmap v5548: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:30.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:30.376696+0000) 2023-08-27T23:58:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:30.897Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:30.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:31.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4052710356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:31.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:31.376912+0000) 2023-08-27T23:58:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4052710356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4052710356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:32 smithi136 ceph-mon[113531]: pgmap v5549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:32.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:32.377088+0000) 2023-08-27T23:58:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:32 smithi093 ceph-mon[111335]: pgmap v5549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:32 smithi093 ceph-mon[118902]: pgmap v5549: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:32.372Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:32.373Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:33.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:33.377289+0000) 2023-08-27T23:58:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:34 smithi136 ceph-mon[113531]: pgmap v5550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/956790113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:34.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:34.377551+0000) 2023-08-27T23:58:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:34 smithi093 ceph-mon[111335]: pgmap v5550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/956790113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:34 smithi093 ceph-mon[118902]: pgmap v5550: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/956790113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:35.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:35.377854+0000) 2023-08-27T23:58:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:58:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:58:35] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-27T23:58:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:36.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:36.378185+0000) 2023-08-27T23:58:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:36 smithi136 ceph-mon[113531]: pgmap v5551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/994709363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:36 smithi093 ceph-mon[111335]: pgmap v5551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/994709363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:36 smithi093 ceph-mon[118902]: pgmap v5551: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/994709363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:37.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:37.378395+0000) 2023-08-27T23:58:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:38 smithi136 ceph-mon[113531]: pgmap v5552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/284054254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:38.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:38.378575+0000) 2023-08-27T23:58:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:38 smithi093 ceph-mon[111335]: pgmap v5552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/284054254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:38 smithi093 ceph-mon[118902]: pgmap v5552: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/284054254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:39.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:39.378801+0000) 2023-08-27T23:58:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:40 smithi136 ceph-mon[113531]: pgmap v5553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:40.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:40.379060+0000) 2023-08-27T23:58:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:40 smithi093 ceph-mon[111335]: pgmap v5553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:40 smithi093 ceph-mon[118902]: pgmap v5553: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:40.897Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:40.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:41.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1833350781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:41.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:41.379294+0000) 2023-08-27T23:58:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1833350781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1833350781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:42 smithi136 ceph-mon[113531]: pgmap v5554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:42.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:42.379539+0000) 2023-08-27T23:58:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:42 smithi093 ceph-mon[111335]: pgmap v5554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:42 smithi093 ceph-mon[118902]: pgmap v5554: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:42.373Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:42.374Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4282085797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:43.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:43.379765+0000) 2023-08-27T23:58:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4282085797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4282085797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:44 smithi136 ceph-mon[113531]: pgmap v5555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:44.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:44.380023+0000) 2023-08-27T23:58:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:44 smithi093 ceph-mon[111335]: pgmap v5555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:44 smithi093 ceph-mon[118902]: pgmap v5555: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:45.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:45.380259+0000) 2023-08-27T23:58:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:58:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:58:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:58:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:46 smithi136 ceph-mon[113531]: pgmap v5556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2383752013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:46.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:46.380524+0000) 2023-08-27T23:58:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:46 smithi093 ceph-mon[111335]: pgmap v5556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2383752013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:46 smithi093 ceph-mon[118902]: pgmap v5556: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2383752013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:47.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:47.380777+0000) 2023-08-27T23:58:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:48 smithi136 ceph-mon[113531]: pgmap v5557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3921992510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:48.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:48.381051+0000) 2023-08-27T23:58:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:48 smithi093 ceph-mon[111335]: pgmap v5557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3921992510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:48 smithi093 ceph-mon[118902]: pgmap v5557: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3921992510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:49.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:49.381301+0000) 2023-08-27T23:58:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:50 smithi136 ceph-mon[113531]: pgmap v5558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2053561210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:50.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:50.381475+0000) 2023-08-27T23:58:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:50 smithi093 ceph-mon[111335]: pgmap v5558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2053561210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:50 smithi093 ceph-mon[118902]: pgmap v5558: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2053561210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:50.898Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:50.899Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:51.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:51.381720+0000) 2023-08-27T23:58:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:52 smithi136 ceph-mon[113531]: pgmap v5559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:52.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:52.381959+0000) 2023-08-27T23:58:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:52 smithi093 ceph-mon[111335]: pgmap v5559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:52 smithi093 ceph-mon[118902]: pgmap v5559: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:52.374Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:58:52.374Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:58:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3275747320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:53.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:53.382195+0000) 2023-08-27T23:58:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3275747320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3275747320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:54 smithi136 ceph-mon[113531]: pgmap v5560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:54.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:54.382468+0000) 2023-08-27T23:58:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:54 smithi093 ceph-mon[111335]: pgmap v5560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:54 smithi093 ceph-mon[118902]: pgmap v5560: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:58:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/6446852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:55.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:55.382738+0000) 2023-08-27T23:58:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/6446852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:58:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:58:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:58:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/6446852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:56 smithi136 ceph-mon[113531]: pgmap v5561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:56.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:56.383012+0000) 2023-08-27T23:58:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:56 smithi093 ceph-mon[111335]: pgmap v5561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:56 smithi093 ceph-mon[118902]: pgmap v5561: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:57.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:57.383269+0000) 2023-08-27T23:58:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:58 smithi136 ceph-mon[113531]: pgmap v5562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:58:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3661479018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:58.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:58.383499+0000) 2023-08-27T23:58:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:58 smithi093 ceph-mon[111335]: pgmap v5562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:58:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3661479018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:58 smithi093 ceph-mon[118902]: pgmap v5562: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:58:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:58:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3661479018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:58:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:58:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:58:59.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:58:59.383754+0000) 2023-08-27T23:59:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:00 smithi136 ceph-mon[113531]: pgmap v5563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3491429136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:00.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:00.383943+0000) 2023-08-27T23:59:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:00 smithi093 ceph-mon[111335]: pgmap v5563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3491429136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:00 smithi093 ceph-mon[118902]: pgmap v5563: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3491429136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:00.899Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:00.900Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:01.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:01.384179+0000) 2023-08-27T23:59:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:02 smithi136 ceph-mon[113531]: pgmap v5564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3480702529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:02.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:02.384408+0000) 2023-08-27T23:59:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:02 smithi093 ceph-mon[111335]: pgmap v5564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3480702529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:02 smithi093 ceph-mon[118902]: pgmap v5564: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3480702529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:02.374Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:02.375Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:03.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:03.384618+0000) 2023-08-27T23:59:04.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:04 smithi136 ceph-mon[113531]: pgmap v5565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:04.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:04.384858+0000) 2023-08-27T23:59:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:04 smithi093 ceph-mon[111335]: pgmap v5565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:04 smithi093 ceph-mon[118902]: pgmap v5565: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4120900033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:05.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:05.385143+0000) 2023-08-27T23:59:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4120900033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4120900033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:59:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:59:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-27T23:59:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:06 smithi136 ceph-mon[113531]: pgmap v5566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:06.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:06.385412+0000) 2023-08-27T23:59:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:06 smithi093 ceph-mon[111335]: pgmap v5566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:06 smithi093 ceph-mon[118902]: pgmap v5566: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2681724294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:07.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:07.385627+0000) 2023-08-27T23:59:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2681724294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2681724294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:08 smithi136 ceph-mon[113531]: pgmap v5567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:08.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:08.385896+0000) 2023-08-27T23:59:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:08 smithi093 ceph-mon[111335]: pgmap v5567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:08 smithi093 ceph-mon[118902]: pgmap v5567: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1787049940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:09.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:09.386149+0000) 2023-08-27T23:59:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1787049940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1787049940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:10 smithi136 ceph-mon[113531]: pgmap v5568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:10.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:10.386370+0000) 2023-08-27T23:59:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:10 smithi093 ceph-mon[111335]: pgmap v5568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:10 smithi093 ceph-mon[118902]: pgmap v5568: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:10.900Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:10.901Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:11.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:11.386621+0000) 2023-08-27T23:59:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:12 smithi136 ceph-mon[113531]: pgmap v5569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/416459977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:12.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:12.386865+0000) 2023-08-27T23:59:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:12 smithi093 ceph-mon[111335]: pgmap v5569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/416459977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:12 smithi093 ceph-mon[118902]: pgmap v5569: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/416459977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:12.375Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:12.376Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:13.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:13.387117+0000) 2023-08-27T23:59:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:14 smithi136 ceph-mon[113531]: pgmap v5570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3986984758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:14.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:14.387382+0000) 2023-08-27T23:59:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:14 smithi093 ceph-mon[111335]: pgmap v5570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3986984758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:14 smithi093 ceph-mon[118902]: pgmap v5570: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3986984758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:15.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:15.387593+0000) 2023-08-27T23:59:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:59:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:59:15] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-27T23:59:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:16 smithi136 ceph-mon[113531]: pgmap v5571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:16.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:16.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:16.387852+0000) 2023-08-27T23:59:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:16 smithi093 ceph-mon[111335]: pgmap v5571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:16 smithi093 ceph-mon[118902]: pgmap v5571: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1936740166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:17.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:17.388085+0000) 2023-08-27T23:59:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1936740166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1936740166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:18 smithi136 ceph-mon[113531]: pgmap v5572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:18.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:18.388366+0000) 2023-08-27T23:59:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:18 smithi093 ceph-mon[111335]: pgmap v5572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:18 smithi093 ceph-mon[118902]: pgmap v5572: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2878083074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:19.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:19.388596+0000) 2023-08-27T23:59:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2878083074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2878083074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:20.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:20 smithi136 ceph-mon[113531]: pgmap v5573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:20.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:20.388856+0000) 2023-08-27T23:59:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:20 smithi093 ceph-mon[111335]: pgmap v5573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:20 smithi093 ceph-mon[118902]: pgmap v5573: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:20.901Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:20.902Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4143098018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:21.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:21.389101+0000) 2023-08-27T23:59:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4143098018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4143098018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:22 smithi136 ceph-mon[113531]: pgmap v5574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:22.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:22.389344+0000) 2023-08-27T23:59:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:22 smithi093 ceph-mon[111335]: pgmap v5574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:22 smithi093 ceph-mon[118902]: pgmap v5574: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:22.376Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:22.377Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:23.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:23.389559+0000) 2023-08-27T23:59:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:24 smithi136 ceph-mon[113531]: pgmap v5575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/694967012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:24.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:24.389811+0000) 2023-08-27T23:59:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:24 smithi093 ceph-mon[111335]: pgmap v5575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/694967012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:24 smithi093 ceph-mon[118902]: pgmap v5575: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/694967012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:25.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:25.390062+0000) 2023-08-27T23:59:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:59:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:59:25] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:59:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:26.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:26.390219+0000) 2023-08-27T23:59:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:26 smithi136 ceph-mon[113531]: pgmap v5576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:59:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2459313154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:26 smithi093 ceph-mon[111335]: pgmap v5576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:59:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2459313154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:26.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:26 smithi093 ceph-mon[118902]: pgmap v5576: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:26.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-27T23:59:26.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2459313154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:27.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:27.390828+0000) 2023-08-27T23:59:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[111335]: pgmap v5577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[118902]: pgmap v5577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:28.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:28.391100+0000) 2023-08-27T23:59:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:28 smithi136 ceph-mon[113531]: pgmap v5577: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1141268084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:29.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:59:29.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:59:29.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:29.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1141268084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-27T23:59:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-27T23:59:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-27T23:59:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1141268084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:29.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:29.391355+0000) 2023-08-27T23:59:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:30 smithi136 ceph-mon[113531]: pgmap v5578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:30.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:30.391612+0000) 2023-08-27T23:59:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:30 smithi093 ceph-mon[111335]: pgmap v5578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:30 smithi093 ceph-mon[118902]: pgmap v5578: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:30.901Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:30.902Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:31.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4027256152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:31.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:31.391858+0000) 2023-08-27T23:59:31.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4027256152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4027256152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:32 smithi136 ceph-mon[113531]: pgmap v5579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:32.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:32.392125+0000) 2023-08-27T23:59:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:32 smithi093 ceph-mon[111335]: pgmap v5579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:32 smithi093 ceph-mon[118902]: pgmap v5579: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:32.377Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:32.378Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4106533800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:33.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:33.392415+0000) 2023-08-27T23:59:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4106533800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4106533800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:34 smithi136 ceph-mon[113531]: pgmap v5580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:34.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:34.392675+0000) 2023-08-27T23:59:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:34 smithi093 ceph-mon[111335]: pgmap v5580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:34 smithi093 ceph-mon[118902]: pgmap v5580: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:35.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:35.392928+0000) 2023-08-27T23:59:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:59:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:59:35] "GET /metrics HTTP/1.1" 200 239919 "" "Prometheus/2.43.0" 2023-08-27T23:59:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:36 smithi136 ceph-mon[113531]: pgmap v5581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3183809531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:36.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:36.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:36.393223+0000) 2023-08-27T23:59:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:36 smithi093 ceph-mon[111335]: pgmap v5581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3183809531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:36 smithi093 ceph-mon[118902]: pgmap v5581: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3183809531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:37.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:37.393432+0000) 2023-08-27T23:59:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:38 smithi136 ceph-mon[113531]: pgmap v5582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3889128568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:38.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:38.393621+0000) 2023-08-27T23:59:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:38 smithi093 ceph-mon[111335]: pgmap v5582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3889128568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:38 smithi093 ceph-mon[118902]: pgmap v5582: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3889128568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:39.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:39.393841+0000) 2023-08-27T23:59:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:40 smithi136 ceph-mon[113531]: pgmap v5583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/775336066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:40.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:40.394141+0000) 2023-08-27T23:59:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:40 smithi093 ceph-mon[111335]: pgmap v5583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/775336066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:40 smithi093 ceph-mon[118902]: pgmap v5583: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/775336066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:40.902Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:40.903Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:41.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:41.394416+0000) 2023-08-27T23:59:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:42 smithi136 ceph-mon[113531]: pgmap v5584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:42.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:42.394729+0000) 2023-08-27T23:59:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:42 smithi093 ceph-mon[111335]: pgmap v5584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:42 smithi093 ceph-mon[118902]: pgmap v5584: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:42.378Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:42.379Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:43.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4214278089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:43.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:43.394960+0000) 2023-08-27T23:59:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4214278089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4214278089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:44 smithi136 ceph-mon[113531]: pgmap v5585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:44.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:44.395214+0000) 2023-08-27T23:59:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:44 smithi093 ceph-mon[111335]: pgmap v5585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:44 smithi093 ceph-mon[118902]: pgmap v5585: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:45.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/10497830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:45.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:45.395398+0000) 2023-08-27T23:59:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/10497830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:59:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:59:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-27T23:59:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/10497830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:46 smithi136 ceph-mon[113531]: pgmap v5586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:46.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:46.395646+0000) 2023-08-27T23:59:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:46 smithi093 ceph-mon[111335]: pgmap v5586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:46 smithi093 ceph-mon[118902]: pgmap v5586: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:47.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:47.395827+0000) 2023-08-27T23:59:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:48 smithi136 ceph-mon[113531]: pgmap v5587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/247884003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:48.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:48.396114+0000) 2023-08-27T23:59:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:48 smithi093 ceph-mon[111335]: pgmap v5587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/247884003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:48 smithi093 ceph-mon[118902]: pgmap v5587: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/247884003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:49.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:49.396393+0000) 2023-08-27T23:59:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:50 smithi136 ceph-mon[113531]: pgmap v5588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/802620651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:50.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:50.396637+0000) 2023-08-27T23:59:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:50 smithi093 ceph-mon[111335]: pgmap v5588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/802620651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:50 smithi093 ceph-mon[118902]: pgmap v5588: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/802620651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:50.903Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:50.904Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:51.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:51.396855+0000) 2023-08-27T23:59:52.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:52 smithi136 ceph-mon[113531]: pgmap v5589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1363334470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:52.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:52.397141+0000) 2023-08-27T23:59:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:52 smithi093 ceph-mon[118902]: pgmap v5589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1363334470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:52 smithi093 ceph-mon[111335]: pgmap v5589: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1363334470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:52.379Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 27 23:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-27T23:59:52.380Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-27T23:59:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:53.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:53.397434+0000) 2023-08-27T23:59:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:54 smithi136 ceph-mon[113531]: pgmap v5590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:54.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:54.397659+0000) 2023-08-27T23:59:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:54 smithi093 ceph-mon[111335]: pgmap v5590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:54 smithi093 ceph-mon[118902]: pgmap v5590: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-27T23:59:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1675246398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:55.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:55.397924+0000) 2023-08-27T23:59:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1675246398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1675246398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 27 23:59:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [27/Aug/2023:23:59:55] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-27T23:59:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:56 smithi136 ceph-mon[113531]: pgmap v5591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:56.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:56.398206+0000) 2023-08-27T23:59:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:56 smithi093 ceph-mon[111335]: pgmap v5591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:56 smithi093 ceph-mon[118902]: pgmap v5591: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:57.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/860691059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:57.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:57.398409+0000) 2023-08-27T23:59:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/860691059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/860691059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:58.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:58 smithi136 ceph-mon[113531]: pgmap v5592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:58.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:58.398681+0000) 2023-08-27T23:59:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:58 smithi093 ceph-mon[111335]: pgmap v5592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:58 smithi093 ceph-mon[118902]: pgmap v5592: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-27T23:59:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 27 23:59:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4055288803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 27 23:59:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-27T23:59:59.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T22:59:59.398959+0000) 2023-08-27T23:59:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 27 23:59:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4055288803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-27T23:59:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 27 23:59:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4055288803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:00 smithi136 ceph-mon[113531]: pgmap v5593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:00:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:00.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:00.399243+0000) 2023-08-28T00:00:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:00 smithi093 ceph-mon[111335]: pgmap v5593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:00:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:00 smithi093 ceph-mon[118902]: pgmap v5593: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:00:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:00.904Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:00.905Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:01.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:01.399529+0000) 2023-08-28T00:00:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:02 smithi136 ceph-mon[113531]: pgmap v5594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/380399121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:02.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:02.399823+0000) 2023-08-28T00:00:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:02 smithi093 ceph-mon[111335]: pgmap v5594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/380399121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:02 smithi093 ceph-mon[118902]: pgmap v5594: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/380399121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:02.379Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:02.381Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:03.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:03.400151+0000) 2023-08-28T00:00:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:04 smithi136 ceph-mon[113531]: pgmap v5595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1113151363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:04.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:04.400443+0000) 2023-08-28T00:00:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:04 smithi093 ceph-mon[111335]: pgmap v5595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1113151363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:04 smithi093 ceph-mon[118902]: pgmap v5595: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1113151363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:05.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:05.400702+0000) 2023-08-28T00:00:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:00:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:00:05] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:00:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:06 smithi136 ceph-mon[113531]: pgmap v5596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:06.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:06.400990+0000) 2023-08-28T00:00:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:06 smithi093 ceph-mon[111335]: pgmap v5596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:06 smithi093 ceph-mon[118902]: pgmap v5596: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3434403127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:07.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:07.401109+0000) 2023-08-28T00:00:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3434403127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3434403127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:08 smithi136 ceph-mon[113531]: pgmap v5597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:08.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:08.401364+0000) 2023-08-28T00:00:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:08 smithi093 ceph-mon[111335]: pgmap v5597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:08 smithi093 ceph-mon[118902]: pgmap v5597: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2082920159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:09.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:09.401655+0000) 2023-08-28T00:00:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2082920159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2082920159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:10 smithi136 ceph-mon[113531]: pgmap v5598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:10.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:10.401899+0000) 2023-08-28T00:00:10.905 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:10 smithi093 ceph-mon[111335]: pgmap v5598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:10.906 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:10 smithi093 ceph-mon[118902]: pgmap v5598: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:10.904Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:10.906Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2634925108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:11.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:11.402174+0000) 2023-08-28T00:00:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2634925108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2634925108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:12 smithi136 ceph-mon[113531]: pgmap v5599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:12.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:12.402395+0000) 2023-08-28T00:00:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:12 smithi093 ceph-mon[111335]: pgmap v5599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:12 smithi093 ceph-mon[118902]: pgmap v5599: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:12.380Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:12.381Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:13.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:13.402664+0000) 2023-08-28T00:00:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:14 smithi136 ceph-mon[113531]: pgmap v5600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3114100782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:14.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:14.402954+0000) 2023-08-28T00:00:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:14 smithi093 ceph-mon[111335]: pgmap v5600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3114100782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:14 smithi093 ceph-mon[118902]: pgmap v5600: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3114100782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:15.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:15.403214+0000) 2023-08-28T00:00:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:00:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:00:15] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-28T00:00:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:16 smithi136 ceph-mon[113531]: pgmap v5601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3845851188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:16.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:16.403494+0000) 2023-08-28T00:00:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:16 smithi093 ceph-mon[111335]: pgmap v5601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3845851188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:16 smithi093 ceph-mon[118902]: pgmap v5601: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3845851188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:17.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:17.403761+0000) 2023-08-28T00:00:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:18 smithi136 ceph-mon[113531]: pgmap v5602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4252992924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:18.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:18.404007+0000) 2023-08-28T00:00:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:18 smithi093 ceph-mon[111335]: pgmap v5602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4252992924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:18 smithi093 ceph-mon[118902]: pgmap v5602: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4252992924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:19.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:19.404206+0000) 2023-08-28T00:00:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:20 smithi136 ceph-mon[113531]: pgmap v5603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:20.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:20.404400+0000) 2023-08-28T00:00:20.906 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:20 smithi093 ceph-mon[111335]: pgmap v5603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:20.907 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:20 smithi093 ceph-mon[118902]: pgmap v5603: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:20.905Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:20.906Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2826858639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:21.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:21.404666+0000) 2023-08-28T00:00:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2826858639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2826858639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:22 smithi136 ceph-mon[113531]: pgmap v5604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:22.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:22.404896+0000) 2023-08-28T00:00:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:22 smithi093 ceph-mon[111335]: pgmap v5604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:22 smithi093 ceph-mon[118902]: pgmap v5604: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:22.381Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:22.381Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:23.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3872823490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:23.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:23.405172+0000) 2023-08-28T00:00:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3872823490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3872823490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:24 smithi136 ceph-mon[113531]: pgmap v5605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:24.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:24.405368+0000) 2023-08-28T00:00:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:24 smithi093 ceph-mon[111335]: pgmap v5605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:24 smithi093 ceph-mon[118902]: pgmap v5605: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:25.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:25.405626+0000) 2023-08-28T00:00:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:00:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:00:25] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:00:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:26 smithi136 ceph-mon[113531]: pgmap v5606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2707644349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:26.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:26.405953+0000) 2023-08-28T00:00:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:26 smithi093 ceph-mon[111335]: pgmap v5606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2707644349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:26 smithi093 ceph-mon[118902]: pgmap v5606: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2707644349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:27.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:27.406224+0000) 2023-08-28T00:00:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:28 smithi136 ceph-mon[113531]: pgmap v5607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/299724722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:00:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:28.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:28.406509+0000) 2023-08-28T00:00:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:28 smithi093 ceph-mon[111335]: pgmap v5607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/299724722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:00:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:28 smithi093 ceph-mon[118902]: pgmap v5607: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/299724722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:00:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:29.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:29.406801+0000) 2023-08-28T00:00:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:30.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:30.406997+0000) 2023-08-28T00:00:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:30 smithi136 ceph-mon[113531]: pgmap v5608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:30.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:30.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2821839064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:30.909 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[111335]: pgmap v5608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:30.909 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:30.909 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:30.909 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2821839064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:30.909 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[118902]: pgmap v5608: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:30.909 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:30.909 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:30.910 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2821839064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:30.906Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:30.908Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:31.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:31.407220+0000) 2023-08-28T00:00:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[111335]: pgmap v5609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:00:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:00:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[118902]: pgmap v5609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:00:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:00:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:31 smithi136 ceph-mon[113531]: pgmap v5609: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:32.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:00:32.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:00:32.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:00:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:32.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:32.407457+0000) 2023-08-28T00:00:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:32.381Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:32.382Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4108067744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4108067744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:33.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4108067744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:33.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:33.407723+0000) 2023-08-28T00:00:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:33 smithi093 ceph-mon[111335]: pgmap v5610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:33 smithi093 ceph-mon[118902]: pgmap v5610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:34.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:33 smithi136 ceph-mon[113531]: pgmap v5610: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:34.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:34.407978+0000) 2023-08-28T00:00:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:35.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:35.408259+0000) 2023-08-28T00:00:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:00:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:00:35] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:00:36.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:36 smithi136 ceph-mon[113531]: pgmap v5611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:36.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3334348339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:36.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:36.408553+0000) 2023-08-28T00:00:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:36 smithi093 ceph-mon[111335]: pgmap v5611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3334348339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:36 smithi093 ceph-mon[118902]: pgmap v5611: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:36.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3334348339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:37.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:37.408790+0000) 2023-08-28T00:00:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:38 smithi136 ceph-mon[113531]: pgmap v5612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2558377912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:38.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:38.409098+0000) 2023-08-28T00:00:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:38 smithi093 ceph-mon[118902]: pgmap v5612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2558377912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:38.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:38 smithi093 ceph-mon[111335]: pgmap v5612: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:38.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2558377912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:39.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:39.409302+0000) 2023-08-28T00:00:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:40.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:40 smithi136 ceph-mon[113531]: pgmap v5613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:40.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1317089990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:40.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:40.409546+0000) 2023-08-28T00:00:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:40 smithi093 ceph-mon[111335]: pgmap v5613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1317089990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:40.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:40 smithi093 ceph-mon[118902]: pgmap v5613: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:40.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1317089990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:40.908Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:40.909Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:41.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:41.409746+0000) 2023-08-28T00:00:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:42 smithi136 ceph-mon[113531]: pgmap v5614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3350951839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:42.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:42.409997+0000) 2023-08-28T00:00:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:42 smithi093 ceph-mon[111335]: pgmap v5614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3350951839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:42 smithi093 ceph-mon[118902]: pgmap v5614: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3350951839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:42.382Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:42.383Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:43.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:43.410232+0000) 2023-08-28T00:00:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:44 smithi136 ceph-mon[113531]: pgmap v5615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:44.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:44.410455+0000) 2023-08-28T00:00:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:44 smithi093 ceph-mon[111335]: pgmap v5615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:44 smithi093 ceph-mon[118902]: pgmap v5615: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:45.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/642489099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:45.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:45.410705+0000) 2023-08-28T00:00:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/642489099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:00:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:00:45] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:00:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/642489099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:46 smithi136 ceph-mon[113531]: pgmap v5616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:46.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:46.410935+0000) 2023-08-28T00:00:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:46 smithi093 ceph-mon[111335]: pgmap v5616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:46 smithi093 ceph-mon[118902]: pgmap v5616: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:47.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3409260783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:47.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:47.411100+0000) 2023-08-28T00:00:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3409260783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3409260783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:48 smithi136 ceph-mon[113531]: pgmap v5617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:48.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:48.411338+0000) 2023-08-28T00:00:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:48 smithi093 ceph-mon[118902]: pgmap v5617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:48 smithi093 ceph-mon[111335]: pgmap v5617: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:49.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:49.411587+0000) 2023-08-28T00:00:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:50 smithi136 ceph-mon[113531]: pgmap v5618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2939835589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:50.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:50.411840+0000) 2023-08-28T00:00:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:50 smithi093 ceph-mon[118902]: pgmap v5618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2939835589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:50 smithi093 ceph-mon[111335]: pgmap v5618: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2939835589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:50.909Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:50.910Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:51.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:51.412131+0000) 2023-08-28T00:00:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:52 smithi136 ceph-mon[113531]: pgmap v5619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1667884011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:52.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:52.412394+0000) 2023-08-28T00:00:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:52 smithi093 ceph-mon[111335]: pgmap v5619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1667884011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:52 smithi093 ceph-mon[118902]: pgmap v5619: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1667884011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:52.383Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:00:52.383Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:00:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:53.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:53.412647+0000) 2023-08-28T00:00:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:54 smithi136 ceph-mon[113531]: pgmap v5620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2214757743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:54.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:54.412884+0000) 2023-08-28T00:00:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:54 smithi093 ceph-mon[111335]: pgmap v5620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2214757743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:54 smithi093 ceph-mon[118902]: pgmap v5620: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:00:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2214757743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:55.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:55.413125+0000) 2023-08-28T00:00:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:00:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:00:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:00:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:56 smithi136 ceph-mon[113531]: pgmap v5621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:56.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:56.413371+0000) 2023-08-28T00:00:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:56 smithi093 ceph-mon[111335]: pgmap v5621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:56 smithi093 ceph-mon[118902]: pgmap v5621: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:57.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2274945868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:57.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:57.413609+0000) 2023-08-28T00:00:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2274945868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2274945868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:58 smithi136 ceph-mon[113531]: pgmap v5622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:58.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:58.413869+0000) 2023-08-28T00:00:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:58 smithi093 ceph-mon[111335]: pgmap v5622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:58 smithi093 ceph-mon[118902]: pgmap v5622: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:00:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:00:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/796006031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:00:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:00:59.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:00:59.414150+0000) 2023-08-28T00:00:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:00:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/796006031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:00:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:00:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/796006031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:00.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:00 smithi136 ceph-mon[113531]: pgmap v5623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:00.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:00.414412+0000) 2023-08-28T00:01:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:00 smithi093 ceph-mon[118902]: pgmap v5623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:00 smithi093 ceph-mon[111335]: pgmap v5623: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:01.173 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:00.910Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:01.173 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:00.911Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:01.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1326531423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:01.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:01.414644+0000) 2023-08-28T00:01:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1326531423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1326531423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:02 smithi136 ceph-mon[113531]: pgmap v5624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:02.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:02.414913+0000) 2023-08-28T00:01:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:02 smithi093 ceph-mon[111335]: pgmap v5624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:02 smithi093 ceph-mon[118902]: pgmap v5624: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:02.383Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:02.384Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:03.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:03.415154+0000) 2023-08-28T00:01:04.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:04 smithi136 ceph-mon[113531]: pgmap v5625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:04.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3684544074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:04.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:04.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:04.415436+0000) 2023-08-28T00:01:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:04 smithi093 ceph-mon[111335]: pgmap v5625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3684544074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:04 smithi093 ceph-mon[118902]: pgmap v5625: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3684544074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:05.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:05.415702+0000) 2023-08-28T00:01:05.690 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 00:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T00:01:05.318576051Z level=info msg="Completed cleanup jobs" duration=13.012841ms 2023-08-28T00:01:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:01:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:01:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:01:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:06 smithi136 ceph-mon[113531]: pgmap v5626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2424488809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:06.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:06.415928+0000) 2023-08-28T00:01:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:06 smithi093 ceph-mon[111335]: pgmap v5626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2424488809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:06 smithi093 ceph-mon[118902]: pgmap v5626: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2424488809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:07.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:07.416152+0000) 2023-08-28T00:01:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:08 smithi136 ceph-mon[113531]: pgmap v5627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:08.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:08.416412+0000) 2023-08-28T00:01:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:08 smithi093 ceph-mon[111335]: pgmap v5627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:08 smithi093 ceph-mon[118902]: pgmap v5627: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2435152859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:09.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:09.416642+0000) 2023-08-28T00:01:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2435152859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2435152859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:10 smithi136 ceph-mon[113531]: pgmap v5628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:10.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:10.416861+0000) 2023-08-28T00:01:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:10 smithi093 ceph-mon[111335]: pgmap v5628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:10 smithi093 ceph-mon[118902]: pgmap v5628: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:10.911Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:10.912Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/143172440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:11.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:11.417140+0000) 2023-08-28T00:01:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/143172440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/143172440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:12 smithi136 ceph-mon[113531]: pgmap v5629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:12.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:12.417518+0000) 2023-08-28T00:01:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:12 smithi093 ceph-mon[111335]: pgmap v5629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:12 smithi093 ceph-mon[118902]: pgmap v5629: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:12.385Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:12.386Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:13.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2057482840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:13.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:13.417794+0000) 2023-08-28T00:01:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2057482840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2057482840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:14 smithi136 ceph-mon[113531]: pgmap v5630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:14.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:14.418041+0000) 2023-08-28T00:01:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:14 smithi093 ceph-mon[111335]: pgmap v5630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:14 smithi093 ceph-mon[118902]: pgmap v5630: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:15.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:15.418226+0000) 2023-08-28T00:01:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:01:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:01:15] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:01:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:16 smithi136 ceph-mon[113531]: pgmap v5631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1305371470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:16.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:16.418455+0000) 2023-08-28T00:01:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:16 smithi093 ceph-mon[111335]: pgmap v5631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1305371470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:16 smithi093 ceph-mon[118902]: pgmap v5631: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1305371470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:17.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:17.418622+0000) 2023-08-28T00:01:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:18 smithi136 ceph-mon[113531]: pgmap v5632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2939368718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:18.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:18.418846+0000) 2023-08-28T00:01:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:18 smithi093 ceph-mon[111335]: pgmap v5632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2939368718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:18 smithi093 ceph-mon[118902]: pgmap v5632: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2939368718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:19.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:19.419110+0000) 2023-08-28T00:01:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:20 smithi136 ceph-mon[113531]: pgmap v5633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/583523354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:20.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:20.419395+0000) 2023-08-28T00:01:20.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:20 smithi093 ceph-mon[111335]: pgmap v5633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:20.912 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/583523354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:20.913 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:20 smithi093 ceph-mon[118902]: pgmap v5633: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:20.913 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/583523354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:20.911Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:20.912Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:21.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:21.419648+0000) 2023-08-28T00:01:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:22 smithi136 ceph-mon[113531]: pgmap v5634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:22.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:22.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:22.419870+0000) 2023-08-28T00:01:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:22 smithi093 ceph-mon[111335]: pgmap v5634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:22 smithi093 ceph-mon[118902]: pgmap v5634: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:22.386Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:22.387Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1053108322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:23.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:23.420158+0000) 2023-08-28T00:01:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1053108322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1053108322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:24 smithi136 ceph-mon[113531]: pgmap v5635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:24.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:24.420408+0000) 2023-08-28T00:01:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:24 smithi093 ceph-mon[111335]: pgmap v5635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:24 smithi093 ceph-mon[118902]: pgmap v5635: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1413393978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:25.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:25.420669+0000) 2023-08-28T00:01:25.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1413393978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:25.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:01:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:01:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:01:25.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1413393978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:26 smithi136 ceph-mon[113531]: pgmap v5636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:26.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:26.420908+0000) 2023-08-28T00:01:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:26 smithi093 ceph-mon[111335]: pgmap v5636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:26 smithi093 ceph-mon[118902]: pgmap v5636: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:27.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:27.421146+0000) 2023-08-28T00:01:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:28 smithi136 ceph-mon[113531]: pgmap v5637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2849927356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:28.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:28.421399+0000) 2023-08-28T00:01:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:28 smithi093 ceph-mon[111335]: pgmap v5637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2849927356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:28 smithi093 ceph-mon[118902]: pgmap v5637: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2849927356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:29.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:29.421614+0000) 2023-08-28T00:01:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:30 smithi136 ceph-mon[113531]: pgmap v5638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3562131320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:30.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:30.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:30.421874+0000) 2023-08-28T00:01:30.913 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:30 smithi093 ceph-mon[111335]: pgmap v5638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:30.913 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3562131320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:30.914 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:30 smithi093 ceph-mon[118902]: pgmap v5638: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:30.914 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3562131320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:30.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:30.913Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:31.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:01:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:31.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:31.422107+0000) 2023-08-28T00:01:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:01:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:01:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:32 smithi136 ceph-mon[113531]: pgmap v5639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1399142643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:32.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:32.422215+0000) 2023-08-28T00:01:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:32 smithi093 ceph-mon[118902]: pgmap v5639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1399142643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:32.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:32 smithi093 ceph-mon[111335]: pgmap v5639: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:32.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1399142643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:32.387Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:32.388Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:33.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:33.422357+0000) 2023-08-28T00:01:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:34.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:34.422580+0000) 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: pgmap v5640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:01:34.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: pgmap v5640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: pgmap v5640: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:01:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:01:34.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:01:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1177396367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:35.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:35.422793+0000) 2023-08-28T00:01:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1177396367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:01:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:01:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:01:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1177396367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:36 smithi136 ceph-mon[113531]: pgmap v5641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:36.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:36.423002+0000) 2023-08-28T00:01:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:36 smithi093 ceph-mon[111335]: pgmap v5641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:36 smithi093 ceph-mon[118902]: pgmap v5641: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1924112902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:37.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:37.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:37.423196+0000) 2023-08-28T00:01:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1924112902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1924112902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:38 smithi136 ceph-mon[113531]: pgmap v5642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:38.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:38.423407+0000) 2023-08-28T00:01:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:38 smithi093 ceph-mon[111335]: pgmap v5642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:38 smithi093 ceph-mon[118902]: pgmap v5642: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3057804902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:39.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:39.423664+0000) 2023-08-28T00:01:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3057804902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3057804902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:40 smithi136 ceph-mon[113531]: pgmap v5643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:40.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:40.423922+0000) 2023-08-28T00:01:40.914 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:40 smithi093 ceph-mon[111335]: pgmap v5643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:40.914 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:40 smithi093 ceph-mon[118902]: pgmap v5643: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:40.913Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:40.914Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:41.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:41.424121+0000) 2023-08-28T00:01:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:42 smithi136 ceph-mon[113531]: pgmap v5644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/646375720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:42.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:42.424294+0000) 2023-08-28T00:01:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:42 smithi093 ceph-mon[111335]: pgmap v5644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/646375720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:42 smithi093 ceph-mon[118902]: pgmap v5644: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/646375720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:42.387Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:42.388Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:43.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:43.424537+0000) 2023-08-28T00:01:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:44 smithi136 ceph-mon[113531]: pgmap v5645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2843327082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:44.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:44.424788+0000) 2023-08-28T00:01:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:44 smithi093 ceph-mon[111335]: pgmap v5645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2843327082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:44 smithi093 ceph-mon[118902]: pgmap v5645: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2843327082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:45.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:45.425067+0000) 2023-08-28T00:01:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:01:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:01:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:01:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:46 smithi136 ceph-mon[113531]: pgmap v5646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:46.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:46.425344+0000) 2023-08-28T00:01:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:46 smithi093 ceph-mon[111335]: pgmap v5646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:46 smithi093 ceph-mon[118902]: pgmap v5646: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:47.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/718285454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:47.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:47.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:47.425546+0000) 2023-08-28T00:01:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/718285454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/718285454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:48 smithi136 ceph-mon[113531]: pgmap v5647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:48.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:48.425824+0000) 2023-08-28T00:01:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:48 smithi093 ceph-mon[111335]: pgmap v5647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:48 smithi093 ceph-mon[118902]: pgmap v5647: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:49.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1677044959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:49.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:49.426069+0000) 2023-08-28T00:01:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1677044959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1677044959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:50.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:50 smithi136 ceph-mon[113531]: pgmap v5648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:50.692 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:50.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:50.426296+0000) 2023-08-28T00:01:50.915 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:50 smithi093 ceph-mon[111335]: pgmap v5648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:50.916 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:50 smithi093 ceph-mon[118902]: pgmap v5648: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:50.914Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:50.915Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:51.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3564641795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:51.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:51.426509+0000) 2023-08-28T00:01:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3564641795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3564641795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:52.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:52 smithi136 ceph-mon[113531]: pgmap v5649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:52.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:52.426746+0000) 2023-08-28T00:01:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:52 smithi093 ceph-mon[111335]: pgmap v5649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:52 smithi093 ceph-mon[118902]: pgmap v5649: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:52.388Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:01:52.388Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:01:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:53.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:53.426981+0000) 2023-08-28T00:01:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:54 smithi136 ceph-mon[113531]: pgmap v5650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/818132025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:54.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:54.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:54.427192+0000) 2023-08-28T00:01:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:54 smithi093 ceph-mon[111335]: pgmap v5650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/818132025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:54 smithi093 ceph-mon[118902]: pgmap v5650: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:01:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/818132025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:55.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:55.427428+0000) 2023-08-28T00:01:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:01:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:01:55] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:01:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:56 smithi136 ceph-mon[113531]: pgmap v5651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:56.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1623013683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:56.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:56.427688+0000) 2023-08-28T00:01:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:56 smithi093 ceph-mon[111335]: pgmap v5651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1623013683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:56 smithi093 ceph-mon[118902]: pgmap v5651: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1623013683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:57.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:57.427938+0000) 2023-08-28T00:01:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:58 smithi136 ceph-mon[113531]: pgmap v5652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:58.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:01:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1213784084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:58.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:58.428162+0000) 2023-08-28T00:01:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:58 smithi093 ceph-mon[111335]: pgmap v5652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:01:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1213784084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:58 smithi093 ceph-mon[118902]: pgmap v5652: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:01:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:01:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1213784084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:01:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:01:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:01:59.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:01:59.428363+0000) 2023-08-28T00:02:00.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:00 smithi136 ceph-mon[113531]: pgmap v5653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:00.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:00.428560+0000) 2023-08-28T00:02:00.915 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:00 smithi093 ceph-mon[111335]: pgmap v5653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:00.916 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:00 smithi093 ceph-mon[118902]: pgmap v5653: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:00.914Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:00.915Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/929482682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:01.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:01.428708+0000) 2023-08-28T00:02:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/929482682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/929482682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:02 smithi136 ceph-mon[113531]: pgmap v5654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:02.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:02.428930+0000) 2023-08-28T00:02:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:02 smithi093 ceph-mon[111335]: pgmap v5654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:02 smithi093 ceph-mon[118902]: pgmap v5654: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:02.388Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:02.389Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1568228877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:03.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:03.429195+0000) 2023-08-28T00:02:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1568228877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1568228877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:04 smithi136 ceph-mon[113531]: pgmap v5655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:04.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:04.429446+0000) 2023-08-28T00:02:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:04 smithi093 ceph-mon[111335]: pgmap v5655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:04 smithi093 ceph-mon[118902]: pgmap v5655: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:05.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:05.429701+0000) 2023-08-28T00:02:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:02:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:02:05] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:02:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:06 smithi136 ceph-mon[113531]: pgmap v5656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1500301680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:06.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:06.429919+0000) 2023-08-28T00:02:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:06 smithi093 ceph-mon[111335]: pgmap v5656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1500301680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:06 smithi093 ceph-mon[118902]: pgmap v5656: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1500301680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:07.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:07.430129+0000) 2023-08-28T00:02:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:08 smithi136 ceph-mon[113531]: pgmap v5657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/269476699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:08.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:08.430310+0000) 2023-08-28T00:02:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:08 smithi093 ceph-mon[111335]: pgmap v5657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/269476699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:08 smithi093 ceph-mon[118902]: pgmap v5657: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/269476699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:09.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:09.430488+0000) 2023-08-28T00:02:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:10 smithi136 ceph-mon[113531]: pgmap v5658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2035509252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:10.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:10.430695+0000) 2023-08-28T00:02:10.916 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:10 smithi093 ceph-mon[111335]: pgmap v5658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:10.916 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2035509252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:10.916 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:10 smithi093 ceph-mon[118902]: pgmap v5658: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:10.916 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2035509252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:10.915Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:10.916Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:11.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:11.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:11.430904+0000) 2023-08-28T00:02:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:12 smithi136 ceph-mon[113531]: pgmap v5659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:12.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:12.431151+0000) 2023-08-28T00:02:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:12 smithi093 ceph-mon[111335]: pgmap v5659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:12 smithi093 ceph-mon[118902]: pgmap v5659: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:12.389Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:12.390Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:13.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3455888288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:13.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:13.431386+0000) 2023-08-28T00:02:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3455888288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3455888288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:14 smithi136 ceph-mon[113531]: pgmap v5660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:14.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:14.431622+0000) 2023-08-28T00:02:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:14 smithi093 ceph-mon[111335]: pgmap v5660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:14 smithi093 ceph-mon[118902]: pgmap v5660: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:15.652 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:02:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:02:15] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:02:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2216623980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:15.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:15.431842+0000) 2023-08-28T00:02:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2216623980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2216623980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:16.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:16 smithi136 ceph-mon[113531]: pgmap v5661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:16.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:16.432098+0000) 2023-08-28T00:02:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:16 smithi093 ceph-mon[111335]: pgmap v5661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:16 smithi093 ceph-mon[118902]: pgmap v5661: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/337350687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:17.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:17.432280+0000) 2023-08-28T00:02:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/337350687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/337350687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:18.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:18 smithi136 ceph-mon[113531]: pgmap v5662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:18.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:18.432507+0000) 2023-08-28T00:02:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:18 smithi093 ceph-mon[111335]: pgmap v5662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:18 smithi093 ceph-mon[118902]: pgmap v5662: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:19.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:19.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:19.432682+0000) 2023-08-28T00:02:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:20 smithi136 ceph-mon[113531]: pgmap v5663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/774489327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:20.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:20.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:20.432904+0000) 2023-08-28T00:02:20.917 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:20 smithi093 ceph-mon[111335]: pgmap v5663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:20.917 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/774489327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:20.917 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:20 smithi093 ceph-mon[118902]: pgmap v5663: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:20.917 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/774489327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:20.916Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:20.916Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:21.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:21.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:21.433109+0000) 2023-08-28T00:02:22.664 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:22.390Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:22.665 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:22.391Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:22 smithi136 ceph-mon[113531]: pgmap v5664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3396090507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:22.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:22.433336+0000) 2023-08-28T00:02:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:22 smithi093 ceph-mon[118902]: pgmap v5664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3396090507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:22 smithi093 ceph-mon[111335]: pgmap v5664: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:22.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3396090507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:23.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:23.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:23.433587+0000) 2023-08-28T00:02:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:24 smithi136 ceph-mon[113531]: pgmap v5665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:24.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:24.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:24.433883+0000) 2023-08-28T00:02:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:24 smithi093 ceph-mon[111335]: pgmap v5665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:24 smithi093 ceph-mon[118902]: pgmap v5665: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:25.674 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:02:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:02:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:02:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3959512168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:25.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:25.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:25.434158+0000) 2023-08-28T00:02:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3959512168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3959512168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:26 smithi136 ceph-mon[113531]: pgmap v5666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:26.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:26.434420+0000) 2023-08-28T00:02:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:26 smithi093 ceph-mon[111335]: pgmap v5666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:26 smithi093 ceph-mon[118902]: pgmap v5666: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3248489860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:27.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:27.434654+0000) 2023-08-28T00:02:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3248489860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3248489860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:28.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:28 smithi136 ceph-mon[113531]: pgmap v5667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:28.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:28.434952+0000) 2023-08-28T00:02:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:28 smithi093 ceph-mon[111335]: pgmap v5667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:28 smithi093 ceph-mon[118902]: pgmap v5667: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1298313364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:29.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:29.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:29.435178+0000) 2023-08-28T00:02:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1298313364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1298313364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:30.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:30.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:30.435437+0000) 2023-08-28T00:02:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:30 smithi136 ceph-mon[113531]: pgmap v5668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:30 smithi093 ceph-mon[111335]: pgmap v5668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:30 smithi093 ceph-mon[118902]: pgmap v5668: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:30.916Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:30.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:31.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:31.435691+0000) 2023-08-28T00:02:32.689 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:32.391Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:32.689 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:32.392Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:32.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:32.435902+0000) 2023-08-28T00:02:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:32 smithi136 ceph-mon[113531]: pgmap v5669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778188037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:32 smithi093 ceph-mon[111335]: pgmap v5669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778188037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:32.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:32 smithi093 ceph-mon[118902]: pgmap v5669: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:32.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778188037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:33.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:33.436170+0000) 2023-08-28T00:02:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:34.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:34.436418+0000) 2023-08-28T00:02:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:34 smithi093 ceph-mon[111335]: pgmap v5670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4108231954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:02:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:34 smithi093 ceph-mon[118902]: pgmap v5670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4108231954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:02:35.138 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:34 smithi136 ceph-mon[113531]: pgmap v5670: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:35.138 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4108231954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:35.138 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:02:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:02:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:02:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:02:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:35.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:35.436604+0000) 2023-08-28T00:02:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:36.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:36.436767+0000) 2023-08-28T00:02:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[111335]: pgmap v5671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2118110921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[118902]: pgmap v5671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2118110921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:36 smithi136 ceph-mon[113531]: pgmap v5671: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2118110921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:37.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:37.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:37.436957+0000) 2023-08-28T00:02:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[111335]: pgmap v5672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[118902]: pgmap v5672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:02:38.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:37 smithi136 ceph-mon[113531]: pgmap v5672: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:02:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:02:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:02:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:38.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:38.437226+0000) 2023-08-28T00:02:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:39.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:39.437437+0000) 2023-08-28T00:02:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3784995893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:40 smithi136 ceph-mon[113531]: pgmap v5673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3784995893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:40 smithi093 ceph-mon[111335]: pgmap v5673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3784995893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:40 smithi093 ceph-mon[118902]: pgmap v5673: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:40.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:40.437648+0000) 2023-08-28T00:02:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:40.917Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:40.918Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:41.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:41.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:41.437898+0000) 2023-08-28T00:02:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:42 smithi136 ceph-mon[113531]: pgmap v5674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:42.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3202767375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:42.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:42.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:42.438116+0000) 2023-08-28T00:02:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:42 smithi093 ceph-mon[111335]: pgmap v5674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3202767375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:42 smithi093 ceph-mon[118902]: pgmap v5674: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3202767375' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:42.392Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:42.393Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:43.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:43.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:43.438363+0000) 2023-08-28T00:02:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:44 smithi136 ceph-mon[113531]: pgmap v5675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2655081918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:44.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:44.438615+0000) 2023-08-28T00:02:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:44 smithi093 ceph-mon[111335]: pgmap v5675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:44.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2655081918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:44.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:44 smithi093 ceph-mon[118902]: pgmap v5675: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:44.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2655081918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:45.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:45.438837+0000) 2023-08-28T00:02:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:02:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:02:45] "GET /metrics HTTP/1.1" 200 239920 "" "Prometheus/2.43.0" 2023-08-28T00:02:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:46 smithi136 ceph-mon[113531]: pgmap v5676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/826250988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:46.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:46.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:46.439118+0000) 2023-08-28T00:02:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:46 smithi093 ceph-mon[118902]: pgmap v5676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/826250988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:46 smithi093 ceph-mon[111335]: pgmap v5676: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/826250988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:47.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:47.439316+0000) 2023-08-28T00:02:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:48 smithi136 ceph-mon[113531]: pgmap v5677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:48.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:48.439597+0000) 2023-08-28T00:02:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:48 smithi093 ceph-mon[111335]: pgmap v5677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:48 smithi093 ceph-mon[118902]: pgmap v5677: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:49.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/213876082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:49.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:49.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:49.439807+0000) 2023-08-28T00:02:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/213876082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/213876082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:50 smithi136 ceph-mon[113531]: pgmap v5678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:50.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:50.439994+0000) 2023-08-28T00:02:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:50 smithi093 ceph-mon[111335]: pgmap v5678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:50 smithi093 ceph-mon[118902]: pgmap v5678: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:50.917Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:50.918Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:51.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3073964766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3073964766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3073964766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:51.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:51.440116+0000) 2023-08-28T00:02:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:52 smithi093 ceph-mon[111335]: pgmap v5679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:52 smithi093 ceph-mon[118902]: pgmap v5679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:52.392Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:02:52.393Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:02:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:52 smithi136 ceph-mon[113531]: pgmap v5679: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:52.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:52.440260+0000) 2023-08-28T00:02:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1389548990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1389548990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1389548990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:53.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:53.440500+0000) 2023-08-28T00:02:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:54 smithi093 ceph-mon[111335]: pgmap v5680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:54 smithi093 ceph-mon[118902]: pgmap v5680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:54 smithi136 ceph-mon[113531]: pgmap v5680: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:02:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:54.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:54.440770+0000) 2023-08-28T00:02:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:02:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:02:55] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:02:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:55.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:55.441027+0000) 2023-08-28T00:02:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:56 smithi093 ceph-mon[111335]: pgmap v5681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/892396509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:56 smithi093 ceph-mon[118902]: pgmap v5681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/892396509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:56.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:56.441285+0000) 2023-08-28T00:02:56.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:56 smithi136 ceph-mon[113531]: pgmap v5681: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:56.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/892396509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:57.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:57.441554+0000) 2023-08-28T00:02:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:58 smithi093 ceph-mon[111335]: pgmap v5682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:02:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2156458111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:58 smithi093 ceph-mon[118902]: pgmap v5682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:02:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2156458111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:58 smithi136 ceph-mon[113531]: pgmap v5682: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:02:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:02:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2156458111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:02:58.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:58.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:58.441805+0000) 2023-08-28T00:02:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:02:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:02:59.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:02:59.442076+0000) 2023-08-28T00:03:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:00 smithi093 ceph-mon[111335]: pgmap v5683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:00 smithi093 ceph-mon[118902]: pgmap v5683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:00 smithi136 ceph-mon[113531]: pgmap v5683: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:00.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:00.442267+0000) 2023-08-28T00:03:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:00.918Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:00.919Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/765038225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:01.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/765038225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/765038225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:01.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:01.442541+0000) 2023-08-28T00:03:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:02 smithi093 ceph-mon[111335]: pgmap v5684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:02 smithi093 ceph-mon[118902]: pgmap v5684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:02.393Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:02.394Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:02 smithi136 ceph-mon[113531]: pgmap v5684: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:02.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:02.442813+0000) 2023-08-28T00:03:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3365428212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3365428212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3365428212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:03.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:03.443089+0000) 2023-08-28T00:03:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:04 smithi093 ceph-mon[111335]: pgmap v5685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:04 smithi093 ceph-mon[118902]: pgmap v5685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:04 smithi136 ceph-mon[113531]: pgmap v5685: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:04.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:04.443339+0000) 2023-08-28T00:03:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3908050535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:03:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:03:05] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:03:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3908050535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3908050535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:05.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:05.443592+0000) 2023-08-28T00:03:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:06 smithi136 ceph-mon[113531]: pgmap v5686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:06.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:06.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:06.443824+0000) 2023-08-28T00:03:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:06 smithi093 ceph-mon[111335]: pgmap v5686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:07.050 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:06 smithi093 ceph-mon[118902]: pgmap v5686: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:07.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:07.443996+0000) 2023-08-28T00:03:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:08 smithi136 ceph-mon[113531]: pgmap v5687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020377751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:08.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:08.444238+0000) 2023-08-28T00:03:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:08 smithi093 ceph-mon[111335]: pgmap v5687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020377751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:08 smithi093 ceph-mon[118902]: pgmap v5687: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:08.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020377751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:09.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:09.444483+0000) 2023-08-28T00:03:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:10.919 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:10 smithi093 ceph-mon[111335]: pgmap v5688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:10.919 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/371287716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:10.919 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:10 smithi093 ceph-mon[118902]: pgmap v5688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:10.919 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/371287716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:10 smithi136 ceph-mon[113531]: pgmap v5688: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/371287716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:10.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:10.444739+0000) 2023-08-28T00:03:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:10.918Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:10.919Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:11.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:11.444954+0000) 2023-08-28T00:03:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:12 smithi093 ceph-mon[111335]: pgmap v5689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3866021348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:12 smithi093 ceph-mon[118902]: pgmap v5689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3866021348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:12.394Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:12.395Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:12 smithi136 ceph-mon[113531]: pgmap v5689: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3866021348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:12.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:12.445183+0000) 2023-08-28T00:03:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:13.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:13.445416+0000) 2023-08-28T00:03:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:14 smithi136 ceph-mon[113531]: pgmap v5690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:14.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:14.445607+0000) 2023-08-28T00:03:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:14 smithi093 ceph-mon[111335]: pgmap v5690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:14 smithi093 ceph-mon[118902]: pgmap v5690: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/222695643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/222695643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:15.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:03:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:03:15] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:03:15.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/222695643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:15.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:15.445863+0000) 2023-08-28T00:03:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:16 smithi136 ceph-mon[113531]: pgmap v5691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:16.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:16.446141+0000) 2023-08-28T00:03:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:16 smithi093 ceph-mon[111335]: pgmap v5691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:16 smithi093 ceph-mon[118902]: pgmap v5691: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1503266327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:17.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:17.446378+0000) 2023-08-28T00:03:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1503266327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1503266327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:18 smithi136 ceph-mon[113531]: pgmap v5692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:18.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:18.446602+0000) 2023-08-28T00:03:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:18 smithi093 ceph-mon[111335]: pgmap v5692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:18 smithi093 ceph-mon[118902]: pgmap v5692: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:19.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:19.446883+0000) 2023-08-28T00:03:20.920 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:20 smithi093 ceph-mon[111335]: pgmap v5693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:20.920 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2653545224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:20.920 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:20 smithi093 ceph-mon[118902]: pgmap v5693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:20.920 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2653545224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:20.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:20.447179+0000) 2023-08-28T00:03:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:20 smithi136 ceph-mon[113531]: pgmap v5693: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2653545224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:20.919Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:20.920Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:21.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:21.447424+0000) 2023-08-28T00:03:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:22 smithi093 ceph-mon[111335]: pgmap v5694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3632222734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:22 smithi093 ceph-mon[118902]: pgmap v5694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3632222734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:22.395Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:22.395Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:22.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:22.447664+0000) 2023-08-28T00:03:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:22 smithi136 ceph-mon[113531]: pgmap v5694: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3632222734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:23.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:23.447944+0000) 2023-08-28T00:03:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:24 smithi136 ceph-mon[113531]: pgmap v5695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3476567943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:24.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:24.448163+0000) 2023-08-28T00:03:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:24 smithi093 ceph-mon[111335]: pgmap v5695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3476567943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:24 smithi093 ceph-mon[118902]: pgmap v5695: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3476567943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:03:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:03:25] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:03:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:25.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:25.448398+0000) 2023-08-28T00:03:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:26 smithi136 ceph-mon[113531]: pgmap v5696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:26.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:26.448678+0000) 2023-08-28T00:03:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:26 smithi093 ceph-mon[111335]: pgmap v5696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:26 smithi093 ceph-mon[118902]: pgmap v5696: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:27.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4215245014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:27.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:27.448933+0000) 2023-08-28T00:03:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4215245014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4215245014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:28 smithi136 ceph-mon[113531]: pgmap v5697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:28.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:28.449141+0000) 2023-08-28T00:03:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:28 smithi093 ceph-mon[111335]: pgmap v5697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:28 smithi093 ceph-mon[118902]: pgmap v5697: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:29.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2705740620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:29.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:29.449346+0000) 2023-08-28T00:03:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2705740620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2705740620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:30.921 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:30 smithi093 ceph-mon[111335]: pgmap v5698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:30.921 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:30 smithi093 ceph-mon[118902]: pgmap v5698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:30 smithi136 ceph-mon[113531]: pgmap v5698: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:30.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:30.449598+0000) 2023-08-28T00:03:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:30.920Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:30.921Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:31.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4027374858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:31.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:31.449867+0000) 2023-08-28T00:03:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4027374858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4027374858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:32 smithi093 ceph-mon[111335]: pgmap v5699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:32 smithi093 ceph-mon[118902]: pgmap v5699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:32.395Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:32.396Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:32 smithi136 ceph-mon[113531]: pgmap v5699: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:32.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:32.450100+0000) 2023-08-28T00:03:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:33.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:33.450372+0000) 2023-08-28T00:03:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:34 smithi136 ceph-mon[113531]: pgmap v5700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/756086018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:34.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:34.450655+0000) 2023-08-28T00:03:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:34 smithi093 ceph-mon[111335]: pgmap v5700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/756086018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:34 smithi093 ceph-mon[118902]: pgmap v5700: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/756086018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:03:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:03:35] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:03:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:35.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:35.450915+0000) 2023-08-28T00:03:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:36 smithi136 ceph-mon[113531]: pgmap v5701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/346896676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:36.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:36.451166+0000) 2023-08-28T00:03:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:36 smithi093 ceph-mon[111335]: pgmap v5701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/346896676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:36 smithi093 ceph-mon[118902]: pgmap v5701: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/346896676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:03:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:37.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:37.451405+0000) 2023-08-28T00:03:37.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:03:38.053 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:03:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:38 smithi136 ceph-mon[113531]: pgmap v5702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:38.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:38.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:38.451728+0000) 2023-08-28T00:03:38.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:38 smithi093 ceph-mon[111335]: pgmap v5702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:38.962 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:38 smithi093 ceph-mon[118902]: pgmap v5702: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:39.224 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:38 smithi093 sudo[391949]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:39.224 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:38 smithi093 sudo[391949]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:39.224 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:38 smithi093 sudo[391949]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:39.224 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:38 smithi093 sudo[391949]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:39.490 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392005]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:39.490 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392005]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:39.490 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392005]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:39.490 INFO:journalctl@ceph.osd.0.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392005]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:39.764 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392100]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:39.764 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392100]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:39.765 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392100]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:39.765 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392100]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:39.765 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3607627267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:39.765 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:39.765 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.765 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.765 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.765 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3607627267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:39.766 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:39.766 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.766 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.766 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3607627267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:39.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:39.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.767 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:39.767 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:39.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:39.451900+0000) 2023-08-28T00:03:40.022 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392136]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:40.022 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392136]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:40.022 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392136]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:40.023 INFO:journalctl@ceph.osd.1.smithi093.stdout:Aug 28 00:03:39 smithi093 sudo[392136]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:40.023 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392139]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:40.023 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392139]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:40.023 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392139]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:40.023 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392139]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:40.490 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392142]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:40.490 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392142]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:40.490 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392142]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:40.490 INFO:journalctl@ceph.osd.2.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392142]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:40.755 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392181]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:40.755 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392181]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:40.755 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392181]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:40.755 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392181]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:40.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[111335]: pgmap v5703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:40.755 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[118902]: pgmap v5703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:03:40.756 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:40.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:40.452111+0000) 2023-08-28T00:03:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:40 smithi136 ceph-mon[113531]: pgmap v5703: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:03:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:03:40.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:41.240 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392184]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:41.240 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392184]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:41.240 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392184]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:41.240 INFO:journalctl@ceph.osd.3.smithi093.stdout:Aug 28 00:03:40 smithi093 sudo[392184]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:40.921Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:40.922Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:41.440 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:40 smithi136 sudo[279121]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:41.440 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:40 smithi136 sudo[279121]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:41.440 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:40 smithi136 sudo[279121]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:41.440 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279121]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:41.440 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279124]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:41.440 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279124]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:41.441 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279124]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:41.441 INFO:journalctl@ceph.osd.4.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279124]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:41.753 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/652639529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:41.753 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:41.754 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:41.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:41.452267+0000) 2023-08-28T00:03:41.754 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279127]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:41.754 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279127]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:41.754 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279127]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:41.754 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279127]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/652639529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/652639529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:41.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:03:42.006 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279130]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:42.006 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279130]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:42.006 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279130]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:42.006 INFO:journalctl@ceph.osd.5.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279130]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:42.006 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279133]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:42.006 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279133]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:42.007 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:41 smithi136 sudo[279133]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:42.007 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279133]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:42.440 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279136]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:42.440 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279136]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:42.440 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279136]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:42.440 INFO:journalctl@ceph.osd.6.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279136]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:42.731 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:42 smithi136 ceph-mon[113531]: pgmap v5704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-08-28T00:03:42.732 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:42.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:42.452426+0000) 2023-08-28T00:03:42.732 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279139]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:42.732 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279139]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:42.732 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279139]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:42.732 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279139]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:42 smithi093 ceph-mon[111335]: pgmap v5704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-08-28T00:03:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:42 smithi093 ceph-mon[118902]: pgmap v5704: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-08-28T00:03:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:42.396Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:42.397Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:43.190 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279142]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:43.190 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279142]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:43.190 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279142]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:43.190 INFO:journalctl@ceph.osd.7.smithi136.stdout:Aug 28 00:03:42 smithi136 sudo[279142]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:42 smithi093 sudo[392212]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:42 smithi093 sudo[392212]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:42 smithi093 sudo[392212]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:42 smithi093 sudo[392212]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392237]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392237]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392237]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392237]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:43.741 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279145]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:43.741 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279145]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:43.741 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279145]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:43.741 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279145]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:43.741 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:43.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:43.452607+0000) 2023-08-28T00:03:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279148]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279148]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279148]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:43 smithi136 sudo[279148]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:44.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392243]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/smartctl -x --json=o /dev/nvme0n1 2023-08-28T00:03:44.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392243]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392243]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:43 smithi093 sudo[392243]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:44.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2040162835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:44.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 sudo[392246]: ceph : TTY=unknown ; PWD=/ ; USER=root ; COMMAND=/sbin/nvme intel smart-log-add --json /dev/nvme0n1 2023-08-28T00:03:44.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 sudo[392246]: pam_systemd(sudo:session): Failed to connect to system bus: No such file or directory 2023-08-28T00:03:44.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 sudo[392246]: pam_unix(sudo:session): session opened for user root by (uid=0) 2023-08-28T00:03:44.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 sudo[392246]: pam_unix(sudo:session): session closed for user root 2023-08-28T00:03:44.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:44.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:44.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:44.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:44.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:44.494 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:44.494 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:44.494 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:44.494 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2040162835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:44.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:44.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:44.452825+0000) 2023-08-28T00:03:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:44.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:44.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:44.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:44.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2040162835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:45.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:45.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:45.453094+0000) 2023-08-28T00:03:45.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: pgmap v5705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-08-28T00:03:45.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:45.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:45 smithi136 ceph-mon[113531]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:03:45] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-28T00:03:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: pgmap v5705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[111335]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: pgmap v5705: 73 pgs: 73 active+clean; 580 KiB data, 205 MiB used, 715 GiB / 715 GiB avail; 17 KiB/s rd, 33 KiB/s wr, 3 op/s 2023-08-28T00:03:45.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:45.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:45.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd='smart' args=[json]: dispatch 2023-08-28T00:03:45.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "a"}]: dispatch 2023-08-28T00:03:45.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "b"}]: dispatch 2023-08-28T00:03:45.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "mon metadata", "id": "c"}]: dispatch 2023-08-28T00:03:45.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:45 smithi093 ceph-mon[118902]: from='admin socket' entity='admin socket' cmd=smart args=[json]: finished 2023-08-28T00:03:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:46 smithi136 ceph-mon[113531]: pgmap v5706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3309817395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:46.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:46.453353+0000) 2023-08-28T00:03:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:46 smithi093 ceph-mon[111335]: pgmap v5706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3309817395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:46 smithi093 ceph-mon[118902]: pgmap v5706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3309817395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:47.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:47.453558+0000) 2023-08-28T00:03:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:48 smithi136 ceph-mon[113531]: pgmap v5707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2831429734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:48.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:48.453813+0000) 2023-08-28T00:03:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:48 smithi093 ceph-mon[111335]: pgmap v5707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2831429734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:48 smithi093 ceph-mon[118902]: pgmap v5707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2831429734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:49.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:49.454093+0000) 2023-08-28T00:03:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:50 smithi136 ceph-mon[113531]: pgmap v5708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:50.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:50.454310+0000) 2023-08-28T00:03:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:50 smithi093 ceph-mon[111335]: pgmap v5708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:50 smithi093 ceph-mon[118902]: pgmap v5708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:50.922Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:50.923Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:51.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1373772387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1373772387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1373772387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:51.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:51.454574+0000) 2023-08-28T00:03:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:52 smithi093 ceph-mon[111335]: pgmap v5709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:52 smithi093 ceph-mon[118902]: pgmap v5709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:52.397Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:03:52.398Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:03:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:52 smithi136 ceph-mon[113531]: pgmap v5709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 19 KiB/s rd, 87 KiB/s wr, 7 op/s 2023-08-28T00:03:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:52.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:52.454850+0000) 2023-08-28T00:03:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/900389110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/900389110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/900389110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:53.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:53.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:53.455050+0000) 2023-08-28T00:03:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:54 smithi093 ceph-mon[111335]: pgmap v5710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 54 KiB/s wr, 4 op/s 2023-08-28T00:03:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:54 smithi093 ceph-mon[118902]: pgmap v5710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 54 KiB/s wr, 4 op/s 2023-08-28T00:03:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:54 smithi136 ceph-mon[113531]: pgmap v5710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 54 KiB/s wr, 4 op/s 2023-08-28T00:03:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:03:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:54.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:54.455207+0000) 2023-08-28T00:03:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1165662741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1165662741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:03:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:03:55] "GET /metrics HTTP/1.1" 200 239952 "" "Prometheus/2.43.0" 2023-08-28T00:03:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1165662741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:55.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:55.455396+0000) 2023-08-28T00:03:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:56 smithi093 ceph-mon[111335]: pgmap v5711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 54 KiB/s wr, 4 op/s 2023-08-28T00:03:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:56 smithi093 ceph-mon[118902]: pgmap v5711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 54 KiB/s wr, 4 op/s 2023-08-28T00:03:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:56 smithi136 ceph-mon[113531]: pgmap v5711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail; 1.5 KiB/s rd, 54 KiB/s wr, 4 op/s 2023-08-28T00:03:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:56.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:56.455617+0000) 2023-08-28T00:03:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:57.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:57.455850+0000) 2023-08-28T00:03:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:58 smithi093 ceph-mon[111335]: pgmap v5712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:03:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/501691165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:58 smithi093 ceph-mon[118902]: pgmap v5712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:03:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/501691165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:58 smithi136 ceph-mon[113531]: pgmap v5712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:03:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:03:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/501691165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:03:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:58.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:58.456132+0000) 2023-08-28T00:03:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:03:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:03:59.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:03:59.456381+0000) 2023-08-28T00:04:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:00 smithi093 ceph-mon[111335]: pgmap v5713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3724160612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:00 smithi093 ceph-mon[118902]: pgmap v5713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3724160612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:00 smithi136 ceph-mon[113531]: pgmap v5713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3724160612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:00.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:00.456622+0000) 2023-08-28T00:04:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:00.923Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:00.924Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:01.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:01.456870+0000) 2023-08-28T00:04:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:02 smithi093 ceph-mon[111335]: pgmap v5714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:02 smithi093 ceph-mon[118902]: pgmap v5714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:02.398Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:02.399Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:02 smithi136 ceph-mon[113531]: pgmap v5714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:02.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:02.457130+0000) 2023-08-28T00:04:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3814435364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3814435364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3814435364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:03.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:03.457384+0000) 2023-08-28T00:04:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:04 smithi093 ceph-mon[111335]: pgmap v5715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:04 smithi093 ceph-mon[118902]: pgmap v5715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:04 smithi136 ceph-mon[113531]: pgmap v5715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:04.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:04.457646+0000) 2023-08-28T00:04:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2696820212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2696820212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:04:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:04:05] "GET /metrics HTTP/1.1" 200 239952 "" "Prometheus/2.43.0" 2023-08-28T00:04:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2696820212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:05.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:05.457866+0000) 2023-08-28T00:04:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:06 smithi093 ceph-mon[111335]: pgmap v5716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:06 smithi093 ceph-mon[118902]: pgmap v5716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:06 smithi136 ceph-mon[113531]: pgmap v5716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:06.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:06.458069+0000) 2023-08-28T00:04:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2937553870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:07.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:07.458290+0000) 2023-08-28T00:04:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2937553870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2937553870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:08 smithi136 ceph-mon[113531]: pgmap v5717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:08.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:08.458516+0000) 2023-08-28T00:04:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:08 smithi093 ceph-mon[118902]: pgmap v5717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:08.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:08 smithi093 ceph-mon[111335]: pgmap v5717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:09.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:09.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:09.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:09.458803+0000) 2023-08-28T00:04:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:10.924 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:10 smithi093 ceph-mon[111335]: pgmap v5718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:10.924 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1225911102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:10.924 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:10 smithi093 ceph-mon[118902]: pgmap v5718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:10.924 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1225911102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:10 smithi136 ceph-mon[113531]: pgmap v5718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1225911102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:10.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:10.459077+0000) 2023-08-28T00:04:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:10.923Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:10.924Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:11.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:11.459294+0000) 2023-08-28T00:04:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:12 smithi093 ceph-mon[111335]: pgmap v5719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/290339182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:12 smithi093 ceph-mon[118902]: pgmap v5719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/290339182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:12.398Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:12.399Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:12 smithi136 ceph-mon[113531]: pgmap v5719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/290339182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:12.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:12.459458+0000) 2023-08-28T00:04:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:13.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:13.459675+0000) 2023-08-28T00:04:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:14 smithi136 ceph-mon[113531]: pgmap v5720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3096625366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:14.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:14.459862+0000) 2023-08-28T00:04:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:14 smithi093 ceph-mon[111335]: pgmap v5720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3096625366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:14 smithi093 ceph-mon[118902]: pgmap v5720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3096625366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:04:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:04:15] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:04:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:15.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:15.460113+0000) 2023-08-28T00:04:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:16 smithi136 ceph-mon[113531]: pgmap v5721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:16.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:16.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:16.460383+0000) 2023-08-28T00:04:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:16 smithi093 ceph-mon[111335]: pgmap v5721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:16 smithi093 ceph-mon[118902]: pgmap v5721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/743064997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:17.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:17.460639+0000) 2023-08-28T00:04:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/743064997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/743064997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:18 smithi136 ceph-mon[113531]: pgmap v5722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:18.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:18.460889+0000) 2023-08-28T00:04:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:18 smithi093 ceph-mon[111335]: pgmap v5722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:18 smithi093 ceph-mon[118902]: pgmap v5722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2032355767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:19.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:19.461131+0000) 2023-08-28T00:04:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2032355767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:19.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2032355767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:20.924 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:20 smithi093 ceph-mon[111335]: pgmap v5723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:20.925 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:20 smithi093 ceph-mon[118902]: pgmap v5723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:20 smithi136 ceph-mon[113531]: pgmap v5723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:20.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:20.461371+0000) 2023-08-28T00:04:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:20.923Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:20.924Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:21.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:21.461612+0000) 2023-08-28T00:04:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:22 smithi093 ceph-mon[111335]: pgmap v5724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2168298804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:22 smithi093 ceph-mon[118902]: pgmap v5724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2168298804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:22.399Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:22.400Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:22 smithi136 ceph-mon[113531]: pgmap v5724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2168298804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:22.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:22.461795+0000) 2023-08-28T00:04:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:23.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:23.462093+0000) 2023-08-28T00:04:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:24 smithi136 ceph-mon[113531]: pgmap v5725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:24.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:24.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3216314244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:24.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:24.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:24.462365+0000) 2023-08-28T00:04:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:24 smithi093 ceph-mon[118902]: pgmap v5725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3216314244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:24 smithi093 ceph-mon[111335]: pgmap v5725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3216314244' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:04:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:04:25] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:04:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:25.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:25.462611+0000) 2023-08-28T00:04:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:26 smithi136 ceph-mon[113531]: pgmap v5726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1371079526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:26.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:26.462927+0000) 2023-08-28T00:04:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:26 smithi093 ceph-mon[111335]: pgmap v5726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1371079526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:26 smithi093 ceph-mon[118902]: pgmap v5726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1371079526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:27.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:27.463135+0000) 2023-08-28T00:04:28.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:28 smithi136 ceph-mon[113531]: pgmap v5727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:28.943 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:28.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:28.463417+0000) 2023-08-28T00:04:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:28 smithi093 ceph-mon[111335]: pgmap v5727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:28 smithi093 ceph-mon[118902]: pgmap v5727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:29.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1305332480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:29.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:29.463656+0000) 2023-08-28T00:04:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1305332480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:29.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1305332480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:30.925 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:30 smithi093 ceph-mon[111335]: pgmap v5728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:30.925 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:30 smithi093 ceph-mon[118902]: pgmap v5728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:30 smithi136 ceph-mon[113531]: pgmap v5728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:30.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:30.463945+0000) 2023-08-28T00:04:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:30.924Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:30.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:31.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1154753666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:31.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:31.464213+0000) 2023-08-28T00:04:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1154753666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1154753666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:32 smithi093 ceph-mon[111335]: pgmap v5729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:32 smithi093 ceph-mon[118902]: pgmap v5729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:32.399Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:32.401Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:32 smithi136 ceph-mon[113531]: pgmap v5729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:32.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:32.464457+0000) 2023-08-28T00:04:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3978863638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:33.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:33.464707+0000) 2023-08-28T00:04:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3978863638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3978863638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:34 smithi136 ceph-mon[113531]: pgmap v5730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:34.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:34.464949+0000) 2023-08-28T00:04:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:34 smithi093 ceph-mon[118902]: pgmap v5730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:34 smithi093 ceph-mon[111335]: pgmap v5730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:04:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:04:35] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:04:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:35.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:35.465191+0000) 2023-08-28T00:04:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:36 smithi136 ceph-mon[113531]: pgmap v5731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1025487045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:36.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:36.465437+0000) 2023-08-28T00:04:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:36 smithi093 ceph-mon[111335]: pgmap v5731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1025487045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:36 smithi093 ceph-mon[118902]: pgmap v5731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1025487045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:37.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:37.465657+0000) 2023-08-28T00:04:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:38 smithi136 ceph-mon[113531]: pgmap v5732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4286879894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:38.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:38.465897+0000) 2023-08-28T00:04:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:38 smithi093 ceph-mon[111335]: pgmap v5732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4286879894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:38 smithi093 ceph-mon[118902]: pgmap v5732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4286879894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:39.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:39.466108+0000) 2023-08-28T00:04:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:40.926 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:40 smithi093 ceph-mon[111335]: pgmap v5733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:40.927 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:04:40.927 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:40 smithi093 ceph-mon[118902]: pgmap v5733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:40.928 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:04:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:40 smithi136 ceph-mon[113531]: pgmap v5733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:04:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:40.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:40.466317+0000) 2023-08-28T00:04:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:40.925Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:40.926Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:41.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/659279830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:41.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:41.466482+0000) 2023-08-28T00:04:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/659279830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/659279830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-mon[111335]: pgmap v5734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-mon[118902]: pgmap v5734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:42.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:42.400Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:42.402Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:42.772 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:42 smithi136 ceph-mon[113531]: pgmap v5734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:42.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:42.773 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:42.773 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:42.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:42.466640+0000) 2023-08-28T00:04:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:43.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:43.466859+0000) 2023-08-28T00:04:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2197365995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-mon[113531]: pgmap v5735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:04:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:04:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2197365995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[111335]: pgmap v5735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2197365995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[118902]: pgmap v5735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:04:44.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:04:44.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:04:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:44.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:44.467128+0000) 2023-08-28T00:04:45.744 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:04:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:04:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T00:04:45.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:45.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:45.467325+0000) 2023-08-28T00:04:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:46 smithi093 ceph-mon[111335]: pgmap v5736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2913551356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:46 smithi093 ceph-mon[118902]: pgmap v5736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2913551356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:46 smithi136 ceph-mon[113531]: pgmap v5736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2913551356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:46.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:46.467573+0000) 2023-08-28T00:04:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:47.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:47.467825+0000) 2023-08-28T00:04:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:48 smithi093 ceph-mon[118902]: pgmap v5737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2883816763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:48 smithi093 ceph-mon[111335]: pgmap v5737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2883816763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:48 smithi136 ceph-mon[113531]: pgmap v5737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2883816763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:48.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:48.468081+0000) 2023-08-28T00:04:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:49.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:49.468324+0000) 2023-08-28T00:04:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:50 smithi093 ceph-mon[111335]: pgmap v5738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1538660609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:50.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:50 smithi093 ceph-mon[118902]: pgmap v5738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1538660609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:50 smithi136 ceph-mon[113531]: pgmap v5738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1538660609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:50.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:50.468573+0000) 2023-08-28T00:04:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:50.925Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:50.926Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:51.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:51.468784+0000) 2023-08-28T00:04:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:52 smithi093 ceph-mon[111335]: pgmap v5739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:52 smithi093 ceph-mon[118902]: pgmap v5739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:52.402Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:04:52.403Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:04:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:52 smithi136 ceph-mon[113531]: pgmap v5739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:52.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:52.469079+0000) 2023-08-28T00:04:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/262067235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/262067235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/262067235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:53.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:53.469327+0000) 2023-08-28T00:04:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:54 smithi093 ceph-mon[118902]: pgmap v5740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:54 smithi093 ceph-mon[111335]: pgmap v5740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:54 smithi136 ceph-mon[113531]: pgmap v5740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:04:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:54.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:54.469527+0000) 2023-08-28T00:04:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4018372384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:04:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:04:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T00:04:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4018372384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4018372384' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:55.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:55.469801+0000) 2023-08-28T00:04:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:56 smithi093 ceph-mon[111335]: pgmap v5741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:56.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:56 smithi093 ceph-mon[118902]: pgmap v5741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:56 smithi136 ceph-mon[113531]: pgmap v5741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:56.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:56.470101+0000) 2023-08-28T00:04:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3976500026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3976500026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3976500026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:04:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:57.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:57.470286+0000) 2023-08-28T00:04:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:04:58 smithi093 ceph-mon[111335]: pgmap v5742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:58.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:04:58 smithi093 ceph-mon[118902]: pgmap v5742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:04:58 smithi136 ceph-mon[113531]: pgmap v5742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:04:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:58.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:58.470445+0000) 2023-08-28T00:04:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:04:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:04:59.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:04:59.470653+0000) 2023-08-28T00:05:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:00 smithi093 ceph-mon[111335]: pgmap v5743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2702170049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:00 smithi093 ceph-mon[118902]: pgmap v5743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:00.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2702170049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:00 smithi136 ceph-mon[113531]: pgmap v5743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2702170049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:00.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:00.470923+0000) 2023-08-28T00:05:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:00.925Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:00.927Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:01.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:01.471188+0000) 2023-08-28T00:05:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:02 smithi093 ceph-mon[111335]: pgmap v5744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/91467856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:02 smithi093 ceph-mon[118902]: pgmap v5744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/91467856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:02.403Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:02.404Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:02 smithi136 ceph-mon[113531]: pgmap v5744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/91467856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:02.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:02.471413+0000) 2023-08-28T00:05:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:03.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:03.471654+0000) 2023-08-28T00:05:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:04 smithi093 ceph-mon[111335]: pgmap v5745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:04 smithi093 ceph-mon[118902]: pgmap v5745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:04 smithi136 ceph-mon[113531]: pgmap v5745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:04.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:04.471874+0000) 2023-08-28T00:05:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3396319329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:05:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:05:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T00:05:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3396319329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3396319329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:05.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:05.472163+0000) 2023-08-28T00:05:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:06 smithi093 ceph-mon[111335]: pgmap v5746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:06 smithi093 ceph-mon[118902]: pgmap v5746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:06 smithi136 ceph-mon[113531]: pgmap v5746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:06.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:06.472440+0000) 2023-08-28T00:05:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/538830083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:07.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:07.472658+0000) 2023-08-28T00:05:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/538830083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/538830083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:08 smithi136 ceph-mon[113531]: pgmap v5747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:08.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:08.472892+0000) 2023-08-28T00:05:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:08 smithi093 ceph-mon[111335]: pgmap v5747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:08 smithi093 ceph-mon[118902]: pgmap v5747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/436177046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:09.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:09.473165+0000) 2023-08-28T00:05:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/436177046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/436177046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:10.928 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:10 smithi093 ceph-mon[118902]: pgmap v5748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:10.928 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:10 smithi093 ceph-mon[111335]: pgmap v5748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:10 smithi136 ceph-mon[113531]: pgmap v5748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:10.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:10.473392+0000) 2023-08-28T00:05:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:10.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:10.928Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:11.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:11.473626+0000) 2023-08-28T00:05:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:12 smithi093 ceph-mon[111335]: pgmap v5749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1475215674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:12 smithi093 ceph-mon[118902]: pgmap v5749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1475215674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:12.403Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:12.404Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:12 smithi136 ceph-mon[113531]: pgmap v5749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1475215674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:12.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:12.473868+0000) 2023-08-28T00:05:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:13.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:13.474122+0000) 2023-08-28T00:05:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:14 smithi136 ceph-mon[113531]: pgmap v5750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/176993194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:14.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:14.474371+0000) 2023-08-28T00:05:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:14 smithi093 ceph-mon[111335]: pgmap v5750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/176993194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:14 smithi093 ceph-mon[118902]: pgmap v5750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/176993194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:05:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:05:15] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T00:05:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:15.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:15.474617+0000) 2023-08-28T00:05:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:16 smithi136 ceph-mon[113531]: pgmap v5751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/148174287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:16.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:16.474882+0000) 2023-08-28T00:05:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:16 smithi093 ceph-mon[118902]: pgmap v5751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/148174287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:16 smithi093 ceph-mon[111335]: pgmap v5751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/148174287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:17.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:17.475138+0000) 2023-08-28T00:05:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:18 smithi136 ceph-mon[113531]: pgmap v5752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:18.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:18.475390+0000) 2023-08-28T00:05:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:18 smithi093 ceph-mon[111335]: pgmap v5752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:18 smithi093 ceph-mon[118902]: pgmap v5752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1764329044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:19.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:19.475590+0000) 2023-08-28T00:05:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1764329044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1764329044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:20.928 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:20 smithi093 ceph-mon[111335]: pgmap v5753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:20.928 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:20 smithi093 ceph-mon[118902]: pgmap v5753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:20 smithi136 ceph-mon[113531]: pgmap v5753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:20.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:20.475815+0000) 2023-08-28T00:05:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:20.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:20.928Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:21.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/139230945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:21.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:21.476086+0000) 2023-08-28T00:05:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/139230945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/139230945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:22 smithi093 ceph-mon[111335]: pgmap v5754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:22 smithi093 ceph-mon[118902]: pgmap v5754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:22.405Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:22.405Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:22 smithi136 ceph-mon[113531]: pgmap v5754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:22.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:22.476339+0000) 2023-08-28T00:05:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:23.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:23.476593+0000) 2023-08-28T00:05:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:24 smithi136 ceph-mon[113531]: pgmap v5755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2682760296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:24.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:24.476848+0000) 2023-08-28T00:05:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:24 smithi093 ceph-mon[111335]: pgmap v5755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2682760296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:24 smithi093 ceph-mon[118902]: pgmap v5755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2682760296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:05:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:05:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:05:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:25.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:25.477155+0000) 2023-08-28T00:05:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:26 smithi136 ceph-mon[113531]: pgmap v5756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/619828011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:26.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:26.477375+0000) 2023-08-28T00:05:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:26 smithi093 ceph-mon[111335]: pgmap v5756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:26.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/619828011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:26 smithi093 ceph-mon[118902]: pgmap v5756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/619828011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:27.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:27.477617+0000) 2023-08-28T00:05:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:28.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:28.477868+0000) 2023-08-28T00:05:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:28 smithi136 ceph-mon[113531]: pgmap v5757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/813599013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:28.993 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:28 smithi093 ceph-mon[111335]: pgmap v5757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:28.993 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/813599013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:28.993 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:28 smithi093 ceph-mon[118902]: pgmap v5757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:28.994 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/813599013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:29.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:29.478120+0000) 2023-08-28T00:05:30.929 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:30 smithi093 ceph-mon[111335]: pgmap v5758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:30.929 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:30 smithi093 ceph-mon[118902]: pgmap v5758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:30 smithi136 ceph-mon[113531]: pgmap v5758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:30.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:30.478323+0000) 2023-08-28T00:05:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:30.928Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:30.929Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:31.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/813077839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:31.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:31.478569+0000) 2023-08-28T00:05:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/813077839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/813077839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:32 smithi093 ceph-mon[111335]: pgmap v5759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:32 smithi093 ceph-mon[118902]: pgmap v5759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:32.405Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:32.406Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:32 smithi136 ceph-mon[113531]: pgmap v5759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:32.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:32.478746+0000) 2023-08-28T00:05:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2096972166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:33.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:33.479000+0000) 2023-08-28T00:05:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2096972166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2096972166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:34 smithi136 ceph-mon[113531]: pgmap v5760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:34.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:34.479277+0000) 2023-08-28T00:05:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:34 smithi093 ceph-mon[111335]: pgmap v5760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:34 smithi093 ceph-mon[118902]: pgmap v5760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:05:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:05:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:05:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:35.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:35.479542+0000) 2023-08-28T00:05:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:36 smithi093 ceph-mon[111335]: pgmap v5761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1056246339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:36 smithi093 ceph-mon[118902]: pgmap v5761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:36.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1056246339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:36 smithi136 ceph-mon[113531]: pgmap v5761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1056246339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:36.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:36.479826+0000) 2023-08-28T00:05:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:37.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:37.480102+0000) 2023-08-28T00:05:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:38 smithi093 ceph-mon[111335]: pgmap v5762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3184253337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:38 smithi093 ceph-mon[118902]: pgmap v5762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:38.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3184253337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:38 smithi136 ceph-mon[113531]: pgmap v5762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3184253337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:38.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:38.480320+0000) 2023-08-28T00:05:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:39.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:39.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:39.480585+0000) 2023-08-28T00:05:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:40 smithi093 ceph-mon[111335]: pgmap v5763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2780857854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:40 smithi093 ceph-mon[118902]: pgmap v5763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2780857854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:40 smithi136 ceph-mon[113531]: pgmap v5763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2780857854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:40.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:40.480853+0000) 2023-08-28T00:05:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:40.929Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:40.930Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:41.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:41.481123+0000) 2023-08-28T00:05:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:42 smithi093 ceph-mon[111335]: pgmap v5764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:42 smithi093 ceph-mon[118902]: pgmap v5764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:42.406Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:42.407Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:42 smithi136 ceph-mon[113531]: pgmap v5764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:42.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:42.481382+0000) 2023-08-28T00:05:43.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3376299291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:43.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3376299291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3376299291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:43.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:43.481647+0000) 2023-08-28T00:05:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:44 smithi093 ceph-mon[118902]: pgmap v5765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:05:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:44 smithi093 ceph-mon[111335]: pgmap v5765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:44.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:05:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:44 smithi136 ceph-mon[113531]: pgmap v5765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:05:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:44.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:44.481846+0000) 2023-08-28T00:05:45.469 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:05:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:05:45] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:05:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2279325926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2279325926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:45.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2279325926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:45.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:45.759 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:45.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:45.482054+0000) 2023-08-28T00:05:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[111335]: pgmap v5766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[118902]: pgmap v5766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:05:46.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:05:46.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:46.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:46.482223+0000) 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-mon[113531]: pgmap v5766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:05:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:05:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2498501984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2498501984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2498501984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:47.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:47.482408+0000) 2023-08-28T00:05:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:48 smithi093 ceph-mon[111335]: pgmap v5767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:48.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:48 smithi093 ceph-mon[118902]: pgmap v5767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:48 smithi136 ceph-mon[113531]: pgmap v5767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:48.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:48.482619+0000) 2023-08-28T00:05:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:49.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:49.482866+0000) 2023-08-28T00:05:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:50 smithi093 ceph-mon[111335]: pgmap v5768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/638136570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:50 smithi093 ceph-mon[118902]: pgmap v5768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/638136570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:50 smithi136 ceph-mon[113531]: pgmap v5768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/638136570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:50.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:50.483082+0000) 2023-08-28T00:05:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:50.929Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:50.930Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:51.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:51.483247+0000) 2023-08-28T00:05:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:52 smithi093 ceph-mon[111335]: pgmap v5769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3857682874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:52 smithi093 ceph-mon[118902]: pgmap v5769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3857682874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:52.406Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:05:52.407Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:05:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:52 smithi136 ceph-mon[113531]: pgmap v5769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3857682874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:52.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:52.483412+0000) 2023-08-28T00:05:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:53.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:53.483570+0000) 2023-08-28T00:05:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:54 smithi093 ceph-mon[111335]: pgmap v5770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:54 smithi093 ceph-mon[118902]: pgmap v5770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:54 smithi136 ceph-mon[113531]: pgmap v5770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:05:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:54.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:54.483754+0000) 2023-08-28T00:05:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4049344742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:05:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:05:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T00:05:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4049344742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4049344742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:55.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:55.484004+0000) 2023-08-28T00:05:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:56 smithi136 ceph-mon[113531]: pgmap v5771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:56.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:56.484296+0000) 2023-08-28T00:05:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:56 smithi093 ceph-mon[111335]: pgmap v5771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:56 smithi093 ceph-mon[118902]: pgmap v5771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1262134371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:57.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:57.484549+0000) 2023-08-28T00:05:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1262134371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1262134371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:58 smithi136 ceph-mon[113531]: pgmap v5772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:58.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:58.484811+0000) 2023-08-28T00:05:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:58 smithi093 ceph-mon[111335]: pgmap v5772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:58 smithi093 ceph-mon[118902]: pgmap v5772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:05:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:05:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1577768769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:05:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:05:59.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:05:59.485092+0000) 2023-08-28T00:05:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:05:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1577768769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:05:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:05:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1577768769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:00.932 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:00 smithi093 ceph-mon[111335]: pgmap v5773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:00.932 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:00 smithi093 ceph-mon[118902]: pgmap v5773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:00 smithi136 ceph-mon[113531]: pgmap v5773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:00.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:00.485408+0000) 2023-08-28T00:06:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:00.930Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:00.931Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:01.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:01.485620+0000) 2023-08-28T00:06:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:02 smithi093 ceph-mon[111335]: pgmap v5774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/770769480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:02 smithi093 ceph-mon[118902]: pgmap v5774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/770769480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:02.407Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:02.408Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:02 smithi136 ceph-mon[113531]: pgmap v5774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/770769480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:02.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:02.485899+0000) 2023-08-28T00:06:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:03.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:03.486195+0000) 2023-08-28T00:06:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:04 smithi136 ceph-mon[113531]: pgmap v5775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/404130266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:04.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:04.486466+0000) 2023-08-28T00:06:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:04 smithi093 ceph-mon[118902]: pgmap v5775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/404130266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:04 smithi093 ceph-mon[111335]: pgmap v5775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/404130266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:06:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:06:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T00:06:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:05.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:05.486761+0000) 2023-08-28T00:06:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:06 smithi136 ceph-mon[113531]: pgmap v5776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:06.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:06.487070+0000) 2023-08-28T00:06:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:06 smithi093 ceph-mon[111335]: pgmap v5776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:06 smithi093 ceph-mon[118902]: pgmap v5776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2026871781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:07.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:07.487351+0000) 2023-08-28T00:06:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2026871781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2026871781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:08 smithi136 ceph-mon[113531]: pgmap v5777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:08.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:08.487628+0000) 2023-08-28T00:06:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:08 smithi093 ceph-mon[111335]: pgmap v5777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:08 smithi093 ceph-mon[118902]: pgmap v5777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2180688049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:09.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:09.487841+0000) 2023-08-28T00:06:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2180688049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2180688049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:10.934 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:10 smithi093 ceph-mon[111335]: pgmap v5778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:10.934 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:10 smithi093 ceph-mon[118902]: pgmap v5778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:10 smithi136 ceph-mon[113531]: pgmap v5778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:10.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:10.488105+0000) 2023-08-28T00:06:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:10.931Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:10.932Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:11.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3150368882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:11.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:11.488354+0000) 2023-08-28T00:06:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3150368882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3150368882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:12 smithi093 ceph-mon[111335]: pgmap v5779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:12 smithi093 ceph-mon[118902]: pgmap v5779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:12.407Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:12.408Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:12 smithi136 ceph-mon[113531]: pgmap v5779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:12.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:12.488617+0000) 2023-08-28T00:06:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:13.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:13.488931+0000) 2023-08-28T00:06:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:14 smithi136 ceph-mon[113531]: pgmap v5780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1990991807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:14.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:14.489213+0000) 2023-08-28T00:06:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:14 smithi093 ceph-mon[111335]: pgmap v5780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1990991807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:14 smithi093 ceph-mon[118902]: pgmap v5780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1990991807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:06:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:06:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:06:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:15.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:15.489462+0000) 2023-08-28T00:06:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:16 smithi136 ceph-mon[113531]: pgmap v5781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/77874784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:16.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:16.489730+0000) 2023-08-28T00:06:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:16 smithi093 ceph-mon[111335]: pgmap v5781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/77874784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:16 smithi093 ceph-mon[118902]: pgmap v5781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/77874784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:17.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:17.489971+0000) 2023-08-28T00:06:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:18 smithi136 ceph-mon[113531]: pgmap v5782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1120381146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:18.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:18.490190+0000) 2023-08-28T00:06:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:18 smithi093 ceph-mon[111335]: pgmap v5782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1120381146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:18 smithi093 ceph-mon[118902]: pgmap v5782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1120381146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:19.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:19.490407+0000) 2023-08-28T00:06:20.933 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:20 smithi093 ceph-mon[111335]: pgmap v5783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:20.933 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:20 smithi093 ceph-mon[118902]: pgmap v5783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:20 smithi136 ceph-mon[113531]: pgmap v5783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:20.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:20.490654+0000) 2023-08-28T00:06:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:20.932Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:20.933Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:21.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/957003638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:21.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:21.490941+0000) 2023-08-28T00:06:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/957003638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/957003638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:22 smithi093 ceph-mon[111335]: pgmap v5784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:22 smithi093 ceph-mon[118902]: pgmap v5784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:22.408Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:22.409Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:22 smithi136 ceph-mon[113531]: pgmap v5784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:22.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:22.491159+0000) 2023-08-28T00:06:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1390763082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:23.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:23.491350+0000) 2023-08-28T00:06:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1390763082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1390763082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:24 smithi136 ceph-mon[113531]: pgmap v5785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:24.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:24.491548+0000) 2023-08-28T00:06:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:24 smithi093 ceph-mon[118902]: pgmap v5785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:24 smithi093 ceph-mon[111335]: pgmap v5785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:06:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:06:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:06:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:25.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:25.491826+0000) 2023-08-28T00:06:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:26 smithi093 ceph-mon[118902]: pgmap v5786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2797201940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:26 smithi093 ceph-mon[111335]: pgmap v5786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2797201940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:26 smithi136 ceph-mon[113531]: pgmap v5786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2797201940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:26.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:26.492116+0000) 2023-08-28T00:06:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:27.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:27.492305+0000) 2023-08-28T00:06:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:28 smithi093 ceph-mon[118902]: pgmap v5787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1268026178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:28 smithi093 ceph-mon[111335]: pgmap v5787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:28.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1268026178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:28 smithi136 ceph-mon[113531]: pgmap v5787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1268026178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:28.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:28.492560+0000) 2023-08-28T00:06:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:29.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:29.492849+0000) 2023-08-28T00:06:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:30 smithi093 ceph-mon[111335]: pgmap v5788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3235499347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:30 smithi093 ceph-mon[118902]: pgmap v5788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:30.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3235499347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:30 smithi136 ceph-mon[113531]: pgmap v5788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3235499347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:30.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:30.493172+0000) 2023-08-28T00:06:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:30.933Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:30.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:31.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:31.493466+0000) 2023-08-28T00:06:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:32 smithi093 ceph-mon[111335]: pgmap v5789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:32 smithi093 ceph-mon[118902]: pgmap v5789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:32.409Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:32.410Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:32 smithi136 ceph-mon[113531]: pgmap v5789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:32.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:32.493719+0000) 2023-08-28T00:06:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1950102388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1950102388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1950102388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:33.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:33.493995+0000) 2023-08-28T00:06:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:34 smithi093 ceph-mon[111335]: pgmap v5790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:34 smithi093 ceph-mon[118902]: pgmap v5790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:34 smithi136 ceph-mon[113531]: pgmap v5790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:34.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:34.494257+0000) 2023-08-28T00:06:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2016691628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:06:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:06:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:06:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2016691628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2016691628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:35.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:35.494529+0000) 2023-08-28T00:06:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:36 smithi093 ceph-mon[111335]: pgmap v5791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:36 smithi093 ceph-mon[118902]: pgmap v5791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:36 smithi136 ceph-mon[113531]: pgmap v5791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:36.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:36.494776+0000) 2023-08-28T00:06:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4158871347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:37.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:37.494958+0000) 2023-08-28T00:06:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4158871347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4158871347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:38 smithi136 ceph-mon[113531]: pgmap v5792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:38.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:38.495184+0000) 2023-08-28T00:06:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:38 smithi093 ceph-mon[111335]: pgmap v5792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:38 smithi093 ceph-mon[118902]: pgmap v5792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:39.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:39.495377+0000) 2023-08-28T00:06:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:40.934 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:40 smithi093 ceph-mon[111335]: pgmap v5793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:40.935 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3100333090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:40.935 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:40 smithi093 ceph-mon[118902]: pgmap v5793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:40.935 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3100333090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:40.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:40 smithi136 ceph-mon[113531]: pgmap v5793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3100333090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:40.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:40.495586+0000) 2023-08-28T00:06:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:40.934Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:40.935Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:41.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:41.495789+0000) 2023-08-28T00:06:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:42 smithi093 ceph-mon[111335]: pgmap v5794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/59798925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:42 smithi093 ceph-mon[118902]: pgmap v5794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/59798925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:42.409Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:42.410Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:42 smithi136 ceph-mon[113531]: pgmap v5794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/59798925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:42.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:42.496101+0000) 2023-08-28T00:06:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:43.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:43.496402+0000) 2023-08-28T00:06:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:44 smithi136 ceph-mon[113531]: pgmap v5795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:44.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:44.496650+0000) 2023-08-28T00:06:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:44 smithi093 ceph-mon[111335]: pgmap v5795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:44 smithi093 ceph-mon[118902]: pgmap v5795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3939339447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3939339447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:45.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:06:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:06:45] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T00:06:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3939339447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:45.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:45.496916+0000) 2023-08-28T00:06:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:46 smithi136 ceph-mon[113531]: pgmap v5796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:46.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:46.497186+0000) 2023-08-28T00:06:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:46 smithi093 ceph-mon[111335]: pgmap v5796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:46 smithi093 ceph-mon[118902]: pgmap v5796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:06:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2335244936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:06:47.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:47.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:47.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2335244936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:47.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:47.497429+0000) 2023-08-28T00:06:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:06:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2335244936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:48.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:48.497662+0000) 2023-08-28T00:06:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:48 smithi136 ceph-mon[113531]: pgmap v5797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:48 smithi093 ceph-mon[111335]: pgmap v5797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:48 smithi093 ceph-mon[118902]: pgmap v5797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:49.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:49.497857+0000) 2023-08-28T00:06:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:06:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2186726254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2186726254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:06:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:06:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:06:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2186726254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:50.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:50.498127+0000) 2023-08-28T00:06:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:50 smithi093 ceph-mon[111335]: pgmap v5798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:50 smithi093 ceph-mon[118902]: pgmap v5798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:50.935Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:50.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:50 smithi136 ceph-mon[113531]: pgmap v5798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:51.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:51.498343+0000) 2023-08-28T00:06:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1752628062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1752628062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1752628062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:52.410Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:06:52.411Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:06:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:52.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:52.498556+0000) 2023-08-28T00:06:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:52 smithi093 ceph-mon[111335]: pgmap v5799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:52 smithi093 ceph-mon[118902]: pgmap v5799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:52 smithi136 ceph-mon[113531]: pgmap v5799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:53.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:53.498778+0000) 2023-08-28T00:06:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:06:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:54.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:54.499056+0000) 2023-08-28T00:06:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:54 smithi093 ceph-mon[111335]: pgmap v5800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3828978711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:54 smithi093 ceph-mon[118902]: pgmap v5800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3828978711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:54 smithi136 ceph-mon[113531]: pgmap v5800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3828978711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:06:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:06:55] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:06:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:55.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:55.499345+0000) 2023-08-28T00:06:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:56.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:56.499623+0000) 2023-08-28T00:06:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:56 smithi093 ceph-mon[111335]: pgmap v5801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1574363912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:56 smithi093 ceph-mon[118902]: pgmap v5801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:57.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1574363912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:56 smithi136 ceph-mon[113531]: pgmap v5801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1574363912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:57.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:57.499813+0000) 2023-08-28T00:06:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:58.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:58.499994+0000) 2023-08-28T00:06:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:58 smithi093 ceph-mon[111335]: pgmap v5802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:06:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/106841146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:58 smithi093 ceph-mon[118902]: pgmap v5802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:06:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/106841146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:58 smithi136 ceph-mon[113531]: pgmap v5802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:06:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:06:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/106841146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:06:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:06:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:06:59.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:06:59.500270+0000) 2023-08-28T00:07:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:00.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:00.500577+0000) 2023-08-28T00:07:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:00 smithi093 ceph-mon[111335]: pgmap v5803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:00 smithi093 ceph-mon[118902]: pgmap v5803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:00.935Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:00.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:00 smithi136 ceph-mon[113531]: pgmap v5803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:01.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:01.500853+0000) 2023-08-28T00:07:02.412 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1638448388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:02.412 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1638448388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1638448388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:02.411Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:02.412Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:02.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:02.501147+0000) 2023-08-28T00:07:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:02 smithi136 ceph-mon[113531]: pgmap v5804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:02 smithi093 ceph-mon[111335]: pgmap v5804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:02 smithi093 ceph-mon[118902]: pgmap v5804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:03.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:03.501368+0000) 2023-08-28T00:07:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3627783526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3627783526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3627783526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:04.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:04.501656+0000) 2023-08-28T00:07:05.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:05 smithi093 ceph-mon[111335]: pgmap v5805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:05 smithi093 ceph-mon[118902]: pgmap v5805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:05 smithi136 ceph-mon[113531]: pgmap v5805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:07:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:07:05] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:07:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:05.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:05.501852+0000) 2023-08-28T00:07:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:06 smithi093 ceph-mon[111335]: pgmap v5806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:06.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1563866864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:06 smithi093 ceph-mon[118902]: pgmap v5806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:06.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1563866864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:06 smithi136 ceph-mon[113531]: pgmap v5806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1563866864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:06.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:06.502145+0000) 2023-08-28T00:07:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:07.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:07.502339+0000) 2023-08-28T00:07:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:08 smithi093 ceph-mon[111335]: pgmap v5807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:08 smithi093 ceph-mon[118902]: pgmap v5807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:08 smithi136 ceph-mon[113531]: pgmap v5807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:08.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:08.502519+0000) 2023-08-28T00:07:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4090358219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:09.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4090358219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4090358219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:09.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:09.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:09.502711+0000) 2023-08-28T00:07:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:10 smithi093 ceph-mon[111335]: pgmap v5808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:10.838 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:10 smithi093 ceph-mon[118902]: pgmap v5808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:10.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:10 smithi136 ceph-mon[113531]: pgmap v5808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:10.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:10.503012+0000) 2023-08-28T00:07:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:10.935Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:10.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:11.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3589524747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:11.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3589524747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:11.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3589524747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:11.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:11.503242+0000) 2023-08-28T00:07:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:12 smithi093 ceph-mon[111335]: pgmap v5809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:12 smithi093 ceph-mon[118902]: pgmap v5809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:12.411Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:12.412Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:12 smithi136 ceph-mon[113531]: pgmap v5809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:12.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:12.503504+0000) 2023-08-28T00:07:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1607475609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1607475609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1607475609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:13.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:13.503725+0000) 2023-08-28T00:07:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:14 smithi093 ceph-mon[111335]: pgmap v5810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:14 smithi093 ceph-mon[118902]: pgmap v5810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:14 smithi136 ceph-mon[113531]: pgmap v5810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:14.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:14.503991+0000) 2023-08-28T00:07:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:07:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:07:15] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:07:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:15.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:15.504255+0000) 2023-08-28T00:07:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:16 smithi093 ceph-mon[111335]: pgmap v5811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2137674861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:16 smithi093 ceph-mon[118902]: pgmap v5811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2137674861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:16 smithi136 ceph-mon[113531]: pgmap v5811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2137674861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:16.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:16.504524+0000) 2023-08-28T00:07:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:17.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:17.504785+0000) 2023-08-28T00:07:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:18 smithi093 ceph-mon[111335]: pgmap v5812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3334001068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:18 smithi093 ceph-mon[118902]: pgmap v5812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:18.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3334001068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:18 smithi136 ceph-mon[113531]: pgmap v5812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3334001068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:18.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:18.505082+0000) 2023-08-28T00:07:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:19.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:19.505327+0000) 2023-08-28T00:07:20.937 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:20 smithi093 ceph-mon[111335]: pgmap v5813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:20.937 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2055440460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:20.937 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:20 smithi093 ceph-mon[118902]: pgmap v5813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:20.937 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2055440460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:20.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:20 smithi136 ceph-mon[113531]: pgmap v5813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2055440460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:20.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:20.505615+0000) 2023-08-28T00:07:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:20.936Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:20.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:21.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:21.505878+0000) 2023-08-28T00:07:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:22 smithi093 ceph-mon[111335]: pgmap v5814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:22 smithi093 ceph-mon[118902]: pgmap v5814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:22.411Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:22.412Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:22.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:22 smithi136 ceph-mon[113531]: pgmap v5814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:22.943 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:22.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:22.506060+0000) 2023-08-28T00:07:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3812272563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:23.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:23.506210+0000) 2023-08-28T00:07:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3812272563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3812272563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:24 smithi136 ceph-mon[113531]: pgmap v5815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:24.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:24.506400+0000) 2023-08-28T00:07:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:24 smithi093 ceph-mon[111335]: pgmap v5815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:24 smithi093 ceph-mon[118902]: pgmap v5815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1897764820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:07:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:07:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:07:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1897764820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1897764820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:25.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:25.506572+0000) 2023-08-28T00:07:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:26 smithi136 ceph-mon[113531]: pgmap v5816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:26.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:26.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:26.506810+0000) 2023-08-28T00:07:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:26 smithi093 ceph-mon[111335]: pgmap v5816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:26 smithi093 ceph-mon[118902]: pgmap v5816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:27.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:27.507047+0000) 2023-08-28T00:07:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:28 smithi136 ceph-mon[113531]: pgmap v5817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/620752606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:28.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:28.507233+0000) 2023-08-28T00:07:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:28 smithi093 ceph-mon[111335]: pgmap v5817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/620752606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:28 smithi093 ceph-mon[118902]: pgmap v5817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/620752606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:29.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:29.507509+0000) 2023-08-28T00:07:30.937 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:30 smithi093 ceph-mon[111335]: pgmap v5818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:30.937 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2274113873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:30.938 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:30 smithi093 ceph-mon[118902]: pgmap v5818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:30.938 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2274113873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:30 smithi136 ceph-mon[113531]: pgmap v5818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2274113873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:30.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:30.507770+0000) 2023-08-28T00:07:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:30.936Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:30.937Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:31.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:31.508055+0000) 2023-08-28T00:07:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:32 smithi093 ceph-mon[111335]: pgmap v5819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/99584369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:32 smithi093 ceph-mon[118902]: pgmap v5819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/99584369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:32.412Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:32.413Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:32 smithi136 ceph-mon[113531]: pgmap v5819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/99584369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:32.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:32.508312+0000) 2023-08-28T00:07:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:33.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:33.508515+0000) 2023-08-28T00:07:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:34 smithi136 ceph-mon[113531]: pgmap v5820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:34.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:34.508752+0000) 2023-08-28T00:07:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:34 smithi093 ceph-mon[111335]: pgmap v5820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:34 smithi093 ceph-mon[118902]: pgmap v5820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/991092521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/991092521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:07:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:07:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:07:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/991092521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:35.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:35.509060+0000) 2023-08-28T00:07:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:36 smithi136 ceph-mon[113531]: pgmap v5821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:36.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:36.509333+0000) 2023-08-28T00:07:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:36 smithi093 ceph-mon[111335]: pgmap v5821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:36 smithi093 ceph-mon[118902]: pgmap v5821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1985926448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:37.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:37.509585+0000) 2023-08-28T00:07:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1985926448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:37.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1985926448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:38 smithi136 ceph-mon[113531]: pgmap v5822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:38.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:38.509852+0000) 2023-08-28T00:07:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:38 smithi093 ceph-mon[111335]: pgmap v5822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:38 smithi093 ceph-mon[118902]: pgmap v5822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3185788848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:39.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:39.510116+0000) 2023-08-28T00:07:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3185788848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3185788848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:40.938 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:40 smithi093 ceph-mon[111335]: pgmap v5823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:40.938 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:40 smithi093 ceph-mon[118902]: pgmap v5823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:40 smithi136 ceph-mon[113531]: pgmap v5823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:40.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:40.510421+0000) 2023-08-28T00:07:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:40.937Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:40.938Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:41.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:41.510651+0000) 2023-08-28T00:07:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:42 smithi093 ceph-mon[118902]: pgmap v5824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3992005013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:42 smithi093 ceph-mon[111335]: pgmap v5824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3992005013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:42.413Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:42.414Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:42 smithi136 ceph-mon[113531]: pgmap v5824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3992005013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:42.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:42.510885+0000) 2023-08-28T00:07:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:43.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:43.511153+0000) 2023-08-28T00:07:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:44 smithi136 ceph-mon[113531]: pgmap v5825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2993948505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:44.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:44.511391+0000) 2023-08-28T00:07:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:44 smithi093 ceph-mon[111335]: pgmap v5825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2993948505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:44 smithi093 ceph-mon[118902]: pgmap v5825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2993948505' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:07:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:07:45] "GET /metrics HTTP/1.1" 200 239951 "" "Prometheus/2.43.0" 2023-08-28T00:07:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:45.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:45.511656+0000) 2023-08-28T00:07:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:46 smithi136 ceph-mon[113531]: pgmap v5826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:46.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:46.511923+0000) 2023-08-28T00:07:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:46 smithi093 ceph-mon[111335]: pgmap v5826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:46 smithi093 ceph-mon[118902]: pgmap v5826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1946394874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:47.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:47.512135+0000) 2023-08-28T00:07:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1946394874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1946394874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:48 smithi136 ceph-mon[113531]: pgmap v5827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:48.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:48.512324+0000) 2023-08-28T00:07:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:48 smithi093 ceph-mon[111335]: pgmap v5827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:48 smithi093 ceph-mon[118902]: pgmap v5827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:49.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3244046758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:49.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:07:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:49.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:49.512533+0000) 2023-08-28T00:07:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3244046758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:49.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:07:49.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3244046758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:49.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:07:50.837 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:50.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:50.512693+0000) 2023-08-28T00:07:50.939 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-mon[111335]: pgmap v5828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:50.939 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:50.939 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:50.939 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-mon[118902]: pgmap v5828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:50.939 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:50.939 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:50 smithi136 ceph-mon[113531]: pgmap v5828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:50.937Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:50.938Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:51.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:51.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:51.512874+0000) 2023-08-28T00:07:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:52 smithi136 ceph-mon[113531]: pgmap v5829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/814268553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[118902]: pgmap v5829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/814268553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:52.414Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:07:52.415Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:07:52.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[111335]: pgmap v5829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:52.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:52.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:52.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/814268553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:52.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:52.513069+0000) 2023-08-28T00:07:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:53.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:53.513278+0000) 2023-08-28T00:07:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:07:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:07:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:53.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:07:53.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:07:53.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:07:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:54.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:54.513546+0000) 2023-08-28T00:07:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:54 smithi136 ceph-mon[113531]: pgmap v5830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/52347976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:54 smithi093 ceph-mon[111335]: pgmap v5830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/52347976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:54 smithi093 ceph-mon[118902]: pgmap v5830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/52347976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:54.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:07:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:07:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:07:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:07:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:55.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:55.513797+0000) 2023-08-28T00:07:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:56 smithi093 ceph-mon[111335]: pgmap v5831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/266877348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:56 smithi093 ceph-mon[118902]: pgmap v5831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/266877348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:56.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:56 smithi136 ceph-mon[113531]: pgmap v5831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:56.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/266877348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:56.943 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:56.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:56.514112+0000) 2023-08-28T00:07:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:57.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:57.514335+0000) 2023-08-28T00:07:58.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:58 smithi093 ceph-mon[111335]: pgmap v5832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:58.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:58 smithi093 ceph-mon[118902]: pgmap v5832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:58 smithi136 ceph-mon[113531]: pgmap v5832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:07:58.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:58.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:58.514601+0000) 2023-08-28T00:07:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:07:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4092777576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:59.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:07:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4092777576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:07:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4092777576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:07:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:07:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:07:59.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:07:59.514841+0000) 2023-08-28T00:08:00.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:00 smithi093 ceph-mon[111335]: pgmap v5833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:00 smithi093 ceph-mon[118902]: pgmap v5833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:00 smithi136 ceph-mon[113531]: pgmap v5833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:00.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:00.515064+0000) 2023-08-28T00:08:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:00.938Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:00.939Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:01.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1127134399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:01.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1127134399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1127134399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:01.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:01.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:01.515346+0000) 2023-08-28T00:08:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:02 smithi093 ceph-mon[111335]: pgmap v5834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:02 smithi093 ceph-mon[118902]: pgmap v5834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:02.414Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:02.416Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:02 smithi136 ceph-mon[113531]: pgmap v5834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:02.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:02.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:02.515581+0000) 2023-08-28T00:08:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2925666313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:03.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:03.515885+0000) 2023-08-28T00:08:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2925666313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2925666313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:04 smithi136 ceph-mon[113531]: pgmap v5835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:04.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:04.516167+0000) 2023-08-28T00:08:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:04 smithi093 ceph-mon[111335]: pgmap v5835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:04 smithi093 ceph-mon[118902]: pgmap v5835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:08:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:08:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:08:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:05.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:05.516408+0000) 2023-08-28T00:08:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:06 smithi136 ceph-mon[113531]: pgmap v5836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1926962123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:06.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:06.516627+0000) 2023-08-28T00:08:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:06 smithi093 ceph-mon[111335]: pgmap v5836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1926962123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:06 smithi093 ceph-mon[118902]: pgmap v5836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1926962123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:07.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:07.516849+0000) 2023-08-28T00:08:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:08 smithi136 ceph-mon[113531]: pgmap v5837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/820020352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:08.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:08.517119+0000) 2023-08-28T00:08:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:08 smithi093 ceph-mon[111335]: pgmap v5837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/820020352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:08 smithi093 ceph-mon[118902]: pgmap v5837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/820020352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:09.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:09.517365+0000) 2023-08-28T00:08:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:10 smithi136 ceph-mon[113531]: pgmap v5838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1752460255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:10.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:10.517638+0000) 2023-08-28T00:08:10.941 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:10 smithi093 ceph-mon[111335]: pgmap v5838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:10.941 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1752460255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:10.941 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:10 smithi093 ceph-mon[118902]: pgmap v5838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:10.941 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1752460255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:10.939Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:10.940Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:11.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:11.517891+0000) 2023-08-28T00:08:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:12 smithi093 ceph-mon[111335]: pgmap v5839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:12 smithi093 ceph-mon[118902]: pgmap v5839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:12.415Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:12.416Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:12 smithi136 ceph-mon[113531]: pgmap v5839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:12.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:12.518138+0000) 2023-08-28T00:08:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3382428297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:13.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:13.518308+0000) 2023-08-28T00:08:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3382428297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3382428297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:14 smithi136 ceph-mon[113531]: pgmap v5840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:14.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:14.518498+0000) 2023-08-28T00:08:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:14 smithi093 ceph-mon[111335]: pgmap v5840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:14 smithi093 ceph-mon[118902]: pgmap v5840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2117960799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2117960799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:15.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:08:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:08:15] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:08:15.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2117960799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:15.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:15.518690+0000) 2023-08-28T00:08:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:16 smithi136 ceph-mon[113531]: pgmap v5841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:16.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:16.518940+0000) 2023-08-28T00:08:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:16 smithi093 ceph-mon[111335]: pgmap v5841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:16 smithi093 ceph-mon[118902]: pgmap v5841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:17.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:17.519195+0000) 2023-08-28T00:08:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:18 smithi136 ceph-mon[113531]: pgmap v5842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/675661089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:18.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:18.519485+0000) 2023-08-28T00:08:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:18 smithi093 ceph-mon[111335]: pgmap v5842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:18.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/675661089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:18 smithi093 ceph-mon[118902]: pgmap v5842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/675661089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:19.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:19.519753+0000) 2023-08-28T00:08:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:20 smithi136 ceph-mon[113531]: pgmap v5843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2696655979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:20.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:20.520007+0000) 2023-08-28T00:08:20.941 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:20 smithi093 ceph-mon[111335]: pgmap v5843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:20.941 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2696655979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:20.941 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:20 smithi093 ceph-mon[118902]: pgmap v5843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:20.941 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2696655979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:20.940Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:20.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:21.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:21.520233+0000) 2023-08-28T00:08:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:22 smithi093 ceph-mon[111335]: pgmap v5844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2584070287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:22 smithi093 ceph-mon[118902]: pgmap v5844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2584070287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:22.415Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:22.416Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:22 smithi136 ceph-mon[113531]: pgmap v5844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2584070287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:22.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:22.520426+0000) 2023-08-28T00:08:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:23.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:23.520670+0000) 2023-08-28T00:08:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:24 smithi136 ceph-mon[113531]: pgmap v5845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:24.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:24.520940+0000) 2023-08-28T00:08:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:24 smithi093 ceph-mon[111335]: pgmap v5845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:24 smithi093 ceph-mon[118902]: pgmap v5845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4284286575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:08:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:08:25] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:08:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4284286575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4284286575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:25.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:25.521191+0000) 2023-08-28T00:08:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:26 smithi136 ceph-mon[113531]: pgmap v5846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:26.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:26.521427+0000) 2023-08-28T00:08:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:26 smithi093 ceph-mon[111335]: pgmap v5846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:26 smithi093 ceph-mon[118902]: pgmap v5846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:27.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3935822359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:27.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:27.521627+0000) 2023-08-28T00:08:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3935822359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3935822359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:28 smithi136 ceph-mon[113531]: pgmap v5847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:28.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:28.521876+0000) 2023-08-28T00:08:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:28 smithi093 ceph-mon[111335]: pgmap v5847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:28 smithi093 ceph-mon[118902]: pgmap v5847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:29.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/560911182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:29.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:29.522156+0000) 2023-08-28T00:08:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/560911182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/560911182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:30 smithi136 ceph-mon[113531]: pgmap v5848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:30.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:30.522415+0000) 2023-08-28T00:08:30.942 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:30 smithi093 ceph-mon[111335]: pgmap v5848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:30.942 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:30 smithi093 ceph-mon[118902]: pgmap v5848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:30.941Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:30.942Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:31.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:31.522722+0000) 2023-08-28T00:08:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:32 smithi093 ceph-mon[111335]: pgmap v5849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3319346039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:32 smithi093 ceph-mon[118902]: pgmap v5849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3319346039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:32.416Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:32.417Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:32 smithi136 ceph-mon[113531]: pgmap v5849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3319346039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:32.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:32.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:32.522937+0000) 2023-08-28T00:08:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:33.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:33.523175+0000) 2023-08-28T00:08:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:34 smithi136 ceph-mon[113531]: pgmap v5850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2633085629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:34.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:34.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:34.523425+0000) 2023-08-28T00:08:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:34 smithi093 ceph-mon[111335]: pgmap v5850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2633085629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:34 smithi093 ceph-mon[118902]: pgmap v5850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2633085629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:08:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:08:35] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:08:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:35.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:35.523650+0000) 2023-08-28T00:08:36.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:36 smithi136 ceph-mon[113531]: pgmap v5851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:36.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:36.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:36.523916+0000) 2023-08-28T00:08:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:36 smithi093 ceph-mon[111335]: pgmap v5851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:36 smithi093 ceph-mon[118902]: pgmap v5851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/225940307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:37.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:37.524160+0000) 2023-08-28T00:08:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/225940307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:37.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/225940307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:38 smithi136 ceph-mon[113531]: pgmap v5852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:38.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:38.524432+0000) 2023-08-28T00:08:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:38 smithi093 ceph-mon[111335]: pgmap v5852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:38 smithi093 ceph-mon[118902]: pgmap v5852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:39.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:39.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3523031452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:39.943 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:39.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:39.524619+0000) 2023-08-28T00:08:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3523031452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3523031452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:40 smithi136 ceph-mon[113531]: pgmap v5853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:40.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:40.524857+0000) 2023-08-28T00:08:40.942 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:40 smithi093 ceph-mon[111335]: pgmap v5853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:40.943 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:40 smithi093 ceph-mon[118902]: pgmap v5853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:40.942Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:40.943Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:41.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2017902861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:41.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:41.525146+0000) 2023-08-28T00:08:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2017902861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2017902861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:42 smithi093 ceph-mon[111335]: pgmap v5854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:42 smithi093 ceph-mon[118902]: pgmap v5854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:42.417Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:42.418Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:42 smithi136 ceph-mon[113531]: pgmap v5854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:42.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:42.525378+0000) 2023-08-28T00:08:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:43.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:43.525542+0000) 2023-08-28T00:08:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:44 smithi136 ceph-mon[113531]: pgmap v5855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1917330770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:44.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:44.525803+0000) 2023-08-28T00:08:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:44 smithi093 ceph-mon[111335]: pgmap v5855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1917330770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:44 smithi093 ceph-mon[118902]: pgmap v5855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1917330770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:08:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:08:45] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:08:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:45.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:45.526025+0000) 2023-08-28T00:08:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:46 smithi136 ceph-mon[113531]: pgmap v5856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3272078225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:46.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:46.526318+0000) 2023-08-28T00:08:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:46 smithi093 ceph-mon[111335]: pgmap v5856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3272078225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:46 smithi093 ceph-mon[118902]: pgmap v5856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3272078225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:47.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:47.526543+0000) 2023-08-28T00:08:48.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:48 smithi136 ceph-mon[113531]: pgmap v5857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:48.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1760940032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:48.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:48.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:48.526790+0000) 2023-08-28T00:08:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:48 smithi093 ceph-mon[111335]: pgmap v5857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1760940032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:48 smithi093 ceph-mon[118902]: pgmap v5857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1760940032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:49.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:49.527084+0000) 2023-08-28T00:08:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:50 smithi136 ceph-mon[113531]: pgmap v5858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:50.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:50.527267+0000) 2023-08-28T00:08:50.944 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:50 smithi093 ceph-mon[111335]: pgmap v5858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:50.944 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:50 smithi093 ceph-mon[118902]: pgmap v5858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:50.943Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:50.944Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/781821150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:51.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:51.527498+0000) 2023-08-28T00:08:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/781821150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:51.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/781821150' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:52 smithi093 ceph-mon[111335]: pgmap v5859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:52 smithi093 ceph-mon[118902]: pgmap v5859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:52.418Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:08:52.419Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:08:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:52 smithi136 ceph-mon[113531]: pgmap v5859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:52.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:52.527697+0000) 2023-08-28T00:08:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:08:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1980281042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:53.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:53.527894+0000) 2023-08-28T00:08:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:08:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1980281042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:08:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1980281042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:54 smithi136 ceph-mon[113531]: pgmap v5860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:54.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:54.528076+0000) 2023-08-28T00:08:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:54 smithi093 ceph-mon[111335]: pgmap v5860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:54 smithi093 ceph-mon[118902]: pgmap v5860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:08:55.723 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:08:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:08:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:55.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:55.528222+0000) 2023-08-28T00:08:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:55.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:56.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:56.528474+0000) 2023-08-28T00:08:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:56 smithi136 ceph-mon[113531]: pgmap v5861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3024638474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:08:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:08:56.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[118902]: pgmap v5861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3024638474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:08:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:08:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:56.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[111335]: pgmap v5861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:56.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3024638474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:56.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:08:56.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:08:56.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:08:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:57.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:57.528728+0000) 2023-08-28T00:08:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:58.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:58.528990+0000) 2023-08-28T00:08:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:58 smithi136 ceph-mon[113531]: pgmap v5862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:58.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:08:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1741847615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:58 smithi093 ceph-mon[118902]: pgmap v5862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:08:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1741847615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:58 smithi093 ceph-mon[111335]: pgmap v5862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:08:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:08:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1741847615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:08:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:08:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:08:59.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:08:59.529248+0000) 2023-08-28T00:09:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:00 smithi136 ceph-mon[113531]: pgmap v5863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1738952485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:00.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:00.529492+0000) 2023-08-28T00:09:00.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:00 smithi093 ceph-mon[118902]: pgmap v5863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:00.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1738952485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:00.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:00 smithi093 ceph-mon[111335]: pgmap v5863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:00.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1738952485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:00.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:00.943Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:00.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:00.944Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:01.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:01.529786+0000) 2023-08-28T00:09:02.736 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:02.419Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:02.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:02.420Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:02 smithi136 ceph-mon[113531]: pgmap v5864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:02.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:02.530064+0000) 2023-08-28T00:09:02.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:02 smithi093 ceph-mon[111335]: pgmap v5864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:02 smithi093 ceph-mon[118902]: pgmap v5864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4155420031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:03.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:03.530337+0000) 2023-08-28T00:09:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4155420031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4155420031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:04 smithi136 ceph-mon[113531]: pgmap v5865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:04.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:04.530551+0000) 2023-08-28T00:09:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:04 smithi093 ceph-mon[111335]: pgmap v5865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:04 smithi093 ceph-mon[118902]: pgmap v5865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:09:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:09:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:09:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/702758657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:05.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:05.530799+0000) 2023-08-28T00:09:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/702758657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/702758657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:06 smithi136 ceph-mon[113531]: pgmap v5866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:06.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:06.531081+0000) 2023-08-28T00:09:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:06 smithi093 ceph-mon[111335]: pgmap v5866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:06 smithi093 ceph-mon[118902]: pgmap v5866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/757901165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:07.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:07.531341+0000) 2023-08-28T00:09:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/757901165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/757901165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:08 smithi136 ceph-mon[113531]: pgmap v5867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:08.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:08.531603+0000) 2023-08-28T00:09:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:08 smithi093 ceph-mon[111335]: pgmap v5867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:08 smithi093 ceph-mon[118902]: pgmap v5867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:09.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:09.531794+0000) 2023-08-28T00:09:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:10 smithi136 ceph-mon[113531]: pgmap v5868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3300514934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:10.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:10.532073+0000) 2023-08-28T00:09:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:10 smithi093 ceph-mon[111335]: pgmap v5868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3300514934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:10 smithi093 ceph-mon[118902]: pgmap v5868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3300514934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:10.944Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:10.945Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:11.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:11.532311+0000) 2023-08-28T00:09:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:12.420Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:12.421Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:12 smithi136 ceph-mon[113531]: pgmap v5869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/613507378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:12.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:12.532544+0000) 2023-08-28T00:09:13.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:12 smithi093 ceph-mon[111335]: pgmap v5869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:13.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/613507378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:13.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:12 smithi093 ceph-mon[118902]: pgmap v5869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:13.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/613507378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:13.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:13.532827+0000) 2023-08-28T00:09:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:14 smithi136 ceph-mon[113531]: pgmap v5870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:14.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:14.533049+0000) 2023-08-28T00:09:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:14 smithi093 ceph-mon[111335]: pgmap v5870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:14 smithi093 ceph-mon[118902]: pgmap v5870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:09:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:09:15] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:09:15.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/964066295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:15.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:15.533243+0000) 2023-08-28T00:09:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/964066295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/964066295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:16 smithi136 ceph-mon[113531]: pgmap v5871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:16.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:16.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:16.533456+0000) 2023-08-28T00:09:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:16 smithi093 ceph-mon[111335]: pgmap v5871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:16 smithi093 ceph-mon[118902]: pgmap v5871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:17.784 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:17.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:17.533689+0000) 2023-08-28T00:09:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/495119898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/495119898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/495119898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:18.786 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:18.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:18.533987+0000) 2023-08-28T00:09:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:18 smithi136 ceph-mon[113531]: pgmap v5872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:18 smithi093 ceph-mon[111335]: pgmap v5872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:18 smithi093 ceph-mon[118902]: pgmap v5872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:19.788 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:19.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:19.534235+0000) 2023-08-28T00:09:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1711331608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1711331608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1711331608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:20.793 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:20.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:20.534507+0000) 2023-08-28T00:09:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:20 smithi136 ceph-mon[113531]: pgmap v5873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:20 smithi093 ceph-mon[111335]: pgmap v5873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:20 smithi093 ceph-mon[118902]: pgmap v5873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:20.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:20.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:21.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:21.534772+0000) 2023-08-28T00:09:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:22.420Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:22.421Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:22.795 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:22.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:22.535014+0000) 2023-08-28T00:09:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:22 smithi136 ceph-mon[113531]: pgmap v5874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2263667623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:22 smithi093 ceph-mon[111335]: pgmap v5874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2263667623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:22 smithi093 ceph-mon[118902]: pgmap v5874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2263667623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:23.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:23.535271+0000) 2023-08-28T00:09:24.798 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:24.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:24.535528+0000) 2023-08-28T00:09:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:24 smithi136 ceph-mon[113531]: pgmap v5875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/388046164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:24 smithi093 ceph-mon[111335]: pgmap v5875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/388046164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:24 smithi093 ceph-mon[118902]: pgmap v5875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/388046164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:09:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:09:25] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:09:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:25.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:25.535748+0000) 2023-08-28T00:09:26.803 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:26.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:26.536053+0000) 2023-08-28T00:09:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:26 smithi136 ceph-mon[113531]: pgmap v5876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1183762383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:26 smithi093 ceph-mon[111335]: pgmap v5876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1183762383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:26 smithi093 ceph-mon[118902]: pgmap v5876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1183762383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:27.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:27.536311+0000) 2023-08-28T00:09:28.805 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:28.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:28.536522+0000) 2023-08-28T00:09:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:28 smithi136 ceph-mon[113531]: pgmap v5877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:28 smithi093 ceph-mon[111335]: pgmap v5877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:28 smithi093 ceph-mon[118902]: pgmap v5877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:29.812 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:29.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:29.536767+0000) 2023-08-28T00:09:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3909269197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3909269197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3909269197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:30.815 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:30.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:30.537083+0000) 2023-08-28T00:09:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:30 smithi136 ceph-mon[113531]: pgmap v5878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:30 smithi093 ceph-mon[111335]: pgmap v5878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:30 smithi093 ceph-mon[118902]: pgmap v5878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:30.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:30.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:31.825 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:31.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:31.537329+0000) 2023-08-28T00:09:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2738698642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2738698642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2738698642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:32.421Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:32.422Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:32.826 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:32.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:32.537546+0000) 2023-08-28T00:09:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:32 smithi136 ceph-mon[113531]: pgmap v5879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:32 smithi093 ceph-mon[111335]: pgmap v5879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:32 smithi093 ceph-mon[118902]: pgmap v5879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:33.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:33.537801+0000) 2023-08-28T00:09:34.828 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:34.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:34.538078+0000) 2023-08-28T00:09:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:34 smithi136 ceph-mon[113531]: pgmap v5880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/449937691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:34 smithi093 ceph-mon[111335]: pgmap v5880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/449937691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:34 smithi093 ceph-mon[118902]: pgmap v5880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/449937691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:09:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:09:35] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:09:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:35.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:35.538361+0000) 2023-08-28T00:09:36.833 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:36.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:36.538623+0000) 2023-08-28T00:09:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:36 smithi136 ceph-mon[113531]: pgmap v5881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1520212936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:36 smithi093 ceph-mon[111335]: pgmap v5881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1520212936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:36 smithi093 ceph-mon[118902]: pgmap v5881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1520212936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:37.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:37.538863+0000) 2023-08-28T00:09:38.841 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:38.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:38.539139+0000) 2023-08-28T00:09:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:38 smithi136 ceph-mon[113531]: pgmap v5882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471236083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:39.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:38 smithi093 ceph-mon[111335]: pgmap v5882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:39.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471236083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:39.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:39.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:38 smithi093 ceph-mon[118902]: pgmap v5882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:39.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471236083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:39.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:39.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:39.539404+0000) 2023-08-28T00:09:40.844 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:40.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:40.539658+0000) 2023-08-28T00:09:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:40 smithi136 ceph-mon[113531]: pgmap v5883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:40 smithi093 ceph-mon[118902]: pgmap v5883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:40 smithi093 ceph-mon[111335]: pgmap v5883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:40.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:40.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:41.847 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:41.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:41.539907+0000) 2023-08-28T00:09:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/618367202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/618367202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/618367202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:42.422Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:42.422Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:42.849 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:42.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:42.540145+0000) 2023-08-28T00:09:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:42 smithi136 ceph-mon[113531]: pgmap v5884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:42 smithi093 ceph-mon[111335]: pgmap v5884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:42 smithi093 ceph-mon[118902]: pgmap v5884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:43.856 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:43.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:43.540398+0000) 2023-08-28T00:09:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2818086976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2818086976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2818086976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:44.855 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:44.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:44.540656+0000) 2023-08-28T00:09:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:44 smithi136 ceph-mon[113531]: pgmap v5885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:44 smithi093 ceph-mon[111335]: pgmap v5885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:44 smithi093 ceph-mon[118902]: pgmap v5885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:09:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:09:45] "GET /metrics HTTP/1.1" 200 239951 "" "Prometheus/2.43.0" 2023-08-28T00:09:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:45.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:45.540923+0000) 2023-08-28T00:09:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:46 smithi093 ceph-mon[111335]: pgmap v5886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3264329849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:46 smithi093 ceph-mon[118902]: pgmap v5886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:46.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3264329849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:46 smithi136 ceph-mon[113531]: pgmap v5886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3264329849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:46.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:46.541233+0000) 2023-08-28T00:09:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:47.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:47.541439+0000) 2023-08-28T00:09:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:48 smithi136 ceph-mon[113531]: pgmap v5887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2271323044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:48.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:48.541684+0000) 2023-08-28T00:09:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:48 smithi093 ceph-mon[118902]: pgmap v5887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2271323044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:48 smithi093 ceph-mon[111335]: pgmap v5887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2271323044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:49.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:49.541927+0000) 2023-08-28T00:09:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:50 smithi136 ceph-mon[113531]: pgmap v5888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:50.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:50.542176+0000) 2023-08-28T00:09:50.947 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:50 smithi093 ceph-mon[111335]: pgmap v5888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:50.948 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:50 smithi093 ceph-mon[118902]: pgmap v5888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:50.946Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:50.947Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2725625264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:51.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:51.542377+0000) 2023-08-28T00:09:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2725625264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:51.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2725625264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:52 smithi093 ceph-mon[111335]: pgmap v5889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:52 smithi093 ceph-mon[118902]: pgmap v5889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:52.422Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:09:52.423Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:09:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:52 smithi136 ceph-mon[113531]: pgmap v5889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:52.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:52.542618+0000) 2023-08-28T00:09:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2833445417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:53.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:53.542889+0000) 2023-08-28T00:09:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2833445417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2833445417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:54 smithi136 ceph-mon[113531]: pgmap v5890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:54.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:54.543082+0000) 2023-08-28T00:09:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:54 smithi093 ceph-mon[111335]: pgmap v5890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:54 smithi093 ceph-mon[118902]: pgmap v5890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:09:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/712368028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:09:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:09:55] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:09:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/712368028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/712368028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:55.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:55.543347+0000) 2023-08-28T00:09:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:56 smithi093 ceph-mon[111335]: pgmap v5891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:56.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:09:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:56 smithi093 ceph-mon[118902]: pgmap v5891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:56.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:09:56.794 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:56 smithi136 ceph-mon[113531]: pgmap v5891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:56.794 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:09:56.794 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:56.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:56.543570+0000) 2023-08-28T00:09:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:57.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:57.543727+0000) 2023-08-28T00:09:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:58.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:58.543904+0000) 2023-08-28T00:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:58 smithi136 ceph-mon[113531]: pgmap v5892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3573954118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:58.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[111335]: pgmap v5892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3573954118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[118902]: pgmap v5892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3573954118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:09:58.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:58.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:09:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:09:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:09:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:09:59.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:09:59.544112+0000) 2023-08-28T00:09:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:09:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:09:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:09:59.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:09:59.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:09:59.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:10:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:00.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:00.544287+0000) 2023-08-28T00:10:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:00 smithi136 ceph-mon[113531]: pgmap v5893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:10:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2704083130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:00.949 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-mon[111335]: pgmap v5893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:00.949 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:10:00.949 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2704083130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:00.949 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-mon[118902]: pgmap v5893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:00.949 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:10:00.949 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2704083130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:00.947Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:00.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:01.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:01.544574+0000) 2023-08-28T00:10:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:02 smithi093 ceph-mon[111335]: pgmap v5894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2530806373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:02 smithi093 ceph-mon[118902]: pgmap v5894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2530806373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:02.423Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:02.424Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:02 smithi136 ceph-mon[113531]: pgmap v5894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2530806373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:02.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:02.544817+0000) 2023-08-28T00:10:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:03.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:03.545000+0000) 2023-08-28T00:10:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:04 smithi136 ceph-mon[113531]: pgmap v5895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:04.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:04.545250+0000) 2023-08-28T00:10:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:04 smithi093 ceph-mon[111335]: pgmap v5895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:04 smithi093 ceph-mon[118902]: pgmap v5895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:05.654 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:10:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:10:05] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:10:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2793315897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:05.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:05.545420+0000) 2023-08-28T00:10:05.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2793315897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2793315897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:06 smithi136 ceph-mon[113531]: pgmap v5896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:06.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:06.545658+0000) 2023-08-28T00:10:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:06 smithi093 ceph-mon[111335]: pgmap v5896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:06 smithi093 ceph-mon[118902]: pgmap v5896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3516215685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:07.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:07.545935+0000) 2023-08-28T00:10:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3516215685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3516215685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:08 smithi136 ceph-mon[113531]: pgmap v5897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:08.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:08.546167+0000) 2023-08-28T00:10:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:08 smithi093 ceph-mon[111335]: pgmap v5897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:08 smithi093 ceph-mon[118902]: pgmap v5897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/127099462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:09.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:09.546383+0000) 2023-08-28T00:10:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/127099462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/127099462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:10 smithi136 ceph-mon[113531]: pgmap v5898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:10.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:10.546642+0000) 2023-08-28T00:10:10.949 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:10 smithi093 ceph-mon[111335]: pgmap v5898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:10.949 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:10 smithi093 ceph-mon[118902]: pgmap v5898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:10.948Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:10.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:11.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:11.546911+0000) 2023-08-28T00:10:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:12 smithi093 ceph-mon[111335]: pgmap v5899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4224203589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:12 smithi093 ceph-mon[118902]: pgmap v5899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4224203589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:12.424Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:12.425Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:12 smithi136 ceph-mon[113531]: pgmap v5899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4224203589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:12.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:12.547185+0000) 2023-08-28T00:10:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:13.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:13.547423+0000) 2023-08-28T00:10:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:14 smithi136 ceph-mon[113531]: pgmap v5900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2209410504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:14.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:14.547641+0000) 2023-08-28T00:10:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:14 smithi093 ceph-mon[111335]: pgmap v5900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2209410504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:14 smithi093 ceph-mon[118902]: pgmap v5900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2209410504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:10:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:10:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T00:10:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:15.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:15.547878+0000) 2023-08-28T00:10:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:16 smithi136 ceph-mon[113531]: pgmap v5901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:16.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:16.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:16.548147+0000) 2023-08-28T00:10:16.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:16 smithi093 ceph-mon[111335]: pgmap v5901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:16 smithi093 ceph-mon[118902]: pgmap v5901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/76871407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:17.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:17.548336+0000) 2023-08-28T00:10:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/76871407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/76871407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:18 smithi136 ceph-mon[113531]: pgmap v5902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:18.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:18.548592+0000) 2023-08-28T00:10:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:18 smithi093 ceph-mon[111335]: pgmap v5902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:18 smithi093 ceph-mon[118902]: pgmap v5902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1089782526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:19.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:19.548810+0000) 2023-08-28T00:10:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1089782526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1089782526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:20 smithi136 ceph-mon[113531]: pgmap v5903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:20.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:20.549080+0000) 2023-08-28T00:10:20.949 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:20 smithi093 ceph-mon[111335]: pgmap v5903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:20.949 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:20 smithi093 ceph-mon[118902]: pgmap v5903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:20.948Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:20.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:21.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2949339581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:21.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:21.549346+0000) 2023-08-28T00:10:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2949339581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2949339581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:22.701 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:22.424Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:22.701 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:22.425Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:22 smithi136 ceph-mon[113531]: pgmap v5904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:22.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:22.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:22.549600+0000) 2023-08-28T00:10:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:22 smithi093 ceph-mon[111335]: pgmap v5904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:22 smithi093 ceph-mon[118902]: pgmap v5904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:23.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:23.549819+0000) 2023-08-28T00:10:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:24 smithi136 ceph-mon[113531]: pgmap v5905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1024468856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:24.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:24.550112+0000) 2023-08-28T00:10:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:24 smithi093 ceph-mon[118902]: pgmap v5905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1024468856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:24 smithi093 ceph-mon[111335]: pgmap v5905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:24.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1024468856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:10:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:10:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:10:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:25.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:25.550391+0000) 2023-08-28T00:10:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:26 smithi136 ceph-mon[113531]: pgmap v5906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/790518067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:26.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:26.550645+0000) 2023-08-28T00:10:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:26 smithi093 ceph-mon[111335]: pgmap v5906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/790518067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:26 smithi093 ceph-mon[118902]: pgmap v5906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/790518067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:27.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:27.550940+0000) 2023-08-28T00:10:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:28 smithi136 ceph-mon[113531]: pgmap v5907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2041102555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:28.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:28.551155+0000) 2023-08-28T00:10:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:28 smithi093 ceph-mon[111335]: pgmap v5907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2041102555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:28 smithi093 ceph-mon[118902]: pgmap v5907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2041102555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:29.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:29.551406+0000) 2023-08-28T00:10:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:30 smithi136 ceph-mon[113531]: pgmap v5908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:30.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:30.551679+0000) 2023-08-28T00:10:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:30 smithi093 ceph-mon[118902]: pgmap v5908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:30.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:30 smithi093 ceph-mon[111335]: pgmap v5908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:30.948Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:30.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:30.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:31.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2996804584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:31.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:31.551956+0000) 2023-08-28T00:10:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2996804584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2996804584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:32.718 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:32.424Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:32.718 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:32.425Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:32.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:32 smithi136 ceph-mon[113531]: pgmap v5909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:32.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:32.552164+0000) 2023-08-28T00:10:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:32 smithi093 ceph-mon[111335]: pgmap v5909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:32 smithi093 ceph-mon[118902]: pgmap v5909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1487034259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:33.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:33.552384+0000) 2023-08-28T00:10:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1487034259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:33.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1487034259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:34 smithi136 ceph-mon[113531]: pgmap v5910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:34.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:34.552586+0000) 2023-08-28T00:10:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:34 smithi093 ceph-mon[111335]: pgmap v5910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:34 smithi093 ceph-mon[118902]: pgmap v5910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:10:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:10:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:10:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:35.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:35.552833+0000) 2023-08-28T00:10:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:36 smithi136 ceph-mon[113531]: pgmap v5911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4181131677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:36.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:36.553115+0000) 2023-08-28T00:10:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:36 smithi093 ceph-mon[111335]: pgmap v5911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4181131677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:36 smithi093 ceph-mon[118902]: pgmap v5911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4181131677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:37.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:37.553328+0000) 2023-08-28T00:10:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:38 smithi136 ceph-mon[113531]: pgmap v5912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2748948862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:38.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:38.553593+0000) 2023-08-28T00:10:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:38 smithi093 ceph-mon[111335]: pgmap v5912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2748948862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:38 smithi093 ceph-mon[118902]: pgmap v5912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2748948862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:39.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:39.553875+0000) 2023-08-28T00:10:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:40 smithi136 ceph-mon[113531]: pgmap v5913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2822280587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:40.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:40.554087+0000) 2023-08-28T00:10:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:40 smithi093 ceph-mon[111335]: pgmap v5913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2822280587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:40 smithi093 ceph-mon[118902]: pgmap v5913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2822280587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:40.949Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:40.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:40.950Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:41.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:41.554331+0000) 2023-08-28T00:10:42.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:42.425Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:42.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:42.426Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:42 smithi136 ceph-mon[113531]: pgmap v5914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:42.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:42.554578+0000) 2023-08-28T00:10:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:42 smithi093 ceph-mon[111335]: pgmap v5914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:42 smithi093 ceph-mon[118902]: pgmap v5914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:43.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1629609091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:43.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:43.554812+0000) 2023-08-28T00:10:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1629609091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1629609091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:44 smithi136 ceph-mon[113531]: pgmap v5915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:44.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:44.555082+0000) 2023-08-28T00:10:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:44 smithi093 ceph-mon[111335]: pgmap v5915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:44 smithi093 ceph-mon[118902]: pgmap v5915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:10:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:10:45] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:10:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2489001227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:45.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:45.555318+0000) 2023-08-28T00:10:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2489001227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2489001227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:46 smithi136 ceph-mon[113531]: pgmap v5916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:46.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:46.555608+0000) 2023-08-28T00:10:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:46 smithi093 ceph-mon[111335]: pgmap v5916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:46 smithi093 ceph-mon[118902]: pgmap v5916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:47.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:47.555879+0000) 2023-08-28T00:10:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:48 smithi136 ceph-mon[113531]: pgmap v5917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3530653382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:48.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:48.556148+0000) 2023-08-28T00:10:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:48 smithi093 ceph-mon[111335]: pgmap v5917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3530653382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:48 smithi093 ceph-mon[118902]: pgmap v5917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3530653382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:49.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:49.556428+0000) 2023-08-28T00:10:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:50 smithi136 ceph-mon[113531]: pgmap v5918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/390599732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:50.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:50.556682+0000) 2023-08-28T00:10:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:50 smithi093 ceph-mon[111335]: pgmap v5918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/390599732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:51.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:50 smithi093 ceph-mon[118902]: pgmap v5918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:51.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/390599732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:50.950Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:50.951Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:51.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:51.556889+0000) 2023-08-28T00:10:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:52.426Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:10:52.427Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:10:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:52.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:52.557159+0000) 2023-08-28T00:10:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:52 smithi136 ceph-mon[113531]: pgmap v5919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3185746303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:52 smithi093 ceph-mon[111335]: pgmap v5919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3185746303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:52 smithi093 ceph-mon[118902]: pgmap v5919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3185746303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:53.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:53.557427+0000) 2023-08-28T00:10:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:54 smithi136 ceph-mon[113531]: pgmap v5920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:54.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:54.557687+0000) 2023-08-28T00:10:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:54 smithi093 ceph-mon[111335]: pgmap v5920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:55.278 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:55.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:54 smithi093 ceph-mon[118902]: pgmap v5920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:55.279 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:10:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:10:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:10:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:10:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1769914539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:55.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:55.557955+0000) 2023-08-28T00:10:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1769914539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1769914539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:56 smithi136 ceph-mon[113531]: pgmap v5921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:56.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:56.558203+0000) 2023-08-28T00:10:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:56 smithi093 ceph-mon[111335]: pgmap v5921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:56 smithi093 ceph-mon[118902]: pgmap v5921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2564478995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:57.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:57.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:57.558406+0000) 2023-08-28T00:10:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2564478995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2564478995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:10:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:58 smithi136 ceph-mon[113531]: pgmap v5922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:10:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:58.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:58.558659+0000) 2023-08-28T00:10:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:58 smithi093 ceph-mon[111335]: pgmap v5922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:10:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:58 smithi093 ceph-mon[118902]: pgmap v5922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:10:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:10:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:10:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:10:59.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:10:59.558828+0000) 2023-08-28T00:11:00.131 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:10:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1045394638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:00.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:10:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1045394638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:10:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1045394638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:00 smithi136 ceph-mon[113531]: pgmap v5923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:00.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:00.558976+0000) 2023-08-28T00:11:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-mon[111335]: pgmap v5923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-mon[118902]: pgmap v5923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:00.951Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:00.952Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:01.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:01.559215+0000) 2023-08-28T00:11:02.427 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:11:02.428 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:11:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:11:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:11:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:02.426Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:02.427Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:02.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:02.559478+0000) 2023-08-28T00:11:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:03 smithi136 ceph-mon[113531]: pgmap v5924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1172814758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:03 smithi093 ceph-mon[111335]: pgmap v5924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1172814758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:03 smithi093 ceph-mon[118902]: pgmap v5924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:03.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1172814758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:03.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:03.559751+0000) 2023-08-28T00:11:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:04.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:04.560059+0000) 2023-08-28T00:11:05.317 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:05 smithi136 ceph-mon[113531]: pgmap v5925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:05.318 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1576934662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:05 smithi093 ceph-mon[111335]: pgmap v5925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:05.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1576934662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:05 smithi093 ceph-mon[118902]: pgmap v5925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1576934662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:05.690 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 00:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T00:11:05.317184004Z level=info msg="Completed cleanup jobs" duration=12.680117ms 2023-08-28T00:11:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:05.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:05.560312+0000) 2023-08-28T00:11:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:11:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:11:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:11:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:06.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:06.560538+0000) 2023-08-28T00:11:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:07 smithi136 ceph-mon[113531]: pgmap v5926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2747597073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:07 smithi093 ceph-mon[111335]: pgmap v5926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2747597073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:07 smithi093 ceph-mon[118902]: pgmap v5926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2747597073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:07.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:07.560745+0000) 2023-08-28T00:11:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:08.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:08.561009+0000) 2023-08-28T00:11:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:09 smithi136 ceph-mon[113531]: pgmap v5927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:09 smithi093 ceph-mon[111335]: pgmap v5927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:09 smithi093 ceph-mon[118902]: pgmap v5927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:09.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:09.561284+0000) 2023-08-28T00:11:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/297577517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/297577517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/297577517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:10.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:10.561503+0000) 2023-08-28T00:11:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:11 smithi093 ceph-mon[111335]: pgmap v5928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:11 smithi093 ceph-mon[118902]: pgmap v5928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:10.952Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:10.953Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:11 smithi136 ceph-mon[113531]: pgmap v5928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:11.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:11.561760+0000) 2023-08-28T00:11:12.428 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2936512410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:12.428 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2936512410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2936512410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:12.427Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:12.428Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:12.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:12.562017+0000) 2023-08-28T00:11:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:13 smithi136 ceph-mon[113531]: pgmap v5929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:13 smithi093 ceph-mon[111335]: pgmap v5929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:13 smithi093 ceph-mon[118902]: pgmap v5929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:13.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:13.562305+0000) 2023-08-28T00:11:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/534209779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/534209779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/534209779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:14.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:14.562553+0000) 2023-08-28T00:11:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:15 smithi093 ceph-mon[111335]: pgmap v5930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:15 smithi093 ceph-mon[118902]: pgmap v5930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:15 smithi136 ceph-mon[113531]: pgmap v5930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:11:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:11:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:11:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:15.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:15.562823+0000) 2023-08-28T00:11:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:16 smithi136 ceph-mon[113531]: pgmap v5931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2212529611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:16.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:16.563112+0000) 2023-08-28T00:11:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:16 smithi093 ceph-mon[111335]: pgmap v5931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2212529611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:16 smithi093 ceph-mon[118902]: pgmap v5931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2212529611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:17.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:17.563360+0000) 2023-08-28T00:11:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:18 smithi136 ceph-mon[113531]: pgmap v5932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:18.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:18.563573+0000) 2023-08-28T00:11:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:18 smithi093 ceph-mon[111335]: pgmap v5932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:18 smithi093 ceph-mon[118902]: pgmap v5932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3930986752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:19.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:19.563817+0000) 2023-08-28T00:11:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3930986752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3930986752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:20 smithi136 ceph-mon[113531]: pgmap v5933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:20.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:20.564076+0000) 2023-08-28T00:11:20.954 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:20 smithi093 ceph-mon[111335]: pgmap v5933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:20.954 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:20 smithi093 ceph-mon[118902]: pgmap v5933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:20.953Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:20.954Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:21.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3630662748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:21.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:21.564314+0000) 2023-08-28T00:11:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3630662748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:21.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3630662748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:22 smithi093 ceph-mon[111335]: pgmap v5934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:22 smithi093 ceph-mon[118902]: pgmap v5934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:22.428Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:22.429Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:22 smithi136 ceph-mon[113531]: pgmap v5934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:22.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:22.564562+0000) 2023-08-28T00:11:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3573145243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:23.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:23.564813+0000) 2023-08-28T00:11:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3573145243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3573145243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:24 smithi136 ceph-mon[113531]: pgmap v5935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:24.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:24.565091+0000) 2023-08-28T00:11:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:24 smithi093 ceph-mon[111335]: pgmap v5935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:24 smithi093 ceph-mon[118902]: pgmap v5935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:11:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:11:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:11:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:25.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:25.565330+0000) 2023-08-28T00:11:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:26 smithi136 ceph-mon[113531]: pgmap v5936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2441491331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:26.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:26.565615+0000) 2023-08-28T00:11:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:26 smithi093 ceph-mon[111335]: pgmap v5936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2441491331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:26 smithi093 ceph-mon[118902]: pgmap v5936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2441491331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:27.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:27.565897+0000) 2023-08-28T00:11:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:28 smithi136 ceph-mon[113531]: pgmap v5937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/759068306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:28.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:28.566115+0000) 2023-08-28T00:11:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:28 smithi093 ceph-mon[111335]: pgmap v5937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/759068306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:28 smithi093 ceph-mon[118902]: pgmap v5937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/759068306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:29.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:29.566392+0000) 2023-08-28T00:11:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:30 smithi136 ceph-mon[113531]: pgmap v5938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:30.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:30.566667+0000) 2023-08-28T00:11:30.955 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:30 smithi093 ceph-mon[111335]: pgmap v5938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:30.955 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:30 smithi093 ceph-mon[118902]: pgmap v5938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:30.954Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:30.955Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:31.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1540504317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:31.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:31.566906+0000) 2023-08-28T00:11:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1540504317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1540504317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:32 smithi093 ceph-mon[111335]: pgmap v5939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:32 smithi093 ceph-mon[118902]: pgmap v5939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:32.429Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:32.429Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:32 smithi136 ceph-mon[113531]: pgmap v5939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:32.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:32.567157+0000) 2023-08-28T00:11:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3231228310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:33.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:33.567402+0000) 2023-08-28T00:11:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3231228310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3231228310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:34 smithi136 ceph-mon[113531]: pgmap v5940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:34.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:34.567657+0000) 2023-08-28T00:11:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:34 smithi093 ceph-mon[111335]: pgmap v5940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:34 smithi093 ceph-mon[118902]: pgmap v5940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2377221466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:11:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:11:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:11:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2377221466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2377221466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:35.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:35.567927+0000) 2023-08-28T00:11:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:36 smithi136 ceph-mon[113531]: pgmap v5941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:36.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:36.568144+0000) 2023-08-28T00:11:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:36 smithi093 ceph-mon[111335]: pgmap v5941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:36 smithi093 ceph-mon[118902]: pgmap v5941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:37.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:37.568340+0000) 2023-08-28T00:11:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:38 smithi136 ceph-mon[113531]: pgmap v5942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1002524314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:38.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:38.568493+0000) 2023-08-28T00:11:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:38 smithi093 ceph-mon[118902]: pgmap v5942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1002524314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:38 smithi093 ceph-mon[111335]: pgmap v5942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:38.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1002524314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:39.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:39.568718+0000) 2023-08-28T00:11:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:40 smithi136 ceph-mon[113531]: pgmap v5943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2501485501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:40.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:40.568966+0000) 2023-08-28T00:11:40.955 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:40 smithi093 ceph-mon[111335]: pgmap v5943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:40.956 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2501485501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:40.956 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:40 smithi093 ceph-mon[118902]: pgmap v5943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:40.956 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2501485501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:40.955Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:40.956Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:41.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:41.569226+0000) 2023-08-28T00:11:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:42 smithi093 ceph-mon[111335]: pgmap v5944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1155171282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:42 smithi093 ceph-mon[118902]: pgmap v5944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1155171282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:42.430Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:42.430Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:42 smithi136 ceph-mon[113531]: pgmap v5944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1155171282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:42.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:42.569491+0000) 2023-08-28T00:11:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:43.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:43.569735+0000) 2023-08-28T00:11:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:44 smithi136 ceph-mon[113531]: pgmap v5945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:44.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:44.569969+0000) 2023-08-28T00:11:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:44 smithi093 ceph-mon[111335]: pgmap v5945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:44 smithi093 ceph-mon[118902]: pgmap v5945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:45.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2138371101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:11:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:11:45] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:11:45.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2138371101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2138371101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:45.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:45.570219+0000) 2023-08-28T00:11:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:46 smithi136 ceph-mon[113531]: pgmap v5946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:46.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:46.570489+0000) 2023-08-28T00:11:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:46 smithi093 ceph-mon[111335]: pgmap v5946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:46 smithi093 ceph-mon[118902]: pgmap v5946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1598710059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:47.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:47.570768+0000) 2023-08-28T00:11:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1598710059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1598710059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:48 smithi136 ceph-mon[113531]: pgmap v5947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:48.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:48.571028+0000) 2023-08-28T00:11:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:48 smithi093 ceph-mon[111335]: pgmap v5947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:48 smithi093 ceph-mon[118902]: pgmap v5947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:49.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:49.571312+0000) 2023-08-28T00:11:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:50 smithi136 ceph-mon[113531]: pgmap v5948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1903027076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:50.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:50.571526+0000) 2023-08-28T00:11:50.956 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:50 smithi093 ceph-mon[111335]: pgmap v5948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:50.956 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1903027076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:50.957 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:50 smithi093 ceph-mon[118902]: pgmap v5948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:50.957 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1903027076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:50.956Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:50.957Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:51.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:51.571762+0000) 2023-08-28T00:11:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:52 smithi093 ceph-mon[111335]: pgmap v5949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3631279671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:52 smithi093 ceph-mon[118902]: pgmap v5949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3631279671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:52.431Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:11:52.431Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:11:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:52 smithi136 ceph-mon[113531]: pgmap v5949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3631279671' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:52.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:52.572012+0000) 2023-08-28T00:11:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:53.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:53.572272+0000) 2023-08-28T00:11:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:54 smithi136 ceph-mon[113531]: pgmap v5950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3405104512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:54.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:54.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:54.572516+0000) 2023-08-28T00:11:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:54 smithi093 ceph-mon[111335]: pgmap v5950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:54.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3405104512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:54 smithi093 ceph-mon[118902]: pgmap v5950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:11:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3405104512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:11:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:11:55] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:11:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:55.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:55.572761+0000) 2023-08-28T00:11:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:56 smithi136 ceph-mon[113531]: pgmap v5951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:56.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:56.573054+0000) 2023-08-28T00:11:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:56 smithi093 ceph-mon[111335]: pgmap v5951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:56 smithi093 ceph-mon[118902]: pgmap v5951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3723086191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:57.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:57.573272+0000) 2023-08-28T00:11:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3723086191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3723086191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:58 smithi136 ceph-mon[113531]: pgmap v5952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:58.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:58.573532+0000) 2023-08-28T00:11:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:58 smithi093 ceph-mon[111335]: pgmap v5952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:58 smithi093 ceph-mon[118902]: pgmap v5952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:11:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:11:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4237779188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:11:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:11:59.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:11:59.573806+0000) 2023-08-28T00:11:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:11:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4237779188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:11:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:11:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4237779188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:00 smithi136 ceph-mon[113531]: pgmap v5953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:00.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:00.574097+0000) 2023-08-28T00:12:00.957 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:00 smithi093 ceph-mon[111335]: pgmap v5953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:00.957 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:00 smithi093 ceph-mon[118902]: pgmap v5953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:00.956Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:00.957Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:01.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:12:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:01.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:01.574367+0000) 2023-08-28T00:12:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:12:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:12:02.725 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:02 smithi093 ceph-mon[111335]: pgmap v5954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:02.726 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1397682746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:02.726 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:02 smithi093 ceph-mon[118902]: pgmap v5954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:02.726 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1397682746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:02.726 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:02.431Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:02.726 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:02.432Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:02 smithi136 ceph-mon[113531]: pgmap v5954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1397682746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:02.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:02.574535+0000) 2023-08-28T00:12:03.798 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:03.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:03.574685+0000) 2023-08-28T00:12:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[111335]: pgmap v5955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1184629261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[118902]: pgmap v5955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1184629261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:04.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:04.574842+0000) 2023-08-28T00:12:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:04 smithi136 ceph-mon[113531]: pgmap v5955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1184629261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:12:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:12:05] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:12:05.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:12:05.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:12:05.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:12:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:12:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:12:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:12:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:12:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:05.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:05.574996+0000) 2023-08-28T00:12:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:06 smithi136 ceph-mon[113531]: pgmap v5956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1400622748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:06.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:06.575264+0000) 2023-08-28T00:12:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:06 smithi093 ceph-mon[111335]: pgmap v5956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1400622748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:06 smithi093 ceph-mon[118902]: pgmap v5956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1400622748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:07.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:07.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:07.575489+0000) 2023-08-28T00:12:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:08 smithi136 ceph-mon[113531]: pgmap v5957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:08.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:08.575749+0000) 2023-08-28T00:12:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:08 smithi093 ceph-mon[111335]: pgmap v5957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:08 smithi093 ceph-mon[118902]: pgmap v5957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3996346419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:09.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:09.576008+0000) 2023-08-28T00:12:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3996346419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3996346419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:10 smithi136 ceph-mon[113531]: pgmap v5958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:10.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:10.576246+0000) 2023-08-28T00:12:10.958 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:10 smithi093 ceph-mon[111335]: pgmap v5958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:10.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:10 smithi093 ceph-mon[118902]: pgmap v5958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:10.957Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:10.958Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:11.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1531218014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:11.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:11.576449+0000) 2023-08-28T00:12:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1531218014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1531218014' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:12 smithi093 ceph-mon[111335]: pgmap v5959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:12 smithi093 ceph-mon[118902]: pgmap v5959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:12.432Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:12.433Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:12 smithi136 ceph-mon[113531]: pgmap v5959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:12.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:12.576717+0000) 2023-08-28T00:12:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:13.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:13.576966+0000) 2023-08-28T00:12:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:14 smithi136 ceph-mon[113531]: pgmap v5960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/271723487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:14.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:14.577209+0000) 2023-08-28T00:12:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:14 smithi093 ceph-mon[111335]: pgmap v5960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/271723487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:14 smithi093 ceph-mon[118902]: pgmap v5960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/271723487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:12:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:12:15] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:12:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:15.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:15.577415+0000) 2023-08-28T00:12:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:16 smithi136 ceph-mon[113531]: pgmap v5961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2417933896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:16.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:16.577688+0000) 2023-08-28T00:12:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:16 smithi093 ceph-mon[111335]: pgmap v5961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2417933896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:16 smithi093 ceph-mon[118902]: pgmap v5961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2417933896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:17.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:17.577929+0000) 2023-08-28T00:12:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:18 smithi136 ceph-mon[113531]: pgmap v5962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2077764616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:18.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:18.578186+0000) 2023-08-28T00:12:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:18 smithi093 ceph-mon[111335]: pgmap v5962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:18.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2077764616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:18 smithi093 ceph-mon[118902]: pgmap v5962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2077764616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:19.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:19.578381+0000) 2023-08-28T00:12:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:20 smithi136 ceph-mon[113531]: pgmap v5963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:20.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:20.578625+0000) 2023-08-28T00:12:20.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:20 smithi093 ceph-mon[111335]: pgmap v5963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:20.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:20 smithi093 ceph-mon[118902]: pgmap v5963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:20.958Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:20.959Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:21.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1047301091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:21.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:21.578822+0000) 2023-08-28T00:12:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1047301091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1047301091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:22 smithi093 ceph-mon[111335]: pgmap v5964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:22 smithi093 ceph-mon[118902]: pgmap v5964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:22.432Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:22.433Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:22 smithi136 ceph-mon[113531]: pgmap v5964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:22.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:22.579089+0000) 2023-08-28T00:12:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/644472225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:23.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:23.579321+0000) 2023-08-28T00:12:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/644472225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/644472225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:24 smithi136 ceph-mon[113531]: pgmap v5965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:24.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:24.579546+0000) 2023-08-28T00:12:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:24 smithi093 ceph-mon[111335]: pgmap v5965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:24 smithi093 ceph-mon[118902]: pgmap v5965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2614331218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:12:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:12:25] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:12:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2614331218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2614331218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:25.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:25.579760+0000) 2023-08-28T00:12:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:26 smithi136 ceph-mon[113531]: pgmap v5966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:26.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:26.580046+0000) 2023-08-28T00:12:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:26 smithi093 ceph-mon[111335]: pgmap v5966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:26 smithi093 ceph-mon[118902]: pgmap v5966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:27.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:27.580298+0000) 2023-08-28T00:12:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:28 smithi136 ceph-mon[113531]: pgmap v5967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3037114647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:28.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:28.580472+0000) 2023-08-28T00:12:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:28 smithi093 ceph-mon[118902]: pgmap v5967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3037114647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:28.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:28 smithi093 ceph-mon[111335]: pgmap v5967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:28.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3037114647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:29.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:29.580716+0000) 2023-08-28T00:12:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:30 smithi136 ceph-mon[113531]: pgmap v5968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1701210901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:30.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:30.580956+0000) 2023-08-28T00:12:30.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:30 smithi093 ceph-mon[111335]: pgmap v5968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:30.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1701210901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:30.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:30 smithi093 ceph-mon[118902]: pgmap v5968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:30.960 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1701210901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:30.958Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:30.959Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:31.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:31.581148+0000) 2023-08-28T00:12:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:32 smithi093 ceph-mon[111335]: pgmap v5969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:32 smithi093 ceph-mon[118902]: pgmap v5969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:32.432Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:32.433Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:32 smithi136 ceph-mon[113531]: pgmap v5969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:32.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:32.581300+0000) 2023-08-28T00:12:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2159142740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:33.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:33.581460+0000) 2023-08-28T00:12:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2159142740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2159142740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:34 smithi136 ceph-mon[113531]: pgmap v5970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:34.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:34.581659+0000) 2023-08-28T00:12:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:34 smithi093 ceph-mon[111335]: pgmap v5970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:34 smithi093 ceph-mon[118902]: pgmap v5970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3888367468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:12:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:12:35] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:12:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3888367468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3888367468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:35.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:35.581869+0000) 2023-08-28T00:12:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:36 smithi136 ceph-mon[113531]: pgmap v5971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:36.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:36.582098+0000) 2023-08-28T00:12:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:36 smithi093 ceph-mon[111335]: pgmap v5971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:36 smithi093 ceph-mon[118902]: pgmap v5971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2323773370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:37.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:37.582241+0000) 2023-08-28T00:12:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2323773370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2323773370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:38 smithi136 ceph-mon[113531]: pgmap v5972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:38.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:38.582405+0000) 2023-08-28T00:12:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:38 smithi093 ceph-mon[111335]: pgmap v5972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:38 smithi093 ceph-mon[118902]: pgmap v5972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:39.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:39.582658+0000) 2023-08-28T00:12:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:40 smithi136 ceph-mon[113531]: pgmap v5973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2085666185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:40.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:40.582909+0000) 2023-08-28T00:12:40.960 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:40 smithi093 ceph-mon[118902]: pgmap v5973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:40.960 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2085666185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:40.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:40 smithi093 ceph-mon[111335]: pgmap v5973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:40.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2085666185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:40.959Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:40.960Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:41.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:41.583110+0000) 2023-08-28T00:12:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:42 smithi093 ceph-mon[111335]: pgmap v5974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1508216385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:42 smithi093 ceph-mon[118902]: pgmap v5974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1508216385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:42.433Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:42.434Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:42 smithi136 ceph-mon[113531]: pgmap v5974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1508216385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:42.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:42.583257+0000) 2023-08-28T00:12:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:43.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:43.583403+0000) 2023-08-28T00:12:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:44 smithi136 ceph-mon[113531]: pgmap v5975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3642154309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:44.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:44.583564+0000) 2023-08-28T00:12:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:44 smithi093 ceph-mon[111335]: pgmap v5975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3642154309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:44 smithi093 ceph-mon[118902]: pgmap v5975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3642154309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:12:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:12:45] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:12:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:45.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:45.583716+0000) 2023-08-28T00:12:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:46 smithi136 ceph-mon[113531]: pgmap v5976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:46.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:46.584060+0000) 2023-08-28T00:12:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:46 smithi093 ceph-mon[111335]: pgmap v5976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:46 smithi093 ceph-mon[118902]: pgmap v5976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3484903162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:47.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:47.584333+0000) 2023-08-28T00:12:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3484903162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3484903162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:48 smithi136 ceph-mon[113531]: pgmap v5977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:48.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:48.584596+0000) 2023-08-28T00:12:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:48 smithi093 ceph-mon[111335]: pgmap v5977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:48 smithi093 ceph-mon[118902]: pgmap v5977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:49.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/471344215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:49.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:49.584828+0000) 2023-08-28T00:12:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/471344215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/471344215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:50 smithi136 ceph-mon[113531]: pgmap v5978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:50.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:50.585076+0000) 2023-08-28T00:12:50.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:50 smithi093 ceph-mon[111335]: pgmap v5978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:50.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:50 smithi093 ceph-mon[118902]: pgmap v5978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:50.960Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:50.961Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:51.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:51.585310+0000) 2023-08-28T00:12:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:52 smithi093 ceph-mon[111335]: pgmap v5979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2640226519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:52 smithi093 ceph-mon[118902]: pgmap v5979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2640226519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:52.433Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:12:52.434Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:12:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:52 smithi136 ceph-mon[113531]: pgmap v5979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2640226519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:52.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:52.585584+0000) 2023-08-28T00:12:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:53.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:53.585840+0000) 2023-08-28T00:12:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:54.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:54.586096+0000) 2023-08-28T00:12:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:54 smithi136 ceph-mon[113531]: pgmap v5980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:54.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:54.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3922285498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:54 smithi093 ceph-mon[111335]: pgmap v5980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:54.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3922285498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:54 smithi093 ceph-mon[118902]: pgmap v5980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:12:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3922285498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:12:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:12:55] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:12:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:55.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:55.586307+0000) 2023-08-28T00:12:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:56.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:56.586549+0000) 2023-08-28T00:12:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:56 smithi136 ceph-mon[113531]: pgmap v5981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1378994741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:56 smithi093 ceph-mon[111335]: pgmap v5981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1378994741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:56 smithi093 ceph-mon[118902]: pgmap v5981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1378994741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:57.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:57.586771+0000) 2023-08-28T00:12:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:58 smithi136 ceph-mon[113531]: pgmap v5982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:58.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:58.587012+0000) 2023-08-28T00:12:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:58 smithi093 ceph-mon[111335]: pgmap v5982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:58 smithi093 ceph-mon[118902]: pgmap v5982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:12:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:12:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1101401279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:12:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:12:59.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:12:59.587279+0000) 2023-08-28T00:12:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:12:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1101401279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:12:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:12:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1101401279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:00 smithi136 ceph-mon[113531]: pgmap v5983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:00.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:00.587514+0000) 2023-08-28T00:13:00.962 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:00 smithi093 ceph-mon[111335]: pgmap v5983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:00.962 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:00 smithi093 ceph-mon[118902]: pgmap v5983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:00.963 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:00.960Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:00.962Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1606609682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:01.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:01.587755+0000) 2023-08-28T00:13:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1606609682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1606609682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:02 smithi093 ceph-mon[111335]: pgmap v5984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:02 smithi093 ceph-mon[118902]: pgmap v5984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:02.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:02.434Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:02.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:02.435Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:02 smithi136 ceph-mon[113531]: pgmap v5984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:02.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:02.588059+0000) 2023-08-28T00:13:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2795634360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:03.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:03.588318+0000) 2023-08-28T00:13:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2795634360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2795634360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:04 smithi136 ceph-mon[113531]: pgmap v5985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:13:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:04.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:04.588600+0000) 2023-08-28T00:13:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:04 smithi093 ceph-mon[111335]: pgmap v5985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:13:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:04 smithi093 ceph-mon[118902]: pgmap v5985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:13:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:13:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:13:05] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:13:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:05.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:05.588784+0000) 2023-08-28T00:13:06.753 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:06 smithi136 ceph-mon[113531]: pgmap v5986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:06.754 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/486799297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:06.754 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:06.754 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:06.754 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:06.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:06.588960+0000) 2023-08-28T00:13:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[111335]: pgmap v5986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/486799297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[118902]: pgmap v5986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/486799297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:07.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:07.589139+0000) 2023-08-28T00:13:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:13:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:13:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:13:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:13:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:13:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:13:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:13:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:08.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:08.589401+0000) 2023-08-28T00:13:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:09 smithi136 ceph-mon[113531]: pgmap v5987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2085903702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:09 smithi093 ceph-mon[111335]: pgmap v5987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2085903702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:09 smithi093 ceph-mon[118902]: pgmap v5987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2085903702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:09.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:09.589664+0000) 2023-08-28T00:13:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:10.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:10.589831+0000) 2023-08-28T00:13:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:11 smithi093 ceph-mon[111335]: pgmap v5988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1468825390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:11 smithi093 ceph-mon[118902]: pgmap v5988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1468825390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:10.961Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:10.962Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:11.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:11 smithi136 ceph-mon[113531]: pgmap v5988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:11.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1468825390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:11.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:11.590045+0000) 2023-08-28T00:13:12.435 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:12 smithi093 ceph-mon[111335]: pgmap v5989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:12.435 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:12 smithi093 ceph-mon[118902]: pgmap v5989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:12 smithi136 ceph-mon[113531]: pgmap v5989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:12.434Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:12.435Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:12.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:12.590303+0000) 2023-08-28T00:13:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2266410896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:13.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2266410896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:13.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2266410896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:13.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:13.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:13.590547+0000) 2023-08-28T00:13:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:14 smithi093 ceph-mon[111335]: pgmap v5990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:14 smithi093 ceph-mon[118902]: pgmap v5990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:14.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:14 smithi136 ceph-mon[113531]: pgmap v5990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:14.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:14.590794+0000) 2023-08-28T00:13:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:13:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:13:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:13:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:15.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:15.590995+0000) 2023-08-28T00:13:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:16 smithi136 ceph-mon[113531]: pgmap v5991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2716010223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:16.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:16.591260+0000) 2023-08-28T00:13:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:16 smithi093 ceph-mon[111335]: pgmap v5991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2716010223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:16 smithi093 ceph-mon[118902]: pgmap v5991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2716010223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:17.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:17.591407+0000) 2023-08-28T00:13:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:18 smithi136 ceph-mon[113531]: pgmap v5992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1509769222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:18.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:18.591680+0000) 2023-08-28T00:13:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:18 smithi093 ceph-mon[111335]: pgmap v5992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1509769222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:18 smithi093 ceph-mon[118902]: pgmap v5992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1509769222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:19.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:19.591962+0000) 2023-08-28T00:13:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:20 smithi136 ceph-mon[113531]: pgmap v5993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/906090981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:20.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:20.592218+0000) 2023-08-28T00:13:20.964 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:20 smithi093 ceph-mon[111335]: pgmap v5993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:20.964 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/906090981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:20.964 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:20 smithi093 ceph-mon[118902]: pgmap v5993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:20.964 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/906090981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:20.963Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:20.964Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:21.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:21.592523+0000) 2023-08-28T00:13:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:22 smithi093 ceph-mon[111335]: pgmap v5994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:22 smithi093 ceph-mon[118902]: pgmap v5994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:22.435Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:22.435Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:22 smithi136 ceph-mon[113531]: pgmap v5994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:22.954 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:22.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:22.592784+0000) 2023-08-28T00:13:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1436527367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:23.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:23.593051+0000) 2023-08-28T00:13:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1436527367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1436527367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:24 smithi136 ceph-mon[113531]: pgmap v5995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:24.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:24.593226+0000) 2023-08-28T00:13:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:24 smithi093 ceph-mon[111335]: pgmap v5995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:24 smithi093 ceph-mon[118902]: pgmap v5995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1734407224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:13:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:13:25] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:13:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1734407224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1734407224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:25.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:25.593391+0000) 2023-08-28T00:13:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:26 smithi136 ceph-mon[113531]: pgmap v5996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:26.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:26.593652+0000) 2023-08-28T00:13:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:26 smithi093 ceph-mon[111335]: pgmap v5996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:26 smithi093 ceph-mon[118902]: pgmap v5996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:27.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/980802674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:27.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:27.593847+0000) 2023-08-28T00:13:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/980802674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/980802674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:28 smithi136 ceph-mon[113531]: pgmap v5997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:28.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:28.594099+0000) 2023-08-28T00:13:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:28 smithi093 ceph-mon[111335]: pgmap v5997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:28 smithi093 ceph-mon[118902]: pgmap v5997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:29.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:29.594341+0000) 2023-08-28T00:13:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:30 smithi136 ceph-mon[113531]: pgmap v5998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4005213896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:30.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:30.594589+0000) 2023-08-28T00:13:30.965 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:30 smithi093 ceph-mon[111335]: pgmap v5998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:30.965 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4005213896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:30.965 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:30 smithi093 ceph-mon[118902]: pgmap v5998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:30.965 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4005213896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:30.964Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:30.965Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:31.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:31.594850+0000) 2023-08-28T00:13:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:32 smithi093 ceph-mon[111335]: pgmap v5999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3400308426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:32 smithi093 ceph-mon[118902]: pgmap v5999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3400308426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:32.435Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:32.436Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:32 smithi136 ceph-mon[113531]: pgmap v5999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3400308426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:32.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:32.595092+0000) 2023-08-28T00:13:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:33.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:33.595357+0000) 2023-08-28T00:13:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:34 smithi136 ceph-mon[113531]: pgmap v6000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:34.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:34.595568+0000) 2023-08-28T00:13:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:34 smithi093 ceph-mon[111335]: pgmap v6000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:34 smithi093 ceph-mon[118902]: pgmap v6000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/266905949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:13:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:13:35] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T00:13:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/266905949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/266905949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:35.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:35.595741+0000) 2023-08-28T00:13:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:36 smithi136 ceph-mon[113531]: pgmap v6001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:36.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:36.595978+0000) 2023-08-28T00:13:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:36 smithi093 ceph-mon[111335]: pgmap v6001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:36 smithi093 ceph-mon[118902]: pgmap v6001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1200388840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:37.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:37.596133+0000) 2023-08-28T00:13:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1200388840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1200388840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:38 smithi136 ceph-mon[113531]: pgmap v6002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:38.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:38.596302+0000) 2023-08-28T00:13:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:38 smithi093 ceph-mon[111335]: pgmap v6002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:38 smithi093 ceph-mon[118902]: pgmap v6002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3301680089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:39.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:39.596487+0000) 2023-08-28T00:13:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3301680089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3301680089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:40 smithi136 ceph-mon[113531]: pgmap v6003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:40.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:40.596734+0000) 2023-08-28T00:13:40.966 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:40 smithi093 ceph-mon[111335]: pgmap v6003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:40.966 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:40 smithi093 ceph-mon[118902]: pgmap v6003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:40.965Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:40.966Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:41.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:41.596980+0000) 2023-08-28T00:13:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:42 smithi093 ceph-mon[111335]: pgmap v6004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1967368023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:42 smithi093 ceph-mon[118902]: pgmap v6004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:42.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1967368023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:42.436Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:42.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:42.437Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:42.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:42.597244+0000) 2023-08-28T00:13:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:42 smithi136 ceph-mon[113531]: pgmap v6004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1967368023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:43.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:43.597483+0000) 2023-08-28T00:13:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:44 smithi136 ceph-mon[113531]: pgmap v6005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3476023099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:44.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:44.597779+0000) 2023-08-28T00:13:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:44 smithi093 ceph-mon[111335]: pgmap v6005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3476023099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:44 smithi093 ceph-mon[118902]: pgmap v6005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3476023099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:13:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:13:45] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T00:13:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:45.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:45.598077+0000) 2023-08-28T00:13:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:46 smithi136 ceph-mon[113531]: pgmap v6006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2313580438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:46.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:46.598403+0000) 2023-08-28T00:13:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:46 smithi093 ceph-mon[111335]: pgmap v6006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2313580438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:46 smithi093 ceph-mon[118902]: pgmap v6006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:46.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2313580438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:47.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:47.598614+0000) 2023-08-28T00:13:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:48 smithi136 ceph-mon[113531]: pgmap v6007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:48.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:48.598864+0000) 2023-08-28T00:13:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:48 smithi093 ceph-mon[111335]: pgmap v6007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:48 smithi093 ceph-mon[118902]: pgmap v6007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:49.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/365270960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:49.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:49.599150+0000) 2023-08-28T00:13:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/365270960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/365270960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:50 smithi136 ceph-mon[113531]: pgmap v6008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:50.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:50.599410+0000) 2023-08-28T00:13:50.967 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:50 smithi093 ceph-mon[111335]: pgmap v6008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:50.967 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:50 smithi093 ceph-mon[118902]: pgmap v6008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:50.966Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:50.967Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/265181899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:51.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:51.599672+0000) 2023-08-28T00:13:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/265181899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/265181899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:52 smithi093 ceph-mon[111335]: pgmap v6009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:52 smithi093 ceph-mon[118902]: pgmap v6009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:52.436Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:13:52.437Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:13:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:52 smithi136 ceph-mon[113531]: pgmap v6009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:52.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:52.599898+0000) 2023-08-28T00:13:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:53.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:53.600179+0000) 2023-08-28T00:13:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:54.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:54.600392+0000) 2023-08-28T00:13:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:54 smithi136 ceph-mon[113531]: pgmap v6010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4024558916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:54 smithi093 ceph-mon[111335]: pgmap v6010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4024558916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:54 smithi093 ceph-mon[118902]: pgmap v6010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4024558916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:13:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:13:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:13:55] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:13:55.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:55.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:55.600647+0000) 2023-08-28T00:13:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:56 smithi136 ceph-mon[113531]: pgmap v6011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1866713636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:56.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:56.600885+0000) 2023-08-28T00:13:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:56 smithi093 ceph-mon[111335]: pgmap v6011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1866713636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:56 smithi093 ceph-mon[118902]: pgmap v6011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1866713636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:57.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:57.601175+0000) 2023-08-28T00:13:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:58 smithi136 ceph-mon[113531]: pgmap v6012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:13:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3819535331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:58.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:58.601411+0000) 2023-08-28T00:13:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:58 smithi093 ceph-mon[111335]: pgmap v6012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:13:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3819535331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:58 smithi093 ceph-mon[118902]: pgmap v6012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:13:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:13:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3819535331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:13:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:13:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:13:59.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:13:59.601683+0000) 2023-08-28T00:14:00.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:00 smithi136 ceph-mon[113531]: pgmap v6013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:00.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:00.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:00.601979+0000) 2023-08-28T00:14:00.967 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:00 smithi093 ceph-mon[111335]: pgmap v6013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:00.967 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:00 smithi093 ceph-mon[118902]: pgmap v6013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:00.967Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:00.967Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3807511048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:01.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:01.602161+0000) 2023-08-28T00:14:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3807511048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3807511048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:02 smithi093 ceph-mon[111335]: pgmap v6014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:02 smithi093 ceph-mon[118902]: pgmap v6014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:02.437Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:02.438Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:02 smithi136 ceph-mon[113531]: pgmap v6014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:02.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:02.602383+0000) 2023-08-28T00:14:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2789552743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:03.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:03.602612+0000) 2023-08-28T00:14:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2789552743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2789552743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:04 smithi136 ceph-mon[113531]: pgmap v6015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:04.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:04.602819+0000) 2023-08-28T00:14:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:04 smithi093 ceph-mon[111335]: pgmap v6015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:04 smithi093 ceph-mon[118902]: pgmap v6015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:14:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:14:05] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:14:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:05.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:05.603079+0000) 2023-08-28T00:14:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:06 smithi136 ceph-mon[113531]: pgmap v6016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2498583814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:06.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:06.603291+0000) 2023-08-28T00:14:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:06 smithi093 ceph-mon[111335]: pgmap v6016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2498583814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:06 smithi093 ceph-mon[118902]: pgmap v6016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2498583814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:07.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:07.603529+0000) 2023-08-28T00:14:08.904 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:08 smithi093 ceph-mon[111335]: pgmap v6017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:08.905 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:14:08.905 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3380593709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:08.905 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:08 smithi093 ceph-mon[118902]: pgmap v6017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:08.905 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:14:08.905 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3380593709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:08 smithi136 ceph-mon[113531]: pgmap v6017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:14:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3380593709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:08.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:08.603708+0000) 2023-08-28T00:14:09.745 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:09.745 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:09.745 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:09.745 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:09.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:09.603860+0000) 2023-08-28T00:14:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:10.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:10.604068+0000) 2023-08-28T00:14:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[111335]: pgmap v6018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2247777726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:11.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:14:11.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:14:11.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:10.967Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:10.969Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[118902]: pgmap v6018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2247777726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:11.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:14:11.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:14:11.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-mon[113531]: pgmap v6018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2247777726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:14:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:14:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:14:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:11.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:11.604299+0000) 2023-08-28T00:14:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:12.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:12.438Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:12.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:12.604533+0000) 2023-08-28T00:14:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:13 smithi136 ceph-mon[113531]: pgmap v6019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1333672780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:13 smithi093 ceph-mon[118902]: pgmap v6019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1333672780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:13 smithi093 ceph-mon[111335]: pgmap v6019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1333672780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:13.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:13.604785+0000) 2023-08-28T00:14:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:14.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:14.605093+0000) 2023-08-28T00:14:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:15 smithi093 ceph-mon[111335]: pgmap v6020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:15 smithi093 ceph-mon[118902]: pgmap v6020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:15 smithi136 ceph-mon[113531]: pgmap v6020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:14:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:14:15] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:14:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:15.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:15.605295+0000) 2023-08-28T00:14:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2140894332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2140894332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2140894332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:16.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:16.605605+0000) 2023-08-28T00:14:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:17 smithi136 ceph-mon[113531]: pgmap v6021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:17 smithi093 ceph-mon[111335]: pgmap v6021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:17 smithi093 ceph-mon[118902]: pgmap v6021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:17.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:17.605837+0000) 2023-08-28T00:14:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/288523125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/288523125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/288523125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:18.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:18.606104+0000) 2023-08-28T00:14:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:19 smithi136 ceph-mon[113531]: pgmap v6022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:19 smithi093 ceph-mon[111335]: pgmap v6022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:19 smithi093 ceph-mon[118902]: pgmap v6022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:19.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:19.606330+0000) 2023-08-28T00:14:20.069 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2325190744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2325190744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2325190744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:20.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:20.606587+0000) 2023-08-28T00:14:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:21 smithi093 ceph-mon[118902]: pgmap v6023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:21.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:21 smithi093 ceph-mon[111335]: pgmap v6023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:20.969Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:20.969Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:21 smithi136 ceph-mon[113531]: pgmap v6023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:21.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:21.606822+0000) 2023-08-28T00:14:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:22.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:22.439Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:22.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:22.607128+0000) 2023-08-28T00:14:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:23 smithi136 ceph-mon[113531]: pgmap v6024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2369990917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:23 smithi093 ceph-mon[118902]: pgmap v6024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2369990917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:23 smithi093 ceph-mon[111335]: pgmap v6024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2369990917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:23.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:23.607320+0000) 2023-08-28T00:14:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:24.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:24.607561+0000) 2023-08-28T00:14:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:25 smithi093 ceph-mon[111335]: pgmap v6025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2506794808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:25.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:25 smithi093 ceph-mon[118902]: pgmap v6025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:25.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2506794808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:25 smithi136 ceph-mon[113531]: pgmap v6025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2506794808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:14:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:14:25] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:14:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:25.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:25.607795+0000) 2023-08-28T00:14:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:26 smithi136 ceph-mon[113531]: pgmap v6026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:26.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:26.608062+0000) 2023-08-28T00:14:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:26 smithi093 ceph-mon[111335]: pgmap v6026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:26 smithi093 ceph-mon[118902]: pgmap v6026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:27.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4112221855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:27.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:27.608351+0000) 2023-08-28T00:14:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4112221855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4112221855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:28 smithi136 ceph-mon[113531]: pgmap v6027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:28.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:28.608595+0000) 2023-08-28T00:14:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:28 smithi093 ceph-mon[111335]: pgmap v6027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:28 smithi093 ceph-mon[118902]: pgmap v6027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:29.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1394231472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:29.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:29.608800+0000) 2023-08-28T00:14:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1394231472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1394231472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:30 smithi136 ceph-mon[113531]: pgmap v6028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:30.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:30.609085+0000) 2023-08-28T00:14:30.971 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:30 smithi093 ceph-mon[111335]: pgmap v6028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:30.971 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:30 smithi093 ceph-mon[118902]: pgmap v6028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:30.970Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:30.971Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:31.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:31.609291+0000) 2023-08-28T00:14:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:32 smithi093 ceph-mon[111335]: pgmap v6029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/640016451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:32 smithi093 ceph-mon[118902]: pgmap v6029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/640016451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:32.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:32.439Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:32 smithi136 ceph-mon[113531]: pgmap v6029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/640016451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:32.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:32.609467+0000) 2023-08-28T00:14:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:33.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:33.609710+0000) 2023-08-28T00:14:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:34 smithi136 ceph-mon[113531]: pgmap v6030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/651733111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:34.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:34.609958+0000) 2023-08-28T00:14:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:34 smithi093 ceph-mon[111335]: pgmap v6030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/651733111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:34 smithi093 ceph-mon[118902]: pgmap v6030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/651733111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:14:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:14:35] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T00:14:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:35.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:35.610149+0000) 2023-08-28T00:14:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:36 smithi136 ceph-mon[113531]: pgmap v6031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:36.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:36.610397+0000) 2023-08-28T00:14:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:36 smithi093 ceph-mon[111335]: pgmap v6031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:36 smithi093 ceph-mon[118902]: pgmap v6031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1665020489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:37.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:37.610652+0000) 2023-08-28T00:14:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1665020489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1665020489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:38 smithi136 ceph-mon[113531]: pgmap v6032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:38.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:38.610889+0000) 2023-08-28T00:14:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:38 smithi093 ceph-mon[111335]: pgmap v6032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:38 smithi093 ceph-mon[118902]: pgmap v6032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/357147576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:39.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:39.611121+0000) 2023-08-28T00:14:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/357147576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/357147576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:40 smithi136 ceph-mon[113531]: pgmap v6033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:40.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:40.611353+0000) 2023-08-28T00:14:40.971 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:40 smithi093 ceph-mon[111335]: pgmap v6033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:40.971 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:40 smithi093 ceph-mon[118902]: pgmap v6033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:40.970Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:40.972Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:41.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1614342342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:41.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:41.611615+0000) 2023-08-28T00:14:41.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1614342342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1614342342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:42 smithi093 ceph-mon[111335]: pgmap v6034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:42 smithi093 ceph-mon[118902]: pgmap v6034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:42.439Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:42.440Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:42 smithi136 ceph-mon[113531]: pgmap v6034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:42.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:42.611886+0000) 2023-08-28T00:14:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:43.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:43.612159+0000) 2023-08-28T00:14:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:44 smithi136 ceph-mon[113531]: pgmap v6035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2350018587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:44.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:44.612396+0000) 2023-08-28T00:14:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:44 smithi093 ceph-mon[111335]: pgmap v6035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2350018587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:44 smithi093 ceph-mon[118902]: pgmap v6035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2350018587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:14:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:14:45] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:14:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:45.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:45.612571+0000) 2023-08-28T00:14:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:46.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:46.612789+0000) 2023-08-28T00:14:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:46 smithi136 ceph-mon[113531]: pgmap v6036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2956345051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:46 smithi093 ceph-mon[111335]: pgmap v6036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2956345051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:46 smithi093 ceph-mon[118902]: pgmap v6036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2956345051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:47.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:47.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:47.612991+0000) 2023-08-28T00:14:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:48 smithi136 ceph-mon[113531]: pgmap v6037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3652420016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:48.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:48.613228+0000) 2023-08-28T00:14:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:48 smithi093 ceph-mon[111335]: pgmap v6037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3652420016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:48 smithi093 ceph-mon[118902]: pgmap v6037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3652420016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:49.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:49.613499+0000) 2023-08-28T00:14:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:50 smithi136 ceph-mon[113531]: pgmap v6038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:50.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:50.613759+0000) 2023-08-28T00:14:50.972 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:50 smithi093 ceph-mon[111335]: pgmap v6038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:50.972 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:50 smithi093 ceph-mon[118902]: pgmap v6038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:50.971Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:50.972Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1071718554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:51.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:51.613974+0000) 2023-08-28T00:14:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1071718554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:51.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1071718554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:52 smithi093 ceph-mon[111335]: pgmap v6039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:52 smithi093 ceph-mon[118902]: pgmap v6039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:52.440Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:14:52.441Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:14:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:52 smithi136 ceph-mon[113531]: pgmap v6039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:52.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:52.614175+0000) 2023-08-28T00:14:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3503149414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:53.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:53.614358+0000) 2023-08-28T00:14:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3503149414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3503149414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:54 smithi136 ceph-mon[113531]: pgmap v6040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:54.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:54.614524+0000) 2023-08-28T00:14:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:54 smithi093 ceph-mon[118902]: pgmap v6040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:54 smithi093 ceph-mon[111335]: pgmap v6040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:54.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:14:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:14:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:14:55] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T00:14:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:55.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:55.614729+0000) 2023-08-28T00:14:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:56 smithi136 ceph-mon[113531]: pgmap v6041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2129158844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:56.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:56.614963+0000) 2023-08-28T00:14:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:56 smithi093 ceph-mon[118902]: pgmap v6041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2129158844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:56 smithi093 ceph-mon[111335]: pgmap v6041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:56.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2129158844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:57.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:57.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:57.615189+0000) 2023-08-28T00:14:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:58.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:58.615468+0000) 2023-08-28T00:14:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:58 smithi136 ceph-mon[113531]: pgmap v6042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:14:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2404103116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:58 smithi093 ceph-mon[118902]: pgmap v6042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:14:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2404103116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:58 smithi093 ceph-mon[111335]: pgmap v6042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:14:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:14:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2404103116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:14:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:14:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:14:59.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:14:59.615760+0000) 2023-08-28T00:15:00.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:00 smithi136 ceph-mon[113531]: pgmap v6043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2252416392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:00.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:00.616024+0000) 2023-08-28T00:15:00.972 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:00 smithi093 ceph-mon[111335]: pgmap v6043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:00.973 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2252416392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:00.973 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:00 smithi093 ceph-mon[118902]: pgmap v6043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:00.973 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2252416392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:00.972Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:00.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:01.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:01.616221+0000) 2023-08-28T00:15:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:02 smithi093 ceph-mon[111335]: pgmap v6044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:02 smithi093 ceph-mon[118902]: pgmap v6044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:02.440Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:02.441Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:02 smithi136 ceph-mon[113531]: pgmap v6044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:02.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:02.616436+0000) 2023-08-28T00:15:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4071716481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:03.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:03.616707+0000) 2023-08-28T00:15:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4071716481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4071716481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:04 smithi136 ceph-mon[113531]: pgmap v6045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:04.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:04.617011+0000) 2023-08-28T00:15:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:04 smithi093 ceph-mon[111335]: pgmap v6045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:04 smithi093 ceph-mon[118902]: pgmap v6045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:05.640 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:15:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:15:05] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T00:15:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4269039330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:05.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:05.617263+0000) 2023-08-28T00:15:05.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4269039330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4269039330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:06 smithi136 ceph-mon[113531]: pgmap v6046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:06.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:06.617543+0000) 2023-08-28T00:15:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:06 smithi093 ceph-mon[111335]: pgmap v6046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:06 smithi093 ceph-mon[118902]: pgmap v6046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1715064507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:07.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:07.617826+0000) 2023-08-28T00:15:07.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1715064507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:07.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1715064507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:08 smithi136 ceph-mon[113531]: pgmap v6047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:08.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:08.618129+0000) 2023-08-28T00:15:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:08 smithi093 ceph-mon[111335]: pgmap v6047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:08 smithi093 ceph-mon[118902]: pgmap v6047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:09.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:09.618405+0000) 2023-08-28T00:15:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:10 smithi136 ceph-mon[113531]: pgmap v6048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/974742171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:15:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:10.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:10.618652+0000) 2023-08-28T00:15:10.974 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-mon[111335]: pgmap v6048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:10.974 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/974742171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:10.974 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:15:10.974 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-mon[118902]: pgmap v6048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:10.974 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/974742171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:10.974 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:15:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:10.972Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:10.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:11.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:11.618838+0000) 2023-08-28T00:15:12.634 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:12.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:12.619071+0000) 2023-08-28T00:15:12.657 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:12.442Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:12.657 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:12.443Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:12 smithi136 ceph-mon[113531]: pgmap v6049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1337206708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[111335]: pgmap v6049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1337206708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:12.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:12.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[118902]: pgmap v6049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:12.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1337206708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:12.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:12.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:13.878 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:13.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:13.619287+0000) 2023-08-28T00:15:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:15:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:15:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:15:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:15:14.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:15:14.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:15:14.890 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:14.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:14.619531+0000) 2023-08-28T00:15:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:14 smithi136 ceph-mon[113531]: pgmap v6050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4266948943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:14 smithi093 ceph-mon[111335]: pgmap v6050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4266948943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:14 smithi093 ceph-mon[118902]: pgmap v6050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4266948943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:15:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:15:15] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:15:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:15.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:15.619801+0000) 2023-08-28T00:15:16.895 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:16.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:16.620087+0000) 2023-08-28T00:15:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:16 smithi136 ceph-mon[113531]: pgmap v6051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:16 smithi093 ceph-mon[111335]: pgmap v6051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:16 smithi093 ceph-mon[118902]: pgmap v6051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:17.902 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:17.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:17.620307+0000) 2023-08-28T00:15:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/38419179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/38419179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/38419179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:18.903 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:18.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:18.620558+0000) 2023-08-28T00:15:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:18 smithi136 ceph-mon[113531]: pgmap v6052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:18 smithi093 ceph-mon[111335]: pgmap v6052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:18 smithi093 ceph-mon[118902]: pgmap v6052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:19.908 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:19.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:19.620758+0000) 2023-08-28T00:15:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/71854705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/71854705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/71854705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:20.909 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:20.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:20.621014+0000) 2023-08-28T00:15:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:20 smithi136 ceph-mon[113531]: pgmap v6053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:20 smithi093 ceph-mon[111335]: pgmap v6053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:20 smithi093 ceph-mon[118902]: pgmap v6053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:20.973Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:20.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:21.913 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:21.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:21.621298+0000) 2023-08-28T00:15:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3552092358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3552092358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3552092358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:22.442Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:22.444Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:22.916 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:22.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:22.621559+0000) 2023-08-28T00:15:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:22 smithi136 ceph-mon[113531]: pgmap v6054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:22 smithi093 ceph-mon[111335]: pgmap v6054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:22 smithi093 ceph-mon[118902]: pgmap v6054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:23.920 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:23.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:23.621767+0000) 2023-08-28T00:15:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:24.922 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:24.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:24.621990+0000) 2023-08-28T00:15:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:24 smithi136 ceph-mon[113531]: pgmap v6055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2846082142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:24 smithi093 ceph-mon[111335]: pgmap v6055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2846082142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:24 smithi093 ceph-mon[118902]: pgmap v6055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2846082142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:15:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:15:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:15:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:25.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:25.622141+0000) 2023-08-28T00:15:26.925 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:26.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:26.622395+0000) 2023-08-28T00:15:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:26 smithi136 ceph-mon[113531]: pgmap v6056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1483511035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:26 smithi093 ceph-mon[111335]: pgmap v6056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1483511035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:26 smithi093 ceph-mon[118902]: pgmap v6056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1483511035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:27.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:27.622638+0000) 2023-08-28T00:15:28.928 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:28.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:28.622835+0000) 2023-08-28T00:15:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:28 smithi136 ceph-mon[113531]: pgmap v6057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:28 smithi093 ceph-mon[111335]: pgmap v6057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:28 smithi093 ceph-mon[118902]: pgmap v6057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:29.933 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:29.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:29.623115+0000) 2023-08-28T00:15:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3090444315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3090444315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3090444315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:30.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:30.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:30.623388+0000) 2023-08-28T00:15:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:30 smithi136 ceph-mon[113531]: pgmap v6058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:30 smithi093 ceph-mon[111335]: pgmap v6058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:30 smithi093 ceph-mon[118902]: pgmap v6058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:30.973Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:30.974Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:31.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:31.623552+0000) 2023-08-28T00:15:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4078840278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4078840278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4078840278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:32.443Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:32.444Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:32.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:32.623789+0000) 2023-08-28T00:15:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:32 smithi093 ceph-mon[111335]: pgmap v6059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:32 smithi093 ceph-mon[118902]: pgmap v6059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:32 smithi136 ceph-mon[113531]: pgmap v6059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:33.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:33.624058+0000) 2023-08-28T00:15:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4222061944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4222061944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4222061944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:34.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:34.624328+0000) 2023-08-28T00:15:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:34 smithi093 ceph-mon[111335]: pgmap v6060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:34 smithi093 ceph-mon[118902]: pgmap v6060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:34 smithi136 ceph-mon[113531]: pgmap v6060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:15:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:15:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T00:15:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:35.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:35.624579+0000) 2023-08-28T00:15:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:36.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:36.624849+0000) 2023-08-28T00:15:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:36 smithi093 ceph-mon[111335]: pgmap v6061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3210790570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:36 smithi093 ceph-mon[118902]: pgmap v6061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3210790570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:36 smithi136 ceph-mon[113531]: pgmap v6061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3210790570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:37.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:37.625111+0000) 2023-08-28T00:15:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:38.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:38.625310+0000) 2023-08-28T00:15:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:38 smithi093 ceph-mon[111335]: pgmap v6062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/426484910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:38 smithi093 ceph-mon[118902]: pgmap v6062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/426484910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:38 smithi136 ceph-mon[113531]: pgmap v6062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/426484910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:39.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:39.625588+0000) 2023-08-28T00:15:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:40.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:40.625806+0000) 2023-08-28T00:15:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:40 smithi093 ceph-mon[111335]: pgmap v6063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:41.597 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3108893175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:41.598 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:40 smithi093 ceph-mon[118902]: pgmap v6063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:41.598 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3108893175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:41.598 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:40.974Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:41.598 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:40.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:41.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:40 smithi136 ceph-mon[113531]: pgmap v6063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:41.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3108893175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:41.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:41.626081+0000) 2023-08-28T00:15:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:42.444Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:42.445Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:42.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:42.626355+0000) 2023-08-28T00:15:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:42 smithi093 ceph-mon[111335]: pgmap v6064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:42 smithi093 ceph-mon[118902]: pgmap v6064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:42 smithi136 ceph-mon[113531]: pgmap v6064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:43.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:43.626616+0000) 2023-08-28T00:15:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/844455329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:44.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/844455329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/844455329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:44.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:44.626823+0000) 2023-08-28T00:15:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:44 smithi093 ceph-mon[111335]: pgmap v6065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:44 smithi093 ceph-mon[118902]: pgmap v6065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:44 smithi136 ceph-mon[113531]: pgmap v6065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:15:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:15:45] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T00:15:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:45.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:45.627117+0000) 2023-08-28T00:15:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:46 smithi136 ceph-mon[113531]: pgmap v6066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1590192583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:46.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:46.627352+0000) 2023-08-28T00:15:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:46 smithi093 ceph-mon[111335]: pgmap v6066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1590192583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:46.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:46 smithi093 ceph-mon[118902]: pgmap v6066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:46.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1590192583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:47.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:47.627602+0000) 2023-08-28T00:15:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:48 smithi136 ceph-mon[113531]: pgmap v6067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2025993094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:48.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:48.627877+0000) 2023-08-28T00:15:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:48 smithi093 ceph-mon[118902]: pgmap v6067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2025993094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:48 smithi093 ceph-mon[111335]: pgmap v6067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2025993094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:49.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:49.628089+0000) 2023-08-28T00:15:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:50.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:50.628323+0000) 2023-08-28T00:15:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:50 smithi136 ceph-mon[113531]: pgmap v6068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/869848752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:50.975 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:50 smithi093 ceph-mon[111335]: pgmap v6068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:50.975 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/869848752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:50.976 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:50 smithi093 ceph-mon[118902]: pgmap v6068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:50.976 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/869848752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:50.974Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:50.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:51.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:51.628578+0000) 2023-08-28T00:15:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:52 smithi093 ceph-mon[111335]: pgmap v6069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:52 smithi093 ceph-mon[118902]: pgmap v6069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:52.445Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:15:52.446Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:15:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:52 smithi136 ceph-mon[113531]: pgmap v6069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:52.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:52.628831+0000) 2023-08-28T00:15:53.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1951341105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:53.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:53.629112+0000) 2023-08-28T00:15:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1951341105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:53.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1951341105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:54 smithi136 ceph-mon[113531]: pgmap v6070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:54.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:54.629368+0000) 2023-08-28T00:15:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:54 smithi093 ceph-mon[111335]: pgmap v6070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:54 smithi093 ceph-mon[118902]: pgmap v6070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:15:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1888037867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1888037867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:15:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:15:55] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T00:15:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1888037867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:55.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:55.629650+0000) 2023-08-28T00:15:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:56 smithi136 ceph-mon[113531]: pgmap v6071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:56.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:56.629893+0000) 2023-08-28T00:15:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:56 smithi093 ceph-mon[111335]: pgmap v6071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:56 smithi093 ceph-mon[118902]: pgmap v6071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1796239238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:57.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:57.630112+0000) 2023-08-28T00:15:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1796239238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:57.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1796239238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:15:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:15:58 smithi136 ceph-mon[113531]: pgmap v6072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:58.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:58.630335+0000) 2023-08-28T00:15:58.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:15:58 smithi093 ceph-mon[111335]: pgmap v6072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:15:58 smithi093 ceph-mon[118902]: pgmap v6072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:15:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:15:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:15:59.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:15:59.630614+0000) 2023-08-28T00:16:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:00 smithi136 ceph-mon[113531]: pgmap v6073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4082009312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:00.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:00.630850+0000) 2023-08-28T00:16:00.977 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:00 smithi093 ceph-mon[118902]: pgmap v6073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:00.977 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4082009312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:00.977 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:00 smithi093 ceph-mon[111335]: pgmap v6073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:00.978 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4082009312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:00.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:00.977Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:01.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:01.631096+0000) 2023-08-28T00:16:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:02 smithi093 ceph-mon[118902]: pgmap v6074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1167138425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:02 smithi093 ceph-mon[111335]: pgmap v6074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1167138425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:02.446Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:02.447Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:02 smithi136 ceph-mon[113531]: pgmap v6074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1167138425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:02.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:02.631328+0000) 2023-08-28T00:16:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:03.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:03.631595+0000) 2023-08-28T00:16:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:04 smithi136 ceph-mon[113531]: pgmap v6075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:04.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:04.631847+0000) 2023-08-28T00:16:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:04 smithi093 ceph-mon[111335]: pgmap v6075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:04 smithi093 ceph-mon[118902]: pgmap v6075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1435590041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:16:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:16:05] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T00:16:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1435590041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1435590041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:05.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:05.632088+0000) 2023-08-28T00:16:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:06 smithi136 ceph-mon[113531]: pgmap v6076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:06.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:06.632351+0000) 2023-08-28T00:16:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:06 smithi093 ceph-mon[111335]: pgmap v6076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:06 smithi093 ceph-mon[118902]: pgmap v6076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:07.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1752653493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:07.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:07.632618+0000) 2023-08-28T00:16:07.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1752653493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:07.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1752653493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:08 smithi136 ceph-mon[113531]: pgmap v6077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:08.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:08.632841+0000) 2023-08-28T00:16:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:08 smithi093 ceph-mon[111335]: pgmap v6077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:08 smithi093 ceph-mon[118902]: pgmap v6077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1447769399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:09.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:09.633074+0000) 2023-08-28T00:16:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1447769399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1447769399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:10 smithi136 ceph-mon[113531]: pgmap v6078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:10.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:10.633307+0000) 2023-08-28T00:16:10.977 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:10 smithi093 ceph-mon[111335]: pgmap v6078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:10.977 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:10 smithi093 ceph-mon[118902]: pgmap v6078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:10.976Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:10.977Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:11.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:11.633549+0000) 2023-08-28T00:16:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:12 smithi093 ceph-mon[111335]: pgmap v6079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3048844296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:12 smithi093 ceph-mon[118902]: pgmap v6079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3048844296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:12.446Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:12.447Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:12 smithi136 ceph-mon[113531]: pgmap v6079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3048844296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:12.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:12.633719+0000) 2023-08-28T00:16:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:16:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:13.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:13.633883+0000) 2023-08-28T00:16:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:16:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:16:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:14 smithi136 ceph-mon[113531]: pgmap v6080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4252584149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:14.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:14.634082+0000) 2023-08-28T00:16:14.945 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:14 smithi093 ceph-mon[111335]: pgmap v6080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:14.945 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4252584149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:14.946 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:14 smithi093 ceph-mon[118902]: pgmap v6080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:14.946 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4252584149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:16:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:16:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:16:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:15.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:15.634420+0000) 2023-08-28T00:16:16.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: pgmap v6081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:16.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:16:16.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:16:16.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: pgmap v6081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:16.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:16:16.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:16:16.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: pgmap v6081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:16:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:16:16.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:16:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:16.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:16.634694+0000) 2023-08-28T00:16:17.635 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3828644006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3828644006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:17.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3828644006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:17.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:17.634951+0000) 2023-08-28T00:16:18.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:18 smithi136 ceph-mon[113531]: pgmap v6082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:18.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:18 smithi093 ceph-mon[111335]: pgmap v6082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:18.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:18 smithi093 ceph-mon[118902]: pgmap v6082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:18.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:18.635199+0000) 2023-08-28T00:16:19.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3617465734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3617465734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:19.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3617465734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:19.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:19.635423+0000) 2023-08-28T00:16:20.638 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:20 smithi136 ceph-mon[113531]: pgmap v6083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:20 smithi093 ceph-mon[111335]: pgmap v6083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:20.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:20 smithi093 ceph-mon[118902]: pgmap v6083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:20.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:20.635652+0000) 2023-08-28T00:16:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:20.977Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:20.978Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:21.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3961896382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:21.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:21.635903+0000) 2023-08-28T00:16:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3961896382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3961896382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:22 smithi093 ceph-mon[111335]: pgmap v6084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:22 smithi093 ceph-mon[118902]: pgmap v6084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:22.447Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:22.448Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:22 smithi136 ceph-mon[113531]: pgmap v6084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:22.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:22.636123+0000) 2023-08-28T00:16:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:23.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:23.636364+0000) 2023-08-28T00:16:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:24 smithi136 ceph-mon[113531]: pgmap v6085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1789554788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:24.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:24.636623+0000) 2023-08-28T00:16:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:24 smithi093 ceph-mon[111335]: pgmap v6085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1789554788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:24 smithi093 ceph-mon[118902]: pgmap v6085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1789554788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:16:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:16:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:16:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:25.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:25.636927+0000) 2023-08-28T00:16:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:26 smithi136 ceph-mon[113531]: pgmap v6086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3260254058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:26.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:26.637190+0000) 2023-08-28T00:16:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:26 smithi093 ceph-mon[111335]: pgmap v6086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3260254058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:26 smithi093 ceph-mon[118902]: pgmap v6086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3260254058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:27.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:27.637394+0000) 2023-08-28T00:16:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:28 smithi136 ceph-mon[113531]: pgmap v6087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2719963289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:28.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:28.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:28.637633+0000) 2023-08-28T00:16:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:28 smithi093 ceph-mon[111335]: pgmap v6087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2719963289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:28 smithi093 ceph-mon[118902]: pgmap v6087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2719963289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:29.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:29.637876+0000) 2023-08-28T00:16:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:30 smithi136 ceph-mon[113531]: pgmap v6088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:30.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:30.638075+0000) 2023-08-28T00:16:30.978 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:30 smithi093 ceph-mon[111335]: pgmap v6088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:30.978 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:30 smithi093 ceph-mon[118902]: pgmap v6088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:30.977Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:30.978Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:31.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4207384638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:31.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:31.638272+0000) 2023-08-28T00:16:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4207384638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4207384638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:32 smithi093 ceph-mon[111335]: pgmap v6089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:32 smithi093 ceph-mon[118902]: pgmap v6089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:32.448Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:32.449Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:32 smithi136 ceph-mon[113531]: pgmap v6089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:32.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:32.638480+0000) 2023-08-28T00:16:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3487796346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:33.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:33.638732+0000) 2023-08-28T00:16:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3487796346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3487796346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:34 smithi136 ceph-mon[113531]: pgmap v6090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:34.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:34.638972+0000) 2023-08-28T00:16:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:34 smithi093 ceph-mon[111335]: pgmap v6090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:34 smithi093 ceph-mon[118902]: pgmap v6090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:16:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:16:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:16:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:35.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:35.639206+0000) 2023-08-28T00:16:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:36 smithi136 ceph-mon[113531]: pgmap v6091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2797390294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:36.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:36.639475+0000) 2023-08-28T00:16:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:36 smithi093 ceph-mon[111335]: pgmap v6091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2797390294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:36 smithi093 ceph-mon[118902]: pgmap v6091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2797390294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:37.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:37.639718+0000) 2023-08-28T00:16:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:38 smithi136 ceph-mon[113531]: pgmap v6092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/575166070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:38.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:38.639967+0000) 2023-08-28T00:16:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:38 smithi093 ceph-mon[111335]: pgmap v6092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/575166070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:38 smithi093 ceph-mon[118902]: pgmap v6092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/575166070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:39.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:39.640261+0000) 2023-08-28T00:16:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:40 smithi136 ceph-mon[113531]: pgmap v6093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2023916590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:40.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:40.640492+0000) 2023-08-28T00:16:40.979 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:40 smithi093 ceph-mon[111335]: pgmap v6093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:40.979 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2023916590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:40.979 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:40 smithi093 ceph-mon[118902]: pgmap v6093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:40.979 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2023916590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:40.978Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:40.979Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:41.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:41.640733+0000) 2023-08-28T00:16:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:42 smithi093 ceph-mon[111335]: pgmap v6094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:42 smithi093 ceph-mon[118902]: pgmap v6094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:42.449Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:42.449Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:42 smithi136 ceph-mon[113531]: pgmap v6094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:42.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:42.640989+0000) 2023-08-28T00:16:43.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2386230056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:43.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:43.641249+0000) 2023-08-28T00:16:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2386230056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2386230056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:44 smithi136 ceph-mon[113531]: pgmap v6095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:44.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:44.641450+0000) 2023-08-28T00:16:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:44 smithi093 ceph-mon[111335]: pgmap v6095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:44 smithi093 ceph-mon[118902]: pgmap v6095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:45.620 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:16:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:16:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:16:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2764187222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:45.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:45.641738+0000) 2023-08-28T00:16:45.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2764187222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2764187222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:46 smithi136 ceph-mon[113531]: pgmap v6096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:46.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:46.641995+0000) 2023-08-28T00:16:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:46 smithi093 ceph-mon[111335]: pgmap v6096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:46.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:46 smithi093 ceph-mon[118902]: pgmap v6096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2210139810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:47.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:47.642256+0000) 2023-08-28T00:16:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2210139810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2210139810' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:48 smithi136 ceph-mon[113531]: pgmap v6097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:48.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:48.642506+0000) 2023-08-28T00:16:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:48 smithi093 ceph-mon[111335]: pgmap v6097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:48 smithi093 ceph-mon[118902]: pgmap v6097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:49.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:49.642772+0000) 2023-08-28T00:16:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:50 smithi136 ceph-mon[113531]: pgmap v6098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/524964901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:50.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:50.643015+0000) 2023-08-28T00:16:50.980 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:50 smithi093 ceph-mon[111335]: pgmap v6098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:50.980 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/524964901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:50.980 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:50 smithi093 ceph-mon[118902]: pgmap v6098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:50.981 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/524964901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:50.979Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:50.980Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:51.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:51.643281+0000) 2023-08-28T00:16:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:52 smithi093 ceph-mon[111335]: pgmap v6099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3700905406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:52.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:52 smithi093 ceph-mon[118902]: pgmap v6099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3700905406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:52.449Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:16:52.450Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:16:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:52 smithi136 ceph-mon[113531]: pgmap v6099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3700905406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:52.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:52.643557+0000) 2023-08-28T00:16:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:53.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:53.643805+0000) 2023-08-28T00:16:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:54 smithi136 ceph-mon[113531]: pgmap v6100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:54.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:54.644037+0000) 2023-08-28T00:16:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:54 smithi093 ceph-mon[111335]: pgmap v6100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:54.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:54 smithi093 ceph-mon[118902]: pgmap v6100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:16:55.646 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:16:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:16:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:16:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4239964415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:55.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:55.644200+0000) 2023-08-28T00:16:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4239964415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:55.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4239964415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:56 smithi136 ceph-mon[113531]: pgmap v6101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:56.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:56.644430+0000) 2023-08-28T00:16:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:56 smithi093 ceph-mon[111335]: pgmap v6101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:56 smithi093 ceph-mon[118902]: pgmap v6101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2639659481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:57.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:57.644638+0000) 2023-08-28T00:16:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2639659481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2639659481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:58 smithi136 ceph-mon[113531]: pgmap v6102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:58.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:58.644875+0000) 2023-08-28T00:16:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:58 smithi093 ceph-mon[111335]: pgmap v6102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:58 smithi093 ceph-mon[118902]: pgmap v6102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:16:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:16:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3424520214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:16:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:16:59.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:16:59.645139+0000) 2023-08-28T00:16:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:16:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3424520214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:16:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:16:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3424520214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:00 smithi136 ceph-mon[113531]: pgmap v6103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:00.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:00.645395+0000) 2023-08-28T00:17:00.981 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:00 smithi093 ceph-mon[111335]: pgmap v6103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:00.982 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:00 smithi093 ceph-mon[118902]: pgmap v6103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:00.980Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:00.981Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:01.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:01.645654+0000) 2023-08-28T00:17:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:02 smithi093 ceph-mon[111335]: pgmap v6104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1895458329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:02 smithi093 ceph-mon[118902]: pgmap v6104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1895458329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:02.450Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:02.451Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:02 smithi136 ceph-mon[113531]: pgmap v6104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1895458329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:02.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:02.645876+0000) 2023-08-28T00:17:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:03.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:03.646121+0000) 2023-08-28T00:17:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:04 smithi136 ceph-mon[113531]: pgmap v6105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2719503959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:04.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:04.646367+0000) 2023-08-28T00:17:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:04 smithi093 ceph-mon[111335]: pgmap v6105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2719503959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:04 smithi093 ceph-mon[118902]: pgmap v6105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2719503959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:17:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:17:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:17:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:05.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:05.646588+0000) 2023-08-28T00:17:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:06 smithi136 ceph-mon[113531]: pgmap v6106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1485791502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:06.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:06.646855+0000) 2023-08-28T00:17:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:06 smithi093 ceph-mon[111335]: pgmap v6106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1485791502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:06 smithi093 ceph-mon[118902]: pgmap v6106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1485791502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:07.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:07.647083+0000) 2023-08-28T00:17:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:08 smithi136 ceph-mon[113531]: pgmap v6107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:08.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:08.647326+0000) 2023-08-28T00:17:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:08 smithi093 ceph-mon[111335]: pgmap v6107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:08.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:08 smithi093 ceph-mon[118902]: pgmap v6107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4031439776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:09.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:09.647624+0000) 2023-08-28T00:17:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:09.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4031439776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4031439776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:10 smithi136 ceph-mon[113531]: pgmap v6108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:10.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:10.647874+0000) 2023-08-28T00:17:10.982 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:10 smithi093 ceph-mon[111335]: pgmap v6108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:10.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:10 smithi093 ceph-mon[118902]: pgmap v6108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:10.981Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:11.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:10.982Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:11.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4177131602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:11.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:11.648059+0000) 2023-08-28T00:17:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4177131602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4177131602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:12.702 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:12.450Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:12.702 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:12.451Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:12 smithi136 ceph-mon[113531]: pgmap v6109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:12.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:12.648297+0000) 2023-08-28T00:17:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:12 smithi093 ceph-mon[111335]: pgmap v6109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:12 smithi093 ceph-mon[118902]: pgmap v6109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:13.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:13.648581+0000) 2023-08-28T00:17:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:14 smithi136 ceph-mon[113531]: pgmap v6110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1927259319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:14.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:14.648835+0000) 2023-08-28T00:17:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:14 smithi093 ceph-mon[111335]: pgmap v6110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1927259319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:14 smithi093 ceph-mon[118902]: pgmap v6110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1927259319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:15.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:17:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:17:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:17:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:15.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:15.649065+0000) 2023-08-28T00:17:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:16.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:16.649218+0000) 2023-08-28T00:17:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:16 smithi136 ceph-mon[113531]: pgmap v6111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3639181939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:17:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[111335]: pgmap v6111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3639181939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:17:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[118902]: pgmap v6111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3639181939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:17:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:17.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:17.649442+0000) 2023-08-28T00:17:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:18.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:18.649717+0000) 2023-08-28T00:17:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[111335]: pgmap v6112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:18.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3124301985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[118902]: pgmap v6112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3124301985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:18 smithi136 ceph-mon[113531]: pgmap v6112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3124301985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:19.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:19.649945+0000) 2023-08-28T00:17:20.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:17:20.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:17:20.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:17:20.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:17:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:20.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:20.650194+0000) 2023-08-28T00:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:20 smithi093 ceph-mon[111335]: pgmap v6113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:20 smithi093 ceph-mon[118902]: pgmap v6113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:20.982Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:20.983Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:20 smithi136 ceph-mon[113531]: pgmap v6113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:21.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:21.650446+0000) 2023-08-28T00:17:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/303178960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/303178960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/303178960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:22.451Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:22.452Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:22.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:22.650696+0000) 2023-08-28T00:17:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:22 smithi093 ceph-mon[111335]: pgmap v6114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:22 smithi093 ceph-mon[118902]: pgmap v6114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:22 smithi136 ceph-mon[113531]: pgmap v6114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:23.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:23.650955+0000) 2023-08-28T00:17:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1191936205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:24.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1191936205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:24.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1191936205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:24.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:24.651219+0000) 2023-08-28T00:17:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:24 smithi093 ceph-mon[118902]: pgmap v6115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:24 smithi093 ceph-mon[111335]: pgmap v6115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:24 smithi136 ceph-mon[113531]: pgmap v6115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:17:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:17:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:17:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:25.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:25.651460+0000) 2023-08-28T00:17:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:26 smithi136 ceph-mon[113531]: pgmap v6116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/149664354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:26.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:26.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:26.651705+0000) 2023-08-28T00:17:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:26 smithi093 ceph-mon[111335]: pgmap v6116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/149664354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:26 smithi093 ceph-mon[118902]: pgmap v6116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/149664354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:27.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:27.651928+0000) 2023-08-28T00:17:28.943 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:28 smithi136 ceph-mon[113531]: pgmap v6117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:28.944 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2825687329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:28.944 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:28.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:28.652176+0000) 2023-08-28T00:17:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:28 smithi093 ceph-mon[111335]: pgmap v6117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2825687329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:28 smithi093 ceph-mon[118902]: pgmap v6117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2825687329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:29.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:29.652404+0000) 2023-08-28T00:17:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:30 smithi136 ceph-mon[113531]: pgmap v6118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1188377439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:30.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:30.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:30.652655+0000) 2023-08-28T00:17:30.984 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:30 smithi093 ceph-mon[111335]: pgmap v6118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:30.984 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1188377439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:30.985 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:30 smithi093 ceph-mon[118902]: pgmap v6118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:30.985 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1188377439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:30.983Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:30.984Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:31.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:31.652892+0000) 2023-08-28T00:17:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:32 smithi093 ceph-mon[111335]: pgmap v6119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:32 smithi093 ceph-mon[118902]: pgmap v6119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:32.452Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:32.453Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:32 smithi136 ceph-mon[113531]: pgmap v6119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:32.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:32.653141+0000) 2023-08-28T00:17:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1027729792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:33.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:33.653397+0000) 2023-08-28T00:17:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1027729792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1027729792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:34 smithi136 ceph-mon[113531]: pgmap v6120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:34.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:34.653638+0000) 2023-08-28T00:17:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:34 smithi093 ceph-mon[111335]: pgmap v6120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:34 smithi093 ceph-mon[118902]: pgmap v6120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4265279605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4265279605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:17:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:17:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:17:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4265279605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:35.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:35.653896+0000) 2023-08-28T00:17:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:36 smithi136 ceph-mon[113531]: pgmap v6121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:36.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:36.654185+0000) 2023-08-28T00:17:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:36 smithi093 ceph-mon[111335]: pgmap v6121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:36 smithi093 ceph-mon[118902]: pgmap v6121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:37.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:37.654426+0000) 2023-08-28T00:17:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:38 smithi136 ceph-mon[113531]: pgmap v6122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3157789918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:38.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:38.654677+0000) 2023-08-28T00:17:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:38 smithi093 ceph-mon[111335]: pgmap v6122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3157789918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:38 smithi093 ceph-mon[118902]: pgmap v6122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3157789918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:39.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:39.654968+0000) 2023-08-28T00:17:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:40 smithi136 ceph-mon[113531]: pgmap v6123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/781106083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:40.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:40.655226+0000) 2023-08-28T00:17:40.985 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:40 smithi093 ceph-mon[111335]: pgmap v6123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:40.985 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/781106083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:40.986 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:40 smithi093 ceph-mon[118902]: pgmap v6123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:40.986 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/781106083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:40.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:40.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:41.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:41.655461+0000) 2023-08-28T00:17:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:42 smithi093 ceph-mon[111335]: pgmap v6124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/423465365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:42 smithi093 ceph-mon[118902]: pgmap v6124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/423465365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:42.453Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:42.454Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:42 smithi136 ceph-mon[113531]: pgmap v6124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/423465365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:42.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:42.655687+0000) 2023-08-28T00:17:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:43.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:43.655877+0000) 2023-08-28T00:17:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:44 smithi136 ceph-mon[113531]: pgmap v6125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:44.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:44.656168+0000) 2023-08-28T00:17:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:44 smithi093 ceph-mon[111335]: pgmap v6125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:44 smithi093 ceph-mon[118902]: pgmap v6125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:45.624 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:17:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:17:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:17:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4272728924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:45.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:45.656425+0000) 2023-08-28T00:17:45.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4272728924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4272728924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:46 smithi136 ceph-mon[113531]: pgmap v6126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:46.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:46.656721+0000) 2023-08-28T00:17:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:46 smithi093 ceph-mon[111335]: pgmap v6126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:46 smithi093 ceph-mon[118902]: pgmap v6126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/810614586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:47.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:47.656972+0000) 2023-08-28T00:17:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/810614586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/810614586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:48 smithi136 ceph-mon[113531]: pgmap v6127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:48.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:48.657232+0000) 2023-08-28T00:17:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:48 smithi093 ceph-mon[111335]: pgmap v6127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:48 smithi093 ceph-mon[118902]: pgmap v6127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:49.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3059210051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:49.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:49.657485+0000) 2023-08-28T00:17:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3059210051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3059210051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:50 smithi136 ceph-mon[113531]: pgmap v6128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:50.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:50.657733+0000) 2023-08-28T00:17:50.986 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:50 smithi093 ceph-mon[111335]: pgmap v6128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:50.986 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:50 smithi093 ceph-mon[118902]: pgmap v6128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:50.985Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:50.986Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:51.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:51.658006+0000) 2023-08-28T00:17:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:52 smithi093 ceph-mon[111335]: pgmap v6129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2026854939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:52 smithi093 ceph-mon[118902]: pgmap v6129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2026854939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:52.454Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:17:52.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:17:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:52 smithi136 ceph-mon[113531]: pgmap v6129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2026854939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:52.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:52.658277+0000) 2023-08-28T00:17:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:53.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:53.658508+0000) 2023-08-28T00:17:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:54 smithi136 ceph-mon[113531]: pgmap v6130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2977568576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:54.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:54.658744+0000) 2023-08-28T00:17:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:54 smithi093 ceph-mon[111335]: pgmap v6130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2977568576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:54 smithi093 ceph-mon[118902]: pgmap v6130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:17:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2977568576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:55.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:17:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:17:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:17:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:55.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:55.658994+0000) 2023-08-28T00:17:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:56 smithi136 ceph-mon[113531]: pgmap v6131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:56.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:56.659256+0000) 2023-08-28T00:17:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:56 smithi093 ceph-mon[111335]: pgmap v6131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:56 smithi093 ceph-mon[118902]: pgmap v6131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:57.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1216879792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:57.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:57.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:57.659452+0000) 2023-08-28T00:17:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1216879792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1216879792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:58 smithi136 ceph-mon[113531]: pgmap v6132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:58.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:58.659613+0000) 2023-08-28T00:17:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:58 smithi093 ceph-mon[111335]: pgmap v6132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:58 smithi093 ceph-mon[118902]: pgmap v6132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:17:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:17:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2623360773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:59.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:17:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:17:59.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:17:59.659801+0000) 2023-08-28T00:17:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:17:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2623360773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:17:59.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:17:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2623360773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:00 smithi136 ceph-mon[113531]: pgmap v6133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:00.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:00.660019+0000) 2023-08-28T00:18:00.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:00 smithi093 ceph-mon[111335]: pgmap v6133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:00.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:00 smithi093 ceph-mon[118902]: pgmap v6133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:00.986Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:00.987Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1242594953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:01.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:01.660281+0000) 2023-08-28T00:18:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1242594953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:01.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1242594953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:02 smithi093 ceph-mon[111335]: pgmap v6134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:02 smithi093 ceph-mon[118902]: pgmap v6134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:02.454Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:02.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:02 smithi136 ceph-mon[113531]: pgmap v6134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:02.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:02.660523+0000) 2023-08-28T00:18:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:03.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:03.660775+0000) 2023-08-28T00:18:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:04 smithi136 ceph-mon[113531]: pgmap v6135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2601603395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:04.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:04.661056+0000) 2023-08-28T00:18:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:04 smithi093 ceph-mon[111335]: pgmap v6135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2601603395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:04 smithi093 ceph-mon[118902]: pgmap v6135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2601603395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:18:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:18:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:18:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:05.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:05.661322+0000) 2023-08-28T00:18:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:06 smithi136 ceph-mon[113531]: pgmap v6136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1781840151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:06.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:06.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:06.661544+0000) 2023-08-28T00:18:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:06 smithi093 ceph-mon[111335]: pgmap v6136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1781840151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:06 smithi093 ceph-mon[118902]: pgmap v6136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1781840151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:07.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:07.661828+0000) 2023-08-28T00:18:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:08 smithi136 ceph-mon[113531]: pgmap v6137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2410043436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:08.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:08.662071+0000) 2023-08-28T00:18:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:08 smithi093 ceph-mon[111335]: pgmap v6137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2410043436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:08 smithi093 ceph-mon[118902]: pgmap v6137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2410043436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:09.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:09.662311+0000) 2023-08-28T00:18:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:09.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:10 smithi136 ceph-mon[113531]: pgmap v6138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:10.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:10.662585+0000) 2023-08-28T00:18:10.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:10 smithi093 ceph-mon[111335]: pgmap v6138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:10 smithi093 ceph-mon[118902]: pgmap v6138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:10.986Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:10.987Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:11.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1033565047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:11.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:11.662775+0000) 2023-08-28T00:18:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1033565047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:11.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1033565047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:12 smithi093 ceph-mon[111335]: pgmap v6139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:12 smithi093 ceph-mon[118902]: pgmap v6139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:12.455Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:12.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:12 smithi136 ceph-mon[113531]: pgmap v6139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:12.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:12.663074+0000) 2023-08-28T00:18:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3559722859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:13.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:13.663307+0000) 2023-08-28T00:18:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3559722859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:13.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3559722859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:14 smithi136 ceph-mon[113531]: pgmap v6140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:14.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:14.663477+0000) 2023-08-28T00:18:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:14 smithi093 ceph-mon[111335]: pgmap v6140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:14 smithi093 ceph-mon[118902]: pgmap v6140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:18:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:18:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:18:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:15.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:15.663732+0000) 2023-08-28T00:18:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:16 smithi136 ceph-mon[113531]: pgmap v6141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2168492571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:16.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:16.664009+0000) 2023-08-28T00:18:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:16 smithi093 ceph-mon[111335]: pgmap v6141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2168492571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:16 smithi093 ceph-mon[118902]: pgmap v6141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2168492571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:17.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:17.664261+0000) 2023-08-28T00:18:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:18 smithi136 ceph-mon[113531]: pgmap v6142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1706070625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:18.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:18.664469+0000) 2023-08-28T00:18:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:18 smithi093 ceph-mon[111335]: pgmap v6142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1706070625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:18 smithi093 ceph-mon[118902]: pgmap v6142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1706070625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:19.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:18:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:19.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:19.664685+0000) 2023-08-28T00:18:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:18:19.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:18:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:20 smithi136 ceph-mon[113531]: pgmap v6143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/22541198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:20.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:20.664862+0000) 2023-08-28T00:18:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[118902]: pgmap v6143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/22541198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:20.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[111335]: pgmap v6143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/22541198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:20.987Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:20.988Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:21.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:21.665085+0000) 2023-08-28T00:18:22.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:22.338 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:22.338 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:22 smithi136 ceph-mon[113531]: pgmap v6144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-mon[111335]: pgmap v6144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-mon[118902]: pgmap v6144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:22.455Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:22.456Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:22.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:22.665259+0000) 2023-08-28T00:18:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:23.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:23.665418+0000) 2023-08-28T00:18:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2071278057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:18:23.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:18:23.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2071278057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2071278057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:18:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:18:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:24 smithi136 ceph-mon[113531]: pgmap v6145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:24.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:24.665683+0000) 2023-08-28T00:18:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:24 smithi093 ceph-mon[111335]: pgmap v6145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:24 smithi093 ceph-mon[118902]: pgmap v6145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3861768627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:18:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:18:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:18:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3861768627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3861768627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:25.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:25.665933+0000) 2023-08-28T00:18:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:26 smithi136 ceph-mon[113531]: pgmap v6146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:26.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:26.666209+0000) 2023-08-28T00:18:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:26 smithi093 ceph-mon[111335]: pgmap v6146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:26 smithi093 ceph-mon[118902]: pgmap v6146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:27.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:27.666463+0000) 2023-08-28T00:18:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:28 smithi136 ceph-mon[113531]: pgmap v6147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3146485352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:28.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:28.666706+0000) 2023-08-28T00:18:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:28 smithi093 ceph-mon[111335]: pgmap v6147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3146485352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:28 smithi093 ceph-mon[118902]: pgmap v6147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3146485352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:29.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:29.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:29.666968+0000) 2023-08-28T00:18:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:30 smithi136 ceph-mon[113531]: pgmap v6148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3375196059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:30.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:30.667197+0000) 2023-08-28T00:18:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:30 smithi093 ceph-mon[111335]: pgmap v6148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3375196059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:30 smithi093 ceph-mon[118902]: pgmap v6148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3375196059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:30.988Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:30.989Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:31.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:31.667410+0000) 2023-08-28T00:18:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:32 smithi093 ceph-mon[111335]: pgmap v6149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/660823444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:32 smithi093 ceph-mon[118902]: pgmap v6149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/660823444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:32.456Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:32.457Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:32 smithi136 ceph-mon[113531]: pgmap v6149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/660823444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:32.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:32.667678+0000) 2023-08-28T00:18:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:33.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:33.667930+0000) 2023-08-28T00:18:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:34 smithi136 ceph-mon[113531]: pgmap v6150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:34.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:34.668131+0000) 2023-08-28T00:18:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:34 smithi093 ceph-mon[111335]: pgmap v6150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:34 smithi093 ceph-mon[118902]: pgmap v6150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2250140851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:18:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:18:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:18:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2250140851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2250140851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:35.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:35.668387+0000) 2023-08-28T00:18:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:36 smithi136 ceph-mon[113531]: pgmap v6151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:36.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:36.668652+0000) 2023-08-28T00:18:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:36 smithi093 ceph-mon[111335]: pgmap v6151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:36 smithi093 ceph-mon[118902]: pgmap v6151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:37.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3352507133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:37.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:37.668895+0000) 2023-08-28T00:18:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3352507133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3352507133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:38 smithi136 ceph-mon[113531]: pgmap v6152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:38.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:38.669165+0000) 2023-08-28T00:18:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:38 smithi093 ceph-mon[111335]: pgmap v6152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:38 smithi093 ceph-mon[118902]: pgmap v6152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:39.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:39.669467+0000) 2023-08-28T00:18:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:40 smithi136 ceph-mon[113531]: pgmap v6153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1634861720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:40.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:40.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:40.669663+0000) 2023-08-28T00:18:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:40 smithi093 ceph-mon[111335]: pgmap v6153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1634861720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:40 smithi093 ceph-mon[118902]: pgmap v6153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1634861720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:40.989Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:40.990Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:41.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:41.669895+0000) 2023-08-28T00:18:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:42 smithi093 ceph-mon[118902]: pgmap v6154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:42.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3851169990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:42 smithi093 ceph-mon[111335]: pgmap v6154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:42.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3851169990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:42.457Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:42.458Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:42 smithi136 ceph-mon[113531]: pgmap v6154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3851169990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:42.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:42.670128+0000) 2023-08-28T00:18:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:43.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:43.670365+0000) 2023-08-28T00:18:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:44 smithi136 ceph-mon[113531]: pgmap v6155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:44.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1496436260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:44.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:44.670573+0000) 2023-08-28T00:18:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:44 smithi093 ceph-mon[111335]: pgmap v6155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1496436260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:44 smithi093 ceph-mon[118902]: pgmap v6155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:44.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1496436260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:18:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:18:45] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:18:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:45.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:45.670816+0000) 2023-08-28T00:18:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:46 smithi136 ceph-mon[113531]: pgmap v6156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:46.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:46.671094+0000) 2023-08-28T00:18:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:46 smithi093 ceph-mon[111335]: pgmap v6156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:46 smithi093 ceph-mon[118902]: pgmap v6156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3794435172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:47.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:47.671297+0000) 2023-08-28T00:18:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3794435172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3794435172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:48 smithi136 ceph-mon[113531]: pgmap v6157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:48.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:48.671518+0000) 2023-08-28T00:18:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:48 smithi093 ceph-mon[111335]: pgmap v6157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:48.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:48 smithi093 ceph-mon[118902]: pgmap v6157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:49.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1131215606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:49.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:49.671785+0000) 2023-08-28T00:18:49.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1131215606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:49.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1131215606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:50 smithi136 ceph-mon[113531]: pgmap v6158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:50.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:50.671942+0000) 2023-08-28T00:18:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:50 smithi093 ceph-mon[118902]: pgmap v6158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:50 smithi093 ceph-mon[111335]: pgmap v6158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:50.990Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:50.991Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:51.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3068654362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:51.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:51.672154+0000) 2023-08-28T00:18:51.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3068654362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:51.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3068654362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:52 smithi093 ceph-mon[111335]: pgmap v6159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:52 smithi093 ceph-mon[118902]: pgmap v6159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:52.458Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:18:52.459Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:18:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:52 smithi136 ceph-mon[113531]: pgmap v6159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:52.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:52.672345+0000) 2023-08-28T00:18:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:53.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:53.672587+0000) 2023-08-28T00:18:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:54 smithi136 ceph-mon[113531]: pgmap v6160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3764881247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:54.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:54.672865+0000) 2023-08-28T00:18:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:54 smithi093 ceph-mon[111335]: pgmap v6160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:54.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3764881247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:54 smithi093 ceph-mon[118902]: pgmap v6160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:18:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3764881247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:18:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:18:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:18:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:55.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:55.673126+0000) 2023-08-28T00:18:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:56 smithi136 ceph-mon[113531]: pgmap v6161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2878498023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:56.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:56.673349+0000) 2023-08-28T00:18:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:56 smithi093 ceph-mon[111335]: pgmap v6161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2878498023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:56 smithi093 ceph-mon[118902]: pgmap v6161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2878498023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:57.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:57.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:57.673584+0000) 2023-08-28T00:18:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:58 smithi136 ceph-mon[113531]: pgmap v6162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:58.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:58.673868+0000) 2023-08-28T00:18:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:58 smithi093 ceph-mon[111335]: pgmap v6162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:58 smithi093 ceph-mon[118902]: pgmap v6162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:18:59.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:18:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3945671362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:18:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:18:59.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:18:59.674166+0000) 2023-08-28T00:18:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:18:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3945671362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:18:59.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:18:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3945671362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:00 smithi136 ceph-mon[113531]: pgmap v6163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:00.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:00.674323+0000) 2023-08-28T00:19:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:00 smithi093 ceph-mon[111335]: pgmap v6163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:00 smithi093 ceph-mon[118902]: pgmap v6163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:00.991Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:00.992Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1229105395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:01.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:01.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:01.674567+0000) 2023-08-28T00:19:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1229105395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:01.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1229105395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:02 smithi093 ceph-mon[111335]: pgmap v6164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:02.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:02 smithi093 ceph-mon[118902]: pgmap v6164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:02.459Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:02.460Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:02 smithi136 ceph-mon[113531]: pgmap v6164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:02.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:02.674758+0000) 2023-08-28T00:19:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1168965733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:03.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:03.674940+0000) 2023-08-28T00:19:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1168965733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1168965733' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:04 smithi136 ceph-mon[113531]: pgmap v6165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:04.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:04.675183+0000) 2023-08-28T00:19:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:04 smithi093 ceph-mon[111335]: pgmap v6165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:04 smithi093 ceph-mon[118902]: pgmap v6165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:19:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:19:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:19:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:05.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:05.675438+0000) 2023-08-28T00:19:06.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:06 smithi136 ceph-mon[113531]: pgmap v6166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:06.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/485930278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:06.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:06.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:06.675672+0000) 2023-08-28T00:19:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:06 smithi093 ceph-mon[118902]: pgmap v6166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/485930278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:06.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:06 smithi093 ceph-mon[111335]: pgmap v6166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:06.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/485930278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:07.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:07.675912+0000) 2023-08-28T00:19:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:08 smithi136 ceph-mon[113531]: pgmap v6167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2188938029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:08.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:08.676123+0000) 2023-08-28T00:19:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:08 smithi093 ceph-mon[111335]: pgmap v6167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2188938029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:08 smithi093 ceph-mon[118902]: pgmap v6167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:08.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2188938029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:09.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:09.676339+0000) 2023-08-28T00:19:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:10 smithi136 ceph-mon[113531]: pgmap v6168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1496603539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:10.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:10.676600+0000) 2023-08-28T00:19:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:10 smithi093 ceph-mon[118902]: pgmap v6168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1496603539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:10.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:10 smithi093 ceph-mon[111335]: pgmap v6168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:10.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1496603539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:10.992Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:10.993Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:11.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:11.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:11.676873+0000) 2023-08-28T00:19:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:12 smithi093 ceph-mon[111335]: pgmap v6169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:12 smithi093 ceph-mon[118902]: pgmap v6169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:12.460Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:12.461Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:12 smithi136 ceph-mon[113531]: pgmap v6169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:12.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:12.677149+0000) 2023-08-28T00:19:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/46605843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:13.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:13.677393+0000) 2023-08-28T00:19:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/46605843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/46605843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:14 smithi136 ceph-mon[113531]: pgmap v6170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:14.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:14.677628+0000) 2023-08-28T00:19:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:14 smithi093 ceph-mon[111335]: pgmap v6170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:14 smithi093 ceph-mon[118902]: pgmap v6170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:15.707 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:19:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:19:15] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:19:15.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/143340735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:15.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:15.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:15.677900+0000) 2023-08-28T00:19:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/143340735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/143340735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:16 smithi136 ceph-mon[113531]: pgmap v6171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:16.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:16.678192+0000) 2023-08-28T00:19:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:16 smithi093 ceph-mon[111335]: pgmap v6171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:16 smithi093 ceph-mon[118902]: pgmap v6171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:17.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:17.678445+0000) 2023-08-28T00:19:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:18 smithi136 ceph-mon[113531]: pgmap v6172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/254153102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:18.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:18.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:18.678684+0000) 2023-08-28T00:19:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:18 smithi093 ceph-mon[118902]: pgmap v6172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/254153102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:18.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:18 smithi093 ceph-mon[111335]: pgmap v6172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:18.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/254153102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:19.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:19.678897+0000) 2023-08-28T00:19:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:20 smithi136 ceph-mon[113531]: pgmap v6173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3030249272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:20.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:20.679200+0000) 2023-08-28T00:19:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:20 smithi093 ceph-mon[111335]: pgmap v6173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3030249272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:20 smithi093 ceph-mon[118902]: pgmap v6173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3030249272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:20.992Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:20.993Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:21.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:21.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:21.679365+0000) 2023-08-28T00:19:22.719 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:22.461Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:22.719 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:22.462Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:22.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:22 smithi136 ceph-mon[113531]: pgmap v6174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:22.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3573718377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:22.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:22.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:22.679575+0000) 2023-08-28T00:19:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:22 smithi093 ceph-mon[111335]: pgmap v6174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3573718377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:22 smithi093 ceph-mon[118902]: pgmap v6174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3573718377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:23.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:19:23.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:23.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:23.679774+0000) 2023-08-28T00:19:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:19:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:19:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:24 smithi136 ceph-mon[113531]: pgmap v6175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:24.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:24.679939+0000) 2023-08-28T00:19:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:24 smithi093 ceph-mon[111335]: pgmap v6175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:24 smithi093 ceph-mon[118902]: pgmap v6175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:25.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:19:25] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:25.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:25.680118+0000) 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2249208559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:19:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:19:25.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2249208559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2249208559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:19:26.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:19:26.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:19:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:26 smithi136 ceph-mon[113531]: pgmap v6176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:26.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:26.680311+0000) 2023-08-28T00:19:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:26 smithi093 ceph-mon[118902]: pgmap v6176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:26 smithi093 ceph-mon[111335]: pgmap v6176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:27.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2986568282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:27.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:27.680471+0000) 2023-08-28T00:19:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2986568282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2986568282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:28 smithi136 ceph-mon[113531]: pgmap v6177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:28.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:28.680728+0000) 2023-08-28T00:19:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:28 smithi093 ceph-mon[111335]: pgmap v6177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:28 smithi093 ceph-mon[118902]: pgmap v6177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:29.933 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:29.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:29.680992+0000) 2023-08-28T00:19:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1249366229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1249366229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1249366229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:30.934 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:30.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:30.681270+0000) 2023-08-28T00:19:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:30 smithi136 ceph-mon[113531]: pgmap v6178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:30 smithi093 ceph-mon[111335]: pgmap v6178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:31.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:30 smithi093 ceph-mon[118902]: pgmap v6178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:30.993Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:30 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:30.994Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:31.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:31.681488+0000) 2023-08-28T00:19:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:32.462Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:32.463Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:32.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:32.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:32.681724+0000) 2023-08-28T00:19:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:32 smithi136 ceph-mon[113531]: pgmap v6179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1250452769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:32 smithi093 ceph-mon[111335]: pgmap v6179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1250452769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:32 smithi093 ceph-mon[118902]: pgmap v6179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1250452769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:33.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:33.682006+0000) 2023-08-28T00:19:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:34.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:34.682271+0000) 2023-08-28T00:19:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:34 smithi093 ceph-mon[111335]: pgmap v6180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3863982626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:34 smithi093 ceph-mon[118902]: pgmap v6180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3863982626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:34 smithi136 ceph-mon[113531]: pgmap v6180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3863982626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:19:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:19:35] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:19:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:35.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:35.682502+0000) 2023-08-28T00:19:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:36.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:36.682787+0000) 2023-08-28T00:19:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:36 smithi093 ceph-mon[111335]: pgmap v6181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2458935483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:36 smithi093 ceph-mon[118902]: pgmap v6181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2458935483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:36 smithi136 ceph-mon[113531]: pgmap v6181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2458935483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:37.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:37.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:37.682986+0000) 2023-08-28T00:19:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:38.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:38.683207+0000) 2023-08-28T00:19:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:38 smithi093 ceph-mon[118902]: pgmap v6182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:38 smithi093 ceph-mon[111335]: pgmap v6182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:38 smithi136 ceph-mon[113531]: pgmap v6182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:39.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:39.683444+0000) 2023-08-28T00:19:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1938811511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1938811511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1938811511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:40.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:40.683688+0000) 2023-08-28T00:19:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:40 smithi093 ceph-mon[111335]: pgmap v6183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:40 smithi093 ceph-mon[118902]: pgmap v6183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:40.994Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:40 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:40.995Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:40 smithi136 ceph-mon[113531]: pgmap v6183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:41.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:41.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:41.683946+0000) 2023-08-28T00:19:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3960292647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3960292647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3960292647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:42.463Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:42.464Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:42.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:42.684244+0000) 2023-08-28T00:19:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:42 smithi093 ceph-mon[111335]: pgmap v6184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:42 smithi093 ceph-mon[118902]: pgmap v6184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:42 smithi136 ceph-mon[113531]: pgmap v6184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:43.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:43.684480+0000) 2023-08-28T00:19:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:44.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:44.684728+0000) 2023-08-28T00:19:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:44 smithi093 ceph-mon[111335]: pgmap v6185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3343243530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:44 smithi093 ceph-mon[118902]: pgmap v6185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3343243530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:44 smithi136 ceph-mon[113531]: pgmap v6185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3343243530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:19:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:19:45] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:19:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:45.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:45.685007+0000) 2023-08-28T00:19:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:46.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:46.685269+0000) 2023-08-28T00:19:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:46 smithi093 ceph-mon[111335]: pgmap v6186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2164244334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:46 smithi093 ceph-mon[118902]: pgmap v6186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2164244334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:46 smithi136 ceph-mon[113531]: pgmap v6186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2164244334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:47.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:47.685537+0000) 2023-08-28T00:19:48.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:48.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:48.685740+0000) 2023-08-28T00:19:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:48 smithi093 ceph-mon[111335]: pgmap v6187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4240934218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:48 smithi093 ceph-mon[118902]: pgmap v6187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4240934218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:48 smithi136 ceph-mon[113531]: pgmap v6187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4240934218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:49.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:49.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:49.686016+0000) 2023-08-28T00:19:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:50.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:50.686234+0000) 2023-08-28T00:19:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:50 smithi093 ceph-mon[111335]: pgmap v6188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:50 smithi093 ceph-mon[118902]: pgmap v6188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:50.995Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:50 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:50.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:50 smithi136 ceph-mon[113531]: pgmap v6188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:51.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:51.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:51.686474+0000) 2023-08-28T00:19:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3707093942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3707093942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3707093942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:52.464Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:19:52.465Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:19:52.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:52.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:52.686717+0000) 2023-08-28T00:19:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:52 smithi093 ceph-mon[111335]: pgmap v6189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:52 smithi093 ceph-mon[118902]: pgmap v6189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:52 smithi136 ceph-mon[113531]: pgmap v6189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:53.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:53.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:53.686963+0000) 2023-08-28T00:19:54.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3112296213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:54.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3112296213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:54.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3112296213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:19:54.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:54.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:54.687161+0000) 2023-08-28T00:19:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:54 smithi093 ceph-mon[111335]: pgmap v6190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:54 smithi093 ceph-mon[118902]: pgmap v6190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:54 smithi136 ceph-mon[113531]: pgmap v6190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:19:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:19:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:19:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:55.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:55.687405+0000) 2023-08-28T00:19:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:56 smithi136 ceph-mon[113531]: pgmap v6191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1131083805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:56.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:56.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:56.687654+0000) 2023-08-28T00:19:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:56 smithi093 ceph-mon[111335]: pgmap v6191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1131083805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:56 smithi093 ceph-mon[118902]: pgmap v6191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:56.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1131083805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:57.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:57.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:57.687906+0000) 2023-08-28T00:19:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:58 smithi136 ceph-mon[113531]: pgmap v6192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:58.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:19:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2276155833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:58.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:58.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:58.688199+0000) 2023-08-28T00:19:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:58 smithi093 ceph-mon[111335]: pgmap v6192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:19:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2276155833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:58 smithi093 ceph-mon[118902]: pgmap v6192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:19:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:19:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2276155833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:19:59.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:19:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:19:59.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:19:59.688445+0000) 2023-08-28T00:20:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:00 smithi136 ceph-mon[113531]: pgmap v6193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:00.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:20:00.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:00.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:00.688678+0000) 2023-08-28T00:20:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:00 smithi093 ceph-mon[111335]: pgmap v6193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:20:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:00 smithi093 ceph-mon[118902]: pgmap v6193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:00.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:20:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:00.995Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:00 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:00.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:01.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1485512923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:01.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:01.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:01.688938+0000) 2023-08-28T00:20:01.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1485512923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:01.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1485512923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:02.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:02 smithi093 ceph-mon[111335]: pgmap v6194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:02 smithi093 ceph-mon[118902]: pgmap v6194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:02.465Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:02.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:02.466Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:02.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:02 smithi136 ceph-mon[113531]: pgmap v6194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:02.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:02.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:02.689201+0000) 2023-08-28T00:20:03.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/591038831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:03.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:03.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:03.689462+0000) 2023-08-28T00:20:03.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/591038831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:03.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/591038831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:04.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:04 smithi136 ceph-mon[113531]: pgmap v6195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:04.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:04.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:04.689699+0000) 2023-08-28T00:20:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:04 smithi093 ceph-mon[111335]: pgmap v6195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:04 smithi093 ceph-mon[118902]: pgmap v6195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:05.630 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:20:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:20:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:20:05.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/663319957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:05.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:05.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:05.689952+0000) 2023-08-28T00:20:05.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/663319957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:05.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/663319957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:06 smithi136 ceph-mon[113531]: pgmap v6196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:06.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:06.690224+0000) 2023-08-28T00:20:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:06 smithi093 ceph-mon[111335]: pgmap v6196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:06 smithi093 ceph-mon[118902]: pgmap v6196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:07.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:07.690474+0000) 2023-08-28T00:20:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:08.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:08.690723+0000) 2023-08-28T00:20:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:08 smithi136 ceph-mon[113531]: pgmap v6197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/427008409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:08 smithi093 ceph-mon[111335]: pgmap v6197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/427008409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:08.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:08 smithi093 ceph-mon[118902]: pgmap v6197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:08.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/427008409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:09.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:09.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:09.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:09.691006+0000) 2023-08-28T00:20:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:10 smithi136 ceph-mon[113531]: pgmap v6198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:10.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3190321963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:10.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:10.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:10.691228+0000) 2023-08-28T00:20:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:10 smithi093 ceph-mon[111335]: pgmap v6198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3190321963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:10 smithi093 ceph-mon[118902]: pgmap v6198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3190321963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:10.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:10 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:10.997Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:11.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:11.691391+0000) 2023-08-28T00:20:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:12 smithi093 ceph-mon[111335]: pgmap v6199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2456649509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:12 smithi093 ceph-mon[118902]: pgmap v6199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2456649509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:12.465Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:12.466Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:12.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:12.691668+0000) 2023-08-28T00:20:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:12 smithi136 ceph-mon[113531]: pgmap v6199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2456649509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:13.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:13.691911+0000) 2023-08-28T00:20:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:14 smithi136 ceph-mon[113531]: pgmap v6200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:14.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:14.692198+0000) 2023-08-28T00:20:14.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:14 smithi093 ceph-mon[111335]: pgmap v6200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:14.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:14 smithi093 ceph-mon[118902]: pgmap v6200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:15.651 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:20:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:20:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:20:15.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3773297576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:15.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:15.692438+0000) 2023-08-28T00:20:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3773297576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3773297576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:16 smithi136 ceph-mon[113531]: pgmap v6201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:16.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:16.692705+0000) 2023-08-28T00:20:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:16 smithi093 ceph-mon[111335]: pgmap v6201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:16 smithi093 ceph-mon[118902]: pgmap v6201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:17.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1509064079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:17.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:17.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:17.692914+0000) 2023-08-28T00:20:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1509064079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1509064079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:18 smithi136 ceph-mon[113531]: pgmap v6202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:18.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:18.693167+0000) 2023-08-28T00:20:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:18 smithi093 ceph-mon[111335]: pgmap v6202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:18 smithi093 ceph-mon[118902]: pgmap v6202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:19.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:19.693423+0000) 2023-08-28T00:20:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:20 smithi136 ceph-mon[113531]: pgmap v6203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:20.941 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2794350606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:20.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:20.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:20.693686+0000) 2023-08-28T00:20:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:20 smithi093 ceph-mon[111335]: pgmap v6203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2794350606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:20 smithi093 ceph-mon[118902]: pgmap v6203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2794350606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:20.997Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:20 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:20.998Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:21.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:21.693938+0000) 2023-08-28T00:20:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:22 smithi093 ceph-mon[118902]: pgmap v6204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3579846770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:22.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:22 smithi093 ceph-mon[111335]: pgmap v6204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:22.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3579846770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:22.466Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:22.467Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:22 smithi136 ceph-mon[113531]: pgmap v6204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3579846770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:22.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:22.694163+0000) 2023-08-28T00:20:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:23.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:23.694427+0000) 2023-08-28T00:20:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:24 smithi136 ceph-mon[113531]: pgmap v6205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1254533995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:24.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:24.694699+0000) 2023-08-28T00:20:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:24 smithi093 ceph-mon[111335]: pgmap v6205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1254533995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:24 smithi093 ceph-mon[118902]: pgmap v6205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1254533995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:20:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:20:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:20:26.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:25.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:25.694976+0000) 2023-08-28T00:20:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:26 smithi136 ceph-mon[113531]: pgmap v6206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:20:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:26.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:26.695222+0000) 2023-08-28T00:20:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:26 smithi093 ceph-mon[111335]: pgmap v6206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:20:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:26 smithi093 ceph-mon[118902]: pgmap v6206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:20:27.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1189864711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:27.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:27.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:27.695394+0000) 2023-08-28T00:20:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1189864711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:27.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1189864711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:28.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:28.695549+0000) 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: pgmap v6207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:20:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: pgmap v6207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: pgmap v6207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:20:29.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:20:29.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:20:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2360059641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:29.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:29.695792+0000) 2023-08-28T00:20:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2360059641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2360059641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:31.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:30 smithi136 ceph-mon[113531]: pgmap v6208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:31.192 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:30.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:30.696084+0000) 2023-08-28T00:20:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:30 smithi093 ceph-mon[111335]: pgmap v6208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:31.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:30 smithi093 ceph-mon[118902]: pgmap v6208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:30.998Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:31.000Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1584620564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:31.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:31.696367+0000) 2023-08-28T00:20:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1584620564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1584620564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:32.467Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:32.467Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:32 smithi136 ceph-mon[113531]: pgmap v6209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:32.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:32.696601+0000) 2023-08-28T00:20:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:32 smithi093 ceph-mon[111335]: pgmap v6209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:32 smithi093 ceph-mon[118902]: pgmap v6209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:33.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:33.696854+0000) 2023-08-28T00:20:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:34 smithi136 ceph-mon[113531]: pgmap v6210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/92186116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:34.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:34.697131+0000) 2023-08-28T00:20:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:34 smithi093 ceph-mon[111335]: pgmap v6210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/92186116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:34 smithi093 ceph-mon[118902]: pgmap v6210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:35.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/92186116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:20:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:20:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:20:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:35.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:35.697377+0000) 2023-08-28T00:20:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:36 smithi136 ceph-mon[113531]: pgmap v6211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3881119743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:36.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:36.697659+0000) 2023-08-28T00:20:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:36 smithi093 ceph-mon[111335]: pgmap v6211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3881119743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:36 smithi093 ceph-mon[118902]: pgmap v6211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3881119743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:37.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:37.697919+0000) 2023-08-28T00:20:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:38.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:38.698193+0000) 2023-08-28T00:20:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:38 smithi136 ceph-mon[113531]: pgmap v6212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:39.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:39.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4125702759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:38 smithi093 ceph-mon[111335]: pgmap v6212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4125702759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:38 smithi093 ceph-mon[118902]: pgmap v6212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4125702759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:39.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:39.698515+0000) 2023-08-28T00:20:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:40 smithi136 ceph-mon[113531]: pgmap v6213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:41.267 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:40.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:40.698819+0000) 2023-08-28T00:20:41.268 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:40 smithi093 ceph-mon[111335]: pgmap v6213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:41.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:40 smithi093 ceph-mon[118902]: pgmap v6213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:41.268 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:41.000Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:41.268 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:41.001Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4177053092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:41.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:41.699099+0000) 2023-08-28T00:20:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4177053092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:42.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4177053092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:42.467Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:42.469Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:42 smithi136 ceph-mon[113531]: pgmap v6214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:42.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:42.699309+0000) 2023-08-28T00:20:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:42 smithi093 ceph-mon[111335]: pgmap v6214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:42 smithi093 ceph-mon[118902]: pgmap v6214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4253992026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:44.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:43.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:43.699540+0000) 2023-08-28T00:20:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4253992026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:44.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4253992026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:44 smithi136 ceph-mon[113531]: pgmap v6215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:45.351 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:44.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:44.699780+0000) 2023-08-28T00:20:45.352 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:44 smithi093 ceph-mon[111335]: pgmap v6215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:45.352 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:44 smithi093 ceph-mon[118902]: pgmap v6215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:45.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:20:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:20:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:20:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:45.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:45.700027+0000) 2023-08-28T00:20:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:46 smithi136 ceph-mon[113531]: pgmap v6216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2042220577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:46.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:46.700319+0000) 2023-08-28T00:20:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:46 smithi093 ceph-mon[111335]: pgmap v6216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2042220577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:46 smithi093 ceph-mon[118902]: pgmap v6216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2042220577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:47.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:47.700561+0000) 2023-08-28T00:20:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:48 smithi136 ceph-mon[113531]: pgmap v6217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/189372388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:48.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:48.700795+0000) 2023-08-28T00:20:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:48 smithi093 ceph-mon[111335]: pgmap v6217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/189372388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:48 smithi093 ceph-mon[118902]: pgmap v6217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/189372388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:49.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:49.701072+0000) 2023-08-28T00:20:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:50 smithi136 ceph-mon[113531]: pgmap v6218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2409436346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:50.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:50.701308+0000) 2023-08-28T00:20:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:50 smithi093 ceph-mon[111335]: pgmap v6218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2409436346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:50 smithi093 ceph-mon[118902]: pgmap v6218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:51.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2409436346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:51.000Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:51.001Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:51.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:51.701594+0000) 2023-08-28T00:20:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:52.468Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:20:52.469Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:20:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:52 smithi136 ceph-mon[113531]: pgmap v6219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:52.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:52.701833+0000) 2023-08-28T00:20:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:52 smithi093 ceph-mon[111335]: pgmap v6219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:52 smithi093 ceph-mon[118902]: pgmap v6219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1415098928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:53.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:53.702146+0000) 2023-08-28T00:20:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1415098928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1415098928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:20:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:54 smithi136 ceph-mon[113531]: pgmap v6220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:54.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:54.702377+0000) 2023-08-28T00:20:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:54 smithi093 ceph-mon[111335]: pgmap v6220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:54 smithi093 ceph-mon[118902]: pgmap v6220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:20:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:20:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:20:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1785163411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:55.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:55.702630+0000) 2023-08-28T00:20:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1785163411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1785163411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:56 smithi136 ceph-mon[113531]: pgmap v6221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:56.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:56.702892+0000) 2023-08-28T00:20:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:56 smithi093 ceph-mon[111335]: pgmap v6221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:56 smithi093 ceph-mon[118902]: pgmap v6221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:57.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:57.703115+0000) 2023-08-28T00:20:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:58 smithi136 ceph-mon[113531]: pgmap v6222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:20:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2764090881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:59.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:58.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:58.703311+0000) 2023-08-28T00:20:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:58 smithi093 ceph-mon[111335]: pgmap v6222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:20:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2764090881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:20:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:58 smithi093 ceph-mon[118902]: pgmap v6222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:20:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:20:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2764090881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:20:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:20:59.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:20:59.703555+0000) 2023-08-28T00:21:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:00 smithi136 ceph-mon[113531]: pgmap v6223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3588427546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:00.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:00.703836+0000) 2023-08-28T00:21:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:00 smithi093 ceph-mon[111335]: pgmap v6223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3588427546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:00 smithi093 ceph-mon[118902]: pgmap v6223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3588427546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:01.001Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:01.002Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:01.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:01.704124+0000) 2023-08-28T00:21:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:02.469Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:02.470Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:02 smithi136 ceph-mon[113531]: pgmap v6224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3848305624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:02.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:02.704397+0000) 2023-08-28T00:21:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:02 smithi093 ceph-mon[118902]: pgmap v6224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3848305624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:03.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:02 smithi093 ceph-mon[111335]: pgmap v6224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:03.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3848305624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:03.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:03.704668+0000) 2023-08-28T00:21:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:04 smithi136 ceph-mon[113531]: pgmap v6225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:04.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:04.704881+0000) 2023-08-28T00:21:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:04 smithi093 ceph-mon[111335]: pgmap v6225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:04 smithi093 ceph-mon[118902]: pgmap v6225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:05.690 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 00:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T00:21:05.320975038Z level=info msg="Completed cleanup jobs" duration=16.319424ms 2023-08-28T00:21:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:21:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:21:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:21:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1870185305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:05.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:05.705145+0000) 2023-08-28T00:21:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1870185305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1870185305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:06 smithi136 ceph-mon[113531]: pgmap v6226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:06.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:06.705387+0000) 2023-08-28T00:21:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:06 smithi093 ceph-mon[111335]: pgmap v6226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:06 smithi093 ceph-mon[118902]: pgmap v6226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1829061251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:07.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:07.705653+0000) 2023-08-28T00:21:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1829061251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1829061251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:08 smithi136 ceph-mon[113531]: pgmap v6227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:08.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:08.705912+0000) 2023-08-28T00:21:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:08 smithi093 ceph-mon[111335]: pgmap v6227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:08 smithi093 ceph-mon[118902]: pgmap v6227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3066320961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:09.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:09.706201+0000) 2023-08-28T00:21:10.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3066320961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:10.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3066320961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:10 smithi136 ceph-mon[113531]: pgmap v6228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:10.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:10.706494+0000) 2023-08-28T00:21:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:10 smithi093 ceph-mon[111335]: pgmap v6228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:10 smithi093 ceph-mon[118902]: pgmap v6228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:11.002Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:11.003Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:11.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:11.706741+0000) 2023-08-28T00:21:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:12.470Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:12.470Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:12 smithi136 ceph-mon[113531]: pgmap v6229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4048927275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:12.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:12.707025+0000) 2023-08-28T00:21:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:12 smithi093 ceph-mon[118902]: pgmap v6229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4048927275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:12 smithi093 ceph-mon[111335]: pgmap v6229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:13.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4048927275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:13.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:13.707233+0000) 2023-08-28T00:21:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:14 smithi136 ceph-mon[113531]: pgmap v6230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/296467340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:14.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:14.707479+0000) 2023-08-28T00:21:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:14 smithi093 ceph-mon[111335]: pgmap v6230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/296467340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:14 smithi093 ceph-mon[118902]: pgmap v6230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:15.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/296467340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:21:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:21:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:21:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:15.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:15.707754+0000) 2023-08-28T00:21:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:16 smithi136 ceph-mon[113531]: pgmap v6231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:16.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:16.708051+0000) 2023-08-28T00:21:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:16 smithi093 ceph-mon[118902]: pgmap v6231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:16 smithi093 ceph-mon[111335]: pgmap v6231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3808859876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:17.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:17.708310+0000) 2023-08-28T00:21:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3808859876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:18.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3808859876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:18 smithi136 ceph-mon[113531]: pgmap v6232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:18.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:18.708550+0000) 2023-08-28T00:21:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:18 smithi093 ceph-mon[111335]: pgmap v6232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:18 smithi093 ceph-mon[118902]: pgmap v6232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1047701951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:19.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:19.708818+0000) 2023-08-28T00:21:20.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1047701951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:20.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1047701951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:20 smithi136 ceph-mon[113531]: pgmap v6233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:20.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:20.709089+0000) 2023-08-28T00:21:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:20 smithi093 ceph-mon[118902]: pgmap v6233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:21.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:20 smithi093 ceph-mon[111335]: pgmap v6233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:21.002Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:21.003Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2080193407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:21.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:21.709322+0000) 2023-08-28T00:21:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2080193407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2080193407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:22.471Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:22.472Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:22 smithi136 ceph-mon[113531]: pgmap v6234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:22.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:22.709541+0000) 2023-08-28T00:21:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:22 smithi093 ceph-mon[111335]: pgmap v6234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:22 smithi093 ceph-mon[118902]: pgmap v6234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:23.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:23.709722+0000) 2023-08-28T00:21:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:24 smithi136 ceph-mon[113531]: pgmap v6235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3333600989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:24.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:24.709925+0000) 2023-08-28T00:21:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:24 smithi093 ceph-mon[111335]: pgmap v6235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3333600989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:24 smithi093 ceph-mon[118902]: pgmap v6235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3333600989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:21:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:21:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:21:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:25.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:25.710148+0000) 2023-08-28T00:21:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:26 smithi136 ceph-mon[113531]: pgmap v6236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3793172738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:26.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:26.710408+0000) 2023-08-28T00:21:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:26 smithi093 ceph-mon[111335]: pgmap v6236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3793172738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:26 smithi093 ceph-mon[118902]: pgmap v6236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3793172738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:27.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:27.710683+0000) 2023-08-28T00:21:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:28 smithi136 ceph-mon[113531]: pgmap v6237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3223896423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:21:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:28.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:28.710936+0000) 2023-08-28T00:21:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:28 smithi093 ceph-mon[118902]: pgmap v6237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3223896423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:21:29.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:28 smithi093 ceph-mon[111335]: pgmap v6237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:29.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3223896423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:29.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:21:29.836 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:29.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:29.711135+0000) 2023-08-28T00:21:30.964 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:30.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:30.711301+0000) 2023-08-28T00:21:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:30 smithi093 ceph-mon[111335]: pgmap v6238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:31.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:31.004Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:31.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:30 smithi093 ceph-mon[118902]: pgmap v6238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:31.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:31.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:30 smithi136 ceph-mon[113531]: pgmap v6238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:31.965 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:31.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:31.711460+0000) 2023-08-28T00:21:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/342638669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/342638669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:21:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/342638669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:21:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:21:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:21:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:32.472Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:32.473Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:32.965 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:32.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:32.711609+0000) 2023-08-28T00:21:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:32 smithi093 ceph-mon[111335]: pgmap v6239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:32 smithi093 ceph-mon[118902]: pgmap v6239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:32 smithi136 ceph-mon[113531]: pgmap v6239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:33.970 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:33.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:33.711877+0000) 2023-08-28T00:21:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/243524148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/243524148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/243524148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:34.971 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:34.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:34.712195+0000) 2023-08-28T00:21:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:34 smithi093 ceph-mon[111335]: pgmap v6240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:34 smithi093 ceph-mon[118902]: pgmap v6240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:34 smithi136 ceph-mon[113531]: pgmap v6240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:21:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:21:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:21:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:35.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:35.712387+0000) 2023-08-28T00:21:36.975 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:36.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:36.712603+0000) 2023-08-28T00:21:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:36 smithi093 ceph-mon[111335]: pgmap v6241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1123826878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:36 smithi093 ceph-mon[118902]: pgmap v6241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:37.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1123826878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:36 smithi136 ceph-mon[113531]: pgmap v6241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1123826878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:37.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:37.712786+0000) 2023-08-28T00:21:38.977 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:38.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:38.713088+0000) 2023-08-28T00:21:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:38 smithi093 ceph-mon[111335]: pgmap v6242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2374370025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:38 smithi093 ceph-mon[118902]: pgmap v6242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2374370025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:38 smithi136 ceph-mon[113531]: pgmap v6242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2374370025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:39.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:39.713355+0000) 2023-08-28T00:21:40.980 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:40.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:40.713625+0000) 2023-08-28T00:21:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:40 smithi093 ceph-mon[111335]: pgmap v6243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4293064647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:40 smithi093 ceph-mon[118902]: pgmap v6243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4293064647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:41.004Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:41.005Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:40 smithi136 ceph-mon[113531]: pgmap v6243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4293064647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:41.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:41.713889+0000) 2023-08-28T00:21:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:42.473Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:42.473Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:42.983 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:42.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:42.714162+0000) 2023-08-28T00:21:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:42 smithi093 ceph-mon[111335]: pgmap v6244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:42 smithi093 ceph-mon[118902]: pgmap v6244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:42 smithi136 ceph-mon[113531]: pgmap v6244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:43.992 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:43.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:43.714378+0000) 2023-08-28T00:21:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/280550602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/280550602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/280550602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:44.993 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:44.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:44.714585+0000) 2023-08-28T00:21:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:44 smithi093 ceph-mon[111335]: pgmap v6245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:44 smithi093 ceph-mon[118902]: pgmap v6245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:44 smithi136 ceph-mon[113531]: pgmap v6245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:21:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:21:45] "GET /metrics HTTP/1.1" 200 239921 "" "Prometheus/2.43.0" 2023-08-28T00:21:46.003 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:45.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:45.714835+0000) 2023-08-28T00:21:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2693163196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2693163196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2693163196' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:47.004 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:46.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:46.715106+0000) 2023-08-28T00:21:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:47 smithi136 ceph-mon[113531]: pgmap v6246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:47 smithi093 ceph-mon[111335]: pgmap v6246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:47 smithi093 ceph-mon[118902]: pgmap v6246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:48.011 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:47.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:47.715327+0000) 2023-08-28T00:21:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/842415698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/842415698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/842415698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:49.013 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:48.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:48.715545+0000) 2023-08-28T00:21:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:49 smithi136 ceph-mon[113531]: pgmap v6247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:49 smithi093 ceph-mon[111335]: pgmap v6247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:49.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:49 smithi093 ceph-mon[118902]: pgmap v6247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:49.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:49.715762+0000) 2023-08-28T00:21:51.016 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:50.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:50.716024+0000) 2023-08-28T00:21:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:51 smithi136 ceph-mon[113531]: pgmap v6248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/457776490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:51 smithi093 ceph-mon[111335]: pgmap v6248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/457776490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:51 smithi093 ceph-mon[118902]: pgmap v6248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/457776490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:51.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:51.005Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:51.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:51.716253+0000) 2023-08-28T00:21:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:52.473Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:21:52.474Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:21:53.018 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:52.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:52.716474+0000) 2023-08-28T00:21:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:53 smithi136 ceph-mon[113531]: pgmap v6249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3916336610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:53.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:53 smithi093 ceph-mon[111335]: pgmap v6249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:53.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3916336610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:53 smithi093 ceph-mon[118902]: pgmap v6249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:53.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3916336610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:54.023 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:53.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:53.716721+0000) 2023-08-28T00:21:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:21:55.023 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:54.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:54.716968+0000) 2023-08-28T00:21:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:55 smithi093 ceph-mon[111335]: pgmap v6250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:55 smithi093 ceph-mon[118902]: pgmap v6250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:55 smithi136 ceph-mon[113531]: pgmap v6250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:21:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:21:55] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:21:56.028 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:55.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:55.717227+0000) 2023-08-28T00:21:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1127028153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1127028153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1127028153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:57.030 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:56.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:56.717491+0000) 2023-08-28T00:21:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:57 smithi136 ceph-mon[113531]: pgmap v6251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:57 smithi093 ceph-mon[111335]: pgmap v6251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:57 smithi093 ceph-mon[118902]: pgmap v6251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:58.035 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:57.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:57.717689+0000) 2023-08-28T00:21:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3844601515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3844601515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3844601515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:21:59.036 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:58.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:58.717933+0000) 2023-08-28T00:21:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:21:59 smithi136 ceph-mon[113531]: pgmap v6252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:21:59 smithi093 ceph-mon[111335]: pgmap v6252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:21:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:21:59 smithi093 ceph-mon[118902]: pgmap v6252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:00.043 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:21:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:21:59.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:21:59.718170+0000) 2023-08-28T00:22:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2682023544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2682023544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2682023544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:01.044 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:00.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:00.718407+0000) 2023-08-28T00:22:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:01 smithi136 ceph-mon[113531]: pgmap v6253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:01 smithi093 ceph-mon[111335]: pgmap v6253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:01 smithi093 ceph-mon[118902]: pgmap v6253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:01.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:01.007Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:01.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:01.718661+0000) 2023-08-28T00:22:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:02.474Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:02.475Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:03.047 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:02.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:02.718888+0000) 2023-08-28T00:22:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:03 smithi136 ceph-mon[113531]: pgmap v6254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1440616913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:03 smithi093 ceph-mon[111335]: pgmap v6254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1440616913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:03 smithi093 ceph-mon[118902]: pgmap v6254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:03.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1440616913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:03.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:03.719128+0000) 2023-08-28T00:22:05.050 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:04.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:04.719347+0000) 2023-08-28T00:22:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:05 smithi093 ceph-mon[111335]: pgmap v6255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:05.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3397570506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:05 smithi093 ceph-mon[118902]: pgmap v6255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3397570506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:05 smithi136 ceph-mon[113531]: pgmap v6255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3397570506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:22:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:22:05] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:22:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:05.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:05.719597+0000) 2023-08-28T00:22:07.053 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:06.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:06.719885+0000) 2023-08-28T00:22:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:07 smithi136 ceph-mon[113531]: pgmap v6256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2080502899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:07 smithi093 ceph-mon[111335]: pgmap v6256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2080502899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:07 smithi093 ceph-mon[118902]: pgmap v6256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2080502899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:07.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:07.720123+0000) 2023-08-28T00:22:09.055 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:08.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:08.720378+0000) 2023-08-28T00:22:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:09 smithi136 ceph-mon[113531]: pgmap v6257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:09 smithi093 ceph-mon[111335]: pgmap v6257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:09 smithi093 ceph-mon[118902]: pgmap v6257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:10.060 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:09.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:09.720637+0000) 2023-08-28T00:22:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2298125335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2298125335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2298125335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:11.062 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:10.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:10.720909+0000) 2023-08-28T00:22:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:11 smithi136 ceph-mon[113531]: pgmap v6258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:11 smithi093 ceph-mon[118902]: pgmap v6258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:11.006Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:11.007Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:11.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:11 smithi093 ceph-mon[111335]: pgmap v6258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:12.066 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:11.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:11.721193+0000) 2023-08-28T00:22:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2677998269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:12.475 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2677998269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:12.475 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2677998269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:12.474Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:12.475Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:13.068 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:12.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:12.721445+0000) 2023-08-28T00:22:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:13 smithi136 ceph-mon[113531]: pgmap v6259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:13 smithi093 ceph-mon[111335]: pgmap v6259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:13 smithi093 ceph-mon[118902]: pgmap v6259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:13.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:13.721731+0000) 2023-08-28T00:22:15.071 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:14.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:14.721984+0000) 2023-08-28T00:22:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:15 smithi093 ceph-mon[111335]: pgmap v6260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3755325294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:15 smithi093 ceph-mon[118902]: pgmap v6260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3755325294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:15 smithi136 ceph-mon[113531]: pgmap v6260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3755325294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:22:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:22:15] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:22:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:15.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:15.722200+0000) 2023-08-28T00:22:17.075 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:16.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:16.722407+0000) 2023-08-28T00:22:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:17 smithi136 ceph-mon[113531]: pgmap v6261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1467503892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:17 smithi093 ceph-mon[111335]: pgmap v6261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1467503892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:17 smithi093 ceph-mon[118902]: pgmap v6261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1467503892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:17.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:17.722644+0000) 2023-08-28T00:22:19.078 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:18.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:18.722936+0000) 2023-08-28T00:22:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:19 smithi136 ceph-mon[113531]: pgmap v6262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2215131589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:19.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:19 smithi093 ceph-mon[111335]: pgmap v6262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:19.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2215131589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:19.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:19 smithi093 ceph-mon[118902]: pgmap v6262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:19.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2215131589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:19.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:19.723196+0000) 2023-08-28T00:22:21.080 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:20.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:20.723443+0000) 2023-08-28T00:22:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:21 smithi136 ceph-mon[113531]: pgmap v6263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:21 smithi093 ceph-mon[111335]: pgmap v6263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:21.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:21 smithi093 ceph-mon[118902]: pgmap v6263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:21.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:21.008Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:22.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:21.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:21.723686+0000) 2023-08-28T00:22:22.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2164080038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:22.476 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2164080038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:22.476 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2164080038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:22.475Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:22.476Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:23.086 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:22.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:22.723969+0000) 2023-08-28T00:22:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:23 smithi136 ceph-mon[113531]: pgmap v6264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:23 smithi093 ceph-mon[111335]: pgmap v6264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:23 smithi093 ceph-mon[118902]: pgmap v6264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:24.093 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:23.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:23.724179+0000) 2023-08-28T00:22:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2699188986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2699188986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:24.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2699188986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:24.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:25.095 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:24.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:24.724434+0000) 2023-08-28T00:22:25.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:25 smithi093 ceph-mon[111335]: pgmap v6265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:25.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:25 smithi093 ceph-mon[118902]: pgmap v6265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:25.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:25 smithi136 ceph-mon[113531]: pgmap v6265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:22:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:22:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:22:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:25.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:25.724682+0000) 2023-08-28T00:22:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:26 smithi136 ceph-mon[113531]: pgmap v6266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/501653044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:26.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:26.724958+0000) 2023-08-28T00:22:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:26 smithi093 ceph-mon[111335]: pgmap v6266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/501653044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:26 smithi093 ceph-mon[118902]: pgmap v6266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/501653044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:27.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:27.725196+0000) 2023-08-28T00:22:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:28.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:28.725428+0000) 2023-08-28T00:22:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:28 smithi136 ceph-mon[113531]: pgmap v6267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/27622949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:28 smithi093 ceph-mon[111335]: pgmap v6267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/27622949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:28 smithi093 ceph-mon[118902]: pgmap v6267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/27622949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:29.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:29.725674+0000) 2023-08-28T00:22:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:30 smithi136 ceph-mon[113531]: pgmap v6268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:30.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:30.725927+0000) 2023-08-28T00:22:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:30 smithi093 ceph-mon[111335]: pgmap v6268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:30 smithi093 ceph-mon[118902]: pgmap v6268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:31.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:31.008Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:31.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3464223233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:31.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:31.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:31.726181+0000) 2023-08-28T00:22:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3464223233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:31.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3464223233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:32 smithi093 ceph-mon[111335]: pgmap v6269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:22:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:32 smithi093 ceph-mon[118902]: pgmap v6269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:22:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:32.475Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:32.476Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:32.909 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:32 smithi136 ceph-mon[113531]: pgmap v6269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:32.909 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:22:32.909 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:32.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:32.726342+0000) 2023-08-28T00:22:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1278611971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:33.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:33.726494+0000) 2023-08-28T00:22:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1278611971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1278611971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[111335]: pgmap v6270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[118902]: pgmap v6270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:34.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:34.726730+0000) 2023-08-28T00:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:34 smithi136 ceph-mon[113531]: pgmap v6270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:35.626 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:22:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:22:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:22:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:35.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:35.726923+0000) 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:36.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:36.727165+0000) 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-mon[113531]: pgmap v6271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4153880859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:22:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[111335]: pgmap v6271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4153880859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[118902]: pgmap v6271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4153880859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:22:36.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:22:36.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:22:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:37.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:37.727351+0000) 2023-08-28T00:22:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:38 smithi136 ceph-mon[113531]: pgmap v6272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:38.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2860045164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:38.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:38.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:38.727603+0000) 2023-08-28T00:22:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:38 smithi093 ceph-mon[118902]: pgmap v6272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2860045164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:38.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:38 smithi093 ceph-mon[111335]: pgmap v6272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:38.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2860045164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:39.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:39.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:39.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:39.727873+0000) 2023-08-28T00:22:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:40 smithi136 ceph-mon[113531]: pgmap v6273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:40.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/58893516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:40.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:40.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:40.728149+0000) 2023-08-28T00:22:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:40 smithi093 ceph-mon[111335]: pgmap v6273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/58893516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:40 smithi093 ceph-mon[118902]: pgmap v6273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:40.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/58893516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:41.009Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:41.010Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:41.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:41.728416+0000) 2023-08-28T00:22:42.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:42 smithi093 ceph-mon[111335]: pgmap v6274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:42.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:42 smithi093 ceph-mon[118902]: pgmap v6274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:42.476Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:42.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:42.477Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:42 smithi136 ceph-mon[113531]: pgmap v6274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:42.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:42.728667+0000) 2023-08-28T00:22:43.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3618901915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:43.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:43.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:43.728898+0000) 2023-08-28T00:22:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3618901915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3618901915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:44 smithi136 ceph-mon[113531]: pgmap v6275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:44.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:44.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:44.729163+0000) 2023-08-28T00:22:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:44 smithi093 ceph-mon[111335]: pgmap v6275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:44 smithi093 ceph-mon[118902]: pgmap v6275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:45.656 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:22:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:22:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:22:45.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2838358071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:45.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:45.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:45.729408+0000) 2023-08-28T00:22:45.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2838358071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:45.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2838358071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:46.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:46 smithi136 ceph-mon[113531]: pgmap v6276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:46.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:46.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:46.729693+0000) 2023-08-28T00:22:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:46 smithi093 ceph-mon[111335]: pgmap v6276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:46.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:46 smithi093 ceph-mon[118902]: pgmap v6276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:47.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2613701155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:47.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:47.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:47.729918+0000) 2023-08-28T00:22:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2613701155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:47.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2613701155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:48.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:48 smithi136 ceph-mon[113531]: pgmap v6277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:48.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:48.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:48.730146+0000) 2023-08-28T00:22:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:48 smithi093 ceph-mon[111335]: pgmap v6277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:48 smithi093 ceph-mon[118902]: pgmap v6277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:49.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:49.730411+0000) 2023-08-28T00:22:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:50 smithi136 ceph-mon[113531]: pgmap v6278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:50.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/248792134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:50.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:50.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:50.730660+0000) 2023-08-28T00:22:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:50 smithi093 ceph-mon[111335]: pgmap v6278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/248792134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:50 smithi093 ceph-mon[118902]: pgmap v6278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:50.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/248792134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:51.009Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:51.010Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:51.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:51.730865+0000) 2023-08-28T00:22:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:52 smithi093 ceph-mon[111335]: pgmap v6279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2258898280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:52 smithi093 ceph-mon[118902]: pgmap v6279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:52.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2258898280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:52.476Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:22:52.477Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:22:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:52 smithi136 ceph-mon[113531]: pgmap v6279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:52.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2258898280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:52.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:52.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:52.731150+0000) 2023-08-28T00:22:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:53.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:53.731360+0000) 2023-08-28T00:22:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:54 smithi136 ceph-mon[113531]: pgmap v6280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:54.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:54.731627+0000) 2023-08-28T00:22:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:54 smithi093 ceph-mon[111335]: pgmap v6280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:54 smithi093 ceph-mon[118902]: pgmap v6280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:22:55.682 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:22:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:22:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:22:55.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2417557257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:55.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:55.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:55.731882+0000) 2023-08-28T00:22:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2417557257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2417557257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:56.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:56 smithi136 ceph-mon[113531]: pgmap v6281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:56.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:56.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:56.732121+0000) 2023-08-28T00:22:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:56 smithi093 ceph-mon[111335]: pgmap v6281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:56 smithi093 ceph-mon[118902]: pgmap v6281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:57.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3860837983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:57.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3860837983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3860837983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:57.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:57.732386+0000) 2023-08-28T00:22:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:58 smithi093 ceph-mon[111335]: pgmap v6282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:58.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:58 smithi093 ceph-mon[118902]: pgmap v6282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:58 smithi136 ceph-mon[113531]: pgmap v6282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:22:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:58.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:58.732603+0000) 2023-08-28T00:22:59.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:22:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2427358238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:22:59.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:22:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2427358238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:22:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2427358238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:22:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:22:59.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:22:59.732885+0000) 2023-08-28T00:23:00.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:00 smithi093 ceph-mon[111335]: pgmap v6283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:00 smithi093 ceph-mon[118902]: pgmap v6283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:00 smithi136 ceph-mon[113531]: pgmap v6283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:01.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:00.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:00.733159+0000) 2023-08-28T00:23:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:01.009Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:01.010Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:01.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:01.733385+0000) 2023-08-28T00:23:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:02 smithi093 ceph-mon[111335]: pgmap v6284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4049658420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:02 smithi093 ceph-mon[118902]: pgmap v6284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:02.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4049658420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:02.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:02.477Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:02.742 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:02.478Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:02 smithi136 ceph-mon[113531]: pgmap v6284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4049658420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:03.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:02.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:02.733664+0000) 2023-08-28T00:23:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:03.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:03.733881+0000) 2023-08-28T00:23:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:04 smithi093 ceph-mon[111335]: pgmap v6285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:04.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2494133118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:04.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:04 smithi093 ceph-mon[118902]: pgmap v6285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:04.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2494133118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:04 smithi136 ceph-mon[113531]: pgmap v6285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2494133118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:04.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:04.734114+0000) 2023-08-28T00:23:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:23:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:23:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:23:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:05.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:05.734343+0000) 2023-08-28T00:23:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:06 smithi093 ceph-mon[111335]: pgmap v6286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4006246892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:06 smithi093 ceph-mon[118902]: pgmap v6286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4006246892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:06 smithi136 ceph-mon[113531]: pgmap v6286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4006246892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:06.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:06.734596+0000) 2023-08-28T00:23:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:07.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:07.734814+0000) 2023-08-28T00:23:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:08 smithi093 ceph-mon[111335]: pgmap v6287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:08 smithi093 ceph-mon[118902]: pgmap v6287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:08 smithi136 ceph-mon[113531]: pgmap v6287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:08.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:08.735087+0000) 2023-08-28T00:23:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2661469640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:09.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2661469640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2661469640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:09.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:09.735345+0000) 2023-08-28T00:23:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:10 smithi093 ceph-mon[111335]: pgmap v6288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:10 smithi093 ceph-mon[118902]: pgmap v6288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:10 smithi136 ceph-mon[113531]: pgmap v6288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:10.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:10.735622+0000) 2023-08-28T00:23:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:11.010Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:11.011Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:11.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/194291940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:11.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/194291940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/194291940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:11.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:11.735894+0000) 2023-08-28T00:23:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:12 smithi093 ceph-mon[111335]: pgmap v6289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:12 smithi093 ceph-mon[118902]: pgmap v6289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:12.477Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:12.478Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:12 smithi136 ceph-mon[113531]: pgmap v6289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:12.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:12.736098+0000) 2023-08-28T00:23:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:13.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:13.736363+0000) 2023-08-28T00:23:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:14 smithi093 ceph-mon[111335]: pgmap v6290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1639469537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:14 smithi093 ceph-mon[118902]: pgmap v6290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:14.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1639469537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:14 smithi136 ceph-mon[113531]: pgmap v6290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1639469537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:14.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:14.736631+0000) 2023-08-28T00:23:15.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:23:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:23:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:23:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:15.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:15.736875+0000) 2023-08-28T00:23:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:16 smithi093 ceph-mon[111335]: pgmap v6291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2647546301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:16 smithi093 ceph-mon[118902]: pgmap v6291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:16.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2647546301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:16 smithi136 ceph-mon[113531]: pgmap v6291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2647546301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:16.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:16.737157+0000) 2023-08-28T00:23:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:17.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:17.737391+0000) 2023-08-28T00:23:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:18 smithi093 ceph-mon[111335]: pgmap v6292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2224315872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:18 smithi093 ceph-mon[118902]: pgmap v6292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2224315872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:18 smithi136 ceph-mon[113531]: pgmap v6292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2224315872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:18.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:18.737642+0000) 2023-08-28T00:23:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:19.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:19.737935+0000) 2023-08-28T00:23:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:20 smithi093 ceph-mon[111335]: pgmap v6293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:20 smithi093 ceph-mon[118902]: pgmap v6293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:20 smithi136 ceph-mon[113531]: pgmap v6293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:20.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:20.738124+0000) 2023-08-28T00:23:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:21.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:21.011Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2407762734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:21.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:21.738276+0000) 2023-08-28T00:23:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2407762734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2407762734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:22.478Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:22.479Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:22 smithi136 ceph-mon[113531]: pgmap v6294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:22.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:22.738496+0000) 2023-08-28T00:23:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:22 smithi093 ceph-mon[111335]: pgmap v6294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:22 smithi093 ceph-mon[118902]: pgmap v6294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/356423811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:23.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:23.738712+0000) 2023-08-28T00:23:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/356423811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:24.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/356423811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:24 smithi136 ceph-mon[113531]: pgmap v6295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:24.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:24.738980+0000) 2023-08-28T00:23:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:24 smithi093 ceph-mon[111335]: pgmap v6295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:24 smithi093 ceph-mon[118902]: pgmap v6295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:23:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:23:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:23:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:25.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:25.739229+0000) 2023-08-28T00:23:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:26 smithi136 ceph-mon[113531]: pgmap v6296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1117069312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:26.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:26.739504+0000) 2023-08-28T00:23:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:26 smithi093 ceph-mon[111335]: pgmap v6296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1117069312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:26 smithi093 ceph-mon[118902]: pgmap v6296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1117069312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:27.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:27.739706+0000) 2023-08-28T00:23:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:28 smithi136 ceph-mon[113531]: pgmap v6297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2871289899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:28.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:28.739964+0000) 2023-08-28T00:23:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:28 smithi093 ceph-mon[111335]: pgmap v6297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2871289899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:28 smithi093 ceph-mon[118902]: pgmap v6297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2871289899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:29.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:29.740216+0000) 2023-08-28T00:23:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:30 smithi136 ceph-mon[113531]: pgmap v6298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/310900850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:30.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:30.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:30.740483+0000) 2023-08-28T00:23:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:30 smithi093 ceph-mon[111335]: pgmap v6298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/310900850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:30 smithi093 ceph-mon[118902]: pgmap v6298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/310900850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:31.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:31.012Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:31.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:31.740758+0000) 2023-08-28T00:23:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:32 smithi093 ceph-mon[111335]: pgmap v6299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:32 smithi093 ceph-mon[118902]: pgmap v6299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:32.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:32.480Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:32 smithi136 ceph-mon[113531]: pgmap v6299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:32.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:32.740991+0000) 2023-08-28T00:23:33.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1565256843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:33.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:33.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:33.741233+0000) 2023-08-28T00:23:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1565256843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:33.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1565256843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:34 smithi136 ceph-mon[113531]: pgmap v6300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:34.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:34.741482+0000) 2023-08-28T00:23:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:34 smithi093 ceph-mon[111335]: pgmap v6300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:34 smithi093 ceph-mon[118902]: pgmap v6300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:35.662 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:23:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:23:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:23:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/195188224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:35.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:35.741741+0000) 2023-08-28T00:23:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/195188224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/195188224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:36 smithi136 ceph-mon[113531]: pgmap v6301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:36.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:23:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:36.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:36.741968+0000) 2023-08-28T00:23:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:36 smithi093 ceph-mon[111335]: pgmap v6301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:23:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:36 smithi093 ceph-mon[118902]: pgmap v6301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:23:37.743 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4219315076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:37.992 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4219315076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:37.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4219315076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:37.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:37.742179+0000) 2023-08-28T00:23:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:38.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:38.742327+0000) 2023-08-28T00:23:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:38 smithi136 ceph-mon[113531]: pgmap v6302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[111335]: pgmap v6302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[118902]: pgmap v6302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:23:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:23:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:39.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:39.742504+0000) 2023-08-28T00:23:40.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:23:40.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:23:40.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:40.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:23:40.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:23:40.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:23:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:40 smithi136 ceph-mon[113531]: pgmap v6303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4135939124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:40.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:40.742736+0000) 2023-08-28T00:23:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:40 smithi093 ceph-mon[111335]: pgmap v6303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4135939124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:40 smithi093 ceph-mon[118902]: pgmap v6303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4135939124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:41.012Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:41.014Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:41.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:41.742959+0000) 2023-08-28T00:23:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:42.480Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:42.481Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:42 smithi136 ceph-mon[113531]: pgmap v6304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/446620522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:42.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:42.743251+0000) 2023-08-28T00:23:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:42 smithi093 ceph-mon[111335]: pgmap v6304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/446620522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:42 smithi093 ceph-mon[118902]: pgmap v6304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/446620522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:43.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:43.743506+0000) 2023-08-28T00:23:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:44 smithi136 ceph-mon[113531]: pgmap v6305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/842412061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:44.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:44.743773+0000) 2023-08-28T00:23:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:44 smithi093 ceph-mon[111335]: pgmap v6305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/842412061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:44 smithi093 ceph-mon[118902]: pgmap v6305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/842412061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:23:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:23:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:23:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:45.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:45.744052+0000) 2023-08-28T00:23:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:46 smithi136 ceph-mon[113531]: pgmap v6306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:46.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:46.744264+0000) 2023-08-28T00:23:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:46 smithi093 ceph-mon[111335]: pgmap v6306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:46 smithi093 ceph-mon[118902]: pgmap v6306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3926190308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:47.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:47.744502+0000) 2023-08-28T00:23:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3926190308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3926190308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:48 smithi136 ceph-mon[113531]: pgmap v6307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:48.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:48.744774+0000) 2023-08-28T00:23:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:48 smithi093 ceph-mon[111335]: pgmap v6307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:48 smithi093 ceph-mon[118902]: pgmap v6307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3483352125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:50.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:49.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:49.745071+0000) 2023-08-28T00:23:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3483352125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3483352125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:50 smithi136 ceph-mon[113531]: pgmap v6308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:50.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:50.745267+0000) 2023-08-28T00:23:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:50 smithi093 ceph-mon[111335]: pgmap v6308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:50 smithi093 ceph-mon[118902]: pgmap v6308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:51.014Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:51.015Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1035608658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:51.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:51.745492+0000) 2023-08-28T00:23:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1035608658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1035608658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:52.481Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:23:52.482Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:23:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:52 smithi136 ceph-mon[113531]: pgmap v6309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:52.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:52.745776+0000) 2023-08-28T00:23:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:52 smithi093 ceph-mon[111335]: pgmap v6309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:52 smithi093 ceph-mon[118902]: pgmap v6309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:53.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:53.745997+0000) 2023-08-28T00:23:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:23:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:54 smithi136 ceph-mon[113531]: pgmap v6310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/435685279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:54.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:54.746269+0000) 2023-08-28T00:23:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:54 smithi093 ceph-mon[111335]: pgmap v6310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/435685279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:54 smithi093 ceph-mon[118902]: pgmap v6310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/435685279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:23:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:23:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:23:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:55.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:55.746477+0000) 2023-08-28T00:23:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:56.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:56.746730+0000) 2023-08-28T00:23:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:56 smithi136 ceph-mon[113531]: pgmap v6311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3177978143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:56 smithi093 ceph-mon[111335]: pgmap v6311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3177978143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:56 smithi093 ceph-mon[118902]: pgmap v6311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:57.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3177978143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:23:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:57.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:57.746979+0000) 2023-08-28T00:23:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:58 smithi136 ceph-mon[113531]: pgmap v6312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:58.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:58.747160+0000) 2023-08-28T00:23:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:58 smithi093 ceph-mon[111335]: pgmap v6312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:23:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:58 smithi093 ceph-mon[118902]: pgmap v6312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:23:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1785504317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:23:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:23:59.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:23:59.747423+0000) 2023-08-28T00:24:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:23:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1785504317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:23:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1785504317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:00 smithi136 ceph-mon[113531]: pgmap v6313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:00.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:00.747649+0000) 2023-08-28T00:24:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:00 smithi093 ceph-mon[111335]: pgmap v6313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:00 smithi093 ceph-mon[118902]: pgmap v6313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:01.015Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:01.016Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3962527289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:01.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:01.747903+0000) 2023-08-28T00:24:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3962527289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3962527289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:02.481Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:02.482Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:02 smithi136 ceph-mon[113531]: pgmap v6314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:02.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:02.748141+0000) 2023-08-28T00:24:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:02 smithi093 ceph-mon[111335]: pgmap v6314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:02 smithi093 ceph-mon[118902]: pgmap v6314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1598999155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:03.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:03.748387+0000) 2023-08-28T00:24:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1598999155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:04.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1598999155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:05.192 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:04 smithi136 ceph-mon[113531]: pgmap v6315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:05.192 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:04.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:04.748638+0000) 2023-08-28T00:24:05.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:04 smithi093 ceph-mon[111335]: pgmap v6315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:05.361 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:04 smithi093 ceph-mon[118902]: pgmap v6315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:24:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:24:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:24:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:05.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:05.748864+0000) 2023-08-28T00:24:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:06.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:06.749149+0000) 2023-08-28T00:24:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:06 smithi136 ceph-mon[113531]: pgmap v6316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/958085355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:06 smithi093 ceph-mon[118902]: pgmap v6316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:07.516 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/958085355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:07.516 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:06 smithi093 ceph-mon[111335]: pgmap v6316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:07.516 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/958085355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:07.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:07.749402+0000) 2023-08-28T00:24:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:08 smithi136 ceph-mon[113531]: pgmap v6317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2353202327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:08.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:08.749680+0000) 2023-08-28T00:24:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:08 smithi093 ceph-mon[111335]: pgmap v6317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2353202327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:08 smithi093 ceph-mon[118902]: pgmap v6317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2353202327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:09.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:09.749905+0000) 2023-08-28T00:24:11.002 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:10.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:10.750163+0000) 2023-08-28T00:24:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:11 smithi136 ceph-mon[113531]: pgmap v6318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:11.721 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2074972687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:11.722 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:11 smithi093 ceph-mon[111335]: pgmap v6318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:11.722 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2074972687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:11.723 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:11 smithi093 ceph-mon[118902]: pgmap v6318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:11.723 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2074972687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:11.723 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:11.016Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:11.723 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:11.017Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:11.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:11.750386+0000) 2023-08-28T00:24:12.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:12.482Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:12.755 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:12.483Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:13.004 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:12.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:12.750642+0000) 2023-08-28T00:24:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:13 smithi136 ceph-mon[113531]: pgmap v6319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:13.564 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:13 smithi093 ceph-mon[111335]: pgmap v6319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:13.564 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:13 smithi093 ceph-mon[118902]: pgmap v6319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:14.009 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:13.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:13.750894+0000) 2023-08-28T00:24:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1168655952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1168655952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1168655952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:15.011 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:14.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:14.751137+0000) 2023-08-28T00:24:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:15 smithi093 ceph-mon[111335]: pgmap v6320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:15 smithi093 ceph-mon[118902]: pgmap v6320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:15 smithi136 ceph-mon[113531]: pgmap v6320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:24:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:24:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:24:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:15.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:15.751391+0000) 2023-08-28T00:24:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:16 smithi136 ceph-mon[113531]: pgmap v6321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1892461081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:16.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:16.751662+0000) 2023-08-28T00:24:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:16 smithi093 ceph-mon[118902]: pgmap v6321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1892461081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:16 smithi093 ceph-mon[111335]: pgmap v6321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:16.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1892461081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:17.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:17.751910+0000) 2023-08-28T00:24:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:18 smithi136 ceph-mon[113531]: pgmap v6322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1920811389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:18.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:18.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:18.752176+0000) 2023-08-28T00:24:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:18 smithi093 ceph-mon[111335]: pgmap v6322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1920811389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:18 smithi093 ceph-mon[118902]: pgmap v6322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1920811389' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:19.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:19.752429+0000) 2023-08-28T00:24:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:20 smithi136 ceph-mon[113531]: pgmap v6323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:20.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4146146381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:20.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:20.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:20.752684+0000) 2023-08-28T00:24:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:20 smithi093 ceph-mon[111335]: pgmap v6323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4146146381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:20 smithi093 ceph-mon[118902]: pgmap v6323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4146146381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:21.017Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:21.018Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:21.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:21.752965+0000) 2023-08-28T00:24:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:22 smithi093 ceph-mon[111335]: pgmap v6324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:22 smithi093 ceph-mon[118902]: pgmap v6324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:22.483Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:22.484Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:22 smithi136 ceph-mon[113531]: pgmap v6324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:22.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:22.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:22.753201+0000) 2023-08-28T00:24:23.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2245614442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:23.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:23.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:23.753467+0000) 2023-08-28T00:24:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2245614442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:23.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2245614442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:24 smithi136 ceph-mon[113531]: pgmap v6325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:24.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:24.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:24.753740+0000) 2023-08-28T00:24:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:24 smithi093 ceph-mon[111335]: pgmap v6325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:24 smithi093 ceph-mon[118902]: pgmap v6325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:25.663 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:24:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:24:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:24:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1950153903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:25.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:25.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:25.753992+0000) 2023-08-28T00:24:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1950153903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1950153903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:26 smithi136 ceph-mon[113531]: pgmap v6326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:26.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:26.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:26.754270+0000) 2023-08-28T00:24:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:26 smithi093 ceph-mon[111335]: pgmap v6326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:26 smithi093 ceph-mon[118902]: pgmap v6326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:27.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/124977133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:27.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:27.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:27.754508+0000) 2023-08-28T00:24:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/124977133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/124977133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:28.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:28 smithi136 ceph-mon[113531]: pgmap v6327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:28.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:28.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:28.754716+0000) 2023-08-28T00:24:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:28 smithi093 ceph-mon[111335]: pgmap v6327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:28 smithi093 ceph-mon[118902]: pgmap v6327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:29.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:29.754981+0000) 2023-08-28T00:24:30.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:30 smithi136 ceph-mon[113531]: pgmap v6328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:30.942 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3697729326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:30.942 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:30.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:30.755244+0000) 2023-08-28T00:24:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:30 smithi093 ceph-mon[111335]: pgmap v6328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3697729326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:30 smithi093 ceph-mon[118902]: pgmap v6328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3697729326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:31.017Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:31.018Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:31.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:31.755467+0000) 2023-08-28T00:24:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:32 smithi093 ceph-mon[111335]: pgmap v6329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/658951578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:32 smithi093 ceph-mon[118902]: pgmap v6329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/658951578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:32.484Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:32.485Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:32 smithi136 ceph-mon[113531]: pgmap v6329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:32.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/658951578' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:32.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:32.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:32.755704+0000) 2023-08-28T00:24:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:33.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:33.755966+0000) 2023-08-28T00:24:34.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:34 smithi136 ceph-mon[113531]: pgmap v6330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:34.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:34.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:34.756207+0000) 2023-08-28T00:24:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:34 smithi093 ceph-mon[111335]: pgmap v6330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:34 smithi093 ceph-mon[118902]: pgmap v6330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:35.679 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:24:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:24:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:24:35.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/318411100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:35.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:35.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:35.756433+0000) 2023-08-28T00:24:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/318411100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/318411100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:36.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:36 smithi136 ceph-mon[113531]: pgmap v6331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:36.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:36.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:36.756705+0000) 2023-08-28T00:24:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:36 smithi093 ceph-mon[111335]: pgmap v6331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:36 smithi093 ceph-mon[118902]: pgmap v6331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2968463519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2968463519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:38.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2968463519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:37.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:37.756915+0000) 2023-08-28T00:24:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:38 smithi093 ceph-mon[111335]: pgmap v6332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:38 smithi093 ceph-mon[118902]: pgmap v6332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:38 smithi136 ceph-mon[113531]: pgmap v6332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:38.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:38.757184+0000) 2023-08-28T00:24:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:24:39.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/47615587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:24:39.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/47615587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:40.054 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:40.054 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:24:40.054 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/47615587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:40.054 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:39.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:39.757334+0000) 2023-08-28T00:24:40.758 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:40 smithi136 ceph-mon[113531]: pgmap v6333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:40 smithi093 ceph-mon[111335]: pgmap v6333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:40 smithi093 ceph-mon[118902]: pgmap v6333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:40.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:40.757468+0000) 2023-08-28T00:24:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:41.018Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:41.019Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:41.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:41.757653+0000) 2023-08-28T00:24:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:42.485Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:42.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:42.486Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:42.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:42.757885+0000) 2023-08-28T00:24:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:42 smithi136 ceph-mon[113531]: pgmap v6334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:24:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:24:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2066687838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[111335]: pgmap v6334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:24:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:24:43.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:43.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2066687838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[118902]: pgmap v6334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:24:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:24:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:24:43.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2066687838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:43.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:43.758102+0000) 2023-08-28T00:24:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:44 smithi136 ceph-mon[113531]: pgmap v6335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3691974525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:44.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:44.758341+0000) 2023-08-28T00:24:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:44 smithi093 ceph-mon[111335]: pgmap v6335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3691974525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:44 smithi093 ceph-mon[118902]: pgmap v6335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3691974525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:24:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:24:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:24:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:45.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:45.758638+0000) 2023-08-28T00:24:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:46 smithi136 ceph-mon[113531]: pgmap v6336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3731824537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:46.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:46.758967+0000) 2023-08-28T00:24:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:46 smithi093 ceph-mon[111335]: pgmap v6336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3731824537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:46 smithi093 ceph-mon[118902]: pgmap v6336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3731824537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:47.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:47.759166+0000) 2023-08-28T00:24:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:48 smithi136 ceph-mon[113531]: pgmap v6337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:48.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:48.759469+0000) 2023-08-28T00:24:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:48 smithi093 ceph-mon[111335]: pgmap v6337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:48 smithi093 ceph-mon[118902]: pgmap v6337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1538289094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:49.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:49.759695+0000) 2023-08-28T00:24:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1538289094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1538289094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:50 smithi136 ceph-mon[113531]: pgmap v6338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:50.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:50.759974+0000) 2023-08-28T00:24:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:50 smithi093 ceph-mon[111335]: pgmap v6338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:50 smithi093 ceph-mon[118902]: pgmap v6338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:51.020Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:51.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/179204042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:51.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:51.760227+0000) 2023-08-28T00:24:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/179204042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/179204042' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:52.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:52.486Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:52.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:24:52.487Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:24:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:52 smithi136 ceph-mon[113531]: pgmap v6339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:52.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:52.760472+0000) 2023-08-28T00:24:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:52 smithi093 ceph-mon[111335]: pgmap v6339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:52 smithi093 ceph-mon[118902]: pgmap v6339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1425060496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:53.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:53.760723+0000) 2023-08-28T00:24:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1425060496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:24:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1425060496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:54 smithi136 ceph-mon[113531]: pgmap v6340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:54.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:54.760946+0000) 2023-08-28T00:24:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:54 smithi093 ceph-mon[111335]: pgmap v6340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:54 smithi093 ceph-mon[118902]: pgmap v6340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:24:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:24:55] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:24:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:55.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:55.761152+0000) 2023-08-28T00:24:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:56 smithi136 ceph-mon[113531]: pgmap v6341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2528553238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:56.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:56.761387+0000) 2023-08-28T00:24:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:56 smithi093 ceph-mon[111335]: pgmap v6341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2528553238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:57.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:56 smithi093 ceph-mon[118902]: pgmap v6341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:57.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2528553238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:57.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:57.761624+0000) 2023-08-28T00:24:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:58 smithi136 ceph-mon[113531]: pgmap v6342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:24:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3524639334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:58.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:58.761954+0000) 2023-08-28T00:24:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:58 smithi093 ceph-mon[111335]: pgmap v6342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:24:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3524639334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:24:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:58 smithi093 ceph-mon[118902]: pgmap v6342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:24:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:24:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3524639334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:24:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:24:59.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:24:59.762200+0000) 2023-08-28T00:25:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:00 smithi136 ceph-mon[113531]: pgmap v6343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:01.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:00.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:00.762461+0000) 2023-08-28T00:25:01.243 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:00 smithi093 ceph-mon[111335]: pgmap v6343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:01.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:00 smithi093 ceph-mon[118902]: pgmap v6343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:01.243 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:01.020Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:01.244 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:01.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1292899102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:01.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:01.762742+0000) 2023-08-28T00:25:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1292899102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1292899102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:02.487Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:02.488Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:02 smithi136 ceph-mon[113531]: pgmap v6344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:02.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:02.762994+0000) 2023-08-28T00:25:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:02 smithi093 ceph-mon[111335]: pgmap v6344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:02 smithi093 ceph-mon[118902]: pgmap v6344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/814461280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:03.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:03.763272+0000) 2023-08-28T00:25:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/814461280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/814461280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:04 smithi136 ceph-mon[113531]: pgmap v6345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:04.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:04.763559+0000) 2023-08-28T00:25:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:04 smithi093 ceph-mon[111335]: pgmap v6345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:04 smithi093 ceph-mon[118902]: pgmap v6345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:25:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:25:05] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:25:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:05.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:05.763783+0000) 2023-08-28T00:25:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:06 smithi136 ceph-mon[113531]: pgmap v6346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/935439820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:06.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:06.764006+0000) 2023-08-28T00:25:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:06 smithi093 ceph-mon[111335]: pgmap v6346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/935439820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:06 smithi093 ceph-mon[118902]: pgmap v6346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:06.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/935439820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:07.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:07.764248+0000) 2023-08-28T00:25:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:08 smithi136 ceph-mon[113531]: pgmap v6347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/102734508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:08.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:08.764502+0000) 2023-08-28T00:25:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:08 smithi093 ceph-mon[111335]: pgmap v6347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/102734508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:08 smithi093 ceph-mon[118902]: pgmap v6347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/102734508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:09.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:09.764725+0000) 2023-08-28T00:25:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:10.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:10.764995+0000) 2023-08-28T00:25:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:10 smithi136 ceph-mon[113531]: pgmap v6348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/624246976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:10 smithi093 ceph-mon[118902]: pgmap v6348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:10.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/624246976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:10 smithi093 ceph-mon[111335]: pgmap v6348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/624246976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:11.020Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:11.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:11.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:11.765322+0000) 2023-08-28T00:25:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:12 smithi093 ceph-mon[111335]: pgmap v6349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:12 smithi093 ceph-mon[118902]: pgmap v6349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:12.487Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:12.488Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:12.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:12 smithi136 ceph-mon[113531]: pgmap v6349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:12.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:12.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:12.765586+0000) 2023-08-28T00:25:13.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3171380070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:13.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:13.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:13.765843+0000) 2023-08-28T00:25:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3171380070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:13.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3171380070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:14 smithi136 ceph-mon[113531]: pgmap v6350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:14.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:14.766144+0000) 2023-08-28T00:25:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:14 smithi093 ceph-mon[111335]: pgmap v6350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:14 smithi093 ceph-mon[118902]: pgmap v6350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:15.687 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:25:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:25:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:25:15.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/295632259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:15.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:15.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:15.766423+0000) 2023-08-28T00:25:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/295632259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/295632259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:16 smithi093 ceph-mon[111335]: pgmap v6351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:16 smithi093 ceph-mon[118902]: pgmap v6351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:17.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:16 smithi136 ceph-mon[113531]: pgmap v6351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:17.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:16.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:16.766697+0000) 2023-08-28T00:25:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:17.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:17.766945+0000) 2023-08-28T00:25:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:18 smithi093 ceph-mon[111335]: pgmap v6352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3447255580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:18 smithi093 ceph-mon[118902]: pgmap v6352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3447255580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:18 smithi136 ceph-mon[113531]: pgmap v6352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3447255580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:18.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:18.767253+0000) 2023-08-28T00:25:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:19.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:19.767541+0000) 2023-08-28T00:25:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:20 smithi093 ceph-mon[111335]: pgmap v6353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3188532647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:20 smithi093 ceph-mon[118902]: pgmap v6353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3188532647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:20 smithi136 ceph-mon[113531]: pgmap v6353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3188532647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:20.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:20.767733+0000) 2023-08-28T00:25:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:21.021Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:21.022Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:21.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:21.767970+0000) 2023-08-28T00:25:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:22 smithi093 ceph-mon[111335]: pgmap v6354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3699861727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:22 smithi093 ceph-mon[118902]: pgmap v6354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:22.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3699861727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:22.488Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:22.489Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:22 smithi136 ceph-mon[113531]: pgmap v6354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3699861727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:22.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:22.768190+0000) 2023-08-28T00:25:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:23.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:23.768379+0000) 2023-08-28T00:25:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:24 smithi093 ceph-mon[111335]: pgmap v6355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:24.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:24 smithi093 ceph-mon[118902]: pgmap v6355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:24 smithi136 ceph-mon[113531]: pgmap v6355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:24.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:24.768639+0000) 2023-08-28T00:25:25.710 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:25:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:25:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:25:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1088367608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1088367608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1088367608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:26.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:25.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:25.768929+0000) 2023-08-28T00:25:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:26 smithi093 ceph-mon[111335]: pgmap v6356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:26 smithi093 ceph-mon[118902]: pgmap v6356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:26 smithi136 ceph-mon[113531]: pgmap v6356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:26.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:26.769228+0000) 2023-08-28T00:25:27.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2674838818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:27.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2674838818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2674838818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:27.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:27.769519+0000) 2023-08-28T00:25:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:28 smithi093 ceph-mon[111335]: pgmap v6357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:28 smithi093 ceph-mon[118902]: pgmap v6357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:28 smithi136 ceph-mon[113531]: pgmap v6357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:28.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:28.769822+0000) 2023-08-28T00:25:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3213992947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3213992947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3213992947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:29.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:29.770108+0000) 2023-08-28T00:25:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:30 smithi093 ceph-mon[118902]: pgmap v6358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:30 smithi093 ceph-mon[111335]: pgmap v6358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:30 smithi136 ceph-mon[113531]: pgmap v6358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:30.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:30.770330+0000) 2023-08-28T00:25:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:31.022Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:31.023Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:31.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:31.770519+0000) 2023-08-28T00:25:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:32 smithi093 ceph-mon[111335]: pgmap v6359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2856987705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:32 smithi093 ceph-mon[118902]: pgmap v6359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:32.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2856987705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:32.489Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:32.490Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:32 smithi136 ceph-mon[113531]: pgmap v6359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2856987705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:32.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:32.770710+0000) 2023-08-28T00:25:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:33.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:33.770918+0000) 2023-08-28T00:25:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:34 smithi093 ceph-mon[111335]: pgmap v6360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1317058171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:34 smithi093 ceph-mon[118902]: pgmap v6360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:34.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1317058171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:34 smithi136 ceph-mon[113531]: pgmap v6360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1317058171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:35.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:34.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:34.771200+0000) 2023-08-28T00:25:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:25:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:25:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:25:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:35.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:35.771409+0000) 2023-08-28T00:25:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:36 smithi093 ceph-mon[111335]: pgmap v6361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:36 smithi093 ceph-mon[118902]: pgmap v6361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:36 smithi136 ceph-mon[113531]: pgmap v6361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:36.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:36.771648+0000) 2023-08-28T00:25:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3344524496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:37.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:37.771890+0000) 2023-08-28T00:25:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3344524496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3344524496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:38 smithi136 ceph-mon[113531]: pgmap v6362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:38.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:38.772135+0000) 2023-08-28T00:25:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:38 smithi093 ceph-mon[111335]: pgmap v6362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:38 smithi093 ceph-mon[118902]: pgmap v6362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4132434392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:39.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:39.772336+0000) 2023-08-28T00:25:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4132434392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:40.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4132434392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:41.024 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:40 smithi093 ceph-mon[111335]: pgmap v6363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:41.024 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:40 smithi093 ceph-mon[118902]: pgmap v6363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:40 smithi136 ceph-mon[113531]: pgmap v6363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:40.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:40.772609+0000) 2023-08-28T00:25:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:41.023Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:41.024Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1508887426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:41.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:41.772880+0000) 2023-08-28T00:25:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1508887426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1508887426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:42.757 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:42.490Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:42.757 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:42.491Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:42.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:42 smithi136 ceph-mon[113531]: pgmap v6364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:25:42.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:42.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:42.773088+0000) 2023-08-28T00:25:43.231 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:42 smithi093 ceph-mon[111335]: pgmap v6364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:43.232 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:25:43.232 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:42 smithi093 ceph-mon[118902]: pgmap v6364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:43.232 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:25:44.149 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:43.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:43.773241+0000) 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:44.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:44.773475+0000) 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: pgmap v6365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1288051973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:25:45.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:25:45.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: pgmap v6365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1288051973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: pgmap v6365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1288051973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:25:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:25:45.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:25:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:25:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:25:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:25:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:45.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:45.773736+0000) 2023-08-28T00:25:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:46 smithi136 ceph-mon[113531]: pgmap v6366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3332257594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:46.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:46.774050+0000) 2023-08-28T00:25:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:46 smithi093 ceph-mon[118902]: pgmap v6366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3332257594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:46 smithi093 ceph-mon[111335]: pgmap v6366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3332257594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:47.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:47.774350+0000) 2023-08-28T00:25:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:48 smithi136 ceph-mon[113531]: pgmap v6367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1412922205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:48.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:48.774598+0000) 2023-08-28T00:25:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:48 smithi093 ceph-mon[111335]: pgmap v6367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1412922205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:48 smithi093 ceph-mon[118902]: pgmap v6367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1412922205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:49.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:49.774792+0000) 2023-08-28T00:25:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:50 smithi136 ceph-mon[113531]: pgmap v6368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:50.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:50.775096+0000) 2023-08-28T00:25:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:50 smithi093 ceph-mon[111335]: pgmap v6368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:50 smithi093 ceph-mon[118902]: pgmap v6368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:51.024Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:51.025Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/686538203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:51.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:51.775359+0000) 2023-08-28T00:25:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/686538203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/686538203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:52.836 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:52.490Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:52.837 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:25:52.491Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:25:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:52 smithi136 ceph-mon[113531]: pgmap v6369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:52.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:52.775590+0000) 2023-08-28T00:25:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:52 smithi093 ceph-mon[111335]: pgmap v6369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:52 smithi093 ceph-mon[118902]: pgmap v6369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3374513319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:53.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:53.775822+0000) 2023-08-28T00:25:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3374513319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3374513319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:25:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:54 smithi136 ceph-mon[113531]: pgmap v6370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:54.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:54.776054+0000) 2023-08-28T00:25:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:54 smithi093 ceph-mon[111335]: pgmap v6370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:54 smithi093 ceph-mon[118902]: pgmap v6370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:25:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:25:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:25:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:55.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:55.776311+0000) 2023-08-28T00:25:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:56 smithi136 ceph-mon[113531]: pgmap v6371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3518189926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:56.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:56.776567+0000) 2023-08-28T00:25:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:56 smithi093 ceph-mon[111335]: pgmap v6371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3518189926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:56 smithi093 ceph-mon[118902]: pgmap v6371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3518189926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:57.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:57.776820+0000) 2023-08-28T00:25:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:58.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:58.777084+0000) 2023-08-28T00:25:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:58 smithi136 ceph-mon[113531]: pgmap v6372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:25:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2064158553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:58 smithi093 ceph-mon[111335]: pgmap v6372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:25:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2064158553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:25:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:58 smithi093 ceph-mon[118902]: pgmap v6372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:25:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:25:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2064158553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:25:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:25:59.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:25:59.777347+0000) 2023-08-28T00:26:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:00 smithi136 ceph-mon[113531]: pgmap v6373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/288956376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:00.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:00.777673+0000) 2023-08-28T00:26:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:00 smithi093 ceph-mon[111335]: pgmap v6373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/288956376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:00 smithi093 ceph-mon[118902]: pgmap v6373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/288956376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:01.025Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:01.026Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:01.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:01.777910+0000) 2023-08-28T00:26:02.858 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:02.491Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:02.858 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:02.492Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:02 smithi136 ceph-mon[113531]: pgmap v6374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:02.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:02.778163+0000) 2023-08-28T00:26:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:02 smithi093 ceph-mon[111335]: pgmap v6374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:02 smithi093 ceph-mon[118902]: pgmap v6374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2324033289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:03.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:03.778452+0000) 2023-08-28T00:26:04.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2324033289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:04.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2324033289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:04 smithi136 ceph-mon[113531]: pgmap v6375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:04.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:04.778750+0000) 2023-08-28T00:26:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:04 smithi093 ceph-mon[111335]: pgmap v6375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:04 smithi093 ceph-mon[118902]: pgmap v6375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:26:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:26:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:26:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/601220614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:05.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:05.779021+0000) 2023-08-28T00:26:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/601220614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/601220614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:06 smithi136 ceph-mon[113531]: pgmap v6376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:06.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:06.779345+0000) 2023-08-28T00:26:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:06 smithi093 ceph-mon[111335]: pgmap v6376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:06 smithi093 ceph-mon[118902]: pgmap v6376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1637048064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:07.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:07.779574+0000) 2023-08-28T00:26:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1637048064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1637048064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:08 smithi136 ceph-mon[113531]: pgmap v6377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:08.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:08.779796+0000) 2023-08-28T00:26:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:08 smithi093 ceph-mon[111335]: pgmap v6377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:08 smithi093 ceph-mon[118902]: pgmap v6377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:10.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:09.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:09.780079+0000) 2023-08-28T00:26:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:10 smithi136 ceph-mon[113531]: pgmap v6378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2084753101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:10.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:10.780370+0000) 2023-08-28T00:26:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:10 smithi093 ceph-mon[111335]: pgmap v6378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2084753101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:10 smithi093 ceph-mon[118902]: pgmap v6378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2084753101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:11.026Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:11.027Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:11.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:11.780613+0000) 2023-08-28T00:26:12.891 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:12.491Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:12.891 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:12.492Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:12 smithi136 ceph-mon[113531]: pgmap v6379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/823581698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:12.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:12.780877+0000) 2023-08-28T00:26:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:12 smithi093 ceph-mon[111335]: pgmap v6379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/823581698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:12 smithi093 ceph-mon[118902]: pgmap v6379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/823581698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:13.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:13.781148+0000) 2023-08-28T00:26:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:14 smithi136 ceph-mon[113531]: pgmap v6380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:14.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:14.781417+0000) 2023-08-28T00:26:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:14 smithi093 ceph-mon[111335]: pgmap v6380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:14 smithi093 ceph-mon[118902]: pgmap v6380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:26:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:26:15] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:26:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2728562238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:15.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:15.781657+0000) 2023-08-28T00:26:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2728562238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2728562238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:16 smithi136 ceph-mon[113531]: pgmap v6381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:16.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:16.781950+0000) 2023-08-28T00:26:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:16 smithi093 ceph-mon[111335]: pgmap v6381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:16 smithi093 ceph-mon[118902]: pgmap v6381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3419702757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:18.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:17.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:17.782200+0000) 2023-08-28T00:26:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3419702757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3419702757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:18 smithi136 ceph-mon[113531]: pgmap v6382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:18.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:18.782433+0000) 2023-08-28T00:26:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:18 smithi093 ceph-mon[111335]: pgmap v6382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:18 smithi093 ceph-mon[118902]: pgmap v6382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1471770216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:19.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:19.782668+0000) 2023-08-28T00:26:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1471770216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1471770216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:20 smithi136 ceph-mon[113531]: pgmap v6383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:20.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:20.782905+0000) 2023-08-28T00:26:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:20 smithi093 ceph-mon[111335]: pgmap v6383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:21.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:20 smithi093 ceph-mon[118902]: pgmap v6383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:21.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:21.027Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:21.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:21.028Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:21.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:21.783163+0000) 2023-08-28T00:26:22.924 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:22.492Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:22.924 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:22.493Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:22 smithi136 ceph-mon[113531]: pgmap v6384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1800149488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:22.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:22.783452+0000) 2023-08-28T00:26:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:22 smithi093 ceph-mon[111335]: pgmap v6384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1800149488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:22 smithi093 ceph-mon[118902]: pgmap v6384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1800149488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:23.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:23.783725+0000) 2023-08-28T00:26:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:24 smithi136 ceph-mon[113531]: pgmap v6385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3243776598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:24.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:24.783961+0000) 2023-08-28T00:26:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:24 smithi093 ceph-mon[111335]: pgmap v6385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3243776598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:24 smithi093 ceph-mon[118902]: pgmap v6385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3243776598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:26:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:26:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:26:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:25.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:25.784206+0000) 2023-08-28T00:26:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:26 smithi136 ceph-mon[113531]: pgmap v6386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2459764068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:26.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:26.784466+0000) 2023-08-28T00:26:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:26 smithi093 ceph-mon[111335]: pgmap v6386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2459764068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:26 smithi093 ceph-mon[118902]: pgmap v6386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2459764068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:27.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:27.784693+0000) 2023-08-28T00:26:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:28 smithi136 ceph-mon[113531]: pgmap v6387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:28.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:28.784956+0000) 2023-08-28T00:26:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:28 smithi093 ceph-mon[111335]: pgmap v6387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:28 smithi093 ceph-mon[118902]: pgmap v6387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/403772436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:29.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:29.785228+0000) 2023-08-28T00:26:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/403772436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:30.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/403772436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:30 smithi136 ceph-mon[113531]: pgmap v6388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:30.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:30.785464+0000) 2023-08-28T00:26:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:30 smithi093 ceph-mon[111335]: pgmap v6388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:30 smithi093 ceph-mon[118902]: pgmap v6388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:31.028Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:31.029Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1601700519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:31.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:31.785734+0000) 2023-08-28T00:26:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1601700519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1601700519' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:32.948 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:32.492Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:32.948 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:32.493Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:32 smithi136 ceph-mon[113531]: pgmap v6389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:32.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:32.786002+0000) 2023-08-28T00:26:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:32 smithi093 ceph-mon[111335]: pgmap v6389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:32 smithi093 ceph-mon[118902]: pgmap v6389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:33.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:33.786290+0000) 2023-08-28T00:26:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:34 smithi136 ceph-mon[113531]: pgmap v6390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3996146345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:34.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:34.786497+0000) 2023-08-28T00:26:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:34 smithi093 ceph-mon[118902]: pgmap v6390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3996146345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:34 smithi093 ceph-mon[111335]: pgmap v6390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3996146345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:26:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:26:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:26:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:35.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:35.786743+0000) 2023-08-28T00:26:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:36 smithi136 ceph-mon[113531]: pgmap v6391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1261224288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:36.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:36.786941+0000) 2023-08-28T00:26:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:36 smithi093 ceph-mon[111335]: pgmap v6391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1261224288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:36 smithi093 ceph-mon[118902]: pgmap v6391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1261224288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:37.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:37.787142+0000) 2023-08-28T00:26:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:38.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:38.787389+0000) 2023-08-28T00:26:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:38 smithi136 ceph-mon[113531]: pgmap v6392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4033097397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:38 smithi093 ceph-mon[111335]: pgmap v6392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4033097397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:38 smithi093 ceph-mon[118902]: pgmap v6392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4033097397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:39.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:39.787655+0000) 2023-08-28T00:26:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:40 smithi136 ceph-mon[113531]: pgmap v6393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:40.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:40.787892+0000) 2023-08-28T00:26:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:40 smithi093 ceph-mon[111335]: pgmap v6393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:40 smithi093 ceph-mon[118902]: pgmap v6393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:41.029Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:41.030Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3109234124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:41.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:41.788157+0000) 2023-08-28T00:26:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3109234124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3109234124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:42.969 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:42.493Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:42.969 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:42.494Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:42 smithi136 ceph-mon[113531]: pgmap v6394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:42.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:42.788420+0000) 2023-08-28T00:26:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:42 smithi093 ceph-mon[111335]: pgmap v6394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:42 smithi093 ceph-mon[118902]: pgmap v6394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/833392576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:43.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:43.788689+0000) 2023-08-28T00:26:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/833392576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/833392576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:44 smithi136 ceph-mon[113531]: pgmap v6395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:26:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:44.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:44.788941+0000) 2023-08-28T00:26:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:44 smithi093 ceph-mon[111335]: pgmap v6395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:26:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:44 smithi093 ceph-mon[118902]: pgmap v6395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:26:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:26:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:26:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:26:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:45.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:45.789136+0000) 2023-08-28T00:26:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:46 smithi136 ceph-mon[113531]: pgmap v6396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:46.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3910102374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:46 smithi093 ceph-mon[111335]: pgmap v6396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:46.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3910102374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:46 smithi093 ceph-mon[118902]: pgmap v6396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:46.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3910102374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:47.186 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:46.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:46.789299+0000) 2023-08-28T00:26:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:47.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:47.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:47.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:47.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:47.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:47.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:47.789511+0000) 2023-08-28T00:26:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[111335]: pgmap v6397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:26:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:26:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:48.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1594719589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:48.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[118902]: pgmap v6397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:48.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:26:48.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:26:48.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:48.992 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1594719589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:48 smithi136 ceph-mon[113531]: pgmap v6397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:26:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:26:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:26:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1594719589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:49.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:48.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:48.789744+0000) 2023-08-28T00:26:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:49.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:49.789960+0000) 2023-08-28T00:26:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:50 smithi093 ceph-mon[111335]: pgmap v6398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3951784012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:50.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:50 smithi093 ceph-mon[118902]: pgmap v6398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:50.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3951784012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:50 smithi136 ceph-mon[113531]: pgmap v6398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3951784012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:50.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:50.790193+0000) 2023-08-28T00:26:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:51.029Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:51.030Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:51.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:51.790480+0000) 2023-08-28T00:26:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:52 smithi093 ceph-mon[111335]: pgmap v6399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:52 smithi093 ceph-mon[118902]: pgmap v6399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:52.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:52.494Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:52.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:26:52.495Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:26:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:52 smithi136 ceph-mon[113531]: pgmap v6399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:52.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:52.790680+0000) 2023-08-28T00:26:53.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/324952180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/324952180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/324952180' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:53.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:53.790900+0000) 2023-08-28T00:26:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:54 smithi093 ceph-mon[118902]: pgmap v6400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:54.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:54 smithi093 ceph-mon[111335]: pgmap v6400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:54.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:54 smithi136 ceph-mon[113531]: pgmap v6400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:26:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:54.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:54.791156+0000) 2023-08-28T00:26:55.735 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:26:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:26:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:26:55.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/906418559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:55.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/906418559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/906418559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:55.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:55.791419+0000) 2023-08-28T00:26:56.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:56 smithi093 ceph-mon[111335]: pgmap v6401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:56.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:56 smithi093 ceph-mon[118902]: pgmap v6401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:56 smithi136 ceph-mon[113531]: pgmap v6401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:56.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:56.791654+0000) 2023-08-28T00:26:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:57.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:57.791901+0000) 2023-08-28T00:26:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:58 smithi093 ceph-mon[111335]: pgmap v6402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:58.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:26:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/225651450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:58 smithi093 ceph-mon[118902]: pgmap v6402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:58.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:26:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/225651450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:26:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:58.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:58.792152+0000) 2023-08-28T00:26:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:58 smithi136 ceph-mon[113531]: pgmap v6402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:26:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:26:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/225651450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:26:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:26:59.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:26:59.792365+0000) 2023-08-28T00:27:01.032 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:00 smithi093 ceph-mon[111335]: pgmap v6403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:01.032 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2240924831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:01.033 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:00 smithi093 ceph-mon[118902]: pgmap v6403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:01.033 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2240924831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:00.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:00.792643+0000) 2023-08-28T00:27:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:00 smithi136 ceph-mon[113531]: pgmap v6403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2240924831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:01.030Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:01.031Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:01.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:01.792868+0000) 2023-08-28T00:27:02.748 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:02.494Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:02.748 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:02.495Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:02 smithi136 ceph-mon[113531]: pgmap v6404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3527592643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:02.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:02.793168+0000) 2023-08-28T00:27:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:02 smithi093 ceph-mon[111335]: pgmap v6404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3527592643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:02 smithi093 ceph-mon[118902]: pgmap v6404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3527592643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:03.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:03.793375+0000) 2023-08-28T00:27:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:04 smithi136 ceph-mon[113531]: pgmap v6405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:04.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:04.793608+0000) 2023-08-28T00:27:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:04 smithi093 ceph-mon[111335]: pgmap v6405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:04 smithi093 ceph-mon[118902]: pgmap v6405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:27:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:27:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:27:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/13643529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:05.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:05.793813+0000) 2023-08-28T00:27:06.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/13643529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:06.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/13643529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:06 smithi136 ceph-mon[113531]: pgmap v6406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:06.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:06.794093+0000) 2023-08-28T00:27:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:06 smithi093 ceph-mon[111335]: pgmap v6406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:06 smithi093 ceph-mon[118902]: pgmap v6406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/261458223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:07.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:07.794346+0000) 2023-08-28T00:27:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/261458223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/261458223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:08 smithi136 ceph-mon[113531]: pgmap v6407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:08.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:08.794603+0000) 2023-08-28T00:27:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:08 smithi093 ceph-mon[111335]: pgmap v6407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:08 smithi093 ceph-mon[118902]: pgmap v6407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3560689656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:09.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:09.794872+0000) 2023-08-28T00:27:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3560689656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:10.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:10.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3560689656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:11.032 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:10 smithi093 ceph-mon[111335]: pgmap v6408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:11.032 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:10 smithi093 ceph-mon[118902]: pgmap v6408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:10 smithi136 ceph-mon[113531]: pgmap v6408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:10.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:10.795156+0000) 2023-08-28T00:27:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:11.031Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:11.032Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:11.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:11.795372+0000) 2023-08-28T00:27:12.781 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:12.495Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:12.782 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:12.496Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:12 smithi136 ceph-mon[113531]: pgmap v6409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2209638334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:12.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:12.795638+0000) 2023-08-28T00:27:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:12 smithi093 ceph-mon[111335]: pgmap v6409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2209638334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:12 smithi093 ceph-mon[118902]: pgmap v6409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2209638334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:13.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:13.795909+0000) 2023-08-28T00:27:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:14 smithi136 ceph-mon[113531]: pgmap v6410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3689093820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:14.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:14.796154+0000) 2023-08-28T00:27:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:14 smithi093 ceph-mon[111335]: pgmap v6410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3689093820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:14 smithi093 ceph-mon[118902]: pgmap v6410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:15.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3689093820' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:27:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:27:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:27:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:15.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:15.796403+0000) 2023-08-28T00:27:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:16 smithi136 ceph-mon[113531]: pgmap v6411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:16.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:16.796656+0000) 2023-08-28T00:27:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:16 smithi093 ceph-mon[111335]: pgmap v6411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:16 smithi093 ceph-mon[118902]: pgmap v6411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2681519019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:17.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:17.796895+0000) 2023-08-28T00:27:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2681519019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2681519019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:18 smithi136 ceph-mon[113531]: pgmap v6412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:18.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:18.797092+0000) 2023-08-28T00:27:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:18 smithi093 ceph-mon[111335]: pgmap v6412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:18 smithi093 ceph-mon[118902]: pgmap v6412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2282025807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:19.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:19.797272+0000) 2023-08-28T00:27:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2282025807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2282025807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:20 smithi136 ceph-mon[113531]: pgmap v6413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:20.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:20.797534+0000) 2023-08-28T00:27:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:20 smithi093 ceph-mon[111335]: pgmap v6413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:20 smithi093 ceph-mon[118902]: pgmap v6413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:21.031Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:21.032Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2881479743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:21.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:21.797794+0000) 2023-08-28T00:27:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2881479743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2881479743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:22.806 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:22.495Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:22.806 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:22.496Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:22 smithi136 ceph-mon[113531]: pgmap v6414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:22.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:22.798029+0000) 2023-08-28T00:27:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:22 smithi093 ceph-mon[111335]: pgmap v6414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:22 smithi093 ceph-mon[118902]: pgmap v6414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:23.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:23.798266+0000) 2023-08-28T00:27:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:24 smithi136 ceph-mon[113531]: pgmap v6415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/913318885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:24.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:24.798561+0000) 2023-08-28T00:27:25.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:24 smithi093 ceph-mon[111335]: pgmap v6415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:25.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:25.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/913318885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:24 smithi093 ceph-mon[118902]: pgmap v6415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/913318885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:27:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:27:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:27:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:25.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:25.798764+0000) 2023-08-28T00:27:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:26 smithi136 ceph-mon[113531]: pgmap v6416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1249895376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:26.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:26.799087+0000) 2023-08-28T00:27:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:26 smithi093 ceph-mon[111335]: pgmap v6416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1249895376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:26 smithi093 ceph-mon[118902]: pgmap v6416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1249895376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:27.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:27.799332+0000) 2023-08-28T00:27:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:28 smithi136 ceph-mon[113531]: pgmap v6417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:29.493 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3373128860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:29.493 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:28.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:28.799594+0000) 2023-08-28T00:27:29.494 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:28 smithi093 ceph-mon[111335]: pgmap v6417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:29.495 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3373128860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:29.495 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:28 smithi093 ceph-mon[118902]: pgmap v6417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:29.496 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3373128860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:29.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:29.799813+0000) 2023-08-28T00:27:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:30 smithi136 ceph-mon[113531]: pgmap v6418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:30.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:30.800074+0000) 2023-08-28T00:27:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:30 smithi093 ceph-mon[111335]: pgmap v6418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:30 smithi093 ceph-mon[118902]: pgmap v6418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:31.032Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:31.032Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2771389952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:31.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:31.800352+0000) 2023-08-28T00:27:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2771389952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2771389952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:32.824 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:32.496Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:32.824 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:32.496Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:32 smithi136 ceph-mon[113531]: pgmap v6419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:32.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:32.800614+0000) 2023-08-28T00:27:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:32 smithi093 ceph-mon[111335]: pgmap v6419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:32 smithi093 ceph-mon[118902]: pgmap v6419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1854879184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:33.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:33.800885+0000) 2023-08-28T00:27:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1854879184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1854879184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:34 smithi136 ceph-mon[113531]: pgmap v6420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:34.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:34.801114+0000) 2023-08-28T00:27:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:34 smithi093 ceph-mon[111335]: pgmap v6420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:34 smithi093 ceph-mon[118902]: pgmap v6420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:27:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:27:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:27:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:35.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:35.801288+0000) 2023-08-28T00:27:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:36 smithi136 ceph-mon[113531]: pgmap v6421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1075198072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:36.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:36.801540+0000) 2023-08-28T00:27:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:36 smithi093 ceph-mon[118902]: pgmap v6421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1075198072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:36 smithi093 ceph-mon[111335]: pgmap v6421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1075198072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:37.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:37.801764+0000) 2023-08-28T00:27:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:38.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:38.802010+0000) 2023-08-28T00:27:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:38 smithi136 ceph-mon[113531]: pgmap v6422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4292111091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:38 smithi093 ceph-mon[118902]: pgmap v6422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4292111091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:38 smithi093 ceph-mon[111335]: pgmap v6422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4292111091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:39.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:39.802263+0000) 2023-08-28T00:27:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:40 smithi136 ceph-mon[113531]: pgmap v6423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3883281609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:40.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:40.802520+0000) 2023-08-28T00:27:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:40 smithi093 ceph-mon[111335]: pgmap v6423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3883281609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:40 smithi093 ceph-mon[118902]: pgmap v6423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3883281609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:41.033Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:41.034Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:41.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:41.802732+0000) 2023-08-28T00:27:42.849 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:42.496Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:42.849 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:42.497Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:42 smithi136 ceph-mon[113531]: pgmap v6424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:42.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:42.802937+0000) 2023-08-28T00:27:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:42 smithi093 ceph-mon[111335]: pgmap v6424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:42 smithi093 ceph-mon[118902]: pgmap v6424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/616192513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:43.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:43.803209+0000) 2023-08-28T00:27:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/616192513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/616192513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:44 smithi136 ceph-mon[113531]: pgmap v6425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:44.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:44.803458+0000) 2023-08-28T00:27:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:44 smithi093 ceph-mon[111335]: pgmap v6425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:44 smithi093 ceph-mon[118902]: pgmap v6425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:27:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:27:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:27:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1251061778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:45.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:45.803709+0000) 2023-08-28T00:27:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1251061778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1251061778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:46 smithi136 ceph-mon[113531]: pgmap v6426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:46.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:46.803944+0000) 2023-08-28T00:27:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:46 smithi093 ceph-mon[111335]: pgmap v6426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:46 smithi093 ceph-mon[118902]: pgmap v6426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2210687681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:47.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:47.804192+0000) 2023-08-28T00:27:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2210687681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2210687681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:48 smithi136 ceph-mon[113531]: pgmap v6427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:27:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:49.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:48.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:48.804428+0000) 2023-08-28T00:27:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[118902]: pgmap v6427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:27:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[111335]: pgmap v6427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:27:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:50.143 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:49.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:49.804638+0000) 2023-08-28T00:27:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:50.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:50.804840+0000) 2023-08-28T00:27:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: pgmap v6428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2218713351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:51.033Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:51.034Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: pgmap v6428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2218713351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: pgmap v6428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2218713351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:51.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:52.076 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:51.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:51.805111+0000) 2023-08-28T00:27:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:52.497Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:27:52.498Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:27:53.164 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:52.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:52.805270+0000) 2023-08-28T00:27:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-mon[113531]: pgmap v6429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1612439439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:27:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:27:53.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[118902]: pgmap v6429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1612439439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:27:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:27:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[111335]: pgmap v6429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:53.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1612439439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:53.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:53.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:27:53.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:27:53.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:27:54.171 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:53.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:53.805499+0000) 2023-08-28T00:27:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:27:55.173 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:54.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:54.805753+0000) 2023-08-28T00:27:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:55 smithi136 ceph-mon[113531]: pgmap v6430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4152182176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:55 smithi093 ceph-mon[111335]: pgmap v6430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4152182176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:55 smithi093 ceph-mon[118902]: pgmap v6430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:55.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4152182176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:55.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:27:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:27:55] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:27:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:55.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:55.806008+0000) 2023-08-28T00:27:57.175 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:56.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:56.806263+0000) 2023-08-28T00:27:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:57 smithi136 ceph-mon[113531]: pgmap v6431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:57 smithi093 ceph-mon[111335]: pgmap v6431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:57 smithi093 ceph-mon[118902]: pgmap v6431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:58.183 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:57.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:57.806482+0000) 2023-08-28T00:27:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1828632315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1828632315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1828632315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:27:59.182 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:58.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:58.806797+0000) 2023-08-28T00:27:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:27:59 smithi136 ceph-mon[113531]: pgmap v6432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:27:59 smithi093 ceph-mon[111335]: pgmap v6432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:27:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:27:59 smithi093 ceph-mon[118902]: pgmap v6432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:00.186 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:27:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:27:59.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:27:59.807089+0000) 2023-08-28T00:28:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2670481993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2670481993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2670481993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:00.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:00.807345+0000) 2023-08-28T00:28:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:01 smithi093 ceph-mon[111335]: pgmap v6433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:01 smithi093 ceph-mon[118902]: pgmap v6433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:01.034Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:01.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:01.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:01 smithi136 ceph-mon[113531]: pgmap v6433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:01.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:01.807546+0000) 2023-08-28T00:28:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1873234111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:02.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1873234111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1873234111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:02.497Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:02.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:02.498Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:02.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:02.807768+0000) 2023-08-28T00:28:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:03 smithi093 ceph-mon[111335]: pgmap v6434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:03.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:03 smithi093 ceph-mon[118902]: pgmap v6434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:03 smithi136 ceph-mon[113531]: pgmap v6434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:03.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:03.808023+0000) 2023-08-28T00:28:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:04.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:04.808290+0000) 2023-08-28T00:28:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:05 smithi093 ceph-mon[111335]: pgmap v6435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1720692800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:05.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:28:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:28:05] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:28:05.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:05 smithi093 ceph-mon[118902]: pgmap v6435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:05.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1720692800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:05 smithi136 ceph-mon[113531]: pgmap v6435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1720692800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:05.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:05.808492+0000) 2023-08-28T00:28:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:06.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:06.808768+0000) 2023-08-28T00:28:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:07 smithi093 ceph-mon[111335]: pgmap v6436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/507834476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:07 smithi093 ceph-mon[118902]: pgmap v6436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/507834476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:07 smithi136 ceph-mon[113531]: pgmap v6436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/507834476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:07.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:07.809026+0000) 2023-08-28T00:28:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:08.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:08.809327+0000) 2023-08-28T00:28:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:09 smithi093 ceph-mon[111335]: pgmap v6437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:09 smithi093 ceph-mon[118902]: pgmap v6437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:09 smithi136 ceph-mon[113531]: pgmap v6437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:09.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:09.809603+0000) 2023-08-28T00:28:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4264549726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4264549726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:10.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4264549726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:10.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:10.809804+0000) 2023-08-28T00:28:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:11 smithi093 ceph-mon[111335]: pgmap v6438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:11 smithi093 ceph-mon[118902]: pgmap v6438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:11.035Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:11.036Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:11 smithi136 ceph-mon[113531]: pgmap v6438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:11.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:11.810075+0000) 2023-08-28T00:28:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:12 smithi093 ceph-mon[118902]: pgmap v6439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4259072850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:12.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:12 smithi093 ceph-mon[111335]: pgmap v6439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:12.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4259072850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:12 smithi136 ceph-mon[113531]: pgmap v6439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4259072850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:12.498Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:12.499Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:13.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:12.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:12.810310+0000) 2023-08-28T00:28:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:13.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:13.810547+0000) 2023-08-28T00:28:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:14 smithi136 ceph-mon[113531]: pgmap v6440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:14.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1244648906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:14.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:14.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:14.810757+0000) 2023-08-28T00:28:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:14 smithi093 ceph-mon[111335]: pgmap v6440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1244648906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:14 smithi093 ceph-mon[118902]: pgmap v6440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1244648906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:28:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:28:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:28:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:15.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:15.811017+0000) 2023-08-28T00:28:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:16 smithi136 ceph-mon[113531]: pgmap v6441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:16.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3053615839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:16.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:16.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:16.811300+0000) 2023-08-28T00:28:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:16 smithi093 ceph-mon[111335]: pgmap v6441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3053615839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:16 smithi093 ceph-mon[118902]: pgmap v6441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3053615839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:17.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:17.811520+0000) 2023-08-28T00:28:18.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:18 smithi136 ceph-mon[113531]: pgmap v6442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:18.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:18.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:18.811764+0000) 2023-08-28T00:28:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:18 smithi093 ceph-mon[111335]: pgmap v6442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:18 smithi093 ceph-mon[118902]: pgmap v6442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:19.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3583447407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:19.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:19.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:19.812015+0000) 2023-08-28T00:28:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3583447407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:19.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3583447407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:20 smithi093 ceph-mon[111335]: pgmap v6443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:20 smithi093 ceph-mon[118902]: pgmap v6443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:20 smithi136 ceph-mon[113531]: pgmap v6443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:20.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:20.812292+0000) 2023-08-28T00:28:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:21.036Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:21.036Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1779310512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1779310512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1779310512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:21.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:21.812475+0000) 2023-08-28T00:28:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:22 smithi093 ceph-mon[111335]: pgmap v6444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:22 smithi093 ceph-mon[118902]: pgmap v6444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:22.499Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:22.500Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:22 smithi136 ceph-mon[113531]: pgmap v6444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:22.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:22.812707+0000) 2023-08-28T00:28:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2219380859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2219380859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2219380859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:23.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:23.812955+0000) 2023-08-28T00:28:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:24 smithi093 ceph-mon[111335]: pgmap v6445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:24 smithi093 ceph-mon[118902]: pgmap v6445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:24 smithi136 ceph-mon[113531]: pgmap v6445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:24.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:24.813195+0000) 2023-08-28T00:28:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:28:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:28:25] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:28:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:25.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:25.813428+0000) 2023-08-28T00:28:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:26 smithi093 ceph-mon[111335]: pgmap v6446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2272529093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:26 smithi093 ceph-mon[118902]: pgmap v6446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:26.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2272529093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:26 smithi136 ceph-mon[113531]: pgmap v6446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2272529093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:26.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:26.813614+0000) 2023-08-28T00:28:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:27.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:27.813863+0000) 2023-08-28T00:28:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:28 smithi093 ceph-mon[111335]: pgmap v6447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:28.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2356144738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:28 smithi093 ceph-mon[118902]: pgmap v6447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:28.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2356144738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:28 smithi136 ceph-mon[113531]: pgmap v6447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2356144738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:28.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:28.814123+0000) 2023-08-28T00:28:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:29.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:29.814364+0000) 2023-08-28T00:28:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:30 smithi093 ceph-mon[111335]: pgmap v6448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:30 smithi093 ceph-mon[118902]: pgmap v6448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:30 smithi136 ceph-mon[113531]: pgmap v6448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:30.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:30.814622+0000) 2023-08-28T00:28:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:31.036Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:31.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:31.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2909221863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:31.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2909221863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2909221863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:31.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:31.814903+0000) 2023-08-28T00:28:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:32 smithi093 ceph-mon[111335]: pgmap v6449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:32 smithi093 ceph-mon[118902]: pgmap v6449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:32.500Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:32.500Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:32 smithi136 ceph-mon[113531]: pgmap v6449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:32.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:32.815175+0000) 2023-08-28T00:28:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2805676662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2805676662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2805676662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:33.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:33.815377+0000) 2023-08-28T00:28:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:34 smithi093 ceph-mon[111335]: pgmap v6450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:34 smithi093 ceph-mon[118902]: pgmap v6450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:34 smithi136 ceph-mon[113531]: pgmap v6450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:34.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:34.815572+0000) 2023-08-28T00:28:35.729 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:28:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:28:35] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:28:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/483100881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/483100881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/483100881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:35.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:35.815786+0000) 2023-08-28T00:28:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:36 smithi093 ceph-mon[111335]: pgmap v6451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:36 smithi093 ceph-mon[118902]: pgmap v6451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:36 smithi136 ceph-mon[113531]: pgmap v6451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:36.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:36.816063+0000) 2023-08-28T00:28:38.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:37.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:37.816321+0000) 2023-08-28T00:28:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:38 smithi093 ceph-mon[111335]: pgmap v6452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2476352423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:38 smithi093 ceph-mon[118902]: pgmap v6452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:38.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2476352423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:38 smithi136 ceph-mon[113531]: pgmap v6452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2476352423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:38.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:38.816568+0000) 2023-08-28T00:28:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:39.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:39.816818+0000) 2023-08-28T00:28:40.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:40.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:41.038 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:40 smithi093 ceph-mon[111335]: pgmap v6453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:41.038 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/780775716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:41.038 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:40 smithi093 ceph-mon[118902]: pgmap v6453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:41.038 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/780775716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:40 smithi136 ceph-mon[113531]: pgmap v6453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/780775716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:40.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:40.817135+0000) 2023-08-28T00:28:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:41.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:41.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:41.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:41.817339+0000) 2023-08-28T00:28:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:42 smithi093 ceph-mon[111335]: pgmap v6454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/289431365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:42 smithi093 ceph-mon[118902]: pgmap v6454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/289431365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:42.501Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:42.501Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:42 smithi136 ceph-mon[113531]: pgmap v6454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/289431365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:42.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:42.817555+0000) 2023-08-28T00:28:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:43.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:43.817760+0000) 2023-08-28T00:28:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:44 smithi136 ceph-mon[113531]: pgmap v6455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:44.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:44.817980+0000) 2023-08-28T00:28:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:44 smithi093 ceph-mon[111335]: pgmap v6455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:44 smithi093 ceph-mon[118902]: pgmap v6455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:28:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:28:45] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:28:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2691605220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:45.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:45.818166+0000) 2023-08-28T00:28:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2691605220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2691605220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:46 smithi136 ceph-mon[113531]: pgmap v6456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:47.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:46.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:46.818414+0000) 2023-08-28T00:28:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:46 smithi093 ceph-mon[111335]: pgmap v6456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:46 smithi093 ceph-mon[118902]: pgmap v6456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/631394127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:47.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:47.818631+0000) 2023-08-28T00:28:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/631394127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:48.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/631394127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:48 smithi136 ceph-mon[113531]: pgmap v6457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:48.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:48.818856+0000) 2023-08-28T00:28:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:48 smithi093 ceph-mon[111335]: pgmap v6457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:48 smithi093 ceph-mon[118902]: pgmap v6457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:49.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:49.819090+0000) 2023-08-28T00:28:51.038 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:50 smithi093 ceph-mon[111335]: pgmap v6458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:51.038 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2015973068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:51.038 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:50 smithi093 ceph-mon[118902]: pgmap v6458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:51.039 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2015973068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:50 smithi136 ceph-mon[113531]: pgmap v6458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2015973068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:50.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:50.819350+0000) 2023-08-28T00:28:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:51.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:51.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:52.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:51.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:51.819607+0000) 2023-08-28T00:28:52.773 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:52.502Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:52.774 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:28:52.502Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:28:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:52 smithi136 ceph-mon[113531]: pgmap v6459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/765992668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:52.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:52.819867+0000) 2023-08-28T00:28:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:52 smithi093 ceph-mon[111335]: pgmap v6459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/765992668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:52 smithi093 ceph-mon[118902]: pgmap v6459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/765992668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:54.137 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:28:54.137 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:54.137 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:54.137 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:53.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:53.820050+0000) 2023-08-28T00:28:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:28:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:28:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:54.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:54 smithi136 ceph-mon[113531]: pgmap v6460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:54.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3028575002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:54.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:54.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:54.820204+0000) 2023-08-28T00:28:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:54 smithi093 ceph-mon[111335]: pgmap v6460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3028575002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:54 smithi093 ceph-mon[118902]: pgmap v6460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:28:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3028575002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:28:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:28:55] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:28:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:55.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:55.820421+0000) 2023-08-28T00:28:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:56.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:56.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:56.820586+0000) 2023-08-28T00:28:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:57 smithi136 ceph-mon[113531]: pgmap v6461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:28:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:28:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4091941220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[118902]: pgmap v6461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:28:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:28:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4091941220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:57.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[111335]: pgmap v6461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:57.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:28:57.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:28:57.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:28:57.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4091941220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:28:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:57.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:57.820804+0000) 2023-08-28T00:28:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:28:59 smithi136 ceph-mon[113531]: pgmap v6462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:58.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:58.821105+0000) 2023-08-28T00:28:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:28:59 smithi093 ceph-mon[111335]: pgmap v6462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:28:59.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:28:59 smithi093 ceph-mon[118902]: pgmap v6462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1793336956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:28:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:28:59.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:28:59.821292+0000) 2023-08-28T00:29:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1793336956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1793336956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:01 smithi136 ceph-mon[113531]: pgmap v6463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:00.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:00.821547+0000) 2023-08-28T00:29:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:01 smithi093 ceph-mon[111335]: pgmap v6463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:01 smithi093 ceph-mon[118902]: pgmap v6463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:01.038Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:01.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2905275499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:01.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:01.821811+0000) 2023-08-28T00:29:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2905275499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:02.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2905275499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:02.502Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:02.503Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:03 smithi136 ceph-mon[113531]: pgmap v6464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:02.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:02.822029+0000) 2023-08-28T00:29:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:03 smithi093 ceph-mon[111335]: pgmap v6464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:03 smithi093 ceph-mon[118902]: pgmap v6464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:03.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:03.822283+0000) 2023-08-28T00:29:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:05 smithi136 ceph-mon[113531]: pgmap v6465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/104384831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:04.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:04.822530+0000) 2023-08-28T00:29:05.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:05 smithi093 ceph-mon[111335]: pgmap v6465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:05.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/104384831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:05 smithi093 ceph-mon[118902]: pgmap v6465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/104384831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:29:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:29:05] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:29:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:05.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:05.822725+0000) 2023-08-28T00:29:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:06.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:06.822960+0000) 2023-08-28T00:29:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:07 smithi136 ceph-mon[113531]: pgmap v6466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2043666342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:07 smithi093 ceph-mon[111335]: pgmap v6466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2043666342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:07 smithi093 ceph-mon[118902]: pgmap v6466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2043666342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:07.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:07.823186+0000) 2023-08-28T00:29:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:09 smithi136 ceph-mon[113531]: pgmap v6467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1846322948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:08.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:08.823373+0000) 2023-08-28T00:29:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:09 smithi093 ceph-mon[118902]: pgmap v6467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1846322948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:09 smithi093 ceph-mon[111335]: pgmap v6467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1846322948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:09.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:09.823630+0000) 2023-08-28T00:29:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:11 smithi136 ceph-mon[113531]: pgmap v6468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:10.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:10.823886+0000) 2023-08-28T00:29:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:11 smithi093 ceph-mon[111335]: pgmap v6468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:11 smithi093 ceph-mon[118902]: pgmap v6468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:11.038Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:11.039Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1774893983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:11.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:11.824150+0000) 2023-08-28T00:29:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1774893983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1774893983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:12.503Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:12.504Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:13 smithi136 ceph-mon[113531]: pgmap v6469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:12.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:12.824452+0000) 2023-08-28T00:29:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:13 smithi093 ceph-mon[111335]: pgmap v6469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:13 smithi093 ceph-mon[118902]: pgmap v6469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3465748503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:13.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:13.824698+0000) 2023-08-28T00:29:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3465748503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3465748503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:15 smithi136 ceph-mon[113531]: pgmap v6470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:14.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:14.824970+0000) 2023-08-28T00:29:15.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:15 smithi093 ceph-mon[111335]: pgmap v6470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:15 smithi093 ceph-mon[118902]: pgmap v6470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:29:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:29:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:29:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:15.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:15.825220+0000) 2023-08-28T00:29:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:16 smithi093 ceph-mon[111335]: pgmap v6471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2123452172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:16 smithi093 ceph-mon[118902]: pgmap v6471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2123452172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:16 smithi136 ceph-mon[113531]: pgmap v6471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2123452172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:16.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:16.825477+0000) 2023-08-28T00:29:18.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:17.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:17.825726+0000) 2023-08-28T00:29:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:18 smithi093 ceph-mon[111335]: pgmap v6472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2225076319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:18 smithi093 ceph-mon[118902]: pgmap v6472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:18.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2225076319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:18 smithi136 ceph-mon[113531]: pgmap v6472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2225076319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:18.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:18.826000+0000) 2023-08-28T00:29:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:19.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:19.826282+0000) 2023-08-28T00:29:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:20 smithi093 ceph-mon[111335]: pgmap v6473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:20.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:20 smithi093 ceph-mon[118902]: pgmap v6473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:20 smithi136 ceph-mon[113531]: pgmap v6473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:20.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:20.826537+0000) 2023-08-28T00:29:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:21.039Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:21.040Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:21.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3739540270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:21.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3739540270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3739540270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:21.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:21.826811+0000) 2023-08-28T00:29:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:22 smithi093 ceph-mon[111335]: pgmap v6474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:22 smithi093 ceph-mon[118902]: pgmap v6474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:22.503Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:22.504Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:22 smithi136 ceph-mon[113531]: pgmap v6474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:22.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:22.827105+0000) 2023-08-28T00:29:23.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3379592640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:23.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3379592640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3379592640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:23.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:23.827304+0000) 2023-08-28T00:29:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:24 smithi093 ceph-mon[111335]: pgmap v6475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:24.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:24 smithi093 ceph-mon[118902]: pgmap v6475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:24.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:24 smithi136 ceph-mon[113531]: pgmap v6475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:25.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:24.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:24.827530+0000) 2023-08-28T00:29:25.717 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:29:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:29:25] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:29:25.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1417650825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1417650825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1417650825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:25.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:25.827733+0000) 2023-08-28T00:29:26.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:26 smithi093 ceph-mon[111335]: pgmap v6476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:26.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:26 smithi093 ceph-mon[118902]: pgmap v6476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:26 smithi136 ceph-mon[113531]: pgmap v6476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:26.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:26.828061+0000) 2023-08-28T00:29:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:27.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:27.828204+0000) 2023-08-28T00:29:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:28 smithi093 ceph-mon[118902]: pgmap v6477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:28.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3653501840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:28.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:28 smithi093 ceph-mon[111335]: pgmap v6477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:28.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3653501840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:28 smithi136 ceph-mon[113531]: pgmap v6477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3653501840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:28.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:28.828453+0000) 2023-08-28T00:29:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:29.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:29.828706+0000) 2023-08-28T00:29:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:30 smithi093 ceph-mon[111335]: pgmap v6478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:30.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1543128927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:30.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:30 smithi093 ceph-mon[118902]: pgmap v6478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:30.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1543128927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:30 smithi136 ceph-mon[113531]: pgmap v6478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1543128927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:30.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:30.828967+0000) 2023-08-28T00:29:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:31.040Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:31.041Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:31.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:31.829212+0000) 2023-08-28T00:29:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:32 smithi093 ceph-mon[111335]: pgmap v6479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3184647937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:32 smithi093 ceph-mon[118902]: pgmap v6479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3184647937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:32.504Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:32.505Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:32 smithi136 ceph-mon[113531]: pgmap v6479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3184647937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:32.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:32.829441+0000) 2023-08-28T00:29:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:33.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:33.829695+0000) 2023-08-28T00:29:34.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:34 smithi093 ceph-mon[111335]: pgmap v6480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:34.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:34 smithi093 ceph-mon[118902]: pgmap v6480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:34 smithi136 ceph-mon[113531]: pgmap v6480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:34.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:34.829898+0000) 2023-08-28T00:29:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:29:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:29:35] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:29:35.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2333465057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:35.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2333465057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2333465057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:35.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:35.830128+0000) 2023-08-28T00:29:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:36 smithi136 ceph-mon[113531]: pgmap v6481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:36.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:36.830361+0000) 2023-08-28T00:29:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:36 smithi093 ceph-mon[111335]: pgmap v6481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:36 smithi093 ceph-mon[118902]: pgmap v6481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3232572314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:37.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:37.830611+0000) 2023-08-28T00:29:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3232572314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3232572314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:38 smithi136 ceph-mon[113531]: pgmap v6482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:38.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:38.830814+0000) 2023-08-28T00:29:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:38 smithi093 ceph-mon[111335]: pgmap v6482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:38 smithi093 ceph-mon[118902]: pgmap v6482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:39.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:39.831095+0000) 2023-08-28T00:29:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:41.042 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:40 smithi093 ceph-mon[111335]: pgmap v6483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:41.042 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/256110842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:41.043 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:40 smithi093 ceph-mon[118902]: pgmap v6483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:41.043 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/256110842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:40 smithi136 ceph-mon[113531]: pgmap v6483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/256110842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:40.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:40.831396+0000) 2023-08-28T00:29:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:41.041Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:41.042Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:41.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:41.831603+0000) 2023-08-28T00:29:42.764 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:42.504Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:42.764 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:42.505Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:42 smithi136 ceph-mon[113531]: pgmap v6484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1029978018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:42.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:42.831849+0000) 2023-08-28T00:29:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:42 smithi093 ceph-mon[111335]: pgmap v6484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1029978018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:42 smithi093 ceph-mon[118902]: pgmap v6484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1029978018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:43.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:43.832121+0000) 2023-08-28T00:29:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:44 smithi136 ceph-mon[113531]: pgmap v6485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2669041555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:44.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:44.832333+0000) 2023-08-28T00:29:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:44 smithi093 ceph-mon[118902]: pgmap v6485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2669041555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:44 smithi093 ceph-mon[111335]: pgmap v6485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:45.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2669041555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:29:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:29:45] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:29:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:45.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:45.832514+0000) 2023-08-28T00:29:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:46 smithi136 ceph-mon[113531]: pgmap v6486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:46.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:46.832772+0000) 2023-08-28T00:29:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:46 smithi093 ceph-mon[111335]: pgmap v6486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:46 smithi093 ceph-mon[118902]: pgmap v6486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4167655767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:47.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:47.832989+0000) 2023-08-28T00:29:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4167655767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4167655767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:48 smithi136 ceph-mon[113531]: pgmap v6487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:49.253 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:48.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:48.833212+0000) 2023-08-28T00:29:49.254 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:48 smithi093 ceph-mon[111335]: pgmap v6487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:49.254 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:48 smithi093 ceph-mon[118902]: pgmap v6487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3794105522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:49.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:49.833512+0000) 2023-08-28T00:29:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3794105522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3794105522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:51.043 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:50 smithi093 ceph-mon[111335]: pgmap v6488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:51.043 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:50 smithi093 ceph-mon[118902]: pgmap v6488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:50 smithi136 ceph-mon[113531]: pgmap v6488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:50.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:50.833738+0000) 2023-08-28T00:29:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:51.042Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:51.043Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1732943576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:51.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:51.833978+0000) 2023-08-28T00:29:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1732943576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1732943576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:52.788 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:52.505Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:52.789 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:29:52.506Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:29:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:52 smithi136 ceph-mon[113531]: pgmap v6489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:52.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:52.834225+0000) 2023-08-28T00:29:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:52 smithi093 ceph-mon[111335]: pgmap v6489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:52 smithi093 ceph-mon[118902]: pgmap v6489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:53.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:53.834441+0000) 2023-08-28T00:29:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:54.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:54.834687+0000) 2023-08-28T00:29:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:54 smithi136 ceph-mon[113531]: pgmap v6490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:55.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3221065269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:54 smithi093 ceph-mon[111335]: pgmap v6490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3221065269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:54 smithi093 ceph-mon[118902]: pgmap v6490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:29:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3221065269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:29:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:29:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:29:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:55.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:55.834934+0000) 2023-08-28T00:29:57.123 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:56 smithi093 ceph-mon[118902]: pgmap v6491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:57.124 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3458929924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:57.124 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:29:57.124 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:56 smithi093 ceph-mon[111335]: pgmap v6491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:57.124 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3458929924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:57.124 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:29:57.144 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:56 smithi136 ceph-mon[113531]: pgmap v6491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:57.145 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3458929924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:29:57.145 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:29:57.145 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:56.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:56.835103+0000) 2023-08-28T00:29:58.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:57.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:57.835235+0000) 2023-08-28T00:29:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:58 smithi136 ceph-mon[113531]: pgmap v6492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:29:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:29:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:58.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:58.835377+0000) 2023-08-28T00:29:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:58 smithi093 ceph-mon[118902]: pgmap v6492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:29:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:29:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:58 smithi093 ceph-mon[111335]: pgmap v6492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:29:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:29:59.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:29:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:29:59.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:29:59.835546+0000) 2023-08-28T00:30:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3942786322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:30:00.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:30:00.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3942786322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:30:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:30:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3942786322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:30:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:30:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:30:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:00 smithi136 ceph-mon[113531]: pgmap v6493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:30:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:00.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:00.835806+0000) 2023-08-28T00:30:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:00 smithi093 ceph-mon[111335]: pgmap v6493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:30:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:00 smithi093 ceph-mon[118902]: pgmap v6493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:30:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:01.043Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:01.044Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2027154253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:01.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:01.836039+0000) 2023-08-28T00:30:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2027154253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2027154253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:02.810 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:02.506Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:02.810 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:02.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:02 smithi136 ceph-mon[113531]: pgmap v6494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:02.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:02.836266+0000) 2023-08-28T00:30:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:02 smithi093 ceph-mon[111335]: pgmap v6494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:02 smithi093 ceph-mon[118902]: pgmap v6494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2740546158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:03.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:03.836523+0000) 2023-08-28T00:30:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2740546158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2740546158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:04 smithi136 ceph-mon[113531]: pgmap v6495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:04.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:04.836767+0000) 2023-08-28T00:30:05.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:04 smithi093 ceph-mon[111335]: pgmap v6495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:05.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:04 smithi093 ceph-mon[118902]: pgmap v6495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:30:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:30:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:30:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:05.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:05.836984+0000) 2023-08-28T00:30:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:06.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:06.837223+0000) 2023-08-28T00:30:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:06 smithi136 ceph-mon[113531]: pgmap v6496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2035730711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:06 smithi093 ceph-mon[111335]: pgmap v6496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2035730711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:06 smithi093 ceph-mon[118902]: pgmap v6496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2035730711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:07.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:07.837476+0000) 2023-08-28T00:30:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:08.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:08.837669+0000) 2023-08-28T00:30:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:08 smithi136 ceph-mon[113531]: pgmap v6497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1124963347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:08 smithi093 ceph-mon[111335]: pgmap v6497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1124963347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:08 smithi093 ceph-mon[118902]: pgmap v6497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1124963347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:09.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:09.837947+0000) 2023-08-28T00:30:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:10 smithi136 ceph-mon[113531]: pgmap v6498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:10.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:10.838195+0000) 2023-08-28T00:30:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:10 smithi093 ceph-mon[118902]: pgmap v6498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:10 smithi093 ceph-mon[111335]: pgmap v6498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:11.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:11.045Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2397219798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:11.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:11.838410+0000) 2023-08-28T00:30:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2397219798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2397219798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:12.835 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:12.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:12.835 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:12.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:12 smithi136 ceph-mon[113531]: pgmap v6499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:12.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:12.838614+0000) 2023-08-28T00:30:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:12 smithi093 ceph-mon[111335]: pgmap v6499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:12 smithi093 ceph-mon[118902]: pgmap v6499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/259955271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:13.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:13.838794+0000) 2023-08-28T00:30:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/259955271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/259955271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:14 smithi136 ceph-mon[113531]: pgmap v6500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:14.838+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:14.839943+0000) 2023-08-28T00:30:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:14 smithi093 ceph-mon[111335]: pgmap v6500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:14 smithi093 ceph-mon[118902]: pgmap v6500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:30:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:30:15] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:30:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/191743254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:15.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:15.840204+0000) 2023-08-28T00:30:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/191743254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/191743254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:16 smithi136 ceph-mon[113531]: pgmap v6501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:16.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:16.840446+0000) 2023-08-28T00:30:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:16 smithi093 ceph-mon[111335]: pgmap v6501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:16 smithi093 ceph-mon[118902]: pgmap v6501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:17.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:17.840695+0000) 2023-08-28T00:30:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:18 smithi136 ceph-mon[113531]: pgmap v6502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1074139074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:18.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:18.840911+0000) 2023-08-28T00:30:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:18 smithi093 ceph-mon[111335]: pgmap v6502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1074139074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:19.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:18 smithi093 ceph-mon[118902]: pgmap v6502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:19.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1074139074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:20.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:19.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:19.841112+0000) 2023-08-28T00:30:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:20.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:20.841304+0000) 2023-08-28T00:30:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:20 smithi136 ceph-mon[113531]: pgmap v6503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/188355046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:20 smithi093 ceph-mon[111335]: pgmap v6503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/188355046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:20 smithi093 ceph-mon[118902]: pgmap v6503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/188355046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:21.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:21.045Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:21.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:21.841543+0000) 2023-08-28T00:30:22.859 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:22.508Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:22.859 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:22.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:23.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:22.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:22.841782+0000) 2023-08-28T00:30:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:22 smithi136 ceph-mon[113531]: pgmap v6504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2000390803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:22 smithi093 ceph-mon[111335]: pgmap v6504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2000390803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:22 smithi093 ceph-mon[118902]: pgmap v6504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2000390803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:23.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:23.842026+0000) 2023-08-28T00:30:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:24 smithi136 ceph-mon[113531]: pgmap v6505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:24.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:24.842273+0000) 2023-08-28T00:30:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:24 smithi093 ceph-mon[111335]: pgmap v6505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:24 smithi093 ceph-mon[118902]: pgmap v6505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:30:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:30:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:30:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3086903767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:25.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:25.842529+0000) 2023-08-28T00:30:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3086903767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3086903767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:26 smithi136 ceph-mon[113531]: pgmap v6506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:26.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:26.842830+0000) 2023-08-28T00:30:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:26 smithi093 ceph-mon[118902]: pgmap v6506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:26 smithi093 ceph-mon[111335]: pgmap v6506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1057737740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:27.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:27.843067+0000) 2023-08-28T00:30:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1057737740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1057737740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:28 smithi136 ceph-mon[113531]: pgmap v6507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:28.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:28.843297+0000) 2023-08-28T00:30:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:28 smithi093 ceph-mon[118902]: pgmap v6507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:28 smithi093 ceph-mon[111335]: pgmap v6507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/878327684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:29.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:29.843529+0000) 2023-08-28T00:30:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/878327684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/878327684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:30 smithi136 ceph-mon[113531]: pgmap v6508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:30.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:30.843802+0000) 2023-08-28T00:30:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:30 smithi093 ceph-mon[111335]: pgmap v6508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:30 smithi093 ceph-mon[118902]: pgmap v6508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:31.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:31.045Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:31.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:31.844096+0000) 2023-08-28T00:30:32.949 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:32.508Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:32.949 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:32.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:32 smithi136 ceph-mon[113531]: pgmap v6509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2566693076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:32.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:32.844290+0000) 2023-08-28T00:30:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:32 smithi093 ceph-mon[111335]: pgmap v6509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2566693076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:32 smithi093 ceph-mon[118902]: pgmap v6509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2566693076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:33.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:33.844470+0000) 2023-08-28T00:30:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:34 smithi136 ceph-mon[113531]: pgmap v6510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4157251297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:34.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:34.844710+0000) 2023-08-28T00:30:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:34 smithi093 ceph-mon[111335]: pgmap v6510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4157251297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:34 smithi093 ceph-mon[118902]: pgmap v6510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4157251297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:30:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:30:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:30:36.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:35.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:35.844988+0000) 2023-08-28T00:30:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:36 smithi136 ceph-mon[113531]: pgmap v6511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:36.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:36.845255+0000) 2023-08-28T00:30:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:36 smithi093 ceph-mon[111335]: pgmap v6511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:36 smithi093 ceph-mon[118902]: pgmap v6511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1381158958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:37.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:37.845507+0000) 2023-08-28T00:30:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1381158958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1381158958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:38 smithi136 ceph-mon[113531]: pgmap v6512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:38.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:38.845735+0000) 2023-08-28T00:30:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:38 smithi093 ceph-mon[111335]: pgmap v6512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:38 smithi093 ceph-mon[118902]: pgmap v6512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1096502223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:39.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:39.846055+0000) 2023-08-28T00:30:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1096502223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1096502223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:40 smithi136 ceph-mon[113531]: pgmap v6513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:40.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:40.846301+0000) 2023-08-28T00:30:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:40 smithi093 ceph-mon[111335]: pgmap v6513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:40 smithi093 ceph-mon[118902]: pgmap v6513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:41.045Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:41.046Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1794278664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:41.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:41.846552+0000) 2023-08-28T00:30:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1794278664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1794278664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:42.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:42.509Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:42.973 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:42.510Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:42 smithi136 ceph-mon[113531]: pgmap v6514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:42.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:42.846796+0000) 2023-08-28T00:30:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:42 smithi093 ceph-mon[118902]: pgmap v6514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:42 smithi093 ceph-mon[111335]: pgmap v6514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:43.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:43.847073+0000) 2023-08-28T00:30:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:44 smithi136 ceph-mon[113531]: pgmap v6515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3898158099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:44.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:44.847284+0000) 2023-08-28T00:30:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:44 smithi093 ceph-mon[111335]: pgmap v6515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3898158099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:44 smithi093 ceph-mon[118902]: pgmap v6515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3898158099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:30:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:30:45] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:30:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:45.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:45.847540+0000) 2023-08-28T00:30:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:46 smithi136 ceph-mon[113531]: pgmap v6516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2645518052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:46.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:46.847832+0000) 2023-08-28T00:30:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:46 smithi093 ceph-mon[111335]: pgmap v6516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2645518052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:46 smithi093 ceph-mon[118902]: pgmap v6516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2645518052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:47.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:47.848127+0000) 2023-08-28T00:30:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:48 smithi136 ceph-mon[113531]: pgmap v6517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/13319995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:48.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:48.848321+0000) 2023-08-28T00:30:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:48 smithi093 ceph-mon[111335]: pgmap v6517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/13319995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:48 smithi093 ceph-mon[118902]: pgmap v6517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/13319995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:49.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:49.848545+0000) 2023-08-28T00:30:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:50 smithi136 ceph-mon[113531]: pgmap v6518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:50.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:50.848806+0000) 2023-08-28T00:30:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:50 smithi093 ceph-mon[111335]: pgmap v6518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:50 smithi093 ceph-mon[118902]: pgmap v6518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:51.045Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:51.046Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2777510580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:51.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:51.849073+0000) 2023-08-28T00:30:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2777510580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2777510580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:52.510Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:30:52.511Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:30:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:53 smithi136 ceph-mon[113531]: pgmap v6519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:52.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:52.849307+0000) 2023-08-28T00:30:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:52 smithi093 ceph-mon[111335]: pgmap v6519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:53 smithi093 ceph-mon[118902]: pgmap v6519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3249494825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:53.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:53.849594+0000) 2023-08-28T00:30:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3249494825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3249494825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:30:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:55 smithi136 ceph-mon[113531]: pgmap v6520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:54.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:54.849847+0000) 2023-08-28T00:30:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:55 smithi093 ceph-mon[111335]: pgmap v6520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:55 smithi093 ceph-mon[118902]: pgmap v6520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:30:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:30:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:30:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:55.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:55.850072+0000) 2023-08-28T00:30:57.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:57 smithi136 ceph-mon[113531]: pgmap v6521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1105429406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:56.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:56.850322+0000) 2023-08-28T00:30:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:57 smithi093 ceph-mon[111335]: pgmap v6521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1105429406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:57 smithi093 ceph-mon[118902]: pgmap v6521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1105429406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:57.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:57.850589+0000) 2023-08-28T00:30:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:59 smithi136 ceph-mon[113531]: pgmap v6522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:30:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1947917204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:58.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:58.850828+0000) 2023-08-28T00:30:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:59 smithi093 ceph-mon[111335]: pgmap v6522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:30:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1947917204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:30:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:59 smithi093 ceph-mon[118902]: pgmap v6522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:30:59.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:30:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1947917204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:00.160 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:31:00.160 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:30:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:30:59.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:30:59.851013+0000) 2023-08-28T00:31:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:31:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:31:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:01 smithi136 ceph-mon[113531]: pgmap v6523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2224229705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:00.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:00.851130+0000) 2023-08-28T00:31:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:01 smithi093 ceph-mon[111335]: pgmap v6523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2224229705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:01 smithi093 ceph-mon[118902]: pgmap v6523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2224229705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:01.046Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:01.047Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:01.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:01.851300+0000) 2023-08-28T00:31:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:02 smithi136 ceph-mon[113531]: pgmap v6524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:02.510Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:02.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:02.511Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[111335]: pgmap v6524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[118902]: pgmap v6524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:02.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:02.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:03.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:02.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:02.851540+0000) 2023-08-28T00:31:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:31:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:31:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471118159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:31:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:31:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471118159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:03.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:31:03.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:31:03.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:31:03.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471118159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:03.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:03.851812+0000) 2023-08-28T00:31:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:04 smithi136 ceph-mon[113531]: pgmap v6525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:04 smithi093 ceph-mon[111335]: pgmap v6525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:04 smithi093 ceph-mon[118902]: pgmap v6525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:04.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:04.852007+0000) 2023-08-28T00:31:05.690 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 00:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T00:31:05.316291423Z level=info msg="Completed cleanup jobs" duration=11.288701ms 2023-08-28T00:31:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:31:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:31:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:31:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:05.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:05.852204+0000) 2023-08-28T00:31:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:06 smithi136 ceph-mon[113531]: pgmap v6526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:06.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3650604403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:06.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:06.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:06.852452+0000) 2023-08-28T00:31:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:06 smithi093 ceph-mon[111335]: pgmap v6526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:06.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3650604403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:06 smithi093 ceph-mon[118902]: pgmap v6526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:06.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3650604403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:07.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:07.852697+0000) 2023-08-28T00:31:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:08 smithi136 ceph-mon[113531]: pgmap v6527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:08.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/337394058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:08.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:08.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:08.852941+0000) 2023-08-28T00:31:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:08 smithi093 ceph-mon[111335]: pgmap v6527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:08.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/337394058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:08 smithi093 ceph-mon[118902]: pgmap v6527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:08.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/337394058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:09.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:09.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:09.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:09.853151+0000) 2023-08-28T00:31:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:10 smithi136 ceph-mon[113531]: pgmap v6528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:10.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1603339796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:10.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:10.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:10.853404+0000) 2023-08-28T00:31:10.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:10 smithi093 ceph-mon[111335]: pgmap v6528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:10.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1603339796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:10 smithi093 ceph-mon[118902]: pgmap v6528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:10.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1603339796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:11.046Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:11.501 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:11.048Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:12.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:11.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:11.853682+0000) 2023-08-28T00:31:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:12 smithi093 ceph-mon[111335]: pgmap v6529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:12 smithi093 ceph-mon[118902]: pgmap v6529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:12.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:12.511Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:12.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:12.512Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:12 smithi136 ceph-mon[113531]: pgmap v6529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:12.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:12.853938+0000) 2023-08-28T00:31:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4204811818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:13.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4204811818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4204811818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:13.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:13.854172+0000) 2023-08-28T00:31:14.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:14 smithi093 ceph-mon[111335]: pgmap v6530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:14.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:14 smithi093 ceph-mon[118902]: pgmap v6530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:14 smithi136 ceph-mon[113531]: pgmap v6530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:14.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:14.854424+0000) 2023-08-28T00:31:15.723 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:31:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:31:15] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:31:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2311719719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2311719719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2311719719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:15.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:15.854649+0000) 2023-08-28T00:31:16.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:16 smithi093 ceph-mon[111335]: pgmap v6531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:16.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:16 smithi093 ceph-mon[118902]: pgmap v6531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:16 smithi136 ceph-mon[113531]: pgmap v6531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:16.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:16.854920+0000) 2023-08-28T00:31:17.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2699067022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2699067022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2699067022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:17.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:17.855205+0000) 2023-08-28T00:31:18.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:18 smithi093 ceph-mon[111335]: pgmap v6532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:18.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:18 smithi093 ceph-mon[118902]: pgmap v6532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:18 smithi136 ceph-mon[113531]: pgmap v6532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:18.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:18.855450+0000) 2023-08-28T00:31:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:19.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:19.855694+0000) 2023-08-28T00:31:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:20 smithi093 ceph-mon[111335]: pgmap v6533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:20.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3423504963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:20 smithi093 ceph-mon[118902]: pgmap v6533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:20.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3423504963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:20 smithi136 ceph-mon[113531]: pgmap v6533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3423504963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:20.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:20.855939+0000) 2023-08-28T00:31:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:21.047Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:21.048Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:22.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:21.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:21.856188+0000) 2023-08-28T00:31:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:22 smithi093 ceph-mon[111335]: pgmap v6534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3479403201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:22 smithi093 ceph-mon[118902]: pgmap v6534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:22.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3479403201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:22.512Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:22.513Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:22 smithi136 ceph-mon[113531]: pgmap v6534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3479403201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:22.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:22.856438+0000) 2023-08-28T00:31:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:23.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:23.856696+0000) 2023-08-28T00:31:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:24 smithi136 ceph-mon[113531]: pgmap v6535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3199718573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:24.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:24.856952+0000) 2023-08-28T00:31:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:24 smithi093 ceph-mon[111335]: pgmap v6535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:25.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3199718573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:24 smithi093 ceph-mon[118902]: pgmap v6535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3199718573' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:31:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:31:25] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:31:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:25.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:25.857188+0000) 2023-08-28T00:31:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:26 smithi136 ceph-mon[113531]: pgmap v6536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:26.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:26.857448+0000) 2023-08-28T00:31:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:26 smithi093 ceph-mon[118902]: pgmap v6536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:26 smithi093 ceph-mon[111335]: pgmap v6536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2237336679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:27.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:27.857684+0000) 2023-08-28T00:31:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2237336679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2237336679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:28 smithi136 ceph-mon[113531]: pgmap v6537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:28.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:28.857875+0000) 2023-08-28T00:31:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:28 smithi093 ceph-mon[111335]: pgmap v6537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:28 smithi093 ceph-mon[118902]: pgmap v6537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2490176308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:29.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:29.858153+0000) 2023-08-28T00:31:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2490176308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2490176308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:31.049 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:30 smithi093 ceph-mon[111335]: pgmap v6538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:31.050 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:30 smithi093 ceph-mon[118902]: pgmap v6538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:30 smithi136 ceph-mon[113531]: pgmap v6538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:30.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:30.858426+0000) 2023-08-28T00:31:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:31.048Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:31.050Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:31.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:31.858688+0000) 2023-08-28T00:31:32.764 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:32 smithi093 ceph-mon[111335]: pgmap v6539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:32.764 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1827879785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:32.764 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:32.512Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:32.764 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:32.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:32 smithi136 ceph-mon[113531]: pgmap v6539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1827879785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:33.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:32.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:32.858947+0000) 2023-08-28T00:31:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:32 smithi093 ceph-mon[118902]: pgmap v6539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1827879785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:33.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:33.859203+0000) 2023-08-28T00:31:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:34 smithi136 ceph-mon[113531]: pgmap v6540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/162825084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:34.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:34.859467+0000) 2023-08-28T00:31:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:34 smithi093 ceph-mon[111335]: pgmap v6540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/162825084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:34 smithi093 ceph-mon[118902]: pgmap v6540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/162825084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:31:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:31:35] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:31:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:35.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:35.859753+0000) 2023-08-28T00:31:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:36 smithi136 ceph-mon[113531]: pgmap v6541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1685739045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:36.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:36.860016+0000) 2023-08-28T00:31:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:36 smithi093 ceph-mon[118902]: pgmap v6541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1685739045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:36 smithi093 ceph-mon[111335]: pgmap v6541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:37.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1685739045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:37.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:37.860209+0000) 2023-08-28T00:31:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:38 smithi136 ceph-mon[113531]: pgmap v6542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:38.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:38.860447+0000) 2023-08-28T00:31:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:38 smithi093 ceph-mon[111335]: pgmap v6542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:38 smithi093 ceph-mon[118902]: pgmap v6542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/432527582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:39.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:39.860697+0000) 2023-08-28T00:31:40.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:40.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/432527582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:40.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:40.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/432527582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:41.050 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:40 smithi093 ceph-mon[111335]: pgmap v6543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:41.051 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:40 smithi093 ceph-mon[118902]: pgmap v6543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:40 smithi136 ceph-mon[113531]: pgmap v6543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:40.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:40.860966+0000) 2023-08-28T00:31:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:41.049Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:41.050Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/272242126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:41.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:41.861235+0000) 2023-08-28T00:31:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/272242126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/272242126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:42.784 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:42.513Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:42.784 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:42.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:42 smithi136 ceph-mon[113531]: pgmap v6544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:42.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:42.861483+0000) 2023-08-28T00:31:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:42 smithi093 ceph-mon[111335]: pgmap v6544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:42 smithi093 ceph-mon[118902]: pgmap v6544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1860337255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:43.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:43.861716+0000) 2023-08-28T00:31:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1860337255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1860337255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:44 smithi136 ceph-mon[113531]: pgmap v6545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:44.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:44.861977+0000) 2023-08-28T00:31:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:44 smithi093 ceph-mon[111335]: pgmap v6545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:44 smithi093 ceph-mon[118902]: pgmap v6545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:31:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:31:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:31:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:45.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:45.862267+0000) 2023-08-28T00:31:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:46 smithi136 ceph-mon[113531]: pgmap v6546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1510612007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:46.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:46.862509+0000) 2023-08-28T00:31:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:46 smithi093 ceph-mon[111335]: pgmap v6546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1510612007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:46 smithi093 ceph-mon[118902]: pgmap v6546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1510612007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:47.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:47.862713+0000) 2023-08-28T00:31:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:48 smithi136 ceph-mon[113531]: pgmap v6547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3467986472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:48.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:48.862933+0000) 2023-08-28T00:31:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:48 smithi093 ceph-mon[111335]: pgmap v6547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3467986472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:48 smithi093 ceph-mon[118902]: pgmap v6547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3467986472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:49.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:49.863174+0000) 2023-08-28T00:31:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:50 smithi136 ceph-mon[113531]: pgmap v6548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:50.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:50.863423+0000) 2023-08-28T00:31:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:50 smithi093 ceph-mon[111335]: pgmap v6548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:50 smithi093 ceph-mon[118902]: pgmap v6548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:51.050Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:51.051Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3196014844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:51.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:51.863647+0000) 2023-08-28T00:31:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3196014844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3196014844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:52.812 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:52.513Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:52.812 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:31:52.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:31:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:52 smithi136 ceph-mon[113531]: pgmap v6549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:52.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:52.863856+0000) 2023-08-28T00:31:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:52 smithi093 ceph-mon[111335]: pgmap v6549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:52 smithi093 ceph-mon[118902]: pgmap v6549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/380216058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:53.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:53.864115+0000) 2023-08-28T00:31:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/380216058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/380216058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:54 smithi136 ceph-mon[113531]: pgmap v6550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:54.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:54.864343+0000) 2023-08-28T00:31:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:54 smithi093 ceph-mon[111335]: pgmap v6550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:54 smithi093 ceph-mon[118902]: pgmap v6550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:31:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:31:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:31:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:31:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/890370460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:55.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:55.864597+0000) 2023-08-28T00:31:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/890370460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/890370460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:56 smithi136 ceph-mon[113531]: pgmap v6551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:56.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:56.864836+0000) 2023-08-28T00:31:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:56 smithi093 ceph-mon[111335]: pgmap v6551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:56 smithi093 ceph-mon[118902]: pgmap v6551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:57.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:57.865059+0000) 2023-08-28T00:31:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:58 smithi136 ceph-mon[113531]: pgmap v6552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:31:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2197475881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:58.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:58.865300+0000) 2023-08-28T00:31:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:58 smithi093 ceph-mon[111335]: pgmap v6552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:31:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2197475881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:31:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:58 smithi093 ceph-mon[118902]: pgmap v6552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:31:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:31:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2197475881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:31:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:31:59.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:31:59.865584+0000) 2023-08-28T00:32:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:00 smithi136 ceph-mon[113531]: pgmap v6553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1038249898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:00.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:00.865820+0000) 2023-08-28T00:32:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:00 smithi093 ceph-mon[111335]: pgmap v6553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1038249898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:00 smithi093 ceph-mon[118902]: pgmap v6553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1038249898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:01.051Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:01.052Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:02.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:01.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:01.866069+0000) 2023-08-28T00:32:02.840 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:02.514Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:02.840 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:02.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:03.116 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:02 smithi136 ceph-mon[113531]: pgmap v6554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:03.117 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:32:03.117 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:02.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:02.866230+0000) 2023-08-28T00:32:03.190 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:02 smithi093 ceph-mon[111335]: pgmap v6554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:03.190 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:32:03.190 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:02 smithi093 ceph-mon[118902]: pgmap v6554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:03.190 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:32:04.145 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3377694877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:04.146 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3377694877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3377694877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:03.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:03.866422+0000) 2023-08-28T00:32:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:04 smithi136 ceph-mon[113531]: pgmap v6555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:04.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:04.866607+0000) 2023-08-28T00:32:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:04 smithi093 ceph-mon[111335]: pgmap v6555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:05.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:05.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:05.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:04 smithi093 ceph-mon[118902]: pgmap v6555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:05.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:05.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:32:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:32:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:05.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:05.866775+0000) 2023-08-28T00:32:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3070262077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:32:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:32:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3070262077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3070262077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:06.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:32:06.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:32:06.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:32:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:06 smithi136 ceph-mon[113531]: pgmap v6556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:06.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:06.867061+0000) 2023-08-28T00:32:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:06 smithi093 ceph-mon[111335]: pgmap v6556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:06 smithi093 ceph-mon[118902]: pgmap v6556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/704092211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:07.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:07.867264+0000) 2023-08-28T00:32:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/704092211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:08.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/704092211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:08 smithi136 ceph-mon[113531]: pgmap v6557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:08.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:08.867524+0000) 2023-08-28T00:32:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:08 smithi093 ceph-mon[111335]: pgmap v6557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:09.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:08 smithi093 ceph-mon[118902]: pgmap v6557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:09.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:09.867777+0000) 2023-08-28T00:32:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:10 smithi136 ceph-mon[113531]: pgmap v6558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:11.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1607157679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:11.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:10.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:10.868055+0000) 2023-08-28T00:32:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:10 smithi093 ceph-mon[111335]: pgmap v6558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1607157679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:10 smithi093 ceph-mon[118902]: pgmap v6558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1607157679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:11.051Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:11.052Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:11.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:11.868328+0000) 2023-08-28T00:32:12.928 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:12.514Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:12.953 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:12.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:12 smithi136 ceph-mon[113531]: pgmap v6559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1850518127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:12.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:12.868602+0000) 2023-08-28T00:32:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:12 smithi093 ceph-mon[111335]: pgmap v6559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1850518127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:12 smithi093 ceph-mon[118902]: pgmap v6559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1850518127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:13.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:13.868859+0000) 2023-08-28T00:32:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:14 smithi136 ceph-mon[113531]: pgmap v6560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/608225827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:14.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:14.869110+0000) 2023-08-28T00:32:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:14 smithi093 ceph-mon[111335]: pgmap v6560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/608225827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:14 smithi093 ceph-mon[118902]: pgmap v6560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:15.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/608225827' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:32:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:32:15] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:32:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:15.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:15.869368+0000) 2023-08-28T00:32:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:16 smithi136 ceph-mon[113531]: pgmap v6561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:16.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:16.869595+0000) 2023-08-28T00:32:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:16 smithi093 ceph-mon[111335]: pgmap v6561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:16 smithi093 ceph-mon[118902]: pgmap v6561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340974653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:17.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:17.869875+0000) 2023-08-28T00:32:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340974653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340974653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:18 smithi136 ceph-mon[113531]: pgmap v6562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:18.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:18.870152+0000) 2023-08-28T00:32:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:18 smithi093 ceph-mon[111335]: pgmap v6562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:19.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:18 smithi093 ceph-mon[118902]: pgmap v6562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/941603862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:19.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:19.870413+0000) 2023-08-28T00:32:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/941603862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/941603862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:20 smithi136 ceph-mon[113531]: pgmap v6563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:20.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:20.870663+0000) 2023-08-28T00:32:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:20 smithi093 ceph-mon[111335]: pgmap v6563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:20 smithi093 ceph-mon[118902]: pgmap v6563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:21.052Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:21.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:21.053Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/39865453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:21.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:21.870899+0000) 2023-08-28T00:32:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/39865453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/39865453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:22.957 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:22.515Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:22.957 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:22.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:22 smithi136 ceph-mon[113531]: pgmap v6564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:22.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:22.871147+0000) 2023-08-28T00:32:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:22 smithi093 ceph-mon[111335]: pgmap v6564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:22 smithi093 ceph-mon[118902]: pgmap v6564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:23.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:23.871408+0000) 2023-08-28T00:32:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:24 smithi136 ceph-mon[113531]: pgmap v6565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1717233066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:24.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:24.871634+0000) 2023-08-28T00:32:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:24 smithi093 ceph-mon[111335]: pgmap v6565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1717233066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:24 smithi093 ceph-mon[118902]: pgmap v6565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1717233066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:32:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:32:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:32:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:25.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:25.871888+0000) 2023-08-28T00:32:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:26 smithi136 ceph-mon[113531]: pgmap v6566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2683658041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:26.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:26.872161+0000) 2023-08-28T00:32:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:26 smithi093 ceph-mon[111335]: pgmap v6566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2683658041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:26 smithi093 ceph-mon[118902]: pgmap v6566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2683658041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:27.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:27.872379+0000) 2023-08-28T00:32:29.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:28 smithi136 ceph-mon[113531]: pgmap v6567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:28.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:28.872635+0000) 2023-08-28T00:32:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:28 smithi093 ceph-mon[111335]: pgmap v6567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:28 smithi093 ceph-mon[118902]: pgmap v6567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2166968780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:29.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:29.872886+0000) 2023-08-28T00:32:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2166968780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2166968780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:30 smithi136 ceph-mon[113531]: pgmap v6568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:30.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:30.873186+0000) 2023-08-28T00:32:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:30 smithi093 ceph-mon[111335]: pgmap v6568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:30 smithi093 ceph-mon[118902]: pgmap v6568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:31.053Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:31.054Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1545296232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:31.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:31.873437+0000) 2023-08-28T00:32:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1545296232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1545296232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:32.515Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:32.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:32 smithi136 ceph-mon[113531]: pgmap v6569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:32.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:32.873703+0000) 2023-08-28T00:32:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:32 smithi093 ceph-mon[111335]: pgmap v6569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:32 smithi093 ceph-mon[118902]: pgmap v6569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/718412007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:33.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:33.873944+0000) 2023-08-28T00:32:34.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/718412007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:34.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/718412007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:34 smithi136 ceph-mon[113531]: pgmap v6570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:34.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:34.874191+0000) 2023-08-28T00:32:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:34 smithi093 ceph-mon[111335]: pgmap v6570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:34 smithi093 ceph-mon[118902]: pgmap v6570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:32:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:32:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:32:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:35.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:35.874425+0000) 2023-08-28T00:32:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:36 smithi136 ceph-mon[113531]: pgmap v6571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/962416756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:36.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:36.874710+0000) 2023-08-28T00:32:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:36 smithi093 ceph-mon[111335]: pgmap v6571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/962416756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:36 smithi093 ceph-mon[118902]: pgmap v6571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/962416756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:37.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:37.874953+0000) 2023-08-28T00:32:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:38.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:38.875216+0000) 2023-08-28T00:32:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:38 smithi136 ceph-mon[113531]: pgmap v6572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2227247285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:38 smithi093 ceph-mon[111335]: pgmap v6572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2227247285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:38 smithi093 ceph-mon[118902]: pgmap v6572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2227247285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:39.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:39.875470+0000) 2023-08-28T00:32:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:41 smithi136 ceph-mon[113531]: pgmap v6573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/361413454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:40.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:40.875793+0000) 2023-08-28T00:32:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:41 smithi093 ceph-mon[111335]: pgmap v6573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/361413454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:41 smithi093 ceph-mon[118902]: pgmap v6573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/361413454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:41.054Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:41.055Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:41.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:41.876095+0000) 2023-08-28T00:32:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:42.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:42.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:43 smithi136 ceph-mon[113531]: pgmap v6574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:42.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:42.876257+0000) 2023-08-28T00:32:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:43 smithi093 ceph-mon[111335]: pgmap v6574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:43 smithi093 ceph-mon[118902]: pgmap v6574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2783342983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:43.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:43.876415+0000) 2023-08-28T00:32:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2783342983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:44.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2783342983' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:45 smithi136 ceph-mon[113531]: pgmap v6575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:44.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:44.876626+0000) 2023-08-28T00:32:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:45 smithi093 ceph-mon[111335]: pgmap v6575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:45 smithi093 ceph-mon[118902]: pgmap v6575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:32:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:32:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:32:46.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:45.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:45.876800+0000) 2023-08-28T00:32:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:46 smithi093 ceph-mon[111335]: pgmap v6576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:46.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2757316685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:46.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:46 smithi093 ceph-mon[118902]: pgmap v6576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:46.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2757316685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:46 smithi136 ceph-mon[113531]: pgmap v6576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2757316685' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:46.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:46.877003+0000) 2023-08-28T00:32:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:47.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:47.877205+0000) 2023-08-28T00:32:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:48 smithi093 ceph-mon[111335]: pgmap v6577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:48.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1148261245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:48 smithi093 ceph-mon[118902]: pgmap v6577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:48.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1148261245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:48 smithi136 ceph-mon[113531]: pgmap v6577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1148261245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:48.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:48.877414+0000) 2023-08-28T00:32:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:49.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:49.877627+0000) 2023-08-28T00:32:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:50 smithi093 ceph-mon[111335]: pgmap v6578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:50.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2180961506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:50 smithi093 ceph-mon[118902]: pgmap v6578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:50.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2180961506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:50 smithi136 ceph-mon[113531]: pgmap v6578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2180961506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:50.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:50.877904+0000) 2023-08-28T00:32:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:51.055Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:51.056Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:51.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:51.878157+0000) 2023-08-28T00:32:52.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:52 smithi093 ceph-mon[111335]: pgmap v6579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:52.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:52 smithi093 ceph-mon[118902]: pgmap v6579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:52.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:52.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:52.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:32:52.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:32:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:52 smithi136 ceph-mon[113531]: pgmap v6579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:52.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:52.878384+0000) 2023-08-28T00:32:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3346831093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:53.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:53.878602+0000) 2023-08-28T00:32:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3346831093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3346831093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:54 smithi136 ceph-mon[113531]: pgmap v6580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:54.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:54.878871+0000) 2023-08-28T00:32:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:54 smithi093 ceph-mon[118902]: pgmap v6580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:54 smithi093 ceph-mon[111335]: pgmap v6580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:32:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:32:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:32:55] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:32:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2091961965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:55.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:55.879116+0000) 2023-08-28T00:32:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2091961965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2091961965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:56 smithi136 ceph-mon[113531]: pgmap v6581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:56.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:56.879366+0000) 2023-08-28T00:32:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:56 smithi093 ceph-mon[111335]: pgmap v6581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:57.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:56 smithi093 ceph-mon[118902]: pgmap v6581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/504628501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:57.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:57.879615+0000) 2023-08-28T00:32:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/504628501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/504628501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:32:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:32:58 smithi136 ceph-mon[113531]: pgmap v6582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:58.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:58.879834+0000) 2023-08-28T00:32:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:32:58 smithi093 ceph-mon[111335]: pgmap v6582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:32:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:32:58 smithi093 ceph-mon[118902]: pgmap v6582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:32:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:32:59.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:32:59.880068+0000) 2023-08-28T00:33:01.057 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:00 smithi093 ceph-mon[118902]: pgmap v6583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:01.058 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3309490433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:01.058 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:00 smithi093 ceph-mon[111335]: pgmap v6583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:01.058 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3309490433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:00 smithi136 ceph-mon[113531]: pgmap v6583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3309490433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:00.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:00.880303+0000) 2023-08-28T00:33:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:01.056Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:01.057Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:01.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:01.880505+0000) 2023-08-28T00:33:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:02 smithi093 ceph-mon[118902]: pgmap v6584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3865819289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:02.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:02 smithi093 ceph-mon[111335]: pgmap v6584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:02.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3865819289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:02.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:02.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:02.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:02.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:02 smithi136 ceph-mon[113531]: pgmap v6584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3865819289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:02.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:02.880780+0000) 2023-08-28T00:33:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:03.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:03.881060+0000) 2023-08-28T00:33:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:04 smithi136 ceph-mon[113531]: pgmap v6585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:04.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:04.881293+0000) 2023-08-28T00:33:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:04 smithi093 ceph-mon[111335]: pgmap v6585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:04 smithi093 ceph-mon[118902]: pgmap v6585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:33:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:33:05] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:33:05.882 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1833592548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:05.882 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:33:06.081 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1833592548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:06.081 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:33:06.081 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1833592548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:06.081 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:33:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:05.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:05.881478+0000) 2023-08-28T00:33:06.920 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:06 smithi093 ceph-mon[111335]: pgmap v6586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:06.920 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:06 smithi093 ceph-mon[118902]: pgmap v6586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:06 smithi136 ceph-mon[113531]: pgmap v6586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:06.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:06.881682+0000) 2023-08-28T00:33:07.940 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:07.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:07.881951+0000) 2023-08-28T00:33:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3642118895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3642118895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3642118895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:08.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:08.882188+0000) 2023-08-28T00:33:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[111335]: pgmap v6587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:33:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:33:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[118902]: pgmap v6587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:33:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:33:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:09 smithi136 ceph-mon[113531]: pgmap v6587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:33:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:33:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:33:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:09.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:09.882415+0000) 2023-08-28T00:33:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2119594409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2119594409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:10.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2119594409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:10.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:10.882678+0000) 2023-08-28T00:33:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:11 smithi093 ceph-mon[111335]: pgmap v6588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:11 smithi093 ceph-mon[118902]: pgmap v6588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:11.057Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:11.058Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:11.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:11 smithi136 ceph-mon[113531]: pgmap v6588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:11.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:11.882969+0000) 2023-08-28T00:33:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2337336187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2337336187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2337336187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:12.518Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:12.519Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:12.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:12.883208+0000) 2023-08-28T00:33:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:13 smithi093 ceph-mon[111335]: pgmap v6589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:13 smithi093 ceph-mon[118902]: pgmap v6589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:13.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:13 smithi136 ceph-mon[113531]: pgmap v6589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:13.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:13.883398+0000) 2023-08-28T00:33:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:14.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:14.883656+0000) 2023-08-28T00:33:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:33:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:33:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:33:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:15 smithi093 ceph-mon[111335]: pgmap v6590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:15.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3357258981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:15.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:15 smithi093 ceph-mon[118902]: pgmap v6590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:15.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3357258981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:15 smithi136 ceph-mon[113531]: pgmap v6590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3357258981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:15.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:15.883887+0000) 2023-08-28T00:33:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:16.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:16.884112+0000) 2023-08-28T00:33:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:17 smithi093 ceph-mon[111335]: pgmap v6591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1043785294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:17 smithi093 ceph-mon[118902]: pgmap v6591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1043785294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:17 smithi136 ceph-mon[113531]: pgmap v6591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1043785294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:17.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:17.884343+0000) 2023-08-28T00:33:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:18.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:18.884625+0000) 2023-08-28T00:33:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:19 smithi093 ceph-mon[111335]: pgmap v6592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:19.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/879590963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:19.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:19 smithi093 ceph-mon[118902]: pgmap v6592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:19.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/879590963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:19.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:19 smithi136 ceph-mon[113531]: pgmap v6592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:19.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/879590963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:19.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:19.884897+0000) 2023-08-28T00:33:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:20.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:20.885135+0000) 2023-08-28T00:33:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:21 smithi093 ceph-mon[111335]: pgmap v6593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:21 smithi093 ceph-mon[118902]: pgmap v6593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:21.058Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:21.059Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:21 smithi136 ceph-mon[113531]: pgmap v6593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:21.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:21.885365+0000) 2023-08-28T00:33:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4265510667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4265510667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:22.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4265510667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:22.519Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:22.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:22.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:22.885642+0000) 2023-08-28T00:33:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:23 smithi136 ceph-mon[113531]: pgmap v6594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:23 smithi093 ceph-mon[111335]: pgmap v6594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:23 smithi093 ceph-mon[118902]: pgmap v6594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:23.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:23.885885+0000) 2023-08-28T00:33:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3378229315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3378229315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3378229315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:24.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:24.886153+0000) 2023-08-28T00:33:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:25 smithi136 ceph-mon[113531]: pgmap v6595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:25 smithi093 ceph-mon[111335]: pgmap v6595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:25 smithi093 ceph-mon[118902]: pgmap v6595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:25.744 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:33:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:33:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:33:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:25.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:25.886411+0000) 2023-08-28T00:33:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:26.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:26.886643+0000) 2023-08-28T00:33:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:27 smithi136 ceph-mon[113531]: pgmap v6596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/22327711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:27 smithi093 ceph-mon[118902]: pgmap v6596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/22327711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:27 smithi093 ceph-mon[111335]: pgmap v6596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/22327711' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:27.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:27.886825+0000) 2023-08-28T00:33:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:28.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:28.887087+0000) 2023-08-28T00:33:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:29 smithi136 ceph-mon[113531]: pgmap v6597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1592464104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:29 smithi093 ceph-mon[111335]: pgmap v6597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1592464104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:29 smithi093 ceph-mon[118902]: pgmap v6597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1592464104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:29.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:29.887360+0000) 2023-08-28T00:33:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:30.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:30.887623+0000) 2023-08-28T00:33:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:31 smithi093 ceph-mon[118902]: pgmap v6598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1874785383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:31.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:31 smithi093 ceph-mon[111335]: pgmap v6598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:31.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1874785383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:31.058Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:31.059Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:31.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:31 smithi136 ceph-mon[113531]: pgmap v6598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:31.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1874785383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:31.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:31.887856+0000) 2023-08-28T00:33:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:32.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:32.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:32.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:32.888086+0000) 2023-08-28T00:33:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:33 smithi136 ceph-mon[113531]: pgmap v6599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:33 smithi093 ceph-mon[111335]: pgmap v6599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:33 smithi093 ceph-mon[118902]: pgmap v6599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:34.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:33.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:33.888258+0000) 2023-08-28T00:33:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1111976050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1111976050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1111976050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:34.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:34.888502+0000) 2023-08-28T00:33:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:35 smithi136 ceph-mon[113531]: pgmap v6600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:35 smithi093 ceph-mon[111335]: pgmap v6600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:35 smithi093 ceph-mon[118902]: pgmap v6600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:35.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:33:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:33:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:33:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:35.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:35.888763+0000) 2023-08-28T00:33:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:36 smithi093 ceph-mon[111335]: pgmap v6601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:36.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3981463901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:36 smithi093 ceph-mon[118902]: pgmap v6601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:36.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3981463901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:36 smithi136 ceph-mon[113531]: pgmap v6601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3981463901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:36.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:36.889073+0000) 2023-08-28T00:33:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:37.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:37.889307+0000) 2023-08-28T00:33:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:38 smithi136 ceph-mon[113531]: pgmap v6602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1806387109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:38.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:38.889541+0000) 2023-08-28T00:33:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:38 smithi093 ceph-mon[111335]: pgmap v6602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1806387109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:38 smithi093 ceph-mon[118902]: pgmap v6602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1806387109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:39.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:39.889799+0000) 2023-08-28T00:33:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:41.060 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:40 smithi093 ceph-mon[111335]: pgmap v6603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:41.060 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3747426138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:41.060 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:40 smithi093 ceph-mon[118902]: pgmap v6603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:41.060 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3747426138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:40 smithi136 ceph-mon[113531]: pgmap v6603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3747426138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:40.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:40.890078+0000) 2023-08-28T00:33:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:41.059Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:41.060Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:41.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:41.890371+0000) 2023-08-28T00:33:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:42 smithi093 ceph-mon[111335]: pgmap v6604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:42.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:42 smithi093 ceph-mon[118902]: pgmap v6604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:42.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:42.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:42 smithi136 ceph-mon[113531]: pgmap v6604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:42.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:42.890625+0000) 2023-08-28T00:33:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4211664228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:43.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:43.890882+0000) 2023-08-28T00:33:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4211664228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4211664228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:44 smithi136 ceph-mon[113531]: pgmap v6605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:44.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:44.891146+0000) 2023-08-28T00:33:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:44 smithi093 ceph-mon[111335]: pgmap v6605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:44 smithi093 ceph-mon[118902]: pgmap v6605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:33:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:33:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:33:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1442076877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:45.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:45.891416+0000) 2023-08-28T00:33:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1442076877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1442076877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:46 smithi136 ceph-mon[113531]: pgmap v6606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:46.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:46.891685+0000) 2023-08-28T00:33:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:46 smithi093 ceph-mon[111335]: pgmap v6606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:46 smithi093 ceph-mon[118902]: pgmap v6606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/365404223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:47.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:47.891943+0000) 2023-08-28T00:33:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/365404223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/365404223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:48 smithi136 ceph-mon[113531]: pgmap v6607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:48.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:48.892195+0000) 2023-08-28T00:33:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:48 smithi093 ceph-mon[111335]: pgmap v6607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:48 smithi093 ceph-mon[118902]: pgmap v6607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:49.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:49.892445+0000) 2023-08-28T00:33:51.060 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:50 smithi093 ceph-mon[111335]: pgmap v6608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:51.060 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2979428515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:51.060 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:50 smithi093 ceph-mon[118902]: pgmap v6608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:51.061 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2979428515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:50 smithi136 ceph-mon[113531]: pgmap v6608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2979428515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:50.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:50.892671+0000) 2023-08-28T00:33:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:51.059Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:51.060Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:51.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:51.892928+0000) 2023-08-28T00:33:52.783 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:52.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:52.783 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:33:52.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:33:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:52 smithi136 ceph-mon[113531]: pgmap v6609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/34421600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:52.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:52.893164+0000) 2023-08-28T00:33:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:52 smithi093 ceph-mon[111335]: pgmap v6609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/34421600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:52 smithi093 ceph-mon[118902]: pgmap v6609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/34421600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:53.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:53.893372+0000) 2023-08-28T00:33:55.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:54 smithi136 ceph-mon[113531]: pgmap v6610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:54.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:54.893596+0000) 2023-08-28T00:33:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:54 smithi093 ceph-mon[111335]: pgmap v6610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:54 smithi093 ceph-mon[118902]: pgmap v6610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:33:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:33:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:33:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:33:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4082639201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:55.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:55.893786+0000) 2023-08-28T00:33:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4082639201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4082639201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:56 smithi136 ceph-mon[113531]: pgmap v6611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:56.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:56.893997+0000) 2023-08-28T00:33:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:56 smithi093 ceph-mon[111335]: pgmap v6611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:56 smithi093 ceph-mon[118902]: pgmap v6611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1468921102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:57.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:57.894249+0000) 2023-08-28T00:33:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1468921102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1468921102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:33:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:58 smithi136 ceph-mon[113531]: pgmap v6612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:58.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:58.894494+0000) 2023-08-28T00:33:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:58 smithi093 ceph-mon[111335]: pgmap v6612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:33:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:58 smithi093 ceph-mon[118902]: pgmap v6612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:33:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1812665012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:33:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:33:59.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:33:59.894718+0000) 2023-08-28T00:34:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:33:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1812665012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:33:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1812665012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:01.062 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:00 smithi093 ceph-mon[111335]: pgmap v6613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:01.063 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:00 smithi093 ceph-mon[118902]: pgmap v6613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:00 smithi136 ceph-mon[113531]: pgmap v6613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:00.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:00.894955+0000) 2023-08-28T00:34:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:01.060Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:01.061Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:01.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:01.895227+0000) 2023-08-28T00:34:02.813 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:02.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:02.813 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:02.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:02 smithi136 ceph-mon[113531]: pgmap v6614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/189323076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:02.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:02.895450+0000) 2023-08-28T00:34:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:02 smithi093 ceph-mon[111335]: pgmap v6614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/189323076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:02 smithi093 ceph-mon[118902]: pgmap v6614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/189323076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:03.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:03.895681+0000) 2023-08-28T00:34:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:04 smithi136 ceph-mon[113531]: pgmap v6615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1640308408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:04.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:04.895952+0000) 2023-08-28T00:34:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:04 smithi093 ceph-mon[111335]: pgmap v6615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1640308408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:04 smithi093 ceph-mon[118902]: pgmap v6615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:05.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1640308408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:34:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:34:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:34:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:05.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:05.896221+0000) 2023-08-28T00:34:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:06 smithi136 ceph-mon[113531]: pgmap v6616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1041858806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:06.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:06.896510+0000) 2023-08-28T00:34:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:06 smithi093 ceph-mon[111335]: pgmap v6616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1041858806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:06 smithi093 ceph-mon[118902]: pgmap v6616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1041858806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:07.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:07.896760+0000) 2023-08-28T00:34:09.083 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:08 smithi093 ceph-mon[111335]: pgmap v6617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:09.083 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:34:09.083 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:08 smithi093 ceph-mon[118902]: pgmap v6617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:09.084 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:34:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:08 smithi136 ceph-mon[113531]: pgmap v6617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:34:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:08.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:08.896969+0000) 2023-08-28T00:34:09.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:09.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/676442543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:09.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:09.962 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/676442543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/676442543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:09.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:09.897166+0000) 2023-08-28T00:34:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:10.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:10.897324+0000) 2023-08-28T00:34:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[111335]: pgmap v6618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:11.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[118902]: pgmap v6618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:11.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.493 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:11.061Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:11.493 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:11.063Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:11 smithi136 ceph-mon[113531]: pgmap v6618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:11.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:11.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:11.897567+0000) 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2336386843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:12.524 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2336386843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:34:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:34:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:34:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2336386843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:12.523Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:12.524Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:12.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:12.897827+0000) 2023-08-28T00:34:13.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:13 smithi136 ceph-mon[113531]: pgmap v6619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:13 smithi093 ceph-mon[111335]: pgmap v6619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:13 smithi093 ceph-mon[118902]: pgmap v6619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:13.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:13.898128+0000) 2023-08-28T00:34:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2275067303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2275067303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2275067303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:14.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:14.898367+0000) 2023-08-28T00:34:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:15 smithi136 ceph-mon[113531]: pgmap v6620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:15 smithi093 ceph-mon[111335]: pgmap v6620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:15 smithi093 ceph-mon[118902]: pgmap v6620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:34:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:34:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:34:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:15.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:15.898623+0000) 2023-08-28T00:34:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:16.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:16.898884+0000) 2023-08-28T00:34:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:17 smithi136 ceph-mon[113531]: pgmap v6621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1371790300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:17 smithi093 ceph-mon[111335]: pgmap v6621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1371790300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:17 smithi093 ceph-mon[118902]: pgmap v6621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:17.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1371790300' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:17.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:17.899158+0000) 2023-08-28T00:34:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:18.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:18.899421+0000) 2023-08-28T00:34:19.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:19 smithi136 ceph-mon[113531]: pgmap v6622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:19.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4004444374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:19 smithi093 ceph-mon[111335]: pgmap v6622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4004444374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:19 smithi093 ceph-mon[118902]: pgmap v6622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:19.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4004444374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:19.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:19.899638+0000) 2023-08-28T00:34:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:20.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:20.899884+0000) 2023-08-28T00:34:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:21 smithi093 ceph-mon[118902]: pgmap v6623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4166983690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:21.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:21 smithi093 ceph-mon[111335]: pgmap v6623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:21.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4166983690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:21.063Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:21.063Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:21 smithi136 ceph-mon[113531]: pgmap v6623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4166983690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:21.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:21.900155+0000) 2023-08-28T00:34:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:22.523Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:22.524Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:22.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:22.900403+0000) 2023-08-28T00:34:23.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:23 smithi136 ceph-mon[113531]: pgmap v6624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:23 smithi093 ceph-mon[111335]: pgmap v6624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:23.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:23 smithi093 ceph-mon[118902]: pgmap v6624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:23.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:23.900622+0000) 2023-08-28T00:34:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2255442097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:24.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2255442097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2255442097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:24.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:24.900835+0000) 2023-08-28T00:34:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:25 smithi136 ceph-mon[113531]: pgmap v6625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:25 smithi093 ceph-mon[111335]: pgmap v6625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:25 smithi093 ceph-mon[118902]: pgmap v6625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:25.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:34:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:34:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:34:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:25.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:25.901115+0000) 2023-08-28T00:34:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:26 smithi136 ceph-mon[113531]: pgmap v6626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3394476022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:26.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:26.901409+0000) 2023-08-28T00:34:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:26 smithi093 ceph-mon[111335]: pgmap v6626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3394476022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:26 smithi093 ceph-mon[118902]: pgmap v6626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3394476022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:27.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:27.901597+0000) 2023-08-28T00:34:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:28 smithi136 ceph-mon[113531]: pgmap v6627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3711845709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:28.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:28.901846+0000) 2023-08-28T00:34:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:28 smithi093 ceph-mon[111335]: pgmap v6627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3711845709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:28 smithi093 ceph-mon[118902]: pgmap v6627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3711845709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:29.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:29.902127+0000) 2023-08-28T00:34:31.064 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:30 smithi093 ceph-mon[118902]: pgmap v6628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:31.064 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1371355545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:31.064 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:30 smithi093 ceph-mon[111335]: pgmap v6628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:31.064 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1371355545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:30 smithi136 ceph-mon[113531]: pgmap v6628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1371355545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:30.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:30.902394+0000) 2023-08-28T00:34:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:31.063Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:31.064Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:31.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:31.902643+0000) 2023-08-28T00:34:32.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:32 smithi093 ceph-mon[111335]: pgmap v6629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:32.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:32 smithi093 ceph-mon[118902]: pgmap v6629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:32.524Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:32.525Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:32 smithi136 ceph-mon[113531]: pgmap v6629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:32.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:32.902965+0000) 2023-08-28T00:34:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2751413072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:33.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:33.903192+0000) 2023-08-28T00:34:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2751413072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2751413072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:34 smithi136 ceph-mon[113531]: pgmap v6630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:34.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:34.903438+0000) 2023-08-28T00:34:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:34 smithi093 ceph-mon[111335]: pgmap v6630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:34 smithi093 ceph-mon[118902]: pgmap v6630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:34:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:34:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:34:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1048702347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:35.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:35.903671+0000) 2023-08-28T00:34:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1048702347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1048702347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:36 smithi136 ceph-mon[113531]: pgmap v6631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:36.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:36.903903+0000) 2023-08-28T00:34:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:36 smithi093 ceph-mon[111335]: pgmap v6631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:36 smithi093 ceph-mon[118902]: pgmap v6631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:37.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:37.904093+0000) 2023-08-28T00:34:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:38 smithi136 ceph-mon[113531]: pgmap v6632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/238151165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:38.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:38.904314+0000) 2023-08-28T00:34:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:38 smithi093 ceph-mon[111335]: pgmap v6632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/238151165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:38 smithi093 ceph-mon[118902]: pgmap v6632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/238151165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:39.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:39.904585+0000) 2023-08-28T00:34:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:41.065 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:40 smithi093 ceph-mon[111335]: pgmap v6633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:41.065 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1753994434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:41.065 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:40 smithi093 ceph-mon[118902]: pgmap v6633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:41.065 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1753994434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:40 smithi136 ceph-mon[113531]: pgmap v6633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1753994434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:40.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:40.904848+0000) 2023-08-28T00:34:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:41.064Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:41.065Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:41.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:41.905116+0000) 2023-08-28T00:34:42.782 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:42.524Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:42.782 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:42.525Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:42 smithi136 ceph-mon[113531]: pgmap v6634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3854573348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:42.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:42.905382+0000) 2023-08-28T00:34:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:42 smithi093 ceph-mon[111335]: pgmap v6634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3854573348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:42 smithi093 ceph-mon[118902]: pgmap v6634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3854573348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:43.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:43.905644+0000) 2023-08-28T00:34:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:44 smithi136 ceph-mon[113531]: pgmap v6635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:44.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:44.905905+0000) 2023-08-28T00:34:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:44 smithi093 ceph-mon[111335]: pgmap v6635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:44 smithi093 ceph-mon[118902]: pgmap v6635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:34:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:34:45] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:34:46.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2950174705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:45.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:45.906178+0000) 2023-08-28T00:34:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2950174705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2950174705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:46 smithi136 ceph-mon[113531]: pgmap v6636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:46.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:46.906446+0000) 2023-08-28T00:34:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:46 smithi093 ceph-mon[118902]: pgmap v6636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:46 smithi093 ceph-mon[111335]: pgmap v6636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2148575374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:47.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:47.906718+0000) 2023-08-28T00:34:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2148575374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2148575374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:48 smithi136 ceph-mon[113531]: pgmap v6637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:48.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:48.906976+0000) 2023-08-28T00:34:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:48 smithi093 ceph-mon[111335]: pgmap v6637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:48 smithi093 ceph-mon[118902]: pgmap v6637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:50.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/311788865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:49.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:49.907233+0000) 2023-08-28T00:34:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/311788865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/311788865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:51.066 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:50 smithi093 ceph-mon[111335]: pgmap v6638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:51.066 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:50 smithi093 ceph-mon[118902]: pgmap v6638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:50 smithi136 ceph-mon[113531]: pgmap v6638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:50.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:50.907491+0000) 2023-08-28T00:34:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:51.065Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:51.066Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:51.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:51.907737+0000) 2023-08-28T00:34:52.810 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:52.525Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:52.810 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:34:52.526Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:34:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:52 smithi136 ceph-mon[113531]: pgmap v6639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3688512271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:52.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:52.907978+0000) 2023-08-28T00:34:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:52 smithi093 ceph-mon[111335]: pgmap v6639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3688512271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:52 smithi093 ceph-mon[118902]: pgmap v6639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3688512271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:53.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:53.908243+0000) 2023-08-28T00:34:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:54.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:54.908522+0000) 2023-08-28T00:34:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:54 smithi136 ceph-mon[113531]: pgmap v6640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3373211517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:54 smithi093 ceph-mon[111335]: pgmap v6640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3373211517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:54 smithi093 ceph-mon[118902]: pgmap v6640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:34:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3373211517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:34:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:34:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:34:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:55.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:55.908736+0000) 2023-08-28T00:34:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:56 smithi136 ceph-mon[113531]: pgmap v6641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:56.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:56.908948+0000) 2023-08-28T00:34:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:56 smithi093 ceph-mon[111335]: pgmap v6641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:56 smithi093 ceph-mon[118902]: pgmap v6641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3961646555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:57.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:57.909198+0000) 2023-08-28T00:34:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3961646555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3961646555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:34:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:58 smithi136 ceph-mon[113531]: pgmap v6642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:58.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:58.909409+0000) 2023-08-28T00:34:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:58 smithi093 ceph-mon[111335]: pgmap v6642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:34:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:58 smithi093 ceph-mon[118902]: pgmap v6642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:34:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4210177755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:34:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:34:59.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:34:59.909665+0000) 2023-08-28T00:35:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:34:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4210177755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:00.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:34:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4210177755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:00 smithi136 ceph-mon[113531]: pgmap v6643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:00.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:00.909925+0000) 2023-08-28T00:35:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:00 smithi093 ceph-mon[111335]: pgmap v6643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:00 smithi093 ceph-mon[118902]: pgmap v6643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:01.066Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:01.067Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/517766239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:01.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:01.910205+0000) 2023-08-28T00:35:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/517766239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/517766239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:02.833 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:02.525Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:02.833 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:02.526Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:02 smithi136 ceph-mon[113531]: pgmap v6644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:02.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:02.910417+0000) 2023-08-28T00:35:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:02 smithi093 ceph-mon[111335]: pgmap v6644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:02 smithi093 ceph-mon[118902]: pgmap v6644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:04.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:03.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:03.910693+0000) 2023-08-28T00:35:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:04 smithi136 ceph-mon[113531]: pgmap v6645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/253302829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:04.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:04.910951+0000) 2023-08-28T00:35:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:04 smithi093 ceph-mon[111335]: pgmap v6645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/253302829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:04 smithi093 ceph-mon[118902]: pgmap v6645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/253302829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:35:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:35:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:35:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:05.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:05.911207+0000) 2023-08-28T00:35:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:06 smithi136 ceph-mon[113531]: pgmap v6646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/504382439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:06.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:06.911468+0000) 2023-08-28T00:35:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:06 smithi093 ceph-mon[111335]: pgmap v6646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/504382439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:06 smithi093 ceph-mon[118902]: pgmap v6646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/504382439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:07.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:07.911713+0000) 2023-08-28T00:35:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:08 smithi136 ceph-mon[113531]: pgmap v6647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1593587010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:08.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:08.911986+0000) 2023-08-28T00:35:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:08 smithi093 ceph-mon[118902]: pgmap v6647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1593587010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:08 smithi093 ceph-mon[111335]: pgmap v6647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1593587010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:09.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:09.912223+0000) 2023-08-28T00:35:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:10 smithi136 ceph-mon[113531]: pgmap v6648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:10.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:10.912504+0000) 2023-08-28T00:35:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:10 smithi093 ceph-mon[111335]: pgmap v6648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:10 smithi093 ceph-mon[118902]: pgmap v6648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:11.067Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:11.068Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:12.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3775781775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:35:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:11.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:11.912715+0000) 2023-08-28T00:35:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3775781775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:12.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:35:12.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3775781775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:12.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:35:12.852 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:12.526Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:12.853 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:12.528Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:12 smithi136 ceph-mon[113531]: pgmap v6649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:12.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:12.912862+0000) 2023-08-28T00:35:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:12 smithi093 ceph-mon[111335]: pgmap v6649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:12 smithi093 ceph-mon[118902]: pgmap v6649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:14.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:13.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:13.913018+0000) 2023-08-28T00:35:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4233118415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4233118415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:14.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4233118415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:35:14.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:14.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:14.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:14.913244+0000) 2023-08-28T00:35:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:35:15] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:35:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[111335]: pgmap v6650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2484M 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[118902]: pgmap v6650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2484M 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:35:15.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:15 smithi136 ceph-mon[113531]: pgmap v6650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:15 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2484M 2023-08-28T00:35:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:35:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:35:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:35:16.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:15.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:15.913417+0000) 2023-08-28T00:35:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:16 smithi136 ceph-mon[113531]: pgmap v6651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/782526566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:16.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:16.913671+0000) 2023-08-28T00:35:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:16 smithi093 ceph-mon[111335]: pgmap v6651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/782526566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:17.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:16 smithi093 ceph-mon[118902]: pgmap v6651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:17.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/782526566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:17.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:17.913919+0000) 2023-08-28T00:35:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:18.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:18.914186+0000) 2023-08-28T00:35:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:18 smithi136 ceph-mon[113531]: pgmap v6652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3566357592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:18 smithi093 ceph-mon[118902]: pgmap v6652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3566357592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:18 smithi093 ceph-mon[111335]: pgmap v6652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3566357592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:19.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:19.914427+0000) 2023-08-28T00:35:21.068 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:20 smithi093 ceph-mon[111335]: pgmap v6653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:21.069 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1218514245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:21.069 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:20 smithi093 ceph-mon[118902]: pgmap v6653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:21.069 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1218514245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:20 smithi136 ceph-mon[113531]: pgmap v6653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1218514245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:20.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:20.914681+0000) 2023-08-28T00:35:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:21.068Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:21.069Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:21.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:21.914937+0000) 2023-08-28T00:35:22.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:22 smithi093 ceph-mon[111335]: pgmap v6654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:22.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:22 smithi093 ceph-mon[118902]: pgmap v6654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:22.527Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:22.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:22.528Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:22 smithi136 ceph-mon[113531]: pgmap v6654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:22.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:22.915152+0000) 2023-08-28T00:35:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2154951013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:23.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:23.915428+0000) 2023-08-28T00:35:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2154951013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2154951013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:24 smithi136 ceph-mon[113531]: pgmap v6655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:24.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:24.915650+0000) 2023-08-28T00:35:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:24 smithi093 ceph-mon[111335]: pgmap v6655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:24 smithi093 ceph-mon[118902]: pgmap v6655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:35:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:35:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:35:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2066848219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:25.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:25.915900+0000) 2023-08-28T00:35:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2066848219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2066848219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:26 smithi136 ceph-mon[113531]: pgmap v6656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:26.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:26.916173+0000) 2023-08-28T00:35:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:26 smithi093 ceph-mon[118902]: pgmap v6656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:26 smithi093 ceph-mon[111335]: pgmap v6656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:28.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:27.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:27.916463+0000) 2023-08-28T00:35:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:28 smithi136 ceph-mon[113531]: pgmap v6657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1328829325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:28.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:28.916758+0000) 2023-08-28T00:35:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:28 smithi093 ceph-mon[111335]: pgmap v6657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1328829325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:28 smithi093 ceph-mon[118902]: pgmap v6657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1328829325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:29.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:29.917021+0000) 2023-08-28T00:35:31.070 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:30 smithi093 ceph-mon[111335]: pgmap v6658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:31.070 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3717311881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:31.070 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:30 smithi093 ceph-mon[118902]: pgmap v6658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:31.070 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3717311881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:30 smithi136 ceph-mon[113531]: pgmap v6658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3717311881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:30.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:30.917305+0000) 2023-08-28T00:35:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:31.069Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:31.070Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:31.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:31.917507+0000) 2023-08-28T00:35:32.780 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:32.528Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:32.780 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:32.529Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:32 smithi136 ceph-mon[113531]: pgmap v6659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/625228837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:32.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:32.917760+0000) 2023-08-28T00:35:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:32 smithi093 ceph-mon[118902]: pgmap v6659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/625228837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:32 smithi093 ceph-mon[111335]: pgmap v6659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/625228837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:33.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:33.918008+0000) 2023-08-28T00:35:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:34 smithi136 ceph-mon[113531]: pgmap v6660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:34.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:34.918253+0000) 2023-08-28T00:35:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:34 smithi093 ceph-mon[111335]: pgmap v6660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:34 smithi093 ceph-mon[118902]: pgmap v6660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:35:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:35:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:35:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1122016153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:35.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:35.918480+0000) 2023-08-28T00:35:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1122016153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:36.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1122016153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:36 smithi136 ceph-mon[113531]: pgmap v6661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:36.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:36.918747+0000) 2023-08-28T00:35:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:36 smithi093 ceph-mon[111335]: pgmap v6661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:36 smithi093 ceph-mon[118902]: pgmap v6661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2912039648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:37.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:37.919017+0000) 2023-08-28T00:35:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2912039648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2912039648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:38 smithi136 ceph-mon[113531]: pgmap v6662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:38.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:38.919240+0000) 2023-08-28T00:35:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:38 smithi093 ceph-mon[111335]: pgmap v6662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:38 smithi093 ceph-mon[118902]: pgmap v6662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/382590609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:39.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:39.919508+0000) 2023-08-28T00:35:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/382590609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/382590609' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:41.071 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:40 smithi093 ceph-mon[111335]: pgmap v6663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:41.071 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:40 smithi093 ceph-mon[118902]: pgmap v6663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:40 smithi136 ceph-mon[113531]: pgmap v6663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:40.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:40.919804+0000) 2023-08-28T00:35:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:41.070Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:41.071Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:41.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:41.920100+0000) 2023-08-28T00:35:42.809 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:42.528Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:42.809 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:42.529Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:42 smithi136 ceph-mon[113531]: pgmap v6664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3601100466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:42.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:42.920358+0000) 2023-08-28T00:35:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:42 smithi093 ceph-mon[111335]: pgmap v6664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:43.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3601100466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:42 smithi093 ceph-mon[118902]: pgmap v6664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:43.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3601100466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:43.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:43.920563+0000) 2023-08-28T00:35:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:44 smithi136 ceph-mon[113531]: pgmap v6665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3252118633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:44.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:44.920828+0000) 2023-08-28T00:35:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:44 smithi093 ceph-mon[111335]: pgmap v6665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3252118633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:44 smithi093 ceph-mon[118902]: pgmap v6665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3252118633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:35:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:35:45] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:35:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:45.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:45.921116+0000) 2023-08-28T00:35:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:46 smithi136 ceph-mon[113531]: pgmap v6666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:46.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:46.921311+0000) 2023-08-28T00:35:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:46 smithi093 ceph-mon[111335]: pgmap v6666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:46 smithi093 ceph-mon[118902]: pgmap v6666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2870710821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:47.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:47.921588+0000) 2023-08-28T00:35:48.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2870710821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:48.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2870710821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:48 smithi136 ceph-mon[113531]: pgmap v6667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:48.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:48.921839+0000) 2023-08-28T00:35:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:48 smithi093 ceph-mon[111335]: pgmap v6667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:48 smithi093 ceph-mon[118902]: pgmap v6667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/928611318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:49.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:49.922112+0000) 2023-08-28T00:35:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/928611318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:50.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/928611318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:50 smithi136 ceph-mon[113531]: pgmap v6668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:50.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:50.922328+0000) 2023-08-28T00:35:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:50 smithi093 ceph-mon[111335]: pgmap v6668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:50 smithi093 ceph-mon[118902]: pgmap v6668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:51.070Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:51.071Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/678227268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:51.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:51.922552+0000) 2023-08-28T00:35:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/678227268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/678227268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:52.837 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:52.529Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:52.837 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:35:52.530Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:35:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:52 smithi136 ceph-mon[113531]: pgmap v6669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:52.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:52.922844+0000) 2023-08-28T00:35:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:52 smithi093 ceph-mon[111335]: pgmap v6669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:52 smithi093 ceph-mon[118902]: pgmap v6669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:53.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:53.923159+0000) 2023-08-28T00:35:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:35:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:54 smithi136 ceph-mon[113531]: pgmap v6670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1196506492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:54.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:54.923426+0000) 2023-08-28T00:35:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:54 smithi093 ceph-mon[111335]: pgmap v6670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1196506492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:54 smithi093 ceph-mon[118902]: pgmap v6670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1196506492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:35:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:35:55] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T00:35:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:55.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:55.923657+0000) 2023-08-28T00:35:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:56 smithi136 ceph-mon[113531]: pgmap v6671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3097321285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:56.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:56.923954+0000) 2023-08-28T00:35:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:56 smithi093 ceph-mon[111335]: pgmap v6671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3097321285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:56 smithi093 ceph-mon[118902]: pgmap v6671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3097321285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:57.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:57.924199+0000) 2023-08-28T00:35:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:58 smithi136 ceph-mon[113531]: pgmap v6672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:35:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/332201429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:58.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:58.924444+0000) 2023-08-28T00:35:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:58 smithi093 ceph-mon[111335]: pgmap v6672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:35:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/332201429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:35:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:58 smithi093 ceph-mon[118902]: pgmap v6672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:35:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:35:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/332201429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:35:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:35:59.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:35:59.924704+0000) 2023-08-28T00:36:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:00 smithi136 ceph-mon[113531]: pgmap v6673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:00.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:00.924925+0000) 2023-08-28T00:36:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:00 smithi093 ceph-mon[111335]: pgmap v6673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:00 smithi093 ceph-mon[118902]: pgmap v6673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:01.070Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:01.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:01.071Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2146098278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:01.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:01.925187+0000) 2023-08-28T00:36:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2146098278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2146098278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:02.864 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:02.530Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:02.864 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:02.531Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:02 smithi136 ceph-mon[113531]: pgmap v6674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:02.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:02.925458+0000) 2023-08-28T00:36:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:02 smithi093 ceph-mon[111335]: pgmap v6674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:02 smithi093 ceph-mon[118902]: pgmap v6674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2681692013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:03.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:03.925713+0000) 2023-08-28T00:36:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2681692013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2681692013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:04 smithi136 ceph-mon[113531]: pgmap v6675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:04.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:04.925964+0000) 2023-08-28T00:36:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:04 smithi093 ceph-mon[111335]: pgmap v6675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:04 smithi093 ceph-mon[118902]: pgmap v6675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:36:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:36:05] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T00:36:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:05.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:05.926199+0000) 2023-08-28T00:36:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:06.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:06.926408+0000) 2023-08-28T00:36:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:06 smithi136 ceph-mon[113531]: pgmap v6676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1896503237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:06 smithi093 ceph-mon[111335]: pgmap v6676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1896503237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:06 smithi093 ceph-mon[118902]: pgmap v6676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:07.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1896503237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:07.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:07.926654+0000) 2023-08-28T00:36:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:08.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:08.926872+0000) 2023-08-28T00:36:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:08 smithi136 ceph-mon[113531]: pgmap v6677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2791023725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:08 smithi093 ceph-mon[111335]: pgmap v6677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2791023725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:08 smithi093 ceph-mon[118902]: pgmap v6677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2791023725' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:09.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:09.927154+0000) 2023-08-28T00:36:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:10.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:10.927360+0000) 2023-08-28T00:36:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:10 smithi136 ceph-mon[113531]: pgmap v6678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2055613027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:10 smithi093 ceph-mon[111335]: pgmap v6678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2055613027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:10 smithi093 ceph-mon[118902]: pgmap v6678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2055613027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:11.071Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:11.072Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:11.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:11.927620+0000) 2023-08-28T00:36:12.886 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:12.531Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:12.886 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:12.532Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:12 smithi136 ceph-mon[113531]: pgmap v6679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:12.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:12.927866+0000) 2023-08-28T00:36:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:12 smithi093 ceph-mon[111335]: pgmap v6679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:12 smithi093 ceph-mon[118902]: pgmap v6679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3993176175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:13.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:13.928142+0000) 2023-08-28T00:36:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3993176175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3993176175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:15.154 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:14 smithi093 ceph-mon[111335]: pgmap v6680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:15.154 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:36:15.155 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:14 smithi093 ceph-mon[118902]: pgmap v6680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:15.155 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:36:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:14 smithi136 ceph-mon[113531]: pgmap v6680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:36:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:14.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:14.928410+0000) 2023-08-28T00:36:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:36:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:36:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:36:15.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4194250181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4194250181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4194250181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:15.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:15.928625+0000) 2023-08-28T00:36:17.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:16 smithi136 ceph-mon[113531]: pgmap v6681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:16.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:16.928800+0000) 2023-08-28T00:36:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[118902]: pgmap v6681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[111335]: pgmap v6681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:17.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:17.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:36:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:36:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:18.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:17.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:17.928997+0000) 2023-08-28T00:36:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:36:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:36:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:36:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:36:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:36:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:18 smithi136 ceph-mon[113531]: pgmap v6682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1161694461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:18.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:18.929218+0000) 2023-08-28T00:36:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:18 smithi093 ceph-mon[111335]: pgmap v6682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1161694461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:18 smithi093 ceph-mon[118902]: pgmap v6682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1161694461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:19.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:19.929481+0000) 2023-08-28T00:36:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:20 smithi136 ceph-mon[113531]: pgmap v6683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4167643749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:20.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:20.929696+0000) 2023-08-28T00:36:21.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:20 smithi093 ceph-mon[111335]: pgmap v6683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:21.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4167643749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:21.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:20 smithi093 ceph-mon[118902]: pgmap v6683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:21.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4167643749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:21.243 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:21.071Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:21.243 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:21.072Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:21.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:21.929955+0000) 2023-08-28T00:36:22.916 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:22.532Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:22.916 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:22.532Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:22 smithi136 ceph-mon[113531]: pgmap v6684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1223019081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:23.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:22.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:22.930205+0000) 2023-08-28T00:36:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:22 smithi093 ceph-mon[111335]: pgmap v6684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:23.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1223019081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:22 smithi093 ceph-mon[118902]: pgmap v6684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1223019081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:23.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:23.930424+0000) 2023-08-28T00:36:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:24 smithi136 ceph-mon[113531]: pgmap v6685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:24.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:24.930706+0000) 2023-08-28T00:36:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:24 smithi093 ceph-mon[111335]: pgmap v6685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:24 smithi093 ceph-mon[118902]: pgmap v6685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:36:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:36:25] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:36:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2177767001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:25.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:25.930966+0000) 2023-08-28T00:36:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2177767001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2177767001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:26 smithi136 ceph-mon[113531]: pgmap v6686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:26.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:26.931228+0000) 2023-08-28T00:36:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:26 smithi093 ceph-mon[111335]: pgmap v6686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:26 smithi093 ceph-mon[118902]: pgmap v6686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2676838233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:27.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:27.931482+0000) 2023-08-28T00:36:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2676838233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2676838233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:28 smithi136 ceph-mon[113531]: pgmap v6687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:28.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:28.931764+0000) 2023-08-28T00:36:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:28 smithi093 ceph-mon[111335]: pgmap v6687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:28 smithi093 ceph-mon[118902]: pgmap v6687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1582909060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:29.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:29.932021+0000) 2023-08-28T00:36:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1582909060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1582909060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:30 smithi136 ceph-mon[113531]: pgmap v6688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:30.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:30.932307+0000) 2023-08-28T00:36:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:30 smithi093 ceph-mon[111335]: pgmap v6688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:30 smithi093 ceph-mon[118902]: pgmap v6688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:31.072Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:31.073Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:31.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:31.932568+0000) 2023-08-28T00:36:32.948 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:32.532Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:32.948 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:32.532Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:32 smithi136 ceph-mon[113531]: pgmap v6689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3513321467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:32.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:32.932823+0000) 2023-08-28T00:36:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:32 smithi093 ceph-mon[111335]: pgmap v6689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3513321467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:32 smithi093 ceph-mon[118902]: pgmap v6689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3513321467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:33.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:33.933115+0000) 2023-08-28T00:36:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:34 smithi136 ceph-mon[113531]: pgmap v6690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1539232386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:34.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:34.933352+0000) 2023-08-28T00:36:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:34 smithi093 ceph-mon[111335]: pgmap v6690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1539232386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:34 smithi093 ceph-mon[118902]: pgmap v6690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1539232386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:36:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:36:35] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:36:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:35.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:35.933619+0000) 2023-08-28T00:36:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:36 smithi136 ceph-mon[113531]: pgmap v6691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:36.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:36.933837+0000) 2023-08-28T00:36:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:36 smithi093 ceph-mon[111335]: pgmap v6691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:36 smithi093 ceph-mon[118902]: pgmap v6691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3059370978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:37.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:37.934089+0000) 2023-08-28T00:36:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3059370978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3059370978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:38 smithi136 ceph-mon[113531]: pgmap v6692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:38.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:38.934386+0000) 2023-08-28T00:36:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:38 smithi093 ceph-mon[111335]: pgmap v6692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:38 smithi093 ceph-mon[118902]: pgmap v6692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1774997771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:39.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:39.934665+0000) 2023-08-28T00:36:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1774997771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1774997771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:40 smithi136 ceph-mon[113531]: pgmap v6693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:40.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:40.934957+0000) 2023-08-28T00:36:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:40 smithi093 ceph-mon[111335]: pgmap v6693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:40 smithi093 ceph-mon[118902]: pgmap v6693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:41.073Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:41.074Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:42.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2252619678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:41.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:41.935227+0000) 2023-08-28T00:36:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2252619678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2252619678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:42.975 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:42.532Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:42.975 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:42.533Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:42 smithi136 ceph-mon[113531]: pgmap v6694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:42.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:42.935491+0000) 2023-08-28T00:36:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:42 smithi093 ceph-mon[111335]: pgmap v6694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:42 smithi093 ceph-mon[118902]: pgmap v6694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:43.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:43.935772+0000) 2023-08-28T00:36:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:44 smithi136 ceph-mon[113531]: pgmap v6695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2024702274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:44.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:44.936019+0000) 2023-08-28T00:36:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:44 smithi093 ceph-mon[111335]: pgmap v6695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2024702274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:44 smithi093 ceph-mon[118902]: pgmap v6695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2024702274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:36:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:36:45] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:36:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:45.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:45.936325+0000) 2023-08-28T00:36:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:46 smithi136 ceph-mon[113531]: pgmap v6696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2956355183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:46.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:46.936634+0000) 2023-08-28T00:36:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:46 smithi093 ceph-mon[111335]: pgmap v6696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2956355183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:46 smithi093 ceph-mon[118902]: pgmap v6696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2956355183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:47.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:47.936856+0000) 2023-08-28T00:36:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:48 smithi136 ceph-mon[113531]: pgmap v6697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1973160359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:48.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:48.937122+0000) 2023-08-28T00:36:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:48 smithi093 ceph-mon[111335]: pgmap v6697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1973160359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:48 smithi093 ceph-mon[118902]: pgmap v6697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1973160359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:50.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:49.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:49.937381+0000) 2023-08-28T00:36:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:50 smithi136 ceph-mon[113531]: pgmap v6698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:50.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:50.937653+0000) 2023-08-28T00:36:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:50 smithi093 ceph-mon[111335]: pgmap v6698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:50 smithi093 ceph-mon[118902]: pgmap v6698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:51.074Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:51.075Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/285386925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:51.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:51.937926+0000) 2023-08-28T00:36:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/285386925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/285386925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:52.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:36:52.533Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:36:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:52 smithi136 ceph-mon[113531]: pgmap v6699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:52.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:52.938137+0000) 2023-08-28T00:36:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:52 smithi093 ceph-mon[111335]: pgmap v6699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:52 smithi093 ceph-mon[118902]: pgmap v6699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/162954029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:53.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:53.938345+0000) 2023-08-28T00:36:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/162954029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/162954029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:36:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:55 smithi136 ceph-mon[113531]: pgmap v6700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:54.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:54.938598+0000) 2023-08-28T00:36:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:55 smithi093 ceph-mon[111335]: pgmap v6700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:55 smithi093 ceph-mon[118902]: pgmap v6700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:36:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:36:55] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:36:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:55.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:55.938922+0000) 2023-08-28T00:36:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:57 smithi136 ceph-mon[113531]: pgmap v6701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3803008032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:56.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:56.939204+0000) 2023-08-28T00:36:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:57 smithi093 ceph-mon[111335]: pgmap v6701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3803008032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:57 smithi093 ceph-mon[118902]: pgmap v6701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:57.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3803008032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:57.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:57.939421+0000) 2023-08-28T00:36:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:59 smithi136 ceph-mon[113531]: pgmap v6702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:36:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2324566012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:58.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:58.939669+0000) 2023-08-28T00:36:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:59 smithi093 ceph-mon[111335]: pgmap v6702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:36:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2324566012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:36:59.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:59 smithi093 ceph-mon[118902]: pgmap v6702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:36:59.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:36:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2324566012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:36:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:36:59.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:36:59.939934+0000) 2023-08-28T00:37:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:01 smithi136 ceph-mon[113531]: pgmap v6703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:01.479 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1252065201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:01.479 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:00.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:00.940190+0000) 2023-08-28T00:37:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:01 smithi093 ceph-mon[111335]: pgmap v6703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1252065201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:01 smithi093 ceph-mon[118902]: pgmap v6703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1252065201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:01.075Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:01.076Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:01.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:01.940461+0000) 2023-08-28T00:37:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:02.534Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:02.535Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:03 smithi136 ceph-mon[113531]: pgmap v6704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:02.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:02.940736+0000) 2023-08-28T00:37:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:03 smithi093 ceph-mon[111335]: pgmap v6704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:03 smithi093 ceph-mon[118902]: pgmap v6704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1410645542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:03.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:03.941013+0000) 2023-08-28T00:37:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1410645542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1410645542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:05 smithi093 ceph-mon[111335]: pgmap v6705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:05 smithi093 ceph-mon[118902]: pgmap v6705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:05 smithi136 ceph-mon[113531]: pgmap v6705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:04.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:04.941279+0000) 2023-08-28T00:37:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:37:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:37:05] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:37:06.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2295780957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:05.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:05.941541+0000) 2023-08-28T00:37:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2295780957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2295780957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:07 smithi136 ceph-mon[113531]: pgmap v6706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:06.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:06.941812+0000) 2023-08-28T00:37:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:07 smithi093 ceph-mon[111335]: pgmap v6706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:07 smithi093 ceph-mon[118902]: pgmap v6706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/788706075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:07.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:07.942121+0000) 2023-08-28T00:37:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/788706075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/788706075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:09 smithi136 ceph-mon[113531]: pgmap v6707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:08.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:08.942402+0000) 2023-08-28T00:37:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:09 smithi093 ceph-mon[111335]: pgmap v6707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:09 smithi093 ceph-mon[118902]: pgmap v6707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:09.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:09.942657+0000) 2023-08-28T00:37:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:11 smithi136 ceph-mon[113531]: pgmap v6708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/725238438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:10.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:10.942921+0000) 2023-08-28T00:37:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:11 smithi093 ceph-mon[111335]: pgmap v6708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/725238438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:11 smithi093 ceph-mon[118902]: pgmap v6708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:11.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/725238438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:11.076Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:11.077Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:11.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:11.943160+0000) 2023-08-28T00:37:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:12.534Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:12.535Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:13 smithi136 ceph-mon[113531]: pgmap v6709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1681750710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:13.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:12.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:12.943394+0000) 2023-08-28T00:37:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:13 smithi093 ceph-mon[118902]: pgmap v6709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1681750710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:13.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:13 smithi093 ceph-mon[111335]: pgmap v6709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:13.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1681750710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:13.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:13.943596+0000) 2023-08-28T00:37:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:15 smithi093 ceph-mon[111335]: pgmap v6710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:15 smithi093 ceph-mon[118902]: pgmap v6710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:15 smithi136 ceph-mon[113531]: pgmap v6710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:14.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:14.943852+0000) 2023-08-28T00:37:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:37:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:37:15] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:37:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1695736226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:15.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:15.944134+0000) 2023-08-28T00:37:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1695736226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1695736226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:17 smithi136 ceph-mon[113531]: pgmap v6711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:16.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:16.944388+0000) 2023-08-28T00:37:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:17 smithi093 ceph-mon[111335]: pgmap v6711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:17 smithi093 ceph-mon[118902]: pgmap v6711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:37:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3495612085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:17.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:17.944650+0000) 2023-08-28T00:37:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:37:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3495612085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:18.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:37:18.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3495612085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:19.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:18.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:18.944800+0000) 2023-08-28T00:37:19.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:19 smithi136 ceph-mon[113531]: pgmap v6712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:19 smithi093 ceph-mon[111335]: pgmap v6712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:19 smithi093 ceph-mon[118902]: pgmap v6712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:20.257 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:19.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:19.944954+0000) 2023-08-28T00:37:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2758633123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2758633123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2758633123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:20.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:21.268 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:20.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:20.945129+0000) 2023-08-28T00:37:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[111335]: pgmap v6713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:37:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:37:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:21.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[118902]: pgmap v6713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:21.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:37:21.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:37:21.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:21.076Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:21.077Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:21 smithi136 ceph-mon[113531]: pgmap v6713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:37:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:37:21.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:37:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:21.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:21.945365+0000) 2023-08-28T00:37:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:22.535Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:22.535Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:23.270 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:22.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:22.945633+0000) 2023-08-28T00:37:23.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:23 smithi136 ceph-mon[113531]: pgmap v6714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:23.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2997362167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:23 smithi093 ceph-mon[111335]: pgmap v6714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:23.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2997362167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:23.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:23 smithi093 ceph-mon[118902]: pgmap v6714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:23.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2997362167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:24.277 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:23.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:23.945888+0000) 2023-08-28T00:37:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:24.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:25.277 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:24.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:24.946145+0000) 2023-08-28T00:37:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:25 smithi136 ceph-mon[113531]: pgmap v6715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:25.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/155629909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:25.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:25 smithi093 ceph-mon[111335]: pgmap v6715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:25.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/155629909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:25.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:37:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:37:25] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:37:25.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:25 smithi093 ceph-mon[118902]: pgmap v6715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:25.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/155629909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:25.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:25.946440+0000) 2023-08-28T00:37:27.280 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:26.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:26.946702+0000) 2023-08-28T00:37:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:27 smithi136 ceph-mon[113531]: pgmap v6716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1335516381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:27 smithi093 ceph-mon[111335]: pgmap v6716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1335516381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:27 smithi093 ceph-mon[118902]: pgmap v6716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:27.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1335516381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:27.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:27.946958+0000) 2023-08-28T00:37:29.282 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:28.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:28.947180+0000) 2023-08-28T00:37:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:29 smithi136 ceph-mon[113531]: pgmap v6717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:29 smithi093 ceph-mon[111335]: pgmap v6717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:29 smithi093 ceph-mon[118902]: pgmap v6717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:30.291 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:29.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:29.947427+0000) 2023-08-28T00:37:30.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2640980215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:30.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2640980215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2640980215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:31.290 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:30.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:30.947697+0000) 2023-08-28T00:37:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:31 smithi093 ceph-mon[111335]: pgmap v6718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:31 smithi093 ceph-mon[118902]: pgmap v6718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:31.077Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:31.079Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:31 smithi136 ceph-mon[113531]: pgmap v6718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:32.295 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:31.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:31.947947+0000) 2023-08-28T00:37:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2734758966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2734758966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:32.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2734758966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:32.535Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:32.536Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:33.296 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:32.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:32.948202+0000) 2023-08-28T00:37:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:33 smithi136 ceph-mon[113531]: pgmap v6719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:33 smithi093 ceph-mon[111335]: pgmap v6719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:33 smithi093 ceph-mon[118902]: pgmap v6719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:34.300 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:33.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:33.948435+0000) 2023-08-28T00:37:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:34 smithi136 ceph-mon[113531]: pgmap v6720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:34.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1335036205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:34 smithi093 ceph-mon[111335]: pgmap v6720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1335036205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:34 smithi093 ceph-mon[118902]: pgmap v6720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:34.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1335036205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:34.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:34.948681+0000) 2023-08-28T00:37:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:37:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:37:35] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:37:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:35.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:35.948952+0000) 2023-08-28T00:37:37.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:36.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:36.949221+0000) 2023-08-28T00:37:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:36 smithi136 ceph-mon[113531]: pgmap v6721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3820578272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:36 smithi093 ceph-mon[111335]: pgmap v6721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3820578272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:36 smithi093 ceph-mon[118902]: pgmap v6721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3820578272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:37.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:37.949426+0000) 2023-08-28T00:37:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:38 smithi136 ceph-mon[113531]: pgmap v6722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:38.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:38.949675+0000) 2023-08-28T00:37:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:38 smithi093 ceph-mon[118902]: pgmap v6722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:38 smithi093 ceph-mon[111335]: pgmap v6722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1975171980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:39.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:39.949907+0000) 2023-08-28T00:37:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1975171980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1975171980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:41.079 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:40 smithi093 ceph-mon[111335]: pgmap v6723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:41.079 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:40 smithi093 ceph-mon[118902]: pgmap v6723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:40 smithi136 ceph-mon[113531]: pgmap v6723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:40.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:40.950122+0000) 2023-08-28T00:37:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:41.078Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:41.079Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002083914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:41.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:41.950376+0000) 2023-08-28T00:37:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002083914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002083914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:42.789 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:42.537Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:42.789 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:42.538Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:42 smithi136 ceph-mon[113531]: pgmap v6724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:42.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:42.950569+0000) 2023-08-28T00:37:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:42 smithi093 ceph-mon[111335]: pgmap v6724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:42 smithi093 ceph-mon[118902]: pgmap v6724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:44.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3905036765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:43.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:43.950804+0000) 2023-08-28T00:37:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3905036765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3905036765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:45.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:44 smithi136 ceph-mon[113531]: pgmap v6725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:44.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:44.951097+0000) 2023-08-28T00:37:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:44 smithi093 ceph-mon[111335]: pgmap v6725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:44 smithi093 ceph-mon[118902]: pgmap v6725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:37:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:37:45] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:37:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:45.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:45.951396+0000) 2023-08-28T00:37:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:46 smithi136 ceph-mon[113531]: pgmap v6726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1454578943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:46.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:46.951703+0000) 2023-08-28T00:37:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:46 smithi093 ceph-mon[111335]: pgmap v6726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1454578943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:46 smithi093 ceph-mon[118902]: pgmap v6726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1454578943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:47.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:47.951987+0000) 2023-08-28T00:37:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:48 smithi136 ceph-mon[113531]: pgmap v6727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1138099599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:48.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:48.952305+0000) 2023-08-28T00:37:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:48 smithi093 ceph-mon[111335]: pgmap v6727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1138099599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:48 smithi093 ceph-mon[118902]: pgmap v6727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1138099599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:49.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:49.952579+0000) 2023-08-28T00:37:51.079 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:50 smithi093 ceph-mon[111335]: pgmap v6728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:51.080 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:50 smithi093 ceph-mon[118902]: pgmap v6728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:50 smithi136 ceph-mon[113531]: pgmap v6728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:50.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:50.952853+0000) 2023-08-28T00:37:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:51.079Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:51.080Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1761773593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:51.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:51.953112+0000) 2023-08-28T00:37:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1761773593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1761773593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:52.811 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:52.537Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:52.812 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:37:52.538Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:37:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:52 smithi136 ceph-mon[113531]: pgmap v6729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:52.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:52.953320+0000) 2023-08-28T00:37:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:52 smithi093 ceph-mon[111335]: pgmap v6729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:52 smithi093 ceph-mon[118902]: pgmap v6729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/434200584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:53.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:53.953562+0000) 2023-08-28T00:37:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/434200584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/434200584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:54 smithi136 ceph-mon[113531]: pgmap v6730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:54.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:54.953820+0000) 2023-08-28T00:37:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:54 smithi093 ceph-mon[111335]: pgmap v6730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:54 smithi093 ceph-mon[118902]: pgmap v6730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:37:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:37:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:37:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:37:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3720495451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:55.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:55.954113+0000) 2023-08-28T00:37:56.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3720495451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:56.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3720495451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:56 smithi136 ceph-mon[113531]: pgmap v6731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:56.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:56.954415+0000) 2023-08-28T00:37:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:56 smithi093 ceph-mon[111335]: pgmap v6731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:56 smithi093 ceph-mon[118902]: pgmap v6731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:57.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:57.954617+0000) 2023-08-28T00:37:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:58 smithi136 ceph-mon[113531]: pgmap v6732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:37:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/733252446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:58.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:58.954898+0000) 2023-08-28T00:37:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:58 smithi093 ceph-mon[111335]: pgmap v6732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:37:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/733252446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:37:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:58 smithi093 ceph-mon[118902]: pgmap v6732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:37:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:37:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/733252446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:37:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:37:59.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:37:59.955171+0000) 2023-08-28T00:38:01.081 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:00 smithi093 ceph-mon[111335]: pgmap v6733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:01.082 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2395258823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:01.082 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:00 smithi093 ceph-mon[118902]: pgmap v6733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:01.082 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2395258823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:00 smithi136 ceph-mon[113531]: pgmap v6733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2395258823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:00.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:00.955411+0000) 2023-08-28T00:38:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:01.080Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:01.081Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:01.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:01.955656+0000) 2023-08-28T00:38:02.831 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:02.538Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:02.832 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:02.539Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:02 smithi136 ceph-mon[113531]: pgmap v6734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1816507352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:02.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:02.955898+0000) 2023-08-28T00:38:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:02 smithi093 ceph-mon[111335]: pgmap v6734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1816507352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:02 smithi093 ceph-mon[118902]: pgmap v6734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1816507352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:03.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:03.956137+0000) 2023-08-28T00:38:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:04 smithi136 ceph-mon[113531]: pgmap v6735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:04.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:04.956321+0000) 2023-08-28T00:38:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:04 smithi093 ceph-mon[111335]: pgmap v6735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:04 smithi093 ceph-mon[118902]: pgmap v6735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:38:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:38:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:38:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/205803175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:05.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:05.956568+0000) 2023-08-28T00:38:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/205803175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/205803175' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:06 smithi136 ceph-mon[113531]: pgmap v6736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:06.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:06.956818+0000) 2023-08-28T00:38:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:06 smithi093 ceph-mon[111335]: pgmap v6736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:06 smithi093 ceph-mon[118902]: pgmap v6736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/297096453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:07.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:07.957077+0000) 2023-08-28T00:38:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/297096453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/297096453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:09.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:08 smithi136 ceph-mon[113531]: pgmap v6737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:08.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:08.957292+0000) 2023-08-28T00:38:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:08 smithi093 ceph-mon[111335]: pgmap v6737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:08 smithi093 ceph-mon[118902]: pgmap v6737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:09.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:09.957521+0000) 2023-08-28T00:38:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:10 smithi136 ceph-mon[113531]: pgmap v6738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/973774145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:10.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:10.957787+0000) 2023-08-28T00:38:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:10 smithi093 ceph-mon[111335]: pgmap v6738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/973774145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:10 smithi093 ceph-mon[118902]: pgmap v6738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/973774145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:11.081Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:11.082Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:11.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:11.958066+0000) 2023-08-28T00:38:12.857 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:12.538Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:12.858 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:12.539Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:12 smithi136 ceph-mon[113531]: pgmap v6739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2945464659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:12.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:12.958325+0000) 2023-08-28T00:38:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:12 smithi093 ceph-mon[111335]: pgmap v6739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2945464659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:12 smithi093 ceph-mon[118902]: pgmap v6739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2945464659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:13.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:13.958600+0000) 2023-08-28T00:38:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:14 smithi136 ceph-mon[113531]: pgmap v6740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3528483263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:14.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:14.958819+0000) 2023-08-28T00:38:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:14 smithi093 ceph-mon[111335]: pgmap v6740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3528483263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:14 smithi093 ceph-mon[118902]: pgmap v6740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3528483263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:38:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:38:15] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:38:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:15.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:15.959018+0000) 2023-08-28T00:38:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:16 smithi136 ceph-mon[113531]: pgmap v6741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:16.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:16.959317+0000) 2023-08-28T00:38:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:16 smithi093 ceph-mon[111335]: pgmap v6741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:16 smithi093 ceph-mon[118902]: pgmap v6741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:18.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2938660953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:17.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:17.959576+0000) 2023-08-28T00:38:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2938660953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2938660953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:18 smithi136 ceph-mon[113531]: pgmap v6742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:18.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:18.959859+0000) 2023-08-28T00:38:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:18 smithi093 ceph-mon[111335]: pgmap v6742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:18 smithi093 ceph-mon[118902]: pgmap v6742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2536104422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:19.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:19.960062+0000) 2023-08-28T00:38:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2536104422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2536104422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:20.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:20.960221+0000) 2023-08-28T00:38:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:20 smithi136 ceph-mon[113531]: pgmap v6743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:38:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[111335]: pgmap v6743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:38:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[118902]: pgmap v6743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:38:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:21.081Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:21.082Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:21.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:21.960387+0000) 2023-08-28T00:38:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:22.538Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:22.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:22.539Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:22.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:22 smithi136 ceph-mon[113531]: pgmap v6744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1352130674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:22.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[111335]: pgmap v6744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1352130674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[118902]: pgmap v6744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1352130674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:22.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:22.960650+0000) 2023-08-28T00:38:24.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:23.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:23.960888+0000) 2023-08-28T00:38:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:38:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:38:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:38:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:38:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:38:24.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:24 smithi136 ceph-mon[113531]: pgmap v6745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3187957678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:24.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:24.961156+0000) 2023-08-28T00:38:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:24 smithi093 ceph-mon[111335]: pgmap v6745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3187957678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:24 smithi093 ceph-mon[118902]: pgmap v6745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3187957678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:38:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:38:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:38:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:25.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:25.961420+0000) 2023-08-28T00:38:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:26 smithi136 ceph-mon[113531]: pgmap v6746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3188574516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:26.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:26.961692+0000) 2023-08-28T00:38:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:26 smithi093 ceph-mon[111335]: pgmap v6746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3188574516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:26 smithi093 ceph-mon[118902]: pgmap v6746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3188574516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:27.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:27.961942+0000) 2023-08-28T00:38:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:28 smithi136 ceph-mon[113531]: pgmap v6747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:28.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:28.962202+0000) 2023-08-28T00:38:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:28 smithi093 ceph-mon[111335]: pgmap v6747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:28 smithi093 ceph-mon[118902]: pgmap v6747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1179007319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:29.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:29.962458+0000) 2023-08-28T00:38:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1179007319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1179007319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:30 smithi136 ceph-mon[113531]: pgmap v6748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:30.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:30.962691+0000) 2023-08-28T00:38:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:30 smithi093 ceph-mon[111335]: pgmap v6748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:30 smithi093 ceph-mon[118902]: pgmap v6748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:31.083Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:31.083Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:32.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3727090793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:31.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:31.962943+0000) 2023-08-28T00:38:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3727090793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3727090793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:32.922 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:32.539Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:32.923 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:32.540Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:32 smithi136 ceph-mon[113531]: pgmap v6749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:32.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:32.963180+0000) 2023-08-28T00:38:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:32 smithi093 ceph-mon[111335]: pgmap v6749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:32 smithi093 ceph-mon[118902]: pgmap v6749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3660164035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:33.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:33.963396+0000) 2023-08-28T00:38:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3660164035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3660164035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:34 smithi136 ceph-mon[113531]: pgmap v6750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:34.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:34.963622+0000) 2023-08-28T00:38:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:34 smithi093 ceph-mon[118902]: pgmap v6750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:34 smithi093 ceph-mon[111335]: pgmap v6750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:38:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:38:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:38:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:35.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:35.963862+0000) 2023-08-28T00:38:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:36 smithi136 ceph-mon[113531]: pgmap v6751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4240376741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:36.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:36.964116+0000) 2023-08-28T00:38:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:36 smithi093 ceph-mon[111335]: pgmap v6751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4240376741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:36 smithi093 ceph-mon[118902]: pgmap v6751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4240376741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:37.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:37.964378+0000) 2023-08-28T00:38:39.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:38 smithi136 ceph-mon[113531]: pgmap v6752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/937499060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:38.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:38.964589+0000) 2023-08-28T00:38:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:38 smithi093 ceph-mon[111335]: pgmap v6752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/937499060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:38 smithi093 ceph-mon[118902]: pgmap v6752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/937499060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:39.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:39.964866+0000) 2023-08-28T00:38:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:40 smithi136 ceph-mon[113531]: pgmap v6753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:40.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:40.965140+0000) 2023-08-28T00:38:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:40 smithi093 ceph-mon[111335]: pgmap v6753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:40 smithi093 ceph-mon[118902]: pgmap v6753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:41.084Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:41.084Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3522471117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3522471117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3522471117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:41.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:41.965397+0000) 2023-08-28T00:38:42.945 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:42.540Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:42.945 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:42.541Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:42 smithi093 ceph-mon[111335]: pgmap v6754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:42 smithi093 ceph-mon[118902]: pgmap v6754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:42 smithi136 ceph-mon[113531]: pgmap v6754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:42.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:42.965626+0000) 2023-08-28T00:38:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3290623366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3290623366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3290623366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:43.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:43.965886+0000) 2023-08-28T00:38:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:44 smithi093 ceph-mon[111335]: pgmap v6755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:44 smithi093 ceph-mon[118902]: pgmap v6755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:44 smithi136 ceph-mon[113531]: pgmap v6755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:44.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:44.966131+0000) 2023-08-28T00:38:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:38:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:38:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:38:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4026751778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4026751778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4026751778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:45.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:45.966350+0000) 2023-08-28T00:38:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:46 smithi093 ceph-mon[118902]: pgmap v6756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:46 smithi093 ceph-mon[111335]: pgmap v6756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:46 smithi136 ceph-mon[113531]: pgmap v6756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:46.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:46.966549+0000) 2023-08-28T00:38:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:47.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:47.966825+0000) 2023-08-28T00:38:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:48 smithi093 ceph-mon[118902]: pgmap v6757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1429466724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:48 smithi093 ceph-mon[111335]: pgmap v6757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1429466724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:48 smithi136 ceph-mon[113531]: pgmap v6757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1429466724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:48.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:48.967157+0000) 2023-08-28T00:38:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:49.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:49.967382+0000) 2023-08-28T00:38:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:50 smithi093 ceph-mon[111335]: pgmap v6758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1428730396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:50 smithi093 ceph-mon[118902]: pgmap v6758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:51.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1428730396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:51.084Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:51.085Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:50 smithi136 ceph-mon[113531]: pgmap v6758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1428730396' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:50.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:50.967571+0000) 2023-08-28T00:38:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:51.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:51.967714+0000) 2023-08-28T00:38:52.965 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:52.540Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:52.965 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:38:52.541Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:38:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:52 smithi093 ceph-mon[118902]: pgmap v6759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/933253387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:52 smithi093 ceph-mon[111335]: pgmap v6759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:53.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/933253387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:52 smithi136 ceph-mon[113531]: pgmap v6759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/933253387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:52.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:52.967881+0000) 2023-08-28T00:38:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:38:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:53.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:53.968063+0000) 2023-08-28T00:38:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:54 smithi093 ceph-mon[111335]: pgmap v6760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:54 smithi093 ceph-mon[118902]: pgmap v6760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:54 smithi136 ceph-mon[113531]: pgmap v6760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:55.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:54.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:54.968312+0000) 2023-08-28T00:38:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:38:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:38:55] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:38:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1574079769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:56.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1574079769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:56.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1574079769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:56.443 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:55.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:55.968546+0000) 2023-08-28T00:38:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:56 smithi093 ceph-mon[111335]: pgmap v6761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:56 smithi093 ceph-mon[118902]: pgmap v6761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:56 smithi136 ceph-mon[113531]: pgmap v6761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:56.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:56.968869+0000) 2023-08-28T00:38:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1455792527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:58.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1455792527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1455792527' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:38:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:57.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:57.969155+0000) 2023-08-28T00:38:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:38:58 smithi093 ceph-mon[111335]: pgmap v6762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:38:58 smithi093 ceph-mon[118902]: pgmap v6762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:38:58 smithi136 ceph-mon[113531]: pgmap v6762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:38:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:58.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:58.969389+0000) 2023-08-28T00:39:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:38:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:38:59.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:38:59.969681+0000) 2023-08-28T00:39:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:00 smithi093 ceph-mon[118902]: pgmap v6763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/847020113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:00 smithi093 ceph-mon[111335]: pgmap v6763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/847020113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:01.085Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:01.086Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:00 smithi136 ceph-mon[113531]: pgmap v6763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/847020113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:00.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:00.969939+0000) 2023-08-28T00:39:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:01.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:01.970171+0000) 2023-08-28T00:39:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:02.540Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:02.541Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:02 smithi136 ceph-mon[113531]: pgmap v6764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1142039723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:02.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:02.970402+0000) 2023-08-28T00:39:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:02 smithi093 ceph-mon[111335]: pgmap v6764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1142039723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:02 smithi093 ceph-mon[118902]: pgmap v6764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1142039723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:03.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:03.970664+0000) 2023-08-28T00:39:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:04 smithi093 ceph-mon[111335]: pgmap v6765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2542382579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:05 smithi093 ceph-mon[118902]: pgmap v6765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2542382579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:05 smithi136 ceph-mon[113531]: pgmap v6765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2542382579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:04.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:04.970961+0000) 2023-08-28T00:39:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:39:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:39:05] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:39:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:05.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:05.971236+0000) 2023-08-28T00:39:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:07 smithi136 ceph-mon[113531]: pgmap v6766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:06.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:06.971534+0000) 2023-08-28T00:39:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:07 smithi093 ceph-mon[111335]: pgmap v6766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:07 smithi093 ceph-mon[118902]: pgmap v6766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1166453400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:07.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:07.971797+0000) 2023-08-28T00:39:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1166453400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1166453400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:09 smithi136 ceph-mon[113531]: pgmap v6767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:08.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:08.972020+0000) 2023-08-28T00:39:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:09 smithi093 ceph-mon[111335]: pgmap v6767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:09 smithi093 ceph-mon[118902]: pgmap v6767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2362743913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:09.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:09.972277+0000) 2023-08-28T00:39:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2362743913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2362743913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:11 smithi136 ceph-mon[113531]: pgmap v6768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:10.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:10.972547+0000) 2023-08-28T00:39:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:11 smithi093 ceph-mon[111335]: pgmap v6768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:11 smithi093 ceph-mon[118902]: pgmap v6768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:11.086Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:11.087Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/43811710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:11.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:11.972815+0000) 2023-08-28T00:39:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/43811710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/43811710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:12.541Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:12.542Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:13 smithi136 ceph-mon[113531]: pgmap v6769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:12.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:12.973014+0000) 2023-08-28T00:39:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:13 smithi093 ceph-mon[111335]: pgmap v6769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:13 smithi093 ceph-mon[118902]: pgmap v6769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:13.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:13.973259+0000) 2023-08-28T00:39:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:15 smithi093 ceph-mon[111335]: pgmap v6770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3945291125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:15 smithi093 ceph-mon[118902]: pgmap v6770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3945291125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:15 smithi136 ceph-mon[113531]: pgmap v6770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3945291125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:14.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:14.973481+0000) 2023-08-28T00:39:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:39:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:39:15] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:39:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:15.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:15.973737+0000) 2023-08-28T00:39:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:17 smithi136 ceph-mon[113531]: pgmap v6771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3431216153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:16.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:16.974024+0000) 2023-08-28T00:39:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:17 smithi093 ceph-mon[111335]: pgmap v6771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3431216153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:17 smithi093 ceph-mon[118902]: pgmap v6771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3431216153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:17.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:17.974220+0000) 2023-08-28T00:39:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:19 smithi136 ceph-mon[113531]: pgmap v6772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:18.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:18.974443+0000) 2023-08-28T00:39:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:19 smithi093 ceph-mon[111335]: pgmap v6772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:19 smithi093 ceph-mon[118902]: pgmap v6772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2427305136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:19.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:19.974662+0000) 2023-08-28T00:39:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2427305136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2427305136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:21 smithi136 ceph-mon[113531]: pgmap v6773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:20.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:20.974873+0000) 2023-08-28T00:39:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:21 smithi093 ceph-mon[111335]: pgmap v6773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:21 smithi093 ceph-mon[118902]: pgmap v6773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:21.086Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:21.087Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2630688063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:21.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:21.975141+0000) 2023-08-28T00:39:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2630688063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2630688063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:22.542Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:23.028 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:22.543Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:23 smithi136 ceph-mon[113531]: pgmap v6774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:22.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:22.975354+0000) 2023-08-28T00:39:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:23 smithi093 ceph-mon[111335]: pgmap v6774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:23 smithi093 ceph-mon[118902]: pgmap v6774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:24.410 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:23.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:23.975523+0000) 2023-08-28T00:39:24.410 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:39:24.410 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:24.411 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:24.411 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:24.411 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3396212091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:39:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3396212091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:24.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:39:24.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:24.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:24.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:24.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3396212091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:25.061 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:24.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:24.975704+0000) 2023-08-28T00:39:25.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:25 smithi093 ceph-mon[111335]: pgmap v6775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:25.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:25 smithi093 ceph-mon[118902]: pgmap v6775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:25 smithi136 ceph-mon[113531]: pgmap v6775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:39:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:39:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:39:26.289 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:25.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:25.976146+0000) 2023-08-28T00:39:26.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1912728282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1912728282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:26.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1912728282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:27.297 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:26.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:26.976388+0000) 2023-08-28T00:39:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:27 smithi136 ceph-mon[113531]: pgmap v6776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:39:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:39:27.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[111335]: pgmap v6776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:39:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:39:27.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[118902]: pgmap v6776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:27.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:39:27.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:39:27.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:39:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:27.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:27.976641+0000) 2023-08-28T00:39:29.300 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:28.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:28.976921+0000) 2023-08-28T00:39:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:29 smithi136 ceph-mon[113531]: pgmap v6777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:29.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/717092097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:29 smithi093 ceph-mon[111335]: pgmap v6777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:29.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/717092097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:29 smithi093 ceph-mon[118902]: pgmap v6777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/717092097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:30.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:29.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:29.977178+0000) 2023-08-28T00:39:31.305 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:30.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:30.977437+0000) 2023-08-28T00:39:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:31 smithi093 ceph-mon[111335]: pgmap v6778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/537337245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:31 smithi093 ceph-mon[118902]: pgmap v6778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/537337245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:31.087Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:31.088Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:31 smithi136 ceph-mon[113531]: pgmap v6778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:31.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/537337245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:31.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:31.977698+0000) 2023-08-28T00:39:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:32.543Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:32.544Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:33.308 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:32.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:32.977953+0000) 2023-08-28T00:39:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:33 smithi136 ceph-mon[113531]: pgmap v6779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:33 smithi093 ceph-mon[111335]: pgmap v6779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:33 smithi093 ceph-mon[118902]: pgmap v6779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:34.315 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:33.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:33.978204+0000) 2023-08-28T00:39:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3834237879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3834237879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3834237879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:35.319 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:34.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:34.978371+0000) 2023-08-28T00:39:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:35 smithi136 ceph-mon[113531]: pgmap v6780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:35 smithi093 ceph-mon[111335]: pgmap v6780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:35 smithi093 ceph-mon[118902]: pgmap v6780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:39:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:39:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:39:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:35.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:35.978626+0000) 2023-08-28T00:39:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:36 smithi136 ceph-mon[113531]: pgmap v6781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3471900951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:36.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:36.978917+0000) 2023-08-28T00:39:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:36 smithi093 ceph-mon[111335]: pgmap v6781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3471900951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:36 smithi093 ceph-mon[118902]: pgmap v6781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3471900951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:37.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:37.979198+0000) 2023-08-28T00:39:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:38.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:38.979441+0000) 2023-08-28T00:39:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:38 smithi136 ceph-mon[113531]: pgmap v6782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3174078557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:38 smithi093 ceph-mon[111335]: pgmap v6782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3174078557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:38 smithi093 ceph-mon[118902]: pgmap v6782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3174078557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:39.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:39.979691+0000) 2023-08-28T00:39:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:41.089 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:40 smithi093 ceph-mon[111335]: pgmap v6783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:41.089 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3978121470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:41.089 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:40 smithi093 ceph-mon[118902]: pgmap v6783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:41.089 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3978121470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:40 smithi136 ceph-mon[113531]: pgmap v6783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3978121470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:40.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:40.979927+0000) 2023-08-28T00:39:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:41.088Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:41.089Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:41.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:41.980147+0000) 2023-08-28T00:39:42.810 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:42.543Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:42.810 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:42.544Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:42 smithi136 ceph-mon[113531]: pgmap v6784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:42.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:42.980352+0000) 2023-08-28T00:39:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:42 smithi093 ceph-mon[111335]: pgmap v6784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:42 smithi093 ceph-mon[118902]: pgmap v6784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4137175388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:43.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:43.980607+0000) 2023-08-28T00:39:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4137175388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4137175388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:44 smithi136 ceph-mon[113531]: pgmap v6785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:44.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:44.980874+0000) 2023-08-28T00:39:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:44 smithi093 ceph-mon[111335]: pgmap v6785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:44 smithi093 ceph-mon[118902]: pgmap v6785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:39:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:39:45] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:39:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4129205190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:45.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:45.981111+0000) 2023-08-28T00:39:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4129205190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4129205190' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:46 smithi136 ceph-mon[113531]: pgmap v6786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:46.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:46.981303+0000) 2023-08-28T00:39:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:46 smithi093 ceph-mon[111335]: pgmap v6786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:46 smithi093 ceph-mon[118902]: pgmap v6786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:48.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3966515965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:47.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:47.981456+0000) 2023-08-28T00:39:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3966515965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3966515965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:49.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:48 smithi136 ceph-mon[113531]: pgmap v6787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:48.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:48.981643+0000) 2023-08-28T00:39:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:48 smithi093 ceph-mon[111335]: pgmap v6787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:48 smithi093 ceph-mon[118902]: pgmap v6787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:49.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:49.981880+0000) 2023-08-28T00:39:51.090 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:50 smithi093 ceph-mon[111335]: pgmap v6788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:51.090 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1101595506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:51.090 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:50 smithi093 ceph-mon[118902]: pgmap v6788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:51.090 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1101595506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:50 smithi136 ceph-mon[113531]: pgmap v6788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1101595506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:50.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:50.982150+0000) 2023-08-28T00:39:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:51.089Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:51.090Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:51.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:51.982409+0000) 2023-08-28T00:39:52.833 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:52.544Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:52.833 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:39:52.544Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:39:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:52 smithi136 ceph-mon[113531]: pgmap v6789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3991693639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:52.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:52.982663+0000) 2023-08-28T00:39:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:52 smithi093 ceph-mon[111335]: pgmap v6789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3991693639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:52 smithi093 ceph-mon[118902]: pgmap v6789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3991693639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:53.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:53.982936+0000) 2023-08-28T00:39:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:39:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:54 smithi136 ceph-mon[113531]: pgmap v6790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:54.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:54.983172+0000) 2023-08-28T00:39:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:54 smithi093 ceph-mon[111335]: pgmap v6790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:54 smithi093 ceph-mon[118902]: pgmap v6790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:39:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:39:55] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:39:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/34256541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:55.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:55.983430+0000) 2023-08-28T00:39:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/34256541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:56.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/34256541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:56 smithi136 ceph-mon[113531]: pgmap v6791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:56.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:56.983682+0000) 2023-08-28T00:39:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:56 smithi093 ceph-mon[111335]: pgmap v6791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:56 smithi093 ceph-mon[118902]: pgmap v6791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3686867406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:57.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:57.983882+0000) 2023-08-28T00:39:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3686867406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3686867406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:39:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:58 smithi136 ceph-mon[113531]: pgmap v6792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:58.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:58.984128+0000) 2023-08-28T00:39:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:58 smithi093 ceph-mon[111335]: pgmap v6792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:39:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:58 smithi093 ceph-mon[118902]: pgmap v6792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:00.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:39:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1440972732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:00.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:39:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:39:59.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:39:59.984339+0000) 2023-08-28T00:40:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:39:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1440972732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:39:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1440972732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:00 smithi136 ceph-mon[113531]: pgmap v6793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:40:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:00.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:00.984581+0000) 2023-08-28T00:40:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:00 smithi093 ceph-mon[111335]: pgmap v6793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:40:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:00 smithi093 ceph-mon[118902]: pgmap v6793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:40:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:01.089Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:01.090Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:01.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:01.984839+0000) 2023-08-28T00:40:02.866 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:02.544Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:02.866 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:02.545Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:02 smithi136 ceph-mon[113531]: pgmap v6794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3844565741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:02.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:02.985109+0000) 2023-08-28T00:40:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:02 smithi093 ceph-mon[111335]: pgmap v6794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3844565741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:02 smithi093 ceph-mon[118902]: pgmap v6794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3844565741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:03.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:03.985365+0000) 2023-08-28T00:40:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:04 smithi136 ceph-mon[113531]: pgmap v6795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3305311865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:04.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:04.985620+0000) 2023-08-28T00:40:05.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:04 smithi093 ceph-mon[111335]: pgmap v6795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:05.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3305311865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:05.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:04 smithi093 ceph-mon[118902]: pgmap v6795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:05.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3305311865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:40:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:40:05] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:40:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:05.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:05.985886+0000) 2023-08-28T00:40:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:06 smithi136 ceph-mon[113531]: pgmap v6796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/507759751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:06.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:06.986179+0000) 2023-08-28T00:40:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:06 smithi093 ceph-mon[111335]: pgmap v6796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/507759751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:06 smithi093 ceph-mon[118902]: pgmap v6796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/507759751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:07.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:07.986387+0000) 2023-08-28T00:40:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:08 smithi136 ceph-mon[113531]: pgmap v6797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:08.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:08.986639+0000) 2023-08-28T00:40:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:08 smithi093 ceph-mon[111335]: pgmap v6797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:08 smithi093 ceph-mon[118902]: pgmap v6797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3631521188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:09.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:09.986881+0000) 2023-08-28T00:40:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3631521188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3631521188' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:10 smithi136 ceph-mon[113531]: pgmap v6798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:10.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:10.987108+0000) 2023-08-28T00:40:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:10 smithi093 ceph-mon[111335]: pgmap v6798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:10 smithi093 ceph-mon[118902]: pgmap v6798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:11.090Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:11.091Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1474532826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:11.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:11.987324+0000) 2023-08-28T00:40:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1474532826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1474532826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:12.892 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:12.544Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:12.892 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:12.545Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:12 smithi136 ceph-mon[113531]: pgmap v6799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:12.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:12.987598+0000) 2023-08-28T00:40:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:12 smithi093 ceph-mon[111335]: pgmap v6799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:12 smithi093 ceph-mon[118902]: pgmap v6799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:13.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:13.987856+0000) 2023-08-28T00:40:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:14 smithi136 ceph-mon[113531]: pgmap v6800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2512525364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:14.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:14.988141+0000) 2023-08-28T00:40:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:14 smithi093 ceph-mon[111335]: pgmap v6800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2512525364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:14 smithi093 ceph-mon[118902]: pgmap v6800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2512525364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:40:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:40:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:40:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:15.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:15.988404+0000) 2023-08-28T00:40:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:16 smithi136 ceph-mon[113531]: pgmap v6801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/266911132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:16.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:16.988655+0000) 2023-08-28T00:40:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:16 smithi093 ceph-mon[111335]: pgmap v6801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/266911132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:16 smithi093 ceph-mon[118902]: pgmap v6801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/266911132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:17.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:17.988905+0000) 2023-08-28T00:40:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:18 smithi136 ceph-mon[113531]: pgmap v6802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:20.234 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/200449210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:20.234 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:18.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:18.989204+0000) 2023-08-28T00:40:20.235 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:18 smithi093 ceph-mon[111335]: pgmap v6802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:20.235 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/200449210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:20.235 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:18 smithi093 ceph-mon[118902]: pgmap v6802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:20.235 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/200449210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:19.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:19.989393+0000) 2023-08-28T00:40:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:20 smithi136 ceph-mon[113531]: pgmap v6803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:20.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:20.989649+0000) 2023-08-28T00:40:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:20 smithi093 ceph-mon[111335]: pgmap v6803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:20 smithi093 ceph-mon[118902]: pgmap v6803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:21.091Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:21.092Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2652701084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:21.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:21.989910+0000) 2023-08-28T00:40:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2652701084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2652701084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:22.914 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:22.545Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:22.914 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:22.546Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:22 smithi136 ceph-mon[113531]: pgmap v6804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:22.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:22.990117+0000) 2023-08-28T00:40:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:22 smithi093 ceph-mon[111335]: pgmap v6804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:22 smithi093 ceph-mon[118902]: pgmap v6804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2961075279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:23.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:23.990367+0000) 2023-08-28T00:40:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2961075279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2961075279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:24 smithi136 ceph-mon[113531]: pgmap v6805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:24.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:24.990646+0000) 2023-08-28T00:40:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:24 smithi093 ceph-mon[111335]: pgmap v6805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:24 smithi093 ceph-mon[118902]: pgmap v6805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:40:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:40:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:40:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:25.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:25.990873+0000) 2023-08-28T00:40:27.070 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:26 smithi093 ceph-mon[111335]: pgmap v6806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:27.070 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/262930243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:27.070 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:40:27.070 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:26 smithi093 ceph-mon[118902]: pgmap v6806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:27.071 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/262930243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:27.071 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:40:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:26 smithi136 ceph-mon[113531]: pgmap v6806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/262930243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:40:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:26.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:26.991089+0000) 2023-08-28T00:40:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:27.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:27.991239+0000) 2023-08-28T00:40:29.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:28.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:28.991386+0000) 2023-08-28T00:40:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:28 smithi136 ceph-mon[113531]: pgmap v6807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3710728935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[111335]: pgmap v6807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3710728935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[118902]: pgmap v6807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3710728935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:29.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:29.991597+0000) 2023-08-28T00:40:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:40:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:40:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:40:30.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:40:30.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:40:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:31 smithi136 ceph-mon[113531]: pgmap v6808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3887723487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:30.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:30.991856+0000) 2023-08-28T00:40:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:31 smithi093 ceph-mon[111335]: pgmap v6808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3887723487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:31 smithi093 ceph-mon[118902]: pgmap v6808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3887723487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:31.092Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:31.093Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:31.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:31.992101+0000) 2023-08-28T00:40:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:32.545Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:32.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:32.546Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:33 smithi136 ceph-mon[113531]: pgmap v6809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1926884313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:32.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:32.992319+0000) 2023-08-28T00:40:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:33 smithi093 ceph-mon[111335]: pgmap v6809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1926884313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:33 smithi093 ceph-mon[118902]: pgmap v6809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:33.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1926884313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:33.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:33.992564+0000) 2023-08-28T00:40:35.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:35 smithi093 ceph-mon[111335]: pgmap v6810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:35 smithi093 ceph-mon[118902]: pgmap v6810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:35 smithi136 ceph-mon[113531]: pgmap v6810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:34.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:34.992781+0000) 2023-08-28T00:40:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:40:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:40:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:40:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3103043466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:35.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:35.993041+0000) 2023-08-28T00:40:36.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3103043466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3103043466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:37 smithi136 ceph-mon[113531]: pgmap v6811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:36.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:36.993272+0000) 2023-08-28T00:40:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:37 smithi093 ceph-mon[111335]: pgmap v6811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:37 smithi093 ceph-mon[118902]: pgmap v6811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1659425178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:37.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:37.993506+0000) 2023-08-28T00:40:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1659425178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1659425178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:39 smithi136 ceph-mon[113531]: pgmap v6812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:38.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:38.993701+0000) 2023-08-28T00:40:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:39 smithi093 ceph-mon[111335]: pgmap v6812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:39 smithi093 ceph-mon[118902]: pgmap v6812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:39.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:39.993983+0000) 2023-08-28T00:40:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:41 smithi136 ceph-mon[113531]: pgmap v6813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/818012730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:40.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:40.994320+0000) 2023-08-28T00:40:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:41 smithi093 ceph-mon[118902]: pgmap v6813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/818012730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:41 smithi093 ceph-mon[111335]: pgmap v6813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/818012730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:41.093Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:41.093Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:41.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:41.994564+0000) 2023-08-28T00:40:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:42.546Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:42.547Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:43 smithi136 ceph-mon[113531]: pgmap v6814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/149000485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:42.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:42.994807+0000) 2023-08-28T00:40:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:43 smithi093 ceph-mon[111335]: pgmap v6814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/149000485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:43 smithi093 ceph-mon[118902]: pgmap v6814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:43.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/149000485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:43.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:43.995074+0000) 2023-08-28T00:40:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:45 smithi093 ceph-mon[118902]: pgmap v6815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3424537962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:45 smithi093 ceph-mon[111335]: pgmap v6815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3424537962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:45 smithi136 ceph-mon[113531]: pgmap v6815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3424537962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:44.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:44.995298+0000) 2023-08-28T00:40:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:40:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:40:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:40:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:45.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:45.995516+0000) 2023-08-28T00:40:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:47 smithi136 ceph-mon[113531]: pgmap v6816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:46.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:46.995742+0000) 2023-08-28T00:40:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:47 smithi093 ceph-mon[111335]: pgmap v6816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:47 smithi093 ceph-mon[118902]: pgmap v6816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1051630670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:47.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:47.996000+0000) 2023-08-28T00:40:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1051630670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1051630670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:49 smithi136 ceph-mon[113531]: pgmap v6817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:48.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:48.996252+0000) 2023-08-28T00:40:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:49 smithi093 ceph-mon[111335]: pgmap v6817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:49 smithi093 ceph-mon[118902]: pgmap v6817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/749734833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:49.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:49.996496+0000) 2023-08-28T00:40:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/749734833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/749734833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:51 smithi136 ceph-mon[113531]: pgmap v6818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:50.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:50.996757+0000) 2023-08-28T00:40:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:51 smithi093 ceph-mon[111335]: pgmap v6818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:51 smithi093 ceph-mon[118902]: pgmap v6818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:51.093Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:51.094Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/317332934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:51.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:51.996995+0000) 2023-08-28T00:40:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/317332934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/317332934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:52.547Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:40:52.548Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:40:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:53 smithi136 ceph-mon[113531]: pgmap v6819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:52.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:52.997212+0000) 2023-08-28T00:40:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:53 smithi093 ceph-mon[111335]: pgmap v6819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:53 smithi093 ceph-mon[118902]: pgmap v6819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:53.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:53.997436+0000) 2023-08-28T00:40:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:40:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:55 smithi136 ceph-mon[113531]: pgmap v6820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3140030083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:54.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:54.997656+0000) 2023-08-28T00:40:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:40:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:40:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:40:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:55 smithi093 ceph-mon[111335]: pgmap v6820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3140030083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:55.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:55 smithi093 ceph-mon[118902]: pgmap v6820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:55.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3140030083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:55.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:55.997937+0000) 2023-08-28T00:40:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:57 smithi136 ceph-mon[113531]: pgmap v6821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2573075722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:56.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:56.998202+0000) 2023-08-28T00:40:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:57 smithi093 ceph-mon[111335]: pgmap v6821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2573075722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:57 smithi093 ceph-mon[118902]: pgmap v6821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2573075722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:40:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:57.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:57.998454+0000) 2023-08-28T00:40:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:40:59 smithi136 ceph-mon[113531]: pgmap v6822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:58.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:58.998717+0000) 2023-08-28T00:40:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:40:59 smithi093 ceph-mon[111335]: pgmap v6822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:40:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:40:59 smithi093 ceph-mon[118902]: pgmap v6822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2200676561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:40:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:40:59.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:40:59.998920+0000) 2023-08-28T00:41:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2200676561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2200676561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:01 smithi136 ceph-mon[113531]: pgmap v6823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:00.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:00.999167+0000) 2023-08-28T00:41:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:01 smithi093 ceph-mon[111335]: pgmap v6823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:01 smithi093 ceph-mon[118902]: pgmap v6823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:01.094Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:01.095Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/535428906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:01.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:01.999359+0000) 2023-08-28T00:41:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/535428906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/535428906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:02.548Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:02.549Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:03 smithi136 ceph-mon[113531]: pgmap v6824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:02.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:02.999641+0000) 2023-08-28T00:41:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:03 smithi093 ceph-mon[111335]: pgmap v6824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:03 smithi093 ceph-mon[118902]: pgmap v6824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4278562849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:03.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:03.999921+0000) 2023-08-28T00:41:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4278562849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4278562849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:05.318 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:05 smithi136 ceph-mon[113531]: pgmap v6825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:05.319 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:04.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:05.000181+0000) 2023-08-28T00:41:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:05 smithi093 ceph-mon[111335]: pgmap v6825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:41:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:41:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:41:05.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:05 smithi093 ceph-mon[118902]: pgmap v6825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:05.690 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 00:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T00:41:05.31796185Z level=info msg="Completed cleanup jobs" duration=13.360983ms 2023-08-28T00:41:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:05.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:06.000444+0000) 2023-08-28T00:41:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:07 smithi136 ceph-mon[113531]: pgmap v6826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340413837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:06.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:07.000701+0000) 2023-08-28T00:41:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:07 smithi093 ceph-mon[111335]: pgmap v6826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340413837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:07 smithi093 ceph-mon[118902]: pgmap v6826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340413837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:07.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:08.000978+0000) 2023-08-28T00:41:09.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:09 smithi136 ceph-mon[113531]: pgmap v6827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:09.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2004235045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:09.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:09.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:09.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:09.001230+0000) 2023-08-28T00:41:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:09 smithi093 ceph-mon[111335]: pgmap v6827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2004235045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:09 smithi093 ceph-mon[118902]: pgmap v6827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2004235045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:09.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:10.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:10.001456+0000) 2023-08-28T00:41:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:11 smithi136 ceph-mon[113531]: pgmap v6828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/338619252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:11.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:11.001682+0000) 2023-08-28T00:41:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:11 smithi093 ceph-mon[111335]: pgmap v6828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/338619252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:11 smithi093 ceph-mon[118902]: pgmap v6828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/338619252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:11.094Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:11.095Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:12.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:12.001945+0000) 2023-08-28T00:41:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:12.549Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:12.550Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:13 smithi136 ceph-mon[113531]: pgmap v6829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:13.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:13.002168+0000) 2023-08-28T00:41:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:13 smithi093 ceph-mon[111335]: pgmap v6829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:13 smithi093 ceph-mon[118902]: pgmap v6829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2615885027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:14.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:14.002426+0000) 2023-08-28T00:41:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2615885027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2615885027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:15 smithi136 ceph-mon[113531]: pgmap v6830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:15.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:15.002628+0000) 2023-08-28T00:41:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:15 smithi093 ceph-mon[111335]: pgmap v6830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:15 smithi093 ceph-mon[118902]: pgmap v6830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:41:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:41:15] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:41:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:16.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:16.002880+0000) 2023-08-28T00:41:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:16 smithi136 ceph-mon[113531]: pgmap v6831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1011254938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:17.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:17.003121+0000) 2023-08-28T00:41:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:16 smithi093 ceph-mon[111335]: pgmap v6831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1011254938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:16 smithi093 ceph-mon[118902]: pgmap v6831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1011254938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:18.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:18.003349+0000) 2023-08-28T00:41:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:18 smithi136 ceph-mon[113531]: pgmap v6832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/304662700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:19.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:19.003606+0000) 2023-08-28T00:41:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:18 smithi093 ceph-mon[111335]: pgmap v6832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/304662700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:18 smithi093 ceph-mon[118902]: pgmap v6832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/304662700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:20.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:20.003860+0000) 2023-08-28T00:41:21.096 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:20 smithi093 ceph-mon[118902]: pgmap v6833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:21.096 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2472443103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:21.097 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:20 smithi093 ceph-mon[111335]: pgmap v6833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:21.097 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2472443103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:20 smithi136 ceph-mon[113531]: pgmap v6833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2472443103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:21.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:21.004084+0000) 2023-08-28T00:41:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:21.095Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:21.096Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:22.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:22.004323+0000) 2023-08-28T00:41:22.818 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:22.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:22.818 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:22.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:22 smithi136 ceph-mon[113531]: pgmap v6834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:23.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:23.004543+0000) 2023-08-28T00:41:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:22 smithi093 ceph-mon[111335]: pgmap v6834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:22 smithi093 ceph-mon[118902]: pgmap v6834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020934427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:24.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:24.004800+0000) 2023-08-28T00:41:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020934427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020934427' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:24 smithi136 ceph-mon[113531]: pgmap v6835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:25.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:25.005008+0000) 2023-08-28T00:41:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:24 smithi093 ceph-mon[111335]: pgmap v6835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:24 smithi093 ceph-mon[118902]: pgmap v6835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:41:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:41:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:41:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2718469418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:26.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:26.005277+0000) 2023-08-28T00:41:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2718469418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2718469418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:26 smithi136 ceph-mon[113531]: pgmap v6836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:27.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:27.005562+0000) 2023-08-28T00:41:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:26 smithi093 ceph-mon[111335]: pgmap v6836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:26 smithi093 ceph-mon[118902]: pgmap v6836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:28.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:28.005822+0000) 2023-08-28T00:41:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:28 smithi136 ceph-mon[113531]: pgmap v6837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4173265907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:29.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:29.006001+0000) 2023-08-28T00:41:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:28 smithi093 ceph-mon[111335]: pgmap v6837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4173265907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:28 smithi093 ceph-mon[118902]: pgmap v6837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4173265907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:41:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:30.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:30.006127+0000) 2023-08-28T00:41:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:41:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:41:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:30 smithi136 ceph-mon[113531]: pgmap v6838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:30.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2269920751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:31.093 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:30 smithi093 ceph-mon[111335]: pgmap v6838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:31.094 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2269920751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:31.094 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:30 smithi093 ceph-mon[118902]: pgmap v6838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:31.094 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2269920751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:31.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:31.006261+0000) 2023-08-28T00:41:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:31.096Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:31.097Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:32.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:32.006425+0000) 2023-08-28T00:41:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:32 smithi136 ceph-mon[113531]: pgmap v6839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[118902]: pgmap v6839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:32.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:32.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:32.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:32.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[111335]: pgmap v6839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:32.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:32.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:33.347 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:33.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:33.006678+0000) 2023-08-28T00:41:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:41:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:41:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1374390348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:41:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:41:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1374390348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:33.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:41:33.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:41:33.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:41:33.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1374390348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:34.354 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:34.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:34.006917+0000) 2023-08-28T00:41:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:34 smithi136 ceph-mon[113531]: pgmap v6840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:34.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:34 smithi093 ceph-mon[111335]: pgmap v6840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:34 smithi093 ceph-mon[118902]: pgmap v6840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:35.355 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:35.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:35.007170+0000) 2023-08-28T00:41:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/560389203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:41:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:41:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:41:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/560389203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/560389203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:36.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:36.007406+0000) 2023-08-28T00:41:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:36 smithi136 ceph-mon[113531]: pgmap v6841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:37.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:37.007673+0000) 2023-08-28T00:41:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:36 smithi093 ceph-mon[111335]: pgmap v6841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:36 smithi093 ceph-mon[118902]: pgmap v6841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1498087895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:38.193 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:38.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:38.007862+0000) 2023-08-28T00:41:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1498087895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1498087895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:38 smithi136 ceph-mon[113531]: pgmap v6842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:39.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:39.008136+0000) 2023-08-28T00:41:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:38 smithi093 ceph-mon[111335]: pgmap v6842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:38 smithi093 ceph-mon[118902]: pgmap v6842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/249875611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:40.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:40.008433+0000) 2023-08-28T00:41:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/249875611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/249875611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:41.098 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:40 smithi093 ceph-mon[111335]: pgmap v6843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:41.098 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:40 smithi093 ceph-mon[118902]: pgmap v6843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:40 smithi136 ceph-mon[113531]: pgmap v6843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:41.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:41.008692+0000) 2023-08-28T00:41:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:41.097Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:41.098Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:42.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:42.008929+0000) 2023-08-28T00:41:42.833 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:42.551Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:42.833 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:42.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:43.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:43.009185+0000) 2023-08-28T00:41:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:42 smithi136 ceph-mon[113531]: pgmap v6844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2314247842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:42 smithi093 ceph-mon[111335]: pgmap v6844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2314247842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:42 smithi093 ceph-mon[118902]: pgmap v6844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2314247842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:44.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:44.009430+0000) 2023-08-28T00:41:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:44 smithi136 ceph-mon[113531]: pgmap v6845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/456335399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:45.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:45.009672+0000) 2023-08-28T00:41:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:44 smithi093 ceph-mon[111335]: pgmap v6845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/456335399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:44 smithi093 ceph-mon[118902]: pgmap v6845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/456335399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:41:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:41:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:41:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:46.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:46.009905+0000) 2023-08-28T00:41:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:46 smithi136 ceph-mon[113531]: pgmap v6846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:47.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:47.010145+0000) 2023-08-28T00:41:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:46 smithi093 ceph-mon[111335]: pgmap v6846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:46 smithi093 ceph-mon[118902]: pgmap v6846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:48.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1447906739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:48.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:48.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:48.010396+0000) 2023-08-28T00:41:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1447906739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1447906739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:48 smithi136 ceph-mon[113531]: pgmap v6847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:49.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:49.010643+0000) 2023-08-28T00:41:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:48 smithi093 ceph-mon[111335]: pgmap v6847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:48 smithi093 ceph-mon[118902]: pgmap v6847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2175104121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:50.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:50.010815+0000) 2023-08-28T00:41:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2175104121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2175104121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:50 smithi136 ceph-mon[113531]: pgmap v6848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:51.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:51.011083+0000) 2023-08-28T00:41:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:50 smithi093 ceph-mon[111335]: pgmap v6848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:50 smithi093 ceph-mon[118902]: pgmap v6848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:51.098Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:51.099Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2737134698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:52.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:52.011327+0000) 2023-08-28T00:41:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2737134698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2737134698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:52.866 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:52.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:52.866 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:41:52.553Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:41:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:52 smithi136 ceph-mon[113531]: pgmap v6849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:53.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:53.011543+0000) 2023-08-28T00:41:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:52 smithi093 ceph-mon[111335]: pgmap v6849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:52 smithi093 ceph-mon[118902]: pgmap v6849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:54.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:54.011820+0000) 2023-08-28T00:41:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:41:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:54 smithi136 ceph-mon[113531]: pgmap v6850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/851558412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:55.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:55.012074+0000) 2023-08-28T00:41:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:54 smithi093 ceph-mon[111335]: pgmap v6850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/851558412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:54 smithi093 ceph-mon[118902]: pgmap v6850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:55.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/851558412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:41:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:41:55] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:41:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:56.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:56.012328+0000) 2023-08-28T00:41:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:56 smithi136 ceph-mon[113531]: pgmap v6851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2378012886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:57.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:57.012638+0000) 2023-08-28T00:41:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:56 smithi093 ceph-mon[111335]: pgmap v6851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2378012886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:56 smithi093 ceph-mon[118902]: pgmap v6851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2378012886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:58.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:58.012926+0000) 2023-08-28T00:41:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:58 smithi136 ceph-mon[113531]: pgmap v6852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:41:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4105705404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:41:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:41:59.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:41:59.013255+0000) 2023-08-28T00:41:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:58 smithi093 ceph-mon[111335]: pgmap v6852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:41:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4105705404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:41:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:58 smithi093 ceph-mon[118902]: pgmap v6852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:41:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:41:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4105705404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:00.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:00.013520+0000) 2023-08-28T00:42:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:00 smithi136 ceph-mon[113531]: pgmap v6853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:01.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:01.013755+0000) 2023-08-28T00:42:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:00 smithi093 ceph-mon[111335]: pgmap v6853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:00 smithi093 ceph-mon[118902]: pgmap v6853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:01.099Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:01.099Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1899337292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:02.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:02.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:02.014010+0000) 2023-08-28T00:42:02.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1899337292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:02.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1899337292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:02.889 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:02.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:02.889 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:02.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:02 smithi136 ceph-mon[113531]: pgmap v6854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:03.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:03.014331+0000) 2023-08-28T00:42:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:02 smithi093 ceph-mon[111335]: pgmap v6854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:02 smithi093 ceph-mon[118902]: pgmap v6854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3136028165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:04.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:04.014615+0000) 2023-08-28T00:42:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3136028165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3136028165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:04 smithi136 ceph-mon[113531]: pgmap v6855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:05.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:05.014891+0000) 2023-08-28T00:42:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:04 smithi093 ceph-mon[111335]: pgmap v6855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:04 smithi093 ceph-mon[118902]: pgmap v6855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:42:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:42:05] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:42:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:06.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:06.015181+0000) 2023-08-28T00:42:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:06 smithi136 ceph-mon[113531]: pgmap v6856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/958519896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:07.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:07.015458+0000) 2023-08-28T00:42:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:06 smithi093 ceph-mon[111335]: pgmap v6856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/958519896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:06 smithi093 ceph-mon[118902]: pgmap v6856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/958519896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:08.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:08.015787+0000) 2023-08-28T00:42:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:08 smithi136 ceph-mon[113531]: pgmap v6857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3185515241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:09.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:09.016113+0000) 2023-08-28T00:42:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:08 smithi093 ceph-mon[111335]: pgmap v6857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3185515241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:08 smithi093 ceph-mon[118902]: pgmap v6857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3185515241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:10.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:10.016375+0000) 2023-08-28T00:42:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:10 smithi136 ceph-mon[113531]: pgmap v6858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/89758680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:11.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:11.016683+0000) 2023-08-28T00:42:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:10 smithi093 ceph-mon[118902]: pgmap v6858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/89758680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:10 smithi093 ceph-mon[111335]: pgmap v6858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/89758680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:11.099Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:11.100Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:12.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:12.016947+0000) 2023-08-28T00:42:12.905 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:12.553Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:12.905 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:12.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:13.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:12 smithi136 ceph-mon[113531]: pgmap v6859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:13.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:13.017234+0000) 2023-08-28T00:42:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:12 smithi093 ceph-mon[111335]: pgmap v6859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:12 smithi093 ceph-mon[118902]: pgmap v6859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2034535029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:14.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:14.017477+0000) 2023-08-28T00:42:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2034535029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2034535029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:14 smithi136 ceph-mon[113531]: pgmap v6860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:15.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:15.017736+0000) 2023-08-28T00:42:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:14 smithi093 ceph-mon[111335]: pgmap v6860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:14 smithi093 ceph-mon[118902]: pgmap v6860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:42:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:42:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:42:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1144034482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:16.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:16.018004+0000) 2023-08-28T00:42:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1144034482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1144034482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:16 smithi136 ceph-mon[113531]: pgmap v6861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:17.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:17.018288+0000) 2023-08-28T00:42:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:16 smithi093 ceph-mon[111335]: pgmap v6861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:16 smithi093 ceph-mon[118902]: pgmap v6861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:18.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:18.018579+0000) 2023-08-28T00:42:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:18 smithi136 ceph-mon[113531]: pgmap v6862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/964800271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:19.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:19.018820+0000) 2023-08-28T00:42:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:18 smithi093 ceph-mon[111335]: pgmap v6862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/964800271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:18 smithi093 ceph-mon[118902]: pgmap v6862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/964800271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:20.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:20.019046+0000) 2023-08-28T00:42:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:20 smithi136 ceph-mon[113531]: pgmap v6863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1465458339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:21.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:21.019285+0000) 2023-08-28T00:42:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:20 smithi093 ceph-mon[111335]: pgmap v6863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1465458339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:20 smithi093 ceph-mon[118902]: pgmap v6863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1465458339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:21.100Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:21.101Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:22.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:22.019539+0000) 2023-08-28T00:42:22.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:22.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:22.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:22.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:22 smithi136 ceph-mon[113531]: pgmap v6864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:23.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3639641229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:23.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:23.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:23.019840+0000) 2023-08-28T00:42:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:22 smithi093 ceph-mon[118902]: pgmap v6864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3639641229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:23.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:22 smithi093 ceph-mon[111335]: pgmap v6864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:23.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3639641229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:24.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:24.020153+0000) 2023-08-28T00:42:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:24 smithi093 ceph-mon[111335]: pgmap v6865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:24 smithi093 ceph-mon[118902]: pgmap v6865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:24 smithi136 ceph-mon[113531]: pgmap v6865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:25.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:25.020442+0000) 2023-08-28T00:42:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:42:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:42:25] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:42:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4276905100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4276905100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4276905100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:26.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:26.020756+0000) 2023-08-28T00:42:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:26 smithi093 ceph-mon[111335]: pgmap v6866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:26 smithi093 ceph-mon[118902]: pgmap v6866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:26 smithi136 ceph-mon[113531]: pgmap v6866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:27.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:27.021028+0000) 2023-08-28T00:42:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2887391499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2887391499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2887391499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:28.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:28.021277+0000) 2023-08-28T00:42:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:28 smithi093 ceph-mon[111335]: pgmap v6867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:28 smithi093 ceph-mon[118902]: pgmap v6867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:28 smithi136 ceph-mon[113531]: pgmap v6867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:29.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:29.021466+0000) 2023-08-28T00:42:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/529075308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/529075308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/529075308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:30.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:30.021757+0000) 2023-08-28T00:42:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:30 smithi093 ceph-mon[111335]: pgmap v6868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:30 smithi093 ceph-mon[118902]: pgmap v6868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:31.100Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:31.101Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:31.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:30 smithi136 ceph-mon[113531]: pgmap v6868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:31.443 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:31.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:31.022078+0000) 2023-08-28T00:42:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:32.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:32.022370+0000) 2023-08-28T00:42:32.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:32.555Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:32.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:32.556Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-mon[111335]: pgmap v6869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1371436170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:42:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-mon[118902]: pgmap v6869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1371436170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:42:33.344 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:32 smithi136 ceph-mon[113531]: pgmap v6869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:33.345 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1371436170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:33.345 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:42:33.345 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:33.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:33.022530+0000) 2023-08-28T00:42:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:34.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:34.022684+0000) 2023-08-28T00:42:35.089 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:34 smithi136 ceph-mon[113531]: pgmap v6870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:35.090 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:35.090 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:35.090 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2297886086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:35.090 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:35.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:35.022866+0000) 2023-08-28T00:42:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[118902]: pgmap v6870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2297886086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[111335]: pgmap v6870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:35.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2297886086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:42:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:42:35] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:42:36.331 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:36.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:36.023140+0000) 2023-08-28T00:42:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:42:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:42:36.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:42:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:42:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:36.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:42:36.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:42:36.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:42:37.338 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:37.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:37.023374+0000) 2023-08-28T00:42:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:37 smithi136 ceph-mon[113531]: pgmap v6871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:37.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3263252618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:37 smithi093 ceph-mon[111335]: pgmap v6871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:37.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3263252618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:37.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:37 smithi093 ceph-mon[118902]: pgmap v6871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:37.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3263252618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:38.344 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:38.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:38.023651+0000) 2023-08-28T00:42:38.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:38 smithi136 ceph-mon[113531]: pgmap v6872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:38.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:38 smithi093 ceph-mon[111335]: pgmap v6872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:38.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:38 smithi093 ceph-mon[118902]: pgmap v6872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:39.344 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:39.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:39.023883+0000) 2023-08-28T00:42:39.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:39.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:40.346 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:40.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:40.024094+0000) 2023-08-28T00:42:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3737931710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:40.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:40 smithi136 ceph-mon[113531]: pgmap v6873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3737931710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:40.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:40 smithi093 ceph-mon[111335]: pgmap v6873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:40.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3737931710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:40.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:40 smithi093 ceph-mon[118902]: pgmap v6873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:41.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:41.024321+0000) 2023-08-28T00:42:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:41.102Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:41.103Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1994426452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:42.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:42.024550+0000) 2023-08-28T00:42:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1994426452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1994426452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:42.832 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:42.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:42.832 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:42.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:42 smithi136 ceph-mon[113531]: pgmap v6874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:43.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:43.024759+0000) 2023-08-28T00:42:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:42 smithi093 ceph-mon[111335]: pgmap v6874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:42 smithi093 ceph-mon[118902]: pgmap v6874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:44.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:44.025023+0000) 2023-08-28T00:42:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:45.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:45.025297+0000) 2023-08-28T00:42:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:44 smithi136 ceph-mon[113531]: pgmap v6875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/478557670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:44 smithi093 ceph-mon[111335]: pgmap v6875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/478557670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:44 smithi093 ceph-mon[118902]: pgmap v6875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:45.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/478557670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:42:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:42:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:42:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:46.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:46.025557+0000) 2023-08-28T00:42:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:46 smithi136 ceph-mon[113531]: pgmap v6876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2761451500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:47.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:47.025778+0000) 2023-08-28T00:42:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:46 smithi093 ceph-mon[118902]: pgmap v6876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2761451500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:46 smithi093 ceph-mon[111335]: pgmap v6876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2761451500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:48.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:48.025982+0000) 2023-08-28T00:42:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:48 smithi136 ceph-mon[113531]: pgmap v6877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:49.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:49.026243+0000) 2023-08-28T00:42:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:48 smithi093 ceph-mon[111335]: pgmap v6877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:48 smithi093 ceph-mon[118902]: pgmap v6877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:50.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/376724800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:50.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:50.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:50.026498+0000) 2023-08-28T00:42:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/376724800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/376724800' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:51.103 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:50 smithi093 ceph-mon[111335]: pgmap v6878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:51.103 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:50 smithi093 ceph-mon[118902]: pgmap v6878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:51.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:50 smithi136 ceph-mon[113531]: pgmap v6878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:51.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:51.026791+0000) 2023-08-28T00:42:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:51.102Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:51.103Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:52.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111208884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:52.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:52.027057+0000) 2023-08-28T00:42:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111208884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111208884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:52.852 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:52.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:52.852 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:42:52.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:42:53.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:52 smithi136 ceph-mon[113531]: pgmap v6879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:53.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:53.027259+0000) 2023-08-28T00:42:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:52 smithi093 ceph-mon[111335]: pgmap v6879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:52 smithi093 ceph-mon[118902]: pgmap v6879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2829391331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:54.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:54.027491+0000) 2023-08-28T00:42:54.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2829391331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:54.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2829391331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:54.243 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:42:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:54 smithi136 ceph-mon[113531]: pgmap v6880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:55.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:55.027702+0000) 2023-08-28T00:42:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:54 smithi093 ceph-mon[111335]: pgmap v6880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:54 smithi093 ceph-mon[118902]: pgmap v6880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:42:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:42:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:42:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:56.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:56.027968+0000) 2023-08-28T00:42:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:56 smithi136 ceph-mon[113531]: pgmap v6881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/403725006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:57.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:57.028248+0000) 2023-08-28T00:42:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:56 smithi093 ceph-mon[111335]: pgmap v6881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/403725006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:56 smithi093 ceph-mon[118902]: pgmap v6881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/403725006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:58.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:58.028506+0000) 2023-08-28T00:42:59.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:58 smithi136 ceph-mon[113531]: pgmap v6882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:42:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3195350849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:42:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:42:59.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:42:59.028753+0000) 2023-08-28T00:42:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:58 smithi093 ceph-mon[111335]: pgmap v6882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:42:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3195350849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:42:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:58 smithi093 ceph-mon[118902]: pgmap v6882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:42:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:42:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3195350849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:00.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:00.029067+0000) 2023-08-28T00:43:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:00 smithi136 ceph-mon[113531]: pgmap v6883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1761847681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:01.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:01.029287+0000) 2023-08-28T00:43:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:00 smithi093 ceph-mon[111335]: pgmap v6883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1761847681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:00 smithi093 ceph-mon[118902]: pgmap v6883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1761847681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:01.103Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:01.103Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:02.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:02.029533+0000) 2023-08-28T00:43:02.871 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:02.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:02.871 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:02.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:02 smithi136 ceph-mon[113531]: pgmap v6884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:03.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:03.029782+0000) 2023-08-28T00:43:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:02 smithi093 ceph-mon[111335]: pgmap v6884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:02 smithi093 ceph-mon[118902]: pgmap v6884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/871956969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:04.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:04.030063+0000) 2023-08-28T00:43:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/871956969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/871956969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:04 smithi136 ceph-mon[113531]: pgmap v6885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:05.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:05.030356+0000) 2023-08-28T00:43:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:04 smithi093 ceph-mon[111335]: pgmap v6885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:04 smithi093 ceph-mon[118902]: pgmap v6885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:43:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:43:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:43:06.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/646772326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:06.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:06.030640+0000) 2023-08-28T00:43:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/646772326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/646772326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:06 smithi136 ceph-mon[113531]: pgmap v6886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:07.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:07.030902+0000) 2023-08-28T00:43:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:06 smithi093 ceph-mon[111335]: pgmap v6886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:06 smithi093 ceph-mon[118902]: pgmap v6886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:08.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:08.031153+0000) 2023-08-28T00:43:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:08 smithi136 ceph-mon[113531]: pgmap v6887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1081712073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:09.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:09.031393+0000) 2023-08-28T00:43:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:08 smithi093 ceph-mon[111335]: pgmap v6887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1081712073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:08 smithi093 ceph-mon[118902]: pgmap v6887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1081712073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:10.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:10.031605+0000) 2023-08-28T00:43:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:10 smithi136 ceph-mon[113531]: pgmap v6888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4261226846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:11.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:11.031864+0000) 2023-08-28T00:43:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:10 smithi093 ceph-mon[111335]: pgmap v6888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4261226846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:10 smithi093 ceph-mon[118902]: pgmap v6888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4261226846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:11.103Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:11.104Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:12.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:12.032111+0000) 2023-08-28T00:43:12.899 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:12.558Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:12.899 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:12.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:12 smithi136 ceph-mon[113531]: pgmap v6889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3536208790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:13.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:13.032317+0000) 2023-08-28T00:43:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:12 smithi093 ceph-mon[111335]: pgmap v6889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3536208790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:12 smithi093 ceph-mon[118902]: pgmap v6889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3536208790' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:14.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:14.032509+0000) 2023-08-28T00:43:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:15.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:15.032746+0000) 2023-08-28T00:43:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:14 smithi136 ceph-mon[113531]: pgmap v6890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:14 smithi093 ceph-mon[111335]: pgmap v6890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:14 smithi093 ceph-mon[118902]: pgmap v6890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:43:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:43:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:43:16.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2825872229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:16.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:16.033008+0000) 2023-08-28T00:43:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2825872229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2825872229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:16 smithi136 ceph-mon[113531]: pgmap v6891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:17.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:17.033291+0000) 2023-08-28T00:43:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:16 smithi093 ceph-mon[111335]: pgmap v6891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:16 smithi093 ceph-mon[118902]: pgmap v6891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:18.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3108159852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:18.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:18.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:18.033523+0000) 2023-08-28T00:43:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3108159852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3108159852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:18 smithi136 ceph-mon[113531]: pgmap v6892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:19.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:19.033750+0000) 2023-08-28T00:43:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:18 smithi093 ceph-mon[111335]: pgmap v6892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:18 smithi093 ceph-mon[118902]: pgmap v6892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:20.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1830978452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:20.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:20.034003+0000) 2023-08-28T00:43:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1830978452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1830978452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:20 smithi136 ceph-mon[113531]: pgmap v6893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:21.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:21.034283+0000) 2023-08-28T00:43:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:20 smithi093 ceph-mon[111335]: pgmap v6893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:20 smithi093 ceph-mon[118902]: pgmap v6893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:21.104Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:21.105Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:22.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:22.034571+0000) 2023-08-28T00:43:22.924 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:22.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:22.924 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:22.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:22 smithi136 ceph-mon[113531]: pgmap v6894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3450544755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:23.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:23.034783+0000) 2023-08-28T00:43:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:22 smithi093 ceph-mon[111335]: pgmap v6894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3450544755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:22 smithi093 ceph-mon[118902]: pgmap v6894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3450544755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:24.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:24.035021+0000) 2023-08-28T00:43:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:24 smithi136 ceph-mon[113531]: pgmap v6895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:25.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2330760309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:25.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:25.035286+0000) 2023-08-28T00:43:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:24 smithi093 ceph-mon[111335]: pgmap v6895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2330760309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:24 smithi093 ceph-mon[118902]: pgmap v6895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2330760309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:43:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:43:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:43:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:26.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:26.035542+0000) 2023-08-28T00:43:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:26 smithi136 ceph-mon[113531]: pgmap v6896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:27.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:27.035794+0000) 2023-08-28T00:43:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:26 smithi093 ceph-mon[111335]: pgmap v6896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:26 smithi093 ceph-mon[118902]: pgmap v6896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2573842707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2573842707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2573842707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:28.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:28.036015+0000) 2023-08-28T00:43:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:28 smithi093 ceph-mon[111335]: pgmap v6897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:28 smithi093 ceph-mon[118902]: pgmap v6897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:28 smithi136 ceph-mon[113531]: pgmap v6897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:29.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:29.036285+0000) 2023-08-28T00:43:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2602618836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2602618836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2602618836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:30.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:30.036543+0000) 2023-08-28T00:43:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:30 smithi093 ceph-mon[111335]: pgmap v6898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:30 smithi093 ceph-mon[118902]: pgmap v6898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:31.105Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:31.106Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:30 smithi136 ceph-mon[113531]: pgmap v6898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:31.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:31.036770+0000) 2023-08-28T00:43:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/150726546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/150726546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/150726546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:32.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:32.036997+0000) 2023-08-28T00:43:32.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:32.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:32.961 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:32.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:32 smithi093 ceph-mon[118902]: pgmap v6899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:32 smithi093 ceph-mon[111335]: pgmap v6899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:32 smithi136 ceph-mon[113531]: pgmap v6899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:33.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:33.037268+0000) 2023-08-28T00:43:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:34.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:34.037473+0000) 2023-08-28T00:43:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:34 smithi093 ceph-mon[111335]: pgmap v6900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4066832364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:34 smithi093 ceph-mon[118902]: pgmap v6900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4066832364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:34 smithi136 ceph-mon[113531]: pgmap v6900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4066832364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:35.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:35.037757+0000) 2023-08-28T00:43:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:43:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:43:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:43:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:36.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:36.037948+0000) 2023-08-28T00:43:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:36 smithi093 ceph-mon[118902]: pgmap v6901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:43:36.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2982722726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:36 smithi093 ceph-mon[111335]: pgmap v6901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:43:36.991 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2982722726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:36 smithi136 ceph-mon[113531]: pgmap v6901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:43:37.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2982722726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:37.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:37.038124+0000) 2023-08-28T00:43:38.149 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:38.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:38.038255+0000) 2023-08-28T00:43:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:39.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:39.038478+0000) 2023-08-28T00:43:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:38 smithi136 ceph-mon[113531]: pgmap v6902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[111335]: pgmap v6902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[118902]: pgmap v6902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:39.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:43:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:43:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:40.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/417863202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:40.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:40.038696+0000) 2023-08-28T00:43:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:43:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:43:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:40.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/417863202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:43:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:43:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:43:40.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/417863202' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:41.107 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:40 smithi093 ceph-mon[111335]: pgmap v6903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:41.107 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:40 smithi093 ceph-mon[118902]: pgmap v6903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:41.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:40 smithi136 ceph-mon[113531]: pgmap v6903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:41.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:41.038975+0000) 2023-08-28T00:43:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:41.105Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:41.107Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:42.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2241953323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:42.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:42.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:42.039262+0000) 2023-08-28T00:43:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2241953323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2241953323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:42.857 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:42.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:42.857 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:42.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:42 smithi136 ceph-mon[113531]: pgmap v6904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:43.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:43.039533+0000) 2023-08-28T00:43:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:42 smithi093 ceph-mon[111335]: pgmap v6904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:42 smithi093 ceph-mon[118902]: pgmap v6904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:44.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3132291498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:44.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:44.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:44.039774+0000) 2023-08-28T00:43:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3132291498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3132291498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:44 smithi136 ceph-mon[113531]: pgmap v6905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:45.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:45.040029+0000) 2023-08-28T00:43:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:44 smithi093 ceph-mon[111335]: pgmap v6905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:44 smithi093 ceph-mon[118902]: pgmap v6905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:43:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:43:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:43:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:46.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:46.040338+0000) 2023-08-28T00:43:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:46 smithi136 ceph-mon[113531]: pgmap v6906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2735989302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:47.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:47.040642+0000) 2023-08-28T00:43:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:46 smithi093 ceph-mon[111335]: pgmap v6906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2735989302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:46 smithi093 ceph-mon[118902]: pgmap v6906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2735989302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:48.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:48.040890+0000) 2023-08-28T00:43:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:48 smithi136 ceph-mon[113531]: pgmap v6907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2932889521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:49.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:49.041129+0000) 2023-08-28T00:43:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:48 smithi093 ceph-mon[111335]: pgmap v6907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2932889521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:48 smithi093 ceph-mon[118902]: pgmap v6907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2932889521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:50.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:50.041385+0000) 2023-08-28T00:43:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:50 smithi136 ceph-mon[113531]: pgmap v6908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:51.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:51.041663+0000) 2023-08-28T00:43:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:50 smithi093 ceph-mon[111335]: pgmap v6908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:50 smithi093 ceph-mon[118902]: pgmap v6908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:51.107Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:51.108Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:52.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2993135911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:52.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:52.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:52.041936+0000) 2023-08-28T00:43:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2993135911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2993135911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:52.879 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:52.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:52.880 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:43:52.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:43:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:52 smithi136 ceph-mon[113531]: pgmap v6909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:53.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:53.042160+0000) 2023-08-28T00:43:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:52 smithi093 ceph-mon[111335]: pgmap v6909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:52 smithi093 ceph-mon[118902]: pgmap v6909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3880164050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:54.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:54.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:54.042368+0000) 2023-08-28T00:43:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3880164050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3880164050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:43:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:54 smithi136 ceph-mon[113531]: pgmap v6910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:55.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:55.042622+0000) 2023-08-28T00:43:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:54 smithi093 ceph-mon[111335]: pgmap v6910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:54 smithi093 ceph-mon[118902]: pgmap v6910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:43:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:43:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:43:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2807589191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:56.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:56.042872+0000) 2023-08-28T00:43:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2807589191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2807589191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:56 smithi136 ceph-mon[113531]: pgmap v6911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:57.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:57.043160+0000) 2023-08-28T00:43:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:56 smithi093 ceph-mon[111335]: pgmap v6911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:56 smithi093 ceph-mon[118902]: pgmap v6911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:58.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:58.043389+0000) 2023-08-28T00:43:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:58 smithi136 ceph-mon[113531]: pgmap v6912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:43:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2122058575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:43:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:43:59.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:43:59.043660+0000) 2023-08-28T00:43:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:58 smithi093 ceph-mon[111335]: pgmap v6912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:43:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2122058575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:43:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:58 smithi093 ceph-mon[118902]: pgmap v6912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:43:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:43:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2122058575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:00.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:00.043937+0000) 2023-08-28T00:44:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:00 smithi136 ceph-mon[113531]: pgmap v6913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4169325954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:01.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:01.044225+0000) 2023-08-28T00:44:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:00 smithi093 ceph-mon[111335]: pgmap v6913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4169325954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:00 smithi093 ceph-mon[118902]: pgmap v6913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4169325954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:01.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:01.107Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:01.108Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:02.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:02.044464+0000) 2023-08-28T00:44:02.905 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:02.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:02.906 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:02.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:02 smithi136 ceph-mon[113531]: pgmap v6914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2661227659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:03.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:03.044693+0000) 2023-08-28T00:44:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:02 smithi093 ceph-mon[111335]: pgmap v6914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2661227659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:02 smithi093 ceph-mon[118902]: pgmap v6914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2661227659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:04.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:04.044940+0000) 2023-08-28T00:44:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:04 smithi136 ceph-mon[113531]: pgmap v6915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:05.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:05.045222+0000) 2023-08-28T00:44:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:04 smithi093 ceph-mon[118902]: pgmap v6915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:04 smithi093 ceph-mon[111335]: pgmap v6915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:44:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:44:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:44:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3087386220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:06.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:06.045457+0000) 2023-08-28T00:44:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3087386220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3087386220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:06 smithi136 ceph-mon[113531]: pgmap v6916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:07.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:07.045698+0000) 2023-08-28T00:44:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:06 smithi093 ceph-mon[111335]: pgmap v6916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:06 smithi093 ceph-mon[118902]: pgmap v6916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3499022294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:08.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:08.045945+0000) 2023-08-28T00:44:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3499022294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3499022294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:08 smithi136 ceph-mon[113531]: pgmap v6917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:09.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:09.046178+0000) 2023-08-28T00:44:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:08 smithi093 ceph-mon[111335]: pgmap v6917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:08 smithi093 ceph-mon[118902]: pgmap v6917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4052695082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:10.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:10.046410+0000) 2023-08-28T00:44:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4052695082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4052695082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:10 smithi136 ceph-mon[113531]: pgmap v6918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:11.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:11.046633+0000) 2023-08-28T00:44:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:10 smithi093 ceph-mon[111335]: pgmap v6918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:10 smithi093 ceph-mon[118902]: pgmap v6918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:11.108Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:11.109Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:12.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:12.046867+0000) 2023-08-28T00:44:12.933 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:12.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:12.933 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:12.563Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:12 smithi136 ceph-mon[113531]: pgmap v6919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3072903229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:13.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:13.047136+0000) 2023-08-28T00:44:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:12 smithi093 ceph-mon[111335]: pgmap v6919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3072903229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:12 smithi093 ceph-mon[118902]: pgmap v6919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3072903229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:14.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:14.047326+0000) 2023-08-28T00:44:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:14 smithi136 ceph-mon[113531]: pgmap v6920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2667607341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:15.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:15.047577+0000) 2023-08-28T00:44:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:14 smithi093 ceph-mon[111335]: pgmap v6920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2667607341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:14 smithi093 ceph-mon[118902]: pgmap v6920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2667607341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:44:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:44:15] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:44:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:16.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:16.047850+0000) 2023-08-28T00:44:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:16 smithi136 ceph-mon[113531]: pgmap v6921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:17.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:17.048130+0000) 2023-08-28T00:44:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:16 smithi093 ceph-mon[111335]: pgmap v6921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:16 smithi093 ceph-mon[118902]: pgmap v6921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4148191852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4148191852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4148191852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:18.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:18.048367+0000) 2023-08-28T00:44:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:18 smithi093 ceph-mon[111335]: pgmap v6922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:18 smithi093 ceph-mon[118902]: pgmap v6922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:18 smithi136 ceph-mon[113531]: pgmap v6922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:19.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:19.048631+0000) 2023-08-28T00:44:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2485041851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2485041851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2485041851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:20.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:20.048914+0000) 2023-08-28T00:44:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:20 smithi093 ceph-mon[111335]: pgmap v6923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:20 smithi093 ceph-mon[118902]: pgmap v6923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:21.108Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:21.109Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:20 smithi136 ceph-mon[113531]: pgmap v6923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:21.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:21.049165+0000) 2023-08-28T00:44:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4174791452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:22.254 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4174791452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4174791452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:22.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:22.049415+0000) 2023-08-28T00:44:22.965 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:22.563Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:22.965 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:22.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:22 smithi093 ceph-mon[111335]: pgmap v6924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:22 smithi093 ceph-mon[118902]: pgmap v6924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:22 smithi136 ceph-mon[113531]: pgmap v6924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:23.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:23.049615+0000) 2023-08-28T00:44:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:24.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:24.049874+0000) 2023-08-28T00:44:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:24 smithi093 ceph-mon[111335]: pgmap v6925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/580512248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:24 smithi093 ceph-mon[118902]: pgmap v6925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/580512248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:24 smithi136 ceph-mon[113531]: pgmap v6925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/580512248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:25.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:25.050144+0000) 2023-08-28T00:44:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:44:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:44:25] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:44:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:26.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:26.050398+0000) 2023-08-28T00:44:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:26 smithi093 ceph-mon[111335]: pgmap v6926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1993215872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:26 smithi093 ceph-mon[118902]: pgmap v6926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1993215872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:26 smithi136 ceph-mon[113531]: pgmap v6926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1993215872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:27.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:27.050702+0000) 2023-08-28T00:44:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:28.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:28.050974+0000) 2023-08-28T00:44:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:28 smithi093 ceph-mon[111335]: pgmap v6927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3512263146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:28 smithi093 ceph-mon[118902]: pgmap v6927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3512263146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:28 smithi136 ceph-mon[113531]: pgmap v6927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3512263146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:29.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:29.051192+0000) 2023-08-28T00:44:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:30.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:30.051459+0000) 2023-08-28T00:44:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:30 smithi093 ceph-mon[111335]: pgmap v6928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:30 smithi093 ceph-mon[118902]: pgmap v6928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:31.109Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:31.110Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:30 smithi136 ceph-mon[113531]: pgmap v6928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:31.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:31.051717+0000) 2023-08-28T00:44:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/793827660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/793827660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/793827660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:32.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:32.051950+0000) 2023-08-28T00:44:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:32.564Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:32.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:32 smithi093 ceph-mon[111335]: pgmap v6929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:32 smithi093 ceph-mon[118902]: pgmap v6929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:32 smithi136 ceph-mon[113531]: pgmap v6929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:33.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:33.052194+0000) 2023-08-28T00:44:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/734949714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:34.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:34.052382+0000) 2023-08-28T00:44:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/734949714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/734949714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:34 smithi093 ceph-mon[111335]: pgmap v6930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:35.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:34 smithi093 ceph-mon[118902]: pgmap v6930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:34 smithi136 ceph-mon[113531]: pgmap v6930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:35.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:35.052629+0000) 2023-08-28T00:44:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:44:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:44:35] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:44:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:36.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:36.052903+0000) 2023-08-28T00:44:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:36 smithi136 ceph-mon[113531]: pgmap v6931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2870376690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:37.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:37.053174+0000) 2023-08-28T00:44:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:36 smithi093 ceph-mon[118902]: pgmap v6931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2870376690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:36 smithi093 ceph-mon[111335]: pgmap v6931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2870376690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:38.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:38.053424+0000) 2023-08-28T00:44:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:38 smithi136 ceph-mon[113531]: pgmap v6932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3091461804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:44:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:39.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:39.053589+0000) 2023-08-28T00:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[118902]: pgmap v6932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3091461804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:44:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[111335]: pgmap v6932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:39.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3091461804' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:39.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:39.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:44:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:40.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:40.053739+0000) 2023-08-28T00:44:41.434 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:41.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:41.053927+0000) 2023-08-28T00:44:41.434 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:41 smithi136 ceph-mon[113531]: pgmap v6933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:41.434 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:41.434 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:41.434 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2223691289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[111335]: pgmap v6933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2223691289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[118902]: pgmap v6933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2223691289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:41.109Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:41.110Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:42.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:42.054103+0000) 2023-08-28T00:44:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:42 smithi136 ceph-mon[113531]: pgmap v6934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:44:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:44:42.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[111335]: pgmap v6934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:44:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:44:42.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[118902]: pgmap v6934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:44:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:44:42.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:44:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:42.565Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:42.565Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:43.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:43.054326+0000) 2023-08-28T00:44:43.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3617606356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:43.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3617606356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3617606356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:44.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:44.054581+0000) 2023-08-28T00:44:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:44 smithi136 ceph-mon[113531]: pgmap v6935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:44 smithi093 ceph-mon[111335]: pgmap v6935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:44 smithi093 ceph-mon[118902]: pgmap v6935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:45.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:45.054826+0000) 2023-08-28T00:44:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:44:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:44:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:44:46.055 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1335489999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1335489999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1335489999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:46.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:46.055027+0000) 2023-08-28T00:44:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:46 smithi136 ceph-mon[113531]: pgmap v6936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:47.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:47.055263+0000) 2023-08-28T00:44:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:46 smithi093 ceph-mon[118902]: pgmap v6936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:46 smithi093 ceph-mon[111335]: pgmap v6936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:48.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:48.055469+0000) 2023-08-28T00:44:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:48 smithi136 ceph-mon[113531]: pgmap v6937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:49.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2196917346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:49.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:49.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:49.055733+0000) 2023-08-28T00:44:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:48 smithi093 ceph-mon[111335]: pgmap v6937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2196917346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:48 smithi093 ceph-mon[118902]: pgmap v6937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2196917346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:50.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:50.055978+0000) 2023-08-28T00:44:51.111 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:50 smithi093 ceph-mon[111335]: pgmap v6938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:51.111 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1400936636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:51.111 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:50 smithi093 ceph-mon[118902]: pgmap v6938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:51.111 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1400936636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:50 smithi136 ceph-mon[113531]: pgmap v6938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:51.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1400936636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:51.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:51.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:51.056251+0000) 2023-08-28T00:44:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:51.110Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:51.111Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:52.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:52.056509+0000) 2023-08-28T00:44:52.858 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:52.565Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:52.858 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:44:52.566Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:44:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:52 smithi136 ceph-mon[113531]: pgmap v6939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:53.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/194390439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:53.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:53.056756+0000) 2023-08-28T00:44:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:52 smithi093 ceph-mon[111335]: pgmap v6939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/194390439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:52 smithi093 ceph-mon[118902]: pgmap v6939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:53.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/194390439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:54.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:54.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:54.056960+0000) 2023-08-28T00:44:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:54.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:44:55.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:54 smithi136 ceph-mon[113531]: pgmap v6940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:55.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:55.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:55.057197+0000) 2023-08-28T00:44:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:54 smithi093 ceph-mon[111335]: pgmap v6940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:54 smithi093 ceph-mon[118902]: pgmap v6940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:44:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:44:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:44:56.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4053383154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:56.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:56.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:56.057428+0000) 2023-08-28T00:44:56.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4053383154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:56.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4053383154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:56 smithi136 ceph-mon[113531]: pgmap v6941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:57.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:57.057707+0000) 2023-08-28T00:44:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:56 smithi093 ceph-mon[111335]: pgmap v6941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:56 smithi093 ceph-mon[118902]: pgmap v6941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:58.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3995170450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:58.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:58.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:58.057912+0000) 2023-08-28T00:44:58.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3995170450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:58.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3995170450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:44:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:44:58 smithi136 ceph-mon[113531]: pgmap v6942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:44:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:44:59.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:44:59.058145+0000) 2023-08-28T00:44:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:44:58 smithi093 ceph-mon[111335]: pgmap v6942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:44:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:44:58 smithi093 ceph-mon[118902]: pgmap v6942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:00.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:00.058375+0000) 2023-08-28T00:45:01.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:00 smithi136 ceph-mon[113531]: pgmap v6943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/695682905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:01.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:01.058626+0000) 2023-08-28T00:45:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:00 smithi093 ceph-mon[111335]: pgmap v6943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/695682905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:00 smithi093 ceph-mon[118902]: pgmap v6943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/695682905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:01.110Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:01.112Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:02.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:02.058871+0000) 2023-08-28T00:45:02.888 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:02.566Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:02.889 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:02.567Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:02 smithi136 ceph-mon[113531]: pgmap v6944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/879826129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:03.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:03.059116+0000) 2023-08-28T00:45:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:02 smithi093 ceph-mon[111335]: pgmap v6944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/879826129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:02 smithi093 ceph-mon[118902]: pgmap v6944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/879826129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:04.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:04.059367+0000) 2023-08-28T00:45:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:05.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:05.059630+0000) 2023-08-28T00:45:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:04 smithi136 ceph-mon[113531]: pgmap v6945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:05.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4208253317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:04 smithi093 ceph-mon[111335]: pgmap v6945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4208253317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:04 smithi093 ceph-mon[118902]: pgmap v6945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4208253317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:45:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:45:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:45:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:06.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:06.059880+0000) 2023-08-28T00:45:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:06 smithi136 ceph-mon[113531]: pgmap v6946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:07.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:07.060150+0000) 2023-08-28T00:45:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:06 smithi093 ceph-mon[111335]: pgmap v6946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:06 smithi093 ceph-mon[118902]: pgmap v6946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/34200394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:08.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:08.060406+0000) 2023-08-28T00:45:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/34200394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/34200394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:08 smithi136 ceph-mon[113531]: pgmap v6947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:09.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:09.060636+0000) 2023-08-28T00:45:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:08 smithi093 ceph-mon[111335]: pgmap v6947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:08 smithi093 ceph-mon[118902]: pgmap v6947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1342681049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:10.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:10.060908+0000) 2023-08-28T00:45:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1342681049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1342681049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:10 smithi136 ceph-mon[113531]: pgmap v6948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:11.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:11.061188+0000) 2023-08-28T00:45:11.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:10 smithi093 ceph-mon[111335]: pgmap v6948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:11.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:10 smithi093 ceph-mon[118902]: pgmap v6948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:11.111Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:11.112Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:12.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2320487258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:12.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:12.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:12.061445+0000) 2023-08-28T00:45:12.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2320487258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:12.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2320487258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:12.913 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:12.566Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:12.913 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:12.567Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:12 smithi136 ceph-mon[113531]: pgmap v6949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:13.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:13.061719+0000) 2023-08-28T00:45:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:12 smithi093 ceph-mon[111335]: pgmap v6949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:12 smithi093 ceph-mon[118902]: pgmap v6949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:14.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:14.061941+0000) 2023-08-28T00:45:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:14 smithi136 ceph-mon[113531]: pgmap v6950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/299781844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:15.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:15.062221+0000) 2023-08-28T00:45:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:14 smithi093 ceph-mon[111335]: pgmap v6950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/299781844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:14 smithi093 ceph-mon[118902]: pgmap v6950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/299781844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:45:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:45:15] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:45:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:16.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:16.062477+0000) 2023-08-28T00:45:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:16 smithi136 ceph-mon[113531]: pgmap v6951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1547482494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:17.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:17.062723+0000) 2023-08-28T00:45:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:16 smithi093 ceph-mon[111335]: pgmap v6951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1547482494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:16 smithi093 ceph-mon[118902]: pgmap v6951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1547482494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:18.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:18.062976+0000) 2023-08-28T00:45:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:18 smithi136 ceph-mon[113531]: pgmap v6952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:19.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:19.063232+0000) 2023-08-28T00:45:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:18 smithi093 ceph-mon[111335]: pgmap v6952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:18 smithi093 ceph-mon[118902]: pgmap v6952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:20.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3818686515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:20.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:20.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:20.063466+0000) 2023-08-28T00:45:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3818686515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3818686515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:20 smithi136 ceph-mon[113531]: pgmap v6953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:21.481 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:21.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:21.063725+0000) 2023-08-28T00:45:21.482 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:20 smithi093 ceph-mon[111335]: pgmap v6953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:21.482 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:20 smithi093 ceph-mon[118902]: pgmap v6953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:21.483 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:21.111Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:21.483 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:21.112Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:22.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3540878832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:22.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:22.063973+0000) 2023-08-28T00:45:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3540878832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3540878832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:22.937 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:22.567Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:22.937 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:22.568Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:22 smithi136 ceph-mon[113531]: pgmap v6954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:23.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:23.064233+0000) 2023-08-28T00:45:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:22 smithi093 ceph-mon[111335]: pgmap v6954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:22 smithi093 ceph-mon[118902]: pgmap v6954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3075663959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3075663959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:24.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3075663959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:24.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:24.064459+0000) 2023-08-28T00:45:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:24 smithi093 ceph-mon[118902]: pgmap v6955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:24 smithi093 ceph-mon[111335]: pgmap v6955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:24 smithi136 ceph-mon[113531]: pgmap v6955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:25.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:25.064728+0000) 2023-08-28T00:45:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:45:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:45:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:45:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:26.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:26.064979+0000) 2023-08-28T00:45:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:26 smithi093 ceph-mon[111335]: pgmap v6956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1712728000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:26 smithi093 ceph-mon[118902]: pgmap v6956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1712728000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:26 smithi136 ceph-mon[113531]: pgmap v6956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1712728000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:27.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:27.065223+0000) 2023-08-28T00:45:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:28.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:28.065461+0000) 2023-08-28T00:45:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:28 smithi093 ceph-mon[111335]: pgmap v6957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4248623256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:28 smithi093 ceph-mon[118902]: pgmap v6957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4248623256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:28 smithi136 ceph-mon[113531]: pgmap v6957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4248623256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:29.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:29.065765+0000) 2023-08-28T00:45:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:30.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:30.066019+0000) 2023-08-28T00:45:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:30 smithi093 ceph-mon[111335]: pgmap v6958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:30 smithi093 ceph-mon[118902]: pgmap v6958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:31.112Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:31.113Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:30 smithi136 ceph-mon[113531]: pgmap v6958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:31.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:31.066284+0000) 2023-08-28T00:45:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2506847189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:32.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2506847189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2506847189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:32.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:32.066565+0000) 2023-08-28T00:45:32.958 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:32.567Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:32.959 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:32.568Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:32 smithi093 ceph-mon[111335]: pgmap v6959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:32 smithi093 ceph-mon[118902]: pgmap v6959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:32 smithi136 ceph-mon[113531]: pgmap v6959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:33.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:33.066873+0000) 2023-08-28T00:45:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3815999753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3815999753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3815999753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:34.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:34.067182+0000) 2023-08-28T00:45:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:34 smithi093 ceph-mon[111335]: pgmap v6960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:34 smithi093 ceph-mon[118902]: pgmap v6960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:34 smithi136 ceph-mon[113531]: pgmap v6960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:35.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:35.067504+0000) 2023-08-28T00:45:35.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:45:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:45:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:45:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2375412604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2375412604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2375412604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:36.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:36.067752+0000) 2023-08-28T00:45:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:36 smithi093 ceph-mon[111335]: pgmap v6961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:36 smithi093 ceph-mon[118902]: pgmap v6961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:36 smithi136 ceph-mon[113531]: pgmap v6961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:37.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:37.068008+0000) 2023-08-28T00:45:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:38.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:38.068340+0000) 2023-08-28T00:45:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:38 smithi093 ceph-mon[111335]: pgmap v6962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/759856046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:38 smithi093 ceph-mon[118902]: pgmap v6962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/759856046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:39.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:39.068628+0000) 2023-08-28T00:45:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:38 smithi136 ceph-mon[113531]: pgmap v6962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/759856046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:40.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:40.068984+0000) 2023-08-28T00:45:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:40 smithi093 ceph-mon[111335]: pgmap v6963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3550414683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:40 smithi093 ceph-mon[118902]: pgmap v6963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3550414683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:41.112Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:41.113Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:40 smithi136 ceph-mon[113531]: pgmap v6963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3550414683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:41.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:41.069214+0000) 2023-08-28T00:45:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:42.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:42.069473+0000) 2023-08-28T00:45:42.844 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:42.567Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:42.845 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:42.568Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:43.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:43.069678+0000) 2023-08-28T00:45:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:42 smithi136 ceph-mon[113531]: pgmap v6964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:45:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/302999599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-mon[111335]: pgmap v6964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:45:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/302999599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-mon[118902]: pgmap v6964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:45:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/302999599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:44.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:44.069884+0000) 2023-08-28T00:45:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: pgmap v6965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: pgmap v6965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:45.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.361 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:45.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:45.070055+0000) 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: pgmap v6965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:45:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:45:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:45:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:46.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:46.070250+0000) 2023-08-28T00:45:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:46 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2473M 2023-08-28T00:45:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:45:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:45:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:46.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4188259048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2473M 2023-08-28T00:45:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:45:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:45:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4188259048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2473M 2023-08-28T00:45:46.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:45:46.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:45:46.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:45:46.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4188259048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:47 smithi136 ceph-mon[113531]: pgmap v6966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:47.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:47.070467+0000) 2023-08-28T00:45:47.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:47 smithi093 ceph-mon[111335]: pgmap v6966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:47 smithi093 ceph-mon[118902]: pgmap v6966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3865082305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:48.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:48.070731+0000) 2023-08-28T00:45:48.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3865082305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:48.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3865082305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:49 smithi136 ceph-mon[113531]: pgmap v6967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:49.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:49.070981+0000) 2023-08-28T00:45:49.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:49 smithi093 ceph-mon[111335]: pgmap v6967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:49.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:49 smithi093 ceph-mon[118902]: pgmap v6967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1822649136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:50.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:50.071232+0000) 2023-08-28T00:45:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1822649136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1822649136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:51 smithi136 ceph-mon[113531]: pgmap v6968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:51.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:51.071459+0000) 2023-08-28T00:45:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:51 smithi093 ceph-mon[111335]: pgmap v6968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:51 smithi093 ceph-mon[118902]: pgmap v6968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:51.113Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:51.114Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:52.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:52.071707+0000) 2023-08-28T00:45:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:52.568Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:45:52.569Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:45:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:53 smithi136 ceph-mon[113531]: pgmap v6969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1058001596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:53.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:53.071977+0000) 2023-08-28T00:45:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:53 smithi093 ceph-mon[111335]: pgmap v6969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1058001596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:53.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:53 smithi093 ceph-mon[118902]: pgmap v6969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:53.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1058001596' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:54.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:54.072224+0000) 2023-08-28T00:45:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:45:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:55 smithi136 ceph-mon[113531]: pgmap v6970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1410386977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:55.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:55.072385+0000) 2023-08-28T00:45:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:45:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:45:55] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:45:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:55 smithi093 ceph-mon[111335]: pgmap v6970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:55.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1410386977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:55 smithi093 ceph-mon[118902]: pgmap v6970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1410386977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:56.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:56.072618+0000) 2023-08-28T00:45:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:57 smithi136 ceph-mon[113531]: pgmap v6971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1984821945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:57.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:57.072877+0000) 2023-08-28T00:45:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:57 smithi093 ceph-mon[111335]: pgmap v6971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1984821945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:57.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:57 smithi093 ceph-mon[118902]: pgmap v6971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:57.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1984821945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:45:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:58.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:58.073124+0000) 2023-08-28T00:45:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:45:59 smithi136 ceph-mon[113531]: pgmap v6972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:45:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:45:59.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:45:59.073368+0000) 2023-08-28T00:45:59.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:45:59 smithi093 ceph-mon[111335]: pgmap v6972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:45:59.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:45:59 smithi093 ceph-mon[118902]: pgmap v6972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3321109710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:00.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:00.073619+0000) 2023-08-28T00:46:00.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3321109710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:00.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3321109710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:01 smithi136 ceph-mon[113531]: pgmap v6973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:01.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:01.073866+0000) 2023-08-28T00:46:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:01 smithi093 ceph-mon[111335]: pgmap v6973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:01 smithi093 ceph-mon[118902]: pgmap v6973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:01.114Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:01.115Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/75156050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:02.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:02.074118+0000) 2023-08-28T00:46:02.570 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/75156050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:02.570 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/75156050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:02.569Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:02.570Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:03 smithi136 ceph-mon[113531]: pgmap v6974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:03.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:03.074316+0000) 2023-08-28T00:46:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:03 smithi093 ceph-mon[111335]: pgmap v6974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:03 smithi093 ceph-mon[118902]: pgmap v6974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:04.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:04.074539+0000) 2023-08-28T00:46:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:05 smithi136 ceph-mon[113531]: pgmap v6975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2812814105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:05.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:05.074812+0000) 2023-08-28T00:46:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:05 smithi093 ceph-mon[118902]: pgmap v6975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2812814105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:05 smithi093 ceph-mon[111335]: pgmap v6975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:05.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2812814105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:46:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:46:05] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:46:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:06.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:06.075105+0000) 2023-08-28T00:46:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:07 smithi136 ceph-mon[113531]: pgmap v6976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3926579830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:07.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:07.075379+0000) 2023-08-28T00:46:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:07 smithi093 ceph-mon[118902]: pgmap v6976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3926579830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:07 smithi093 ceph-mon[111335]: pgmap v6976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:07.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3926579830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:08.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:08.075667+0000) 2023-08-28T00:46:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:09.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:09.075927+0000) 2023-08-28T00:46:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:09 smithi136 ceph-mon[113531]: pgmap v6977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4155911564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:09 smithi093 ceph-mon[111335]: pgmap v6977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4155911564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:09 smithi093 ceph-mon[118902]: pgmap v6977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4155911564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:10.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:10.076182+0000) 2023-08-28T00:46:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:11 smithi136 ceph-mon[113531]: pgmap v6978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:11.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:11.076430+0000) 2023-08-28T00:46:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:11 smithi093 ceph-mon[111335]: pgmap v6978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:11 smithi093 ceph-mon[118902]: pgmap v6978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:11.115Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:11.116Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4224497933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:12.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:12.076688+0000) 2023-08-28T00:46:12.570 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4224497933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:12.571 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4224497933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:12.570Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:12.571Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:13 smithi136 ceph-mon[113531]: pgmap v6979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:13.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:13.076959+0000) 2023-08-28T00:46:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:13 smithi093 ceph-mon[111335]: pgmap v6979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:13 smithi093 ceph-mon[118902]: pgmap v6979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1647715625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:14.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:14.077231+0000) 2023-08-28T00:46:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1647715625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1647715625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:15 smithi136 ceph-mon[113531]: pgmap v6980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:15.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:15.077472+0000) 2023-08-28T00:46:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:15 smithi093 ceph-mon[111335]: pgmap v6980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:15 smithi093 ceph-mon[118902]: pgmap v6980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:46:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:46:15] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:46:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:16.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:16.077703+0000) 2023-08-28T00:46:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:16 smithi136 ceph-mon[113531]: pgmap v6981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1515690079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:17.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:17.077992+0000) 2023-08-28T00:46:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:16 smithi093 ceph-mon[111335]: pgmap v6981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1515690079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:16 smithi093 ceph-mon[118902]: pgmap v6981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1515690079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:18.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:18.078266+0000) 2023-08-28T00:46:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:18 smithi136 ceph-mon[113531]: pgmap v6982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/379565582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:19.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:19.078508+0000) 2023-08-28T00:46:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:18 smithi093 ceph-mon[111335]: pgmap v6982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/379565582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:18 smithi093 ceph-mon[118902]: pgmap v6982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/379565582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:20.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:20.078757+0000) 2023-08-28T00:46:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:20 smithi136 ceph-mon[113531]: pgmap v6983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:21.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:21.078954+0000) 2023-08-28T00:46:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:20 smithi093 ceph-mon[111335]: pgmap v6983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:20 smithi093 ceph-mon[118902]: pgmap v6983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:21.115Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:21.116Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:22.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3929058467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:22.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:22.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:22.079190+0000) 2023-08-28T00:46:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3929058467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3929058467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:22.874 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:22.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:22.875 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:22.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:22 smithi136 ceph-mon[113531]: pgmap v6984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:23.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:23.079449+0000) 2023-08-28T00:46:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:22 smithi093 ceph-mon[111335]: pgmap v6984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:22 smithi093 ceph-mon[118902]: pgmap v6984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:24.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4121131627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:24.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:24.079662+0000) 2023-08-28T00:46:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4121131627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4121131627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:24 smithi136 ceph-mon[113531]: pgmap v6985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:25.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:25.079868+0000) 2023-08-28T00:46:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:24 smithi093 ceph-mon[111335]: pgmap v6985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:24 smithi093 ceph-mon[118902]: pgmap v6985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:46:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:46:25] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:46:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4169520672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:26.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:26.080134+0000) 2023-08-28T00:46:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4169520672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4169520672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:26 smithi136 ceph-mon[113531]: pgmap v6986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:27.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:27.080377+0000) 2023-08-28T00:46:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:26 smithi093 ceph-mon[111335]: pgmap v6986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:26 smithi093 ceph-mon[118902]: pgmap v6986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:28.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:28.080607+0000) 2023-08-28T00:46:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:28 smithi136 ceph-mon[113531]: pgmap v6987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3015834593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:29.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:29.080872+0000) 2023-08-28T00:46:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:28 smithi093 ceph-mon[111335]: pgmap v6987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3015834593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:28 smithi093 ceph-mon[118902]: pgmap v6987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3015834593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:30.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:30.081126+0000) 2023-08-28T00:46:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:30 smithi136 ceph-mon[113531]: pgmap v6988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1103308221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:31.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:31.081325+0000) 2023-08-28T00:46:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:30 smithi093 ceph-mon[111335]: pgmap v6988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1103308221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:30 smithi093 ceph-mon[118902]: pgmap v6988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1103308221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:31.116Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:31.117Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:32.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:32.081562+0000) 2023-08-28T00:46:32.893 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:32.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:32.894 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:32.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:32 smithi136 ceph-mon[113531]: pgmap v6989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:33.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:33.081836+0000) 2023-08-28T00:46:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:32 smithi093 ceph-mon[111335]: pgmap v6989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:32 smithi093 ceph-mon[118902]: pgmap v6989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:34.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3746440869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:34.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:34.082157+0000) 2023-08-28T00:46:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3746440869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3746440869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:34 smithi136 ceph-mon[113531]: pgmap v6990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:35.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:35.082323+0000) 2023-08-28T00:46:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:34 smithi093 ceph-mon[111335]: pgmap v6990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:34 smithi093 ceph-mon[118902]: pgmap v6990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:46:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:46:35] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:46:36.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1744067958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:36.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:36.082511+0000) 2023-08-28T00:46:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1744067958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1744067958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:36 smithi136 ceph-mon[113531]: pgmap v6991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:37.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:37.082752+0000) 2023-08-28T00:46:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:36 smithi093 ceph-mon[111335]: pgmap v6991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:36 smithi093 ceph-mon[118902]: pgmap v6991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2075330111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:38.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:38.082957+0000) 2023-08-28T00:46:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2075330111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2075330111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:38 smithi136 ceph-mon[113531]: pgmap v6992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:39.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:39.083218+0000) 2023-08-28T00:46:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:38 smithi093 ceph-mon[111335]: pgmap v6992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:38 smithi093 ceph-mon[118902]: pgmap v6992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:40.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:40.083421+0000) 2023-08-28T00:46:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:40 smithi136 ceph-mon[113531]: pgmap v6993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/318024052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:41.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:41.083641+0000) 2023-08-28T00:46:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:40 smithi093 ceph-mon[111335]: pgmap v6993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/318024052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:40 smithi093 ceph-mon[118902]: pgmap v6993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/318024052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:41.117Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:41.118Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:42.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:42.083868+0000) 2023-08-28T00:46:42.918 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:42.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:42.919 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:42.573Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:43.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:42 smithi136 ceph-mon[113531]: pgmap v6994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:43.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2787764718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:43.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:43.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:43.084128+0000) 2023-08-28T00:46:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:42 smithi093 ceph-mon[111335]: pgmap v6994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2787764718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:42 smithi093 ceph-mon[118902]: pgmap v6994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2787764718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:44.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:44.084391+0000) 2023-08-28T00:46:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:44 smithi136 ceph-mon[113531]: pgmap v6995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:45.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/362429355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:45.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:45.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:45.084595+0000) 2023-08-28T00:46:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:44 smithi093 ceph-mon[111335]: pgmap v6995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/362429355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:44 smithi093 ceph-mon[118902]: pgmap v6995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/362429355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:46:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:46:46.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:46:46.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:46.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:46.084775+0000) 2023-08-28T00:46:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:46:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:46:47.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:46 smithi136 ceph-mon[113531]: pgmap v6996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:47.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:47.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:47.084979+0000) 2023-08-28T00:46:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:46 smithi093 ceph-mon[111335]: pgmap v6996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:46 smithi093 ceph-mon[118902]: pgmap v6996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:48.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:48.085156+0000) 2023-08-28T00:46:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3876533415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3876533415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3876533415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:48.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:49.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:49.085412+0000) 2023-08-28T00:46:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:49 smithi136 ceph-mon[113531]: pgmap v6997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:46:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:46:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[111335]: pgmap v6997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[118902]: pgmap v6997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:46:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:46:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2740953133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:50.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:50.085673+0000) 2023-08-28T00:46:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2740953133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2740953133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:51 smithi136 ceph-mon[113531]: pgmap v6998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:51.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:51.085949+0000) 2023-08-28T00:46:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:51 smithi093 ceph-mon[111335]: pgmap v6998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:51 smithi093 ceph-mon[118902]: pgmap v6998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:51.118Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:51.119Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/979006633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:52.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:52.086194+0000) 2023-08-28T00:46:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/979006633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/979006633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:52.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:46:52.573Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:46:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:53 smithi136 ceph-mon[113531]: pgmap v6999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:53.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:53.086436+0000) 2023-08-28T00:46:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:53 smithi093 ceph-mon[111335]: pgmap v6999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:53 smithi093 ceph-mon[118902]: pgmap v6999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:54.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:54.086673+0000) 2023-08-28T00:46:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:46:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:55 smithi136 ceph-mon[113531]: pgmap v7000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3326994159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:55.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:55.086935+0000) 2023-08-28T00:46:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:46:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:46:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:55 smithi093 ceph-mon[118902]: pgmap v7000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3326994159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:55.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:55 smithi093 ceph-mon[111335]: pgmap v7000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:55.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3326994159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:56.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:56.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:56.087196+0000) 2023-08-28T00:46:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:57 smithi136 ceph-mon[113531]: pgmap v7001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3900639211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:57.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:57.087464+0000) 2023-08-28T00:46:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:57 smithi093 ceph-mon[111335]: pgmap v7001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3900639211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:57 smithi093 ceph-mon[118902]: pgmap v7001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3900639211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:46:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:58.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:58.087674+0000) 2023-08-28T00:46:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:46:59 smithi136 ceph-mon[113531]: pgmap v7002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:46:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:46:59.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:46:59.087912+0000) 2023-08-28T00:46:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:46:59 smithi093 ceph-mon[111335]: pgmap v7002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:46:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:46:59 smithi093 ceph-mon[118902]: pgmap v7002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2547733898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:00.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:00.088138+0000) 2023-08-28T00:47:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2547733898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2547733898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:01 smithi136 ceph-mon[113531]: pgmap v7003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:01.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:01.088350+0000) 2023-08-28T00:47:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:01 smithi093 ceph-mon[111335]: pgmap v7003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:01 smithi093 ceph-mon[118902]: pgmap v7003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:01.119Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:01.120Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/897169831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:02.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:02.088572+0000) 2023-08-28T00:47:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/897169831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/897169831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:02.573Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:02.574Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:03 smithi136 ceph-mon[113531]: pgmap v7004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:03.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:03.088825+0000) 2023-08-28T00:47:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:03 smithi093 ceph-mon[111335]: pgmap v7004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:03 smithi093 ceph-mon[118902]: pgmap v7004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1310928615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:04.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:04.089098+0000) 2023-08-28T00:47:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1310928615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1310928615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:05 smithi136 ceph-mon[113531]: pgmap v7005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:05.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:05.089358+0000) 2023-08-28T00:47:05.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:05 smithi093 ceph-mon[111335]: pgmap v7005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:05 smithi093 ceph-mon[118902]: pgmap v7005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:05.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:47:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:47:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:47:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:06.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:06.089655+0000) 2023-08-28T00:47:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:07 smithi136 ceph-mon[113531]: pgmap v7006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/892916814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:07.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:07.089880+0000) 2023-08-28T00:47:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:07 smithi093 ceph-mon[111335]: pgmap v7006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/892916814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:07 smithi093 ceph-mon[118902]: pgmap v7006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/892916814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:08.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:08.090072+0000) 2023-08-28T00:47:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:09 smithi136 ceph-mon[113531]: pgmap v7007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2771952907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:09.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:09.090341+0000) 2023-08-28T00:47:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:09 smithi093 ceph-mon[111335]: pgmap v7007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2771952907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:09 smithi093 ceph-mon[118902]: pgmap v7007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2771952907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:10.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:10.090602+0000) 2023-08-28T00:47:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:11.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:11.090869+0000) 2023-08-28T00:47:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:11 smithi136 ceph-mon[113531]: pgmap v7008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2936843118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:11 smithi093 ceph-mon[118902]: pgmap v7008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2936843118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:11 smithi093 ceph-mon[111335]: pgmap v7008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2936843118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:11.119Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:11.120Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:12.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:12.091148+0000) 2023-08-28T00:47:12.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:12.573Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:12.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:12.574Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:13 smithi136 ceph-mon[113531]: pgmap v7009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:13.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:13.091365+0000) 2023-08-28T00:47:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:13 smithi093 ceph-mon[111335]: pgmap v7009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:13 smithi093 ceph-mon[118902]: pgmap v7009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2091167562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:14.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:14.091628+0000) 2023-08-28T00:47:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2091167562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2091167562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:15 smithi136 ceph-mon[113531]: pgmap v7010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:15.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:15.091883+0000) 2023-08-28T00:47:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:15 smithi093 ceph-mon[111335]: pgmap v7010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:47:15] "GET /metrics HTTP/1.1" 200 239924 "" "Prometheus/2.43.0" 2023-08-28T00:47:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:15 smithi093 ceph-mon[118902]: pgmap v7010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:16.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:16.092158+0000) 2023-08-28T00:47:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:16 smithi136 ceph-mon[113531]: pgmap v7011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1467696811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:17.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:17.092351+0000) 2023-08-28T00:47:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:16 smithi093 ceph-mon[111335]: pgmap v7011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1467696811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:16 smithi093 ceph-mon[118902]: pgmap v7011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1467696811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:18.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:18.092534+0000) 2023-08-28T00:47:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:18 smithi136 ceph-mon[113531]: pgmap v7012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/782275998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:19.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:19.092788+0000) 2023-08-28T00:47:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:18 smithi093 ceph-mon[111335]: pgmap v7012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/782275998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:18 smithi093 ceph-mon[118902]: pgmap v7012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/782275998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:20.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:20.093025+0000) 2023-08-28T00:47:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:20 smithi136 ceph-mon[113531]: pgmap v7013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4020638765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:21.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:21.093297+0000) 2023-08-28T00:47:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:20 smithi093 ceph-mon[118902]: pgmap v7013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4020638765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:20 smithi093 ceph-mon[111335]: pgmap v7013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4020638765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:21.120Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:21.121Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:22.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:22.093495+0000) 2023-08-28T00:47:22.880 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:22.574Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:22.880 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:22.575Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:22 smithi136 ceph-mon[113531]: pgmap v7014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:23.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:23.093738+0000) 2023-08-28T00:47:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:22 smithi093 ceph-mon[111335]: pgmap v7014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:22 smithi093 ceph-mon[118902]: pgmap v7014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2623475077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:24.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:24.093908+0000) 2023-08-28T00:47:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2623475077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2623475077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:24 smithi136 ceph-mon[113531]: pgmap v7015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:25.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:25.094110+0000) 2023-08-28T00:47:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:24 smithi093 ceph-mon[111335]: pgmap v7015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:24 smithi093 ceph-mon[118902]: pgmap v7015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:47:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:47:25] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:47:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/611579280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:26.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:26.094340+0000) 2023-08-28T00:47:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/611579280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/611579280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:26 smithi136 ceph-mon[113531]: pgmap v7016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:27.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:27.094571+0000) 2023-08-28T00:47:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:26 smithi093 ceph-mon[111335]: pgmap v7016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:26 smithi093 ceph-mon[118902]: pgmap v7016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2418548978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:28.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:28.094836+0000) 2023-08-28T00:47:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2418548978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2418548978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:28 smithi136 ceph-mon[113531]: pgmap v7017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:29.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:29.095068+0000) 2023-08-28T00:47:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:28 smithi093 ceph-mon[111335]: pgmap v7017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:28 smithi093 ceph-mon[118902]: pgmap v7017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:30.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:30.095274+0000) 2023-08-28T00:47:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:30 smithi136 ceph-mon[113531]: pgmap v7018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:31.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3124145223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:31.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:31.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:31.095422+0000) 2023-08-28T00:47:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:30 smithi093 ceph-mon[111335]: pgmap v7018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:31.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3124145223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:31.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:30 smithi093 ceph-mon[118902]: pgmap v7018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:31.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3124145223' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:31.121Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:31.122Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:32.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:32.095676+0000) 2023-08-28T00:47:32.907 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:32.575Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:32.908 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:32.576Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:32 smithi136 ceph-mon[113531]: pgmap v7019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:33.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2403239474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:33.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:33.095937+0000) 2023-08-28T00:47:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:32 smithi093 ceph-mon[111335]: pgmap v7019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2403239474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:32 smithi093 ceph-mon[118902]: pgmap v7019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2403239474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:34.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:34.096189+0000) 2023-08-28T00:47:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:34 smithi136 ceph-mon[113531]: pgmap v7020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:35.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:35.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:35.096450+0000) 2023-08-28T00:47:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:34 smithi093 ceph-mon[111335]: pgmap v7020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:34 smithi093 ceph-mon[118902]: pgmap v7020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:47:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:47:35] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:47:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2033549105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:36.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:36.096702+0000) 2023-08-28T00:47:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2033549105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2033549105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:37.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:36 smithi136 ceph-mon[113531]: pgmap v7021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:37.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:37.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:37.097015+0000) 2023-08-28T00:47:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:36 smithi093 ceph-mon[111335]: pgmap v7021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:36 smithi093 ceph-mon[118902]: pgmap v7021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:38.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2726450572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:38.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:38.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:38.097287+0000) 2023-08-28T00:47:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2726450572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2726450572' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:38 smithi136 ceph-mon[113531]: pgmap v7022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:39.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:39.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:39.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:39.097540+0000) 2023-08-28T00:47:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:38 smithi093 ceph-mon[111335]: pgmap v7022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:38 smithi093 ceph-mon[118902]: pgmap v7022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:40.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3739434955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:40.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:40.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:40.097757+0000) 2023-08-28T00:47:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3739434955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3739434955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:41.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:40 smithi136 ceph-mon[113531]: pgmap v7023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:41.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:41.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:41.098021+0000) 2023-08-28T00:47:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:40 smithi093 ceph-mon[111335]: pgmap v7023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:40 smithi093 ceph-mon[118902]: pgmap v7023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:41.122Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:41.123Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:42.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:42.098317+0000) 2023-08-28T00:47:42.939 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:42.575Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:42.939 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:42.576Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:42 smithi093 ceph-mon[111335]: pgmap v7024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3386370472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:42 smithi093 ceph-mon[118902]: pgmap v7024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3386370472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:42 smithi136 ceph-mon[113531]: pgmap v7024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3386370472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:43.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:43.098582+0000) 2023-08-28T00:47:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:44.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:44.098808+0000) 2023-08-28T00:47:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:44 smithi093 ceph-mon[111335]: pgmap v7025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2966356616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:44 smithi093 ceph-mon[118902]: pgmap v7025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2966356616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:44 smithi136 ceph-mon[113531]: pgmap v7025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2966356616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:45.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:45.099086+0000) 2023-08-28T00:47:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:47:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:47:45] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:47:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:46.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:46.099287+0000) 2023-08-28T00:47:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:46 smithi093 ceph-mon[118902]: pgmap v7026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2962451717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:46 smithi093 ceph-mon[111335]: pgmap v7026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:47.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2962451717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:46 smithi136 ceph-mon[113531]: pgmap v7026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2962451717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:47.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:47.099546+0000) 2023-08-28T00:47:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:48.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:48.099788+0000) 2023-08-28T00:47:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:48 smithi093 ceph-mon[111335]: pgmap v7027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:47:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:48 smithi093 ceph-mon[118902]: pgmap v7027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:47:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:48 smithi136 ceph-mon[113531]: pgmap v7027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:47:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:49.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:49.099981+0000) 2023-08-28T00:47:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2669474299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2669474299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2669474299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:50.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:50.100147+0000) 2023-08-28T00:47:51.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:51.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:51.100368+0000) 2023-08-28T00:47:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[111335]: pgmap v7028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[118902]: pgmap v7028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:51.122Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:51.492 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:51.123Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:51.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:51 smithi136 ceph-mon[113531]: pgmap v7028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:51.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:51.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:52.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:52.100569+0000) 2023-08-28T00:47:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:47:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:47:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3113084329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:52.577 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:47:52.577 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:47:52.577 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:52.577 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3113084329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:52.577 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:47:52.577 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:47:52.577 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:47:52.578 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3113084329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:52.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:47:52.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:47:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:53 smithi136 ceph-mon[113531]: pgmap v7029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:53.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:53.100766+0000) 2023-08-28T00:47:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:53 smithi093 ceph-mon[111335]: pgmap v7029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:53 smithi093 ceph-mon[118902]: pgmap v7029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3914969795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:54.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:54.101011+0000) 2023-08-28T00:47:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:54.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3914969795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:47:54.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3914969795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:55 smithi136 ceph-mon[113531]: pgmap v7030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:55.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:55.101215+0000) 2023-08-28T00:47:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:55 smithi093 ceph-mon[118902]: pgmap v7030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:55 smithi093 ceph-mon[111335]: pgmap v7030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:47:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:47:55] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:47:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:56.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:56.101498+0000) 2023-08-28T00:47:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:57 smithi136 ceph-mon[113531]: pgmap v7031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1373347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:57.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:57.101779+0000) 2023-08-28T00:47:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:57 smithi093 ceph-mon[111335]: pgmap v7031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:57.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1373347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:57 smithi093 ceph-mon[118902]: pgmap v7031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:47:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1373347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:47:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:58.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:58.101993+0000) 2023-08-28T00:47:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:59 smithi136 ceph-mon[113531]: pgmap v7032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:00.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:47:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3669508367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:00.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:47:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:47:59.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:47:59.102204+0000) 2023-08-28T00:48:00.190 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:59 smithi093 ceph-mon[111335]: pgmap v7032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:00.190 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:47:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3669508367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:00.190 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:59 smithi093 ceph-mon[118902]: pgmap v7032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:00.190 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:47:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3669508367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:00.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:00.102462+0000) 2023-08-28T00:48:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:01 smithi136 ceph-mon[113531]: pgmap v7033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/772244631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:01.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:01.102717+0000) 2023-08-28T00:48:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:01 smithi093 ceph-mon[111335]: pgmap v7033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/772244631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:01 smithi093 ceph-mon[118902]: pgmap v7033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/772244631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:01.123Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:01.125Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:02.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:02.102917+0000) 2023-08-28T00:48:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:02.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:02.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:03 smithi136 ceph-mon[113531]: pgmap v7034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:03.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:03.103170+0000) 2023-08-28T00:48:03.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:03 smithi093 ceph-mon[118902]: pgmap v7034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:03 smithi093 ceph-mon[111335]: pgmap v7034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2410206081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:04.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:04.103421+0000) 2023-08-28T00:48:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2410206081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2410206081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:05 smithi136 ceph-mon[113531]: pgmap v7035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:05.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:05.103671+0000) 2023-08-28T00:48:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:05 smithi093 ceph-mon[111335]: pgmap v7035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:05 smithi093 ceph-mon[118902]: pgmap v7035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:48:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:48:05] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:48:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:06.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:06.103924+0000) 2023-08-28T00:48:07.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:06 smithi136 ceph-mon[113531]: pgmap v7036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/588936786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:07.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:07.104167+0000) 2023-08-28T00:48:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:06 smithi093 ceph-mon[111335]: pgmap v7036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/588936786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:06 smithi093 ceph-mon[118902]: pgmap v7036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/588936786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:08.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:08.104390+0000) 2023-08-28T00:48:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:08 smithi136 ceph-mon[113531]: pgmap v7037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2612952412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:09.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:09.104671+0000) 2023-08-28T00:48:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:08 smithi093 ceph-mon[111335]: pgmap v7037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2612952412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:08 smithi093 ceph-mon[118902]: pgmap v7037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2612952412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:10.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:10.104924+0000) 2023-08-28T00:48:11.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:11.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:11.105194+0000) 2023-08-28T00:48:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:10 smithi136 ceph-mon[113531]: pgmap v7038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3972862643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:10 smithi093 ceph-mon[111335]: pgmap v7038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3972862643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:10 smithi093 ceph-mon[118902]: pgmap v7038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3972862643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:11.124Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": context deadline exceeded" 2023-08-28T00:48:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:11.125Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:12.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:12.105467+0000) 2023-08-28T00:48:12.891 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:12.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:12.892 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:12.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:12 smithi136 ceph-mon[113531]: pgmap v7039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:13.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:13.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:13.105659+0000) 2023-08-28T00:48:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:12 smithi093 ceph-mon[111335]: pgmap v7039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:12 smithi093 ceph-mon[118902]: pgmap v7039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:14.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3040327413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:14.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:14.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:14.105931+0000) 2023-08-28T00:48:14.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3040327413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:14.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3040327413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:14 smithi136 ceph-mon[113531]: pgmap v7040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:15.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:15.106168+0000) 2023-08-28T00:48:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:14 smithi093 ceph-mon[111335]: pgmap v7040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:14 smithi093 ceph-mon[118902]: pgmap v7040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:48:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:48:15] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:48:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4024940435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:16.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:16.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:16.106432+0000) 2023-08-28T00:48:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4024940435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4024940435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:16 smithi136 ceph-mon[113531]: pgmap v7041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:17.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:17.106670+0000) 2023-08-28T00:48:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:16 smithi093 ceph-mon[111335]: pgmap v7041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:16 smithi093 ceph-mon[118902]: pgmap v7041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:18.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1499538525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:18.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:18.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:18.106878+0000) 2023-08-28T00:48:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1499538525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1499538525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:18 smithi136 ceph-mon[113531]: pgmap v7042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:19.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:19.107111+0000) 2023-08-28T00:48:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:18 smithi093 ceph-mon[111335]: pgmap v7042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:18 smithi093 ceph-mon[118902]: pgmap v7042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:20.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:20.107368+0000) 2023-08-28T00:48:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:21.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:21.107616+0000) 2023-08-28T00:48:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:20 smithi136 ceph-mon[113531]: pgmap v7043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/967045224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:20 smithi093 ceph-mon[111335]: pgmap v7043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/967045224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:20 smithi093 ceph-mon[118902]: pgmap v7043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/967045224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:21.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:21.124Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:21.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:21.125Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:22.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:22.107850+0000) 2023-08-28T00:48:22.915 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:22.577Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:22.915 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:22.578Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:23.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:22 smithi136 ceph-mon[113531]: pgmap v7044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:23.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3454222502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:23.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:23.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:23.108116+0000) 2023-08-28T00:48:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:22 smithi093 ceph-mon[111335]: pgmap v7044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3454222502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:22 smithi093 ceph-mon[118902]: pgmap v7044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3454222502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:24.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:24.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:24.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:24.108346+0000) 2023-08-28T00:48:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:25.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:24 smithi136 ceph-mon[113531]: pgmap v7045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:25.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:25.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:25.108574+0000) 2023-08-28T00:48:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:24 smithi093 ceph-mon[111335]: pgmap v7045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:24 smithi093 ceph-mon[118902]: pgmap v7045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:48:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:48:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:48:26.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4191330638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:26.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:26.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:26.108856+0000) 2023-08-28T00:48:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4191330638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4191330638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:27.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:26 smithi136 ceph-mon[113531]: pgmap v7046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:27.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:27.109145+0000) 2023-08-28T00:48:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:26 smithi093 ceph-mon[111335]: pgmap v7046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:26 smithi093 ceph-mon[118902]: pgmap v7046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:28.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3956879411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:28.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:28.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:28.109377+0000) 2023-08-28T00:48:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3956879411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3956879411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:28 smithi136 ceph-mon[113531]: pgmap v7047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:29.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:29.109628+0000) 2023-08-28T00:48:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:28 smithi093 ceph-mon[111335]: pgmap v7047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:28 smithi093 ceph-mon[118902]: pgmap v7047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1444994940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:30.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1444994940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1444994940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:30.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:30.109874+0000) 2023-08-28T00:48:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:30 smithi093 ceph-mon[111335]: pgmap v7048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:30 smithi093 ceph-mon[118902]: pgmap v7048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:31.125Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:31.126Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:30 smithi136 ceph-mon[113531]: pgmap v7048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:31.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:31.110113+0000) 2023-08-28T00:48:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:32.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:32.110331+0000) 2023-08-28T00:48:32.950 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:32.578Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:32.950 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:32.578Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:32 smithi093 ceph-mon[111335]: pgmap v7049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:33.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2055651996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:32 smithi093 ceph-mon[118902]: pgmap v7049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:33.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2055651996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:32 smithi136 ceph-mon[113531]: pgmap v7049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2055651996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:33.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:33.110587+0000) 2023-08-28T00:48:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:34.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:34.110842+0000) 2023-08-28T00:48:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:34 smithi093 ceph-mon[118902]: pgmap v7050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2742559008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:34 smithi093 ceph-mon[111335]: pgmap v7050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2742559008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:34 smithi136 ceph-mon[113531]: pgmap v7050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2742559008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:35.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:35.111111+0000) 2023-08-28T00:48:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:48:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:48:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:48:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:36.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:36.111393+0000) 2023-08-28T00:48:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:36 smithi093 ceph-mon[111335]: pgmap v7051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1581049443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:36 smithi093 ceph-mon[118902]: pgmap v7051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1581049443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:36 smithi136 ceph-mon[113531]: pgmap v7051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1581049443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:37.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:37.111658+0000) 2023-08-28T00:48:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:38.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:38.111862+0000) 2023-08-28T00:48:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:38 smithi093 ceph-mon[111335]: pgmap v7052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:38 smithi093 ceph-mon[118902]: pgmap v7052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:38 smithi136 ceph-mon[113531]: pgmap v7052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:39.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:39.112055+0000) 2023-08-28T00:48:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/537784098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/537784098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:40.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/537784098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:40.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:40.112301+0000) 2023-08-28T00:48:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:40 smithi093 ceph-mon[111335]: pgmap v7053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:40 smithi093 ceph-mon[118902]: pgmap v7053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:41.125Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:41.126Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:40 smithi136 ceph-mon[113531]: pgmap v7053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:41.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:41.112555+0000) 2023-08-28T00:48:42.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1994957883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:42.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1994957883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1994957883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:42.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:42.112808+0000) 2023-08-28T00:48:42.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:42.578Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:42.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:42.579Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:42 smithi093 ceph-mon[111335]: pgmap v7054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:42 smithi093 ceph-mon[118902]: pgmap v7054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:42 smithi136 ceph-mon[113531]: pgmap v7054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:43.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:43.113089+0000) 2023-08-28T00:48:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:44.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:44.113348+0000) 2023-08-28T00:48:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:44 smithi093 ceph-mon[111335]: pgmap v7055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/858755747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:44 smithi093 ceph-mon[118902]: pgmap v7055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/858755747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:44 smithi136 ceph-mon[113531]: pgmap v7055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/858755747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:45.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:45.113598+0000) 2023-08-28T00:48:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:48:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:48:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:48:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:46.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:46.113862+0000) 2023-08-28T00:48:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:46 smithi093 ceph-mon[111335]: pgmap v7056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/81083610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:46 smithi093 ceph-mon[118902]: pgmap v7056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/81083610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:46 smithi136 ceph-mon[113531]: pgmap v7056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/81083610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:47.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:47.114135+0000) 2023-08-28T00:48:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:48.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:48.114345+0000) 2023-08-28T00:48:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:48 smithi093 ceph-mon[118902]: pgmap v7057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:49.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3609137581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:48 smithi093 ceph-mon[111335]: pgmap v7057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:49.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3609137581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:48 smithi136 ceph-mon[113531]: pgmap v7057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3609137581' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:49.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:49.114571+0000) 2023-08-28T00:48:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:50.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:50.114831+0000) 2023-08-28T00:48:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:50 smithi093 ceph-mon[111335]: pgmap v7058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:50 smithi093 ceph-mon[118902]: pgmap v7058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:51.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:51.126Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:51.127Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:50 smithi136 ceph-mon[113531]: pgmap v7058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:51.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:51.115080+0000) 2023-08-28T00:48:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2539236497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:52.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:48:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2539236497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:52.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:48:52.346 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2539236497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:52.346 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:48:52.346 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:52.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:52.115234+0000) 2023-08-28T00:48:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:52.579Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:52.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:48:52.580Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:48:53.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:53 smithi136 ceph-mon[113531]: pgmap v7059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:53.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:53.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:53.118 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:53.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:53.115367+0000) 2023-08-28T00:48:53.448 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:53 smithi093 ceph-mon[111335]: pgmap v7059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:53.448 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:53.448 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:53.449 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:53 smithi093 ceph-mon[118902]: pgmap v7059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:53.449 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:53.449 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:54.084 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3450115312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:54.084 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:54.085 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:54.085 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:54.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:54.115538+0000) 2023-08-28T00:48:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3450115312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3450115312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:48:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:55.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:55.115757+0000) 2023-08-28T00:48:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:55 smithi136 ceph-mon[113531]: pgmap v7060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:48:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:48:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[111335]: pgmap v7060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:48:55.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:48:55] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:48:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[118902]: pgmap v7060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:48:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:48:55.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:48:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:56.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:56.116050+0000) 2023-08-28T00:48:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:56 smithi136 ceph-mon[113531]: pgmap v7061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:57.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/44843838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:57.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:57.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:57.116312+0000) 2023-08-28T00:48:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:56 smithi093 ceph-mon[111335]: pgmap v7061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/44843838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:56 smithi093 ceph-mon[118902]: pgmap v7061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/44843838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:58.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:58.116512+0000) 2023-08-28T00:48:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:58 smithi136 ceph-mon[113531]: pgmap v7062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:59.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:48:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1662606056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:59.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:48:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:48:59.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:48:59.116734+0000) 2023-08-28T00:48:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:58 smithi093 ceph-mon[111335]: pgmap v7062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:48:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1662606056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:48:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:58 smithi093 ceph-mon[118902]: pgmap v7062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:48:59.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:48:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1662606056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:00.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:00.116958+0000) 2023-08-28T00:49:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:00 smithi136 ceph-mon[113531]: pgmap v7063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2213392853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:01.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:01.117160+0000) 2023-08-28T00:49:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:00 smithi093 ceph-mon[111335]: pgmap v7063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:01.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2213392853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:00 smithi093 ceph-mon[118902]: pgmap v7063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2213392853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:01.126Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:01.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:01.127Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:02.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:02.117372+0000) 2023-08-28T00:49:02.907 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:02.579Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:02.907 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:02.580Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:03.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:02 smithi136 ceph-mon[113531]: pgmap v7064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:03.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:03.117585+0000) 2023-08-28T00:49:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:02 smithi093 ceph-mon[111335]: pgmap v7064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:02 smithi093 ceph-mon[118902]: pgmap v7064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:04.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/734139005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:04.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:04.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:04.117817+0000) 2023-08-28T00:49:04.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/734139005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:04.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/734139005' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:04 smithi136 ceph-mon[113531]: pgmap v7065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:05.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:05.118141+0000) 2023-08-28T00:49:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:04 smithi093 ceph-mon[111335]: pgmap v7065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:04 smithi093 ceph-mon[118902]: pgmap v7065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:49:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:49:05] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:49:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3234913397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:06.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:06.118400+0000) 2023-08-28T00:49:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3234913397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3234913397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:06 smithi136 ceph-mon[113531]: pgmap v7066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:07.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:07.118686+0000) 2023-08-28T00:49:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:06 smithi093 ceph-mon[111335]: pgmap v7066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:06 smithi093 ceph-mon[118902]: pgmap v7066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:08.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:08.118932+0000) 2023-08-28T00:49:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:09.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:09.119215+0000) 2023-08-28T00:49:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:08 smithi136 ceph-mon[113531]: pgmap v7067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4019402971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:08 smithi093 ceph-mon[111335]: pgmap v7067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4019402971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:08 smithi093 ceph-mon[118902]: pgmap v7067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4019402971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:09.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:10.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:10.119466+0000) 2023-08-28T00:49:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:10 smithi136 ceph-mon[113531]: pgmap v7068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:11.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4152686889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:11.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:11.119762+0000) 2023-08-28T00:49:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:10 smithi093 ceph-mon[111335]: pgmap v7068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4152686889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:10 smithi093 ceph-mon[118902]: pgmap v7068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4152686889' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:11.127Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:11.128Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:12.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:12.120019+0000) 2023-08-28T00:49:12.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:12.580Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:12.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:12.581Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:13.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:12 smithi136 ceph-mon[113531]: pgmap v7069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:13.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3739712032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:13.325 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:13.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:13.120260+0000) 2023-08-28T00:49:13.325 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:12 smithi093 ceph-mon[111335]: pgmap v7069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:13.327 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3739712032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:13.328 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:12 smithi093 ceph-mon[118902]: pgmap v7069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:13.328 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3739712032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:14.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:14.120482+0000) 2023-08-28T00:49:15.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:14 smithi136 ceph-mon[113531]: pgmap v7070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:15.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:15.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:15.120762+0000) 2023-08-28T00:49:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:14 smithi093 ceph-mon[111335]: pgmap v7070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:14 smithi093 ceph-mon[118902]: pgmap v7070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:49:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:49:15] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:49:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2199976287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2199976287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2199976287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:16.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:16.121017+0000) 2023-08-28T00:49:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:16 smithi093 ceph-mon[111335]: pgmap v7071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:16 smithi093 ceph-mon[118902]: pgmap v7071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:16 smithi136 ceph-mon[113531]: pgmap v7071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:17.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:17.121284+0000) 2023-08-28T00:49:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2923738513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2923738513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2923738513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:18.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:18.121553+0000) 2023-08-28T00:49:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:18 smithi093 ceph-mon[111335]: pgmap v7072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:18 smithi093 ceph-mon[118902]: pgmap v7072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:18 smithi136 ceph-mon[113531]: pgmap v7072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:19.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:19.121759+0000) 2023-08-28T00:49:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3577012039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3577012039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3577012039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:20.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:20.122010+0000) 2023-08-28T00:49:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:20 smithi093 ceph-mon[111335]: pgmap v7073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:20 smithi093 ceph-mon[118902]: pgmap v7073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:21.128Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:21.129Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:20 smithi136 ceph-mon[113531]: pgmap v7073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:21.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:21.122283+0000) 2023-08-28T00:49:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:22.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:22.122466+0000) 2023-08-28T00:49:22.969 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:22.580Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:22.970 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:22.581Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:22 smithi093 ceph-mon[111335]: pgmap v7074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/736935523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:22 smithi093 ceph-mon[118902]: pgmap v7074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:23.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/736935523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:22 smithi136 ceph-mon[113531]: pgmap v7074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/736935523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:23.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:23.122723+0000) 2023-08-28T00:49:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:24.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:24.122957+0000) 2023-08-28T00:49:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:24 smithi093 ceph-mon[111335]: pgmap v7075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3830391965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:24 smithi093 ceph-mon[118902]: pgmap v7075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3830391965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:24 smithi136 ceph-mon[113531]: pgmap v7075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3830391965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:25.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:25.123201+0000) 2023-08-28T00:49:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:49:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:49:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:49:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:26.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:26.123398+0000) 2023-08-28T00:49:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:26 smithi093 ceph-mon[111335]: pgmap v7076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:26 smithi093 ceph-mon[118902]: pgmap v7076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:26 smithi136 ceph-mon[113531]: pgmap v7076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:27.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:27.123578+0000) 2023-08-28T00:49:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/238509249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/238509249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/238509249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:28.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:28.123768+0000) 2023-08-28T00:49:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:28 smithi136 ceph-mon[113531]: pgmap v7077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:29.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:29.124093+0000) 2023-08-28T00:49:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:28 smithi093 ceph-mon[111335]: pgmap v7077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:28 smithi093 ceph-mon[118902]: pgmap v7077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/271074405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:30.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:30.124344+0000) 2023-08-28T00:49:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/271074405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/271074405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:31 smithi136 ceph-mon[113531]: pgmap v7078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:31.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:31.124545+0000) 2023-08-28T00:49:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:31 smithi093 ceph-mon[111335]: pgmap v7078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:31 smithi093 ceph-mon[118902]: pgmap v7078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:31.128Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:31.129Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3603063940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:32.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:32.124782+0000) 2023-08-28T00:49:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3603063940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3603063940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:32.581Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:32.582Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:33 smithi136 ceph-mon[113531]: pgmap v7079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:33.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:33.125003+0000) 2023-08-28T00:49:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:33 smithi093 ceph-mon[111335]: pgmap v7079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:33 smithi093 ceph-mon[118902]: pgmap v7079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:34.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:34.125269+0000) 2023-08-28T00:49:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:35 smithi093 ceph-mon[111335]: pgmap v7080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:35.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1535806564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:35 smithi093 ceph-mon[118902]: pgmap v7080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1535806564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:35 smithi136 ceph-mon[113531]: pgmap v7080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1535806564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:35.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:35.125438+0000) 2023-08-28T00:49:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:49:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:49:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:49:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:36.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:36.125692+0000) 2023-08-28T00:49:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:37 smithi136 ceph-mon[113531]: pgmap v7081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/678255619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:37.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:37.125940+0000) 2023-08-28T00:49:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:37 smithi093 ceph-mon[111335]: pgmap v7081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/678255619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:37 smithi093 ceph-mon[118902]: pgmap v7081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/678255619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:38.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:38.126142+0000) 2023-08-28T00:49:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:39 smithi136 ceph-mon[113531]: pgmap v7082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/402466658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:39.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:39.126391+0000) 2023-08-28T00:49:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:39 smithi093 ceph-mon[111335]: pgmap v7082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/402466658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:39 smithi093 ceph-mon[118902]: pgmap v7082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/402466658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:40.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:40.126637+0000) 2023-08-28T00:49:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:41 smithi136 ceph-mon[113531]: pgmap v7083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:41.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:41.126908+0000) 2023-08-28T00:49:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:41 smithi093 ceph-mon[111335]: pgmap v7083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:41 smithi093 ceph-mon[118902]: pgmap v7083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:41.129Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:41.130Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2514081521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:42.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:42.127142+0000) 2023-08-28T00:49:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2514081521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2514081521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:42.581Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:42.582Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:43 smithi136 ceph-mon[113531]: pgmap v7084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:43.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:43.127394+0000) 2023-08-28T00:49:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:43 smithi093 ceph-mon[111335]: pgmap v7084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:43 smithi093 ceph-mon[118902]: pgmap v7084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3895952757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:44.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:44.127652+0000) 2023-08-28T00:49:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3895952757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3895952757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:45 smithi093 ceph-mon[111335]: pgmap v7085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:45 smithi093 ceph-mon[118902]: pgmap v7085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:45 smithi136 ceph-mon[113531]: pgmap v7085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:45.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:45.127870+0000) 2023-08-28T00:49:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:49:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:49:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:49:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:46.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:46.128110+0000) 2023-08-28T00:49:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:47 smithi136 ceph-mon[113531]: pgmap v7086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/397009138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:47.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:47.128336+0000) 2023-08-28T00:49:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:47 smithi093 ceph-mon[111335]: pgmap v7086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/397009138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:47 smithi093 ceph-mon[118902]: pgmap v7086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/397009138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:48.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:48.128591+0000) 2023-08-28T00:49:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:49 smithi136 ceph-mon[113531]: pgmap v7087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/501294241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:49.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:49.128812+0000) 2023-08-28T00:49:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:49 smithi093 ceph-mon[111335]: pgmap v7087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/501294241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:49 smithi093 ceph-mon[118902]: pgmap v7087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/501294241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:50.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:50.129074+0000) 2023-08-28T00:49:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:51 smithi136 ceph-mon[113531]: pgmap v7088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2947036303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:51.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:51.129362+0000) 2023-08-28T00:49:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:51 smithi093 ceph-mon[111335]: pgmap v7088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2947036303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:51 smithi093 ceph-mon[118902]: pgmap v7088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2947036303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:51.130Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:51.131Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:52.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:52.129613+0000) 2023-08-28T00:49:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:52.582Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:49:52.583Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:49:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:53 smithi136 ceph-mon[113531]: pgmap v7089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:53.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:53.129832+0000) 2023-08-28T00:49:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:53 smithi093 ceph-mon[111335]: pgmap v7089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:53 smithi093 ceph-mon[118902]: pgmap v7089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3105308788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:54.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:54.130112+0000) 2023-08-28T00:49:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3105308788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3105308788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:49:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:55 smithi093 ceph-mon[111335]: pgmap v7090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:49:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:55 smithi093 ceph-mon[118902]: pgmap v7090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:49:55.360 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:49:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:49:55] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:49:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:55.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:55.130264+0000) 2023-08-28T00:49:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:55 smithi136 ceph-mon[113531]: pgmap v7090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:49:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/936610601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:56.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:56.130420+0000) 2023-08-28T00:49:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/936610601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/936610601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:57 smithi136 ceph-mon[113531]: pgmap v7091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:57.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:57.130596+0000) 2023-08-28T00:49:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:57 smithi093 ceph-mon[118902]: pgmap v7091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:57 smithi093 ceph-mon[111335]: pgmap v7091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:58.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:58.130747+0000) 2023-08-28T00:49:59.131 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.131 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-mon[113531]: pgmap v7092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:59.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3483348674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:59.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:49:59.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:49:59.132 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:49:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[111335]: pgmap v7092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3483348674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:49:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:49:59.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[118902]: pgmap v7092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:49:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3483348674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:49:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:49:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:49:59.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:49:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:49:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:49:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:49:59.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:49:59.130958+0000) 2023-08-28T00:50:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:00.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:00.131230+0000) 2023-08-28T00:50:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:00 smithi136 ceph-mon[113531]: pgmap v7093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:50:01.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4287690523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:01.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:01.131500+0000) 2023-08-28T00:50:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:00 smithi093 ceph-mon[111335]: pgmap v7093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:50:01.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4287690523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:00 smithi093 ceph-mon[118902]: pgmap v7093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T00:50:01.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4287690523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:01.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:01.131Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:01.242 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:01.133Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:02.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:02.131723+0000) 2023-08-28T00:50:02.913 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:02.582Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:02.913 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:02.583Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:02 smithi136 ceph-mon[113531]: pgmap v7094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:03.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4084035340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:03.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:03.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:03.131934+0000) 2023-08-28T00:50:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:02 smithi093 ceph-mon[111335]: pgmap v7094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:03.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4084035340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:03.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:02 smithi093 ceph-mon[118902]: pgmap v7094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:03.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4084035340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:04.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:04.132187+0000) 2023-08-28T00:50:05.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:04 smithi136 ceph-mon[113531]: pgmap v7095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:05.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:05.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:05.132419+0000) 2023-08-28T00:50:05.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:04 smithi093 ceph-mon[111335]: pgmap v7095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:05.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:04 smithi093 ceph-mon[118902]: pgmap v7095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:50:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:50:05] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:50:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2561182559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:06.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:06.132656+0000) 2023-08-28T00:50:06.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2561182559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:06.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2561182559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:07.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:06 smithi136 ceph-mon[113531]: pgmap v7096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:07.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:07.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:07.132947+0000) 2023-08-28T00:50:07.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:06 smithi093 ceph-mon[111335]: pgmap v7096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:07.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:06 smithi093 ceph-mon[118902]: pgmap v7096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:08.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/52954257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:08.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:08.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:08.133178+0000) 2023-08-28T00:50:08.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/52954257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:08.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/52954257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:08 smithi136 ceph-mon[113531]: pgmap v7097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:09.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:09.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:09.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:09.133400+0000) 2023-08-28T00:50:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:08 smithi093 ceph-mon[111335]: pgmap v7097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:09.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:08 smithi093 ceph-mon[118902]: pgmap v7097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:10.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2614815454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:10.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:10.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:10.133625+0000) 2023-08-28T00:50:10.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2614815454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:10.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2614815454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:11.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:10 smithi136 ceph-mon[113531]: pgmap v7098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:11.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:11.133871+0000) 2023-08-28T00:50:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:10 smithi093 ceph-mon[111335]: pgmap v7098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:10 smithi093 ceph-mon[118902]: pgmap v7098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:11.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:11.132Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:11.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:11.133Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:12.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:12.134153+0000) 2023-08-28T00:50:12.939 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:12.583Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:12.939 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:12.584Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:12 smithi093 ceph-mon[111335]: pgmap v7099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:13.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2539634947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:12 smithi093 ceph-mon[118902]: pgmap v7099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:13.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2539634947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:12 smithi136 ceph-mon[113531]: pgmap v7099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2539634947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:13.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:13.134382+0000) 2023-08-28T00:50:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:14.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:14.134624+0000) 2023-08-28T00:50:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:14 smithi093 ceph-mon[111335]: pgmap v7100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3711945373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:14 smithi093 ceph-mon[118902]: pgmap v7100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3711945373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:14 smithi136 ceph-mon[113531]: pgmap v7100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3711945373' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:15.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:15.134854+0000) 2023-08-28T00:50:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:50:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:50:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:50:16.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:16.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:16.135114+0000) 2023-08-28T00:50:17.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:16 smithi093 ceph-mon[111335]: pgmap v7101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:17.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:16 smithi093 ceph-mon[118902]: pgmap v7101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:16 smithi136 ceph-mon[113531]: pgmap v7101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:17.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:17.135335+0000) 2023-08-28T00:50:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3865901713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3865901713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3865901713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:18.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:18.135537+0000) 2023-08-28T00:50:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:18 smithi093 ceph-mon[111335]: pgmap v7102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:18 smithi093 ceph-mon[118902]: pgmap v7102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:18 smithi136 ceph-mon[113531]: pgmap v7102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:19.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:19.135791+0000) 2023-08-28T00:50:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/107421831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/107421831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:20.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/107421831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:20.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:20.135991+0000) 2023-08-28T00:50:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:20 smithi093 ceph-mon[111335]: pgmap v7103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:20 smithi093 ceph-mon[118902]: pgmap v7103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:21.133Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:21.134Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:20 smithi136 ceph-mon[113531]: pgmap v7103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:21.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:21.136214+0000) 2023-08-28T00:50:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2729176562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2729176562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:22.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2729176562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:22.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:22.136451+0000) 2023-08-28T00:50:22.970 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:22.584Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:22.970 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:22.585Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:22 smithi093 ceph-mon[111335]: pgmap v7104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:22 smithi093 ceph-mon[118902]: pgmap v7104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:22 smithi136 ceph-mon[113531]: pgmap v7104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:23.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:23.136696+0000) 2023-08-28T00:50:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:24.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:24.136953+0000) 2023-08-28T00:50:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:24 smithi093 ceph-mon[111335]: pgmap v7105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:25.336 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/622855323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:25.338 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:24 smithi093 ceph-mon[118902]: pgmap v7105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:25.338 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/622855323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:24 smithi136 ceph-mon[113531]: pgmap v7105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:25.522 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/622855323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:25.522 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:25.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:25.137285+0000) 2023-08-28T00:50:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:50:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:50:25] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-28T00:50:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:26.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:26.137555+0000) 2023-08-28T00:50:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:26 smithi093 ceph-mon[118902]: pgmap v7106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:27.346 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1609617847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:27.346 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:26 smithi093 ceph-mon[111335]: pgmap v7106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:27.346 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1609617847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:26 smithi136 ceph-mon[113531]: pgmap v7106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:27.983 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1609617847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:27.983 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:27.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:27.137849+0000) 2023-08-28T00:50:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:28.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:28.138141+0000) 2023-08-28T00:50:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:28 smithi093 ceph-mon[118902]: pgmap v7107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:29.287 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/612550162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:29.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:28 smithi093 ceph-mon[111335]: pgmap v7107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:29.287 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/612550162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:28 smithi136 ceph-mon[113531]: pgmap v7107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/612550162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:29.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:29.138370+0000) 2023-08-28T00:50:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:30.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:30.138601+0000) 2023-08-28T00:50:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:30 smithi093 ceph-mon[111335]: pgmap v7108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:31.414 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:30 smithi093 ceph-mon[118902]: pgmap v7108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:31.415 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:31.133Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:31.415 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:31.134Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:30 smithi136 ceph-mon[113531]: pgmap v7108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:31.528 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:31.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:31.138872+0000) 2023-08-28T00:50:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/959684378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:32.683 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/959684378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:32.684 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/959684378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:32.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:32.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:32.139159+0000) 2023-08-28T00:50:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:32.584Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:33.304 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:32.585Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:32 smithi136 ceph-mon[113531]: pgmap v7109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:33.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:33.139377+0000) 2023-08-28T00:50:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:32 smithi093 ceph-mon[111335]: pgmap v7109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:33.502 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:32 smithi093 ceph-mon[118902]: pgmap v7109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1601294075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:34.272 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1601294075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:34.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1601294075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:35.054 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:34.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:34.139621+0000) 2023-08-28T00:50:35.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:34 smithi093 ceph-mon[111335]: pgmap v7110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:35.382 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:34 smithi093 ceph-mon[118902]: pgmap v7110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:34 smithi136 ceph-mon[113531]: pgmap v7110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:35.514 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:35.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:35.139859+0000) 2023-08-28T00:50:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:50:35] "GET /metrics HTTP/1.1" 200 239922 "" "Prometheus/2.43.0" 2023-08-28T00:50:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:36.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:36.140140+0000) 2023-08-28T00:50:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:37 smithi136 ceph-mon[113531]: pgmap v7111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3306374077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:37.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:37.140428+0000) 2023-08-28T00:50:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:37 smithi093 ceph-mon[111335]: pgmap v7111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3306374077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:37 smithi093 ceph-mon[118902]: pgmap v7111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:37.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3306374077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:38.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:38.140686+0000) 2023-08-28T00:50:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:39 smithi136 ceph-mon[113531]: pgmap v7112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3748421500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:39.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:39.140960+0000) 2023-08-28T00:50:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:39 smithi093 ceph-mon[111335]: pgmap v7112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3748421500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:39 smithi093 ceph-mon[118902]: pgmap v7112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3748421500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:40.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:40.141162+0000) 2023-08-28T00:50:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:41 smithi136 ceph-mon[113531]: pgmap v7113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2657158147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:41.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:41.141425+0000) 2023-08-28T00:50:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:41 smithi093 ceph-mon[111335]: pgmap v7113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2657158147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:41 smithi093 ceph-mon[118902]: pgmap v7113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2657158147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:41.135Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:41.135Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:42.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:42.141696+0000) 2023-08-28T00:50:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:42.584Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:42.585Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:43 smithi136 ceph-mon[113531]: pgmap v7114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:43.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:43.141967+0000) 2023-08-28T00:50:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:43 smithi093 ceph-mon[111335]: pgmap v7114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:43 smithi093 ceph-mon[118902]: pgmap v7114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2290771323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:44.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:44.142243+0000) 2023-08-28T00:50:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2290771323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2290771323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:45.363 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:45 smithi093 ceph-mon[111335]: pgmap v7115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:45.363 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:45 smithi093 ceph-mon[118902]: pgmap v7115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:45 smithi136 ceph-mon[113531]: pgmap v7115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:45.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:45.142505+0000) 2023-08-28T00:50:45.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:50:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:50:45] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:50:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/492981426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:46.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:46.142763+0000) 2023-08-28T00:50:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/492981426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/492981426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:47 smithi136 ceph-mon[113531]: pgmap v7116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:47.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:47.142997+0000) 2023-08-28T00:50:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:47 smithi093 ceph-mon[111335]: pgmap v7116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:47 smithi093 ceph-mon[118902]: pgmap v7116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2085448386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:48.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:48.143176+0000) 2023-08-28T00:50:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2085448386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2085448386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:49 smithi136 ceph-mon[113531]: pgmap v7117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:49.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:49.143446+0000) 2023-08-28T00:50:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:49 smithi093 ceph-mon[111335]: pgmap v7117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:49 smithi093 ceph-mon[118902]: pgmap v7117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:50.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:50.143687+0000) 2023-08-28T00:50:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:51 smithi136 ceph-mon[113531]: pgmap v7118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1943300308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:51.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:51.143940+0000) 2023-08-28T00:50:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:51 smithi093 ceph-mon[111335]: pgmap v7118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1943300308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:51 smithi093 ceph-mon[118902]: pgmap v7118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:51.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1943300308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:51.135Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:51.136Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:52.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:52.144197+0000) 2023-08-28T00:50:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:52.585Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:50:52.586Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:50:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:53 smithi136 ceph-mon[113531]: pgmap v7119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3883455088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:53.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:53.144450+0000) 2023-08-28T00:50:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:53 smithi093 ceph-mon[111335]: pgmap v7119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3883455088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:53 smithi093 ceph-mon[118902]: pgmap v7119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:53.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3883455088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:54.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:54.144696+0000) 2023-08-28T00:50:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:50:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:55 smithi093 ceph-mon[111335]: pgmap v7120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:55 smithi093 ceph-mon[118902]: pgmap v7120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:55 smithi136 ceph-mon[113531]: pgmap v7120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:55.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:55.144953+0000) 2023-08-28T00:50:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:50:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:50:55] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T00:50:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2704835321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:56.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:56.145187+0000) 2023-08-28T00:50:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2704835321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2704835321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:57 smithi136 ceph-mon[113531]: pgmap v7121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:57.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:57.145441+0000) 2023-08-28T00:50:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:57 smithi093 ceph-mon[111335]: pgmap v7121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:57 smithi093 ceph-mon[118902]: pgmap v7121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1614430391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:58.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:58.145717+0000) 2023-08-28T00:50:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1614430391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1614430391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:50:59.372 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:59 smithi136 ceph-mon[113531]: pgmap v7122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:59.372 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:50:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:50:59.372 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:50:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:50:59.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:50:59.145895+0000) 2023-08-28T00:50:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:59 smithi093 ceph-mon[118902]: pgmap v7122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:50:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:50:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:59 smithi093 ceph-mon[111335]: pgmap v7122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:50:59.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:50:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:51:00.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3981176368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:00.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3981176368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3981176368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:00.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:00.146121+0000) 2023-08-28T00:51:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:01 smithi136 ceph-mon[113531]: pgmap v7123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:01.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:01.146306+0000) 2023-08-28T00:51:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[111335]: pgmap v7123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[118902]: pgmap v7123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:01.136Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:01.137Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:51:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:51:02.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:02.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:02.146532+0000) 2023-08-28T00:51:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:51:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:51:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:51:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:51:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:51:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:02.585Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:02.586Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:03 smithi136 ceph-mon[113531]: pgmap v7124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:03.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3781443305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:03.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:03.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:03.146744+0000) 2023-08-28T00:51:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:03 smithi093 ceph-mon[111335]: pgmap v7124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:03.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3781443305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:03.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:03 smithi093 ceph-mon[118902]: pgmap v7124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:03.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3781443305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:04.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:04.147000+0000) 2023-08-28T00:51:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:05 smithi136 ceph-mon[113531]: pgmap v7125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1649562656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:05.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 00:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T00:51:05.319401597Z level=info msg="Completed cleanup jobs" duration=14.005294ms 2023-08-28T00:51:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:05.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:05.147253+0000) 2023-08-28T00:51:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:05 smithi093 ceph-mon[111335]: pgmap v7125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1649562656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:05 smithi093 ceph-mon[118902]: pgmap v7125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:05.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1649562656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:05.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:51:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:51:05] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T00:51:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:06.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:06.147547+0000) 2023-08-28T00:51:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:07 smithi136 ceph-mon[113531]: pgmap v7126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3330537181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:07.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:07.147793+0000) 2023-08-28T00:51:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:07 smithi093 ceph-mon[111335]: pgmap v7126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3330537181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:07 smithi093 ceph-mon[118902]: pgmap v7126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3330537181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:08.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:08.148048+0000) 2023-08-28T00:51:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:09 smithi136 ceph-mon[113531]: pgmap v7127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:09.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:09.148292+0000) 2023-08-28T00:51:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:09 smithi093 ceph-mon[111335]: pgmap v7127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:09 smithi093 ceph-mon[118902]: pgmap v7127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1570508246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:10.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:10.148567+0000) 2023-08-28T00:51:10.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1570508246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:10.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1570508246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:11 smithi136 ceph-mon[113531]: pgmap v7128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:11.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:11.148840+0000) 2023-08-28T00:51:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:11 smithi093 ceph-mon[111335]: pgmap v7128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:11 smithi093 ceph-mon[118902]: pgmap v7128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:11.137Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:11.137Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1529371367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:12.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:12.149114+0000) 2023-08-28T00:51:12.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1529371367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:12.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1529371367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:12.586Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:12.587Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:13 smithi136 ceph-mon[113531]: pgmap v7129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:13.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:13.149385+0000) 2023-08-28T00:51:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:13 smithi093 ceph-mon[111335]: pgmap v7129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:13 smithi093 ceph-mon[118902]: pgmap v7129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2737823534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:14.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:14.149629+0000) 2023-08-28T00:51:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2737823534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2737823534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:15 smithi136 ceph-mon[113531]: pgmap v7130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:15.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:15.149901+0000) 2023-08-28T00:51:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:15 smithi093 ceph-mon[111335]: pgmap v7130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:51:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:51:15] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:51:15.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:15 smithi093 ceph-mon[118902]: pgmap v7130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:16.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:16.150180+0000) 2023-08-28T00:51:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:17 smithi136 ceph-mon[113531]: pgmap v7131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4260405814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:17.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:17.150452+0000) 2023-08-28T00:51:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:17 smithi093 ceph-mon[111335]: pgmap v7131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:17.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4260405814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:17.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:17 smithi093 ceph-mon[118902]: pgmap v7131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:17.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4260405814' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:18.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:18.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:18.150719+0000) 2023-08-28T00:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:19 smithi136 ceph-mon[113531]: pgmap v7132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1597338973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:19.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:19.150986+0000) 2023-08-28T00:51:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:19 smithi093 ceph-mon[111335]: pgmap v7132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1597338973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:19 smithi093 ceph-mon[118902]: pgmap v7132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:19.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1597338973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:20.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:20.151287+0000) 2023-08-28T00:51:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:21 smithi136 ceph-mon[113531]: pgmap v7133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:21.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:21.151508+0000) 2023-08-28T00:51:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:21 smithi093 ceph-mon[111335]: pgmap v7133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:21 smithi093 ceph-mon[118902]: pgmap v7133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:21.138Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:21.138Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3685986370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:22.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:22.151751+0000) 2023-08-28T00:51:22.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3685986370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3685986370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:22.587Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:22.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:22.588Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:23 smithi136 ceph-mon[113531]: pgmap v7134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:23.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:23.152015+0000) 2023-08-28T00:51:23.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:23 smithi093 ceph-mon[111335]: pgmap v7134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:23.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:23 smithi093 ceph-mon[118902]: pgmap v7134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1096951257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:24.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:24.152294+0000) 2023-08-28T00:51:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1096951257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1096951257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:24.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:25 smithi136 ceph-mon[113531]: pgmap v7135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:25.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:25.152568+0000) 2023-08-28T00:51:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:25 smithi093 ceph-mon[111335]: pgmap v7135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:51:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:51:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:51:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:25 smithi093 ceph-mon[118902]: pgmap v7135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:26.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:26.152843+0000) 2023-08-28T00:51:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:26 smithi136 ceph-mon[113531]: pgmap v7136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:27.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2709864311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:27.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:27.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:27.153200+0000) 2023-08-28T00:51:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:26 smithi093 ceph-mon[111335]: pgmap v7136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2709864311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:26 smithi093 ceph-mon[118902]: pgmap v7136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2709864311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:28.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:28.153443+0000) 2023-08-28T00:51:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:28 smithi136 ceph-mon[113531]: pgmap v7137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:29.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2639098346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:29.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:29.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:29.153657+0000) 2023-08-28T00:51:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:28 smithi093 ceph-mon[111335]: pgmap v7137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2639098346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:28 smithi093 ceph-mon[118902]: pgmap v7137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2639098346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:30.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:30.153889+0000) 2023-08-28T00:51:31.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:30 smithi136 ceph-mon[113531]: pgmap v7138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:31.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:31.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:31.154152+0000) 2023-08-28T00:51:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:30 smithi093 ceph-mon[111335]: pgmap v7138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:30 smithi093 ceph-mon[118902]: pgmap v7138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:31.138Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:31.139Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:32.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1886078924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:32.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:32.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:32.154397+0000) 2023-08-28T00:51:32.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1886078924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:32.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1886078924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:32.933 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:32.588Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:32.934 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:32.589Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:33.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:32 smithi136 ceph-mon[113531]: pgmap v7139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:33.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:33.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:33.154636+0000) 2023-08-28T00:51:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:32 smithi093 ceph-mon[111335]: pgmap v7139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:32 smithi093 ceph-mon[118902]: pgmap v7139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:34.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1939591314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:34.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:34.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:34.154889+0000) 2023-08-28T00:51:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1939591314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1939591314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:34 smithi093 ceph-mon[111335]: pgmap v7140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:34 smithi093 ceph-mon[118902]: pgmap v7140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:34 smithi136 ceph-mon[113531]: pgmap v7140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:35.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:35.155151+0000) 2023-08-28T00:51:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:51:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:51:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:51:36.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1178406874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:36.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1178406874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1178406874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:36.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:36.155392+0000) 2023-08-28T00:51:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:36 smithi093 ceph-mon[111335]: pgmap v7141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:36 smithi093 ceph-mon[118902]: pgmap v7141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:37.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:36 smithi136 ceph-mon[113531]: pgmap v7141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:37.443 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:37.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:37.155696+0000) 2023-08-28T00:51:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:38.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:38.155923+0000) 2023-08-28T00:51:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:38 smithi093 ceph-mon[118902]: pgmap v7142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4220440058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:39.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:38 smithi093 ceph-mon[111335]: pgmap v7142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:39.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4220440058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:39.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:38 smithi136 ceph-mon[113531]: pgmap v7142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4220440058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:39.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:39.156167+0000) 2023-08-28T00:51:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:40.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:40.156357+0000) 2023-08-28T00:51:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:40 smithi093 ceph-mon[111335]: pgmap v7143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4120475247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:40 smithi093 ceph-mon[118902]: pgmap v7143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:41.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4120475247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:41.139Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:41.140Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:40 smithi136 ceph-mon[113531]: pgmap v7143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4120475247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:41.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:41.156615+0000) 2023-08-28T00:51:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:42.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:42.156881+0000) 2023-08-28T00:51:42.956 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:42.588Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:42.956 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:42.589Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:42 smithi093 ceph-mon[111335]: pgmap v7144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2387536699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:42 smithi093 ceph-mon[118902]: pgmap v7144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2387536699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:42 smithi136 ceph-mon[113531]: pgmap v7144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2387536699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:43.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:43.157165+0000) 2023-08-28T00:51:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:44.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:44.157383+0000) 2023-08-28T00:51:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:44 smithi093 ceph-mon[111335]: pgmap v7145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:44 smithi093 ceph-mon[118902]: pgmap v7145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:44 smithi136 ceph-mon[113531]: pgmap v7145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:45.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:45.157593+0000) 2023-08-28T00:51:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:51:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:51:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:51:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3827413795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3827413795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3827413795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:46.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:46.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:46.157836+0000) 2023-08-28T00:51:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:46 smithi093 ceph-mon[111335]: pgmap v7146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:47.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:46 smithi093 ceph-mon[118902]: pgmap v7146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:46 smithi136 ceph-mon[113531]: pgmap v7146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:47.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:47.158114+0000) 2023-08-28T00:51:48.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2218375386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:48.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2218375386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2218375386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:48.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:48.158390+0000) 2023-08-28T00:51:49.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:48 smithi093 ceph-mon[111335]: pgmap v7147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:48 smithi093 ceph-mon[118902]: pgmap v7147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:48 smithi136 ceph-mon[113531]: pgmap v7147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:49.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:49.158628+0000) 2023-08-28T00:51:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:50.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:50.158894+0000) 2023-08-28T00:51:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:50 smithi093 ceph-mon[111335]: pgmap v7148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:51.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1111616271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:51.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:50 smithi093 ceph-mon[118902]: pgmap v7148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:51.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1111616271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:51.140Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:51.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:51.140Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:50 smithi136 ceph-mon[113531]: pgmap v7148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1111616271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:51.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:51.159134+0000) 2023-08-28T00:51:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:52.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:52.159361+0000) 2023-08-28T00:51:52.979 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:52.589Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:52.980 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:51:52.590Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:51:53.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:52 smithi093 ceph-mon[111335]: pgmap v7149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:53.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3839616385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:52 smithi093 ceph-mon[118902]: pgmap v7149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:53.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3839616385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:52 smithi136 ceph-mon[113531]: pgmap v7149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3839616385' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:53.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:53.159606+0000) 2023-08-28T00:51:54.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:54.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:51:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:54.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:54.159863+0000) 2023-08-28T00:51:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:54 smithi093 ceph-mon[111335]: pgmap v7150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:55.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1281505219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:54 smithi093 ceph-mon[118902]: pgmap v7150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1281505219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:54 smithi136 ceph-mon[113531]: pgmap v7150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1281505219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:55.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:55.160133+0000) 2023-08-28T00:51:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:51:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:51:55] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:51:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:56.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:56.160386+0000) 2023-08-28T00:51:57.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:56 smithi093 ceph-mon[111335]: pgmap v7151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:57.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:56 smithi093 ceph-mon[118902]: pgmap v7151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:56 smithi136 ceph-mon[113531]: pgmap v7151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:57.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:57.160671+0000) 2023-08-28T00:51:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2107467520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:58.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:58.160931+0000) 2023-08-28T00:51:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2107467520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2107467520' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:51:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:51:58 smithi136 ceph-mon[113531]: pgmap v7152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:51:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:51:59.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:51:59.161221+0000) 2023-08-28T00:51:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:51:58 smithi093 ceph-mon[111335]: pgmap v7152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:51:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:51:58 smithi093 ceph-mon[118902]: pgmap v7152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2706932264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:00.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:00.161425+0000) 2023-08-28T00:52:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2706932264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2706932264' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:01 smithi136 ceph-mon[113531]: pgmap v7153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:01.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:01.161643+0000) 2023-08-28T00:52:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:01 smithi093 ceph-mon[111335]: pgmap v7153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:01 smithi093 ceph-mon[118902]: pgmap v7153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:01.140Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:01.141Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:02.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:52:02.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/736138228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:02.432 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:02.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:02.161783+0000) 2023-08-28T00:52:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:52:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/736138228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:52:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/736138228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:02.590Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:02.591Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:03 smithi136 ceph-mon[113531]: pgmap v7154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:03.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:03.162021+0000) 2023-08-28T00:52:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:03 smithi093 ceph-mon[111335]: pgmap v7154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:03 smithi093 ceph-mon[118902]: pgmap v7154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1452338331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:04.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:04.162227+0000) 2023-08-28T00:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1452338331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:04.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1452338331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:05 smithi136 ceph-mon[113531]: pgmap v7155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:05.927 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:52:05.927 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:52:05.927 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:05.927 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:05.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:05.162457+0000) 2023-08-28T00:52:05.928 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:52:05] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:52:05.928 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[111335]: pgmap v7155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:05.928 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:52:05.928 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:52:05.929 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:05.929 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[118902]: pgmap v7155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:05.929 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:52:05.929 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:52:05.929 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:52:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:06.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:06.162722+0000) 2023-08-28T00:52:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:07 smithi136 ceph-mon[113531]: pgmap v7156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2359117360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:07.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:07.162962+0000) 2023-08-28T00:52:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:07 smithi093 ceph-mon[111335]: pgmap v7156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2359117360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:07 smithi093 ceph-mon[118902]: pgmap v7156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:07.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2359117360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:08.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:08.163235+0000) 2023-08-28T00:52:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:09.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:09.163503+0000) 2023-08-28T00:52:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:09 smithi136 ceph-mon[113531]: pgmap v7157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/154142869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:09 smithi093 ceph-mon[111335]: pgmap v7157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/154142869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:09 smithi093 ceph-mon[118902]: pgmap v7157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:09.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/154142869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:10.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:10.163759+0000) 2023-08-28T00:52:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:11 smithi136 ceph-mon[113531]: pgmap v7158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:11.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:11.164048+0000) 2023-08-28T00:52:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:11 smithi093 ceph-mon[111335]: pgmap v7158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:11 smithi093 ceph-mon[118902]: pgmap v7158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:11.141Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:11.142Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/948445586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:12.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:12.164254+0000) 2023-08-28T00:52:12.592 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/948445586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:12.592 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/948445586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:12.591Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:12.592Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:13 smithi136 ceph-mon[113531]: pgmap v7159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:13.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:13.164506+0000) 2023-08-28T00:52:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:13 smithi093 ceph-mon[111335]: pgmap v7159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:13.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:13 smithi093 ceph-mon[118902]: pgmap v7159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/663862007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:14.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:14.164728+0000) 2023-08-28T00:52:14.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/663862007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:14.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/663862007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:15 smithi136 ceph-mon[113531]: pgmap v7160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:15.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:15.164959+0000) 2023-08-28T00:52:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:15 smithi093 ceph-mon[111335]: pgmap v7160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:15 smithi093 ceph-mon[118902]: pgmap v7160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:52:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:52:15] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:52:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:16.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:16.165218+0000) 2023-08-28T00:52:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:16 smithi136 ceph-mon[113531]: pgmap v7161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:17.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1655598534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:17.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:17.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:17.165430+0000) 2023-08-28T00:52:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:16 smithi093 ceph-mon[111335]: pgmap v7161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1655598534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:16 smithi093 ceph-mon[118902]: pgmap v7161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1655598534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:18.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:18.165704+0000) 2023-08-28T00:52:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:18 smithi136 ceph-mon[113531]: pgmap v7162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:19.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1390482664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:19.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:19.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:19.165970+0000) 2023-08-28T00:52:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:18 smithi093 ceph-mon[111335]: pgmap v7162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1390482664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:18 smithi093 ceph-mon[118902]: pgmap v7162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:19.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1390482664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:20.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:20.166218+0000) 2023-08-28T00:52:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:20 smithi136 ceph-mon[113531]: pgmap v7163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:21.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:21.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:21.166454+0000) 2023-08-28T00:52:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:20 smithi093 ceph-mon[111335]: pgmap v7163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:21.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:20 smithi093 ceph-mon[118902]: pgmap v7163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:21.142Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:21.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:21.142Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1364168177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1364168177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:22.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1364168177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:22.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:22.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:22.166705+0000) 2023-08-28T00:52:22.944 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:22.592Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:22.944 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:22.593Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:22 smithi093 ceph-mon[111335]: pgmap v7164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:22 smithi093 ceph-mon[118902]: pgmap v7164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:22 smithi136 ceph-mon[113531]: pgmap v7164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:23.561 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:23.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:23.166964+0000) 2023-08-28T00:52:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1940746233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1940746233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1940746233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:24.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:24.167205+0000) 2023-08-28T00:52:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:24 smithi093 ceph-mon[111335]: pgmap v7165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:25.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:24 smithi093 ceph-mon[118902]: pgmap v7165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:24 smithi136 ceph-mon[113531]: pgmap v7165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:25.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:25.167395+0000) 2023-08-28T00:52:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:52:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:52:25] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:52:26.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1499850132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:26.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1499850132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1499850132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:26.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:26.167620+0000) 2023-08-28T00:52:27.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:26 smithi093 ceph-mon[111335]: pgmap v7166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:27.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:26 smithi093 ceph-mon[118902]: pgmap v7166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:26 smithi136 ceph-mon[113531]: pgmap v7166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:27.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:27.167886+0000) 2023-08-28T00:52:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:28.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:28.168144+0000) 2023-08-28T00:52:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:28 smithi093 ceph-mon[111335]: pgmap v7167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3653838428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:28 smithi093 ceph-mon[118902]: pgmap v7167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:29.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3653838428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:28 smithi136 ceph-mon[113531]: pgmap v7167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3653838428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:29.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:29.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:29.168367+0000) 2023-08-28T00:52:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:30.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:30.168619+0000) 2023-08-28T00:52:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:30 smithi093 ceph-mon[118902]: pgmap v7168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2426020409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:30 smithi093 ceph-mon[111335]: pgmap v7168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2426020409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:31.142Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:31.143Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:30 smithi136 ceph-mon[113531]: pgmap v7168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2426020409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:31.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:31.168866+0000) 2023-08-28T00:52:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:32.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:32.169121+0000) 2023-08-28T00:52:32.973 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:32.592Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:32.973 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:32.593Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:32 smithi093 ceph-mon[111335]: pgmap v7169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1336799565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:32 smithi093 ceph-mon[118902]: pgmap v7169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1336799565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:32 smithi136 ceph-mon[113531]: pgmap v7169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1336799565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:33.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:33.169372+0000) 2023-08-28T00:52:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:34.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:34.169616+0000) 2023-08-28T00:52:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:34 smithi093 ceph-mon[111335]: pgmap v7170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:34 smithi093 ceph-mon[118902]: pgmap v7170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:34 smithi136 ceph-mon[113531]: pgmap v7170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:35.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:35.169833+0000) 2023-08-28T00:52:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:52:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:52:35] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:52:36.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2642715838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:36.242 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2642715838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:36.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2642715838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:36.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:36.170126+0000) 2023-08-28T00:52:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:36 smithi093 ceph-mon[111335]: pgmap v7171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:36 smithi093 ceph-mon[118902]: pgmap v7171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:36 smithi136 ceph-mon[113531]: pgmap v7171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:37.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:37.170344+0000) 2023-08-28T00:52:38.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3612653335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:38.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3612653335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3612653335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:38.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:38.170639+0000) 2023-08-28T00:52:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:38 smithi093 ceph-mon[111335]: pgmap v7172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:38 smithi093 ceph-mon[118902]: pgmap v7172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:38 smithi136 ceph-mon[113531]: pgmap v7172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:39.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:39.170886+0000) 2023-08-28T00:52:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:40.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:40.171121+0000) 2023-08-28T00:52:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:40 smithi136 ceph-mon[113531]: pgmap v7173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2324854875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:41.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:41.171378+0000) 2023-08-28T00:52:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:40 smithi093 ceph-mon[118902]: pgmap v7173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2324854875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:40 smithi093 ceph-mon[111335]: pgmap v7173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:41.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2324854875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:41.143Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:41.144Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:42.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:42.171667+0000) 2023-08-28T00:52:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:42.593Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:42.594Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:42 smithi136 ceph-mon[113531]: pgmap v7174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/602513232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:43.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:43.171920+0000) 2023-08-28T00:52:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:42 smithi093 ceph-mon[111335]: pgmap v7174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/602513232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:42 smithi093 ceph-mon[118902]: pgmap v7174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/602513232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:44.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:44.172159+0000) 2023-08-28T00:52:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:45 smithi093 ceph-mon[111335]: pgmap v7175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:45.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3762613027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:45 smithi093 ceph-mon[118902]: pgmap v7175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:45.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3762613027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:45 smithi136 ceph-mon[113531]: pgmap v7175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3762613027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:45.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:45.172377+0000) 2023-08-28T00:52:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:52:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:52:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:52:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:46.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:46.172579+0000) 2023-08-28T00:52:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:47 smithi136 ceph-mon[113531]: pgmap v7176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:47.495 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:47.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:47.172831+0000) 2023-08-28T00:52:47.496 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:47 smithi093 ceph-mon[111335]: pgmap v7176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:47.496 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:47 smithi093 ceph-mon[118902]: pgmap v7176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3288264051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:48.873 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:48.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:48.173121+0000) 2023-08-28T00:52:48.874 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3288264051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:48.883 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3288264051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:49 smithi136 ceph-mon[113531]: pgmap v7177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:49.515 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:49.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:49.173343+0000) 2023-08-28T00:52:49.516 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:49 smithi093 ceph-mon[111335]: pgmap v7177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:49.516 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:49 smithi093 ceph-mon[118902]: pgmap v7177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:50.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3774711104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:50.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:50.173564+0000) 2023-08-28T00:52:50.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3774711104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3774711104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:51 smithi136 ceph-mon[113531]: pgmap v7178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:51.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:51.173826+0000) 2023-08-28T00:52:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:51 smithi093 ceph-mon[111335]: pgmap v7178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:51 smithi093 ceph-mon[118902]: pgmap v7178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:51.144Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:51.145Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3662577891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:52.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:52.174081+0000) 2023-08-28T00:52:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3662577891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3662577891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:52.594Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:52:52.595Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:52:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:53 smithi136 ceph-mon[113531]: pgmap v7179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:53.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:53.174335+0000) 2023-08-28T00:52:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:53 smithi093 ceph-mon[118902]: pgmap v7179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:53 smithi093 ceph-mon[111335]: pgmap v7179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:54.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:54.174558+0000) 2023-08-28T00:52:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:52:55.361 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:55 smithi093 ceph-mon[118902]: pgmap v7180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:55.362 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1422898905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:55.362 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:55 smithi093 ceph-mon[111335]: pgmap v7180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:55.362 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1422898905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:55 smithi136 ceph-mon[113531]: pgmap v7180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1422898905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:55.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:55.174805+0000) 2023-08-28T00:52:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:52:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:52:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:52:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:56.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:56.175069+0000) 2023-08-28T00:52:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:57 smithi136 ceph-mon[113531]: pgmap v7181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1362418888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:57.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:57.175318+0000) 2023-08-28T00:52:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:57 smithi093 ceph-mon[118902]: pgmap v7181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1362418888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:57 smithi093 ceph-mon[111335]: pgmap v7181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1362418888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:52:58.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:58.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:58.175538+0000) 2023-08-28T00:52:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:52:59 smithi136 ceph-mon[113531]: pgmap v7182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:52:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:52:59.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:52:59.175778+0000) 2023-08-28T00:52:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:52:59 smithi093 ceph-mon[111335]: pgmap v7182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:52:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:52:59 smithi093 ceph-mon[118902]: pgmap v7182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:00.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3266289679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:00.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:00.175994+0000) 2023-08-28T00:53:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3266289679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3266289679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:01 smithi136 ceph-mon[113531]: pgmap v7183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:01.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:01.176247+0000) 2023-08-28T00:53:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:01 smithi093 ceph-mon[111335]: pgmap v7183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:01 smithi093 ceph-mon[118902]: pgmap v7183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:01.145Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:01.146Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/244937316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:02.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:02.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:02.176490+0000) 2023-08-28T00:53:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/244937316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/244937316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:02.595Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:02.596Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:03 smithi136 ceph-mon[113531]: pgmap v7184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:03.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:03.176746+0000) 2023-08-28T00:53:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:03 smithi093 ceph-mon[111335]: pgmap v7184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:03 smithi093 ceph-mon[118902]: pgmap v7184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3358002053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:04.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:04.176969+0000) 2023-08-28T00:53:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3358002053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3358002053' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:05 smithi093 ceph-mon[111335]: pgmap v7185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:53:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:05 smithi093 ceph-mon[118902]: pgmap v7185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:53:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:05 smithi136 ceph-mon[113531]: pgmap v7185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:53:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:05.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:05.177123+0000) 2023-08-28T00:53:05.696 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:53:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:53:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:06.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:06.177280+0000) 2023-08-28T00:53:07.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:07 smithi136 ceph-mon[113531]: pgmap v7186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:07.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/77496919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:07.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:07.178 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:07.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:07.177453+0000) 2023-08-28T00:53:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[111335]: pgmap v7186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/77496919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[118902]: pgmap v7186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/77496919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:07.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:08.389 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:08.389 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:08.389 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:08.389 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:08.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:08.177623+0000) 2023-08-28T00:53:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:09.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:09.177893+0000) 2023-08-28T00:53:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: pgmap v7187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:53:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:53:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/107179046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:09.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: pgmap v7187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:53:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:53:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/107179046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:09.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:09.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: pgmap v7187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:09.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.492 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:53:09.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:53:09.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:53:09.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/107179046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:09.493 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:10.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:10.178151+0000) 2023-08-28T00:53:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:11 smithi136 ceph-mon[113531]: pgmap v7188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1528322655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:11.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:11.178390+0000) 2023-08-28T00:53:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:11 smithi093 ceph-mon[111335]: pgmap v7188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1528322655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:11 smithi093 ceph-mon[118902]: pgmap v7188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:11.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1528322655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:11.146Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:11.147Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:12.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:12.178646+0000) 2023-08-28T00:53:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:12.596Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:12.597Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:13 smithi136 ceph-mon[113531]: pgmap v7189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:13.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:13.178919+0000) 2023-08-28T00:53:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:13 smithi093 ceph-mon[111335]: pgmap v7189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:13 smithi093 ceph-mon[118902]: pgmap v7189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:14.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/483252439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:14.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:14.179184+0000) 2023-08-28T00:53:14.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/483252439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/483252439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:15 smithi136 ceph-mon[113531]: pgmap v7190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:15.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:15.179435+0000) 2023-08-28T00:53:15.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:15 smithi093 ceph-mon[111335]: pgmap v7190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:15 smithi093 ceph-mon[118902]: pgmap v7190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:53:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:53:15] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:53:16.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3006936562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:16.443 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:16.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:16.179707+0000) 2023-08-28T00:53:16.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3006936562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3006936562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:17 smithi136 ceph-mon[113531]: pgmap v7191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:17.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:17.179955+0000) 2023-08-28T00:53:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:17 smithi093 ceph-mon[111335]: pgmap v7191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:17 smithi093 ceph-mon[118902]: pgmap v7191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:18.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:18.180216+0000) 2023-08-28T00:53:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:19 smithi136 ceph-mon[113531]: pgmap v7192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3970668831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:19.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:19.180477+0000) 2023-08-28T00:53:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:19 smithi093 ceph-mon[111335]: pgmap v7192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3970668831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:19 smithi093 ceph-mon[118902]: pgmap v7192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3970668831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:20.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:20.180708+0000) 2023-08-28T00:53:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:21 smithi136 ceph-mon[113531]: pgmap v7193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3880728229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:21.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:21.180936+0000) 2023-08-28T00:53:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:21 smithi093 ceph-mon[111335]: pgmap v7193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3880728229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:21 smithi093 ceph-mon[118902]: pgmap v7193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:21.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3880728229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:21.147Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:21.148Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:22.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:22.181179+0000) 2023-08-28T00:53:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:22.597Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:22.598Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:23 smithi136 ceph-mon[113531]: pgmap v7194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/810696838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:23.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:23.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:23.181425+0000) 2023-08-28T00:53:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:23 smithi093 ceph-mon[111335]: pgmap v7194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/810696838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:23 smithi093 ceph-mon[118902]: pgmap v7194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/810696838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:24.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:24.181689+0000) 2023-08-28T00:53:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:25 smithi136 ceph-mon[113531]: pgmap v7195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:25.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:25.181927+0000) 2023-08-28T00:53:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:25 smithi093 ceph-mon[111335]: pgmap v7195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:25 smithi093 ceph-mon[118902]: pgmap v7195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:53:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:53:25] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:53:26.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1932292710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:26.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:26.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:26.182210+0000) 2023-08-28T00:53:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1932292710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:26.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1932292710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:27 smithi136 ceph-mon[113531]: pgmap v7196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:27.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:27.182427+0000) 2023-08-28T00:53:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:27 smithi093 ceph-mon[111335]: pgmap v7196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:27 smithi093 ceph-mon[118902]: pgmap v7196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1029939874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:28.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:28.182655+0000) 2023-08-28T00:53:28.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1029939874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:28.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1029939874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:29 smithi136 ceph-mon[113531]: pgmap v7197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:29.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:29.182927+0000) 2023-08-28T00:53:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:29 smithi093 ceph-mon[111335]: pgmap v7197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:29 smithi093 ceph-mon[118902]: pgmap v7197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4275059512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:30.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:30.183185+0000) 2023-08-28T00:53:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4275059512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4275059512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:31 smithi136 ceph-mon[113531]: pgmap v7198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:31.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:31.183381+0000) 2023-08-28T00:53:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:31 smithi093 ceph-mon[111335]: pgmap v7198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:31 smithi093 ceph-mon[118902]: pgmap v7198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:31.147Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:31.149Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:32.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:32.183631+0000) 2023-08-28T00:53:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:32.597Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:32.598Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:33 smithi136 ceph-mon[113531]: pgmap v7199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/165185524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:33.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:33.183897+0000) 2023-08-28T00:53:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:33 smithi093 ceph-mon[111335]: pgmap v7199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/165185524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:33 smithi093 ceph-mon[118902]: pgmap v7199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/165185524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:34.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:34.184168+0000) 2023-08-28T00:53:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:35 smithi136 ceph-mon[113531]: pgmap v7200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3196634454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:35.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:35.184424+0000) 2023-08-28T00:53:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:53:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:53:35] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:53:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:35 smithi093 ceph-mon[111335]: pgmap v7200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3196634454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:35.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:35 smithi093 ceph-mon[118902]: pgmap v7200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:35.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3196634454' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:36.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:36.184715+0000) 2023-08-28T00:53:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:37 smithi136 ceph-mon[113531]: pgmap v7201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:37.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:37.184968+0000) 2023-08-28T00:53:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:37 smithi093 ceph-mon[111335]: pgmap v7201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:37 smithi093 ceph-mon[118902]: pgmap v7201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:38.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2752930364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:38.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:38.185152+0000) 2023-08-28T00:53:38.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2752930364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2752930364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:39 smithi136 ceph-mon[113531]: pgmap v7202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:39.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:39.185337+0000) 2023-08-28T00:53:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:39 smithi093 ceph-mon[111335]: pgmap v7202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:39 smithi093 ceph-mon[118902]: pgmap v7202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:40.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2172984648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:40.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:40.185531+0000) 2023-08-28T00:53:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2172984648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2172984648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:41 smithi136 ceph-mon[113531]: pgmap v7203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:41.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:41.185757+0000) 2023-08-28T00:53:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:41 smithi093 ceph-mon[111335]: pgmap v7203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:41 smithi093 ceph-mon[118902]: pgmap v7203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:41.148Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:41.149Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:42.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/115262766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:42.444 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:42.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:42.185974+0000) 2023-08-28T00:53:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/115262766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/115262766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:42.598Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:43.944 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:42.599Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:43.945 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:43 smithi136 ceph-mon[113531]: pgmap v7204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:43.945 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:43 smithi093 ceph-mon[111335]: pgmap v7204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:43.945 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:43.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:43.186231+0000) 2023-08-28T00:53:43.946 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:43 smithi093 ceph-mon[118902]: pgmap v7204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:44.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:44.186524+0000) 2023-08-28T00:53:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:45 smithi136 ceph-mon[113531]: pgmap v7205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/673307547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:45.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:45.186785+0000) 2023-08-28T00:53:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:53:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:53:45] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:53:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:45 smithi093 ceph-mon[111335]: pgmap v7205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:45.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/673307547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:45.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:45 smithi093 ceph-mon[118902]: pgmap v7205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:45.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/673307547' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:46.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:46.187088+0000) 2023-08-28T00:53:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:47 smithi136 ceph-mon[113531]: pgmap v7206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:47.509 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2761311027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:47.510 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:47.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:47.187398+0000) 2023-08-28T00:53:47.510 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:47 smithi093 ceph-mon[111335]: pgmap v7206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:47.510 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2761311027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:47.511 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:47 smithi093 ceph-mon[118902]: pgmap v7206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:47.511 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2761311027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:48.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:48.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:48.187705+0000) 2023-08-28T00:53:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:49 smithi136 ceph-mon[113531]: pgmap v7207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3643162277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:49.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:49.187952+0000) 2023-08-28T00:53:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:49 smithi093 ceph-mon[111335]: pgmap v7207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3643162277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:49 smithi093 ceph-mon[118902]: pgmap v7207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:49.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3643162277' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:50.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:50.188206+0000) 2023-08-28T00:53:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:51 smithi136 ceph-mon[113531]: pgmap v7208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:51.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:51.188404+0000) 2023-08-28T00:53:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:51 smithi093 ceph-mon[111335]: pgmap v7208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:51 smithi093 ceph-mon[118902]: pgmap v7208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:51.148Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:51.149Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/456614929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:52.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:52.188658+0000) 2023-08-28T00:53:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/456614929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/456614929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:52.599Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:53:52.600Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:53:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:53 smithi136 ceph-mon[113531]: pgmap v7209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:53.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:53.188922+0000) 2023-08-28T00:53:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:53 smithi093 ceph-mon[111335]: pgmap v7209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:53 smithi093 ceph-mon[118902]: pgmap v7209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1511237965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1511237965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:53:54.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1511237965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:54.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:54.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:54.189124+0000) 2023-08-28T00:53:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:55 smithi093 ceph-mon[118902]: pgmap v7210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:55 smithi093 ceph-mon[111335]: pgmap v7210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:55.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:53:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:53:55] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:53:55.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:55 smithi136 ceph-mon[113531]: pgmap v7210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:55.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:55.189317+0000) 2023-08-28T00:53:56.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:56.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:56.189594+0000) 2023-08-28T00:53:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:57 smithi093 ceph-mon[118902]: pgmap v7211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3666078290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:57 smithi093 ceph-mon[111335]: pgmap v7211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3666078290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:57.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:57 smithi136 ceph-mon[113531]: pgmap v7211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:57.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3666078290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:57.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:57.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:57.189914+0000) 2023-08-28T00:53:58.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:58.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:58.190157+0000) 2023-08-28T00:53:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:59 smithi093 ceph-mon[111335]: pgmap v7212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:53:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2767422667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:59 smithi093 ceph-mon[118902]: pgmap v7212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:59.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:53:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2767422667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:59 smithi136 ceph-mon[113531]: pgmap v7212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:53:59.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:53:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2767422667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:53:59.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:53:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:53:59.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:53:59.190373+0000) 2023-08-28T00:54:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:00.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:00.190626+0000) 2023-08-28T00:54:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:01 smithi093 ceph-mon[118902]: pgmap v7213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1904575491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:01.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:01 smithi093 ceph-mon[111335]: pgmap v7213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:01.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1904575491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:01.149Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:01.150Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:01.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:01 smithi136 ceph-mon[113531]: pgmap v7213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:01.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1904575491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:01.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:01.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:01.190909+0000) 2023-08-28T00:54:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:02.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:02.191197+0000) 2023-08-28T00:54:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:02.600Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:02.601Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:03 smithi093 ceph-mon[111335]: pgmap v7214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:03 smithi093 ceph-mon[118902]: pgmap v7214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:03.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:03 smithi136 ceph-mon[113531]: pgmap v7214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:03.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:03.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:03.191448+0000) 2023-08-28T00:54:04.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3780108837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:04.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:04.191675+0000) 2023-08-28T00:54:04.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3780108837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:04.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3780108837' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:05.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:05 smithi136 ceph-mon[113531]: pgmap v7215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:05.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:05.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:05.191927+0000) 2023-08-28T00:54:05.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:05 smithi093 ceph-mon[111335]: pgmap v7215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:54:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:54:05] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:54:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:05 smithi093 ceph-mon[118902]: pgmap v7215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:06.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:06.192219+0000) 2023-08-28T00:54:06.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1125903322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:06.742 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1125903322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:06.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1125903322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:07.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:07 smithi136 ceph-mon[113531]: pgmap v7216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:07.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:07.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:07.192595+0000) 2023-08-28T00:54:07.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:07 smithi093 ceph-mon[111335]: pgmap v7216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:07.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:07 smithi093 ceph-mon[118902]: pgmap v7216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2075327933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:08.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:08.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:08.192890+0000) 2023-08-28T00:54:08.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2075327933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:08.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2075327933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:09 smithi136 ceph-mon[113531]: pgmap v7217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:54:09.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:09.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:09.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:09.193030+0000) 2023-08-28T00:54:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:09 smithi093 ceph-mon[118902]: pgmap v7217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:54:09.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:09 smithi093 ceph-mon[111335]: pgmap v7217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:09.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:54:09.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:10.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:10.193189+0000) 2023-08-28T00:54:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:11 smithi136 ceph-mon[113531]: pgmap v7218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:11.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:11.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:11.442 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4010770467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:11.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:11.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:11.193324+0000) 2023-08-28T00:54:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[111335]: pgmap v7218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4010770467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:11.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[118902]: pgmap v7218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:11.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:11.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:11.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4010770467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:11.150Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:11.151Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:54:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:54:12.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:12.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:12.193548+0000) 2023-08-28T00:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:54:12.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:54:12.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:54:12.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:54:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:12.600Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:12.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:12.601Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:13.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:13.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:13.193807+0000) 2023-08-28T00:54:13.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:13 smithi136 ceph-mon[113531]: pgmap v7219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:13.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1981478630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:13.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:13 smithi093 ceph-mon[111335]: pgmap v7219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:13.843 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1981478630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:13.844 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:13 smithi093 ceph-mon[118902]: pgmap v7219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:13.844 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1981478630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:14.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:14.194085+0000) 2023-08-28T00:54:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:15 smithi136 ceph-mon[113531]: pgmap v7220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:15.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2719140289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:15.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:15.194304+0000) 2023-08-28T00:54:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:15 smithi093 ceph-mon[111335]: pgmap v7220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:15.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2719140289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:15 smithi093 ceph-mon[118902]: pgmap v7220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:15.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2719140289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:15.741 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:54:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:54:15] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:54:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:16.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:16.194563+0000) 2023-08-28T00:54:17.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:17 smithi136 ceph-mon[113531]: pgmap v7221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:18.903 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:17.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:17.194872+0000) 2023-08-28T00:54:18.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2257992282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:18.904 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:18.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:18.195135+0000) 2023-08-28T00:54:18.904 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:17 smithi093 ceph-mon[111335]: pgmap v7221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:18.904 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:17 smithi093 ceph-mon[118902]: pgmap v7221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:18.905 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2257992282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:18.905 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2257992282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:19.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:19 smithi136 ceph-mon[113531]: pgmap v7222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:21.551 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:19.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:19.195365+0000) 2023-08-28T00:54:21.552 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1445845191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.552 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:20.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:20.195614+0000) 2023-08-28T00:54:21.707 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:19 smithi093 ceph-mon[111335]: pgmap v7222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:21.708 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:21 smithi136 ceph-mon[113531]: pgmap v7223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:21.708 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:19 smithi093 ceph-mon[118902]: pgmap v7222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:21.708 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:21.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:21.195867+0000) 2023-08-28T00:54:21.709 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1445845191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.709 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1445845191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:21.709 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:21.151Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:21.709 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:21.152Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:21.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:21 smithi093 ceph-mon[111335]: pgmap v7223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:21.867 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:21 smithi093 ceph-mon[118902]: pgmap v7223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:22.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:22.196141+0000) 2023-08-28T00:54:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:22.601Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:22.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:23.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:23 smithi136 ceph-mon[113531]: pgmap v7224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:23.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2536871152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:23.792 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:23.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:23.196386+0000) 2023-08-28T00:54:23.792 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:23 smithi093 ceph-mon[118902]: pgmap v7224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:23.797 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2536871152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:23.798 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:23 smithi093 ceph-mon[111335]: pgmap v7224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:23.798 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2536871152' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:24.449 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:24.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:24.196644+0000) 2023-08-28T00:54:24.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:24.757 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:24.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:25.451 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:25.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:25.196931+0000) 2023-08-28T00:54:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:25 smithi093 ceph-mon[111335]: pgmap v7225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:25.744 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3181741041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:25.745 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:54:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:54:25] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:54:25.745 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:25 smithi093 ceph-mon[118902]: pgmap v7225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:25.745 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3181741041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:25 smithi136 ceph-mon[113531]: pgmap v7225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:25.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3181741041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:26.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:26.197159+0000) 2023-08-28T00:54:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:26 smithi093 ceph-mon[111335]: pgmap v7226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:26 smithi093 ceph-mon[118902]: pgmap v7226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:26 smithi136 ceph-mon[113531]: pgmap v7226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:27.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:27.197427+0000) 2023-08-28T00:54:28.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3483215204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:28.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3483215204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:28.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3483215204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:28.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:28.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:28.197623+0000) 2023-08-28T00:54:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:28 smithi093 ceph-mon[111335]: pgmap v7227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:28 smithi093 ceph-mon[118902]: pgmap v7227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:28 smithi136 ceph-mon[113531]: pgmap v7227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:29.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:29.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:29.197881+0000) 2023-08-28T00:54:30.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4033864362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:30.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4033864362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4033864362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:30.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:30.198144+0000) 2023-08-28T00:54:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:30 smithi093 ceph-mon[111335]: pgmap v7228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:30 smithi093 ceph-mon[118902]: pgmap v7228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:31.152Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:31.153Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:30 smithi136 ceph-mon[113531]: pgmap v7228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:31.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:31.198401+0000) 2023-08-28T00:54:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:32.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:32.198674+0000) 2023-08-28T00:54:32.974 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:32.602Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:32.974 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:32.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:32 smithi093 ceph-mon[111335]: pgmap v7229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:33.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1748126364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:32 smithi093 ceph-mon[118902]: pgmap v7229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1748126364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:32 smithi136 ceph-mon[113531]: pgmap v7229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1748126364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:33.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:33.198937+0000) 2023-08-28T00:54:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:34.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:34.199206+0000) 2023-08-28T00:54:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:34 smithi093 ceph-mon[118902]: pgmap v7230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1600031428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:34 smithi093 ceph-mon[111335]: pgmap v7230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:35.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1600031428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:34 smithi136 ceph-mon[113531]: pgmap v7230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1600031428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:35.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:35.199407+0000) 2023-08-28T00:54:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:54:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:54:35] "GET /metrics HTTP/1.1" 200 239923 "" "Prometheus/2.43.0" 2023-08-28T00:54:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:36.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:36.199641+0000) 2023-08-28T00:54:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:36 smithi093 ceph-mon[111335]: pgmap v7231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1745314861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:36 smithi093 ceph-mon[118902]: pgmap v7231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:37.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1745314861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:36 smithi136 ceph-mon[113531]: pgmap v7231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1745314861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:37.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:37.199895+0000) 2023-08-28T00:54:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:38.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:38.200155+0000) 2023-08-28T00:54:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:38 smithi093 ceph-mon[111335]: pgmap v7232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:38 smithi093 ceph-mon[118902]: pgmap v7232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:39.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:38 smithi136 ceph-mon[113531]: pgmap v7232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:39.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:39.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:39.200394+0000) 2023-08-28T00:54:40.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1371549610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1371549610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1371549610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:40.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:40.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:40.200598+0000) 2023-08-28T00:54:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:40 smithi093 ceph-mon[111335]: pgmap v7233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:40 smithi093 ceph-mon[118902]: pgmap v7233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:41.153Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:41.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:41.154Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:40 smithi136 ceph-mon[113531]: pgmap v7233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:41.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:41.200850+0000) 2023-08-28T00:54:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1319786824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:42.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:42.201090+0000) 2023-08-28T00:54:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1319786824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1319786824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:42.602Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:42.603Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:42 smithi136 ceph-mon[113531]: pgmap v7234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:43.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:43.201328+0000) 2023-08-28T00:54:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:42 smithi093 ceph-mon[111335]: pgmap v7234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:42 smithi093 ceph-mon[118902]: pgmap v7234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/65801481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:44.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:44.201605+0000) 2023-08-28T00:54:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/65801481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/65801481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:45 smithi093 ceph-mon[118902]: pgmap v7235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:45 smithi093 ceph-mon[111335]: pgmap v7235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:45 smithi136 ceph-mon[113531]: pgmap v7235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:45.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:45.201861+0000) 2023-08-28T00:54:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:54:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:54:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:54:46.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:46.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:46.202128+0000) 2023-08-28T00:54:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:47 smithi136 ceph-mon[113531]: pgmap v7236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:47.470 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3924588643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:47.470 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:47.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:47.202321+0000) 2023-08-28T00:54:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:47 smithi093 ceph-mon[118902]: pgmap v7236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:47.564 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3924588643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:47.565 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:47 smithi093 ceph-mon[111335]: pgmap v7236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:47.565 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3924588643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:48.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:48.202576+0000) 2023-08-28T00:54:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:49 smithi136 ceph-mon[113531]: pgmap v7237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3715624325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:49.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:49.202832+0000) 2023-08-28T00:54:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:49 smithi093 ceph-mon[111335]: pgmap v7237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3715624325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:49 smithi093 ceph-mon[118902]: pgmap v7237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3715624325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:50.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:50.203117+0000) 2023-08-28T00:54:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:51 smithi136 ceph-mon[113531]: pgmap v7238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:51.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:51.203363+0000) 2023-08-28T00:54:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:51 smithi093 ceph-mon[111335]: pgmap v7238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:51 smithi093 ceph-mon[118902]: pgmap v7238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:51.153Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:51.154Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:52.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1418774388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:52.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:52.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:52.203627+0000) 2023-08-28T00:54:52.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1418774388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1418774388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:52.603Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:54:52.604Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:54:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:53 smithi136 ceph-mon[113531]: pgmap v7239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:53.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:53.203843+0000) 2023-08-28T00:54:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:53 smithi093 ceph-mon[111335]: pgmap v7239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:53 smithi093 ceph-mon[118902]: pgmap v7239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1369187986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:54.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:54.204121+0000) 2023-08-28T00:54:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1369187986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1369187986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:54.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:54:55.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:55 smithi093 ceph-mon[111335]: pgmap v7240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:55 smithi093 ceph-mon[118902]: pgmap v7240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:55 smithi136 ceph-mon[113531]: pgmap v7240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:55.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:55.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:55.204372+0000) 2023-08-28T00:54:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:54:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:54:55] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:54:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/516259879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:56.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:56.204596+0000) 2023-08-28T00:54:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/516259879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/516259879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:57 smithi136 ceph-mon[113531]: pgmap v7241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:57.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:57.204816+0000) 2023-08-28T00:54:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:57 smithi093 ceph-mon[118902]: pgmap v7241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:57 smithi093 ceph-mon[111335]: pgmap v7241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:58.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:58.205062+0000) 2023-08-28T00:54:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:59 smithi136 ceph-mon[113531]: pgmap v7242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:54:59.946 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:54:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/813276072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:54:59.946 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:54:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:54:59.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:54:59.205294+0000) 2023-08-28T00:54:59.947 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:59 smithi093 ceph-mon[111335]: pgmap v7242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:00.464 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:54:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/813276072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:00.792 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:59 smithi093 ceph-mon[118902]: pgmap v7242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:00.792 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:54:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/813276072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:00.793 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:00.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:00.205571+0000) 2023-08-28T00:55:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:01 smithi136 ceph-mon[113531]: pgmap v7243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:01.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2209945556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:01.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:01.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:01.205823+0000) 2023-08-28T00:55:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:01 smithi093 ceph-mon[111335]: pgmap v7243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2209945556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:01 smithi093 ceph-mon[118902]: pgmap v7243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2209945556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:01.154Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:01.155Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:02.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:02.206089+0000) 2023-08-28T00:55:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:02.603Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:02.604Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:03 smithi136 ceph-mon[113531]: pgmap v7244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:03.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:03.206340+0000) 2023-08-28T00:55:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:03 smithi093 ceph-mon[118902]: pgmap v7244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:03 smithi093 ceph-mon[111335]: pgmap v7244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:04.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3081446497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:04.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:04.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:04.206604+0000) 2023-08-28T00:55:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3081446497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3081446497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:05 smithi093 ceph-mon[111335]: pgmap v7245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:05 smithi093 ceph-mon[118902]: pgmap v7245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:05 smithi136 ceph-mon[113531]: pgmap v7245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:05.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:05.206880+0000) 2023-08-28T00:55:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:55:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:55:05] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:55:06.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2655265170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:06.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:06.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:06.207177+0000) 2023-08-28T00:55:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2655265170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2655265170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:07 smithi136 ceph-mon[113531]: pgmap v7246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:07.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:07.207513+0000) 2023-08-28T00:55:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:07 smithi093 ceph-mon[111335]: pgmap v7246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:07 smithi093 ceph-mon[118902]: pgmap v7246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:08.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1781748294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:08.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:08.207733+0000) 2023-08-28T00:55:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1781748294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1781748294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:09 smithi136 ceph-mon[113531]: pgmap v7247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:09.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:09.207977+0000) 2023-08-28T00:55:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:09 smithi093 ceph-mon[111335]: pgmap v7247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:09 smithi093 ceph-mon[118902]: pgmap v7247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:10.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:10.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:10.208247+0000) 2023-08-28T00:55:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:11.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:11.208508+0000) 2023-08-28T00:55:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:11 smithi136 ceph-mon[113531]: pgmap v7248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1083482939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:11 smithi093 ceph-mon[111335]: pgmap v7248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1083482939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:11 smithi093 ceph-mon[118902]: pgmap v7248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1083482939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:11.155Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:11.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:11.156Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:12.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:55:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:12.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:12.208668+0000) 2023-08-28T00:55:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:55:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:55:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:12.604Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:12.605Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:13.428 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:13 smithi136 ceph-mon[113531]: pgmap v7249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:13.428 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1560554371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:13.428 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:13.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:13.208856+0000) 2023-08-28T00:55:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:13 smithi093 ceph-mon[111335]: pgmap v7249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1560554371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:13 smithi093 ceph-mon[118902]: pgmap v7249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:13.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1560554371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:14.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:14.209023+0000) 2023-08-28T00:55:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:14 smithi136 ceph-mon[113531]: pgmap v7250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[111335]: pgmap v7250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[118902]: pgmap v7250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:15.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:15.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:15.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:15.209253+0000) 2023-08-28T00:55:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:55:15] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:55:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:55:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:55:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:16.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3917918576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:55:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:55:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:16.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3917918576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:55:16.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:55:16.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:55:16.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3917918576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:16.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:16.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:16.209447+0000) 2023-08-28T00:55:17.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:16 smithi093 ceph-mon[111335]: pgmap v7251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:17.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:16 smithi093 ceph-mon[118902]: pgmap v7251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:16 smithi136 ceph-mon[113531]: pgmap v7251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:17.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:17.209742+0000) 2023-08-28T00:55:18.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2953322924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:18.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2953322924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2953322924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:18.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:18.209980+0000) 2023-08-28T00:55:19.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:18 smithi093 ceph-mon[111335]: pgmap v7252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:19.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:18 smithi093 ceph-mon[118902]: pgmap v7252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:18 smithi136 ceph-mon[113531]: pgmap v7252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:19.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:19.210194+0000) 2023-08-28T00:55:20.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/145163943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:20.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/145163943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/145163943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:20.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:20.210470+0000) 2023-08-28T00:55:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:20 smithi093 ceph-mon[111335]: pgmap v7253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:21.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:20 smithi093 ceph-mon[118902]: pgmap v7253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:21.156Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:21.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:21.157Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:21.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:20 smithi136 ceph-mon[113531]: pgmap v7253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:21.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:21.210742+0000) 2023-08-28T00:55:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:22.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:22.210958+0000) 2023-08-28T00:55:22.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:22.605Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:22.981 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:22.606Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:22 smithi093 ceph-mon[118902]: pgmap v7254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:23.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2792544574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:22 smithi093 ceph-mon[111335]: pgmap v7254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:23.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2792544574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:22 smithi136 ceph-mon[113531]: pgmap v7254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2792544574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:23.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:23.211184+0000) 2023-08-28T00:55:24.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:24.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:24.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:24.211470+0000) 2023-08-28T00:55:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:24 smithi093 ceph-mon[111335]: pgmap v7255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:25.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1554422237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:24 smithi093 ceph-mon[118902]: pgmap v7255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:25.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1554422237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:24 smithi136 ceph-mon[113531]: pgmap v7255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:25.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1554422237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:25.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:25.211714+0000) 2023-08-28T00:55:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:55:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:55:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:55:26.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:26.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:26.211988+0000) 2023-08-28T00:55:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:26 smithi093 ceph-mon[118902]: pgmap v7256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2118595412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:27.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:26 smithi093 ceph-mon[111335]: pgmap v7256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:27.241 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2118595412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:26 smithi136 ceph-mon[113531]: pgmap v7256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2118595412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:27.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:27.212276+0000) 2023-08-28T00:55:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:28.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:28.212519+0000) 2023-08-28T00:55:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:28 smithi136 ceph-mon[113531]: pgmap v7257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:29.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:29.212784+0000) 2023-08-28T00:55:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:28 smithi093 ceph-mon[111335]: pgmap v7257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:28 smithi093 ceph-mon[118902]: pgmap v7257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:30.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1640756063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:30.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:30.213073+0000) 2023-08-28T00:55:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1640756063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1640756063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:31.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:30 smithi136 ceph-mon[113531]: pgmap v7258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:31.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:31.213350+0000) 2023-08-28T00:55:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:30 smithi093 ceph-mon[111335]: pgmap v7258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:30 smithi093 ceph-mon[118902]: pgmap v7258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:31.157Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:31.158Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:32.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4011648752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:32.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:32.213621+0000) 2023-08-28T00:55:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4011648752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4011648752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:32.606Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:32.607Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:33.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:33 smithi136 ceph-mon[113531]: pgmap v7259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:33.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:33.213895+0000) 2023-08-28T00:55:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:33 smithi093 ceph-mon[111335]: pgmap v7259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:33 smithi093 ceph-mon[118902]: pgmap v7259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:34.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:34.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:34.214139+0000) 2023-08-28T00:55:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:35 smithi093 ceph-mon[111335]: pgmap v7260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:35.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3209366867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:35 smithi093 ceph-mon[118902]: pgmap v7260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:35.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3209366867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:35 smithi136 ceph-mon[113531]: pgmap v7260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:35.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3209366867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:35.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:35.214384+0000) 2023-08-28T00:55:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:55:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:55:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:55:36.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:36.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:36.214620+0000) 2023-08-28T00:55:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:37 smithi136 ceph-mon[113531]: pgmap v7261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:37.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/87406195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:37.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:37.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:37.214893+0000) 2023-08-28T00:55:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:37 smithi093 ceph-mon[111335]: pgmap v7261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/87406195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:37 smithi093 ceph-mon[118902]: pgmap v7261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/87406195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:38.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:38.215142+0000) 2023-08-28T00:55:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:39 smithi136 ceph-mon[113531]: pgmap v7262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3767416707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:39.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:39.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:39.215393+0000) 2023-08-28T00:55:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:39 smithi093 ceph-mon[111335]: pgmap v7262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3767416707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:39 smithi093 ceph-mon[118902]: pgmap v7262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3767416707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:39.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:40.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:40.215627+0000) 2023-08-28T00:55:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:41 smithi136 ceph-mon[113531]: pgmap v7263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:41.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:41.215909+0000) 2023-08-28T00:55:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:41 smithi093 ceph-mon[111335]: pgmap v7263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:41 smithi093 ceph-mon[118902]: pgmap v7263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:41.158Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:41.159Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:42.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/829539997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:42.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:42.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:42.216181+0000) 2023-08-28T00:55:42.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/829539997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/829539997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:42.606Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:42.607Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:43 smithi136 ceph-mon[113531]: pgmap v7264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:43.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:43.216445+0000) 2023-08-28T00:55:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:43 smithi093 ceph-mon[111335]: pgmap v7264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:43 smithi093 ceph-mon[118902]: pgmap v7264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:44.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/205933155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:44.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:44.216666+0000) 2023-08-28T00:55:44.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/205933155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:44.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/205933155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:45 smithi093 ceph-mon[111335]: pgmap v7265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:45 smithi093 ceph-mon[118902]: pgmap v7265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:45 smithi136 ceph-mon[113531]: pgmap v7265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:45.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:45.216914+0000) 2023-08-28T00:55:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:55:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:55:45] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:55:46.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/446970630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:46.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:46.217175+0000) 2023-08-28T00:55:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/446970630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/446970630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:47.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:47 smithi136 ceph-mon[113531]: pgmap v7266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:47.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:47.217405+0000) 2023-08-28T00:55:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:47 smithi093 ceph-mon[118902]: pgmap v7266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:47 smithi093 ceph-mon[111335]: pgmap v7266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:48.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:48.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:48.217660+0000) 2023-08-28T00:55:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:49 smithi136 ceph-mon[113531]: pgmap v7267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1211051664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:49.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:49.217912+0000) 2023-08-28T00:55:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:49 smithi093 ceph-mon[111335]: pgmap v7267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1211051664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:49 smithi093 ceph-mon[118902]: pgmap v7267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1211051664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:50.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:50.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:50.218136+0000) 2023-08-28T00:55:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:51 smithi136 ceph-mon[113531]: pgmap v7268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:51.558 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/760456024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:51.558 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:51.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:51.218393+0000) 2023-08-28T00:55:51.559 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:51 smithi093 ceph-mon[118902]: pgmap v7268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:51.559 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/760456024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:51.559 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:51 smithi093 ceph-mon[111335]: pgmap v7268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:51.559 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/760456024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:51.559 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:51.159Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:51.559 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:51.160Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:52.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:52.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:52.218611+0000) 2023-08-28T00:55:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:52.607Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:53.273 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:55:52.608Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:55:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:53 smithi136 ceph-mon[113531]: pgmap v7269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:53.447 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:53.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:53.218850+0000) 2023-08-28T00:55:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:53 smithi093 ceph-mon[111335]: pgmap v7269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:53.608 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:53 smithi093 ceph-mon[118902]: pgmap v7269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3705965620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:54.741 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:54.742 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:54.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:54.219121+0000) 2023-08-28T00:55:54.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3705965620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:54.743 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:54.743 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3705965620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:54.774 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:55:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:55 smithi093 ceph-mon[111335]: pgmap v7270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:55.410 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:55 smithi093 ceph-mon[118902]: pgmap v7270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:55 smithi136 ceph-mon[113531]: pgmap v7270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:55.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:55.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:55.219332+0000) 2023-08-28T00:55:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:55:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:55:55] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:55:56.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2642631447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:56.546 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:56.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:56.219583+0000) 2023-08-28T00:55:56.547 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2642631447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:56.569 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2642631447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:57.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:57 smithi136 ceph-mon[113531]: pgmap v7271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:57.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:57.219868+0000) 2023-08-28T00:55:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:57 smithi093 ceph-mon[111335]: pgmap v7271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:57 smithi093 ceph-mon[118902]: pgmap v7271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:58.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3882387290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:58.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:58.220155+0000) 2023-08-28T00:55:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3882387290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3882387290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:55:59.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:55:59 smithi136 ceph-mon[113531]: pgmap v7272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:55:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:55:59.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:55:59.220392+0000) 2023-08-28T00:55:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:55:59 smithi093 ceph-mon[111335]: pgmap v7272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:55:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:55:59 smithi093 ceph-mon[118902]: pgmap v7272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:00.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:00.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:00.220629+0000) 2023-08-28T00:56:01.443 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:01 smithi136 ceph-mon[113531]: pgmap v7273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:01.512 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2406461649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:01.512 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:01.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:01.220836+0000) 2023-08-28T00:56:01.513 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:01 smithi093 ceph-mon[111335]: pgmap v7273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:01.513 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2406461649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:01.513 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:01 smithi093 ceph-mon[118902]: pgmap v7273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:01.514 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2406461649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:01.514 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:01.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:01.514 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:01.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:02.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:02.221114+0000) 2023-08-28T00:56:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:02.608Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:02.609Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:03 smithi136 ceph-mon[113531]: pgmap v7274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3716425624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:03.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:03.221346+0000) 2023-08-28T00:56:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:03 smithi093 ceph-mon[118902]: pgmap v7274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3716425624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:03 smithi093 ceph-mon[111335]: pgmap v7274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3716425624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:04.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:04.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:04.221632+0000) 2023-08-28T00:56:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:05 smithi093 ceph-mon[111335]: pgmap v7275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2060339191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:05 smithi093 ceph-mon[118902]: pgmap v7275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2060339191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:05 smithi136 ceph-mon[113531]: pgmap v7275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2060339191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:05.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:05.221860+0000) 2023-08-28T00:56:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:56:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:56:05] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:56:06.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:06.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:06.222124+0000) 2023-08-28T00:56:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:07 smithi136 ceph-mon[113531]: pgmap v7276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:07.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:07.222340+0000) 2023-08-28T00:56:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:07 smithi093 ceph-mon[111335]: pgmap v7276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:07 smithi093 ceph-mon[118902]: pgmap v7276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2861470239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:08.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:08.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:08.222566+0000) 2023-08-28T00:56:08.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2861470239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:08.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2861470239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:09 smithi136 ceph-mon[113531]: pgmap v7277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:09.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:09.222845+0000) 2023-08-28T00:56:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:09 smithi093 ceph-mon[111335]: pgmap v7277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:09 smithi093 ceph-mon[118902]: pgmap v7277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3986722260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:10.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:10.223141+0000) 2023-08-28T00:56:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3986722260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3986722260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:11.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:11 smithi136 ceph-mon[113531]: pgmap v7278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:11.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:11.223362+0000) 2023-08-28T00:56:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:11 smithi093 ceph-mon[111335]: pgmap v7278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:11 smithi093 ceph-mon[118902]: pgmap v7278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:11.160Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:11.161Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:12.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:12.223585+0000) 2023-08-28T00:56:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:12.608Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:12.609Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:13 smithi136 ceph-mon[113531]: pgmap v7279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:13.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3727917753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:13.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:13.223819+0000) 2023-08-28T00:56:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:13 smithi093 ceph-mon[111335]: pgmap v7279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3727917753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:13 smithi093 ceph-mon[118902]: pgmap v7279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3727917753' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:14.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:14.224093+0000) 2023-08-28T00:56:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:15 smithi093 ceph-mon[111335]: pgmap v7280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1307390811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:56:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:15 smithi093 ceph-mon[118902]: pgmap v7280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1307390811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:56:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:15.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:15.224250+0000) 2023-08-28T00:56:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:15 smithi136 ceph-mon[113531]: pgmap v7280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1307390811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:15.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:56:15.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:56:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:56:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:56:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:16.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:16.224385+0000) 2023-08-28T00:56:17.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:17 smithi136 ceph-mon[113531]: pgmap v7281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:17.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:17.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:17.225 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/285398137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[111335]: pgmap v7281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/285398137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[118902]: pgmap v7281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:17.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/285398137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:17.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:17.224599+0000) 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:18.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:18.224838+0000) 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2462M 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:56:18.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:56:18.441 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2462M 2023-08-28T00:56:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2462M 2023-08-28T00:56:18.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:18.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:56:18.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:56:18.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:56:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:19 smithi136 ceph-mon[113531]: pgmap v7282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:19.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:19.225073+0000) 2023-08-28T00:56:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:19 smithi093 ceph-mon[111335]: pgmap v7282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:19 smithi093 ceph-mon[118902]: pgmap v7282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3095353858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:20.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:20.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:20.225309+0000) 2023-08-28T00:56:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3095353858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3095353858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:21 smithi136 ceph-mon[113531]: pgmap v7283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:21.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:21.225560+0000) 2023-08-28T00:56:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:21 smithi093 ceph-mon[111335]: pgmap v7283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:21 smithi093 ceph-mon[118902]: pgmap v7283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:21.161Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:21.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1302923553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:22.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:22.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:22.225822+0000) 2023-08-28T00:56:22.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1302923553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:22.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1302923553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:22.609Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:22.610Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:23 smithi136 ceph-mon[113531]: pgmap v7284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:23.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:23.226109+0000) 2023-08-28T00:56:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:23 smithi093 ceph-mon[111335]: pgmap v7284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:23 smithi093 ceph-mon[118902]: pgmap v7284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1419062274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:24.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:24.226315+0000) 2023-08-28T00:56:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1419062274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1419062274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:25 smithi136 ceph-mon[113531]: pgmap v7285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:25.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:25.226523+0000) 2023-08-28T00:56:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:25 smithi093 ceph-mon[111335]: pgmap v7285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:56:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:56:25] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T00:56:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:25 smithi093 ceph-mon[118902]: pgmap v7285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:26.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:26.226783+0000) 2023-08-28T00:56:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:27 smithi136 ceph-mon[113531]: pgmap v7286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1587630023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:27.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:27.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:27.227009+0000) 2023-08-28T00:56:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:27 smithi093 ceph-mon[111335]: pgmap v7286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1587630023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:27 smithi093 ceph-mon[118902]: pgmap v7286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1587630023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:28.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:28.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:28.227262+0000) 2023-08-28T00:56:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:29 smithi136 ceph-mon[113531]: pgmap v7287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1236870651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:29.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:29.227487+0000) 2023-08-28T00:56:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:29 smithi093 ceph-mon[111335]: pgmap v7287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1236870651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:29 smithi093 ceph-mon[118902]: pgmap v7287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1236870651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:30.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:30.227771+0000) 2023-08-28T00:56:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:31 smithi136 ceph-mon[113531]: pgmap v7288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:31.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:31.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:31.228070+0000) 2023-08-28T00:56:31.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:31 smithi093 ceph-mon[111335]: pgmap v7288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:31 smithi093 ceph-mon[118902]: pgmap v7288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:31.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:31.162Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2740830576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:32.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:32.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:32.228330+0000) 2023-08-28T00:56:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2740830576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:32.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2740830576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:32.609Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:32.610Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:33 smithi136 ceph-mon[113531]: pgmap v7289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:33.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:33.228577+0000) 2023-08-28T00:56:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:33 smithi093 ceph-mon[111335]: pgmap v7289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:33 smithi093 ceph-mon[118902]: pgmap v7289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/317653828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:34.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:34.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:34.228845+0000) 2023-08-28T00:56:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/317653828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/317653828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:35 smithi136 ceph-mon[113531]: pgmap v7290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:35.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:35.229090+0000) 2023-08-28T00:56:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:35 smithi093 ceph-mon[111335]: pgmap v7290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:56:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:56:35] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T00:56:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:35 smithi093 ceph-mon[118902]: pgmap v7290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:36.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:36.229372+0000) 2023-08-28T00:56:37.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:36 smithi136 ceph-mon[113531]: pgmap v7291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:37.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3651033357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:36 smithi093 ceph-mon[111335]: pgmap v7291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:37.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3651033357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:36 smithi093 ceph-mon[118902]: pgmap v7291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:37.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3651033357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:37.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:37.229611+0000) 2023-08-28T00:56:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:38.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:38.229821+0000) 2023-08-28T00:56:39.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:38 smithi136 ceph-mon[113531]: pgmap v7292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:39.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3136438568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:39.230 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:38 smithi093 ceph-mon[111335]: pgmap v7292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3136438568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:39.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:38 smithi093 ceph-mon[118902]: pgmap v7292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3136438568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:39.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:39.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:39.230048+0000) 2023-08-28T00:56:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:40.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:40.230330+0000) 2023-08-28T00:56:41.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:40 smithi136 ceph-mon[113531]: pgmap v7293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:41.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3354322002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:40 smithi093 ceph-mon[111335]: pgmap v7293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:41.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3354322002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:40 smithi093 ceph-mon[118902]: pgmap v7293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:41.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3354322002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:41.162Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:41.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:41.163Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:41.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:41.230650+0000) 2023-08-28T00:56:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:42.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:42.230963+0000) 2023-08-28T00:56:42.977 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:42.610Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:42.977 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:42.611Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:43.231 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:42 smithi136 ceph-mon[113531]: pgmap v7294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:43.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:42 smithi093 ceph-mon[111335]: pgmap v7294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:43.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:42 smithi093 ceph-mon[118902]: pgmap v7294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:43.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:43.231237+0000) 2023-08-28T00:56:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/803397700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:44.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/803397700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/803397700' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:44.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:44.231484+0000) 2023-08-28T00:56:45.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:44 smithi093 ceph-mon[111335]: pgmap v7295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:45.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:44 smithi093 ceph-mon[118902]: pgmap v7295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:45.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:44 smithi136 ceph-mon[113531]: pgmap v7295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:45.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:45.231737+0000) 2023-08-28T00:56:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:56:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:56:45] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:56:46.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2746815954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:46.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2746815954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2746815954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:46.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:46.231990+0000) 2023-08-28T00:56:47.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:46 smithi093 ceph-mon[111335]: pgmap v7296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:47.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:46 smithi093 ceph-mon[118902]: pgmap v7296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:46 smithi136 ceph-mon[113531]: pgmap v7296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:47.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:47.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:47.232246+0000) 2023-08-28T00:56:48.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3037202275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:48.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:48.232505+0000) 2023-08-28T00:56:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3037202275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3037202275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:48 smithi136 ceph-mon[113531]: pgmap v7297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:49.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:49.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:49.232790+0000) 2023-08-28T00:56:49.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:48 smithi093 ceph-mon[111335]: pgmap v7297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:49.494 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:48 smithi093 ceph-mon[118902]: pgmap v7297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:50.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:50.233010+0000) 2023-08-28T00:56:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:50 smithi136 ceph-mon[113531]: pgmap v7298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:51.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2779391841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:51.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:51.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:51.233240+0000) 2023-08-28T00:56:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:50 smithi093 ceph-mon[111335]: pgmap v7298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2779391841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:50 smithi093 ceph-mon[118902]: pgmap v7298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2779391841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:51.163Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:51.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:52.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:52.233502+0000) 2023-08-28T00:56:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:52.611Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:56:52.611Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:56:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:53 smithi136 ceph-mon[113531]: pgmap v7299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/722436561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:53.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:53.233716+0000) 2023-08-28T00:56:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:53 smithi093 ceph-mon[111335]: pgmap v7299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/722436561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:53 smithi093 ceph-mon[118902]: pgmap v7299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/722436561' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:54.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:54.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:54.233956+0000) 2023-08-28T00:56:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:56:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:55 smithi093 ceph-mon[111335]: pgmap v7300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:55 smithi093 ceph-mon[118902]: pgmap v7300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:55 smithi136 ceph-mon[113531]: pgmap v7300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:55.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:55.234204+0000) 2023-08-28T00:56:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:56:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:56:55] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:56:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1659285191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:56.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:56.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:56.234465+0000) 2023-08-28T00:56:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1659285191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1659285191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:57 smithi136 ceph-mon[113531]: pgmap v7301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:57.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:57.234796+0000) 2023-08-28T00:56:57.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:57 smithi093 ceph-mon[111335]: pgmap v7301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:57 smithi093 ceph-mon[118902]: pgmap v7301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3061669877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:58.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:58.235089+0000) 2023-08-28T00:56:58.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3061669877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3061669877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:56:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:56:59 smithi136 ceph-mon[113531]: pgmap v7302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:56:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:56:59.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:56:59.235366+0000) 2023-08-28T00:56:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:56:59 smithi093 ceph-mon[111335]: pgmap v7302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:56:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:56:59 smithi093 ceph-mon[118902]: pgmap v7302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3973323102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:00.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:00.235654+0000) 2023-08-28T00:57:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3973323102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3973323102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:01 smithi136 ceph-mon[113531]: pgmap v7303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:01.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:01.235899+0000) 2023-08-28T00:57:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:01 smithi093 ceph-mon[111335]: pgmap v7303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:01 smithi093 ceph-mon[118902]: pgmap v7303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:01.163Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:01.164Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:02.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:02.236170+0000) 2023-08-28T00:57:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:02.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:02.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:03 smithi136 ceph-mon[113531]: pgmap v7304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:03.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/853162997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:03.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:03.236460+0000) 2023-08-28T00:57:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:03 smithi093 ceph-mon[111335]: pgmap v7304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/853162997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:03 smithi093 ceph-mon[118902]: pgmap v7304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/853162997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:04.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:04.236767+0000) 2023-08-28T00:57:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:05 smithi093 ceph-mon[111335]: pgmap v7305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:05.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1416716331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:05 smithi093 ceph-mon[118902]: pgmap v7305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:05.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1416716331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:05 smithi136 ceph-mon[113531]: pgmap v7305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:05.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1416716331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:05.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:05.237089+0000) 2023-08-28T00:57:05.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:57:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:57:05] "GET /metrics HTTP/1.1" 200 239926 "" "Prometheus/2.43.0" 2023-08-28T00:57:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:06.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:06.237368+0000) 2023-08-28T00:57:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:07 smithi136 ceph-mon[113531]: pgmap v7306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:07.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1136462542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:07.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:07.237632+0000) 2023-08-28T00:57:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:07 smithi093 ceph-mon[111335]: pgmap v7306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1136462542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:07 smithi093 ceph-mon[118902]: pgmap v7306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1136462542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:08.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:08.237923+0000) 2023-08-28T00:57:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:09 smithi136 ceph-mon[113531]: pgmap v7307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:09.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:09.238216+0000) 2023-08-28T00:57:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:09 smithi093 ceph-mon[111335]: pgmap v7307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:09 smithi093 ceph-mon[118902]: pgmap v7307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1179885770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:10.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:10.238461+0000) 2023-08-28T00:57:10.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1179885770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1179885770' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:11 smithi136 ceph-mon[113531]: pgmap v7308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:11.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:11.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:11.238714+0000) 2023-08-28T00:57:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:11 smithi093 ceph-mon[111335]: pgmap v7308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:11 smithi093 ceph-mon[118902]: pgmap v7308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:11.164Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:11.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3063900059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:12.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:12.238989+0000) 2023-08-28T00:57:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3063900059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3063900059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:12.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:12.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:13 smithi136 ceph-mon[113531]: pgmap v7309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:13.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:13.239240+0000) 2023-08-28T00:57:13.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:13 smithi093 ceph-mon[111335]: pgmap v7309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:13 smithi093 ceph-mon[118902]: pgmap v7309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:14.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:14.239471+0000) 2023-08-28T00:57:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:15 smithi093 ceph-mon[111335]: pgmap v7310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/667818146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:15 smithi093 ceph-mon[118902]: pgmap v7310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:15.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/667818146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:15 smithi136 ceph-mon[113531]: pgmap v7310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/667818146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:15.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:15.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:15.239663+0000) 2023-08-28T00:57:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:57:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:57:15] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:57:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:16.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:16.239853+0000) 2023-08-28T00:57:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:17.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:17.240151+0000) 2023-08-28T00:57:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:17 smithi136 ceph-mon[113531]: pgmap v7311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:17.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4051778339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:17 smithi093 ceph-mon[118902]: pgmap v7311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4051778339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:17 smithi093 ceph-mon[111335]: pgmap v7311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4051778339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:57:18.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:18.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:18.240334+0000) 2023-08-28T00:57:18.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:57:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:57:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:19 smithi136 ceph-mon[113531]: pgmap v7312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:19.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1957144013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:19.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:19.240504+0000) 2023-08-28T00:57:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:19 smithi093 ceph-mon[111335]: pgmap v7312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1957144013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:19 smithi093 ceph-mon[118902]: pgmap v7312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1957144013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:20.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:20.240670+0000) 2023-08-28T00:57:21.165 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[111335]: pgmap v7313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[118902]: pgmap v7313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.166 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:20 smithi136 ceph-mon[113531]: pgmap v7313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:21.164Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:21.165Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:21.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:21.240903+0000) 2023-08-28T00:57:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:57:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:57:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:22.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1168236238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:57:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:57:22.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:22.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1168236238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:22.241 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:57:22.241 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:57:22.241 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:57:22.241 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1168236238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:22.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:22.241179+0000) 2023-08-28T00:57:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:22.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:22.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:23 smithi136 ceph-mon[113531]: pgmap v7314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:23.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:23.241457+0000) 2023-08-28T00:57:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:23 smithi093 ceph-mon[111335]: pgmap v7314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:23 smithi093 ceph-mon[118902]: pgmap v7314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4078419636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:24.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:24.241711+0000) 2023-08-28T00:57:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4078419636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4078419636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:25 smithi136 ceph-mon[113531]: pgmap v7315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:25.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:25.241926+0000) 2023-08-28T00:57:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:25 smithi093 ceph-mon[111335]: pgmap v7315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:25 smithi093 ceph-mon[118902]: pgmap v7315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:57:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:57:25] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:57:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:26.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:26.242185+0000) 2023-08-28T00:57:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:26 smithi093 ceph-mon[111335]: pgmap v7316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:27.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4067330353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:26 smithi093 ceph-mon[118902]: pgmap v7316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:27.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4067330353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:27.242 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:26 smithi136 ceph-mon[113531]: pgmap v7316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:27.242 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4067330353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:27.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:27.242485+0000) 2023-08-28T00:57:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:28.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:28.242748+0000) 2023-08-28T00:57:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:28 smithi093 ceph-mon[111335]: pgmap v7317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:29.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/185459560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:28 smithi093 ceph-mon[118902]: pgmap v7317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/185459560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:29.243 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:28 smithi136 ceph-mon[113531]: pgmap v7317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:29.243 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/185459560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:29.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:29.243052+0000) 2023-08-28T00:57:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:30.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:30.243349+0000) 2023-08-28T00:57:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:30 smithi093 ceph-mon[111335]: pgmap v7318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:31.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2405461214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:30 smithi093 ceph-mon[118902]: pgmap v7318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:31.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2405461214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:31.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:31.165Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:31.241 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:31.166Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:31.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:30 smithi136 ceph-mon[113531]: pgmap v7318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:31.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2405461214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:31.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:31.243636+0000) 2023-08-28T00:57:32.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:32.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:32.243923+0000) 2023-08-28T00:57:32.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:32.613Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:32.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:32.614Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:32 smithi093 ceph-mon[111335]: pgmap v7319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:33.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:32 smithi093 ceph-mon[118902]: pgmap v7319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:33.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:32 smithi136 ceph-mon[113531]: pgmap v7319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:33.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:33.244227+0000) 2023-08-28T00:57:34.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3046367627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:34.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3046367627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:34.244 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3046367627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:34.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:34.244514+0000) 2023-08-28T00:57:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:34 smithi093 ceph-mon[111335]: pgmap v7320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:34 smithi093 ceph-mon[118902]: pgmap v7320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:35.245 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:34 smithi136 ceph-mon[113531]: pgmap v7320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:35.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:35.244781+0000) 2023-08-28T00:57:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:57:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:57:35] "GET /metrics HTTP/1.1" 200 239929 "" "Prometheus/2.43.0" 2023-08-28T00:57:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2453724362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:36.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:36.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:36.245067+0000) 2023-08-28T00:57:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2453724362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:36.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2453724362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:36 smithi136 ceph-mon[113531]: pgmap v7321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:37.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:37.245366+0000) 2023-08-28T00:57:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:36 smithi093 ceph-mon[111335]: pgmap v7321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:36 smithi093 ceph-mon[118902]: pgmap v7321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/495311443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:38.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:38.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:38.245668+0000) 2023-08-28T00:57:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/495311443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/495311443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:39 smithi136 ceph-mon[113531]: pgmap v7322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:39.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:39.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:39.245974+0000) 2023-08-28T00:57:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:39 smithi093 ceph-mon[111335]: pgmap v7322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:39 smithi093 ceph-mon[118902]: pgmap v7322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:40.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:40.246258+0000) 2023-08-28T00:57:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:41 smithi136 ceph-mon[113531]: pgmap v7323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:41.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/828231724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:41.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:41.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:41.246510+0000) 2023-08-28T00:57:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:41 smithi093 ceph-mon[111335]: pgmap v7323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/828231724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:41 smithi093 ceph-mon[118902]: pgmap v7323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:41.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/828231724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:41.166Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:41.167Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:42.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:42.246849+0000) 2023-08-28T00:57:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:42.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:42.615Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:43 smithi136 ceph-mon[113531]: pgmap v7324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:43.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1296104697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:43.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:43.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:43.247182+0000) 2023-08-28T00:57:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:43 smithi093 ceph-mon[111335]: pgmap v7324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1296104697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:43 smithi093 ceph-mon[118902]: pgmap v7324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1296104697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:44.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:44.247468+0000) 2023-08-28T00:57:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:45 smithi093 ceph-mon[111335]: pgmap v7325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:45 smithi093 ceph-mon[118902]: pgmap v7325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:45 smithi136 ceph-mon[113531]: pgmap v7325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:45.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:45.247723+0000) 2023-08-28T00:57:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:57:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:57:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T00:57:46.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2158438948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:46.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:46.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:46.248005+0000) 2023-08-28T00:57:46.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2158438948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:46.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2158438948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:47.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:47 smithi136 ceph-mon[113531]: pgmap v7326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:47.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:47.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:47.248287+0000) 2023-08-28T00:57:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:47 smithi093 ceph-mon[111335]: pgmap v7326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:47 smithi093 ceph-mon[118902]: pgmap v7326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:48.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3485347226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:48.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:48.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:48.248503+0000) 2023-08-28T00:57:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3485347226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3485347226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:49 smithi136 ceph-mon[113531]: pgmap v7327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:49.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:49.248724+0000) 2023-08-28T00:57:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:49 smithi093 ceph-mon[111335]: pgmap v7327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:49 smithi093 ceph-mon[118902]: pgmap v7327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3948200046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:50.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:50.248914+0000) 2023-08-28T00:57:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3948200046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3948200046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:51 smithi136 ceph-mon[113531]: pgmap v7328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:51.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:51.249200+0000) 2023-08-28T00:57:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:51 smithi093 ceph-mon[111335]: pgmap v7328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:51 smithi093 ceph-mon[118902]: pgmap v7328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:51.167Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:51.168Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:52.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:52.249453+0000) 2023-08-28T00:57:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:52.615Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:57:52.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:57:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:53 smithi136 ceph-mon[113531]: pgmap v7329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:53.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1044365830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:53.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:53.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:53.249699+0000) 2023-08-28T00:57:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:53 smithi093 ceph-mon[111335]: pgmap v7329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1044365830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:53 smithi093 ceph-mon[118902]: pgmap v7329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1044365830' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:54.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:54.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:54.249943+0000) 2023-08-28T00:57:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:57:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:55 smithi093 ceph-mon[111335]: pgmap v7330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1037198792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:55 smithi093 ceph-mon[118902]: pgmap v7330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1037198792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:55 smithi136 ceph-mon[113531]: pgmap v7330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1037198792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:55.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:55.250195+0000) 2023-08-28T00:57:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:57:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:57:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:57:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:56.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:56.250441+0000) 2023-08-28T00:57:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:57 smithi136 ceph-mon[113531]: pgmap v7331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:57.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:57.250717+0000) 2023-08-28T00:57:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:57 smithi093 ceph-mon[111335]: pgmap v7331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:57 smithi093 ceph-mon[118902]: pgmap v7331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:58.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/184744280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:58.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:58.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:58.250952+0000) 2023-08-28T00:57:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/184744280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/184744280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:57:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:57:59 smithi136 ceph-mon[113531]: pgmap v7332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:57:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:57:59.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:57:59.251242+0000) 2023-08-28T00:57:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:57:59 smithi093 ceph-mon[111335]: pgmap v7332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:57:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:57:59 smithi093 ceph-mon[118902]: pgmap v7332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1605227782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:00.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:00.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:00.251557+0000) 2023-08-28T00:58:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1605227782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1605227782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:01 smithi136 ceph-mon[113531]: pgmap v7333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:01.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:01.251846+0000) 2023-08-28T00:58:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:01 smithi093 ceph-mon[111335]: pgmap v7333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:01 smithi093 ceph-mon[118902]: pgmap v7333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:01.168Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:01.169Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4090811564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:02.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:02.252124+0000) 2023-08-28T00:58:02.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4090811564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4090811564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:02.615Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:03.004 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:02.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:03 smithi136 ceph-mon[113531]: pgmap v7334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:03.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:03.252411+0000) 2023-08-28T00:58:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:03 smithi093 ceph-mon[111335]: pgmap v7334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:03 smithi093 ceph-mon[118902]: pgmap v7334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:04.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:04.252703+0000) 2023-08-28T00:58:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:05 smithi093 ceph-mon[111335]: pgmap v7335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/895709421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:05 smithi093 ceph-mon[118902]: pgmap v7335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/895709421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:05 smithi136 ceph-mon[113531]: pgmap v7335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/895709421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:05.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:05.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:05.252986+0000) 2023-08-28T00:58:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:58:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:58:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:58:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:06.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:06.253204+0000) 2023-08-28T00:58:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:07 smithi136 ceph-mon[113531]: pgmap v7336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3472388720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:07.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:07.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:07.253481+0000) 2023-08-28T00:58:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:07 smithi093 ceph-mon[111335]: pgmap v7336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3472388720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:07 smithi093 ceph-mon[118902]: pgmap v7336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3472388720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:08.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:08.253768+0000) 2023-08-28T00:58:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:09 smithi136 ceph-mon[113531]: pgmap v7337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2316438503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:09.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:09.254068+0000) 2023-08-28T00:58:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:09 smithi093 ceph-mon[111335]: pgmap v7337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2316438503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:09 smithi093 ceph-mon[118902]: pgmap v7337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2316438503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:10.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:10.254309+0000) 2023-08-28T00:58:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:11 smithi136 ceph-mon[113531]: pgmap v7338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:11.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:11.254566+0000) 2023-08-28T00:58:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:11 smithi093 ceph-mon[111335]: pgmap v7338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:11 smithi093 ceph-mon[118902]: pgmap v7338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:11.168Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:11.169Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2937374657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:12.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:12.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:12.254865+0000) 2023-08-28T00:58:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2937374657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2937374657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:12.616Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:12.617Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:13 smithi136 ceph-mon[113531]: pgmap v7339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:13.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:13.255151+0000) 2023-08-28T00:58:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:13 smithi093 ceph-mon[111335]: pgmap v7339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:13 smithi093 ceph-mon[118902]: pgmap v7339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3738218138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:14.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:14.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:14.255489+0000) 2023-08-28T00:58:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3738218138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3738218138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:15.362 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:15 smithi093 ceph-mon[111335]: pgmap v7340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:15.362 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:15 smithi093 ceph-mon[118902]: pgmap v7340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:15 smithi136 ceph-mon[113531]: pgmap v7340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:15.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:15.255746+0000) 2023-08-28T00:58:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:58:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:58:15] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T00:58:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:16.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:16.256049+0000) 2023-08-28T00:58:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:17 smithi136 ceph-mon[113531]: pgmap v7341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3236046248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:17.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:17.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:17.256349+0000) 2023-08-28T00:58:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:17 smithi093 ceph-mon[118902]: pgmap v7341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:17.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3236046248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:17.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:17 smithi093 ceph-mon[111335]: pgmap v7341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:17.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3236046248' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:18.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:18.256644+0000) 2023-08-28T00:58:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:19 smithi136 ceph-mon[113531]: pgmap v7342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3877136312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:19.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:19.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:19.256912+0000) 2023-08-28T00:58:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:19 smithi093 ceph-mon[111335]: pgmap v7342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3877136312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:19 smithi093 ceph-mon[118902]: pgmap v7342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3877136312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:20.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:20.257160+0000) 2023-08-28T00:58:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:21 smithi136 ceph-mon[113531]: pgmap v7343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3364040138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:21.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:21.257343+0000) 2023-08-28T00:58:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:21 smithi093 ceph-mon[111335]: pgmap v7343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3364040138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:21 smithi093 ceph-mon[118902]: pgmap v7343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3364040138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:21.169Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:21.170Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:22.366 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:58:22.367 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:58:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:58:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:22.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:22.257539+0000) 2023-08-28T00:58:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:22.617Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:22.617Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:23 smithi136 ceph-mon[113531]: pgmap v7344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:23.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:23.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:23.257728+0000) 2023-08-28T00:58:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:23 smithi093 ceph-mon[118902]: pgmap v7344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:23.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:23.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:23.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:23 smithi093 ceph-mon[111335]: pgmap v7344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:23.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:23.492 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:24.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:24.257894+0000) 2023-08-28T00:58:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3662004064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3662004064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3662004064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:24.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:25.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:25 smithi136 ceph-mon[113531]: pgmap v7345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:25.518 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:25.519 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:25.519 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:25.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:25.258091+0000) 2023-08-28T00:58:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:58:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:58:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:58:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:25 smithi093 ceph-mon[111335]: pgmap v7345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:25.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:25 smithi093 ceph-mon[118902]: pgmap v7345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:25.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:26.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:26.258235+0000) 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1183709431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1183709431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:26.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:58:26.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:58:26.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1183709431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:26.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:58:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:58:26.940 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:58:27.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:27 smithi136 ceph-mon[113531]: pgmap v7346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:27.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:27.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:27.258416+0000) 2023-08-28T00:58:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:27 smithi093 ceph-mon[111335]: pgmap v7346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:27 smithi093 ceph-mon[118902]: pgmap v7346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:28.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2772073854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:28.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:28.258683+0000) 2023-08-28T00:58:28.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2772073854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:28.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2772073854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:29.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:29 smithi136 ceph-mon[113531]: pgmap v7347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:29.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:29.258982+0000) 2023-08-28T00:58:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:29 smithi093 ceph-mon[111335]: pgmap v7347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:29.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:29 smithi093 ceph-mon[118902]: pgmap v7347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:30.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3078431392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:30.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:30.259161+0000) 2023-08-28T00:58:30.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3078431392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:30.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3078431392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:31.473 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:31.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:31.474 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:31.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:31 smithi136 ceph-mon[113531]: pgmap v7348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:31.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:31.259403+0000) 2023-08-28T00:58:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:31 smithi093 ceph-mon[111335]: pgmap v7348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:31 smithi093 ceph-mon[118902]: pgmap v7348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:32.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:32.259656+0000) 2023-08-28T00:58:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:32.617Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:32.618Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:33 smithi136 ceph-mon[113531]: pgmap v7349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2577299689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:33.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:33.259962+0000) 2023-08-28T00:58:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:33 smithi093 ceph-mon[111335]: pgmap v7349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2577299689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:33 smithi093 ceph-mon[118902]: pgmap v7349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2577299689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:34.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:34.260251+0000) 2023-08-28T00:58:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:35 smithi136 ceph-mon[113531]: pgmap v7350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3034571975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:35.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:35.260519+0000) 2023-08-28T00:58:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:58:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:58:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T00:58:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:35 smithi093 ceph-mon[111335]: pgmap v7350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3034571975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:35 smithi093 ceph-mon[118902]: pgmap v7350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3034571975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:36.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:36.260779+0000) 2023-08-28T00:58:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:37 smithi136 ceph-mon[113531]: pgmap v7351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:37.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:37.261066+0000) 2023-08-28T00:58:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:37 smithi093 ceph-mon[111335]: pgmap v7351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:37 smithi093 ceph-mon[118902]: pgmap v7351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3384847401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:38.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:38.261309+0000) 2023-08-28T00:58:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3384847401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:38.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3384847401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:39 smithi136 ceph-mon[113531]: pgmap v7352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:39.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:39.261531+0000) 2023-08-28T00:58:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:39 smithi093 ceph-mon[111335]: pgmap v7352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:39 smithi093 ceph-mon[118902]: pgmap v7352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/213136199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:40 smithi136 ceph-mon[113531]: pgmap v7353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:40.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:40.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:40.261789+0000) 2023-08-28T00:58:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/213136199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:40.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:40 smithi093 ceph-mon[111335]: pgmap v7353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/213136199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:40 smithi093 ceph-mon[118902]: pgmap v7353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:41.170Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:41.171Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:41.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:41.262027+0000) 2023-08-28T00:58:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:42.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:42.262342+0000) 2023-08-28T00:58:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:42.618Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:42.619Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:43.262 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:42 smithi136 ceph-mon[113531]: pgmap v7354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:43.263 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1064503682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:42 smithi093 ceph-mon[111335]: pgmap v7354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1064503682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:42 smithi093 ceph-mon[118902]: pgmap v7354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:43.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1064503682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:43.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:43.262576+0000) 2023-08-28T00:58:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:44.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:44.262842+0000) 2023-08-28T00:58:45.263 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:44 smithi136 ceph-mon[113531]: pgmap v7355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:45.263 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/974339745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:44 smithi093 ceph-mon[111335]: pgmap v7355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/974339745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:44 smithi093 ceph-mon[118902]: pgmap v7355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/974339745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:45.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:45.263080+0000) 2023-08-28T00:58:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:58:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:58:45] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T00:58:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:46.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:46.263378+0000) 2023-08-28T00:58:47.264 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:47 smithi136 ceph-mon[113531]: pgmap v7356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:47 smithi093 ceph-mon[111335]: pgmap v7356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:47 smithi093 ceph-mon[118902]: pgmap v7356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:47.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:47.263693+0000) 2023-08-28T00:58:48.264 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2552080506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:48.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2552080506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:48.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2552080506' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:48.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:48.263978+0000) 2023-08-28T00:58:49.264 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:49 smithi136 ceph-mon[113531]: pgmap v7357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:49 smithi093 ceph-mon[111335]: pgmap v7357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:49 smithi093 ceph-mon[118902]: pgmap v7357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:49.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:49.264255+0000) 2023-08-28T00:58:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2961416756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:50.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:50.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:50.264519+0000) 2023-08-28T00:58:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2961416756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2961416756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:51 smithi136 ceph-mon[113531]: pgmap v7358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:51.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:51.264786+0000) 2023-08-28T00:58:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:51 smithi093 ceph-mon[111335]: pgmap v7358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:51 smithi093 ceph-mon[118902]: pgmap v7358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:51.171Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:51.172Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2610227881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:52.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:52.265114+0000) 2023-08-28T00:58:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2610227881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2610227881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:52.618Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:58:52.619Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:58:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:53 smithi136 ceph-mon[113531]: pgmap v7359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:53.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:53.265377+0000) 2023-08-28T00:58:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:53 smithi093 ceph-mon[111335]: pgmap v7359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:53 smithi093 ceph-mon[118902]: pgmap v7359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:54.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:54.265636+0000) 2023-08-28T00:58:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:58:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:55 smithi093 ceph-mon[111335]: pgmap v7360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/513267601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:55 smithi093 ceph-mon[118902]: pgmap v7360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/513267601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:55 smithi136 ceph-mon[113531]: pgmap v7360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/513267601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:55.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:55.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:55.265861+0000) 2023-08-28T00:58:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:58:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:58:55] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:58:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:56.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:56.266090+0000) 2023-08-28T00:58:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:57 smithi136 ceph-mon[113531]: pgmap v7361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4131702526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:57.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:57.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:57.266347+0000) 2023-08-28T00:58:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:57 smithi093 ceph-mon[111335]: pgmap v7361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4131702526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:57 smithi093 ceph-mon[118902]: pgmap v7361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4131702526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:58.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:58.266571+0000) 2023-08-28T00:58:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:59 smithi136 ceph-mon[113531]: pgmap v7362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:58:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/770778386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:58:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:58:59.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:58:59.266785+0000) 2023-08-28T00:58:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:59 smithi093 ceph-mon[111335]: pgmap v7362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:58:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/770778386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:58:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:59 smithi093 ceph-mon[118902]: pgmap v7362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:58:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:58:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/770778386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:00.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:00.267064+0000) 2023-08-28T00:59:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:01 smithi136 ceph-mon[113531]: pgmap v7363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:01.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:01.267381+0000) 2023-08-28T00:59:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:01 smithi093 ceph-mon[111335]: pgmap v7363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:01 smithi093 ceph-mon[118902]: pgmap v7363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:01.171Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:01.173Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/624746909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:02.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:02.267581+0000) 2023-08-28T00:59:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/624746909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:02.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/624746909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:02.619Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:02.620Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:03 smithi136 ceph-mon[113531]: pgmap v7364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:03.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:03.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:03.267846+0000) 2023-08-28T00:59:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:03 smithi093 ceph-mon[111335]: pgmap v7364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:03 smithi093 ceph-mon[118902]: pgmap v7364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1102466004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:04.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:04.268101+0000) 2023-08-28T00:59:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1102466004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1102466004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:05 smithi093 ceph-mon[111335]: pgmap v7365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:05 smithi093 ceph-mon[118902]: pgmap v7365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:05 smithi136 ceph-mon[113531]: pgmap v7365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:05.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:05.268362+0000) 2023-08-28T00:59:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:59:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:59:05] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:59:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:06.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:06.268639+0000) 2023-08-28T00:59:07.269 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:06 smithi136 ceph-mon[113531]: pgmap v7366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:07.269 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1989684227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:06 smithi093 ceph-mon[111335]: pgmap v7366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:07.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1989684227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:06 smithi093 ceph-mon[118902]: pgmap v7366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1989684227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:07.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:07.268853+0000) 2023-08-28T00:59:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:08.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:08.269123+0000) 2023-08-28T00:59:09.269 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:09 smithi136 ceph-mon[113531]: pgmap v7367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:09.269 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3268032072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:09.269 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:09 smithi093 ceph-mon[111335]: pgmap v7367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3268032072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:09 smithi093 ceph-mon[118902]: pgmap v7367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3268032072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:09.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:09.269397+0000) 2023-08-28T00:59:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:10.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:10.269651+0000) 2023-08-28T00:59:11.270 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:11 smithi136 ceph-mon[113531]: pgmap v7368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:11.270 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3260950905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:11 smithi093 ceph-mon[111335]: pgmap v7368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:11.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3260950905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:11 smithi093 ceph-mon[118902]: pgmap v7368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3260950905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:11.172Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:11.173Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:11.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:11.269900+0000) 2023-08-28T00:59:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:12.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:12.270180+0000) 2023-08-28T00:59:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:12.620Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:12.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:12.621Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:13.270 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:13 smithi136 ceph-mon[113531]: pgmap v7369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:13 smithi093 ceph-mon[111335]: pgmap v7369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:13 smithi093 ceph-mon[118902]: pgmap v7369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:13.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:13.270422+0000) 2023-08-28T00:59:14.270 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3438398467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3438398467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3438398467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:14.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:14.270646+0000) 2023-08-28T00:59:15.271 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:15 smithi136 ceph-mon[113531]: pgmap v7370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:15 smithi093 ceph-mon[111335]: pgmap v7370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:15 smithi093 ceph-mon[118902]: pgmap v7370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:15.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:15.270868+0000) 2023-08-28T00:59:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:59:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:59:15] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T00:59:16.271 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3192614555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3192614555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3192614555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:16.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:16.271148+0000) 2023-08-28T00:59:17.271 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:17 smithi136 ceph-mon[113531]: pgmap v7371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:17 smithi093 ceph-mon[111335]: pgmap v7371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:17 smithi093 ceph-mon[118902]: pgmap v7371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:17.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:17.271477+0000) 2023-08-28T00:59:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:18.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:18.271732+0000) 2023-08-28T00:59:19.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:19 smithi136 ceph-mon[113531]: pgmap v7372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:19.272 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3420853911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:19 smithi093 ceph-mon[111335]: pgmap v7372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3420853911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:19 smithi093 ceph-mon[118902]: pgmap v7372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3420853911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:19.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:19.272003+0000) 2023-08-28T00:59:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:20.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:20.272309+0000) 2023-08-28T00:59:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:21 smithi136 ceph-mon[113531]: pgmap v7373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3746454579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:21.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:21.272538+0000) 2023-08-28T00:59:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:21 smithi093 ceph-mon[111335]: pgmap v7373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3746454579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:21 smithi093 ceph-mon[118902]: pgmap v7373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3746454579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:21.173Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:21.174Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:22.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:22.272791+0000) 2023-08-28T00:59:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:22.621Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 9 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:22.622Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:23 smithi136 ceph-mon[113531]: pgmap v7374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3148195630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:23.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:23.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:23.273084+0000) 2023-08-28T00:59:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:23 smithi093 ceph-mon[111335]: pgmap v7374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:23.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3148195630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:23 smithi093 ceph-mon[118902]: pgmap v7374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3148195630' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:24.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:24.273377+0000) 2023-08-28T00:59:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:25 smithi093 ceph-mon[111335]: pgmap v7375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:25 smithi093 ceph-mon[118902]: pgmap v7375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:25 smithi136 ceph-mon[113531]: pgmap v7375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:25.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:25.273661+0000) 2023-08-28T00:59:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:59:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:59:25] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:59:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1833329034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:26.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:26.273907+0000) 2023-08-28T00:59:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1833329034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:26.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1833329034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:27.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:27.274106+0000) 2023-08-28T00:59:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:27 smithi136 ceph-mon[113531]: pgmap v7376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:59:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:27.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[118902]: pgmap v7376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:59:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[111335]: pgmap v7376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T00:59:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:27.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:28.274 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1991920124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1991920124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:28.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1991920124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:28.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:28.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:28.274287+0000) 2023-08-28T00:59:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:29.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:29.274434+0000) 2023-08-28T00:59:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:29 smithi136 ceph-mon[113531]: pgmap v7377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[118902]: pgmap v7377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[111335]: pgmap v7377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002039105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:30.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:30.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:30.274642+0000) 2023-08-28T00:59:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:59:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:59:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002039105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T00:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T00:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T00:59:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002039105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:31 smithi093 ceph-mon[111335]: pgmap v7378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:32.091 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:31 smithi093 ceph-mon[118902]: pgmap v7378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:32.091 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:31.174Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:32.091 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:31.175Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:32.092 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:31 smithi136 ceph-mon[113531]: pgmap v7378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:32.092 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:31.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:31.274874+0000) 2023-08-28T00:59:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:32.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:32.275155+0000) 2023-08-28T00:59:33.163 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:32.622Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:33.164 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:32.623Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:33 smithi136 ceph-mon[113531]: pgmap v7379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:33.811 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4228521789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:33.811 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:33.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:33.275423+0000) 2023-08-28T00:59:33.812 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:33 smithi093 ceph-mon[111335]: pgmap v7379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:33.812 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4228521789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:33.812 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:33 smithi093 ceph-mon[118902]: pgmap v7379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:33.812 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4228521789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:34.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:34.275675+0000) 2023-08-28T00:59:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:35 smithi136 ceph-mon[113531]: pgmap v7380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3058511590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:35.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:35.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:35.275924+0000) 2023-08-28T00:59:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:35 smithi093 ceph-mon[111335]: pgmap v7380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3058511590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:35.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:59:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:59:35] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T00:59:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:35 smithi093 ceph-mon[118902]: pgmap v7380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3058511590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:36.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:36.276165+0000) 2023-08-28T00:59:37.278 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:37 smithi136 ceph-mon[113531]: pgmap v7381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:37 smithi093 ceph-mon[111335]: pgmap v7381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:37 smithi093 ceph-mon[118902]: pgmap v7381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:37.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:37.276390+0000) 2023-08-28T00:59:38.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/795856420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/795856420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:38.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/795856420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:38.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:38.276645+0000) 2023-08-28T00:59:39.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:39 smithi136 ceph-mon[113531]: pgmap v7382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:39.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:39 smithi093 ceph-mon[118902]: pgmap v7382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:39 smithi093 ceph-mon[111335]: pgmap v7382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:39.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:39.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:39.276887+0000) 2023-08-28T00:59:40.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1112277500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1112277500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1112277500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:40.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:40.277163+0000) 2023-08-28T00:59:41.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:41 smithi136 ceph-mon[113531]: pgmap v7383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:41 smithi093 ceph-mon[111335]: pgmap v7383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:41 smithi093 ceph-mon[118902]: pgmap v7383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:41.175Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:41.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:41.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:41.277370+0000) 2023-08-28T00:59:42.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2174392015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2174392015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2174392015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:42.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:42.277544+0000) 2023-08-28T00:59:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:42.623Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:42.624Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:43.277 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:43 smithi136 ceph-mon[113531]: pgmap v7384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:43 smithi093 ceph-mon[118902]: pgmap v7384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:43.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:43 smithi093 ceph-mon[111335]: pgmap v7384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:43.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:43.277757+0000) 2023-08-28T00:59:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:44.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:44.277938+0000) 2023-08-28T00:59:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:45 smithi093 ceph-mon[118902]: pgmap v7385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1744932027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:45 smithi093 ceph-mon[111335]: pgmap v7385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:45.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1744932027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:45 smithi136 ceph-mon[113531]: pgmap v7385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1744932027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:45.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:45.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:45.278212+0000) 2023-08-28T00:59:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:59:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:59:45] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:59:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:46.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:46.278510+0000) 2023-08-28T00:59:47.278 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:47 smithi136 ceph-mon[113531]: pgmap v7386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:47.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3308150359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:47 smithi093 ceph-mon[118902]: pgmap v7386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3308150359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:47 smithi093 ceph-mon[111335]: pgmap v7386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3308150359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:47.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:47.278766+0000) 2023-08-28T00:59:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:48.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:48.279028+0000) 2023-08-28T00:59:49.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:49 smithi136 ceph-mon[113531]: pgmap v7387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:49 smithi093 ceph-mon[111335]: pgmap v7387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:49 smithi093 ceph-mon[118902]: pgmap v7387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:49.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:49.279304+0000) 2023-08-28T00:59:50.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2856558390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2856558390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:50.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2856558390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:50.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:50.279530+0000) 2023-08-28T00:59:51.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:51 smithi136 ceph-mon[113531]: pgmap v7388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:51 smithi093 ceph-mon[111335]: pgmap v7388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:51 smithi093 ceph-mon[118902]: pgmap v7388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:51.175Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:51.176Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:51.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:51.279777+0000) 2023-08-28T00:59:52.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/73396756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/73396756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:52.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/73396756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:52.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:52.279995+0000) 2023-08-28T00:59:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:52.623Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:52.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 00:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T00:59:52.624Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T00:59:53.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:53 smithi136 ceph-mon[113531]: pgmap v7389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:53 smithi093 ceph-mon[111335]: pgmap v7389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:53 smithi093 ceph-mon[118902]: pgmap v7389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:53.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:53.280238+0000) 2023-08-28T00:59:54.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2138373216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:54.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2138373216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:54.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2138373216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T00:59:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:54.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:54.280451+0000) 2023-08-28T00:59:55.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:55 smithi136 ceph-mon[113531]: pgmap v7390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:55 smithi093 ceph-mon[111335]: pgmap v7390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:55 smithi093 ceph-mon[118902]: pgmap v7390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:55.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:55.280668+0000) 2023-08-28T00:59:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 00:59:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:00:59:55] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T00:59:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:56.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:56.280967+0000) 2023-08-28T00:59:57.281 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:57 smithi136 ceph-mon[113531]: pgmap v7391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:57.281 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1908553122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:57 smithi093 ceph-mon[111335]: pgmap v7391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1908553122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:57 smithi093 ceph-mon[118902]: pgmap v7391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1908553122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:57.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:57.281224+0000) 2023-08-28T00:59:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:58.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:58.281477+0000) 2023-08-28T00:59:59.282 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:59 smithi136 ceph-mon[113531]: pgmap v7392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:59.282 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 00:59:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2769695475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:59 smithi093 ceph-mon[111335]: pgmap v7392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:59.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 00:59:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2769695475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:59 smithi093 ceph-mon[118902]: pgmap v7392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T00:59:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 00:59:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2769695475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T00:59:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 00:59:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T00:59:59.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-27T23:59:59.281696+0000) 2023-08-28T01:00:00.282 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:00:00.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:00:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:00:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:00.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:00.281902+0000) 2023-08-28T01:00:00.689 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 28 01:00:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-28T01:00:00.408Z caller=compact.go:519 level=info component=tsdb msg="write block" mint=1693173600138 maxt=1693180800000 ulid=01H8WTF6Z6NY9T31QTWCMX1APN duration=50.067003ms 2023-08-28T01:00:00.689 INFO:journalctl@ceph.prometheus.a.smithi136.stdout:Aug 28 01:00:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-prometheus-a[137831]: ts=2023-08-28T01:00:00.411Z caller=head.go:1269 level=info component=tsdb msg="Head GC completed" caller=truncateMemory duration=3.105833ms 2023-08-28T01:00:01.282 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:01 smithi136 ceph-mon[113531]: pgmap v7393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:01.282 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1633836762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:01 smithi093 ceph-mon[111335]: pgmap v7393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1633836762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:01 smithi093 ceph-mon[118902]: pgmap v7393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1633836762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:01.176Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:01.177Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:01.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:01.282160+0000) 2023-08-28T01:00:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:02.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:02.282398+0000) 2023-08-28T01:00:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:02.623Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:02.625Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:03.283 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:03 smithi136 ceph-mon[113531]: pgmap v7394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:03 smithi093 ceph-mon[111335]: pgmap v7394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:03 smithi093 ceph-mon[118902]: pgmap v7394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:03.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:03.282660+0000) 2023-08-28T01:00:04.283 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2966852994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2966852994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2966852994' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:04.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:04.282917+0000) 2023-08-28T01:00:05.284 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:05 smithi136 ceph-mon[113531]: pgmap v7395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:05 smithi093 ceph-mon[111335]: pgmap v7395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:05 smithi093 ceph-mon[118902]: pgmap v7395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:05.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:05.283168+0000) 2023-08-28T01:00:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:00:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:00:05] "GET /metrics HTTP/1.1" 200 239925 "" "Prometheus/2.43.0" 2023-08-28T01:00:06.283 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2228776660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2228776660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2228776660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:06.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:06.283426+0000) 2023-08-28T01:00:07.284 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:07 smithi136 ceph-mon[113531]: pgmap v7396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:07 smithi093 ceph-mon[111335]: pgmap v7396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:07 smithi093 ceph-mon[118902]: pgmap v7396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:07.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:07.283676+0000) 2023-08-28T01:00:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:08.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:08.283924+0000) 2023-08-28T01:00:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:09 smithi136 ceph-mon[113531]: pgmap v7397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/138405485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:09.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:09.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:09.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:09.284185+0000) 2023-08-28T01:00:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:09 smithi093 ceph-mon[111335]: pgmap v7397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/138405485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:09 smithi093 ceph-mon[118902]: pgmap v7397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/138405485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:10.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:10.284414+0000) 2023-08-28T01:00:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:11 smithi136 ceph-mon[113531]: pgmap v7398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1682619199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:11.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:11.284667+0000) 2023-08-28T01:00:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:11 smithi093 ceph-mon[111335]: pgmap v7398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1682619199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:11 smithi093 ceph-mon[118902]: pgmap v7398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1682619199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:11.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:11.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:12.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:12.284928+0000) 2023-08-28T01:00:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:12.624Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:12.625Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:13.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:13.285184+0000) 2023-08-28T01:00:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:13 smithi136 ceph-mon[113531]: pgmap v7399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/218711918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:13 smithi093 ceph-mon[111335]: pgmap v7399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/218711918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:13 smithi093 ceph-mon[118902]: pgmap v7399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/218711918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:14.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:14.285369+0000) 2023-08-28T01:00:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:15 smithi093 ceph-mon[118902]: pgmap v7400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:15 smithi093 ceph-mon[111335]: pgmap v7400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:15 smithi136 ceph-mon[113531]: pgmap v7400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:15.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:15.285615+0000) 2023-08-28T01:00:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:00:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:00:15] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T01:00:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2897292884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:16.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:16.285883+0000) 2023-08-28T01:00:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2897292884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:16.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2897292884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:17 smithi136 ceph-mon[113531]: pgmap v7401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:17.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:17.286147+0000) 2023-08-28T01:00:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:17 smithi093 ceph-mon[111335]: pgmap v7401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:17 smithi093 ceph-mon[118902]: pgmap v7401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1170054224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:18.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:18.286406+0000) 2023-08-28T01:00:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1170054224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:18.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1170054224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:19 smithi136 ceph-mon[113531]: pgmap v7402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:19.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:19.286657+0000) 2023-08-28T01:00:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:19 smithi093 ceph-mon[111335]: pgmap v7402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:19 smithi093 ceph-mon[118902]: pgmap v7402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1099099780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:20.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:20.286887+0000) 2023-08-28T01:00:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1099099780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1099099780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:21 smithi136 ceph-mon[113531]: pgmap v7403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:21.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:21.287181+0000) 2023-08-28T01:00:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:21 smithi093 ceph-mon[111335]: pgmap v7403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:21 smithi093 ceph-mon[118902]: pgmap v7403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:21.177Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:21.178Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:22.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:22.287409+0000) 2023-08-28T01:00:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:22.625Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:22.626Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:23 smithi136 ceph-mon[113531]: pgmap v7404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2169487084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:23.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:23.287700+0000) 2023-08-28T01:00:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:23 smithi093 ceph-mon[111335]: pgmap v7404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2169487084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:23 smithi093 ceph-mon[118902]: pgmap v7404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2169487084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:24.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:24.287972+0000) 2023-08-28T01:00:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:25.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:25 smithi093 ceph-mon[118902]: pgmap v7405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/47022782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:25 smithi093 ceph-mon[111335]: pgmap v7405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/47022782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:25 smithi136 ceph-mon[113531]: pgmap v7405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/47022782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:25.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:25.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:25.288210+0000) 2023-08-28T01:00:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:00:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:00:25] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T01:00:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:26.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:26.288487+0000) 2023-08-28T01:00:27.288 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:27 smithi136 ceph-mon[113531]: pgmap v7406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:27 smithi093 ceph-mon[111335]: pgmap v7406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:27 smithi093 ceph-mon[118902]: pgmap v7406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:27.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:27.288753+0000) 2023-08-28T01:00:28.289 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1075357797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1075357797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:28.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1075357797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:28.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:28.289010+0000) 2023-08-28T01:00:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:29 smithi136 ceph-mon[113531]: pgmap v7407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:29.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:29.289212+0000) 2023-08-28T01:00:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:29 smithi093 ceph-mon[111335]: pgmap v7407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:29 smithi093 ceph-mon[118902]: pgmap v7407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:30.406 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:00:30.406 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/754865433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:30.406 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:30.407 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:30.407 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:30.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:30.289369+0000) 2023-08-28T01:00:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:00:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/754865433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/754865433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:31 smithi136 ceph-mon[113531]: pgmap v7408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:31.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:31.289544+0000) 2023-08-28T01:00:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:31 smithi093 ceph-mon[111335]: pgmap v7408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:31 smithi093 ceph-mon[118902]: pgmap v7408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:31.178Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:31.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:32.626 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.626 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.626 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3923203977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:32.626 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.627 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3923203977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:32.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:32.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:32.289727+0000) 2023-08-28T01:00:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3923203977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:32.626Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:32.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:33 smithi136 ceph-mon[113531]: pgmap v7409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:00:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:00:33.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:33.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:33.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:33.289942+0000) 2023-08-28T01:00:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[111335]: pgmap v7409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:00:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:00:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[118902]: pgmap v7409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:00:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:00:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:00:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:34.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:34.290163+0000) 2023-08-28T01:00:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:35 smithi136 ceph-mon[113531]: pgmap v7410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1308107147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:35.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:35.290405+0000) 2023-08-28T01:00:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:00:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:00:35] "GET /metrics HTTP/1.1" 200 239928 "" "Prometheus/2.43.0" 2023-08-28T01:00:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:35 smithi093 ceph-mon[111335]: pgmap v7410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1308107147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:35 smithi093 ceph-mon[118902]: pgmap v7410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1308107147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:36.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:36.290656+0000) 2023-08-28T01:00:37.291 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:37 smithi136 ceph-mon[113531]: pgmap v7411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:37.291 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3871406643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:37 smithi093 ceph-mon[111335]: pgmap v7411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:37.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3871406643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:37 smithi093 ceph-mon[118902]: pgmap v7411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3871406643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:37.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:37.290899+0000) 2023-08-28T01:00:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:38.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:38.291115+0000) 2023-08-28T01:00:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:39.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:39.291374+0000) 2023-08-28T01:00:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:39 smithi136 ceph-mon[113531]: pgmap v7412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:39 smithi093 ceph-mon[111335]: pgmap v7412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:39 smithi093 ceph-mon[118902]: pgmap v7412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:40.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4235236197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:40.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:40.291622+0000) 2023-08-28T01:00:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4235236197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:40.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4235236197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:41 smithi136 ceph-mon[113531]: pgmap v7413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:41.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:41.291906+0000) 2023-08-28T01:00:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:41 smithi093 ceph-mon[111335]: pgmap v7413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:41 smithi093 ceph-mon[118902]: pgmap v7413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:41.179Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:41.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1391374402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:42.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:42.292140+0000) 2023-08-28T01:00:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1391374402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1391374402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:42.626Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:42.627Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:43 smithi136 ceph-mon[113531]: pgmap v7414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:43.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:43.292388+0000) 2023-08-28T01:00:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:43 smithi093 ceph-mon[111335]: pgmap v7414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:43 smithi093 ceph-mon[118902]: pgmap v7414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:44.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/189472245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:44.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:44.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:44.292646+0000) 2023-08-28T01:00:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/189472245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/189472245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:45.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:45 smithi093 ceph-mon[111335]: pgmap v7415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:45 smithi093 ceph-mon[118902]: pgmap v7415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:45 smithi136 ceph-mon[113531]: pgmap v7415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:45.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:45.292955+0000) 2023-08-28T01:00:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:00:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:00:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T01:00:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:46.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:46.293217+0000) 2023-08-28T01:00:47.293 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:47 smithi136 ceph-mon[113531]: pgmap v7416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:47.293 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1023303993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:47 smithi093 ceph-mon[118902]: pgmap v7416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1023303993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:47 smithi093 ceph-mon[111335]: pgmap v7416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:47.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1023303993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:47.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:47.293467+0000) 2023-08-28T01:00:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:48.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:48.293731+0000) 2023-08-28T01:00:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:49 smithi136 ceph-mon[113531]: pgmap v7417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:49.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4096140522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:49.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:49.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:49.293988+0000) 2023-08-28T01:00:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:49 smithi093 ceph-mon[111335]: pgmap v7417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4096140522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:49 smithi093 ceph-mon[118902]: pgmap v7417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4096140522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:50.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:50.294205+0000) 2023-08-28T01:00:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:51 smithi136 ceph-mon[113531]: pgmap v7418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2048409769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:51.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:51.294473+0000) 2023-08-28T01:00:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:51 smithi093 ceph-mon[118902]: pgmap v7418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2048409769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:51 smithi093 ceph-mon[111335]: pgmap v7418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2048409769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:51.179Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:51.180Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:52.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:52.294687+0000) 2023-08-28T01:00:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:52.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:00:52.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:00:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:53 smithi136 ceph-mon[113531]: pgmap v7419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:53.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:53.294939+0000) 2023-08-28T01:00:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:53 smithi093 ceph-mon[111335]: pgmap v7419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:53 smithi093 ceph-mon[118902]: pgmap v7419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4048471240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:54.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:54.295179+0000) 2023-08-28T01:00:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4048471240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4048471240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:00:55.360 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:55 smithi093 ceph-mon[118902]: pgmap v7420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:55.360 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:55 smithi093 ceph-mon[111335]: pgmap v7420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:55 smithi136 ceph-mon[113531]: pgmap v7420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:55.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:55.295410+0000) 2023-08-28T01:00:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:00:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:00:55] "GET /metrics HTTP/1.1" 200 239934 "" "Prometheus/2.43.0" 2023-08-28T01:00:56.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1181513455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:56.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:56.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:56.295697+0000) 2023-08-28T01:00:56.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1181513455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1181513455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:57 smithi136 ceph-mon[113531]: pgmap v7421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:57.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:57.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:57.295922+0000) 2023-08-28T01:00:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:57 smithi093 ceph-mon[111335]: pgmap v7421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:57 smithi093 ceph-mon[118902]: pgmap v7421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:58.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:58.296182+0000) 2023-08-28T01:00:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:59 smithi136 ceph-mon[113531]: pgmap v7422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:00:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/743970998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:00:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:00:59.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:00:59.296417+0000) 2023-08-28T01:00:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:59 smithi093 ceph-mon[111335]: pgmap v7422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:00:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/743970998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:00:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:59 smithi093 ceph-mon[118902]: pgmap v7422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:00:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:00:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/743970998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:00.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:00.296623+0000) 2023-08-28T01:01:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:01 smithi136 ceph-mon[113531]: pgmap v7423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1213586276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:01.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:01.296841+0000) 2023-08-28T01:01:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:01 smithi093 ceph-mon[111335]: pgmap v7423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1213586276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:01 smithi093 ceph-mon[118902]: pgmap v7423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1213586276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:01.180Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:01.181Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:02.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:02.297047+0000) 2023-08-28T01:01:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:02.628Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:02.629Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:03.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:03.297332+0000) 2023-08-28T01:01:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:03 smithi136 ceph-mon[113531]: pgmap v7424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3022076338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:03 smithi093 ceph-mon[111335]: pgmap v7424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:03.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3022076338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:03 smithi093 ceph-mon[118902]: pgmap v7424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3022076338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:04.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:04.297604+0000) 2023-08-28T01:01:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:05 smithi093 ceph-mon[118902]: pgmap v7425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:05 smithi093 ceph-mon[111335]: pgmap v7425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:05 smithi136 ceph-mon[113531]: pgmap v7425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:05.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:05.297860+0000) 2023-08-28T01:01:05.440 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 01:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T01:01:05.317222577Z level=info msg="Completed cleanup jobs" duration=12.72962ms 2023-08-28T01:01:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:01:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:01:05] "GET /metrics HTTP/1.1" 200 239934 "" "Prometheus/2.43.0" 2023-08-28T01:01:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4019361039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:06.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:06.298153+0000) 2023-08-28T01:01:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4019361039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4019361039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:07 smithi136 ceph-mon[113531]: pgmap v7426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:07.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:07.298398+0000) 2023-08-28T01:01:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:07 smithi093 ceph-mon[111335]: pgmap v7426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:07 smithi093 ceph-mon[118902]: pgmap v7426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/772921173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:08.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:08.298637+0000) 2023-08-28T01:01:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/772921173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/772921173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:09 smithi136 ceph-mon[113531]: pgmap v7427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:09.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:09.298886+0000) 2023-08-28T01:01:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:09 smithi093 ceph-mon[111335]: pgmap v7427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:09 smithi093 ceph-mon[118902]: pgmap v7427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:10.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2650495655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:10.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:10.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:10.299151+0000) 2023-08-28T01:01:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2650495655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2650495655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:11 smithi136 ceph-mon[113531]: pgmap v7428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:11.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:11.299416+0000) 2023-08-28T01:01:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:11 smithi093 ceph-mon[111335]: pgmap v7428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:11 smithi093 ceph-mon[118902]: pgmap v7428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:11.181Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:11.182Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:12.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:12.299657+0000) 2023-08-28T01:01:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:12.629Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:12.630Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:13 smithi136 ceph-mon[113531]: pgmap v7429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1827320929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:13.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:13.299881+0000) 2023-08-28T01:01:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:13 smithi093 ceph-mon[111335]: pgmap v7429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1827320929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:13 smithi093 ceph-mon[118902]: pgmap v7429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:13.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1827320929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:14.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:14.300149+0000) 2023-08-28T01:01:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:15 smithi136 ceph-mon[113531]: pgmap v7430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1432092436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:15.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:15.300427+0000) 2023-08-28T01:01:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:15 smithi093 ceph-mon[111335]: pgmap v7430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1432092436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:01:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:01:15] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T01:01:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:15 smithi093 ceph-mon[118902]: pgmap v7430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1432092436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:16.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:16.300674+0000) 2023-08-28T01:01:17.300 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:17 smithi136 ceph-mon[113531]: pgmap v7431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:17 smithi093 ceph-mon[111335]: pgmap v7431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:17 smithi093 ceph-mon[118902]: pgmap v7431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:17.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:17.300904+0000) 2023-08-28T01:01:18.301 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1648079064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1648079064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:18.561 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1648079064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:18.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:18.301130+0000) 2023-08-28T01:01:19.301 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:19 smithi136 ceph-mon[113531]: pgmap v7432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:19 smithi093 ceph-mon[111335]: pgmap v7432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:19 smithi093 ceph-mon[118902]: pgmap v7432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:19.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:19.301361+0000) 2023-08-28T01:01:20.301 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1644387861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1644387861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1644387861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:20.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:20.301562+0000) 2023-08-28T01:01:21.301 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:21 smithi136 ceph-mon[113531]: pgmap v7433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:21 smithi093 ceph-mon[118902]: pgmap v7433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:21 smithi093 ceph-mon[111335]: pgmap v7433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:21.181Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:21.182Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:21.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:21.301831+0000) 2023-08-28T01:01:22.302 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2615844601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2615844601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2615844601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:22.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:22.302105+0000) 2023-08-28T01:01:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:22.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:22.631Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:23.302 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:23 smithi136 ceph-mon[113531]: pgmap v7434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:23 smithi093 ceph-mon[111335]: pgmap v7434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:23 smithi093 ceph-mon[118902]: pgmap v7434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:23.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:23.302325+0000) 2023-08-28T01:01:24.302 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:24.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:24.302553+0000) 2023-08-28T01:01:25.302 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:25 smithi136 ceph-mon[113531]: pgmap v7435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:25.302 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3462071181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:25 smithi093 ceph-mon[111335]: pgmap v7435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:25.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3462071181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:25 smithi093 ceph-mon[118902]: pgmap v7435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3462071181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:25.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:25.302781+0000) 2023-08-28T01:01:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:01:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:01:25] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T01:01:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:26.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:26.303015+0000) 2023-08-28T01:01:27.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:27 smithi136 ceph-mon[113531]: pgmap v7436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:27.303 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3897624447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:27 smithi093 ceph-mon[111335]: pgmap v7436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3897624447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:27 smithi093 ceph-mon[118902]: pgmap v7436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3897624447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:27.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:27.303302+0000) 2023-08-28T01:01:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:28.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:28.303571+0000) 2023-08-28T01:01:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:29 smithi136 ceph-mon[113531]: pgmap v7437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:29.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:29.303856+0000) 2023-08-28T01:01:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:29 smithi093 ceph-mon[111335]: pgmap v7437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:29 smithi093 ceph-mon[118902]: pgmap v7437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/427200442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:30.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:30.304137+0000) 2023-08-28T01:01:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/427200442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/427200442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:31 smithi136 ceph-mon[113531]: pgmap v7438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:31.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:31.304394+0000) 2023-08-28T01:01:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:31 smithi093 ceph-mon[111335]: pgmap v7438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:31 smithi093 ceph-mon[118902]: pgmap v7438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:31.182Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:31.184Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/422502290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:32.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:32.304637+0000) 2023-08-28T01:01:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/422502290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:32.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/422502290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:32.630Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:32.631Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:33 smithi136 ceph-mon[113531]: pgmap v7439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:01:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:33.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:33.304914+0000) 2023-08-28T01:01:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:33 smithi093 ceph-mon[118902]: pgmap v7439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:01:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:33 smithi093 ceph-mon[111335]: pgmap v7439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:01:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2139245513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:34.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:34.305161+0000) 2023-08-28T01:01:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2139245513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2139245513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:35.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:35.305361+0000) 2023-08-28T01:01:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:35 smithi136 ceph-mon[113531]: pgmap v7440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:01:35] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[111335]: pgmap v7440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[118902]: pgmap v7440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:35.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:01:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:01:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:36.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3373407862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:36.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:36.305545+0000) 2023-08-28T01:01:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:01:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:01:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:36.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3373407862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:01:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:01:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:01:36.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3373407862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:37.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:37 smithi136 ceph-mon[113531]: pgmap v7441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:37.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:37.305791+0000) 2023-08-28T01:01:37.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:37 smithi093 ceph-mon[111335]: pgmap v7441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:37 smithi093 ceph-mon[118902]: pgmap v7441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:38.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:38.306082+0000) 2023-08-28T01:01:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:39 smithi136 ceph-mon[113531]: pgmap v7442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1851818787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:39.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:39.306325+0000) 2023-08-28T01:01:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:39 smithi093 ceph-mon[111335]: pgmap v7442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1851818787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:39 smithi093 ceph-mon[118902]: pgmap v7442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1851818787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:40.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:40.306572+0000) 2023-08-28T01:01:41.467 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:41.183Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:41.467 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:41.184Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:41 smithi136 ceph-mon[113531]: pgmap v7443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4180003544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:41.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:41.306829+0000) 2023-08-28T01:01:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:41 smithi093 ceph-mon[111335]: pgmap v7443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4180003544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:41 smithi093 ceph-mon[118902]: pgmap v7443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:41.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4180003544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:42.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:42.307100+0000) 2023-08-28T01:01:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:42.631Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:42.632Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:43 smithi136 ceph-mon[113531]: pgmap v7444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:43.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:43.307337+0000) 2023-08-28T01:01:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:43 smithi093 ceph-mon[118902]: pgmap v7444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:43 smithi093 ceph-mon[111335]: pgmap v7444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/472948315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:44.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:44.307592+0000) 2023-08-28T01:01:44.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/472948315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/472948315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:45 smithi136 ceph-mon[113531]: pgmap v7445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:45.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:45.307874+0000) 2023-08-28T01:01:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:45 smithi093 ceph-mon[111335]: pgmap v7445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:01:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:01:45] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T01:01:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:45 smithi093 ceph-mon[118902]: pgmap v7445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:46.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/370858929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:46.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:46.308160+0000) 2023-08-28T01:01:46.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/370858929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/370858929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:47 smithi136 ceph-mon[113531]: pgmap v7446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:47.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:47.308414+0000) 2023-08-28T01:01:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:47 smithi093 ceph-mon[111335]: pgmap v7446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:47 smithi093 ceph-mon[118902]: pgmap v7446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2023825365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:48.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:48.308686+0000) 2023-08-28T01:01:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2023825365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2023825365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:49 smithi136 ceph-mon[113531]: pgmap v7447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:49.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:49.308930+0000) 2023-08-28T01:01:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:49 smithi093 ceph-mon[111335]: pgmap v7447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:49 smithi093 ceph-mon[118902]: pgmap v7447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:50.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:50.309125+0000) 2023-08-28T01:01:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:51.184Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:51.185Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:51 smithi136 ceph-mon[113531]: pgmap v7448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3996443077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:51.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:51.309307+0000) 2023-08-28T01:01:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:51 smithi093 ceph-mon[111335]: pgmap v7448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3996443077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:51 smithi093 ceph-mon[118902]: pgmap v7448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:51.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3996443077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:52.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:52.309522+0000) 2023-08-28T01:01:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:52.631Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:01:52.632Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:01:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:53 smithi136 ceph-mon[113531]: pgmap v7449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1457025296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:53.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:53.309751+0000) 2023-08-28T01:01:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:53 smithi093 ceph-mon[111335]: pgmap v7449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1457025296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:53 smithi093 ceph-mon[118902]: pgmap v7449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:53.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1457025296' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:54.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:54.309964+0000) 2023-08-28T01:01:54.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:54.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:01:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:55 smithi136 ceph-mon[113531]: pgmap v7450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4084245247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:55.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:55.310206+0000) 2023-08-28T01:01:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:55 smithi093 ceph-mon[111335]: pgmap v7450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4084245247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:01:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:01:55] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T01:01:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:55 smithi093 ceph-mon[118902]: pgmap v7450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4084245247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:56.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:56.310461+0000) 2023-08-28T01:01:57.310 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:57 smithi136 ceph-mon[113531]: pgmap v7451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:57 smithi093 ceph-mon[111335]: pgmap v7451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:57 smithi093 ceph-mon[118902]: pgmap v7451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:57.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:57.310760+0000) 2023-08-28T01:01:58.311 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2566266129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2566266129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2566266129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:01:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:58.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:58.311004+0000) 2023-08-28T01:01:59.311 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:01:59 smithi136 ceph-mon[113531]: pgmap v7452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:01:59 smithi093 ceph-mon[111335]: pgmap v7452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:01:59 smithi093 ceph-mon[118902]: pgmap v7452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:01:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:01:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:01:59.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:01:59.311290+0000) 2023-08-28T01:02:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:00.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:00.311561+0000) 2023-08-28T01:02:01.311 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:01 smithi136 ceph-mon[113531]: pgmap v7453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:01.312 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/136615620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:01 smithi093 ceph-mon[111335]: pgmap v7453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/136615620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:01 smithi093 ceph-mon[118902]: pgmap v7453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/136615620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:01.185Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:01.186Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:01.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:01.311794+0000) 2023-08-28T01:02:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:02.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:02.312097+0000) 2023-08-28T01:02:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:02.631Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:02.632Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:03.312 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:03 smithi136 ceph-mon[113531]: pgmap v7454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:03.312 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3614848187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:03 smithi093 ceph-mon[111335]: pgmap v7454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3614848187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:03 smithi093 ceph-mon[118902]: pgmap v7454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3614848187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:03.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:03.312347+0000) 2023-08-28T01:02:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:04.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:04.312573+0000) 2023-08-28T01:02:05.312 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:05 smithi136 ceph-mon[113531]: pgmap v7455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:05.312 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1815253214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:05 smithi093 ceph-mon[111335]: pgmap v7455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1815253214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:05 smithi093 ceph-mon[118902]: pgmap v7455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1815253214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:05.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:05.312816+0000) 2023-08-28T01:02:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:02:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:02:05] "GET /metrics HTTP/1.1" 200 239931 "" "Prometheus/2.43.0" 2023-08-28T01:02:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:06.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:06.313107+0000) 2023-08-28T01:02:07.313 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:07 smithi136 ceph-mon[113531]: pgmap v7456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:07 smithi093 ceph-mon[111335]: pgmap v7456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:07 smithi093 ceph-mon[118902]: pgmap v7456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:07.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:07.313343+0000) 2023-08-28T01:02:08.313 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4094897916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4094897916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4094897916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:08.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:08.313601+0000) 2023-08-28T01:02:09.314 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:09 smithi136 ceph-mon[113531]: pgmap v7457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:09.314 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:09 smithi093 ceph-mon[111335]: pgmap v7457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:09 smithi093 ceph-mon[118902]: pgmap v7457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:09.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:09.313866+0000) 2023-08-28T01:02:10.314 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2610877173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2610877173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2610877173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:10.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:10.314133+0000) 2023-08-28T01:02:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:11 smithi136 ceph-mon[113531]: pgmap v7458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:11.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:11.314438+0000) 2023-08-28T01:02:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:11 smithi093 ceph-mon[111335]: pgmap v7458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:11.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:11 smithi093 ceph-mon[118902]: pgmap v7458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:11.186Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:11.187Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4159343459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:12.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:12.314626+0000) 2023-08-28T01:02:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4159343459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4159343459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:12.632Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:12.634Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:13 smithi136 ceph-mon[113531]: pgmap v7459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:13.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:13.314891+0000) 2023-08-28T01:02:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:13 smithi093 ceph-mon[111335]: pgmap v7459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:13 smithi093 ceph-mon[118902]: pgmap v7459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:14.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:14.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:14.315094+0000) 2023-08-28T01:02:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:15 smithi093 ceph-mon[111335]: pgmap v7460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:15.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2022650605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:15 smithi093 ceph-mon[118902]: pgmap v7460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:15.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2022650605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:15 smithi136 ceph-mon[113531]: pgmap v7460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2022650605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:15.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:15.315323+0000) 2023-08-28T01:02:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:02:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:02:15] "GET /metrics HTTP/1.1" 200 239930 "" "Prometheus/2.43.0" 2023-08-28T01:02:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:16.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:16.315566+0000) 2023-08-28T01:02:17.315 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:17 smithi136 ceph-mon[113531]: pgmap v7461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:17.316 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4057861709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:17 smithi093 ceph-mon[111335]: pgmap v7461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4057861709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:17 smithi093 ceph-mon[118902]: pgmap v7461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4057861709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:17.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:17.315818+0000) 2023-08-28T01:02:18.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:18.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:18.316085+0000) 2023-08-28T01:02:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:19 smithi136 ceph-mon[113531]: pgmap v7462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:19.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:19.316294+0000) 2023-08-28T01:02:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:19 smithi093 ceph-mon[111335]: pgmap v7462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:19 smithi093 ceph-mon[118902]: pgmap v7462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:20.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/262612229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:20.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:20.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:20.316576+0000) 2023-08-28T01:02:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/262612229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/262612229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:21 smithi136 ceph-mon[113531]: pgmap v7463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:21.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:21.316831+0000) 2023-08-28T01:02:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:21 smithi093 ceph-mon[111335]: pgmap v7463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:21 smithi093 ceph-mon[118902]: pgmap v7463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:21.186Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:21.187Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2137439017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:22.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:22.317124+0000) 2023-08-28T01:02:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2137439017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2137439017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:22.633Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:22.634Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:23 smithi136 ceph-mon[113531]: pgmap v7464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:23.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:23.317426+0000) 2023-08-28T01:02:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:23 smithi093 ceph-mon[111335]: pgmap v7464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:23 smithi093 ceph-mon[118902]: pgmap v7464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2289242754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:24.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:24.317640+0000) 2023-08-28T01:02:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2289242754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2289242754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:25 smithi136 ceph-mon[113531]: pgmap v7465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:25.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:25.317887+0000) 2023-08-28T01:02:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:25 smithi093 ceph-mon[111335]: pgmap v7465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:25 smithi093 ceph-mon[118902]: pgmap v7465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:02:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:02:25] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T01:02:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:26.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:26.318166+0000) 2023-08-28T01:02:27.318 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:27 smithi136 ceph-mon[113531]: pgmap v7466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:27.318 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/571174751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:27 smithi093 ceph-mon[111335]: pgmap v7466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/571174751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:27 smithi093 ceph-mon[118902]: pgmap v7466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/571174751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:27.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:27.318430+0000) 2023-08-28T01:02:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:28.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:28.318674+0000) 2023-08-28T01:02:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:29.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:29.318867+0000) 2023-08-28T01:02:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:29 smithi136 ceph-mon[113531]: pgmap v7467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:29.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4118677444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:29 smithi093 ceph-mon[111335]: pgmap v7467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4118677444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:29 smithi093 ceph-mon[118902]: pgmap v7467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4118677444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:30.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:30.319178+0000) 2023-08-28T01:02:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:31 smithi136 ceph-mon[113531]: pgmap v7468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:31.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:31.319430+0000) 2023-08-28T01:02:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:31 smithi093 ceph-mon[111335]: pgmap v7468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:31 smithi093 ceph-mon[118902]: pgmap v7468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:31.186Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:31.187Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:32.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3246984391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:32.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:32.319667+0000) 2023-08-28T01:02:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3246984391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3246984391' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:32.634Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:32.635Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:33 smithi136 ceph-mon[113531]: pgmap v7469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:33.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:33.319910+0000) 2023-08-28T01:02:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:33 smithi093 ceph-mon[111335]: pgmap v7469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:33 smithi093 ceph-mon[118902]: pgmap v7469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:34.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/657057435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:34.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:34.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:34.320128+0000) 2023-08-28T01:02:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/657057435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/657057435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:35 smithi136 ceph-mon[113531]: pgmap v7470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:35.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:35.320375+0000) 2023-08-28T01:02:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:35 smithi093 ceph-mon[111335]: pgmap v7470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:35 smithi093 ceph-mon[118902]: pgmap v7470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:02:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:02:35] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T01:02:36.417 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:02:36.417 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4107528947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:36.417 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:02:36.417 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4107528947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:02:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4107528947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:36.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:36.320570+0000) 2023-08-28T01:02:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:37 smithi093 ceph-mon[111335]: pgmap v7471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:37 smithi093 ceph-mon[118902]: pgmap v7471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:37 smithi136 ceph-mon[113531]: pgmap v7471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:37.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:37.320815+0000) 2023-08-28T01:02:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:38 smithi136 ceph-mon[113531]: pgmap v7472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1540510050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:38.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:38.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:38.320998+0000) 2023-08-28T01:02:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[118902]: pgmap v7472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1540510050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[111335]: pgmap v7472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:38.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1540510050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:02:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:02:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:39.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:39.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:39.321225+0000) 2023-08-28T01:02:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:02:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:02:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:39.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:02:39.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:40.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:40 smithi136 ceph-mon[113531]: pgmap v7473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:40.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:40.321469+0000) 2023-08-28T01:02:40.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:40 smithi093 ceph-mon[111335]: pgmap v7473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:40.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:40 smithi093 ceph-mon[118902]: pgmap v7473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:41.187Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:41.189Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1333836633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:41.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:41.321716+0000) 2023-08-28T01:02:41.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1333836633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:41.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1333836633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:42.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:42 smithi136 ceph-mon[113531]: pgmap v7474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:42.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:42.321930+0000) 2023-08-28T01:02:42.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:42 smithi093 ceph-mon[111335]: pgmap v7474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:42.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:42 smithi093 ceph-mon[118902]: pgmap v7474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:42.635Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:42.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:42.636Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3843774855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:43.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:43.322148+0000) 2023-08-28T01:02:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3843774855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:43.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3843774855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:44 smithi136 ceph-mon[113531]: pgmap v7475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:44.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:44.322331+0000) 2023-08-28T01:02:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:44 smithi093 ceph-mon[111335]: pgmap v7475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:44 smithi093 ceph-mon[118902]: pgmap v7475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/635608532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:45.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:45.322573+0000) 2023-08-28T01:02:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/635608532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:02:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:02:45] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T01:02:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/635608532' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:46.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:46.322840+0000) 2023-08-28T01:02:47.323 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:47 smithi136 ceph-mon[113531]: pgmap v7476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:47 smithi093 ceph-mon[111335]: pgmap v7476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:47.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:47 smithi093 ceph-mon[118902]: pgmap v7476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:47.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:47.323111+0000) 2023-08-28T01:02:48.323 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/619244431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/619244431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/619244431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:48.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:48.323374+0000) 2023-08-28T01:02:49.323 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:49 smithi136 ceph-mon[113531]: pgmap v7477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:49 smithi093 ceph-mon[118902]: pgmap v7477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:49 smithi093 ceph-mon[111335]: pgmap v7477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:49.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:49.323633+0000) 2023-08-28T01:02:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:50.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:50.323917+0000) 2023-08-28T01:02:51.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:51 smithi136 ceph-mon[113531]: pgmap v7478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:51.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/790245222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:51 smithi093 ceph-mon[111335]: pgmap v7478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/790245222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:51 smithi093 ceph-mon[118902]: pgmap v7478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/790245222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:51.189Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:51.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:51.190Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:51.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:51.324158+0000) 2023-08-28T01:02:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:52.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:52.324405+0000) 2023-08-28T01:02:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:52.635Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:02:52.637Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:02:53.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:53 smithi136 ceph-mon[113531]: pgmap v7479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:53.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/377492945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:53 smithi093 ceph-mon[111335]: pgmap v7479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/377492945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:53 smithi093 ceph-mon[118902]: pgmap v7479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/377492945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:53.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:53.324655+0000) 2023-08-28T01:02:54.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:02:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:54.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:54.324916+0000) 2023-08-28T01:02:55.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:55 smithi136 ceph-mon[113531]: pgmap v7480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:55.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3152135522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:55 smithi093 ceph-mon[111335]: pgmap v7480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:55.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3152135522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:55 smithi093 ceph-mon[118902]: pgmap v7480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3152135522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:55.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:55.325146+0000) 2023-08-28T01:02:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:02:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:02:55] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T01:02:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:56.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:56.325430+0000) 2023-08-28T01:02:57.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:57 smithi136 ceph-mon[113531]: pgmap v7481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:57 smithi093 ceph-mon[111335]: pgmap v7481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:57 smithi093 ceph-mon[118902]: pgmap v7481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:57.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:57.325687+0000) 2023-08-28T01:02:58.325 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3345657493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3345657493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:58.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3345657493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:02:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:58.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:58.325937+0000) 2023-08-28T01:02:59.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:02:59 smithi136 ceph-mon[113531]: pgmap v7482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:02:59 smithi093 ceph-mon[111335]: pgmap v7482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:02:59 smithi093 ceph-mon[118902]: pgmap v7482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:02:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:02:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:02:59.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:02:59.326203+0000) 2023-08-28T01:03:00.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/141881323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/141881323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/141881323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:00.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:00.326504+0000) 2023-08-28T01:03:01.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:01 smithi136 ceph-mon[113531]: pgmap v7483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:01 smithi093 ceph-mon[118902]: pgmap v7483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:01 smithi093 ceph-mon[111335]: pgmap v7483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:01.190Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:01.191Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:01.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:01.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:01.326727+0000) 2023-08-28T01:03:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:02.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:02.326970+0000) 2023-08-28T01:03:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:02.636Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:02.637Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:03 smithi136 ceph-mon[113531]: pgmap v7484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/577357921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:03.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:03.327142+0000) 2023-08-28T01:03:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:03 smithi093 ceph-mon[111335]: pgmap v7484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/577357921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:03 smithi093 ceph-mon[118902]: pgmap v7484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:03.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/577357921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:04.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:04.327375+0000) 2023-08-28T01:03:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:05 smithi093 ceph-mon[118902]: pgmap v7485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4207915037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:05 smithi093 ceph-mon[111335]: pgmap v7485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4207915037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:05 smithi136 ceph-mon[113531]: pgmap v7485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4207915037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:05.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:05.327634+0000) 2023-08-28T01:03:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:03:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:03:05] "GET /metrics HTTP/1.1" 200 239927 "" "Prometheus/2.43.0" 2023-08-28T01:03:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:06.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:06.327949+0000) 2023-08-28T01:03:07.328 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:07 smithi136 ceph-mon[113531]: pgmap v7486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:07.328 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3040101067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:07 smithi093 ceph-mon[111335]: pgmap v7486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3040101067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:07 smithi093 ceph-mon[118902]: pgmap v7486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3040101067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:07.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:07.328193+0000) 2023-08-28T01:03:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:08.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:08.328410+0000) 2023-08-28T01:03:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:09 smithi136 ceph-mon[113531]: pgmap v7487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:09.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:09.328646+0000) 2023-08-28T01:03:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:09 smithi093 ceph-mon[111335]: pgmap v7487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:09 smithi093 ceph-mon[118902]: pgmap v7487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/584910224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:10.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:10.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:10.328890+0000) 2023-08-28T01:03:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/584910224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/584910224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:11 smithi136 ceph-mon[113531]: pgmap v7488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:11.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:11.329159+0000) 2023-08-28T01:03:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:11 smithi093 ceph-mon[111335]: pgmap v7488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:11 smithi093 ceph-mon[118902]: pgmap v7488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:11.190Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:11.192Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2093257290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:12.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:12.329407+0000) 2023-08-28T01:03:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2093257290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2093257290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:12.636Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:12.637Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:13 smithi136 ceph-mon[113531]: pgmap v7489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:13.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:13.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:13.329653+0000) 2023-08-28T01:03:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:13 smithi093 ceph-mon[111335]: pgmap v7489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:13 smithi093 ceph-mon[118902]: pgmap v7489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3785027584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:14.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:14.329902+0000) 2023-08-28T01:03:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3785027584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3785027584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:15 smithi136 ceph-mon[113531]: pgmap v7490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:15.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:15.330127+0000) 2023-08-28T01:03:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:15 smithi093 ceph-mon[111335]: pgmap v7490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:15 smithi093 ceph-mon[118902]: pgmap v7490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:03:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:03:15] "GET /metrics HTTP/1.1" 200 239933 "" "Prometheus/2.43.0" 2023-08-28T01:03:16.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:16.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:16.330330+0000) 2023-08-28T01:03:17.330 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:17 smithi136 ceph-mon[113531]: pgmap v7491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:17.330 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3790230603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:17 smithi093 ceph-mon[111335]: pgmap v7491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3790230603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:17 smithi093 ceph-mon[118902]: pgmap v7491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3790230603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:17.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:17.330560+0000) 2023-08-28T01:03:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:18.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:18.330813+0000) 2023-08-28T01:03:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:19.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:19.331012+0000) 2023-08-28T01:03:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:19 smithi136 ceph-mon[113531]: pgmap v7492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/497475997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:19 smithi093 ceph-mon[111335]: pgmap v7492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/497475997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:19 smithi093 ceph-mon[118902]: pgmap v7492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/497475997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:20.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:20.331202+0000) 2023-08-28T01:03:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:21 smithi136 ceph-mon[113531]: pgmap v7493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:21.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:21.331439+0000) 2023-08-28T01:03:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:21 smithi093 ceph-mon[111335]: pgmap v7493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:21 smithi093 ceph-mon[118902]: pgmap v7493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:21.192Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:21.192Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:22.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1843291619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:22.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:22.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:22.331634+0000) 2023-08-28T01:03:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1843291619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1843291619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:22.637Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:22.638Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:23 smithi136 ceph-mon[113531]: pgmap v7494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:23.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:23.331886+0000) 2023-08-28T01:03:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:23 smithi093 ceph-mon[111335]: pgmap v7494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:23 smithi093 ceph-mon[118902]: pgmap v7494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:24.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:24.332161+0000) 2023-08-28T01:03:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1615521249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1615521249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1615521249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:24.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:25 smithi136 ceph-mon[113531]: pgmap v7495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:25.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:25.332404+0000) 2023-08-28T01:03:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:25 smithi093 ceph-mon[111335]: pgmap v7495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:25 smithi093 ceph-mon[118902]: pgmap v7495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:25.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:03:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:03:25] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:03:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/617313794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:26.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:26.332675+0000) 2023-08-28T01:03:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/617313794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/617313794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:27 smithi136 ceph-mon[113531]: pgmap v7496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:27.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:27.332880+0000) 2023-08-28T01:03:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:27 smithi093 ceph-mon[111335]: pgmap v7496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:27 smithi093 ceph-mon[118902]: pgmap v7496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:28.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:28.333112+0000) 2023-08-28T01:03:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:29 smithi136 ceph-mon[113531]: pgmap v7497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2129782752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:29.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:29.333342+0000) 2023-08-28T01:03:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:29 smithi093 ceph-mon[111335]: pgmap v7497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2129782752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:29 smithi093 ceph-mon[118902]: pgmap v7497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2129782752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:30.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:30.333571+0000) 2023-08-28T01:03:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:31 smithi136 ceph-mon[113531]: pgmap v7498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/666508326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:31.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:31.333837+0000) 2023-08-28T01:03:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:31 smithi093 ceph-mon[111335]: pgmap v7498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/666508326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:31 smithi093 ceph-mon[118902]: pgmap v7498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/666508326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:31.193Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:31.194Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:32.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:32.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:32.334101+0000) 2023-08-28T01:03:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:32.638Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:32.640Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:33 smithi136 ceph-mon[113531]: pgmap v7499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1754681341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:33.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:33.334327+0000) 2023-08-28T01:03:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:33 smithi093 ceph-mon[111335]: pgmap v7499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1754681341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:33 smithi093 ceph-mon[118902]: pgmap v7499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:33.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1754681341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:34.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:34.334572+0000) 2023-08-28T01:03:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:35 smithi136 ceph-mon[113531]: pgmap v7500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:35.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:35.334851+0000) 2023-08-28T01:03:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:35 smithi093 ceph-mon[111335]: pgmap v7500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:35 smithi093 ceph-mon[118902]: pgmap v7500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:03:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:03:35] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:03:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3871139489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:36.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:36.335068+0000) 2023-08-28T01:03:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3871139489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3871139489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:37 smithi136 ceph-mon[113531]: pgmap v7501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:37.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:37.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:37.335300+0000) 2023-08-28T01:03:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:37 smithi093 ceph-mon[111335]: pgmap v7501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:37 smithi093 ceph-mon[118902]: pgmap v7501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:38.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1657367286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:38.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:38.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:38.335578+0000) 2023-08-28T01:03:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1657367286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1657367286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:39 smithi136 ceph-mon[113531]: pgmap v7502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:03:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:39.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:39.335752+0000) 2023-08-28T01:03:39.465 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:39 smithi093 ceph-mon[111335]: pgmap v7502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:39.465 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:03:39.465 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:39.465 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:39 smithi093 ceph-mon[118902]: pgmap v7502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:39.465 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:03:39.465 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:40.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:40.335912+0000) 2023-08-28T01:03:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:41 smithi136 ceph-mon[113531]: pgmap v7503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/772694090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:41.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:41.336054+0000) 2023-08-28T01:03:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[118902]: pgmap v7503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/772694090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[111335]: pgmap v7503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/772694090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:41.193Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:41.194Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:03:42.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:42.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:42.336238+0000) 2023-08-28T01:03:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:03:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:03:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:03:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:03:42.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:03:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:42.639Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:42.640Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:43 smithi136 ceph-mon[113531]: pgmap v7504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2749854460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:43.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:43.336508+0000) 2023-08-28T01:03:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:43 smithi093 ceph-mon[111335]: pgmap v7504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2749854460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:43 smithi093 ceph-mon[118902]: pgmap v7504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2749854460' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:44.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:44.336737+0000) 2023-08-28T01:03:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:45 smithi136 ceph-mon[113531]: pgmap v7505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2414016305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:45.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:45.336993+0000) 2023-08-28T01:03:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:45 smithi093 ceph-mon[111335]: pgmap v7505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2414016305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:45 smithi093 ceph-mon[118902]: pgmap v7505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:45.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2414016305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:03:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:03:45] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:03:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:46.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:46.337241+0000) 2023-08-28T01:03:47.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:47 smithi136 ceph-mon[113531]: pgmap v7506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:47 smithi093 ceph-mon[111335]: pgmap v7506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:47 smithi093 ceph-mon[118902]: pgmap v7506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:47.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:47.337490+0000) 2023-08-28T01:03:48.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3079358799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3079358799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3079358799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:48.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:48.337747+0000) 2023-08-28T01:03:49.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:49 smithi136 ceph-mon[113531]: pgmap v7507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:49 smithi093 ceph-mon[111335]: pgmap v7507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:49 smithi093 ceph-mon[118902]: pgmap v7507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:49.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:49.338007+0000) 2023-08-28T01:03:50.338 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/871436796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/871436796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/871436796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:50.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:50.338277+0000) 2023-08-28T01:03:51.338 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:51 smithi136 ceph-mon[113531]: pgmap v7508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:51 smithi093 ceph-mon[111335]: pgmap v7508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:51 smithi093 ceph-mon[118902]: pgmap v7508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:51.194Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:51.195Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:51.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:51.338528+0000) 2023-08-28T01:03:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:52.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:52.338789+0000) 2023-08-28T01:03:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:52.640Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:03:52.641Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:03:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:53.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:53.339085+0000) 2023-08-28T01:03:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:53 smithi136 ceph-mon[113531]: pgmap v7509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4222932550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:53 smithi093 ceph-mon[111335]: pgmap v7509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4222932550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:53 smithi093 ceph-mon[118902]: pgmap v7509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4222932550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:54.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:54.339331+0000) 2023-08-28T01:03:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:03:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:55.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:55.339597+0000) 2023-08-28T01:03:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:55 smithi136 ceph-mon[113531]: pgmap v7510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:55.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3129974642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:55.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:03:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:03:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:03:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:55 smithi093 ceph-mon[118902]: pgmap v7510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3129974642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:55 smithi093 ceph-mon[111335]: pgmap v7510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3129974642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:56.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:56.339875+0000) 2023-08-28T01:03:57.340 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:57 smithi136 ceph-mon[113531]: pgmap v7511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:57.340 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3256779382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:57 smithi093 ceph-mon[111335]: pgmap v7511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3256779382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:57 smithi093 ceph-mon[118902]: pgmap v7511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3256779382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:03:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:57.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:57.340143+0000) 2023-08-28T01:03:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:58.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:58.340384+0000) 2023-08-28T01:03:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:03:59 smithi136 ceph-mon[113531]: pgmap v7512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:59.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:03:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:03:59.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:03:59.340589+0000) 2023-08-28T01:03:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:03:59 smithi093 ceph-mon[111335]: pgmap v7512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:03:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:03:59 smithi093 ceph-mon[118902]: pgmap v7512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:00.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3286320487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:00.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:00.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:00.340766+0000) 2023-08-28T01:04:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3286320487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3286320487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:01 smithi136 ceph-mon[113531]: pgmap v7513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:01.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:01.341010+0000) 2023-08-28T01:04:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:01 smithi093 ceph-mon[111335]: pgmap v7513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:01 smithi093 ceph-mon[118902]: pgmap v7513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:01.194Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:01.195Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:02.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3125649145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:02.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:02.341236+0000) 2023-08-28T01:04:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3125649145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3125649145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:02.640Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:02.641Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:03 smithi136 ceph-mon[113531]: pgmap v7514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:03.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:03.341449+0000) 2023-08-28T01:04:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:03 smithi093 ceph-mon[111335]: pgmap v7514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:03 smithi093 ceph-mon[118902]: pgmap v7514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3663515834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:04.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:04.341706+0000) 2023-08-28T01:04:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3663515834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3663515834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:05 smithi136 ceph-mon[113531]: pgmap v7515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:05.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:05.341955+0000) 2023-08-28T01:04:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:05 smithi093 ceph-mon[111335]: pgmap v7515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:04:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:04:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:04:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:05 smithi093 ceph-mon[118902]: pgmap v7515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:06.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:06.342216+0000) 2023-08-28T01:04:07.342 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:07 smithi136 ceph-mon[113531]: pgmap v7516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:07.342 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/574956163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:07 smithi093 ceph-mon[111335]: pgmap v7516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/574956163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:07 smithi093 ceph-mon[118902]: pgmap v7516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/574956163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:07.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:07.342377+0000) 2023-08-28T01:04:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:08.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:08.342596+0000) 2023-08-28T01:04:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:09 smithi136 ceph-mon[113531]: pgmap v7517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/586152999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:09.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:09.342846+0000) 2023-08-28T01:04:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:09 smithi093 ceph-mon[111335]: pgmap v7517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/586152999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:09 smithi093 ceph-mon[118902]: pgmap v7517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/586152999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:10.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:10.343115+0000) 2023-08-28T01:04:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:11 smithi136 ceph-mon[113531]: pgmap v7518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:11.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:11.343375+0000) 2023-08-28T01:04:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:11 smithi093 ceph-mon[111335]: pgmap v7518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:11 smithi093 ceph-mon[118902]: pgmap v7518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:11.195Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:11.196Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:12.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/771198521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:12.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:12.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:12.343645+0000) 2023-08-28T01:04:12.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/771198521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:12.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/771198521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:12.641Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:12.642Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:13 smithi136 ceph-mon[113531]: pgmap v7519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:13.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:13.343843+0000) 2023-08-28T01:04:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:13 smithi093 ceph-mon[111335]: pgmap v7519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:13 smithi093 ceph-mon[118902]: pgmap v7519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4241304359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:14.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:14.344121+0000) 2023-08-28T01:04:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4241304359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4241304359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:15 smithi136 ceph-mon[113531]: pgmap v7520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:15.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:15.344309+0000) 2023-08-28T01:04:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:15 smithi093 ceph-mon[111335]: pgmap v7520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:15 smithi093 ceph-mon[118902]: pgmap v7520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:15.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:04:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:04:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:04:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1790938862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:16.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:16.344568+0000) 2023-08-28T01:04:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1790938862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1790938862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:17 smithi136 ceph-mon[113531]: pgmap v7521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:17.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:17.344775+0000) 2023-08-28T01:04:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:17 smithi093 ceph-mon[111335]: pgmap v7521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:17 smithi093 ceph-mon[118902]: pgmap v7521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:18.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:18.344999+0000) 2023-08-28T01:04:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:19 smithi136 ceph-mon[113531]: pgmap v7522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1986513270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:19.441 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:19.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:19.345224+0000) 2023-08-28T01:04:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:19 smithi093 ceph-mon[111335]: pgmap v7522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1986513270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:19 smithi093 ceph-mon[118902]: pgmap v7522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1986513270' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:20.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:20.345511+0000) 2023-08-28T01:04:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:21 smithi136 ceph-mon[113531]: pgmap v7523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/292398489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:21.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:21.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:21.345713+0000) 2023-08-28T01:04:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:21 smithi093 ceph-mon[111335]: pgmap v7523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:21.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/292398489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:21 smithi093 ceph-mon[118902]: pgmap v7523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:21.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/292398489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:21.195Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:21.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:21.196Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:22.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:22.345906+0000) 2023-08-28T01:04:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:22.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:22.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:23 smithi136 ceph-mon[113531]: pgmap v7524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4229933917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:23.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:23.346135+0000) 2023-08-28T01:04:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:23 smithi093 ceph-mon[111335]: pgmap v7524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4229933917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:23 smithi093 ceph-mon[118902]: pgmap v7524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:23.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4229933917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:24.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:24.346347+0000) 2023-08-28T01:04:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:25 smithi136 ceph-mon[113531]: pgmap v7525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:25.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:25.346571+0000) 2023-08-28T01:04:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:25 smithi093 ceph-mon[111335]: pgmap v7525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:25 smithi093 ceph-mon[118902]: pgmap v7525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:04:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:04:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:04:26.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/506151488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:26.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:26.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:26.346830+0000) 2023-08-28T01:04:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/506151488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/506151488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:27 smithi136 ceph-mon[113531]: pgmap v7526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:27.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:27.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:27.347110+0000) 2023-08-28T01:04:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:27 smithi093 ceph-mon[111335]: pgmap v7526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:27 smithi093 ceph-mon[118902]: pgmap v7526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:28.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3883933486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:28.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:28.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:28.347413+0000) 2023-08-28T01:04:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3883933486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3883933486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:29 smithi136 ceph-mon[113531]: pgmap v7527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:29.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:29.347608+0000) 2023-08-28T01:04:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:29 smithi093 ceph-mon[111335]: pgmap v7527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:29 smithi093 ceph-mon[118902]: pgmap v7527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:30.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:30.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:30.347861+0000) 2023-08-28T01:04:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:31 smithi136 ceph-mon[113531]: pgmap v7528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2285114515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:31.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:31.348072+0000) 2023-08-28T01:04:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:31 smithi093 ceph-mon[111335]: pgmap v7528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2285114515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:31 smithi093 ceph-mon[118902]: pgmap v7528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2285114515' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:31.196Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:31.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:31.197Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:32.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:32.348262+0000) 2023-08-28T01:04:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:32.642Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:32.643Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:33 smithi136 ceph-mon[113531]: pgmap v7529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2951170237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:33.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:33.348529+0000) 2023-08-28T01:04:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:33 smithi093 ceph-mon[111335]: pgmap v7529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2951170237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:33 smithi093 ceph-mon[118902]: pgmap v7529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2951170237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:34.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:34.348761+0000) 2023-08-28T01:04:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:35 smithi136 ceph-mon[113531]: pgmap v7530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2127001376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:35.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:35.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:35.349008+0000) 2023-08-28T01:04:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:35 smithi093 ceph-mon[111335]: pgmap v7530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2127001376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:35 smithi093 ceph-mon[118902]: pgmap v7530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2127001376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:35.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:04:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:04:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:04:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:36.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:36.349264+0000) 2023-08-28T01:04:37.349 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:37 smithi136 ceph-mon[113531]: pgmap v7531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:37 smithi093 ceph-mon[111335]: pgmap v7531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:37 smithi093 ceph-mon[118902]: pgmap v7531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:37.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:37.349509+0000) 2023-08-28T01:04:38.349 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3968352989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3968352989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3968352989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:38.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:38.349748+0000) 2023-08-28T01:04:39.349 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:39 smithi136 ceph-mon[113531]: pgmap v7532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:39.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:39 smithi093 ceph-mon[111335]: pgmap v7532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:39 smithi093 ceph-mon[118902]: pgmap v7532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:39.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:39.349971+0000) 2023-08-28T01:04:40.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1778787940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:40.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1778787940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:40.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1778787940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:40.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:40.350247+0000) 2023-08-28T01:04:41.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:41 smithi136 ceph-mon[113531]: pgmap v7533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:41 smithi093 ceph-mon[111335]: pgmap v7533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:41 smithi093 ceph-mon[118902]: pgmap v7533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:41.197Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:41.198Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:41.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:41.350433+0000) 2023-08-28T01:04:42.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:04:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:04:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:04:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:42.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:42.350612+0000) 2023-08-28T01:04:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:42.643Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:42.645Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:43.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:43 smithi136 ceph-mon[113531]: pgmap v7534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:43.350 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/999083641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:43 smithi093 ceph-mon[111335]: pgmap v7534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/999083641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:43 smithi093 ceph-mon[118902]: pgmap v7534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/999083641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:43.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:43.350820+0000) 2023-08-28T01:04:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:44.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:44.350982+0000) 2023-08-28T01:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:04:44.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:04:44.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:04:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:04:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:45.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:45.351216+0000) 2023-08-28T01:04:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:45 smithi136 ceph-mon[113531]: pgmap v7535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/625578352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:45 smithi093 ceph-mon[111335]: pgmap v7535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/625578352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:45 smithi093 ceph-mon[118902]: pgmap v7535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/625578352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:45.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:04:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:04:45] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:04:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:46.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:46.351439+0000) 2023-08-28T01:04:47.351 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:47 smithi136 ceph-mon[113531]: pgmap v7536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:47.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1347836234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:47 smithi093 ceph-mon[118902]: pgmap v7536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1347836234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:47 smithi093 ceph-mon[111335]: pgmap v7536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1347836234' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:47.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:47.351665+0000) 2023-08-28T01:04:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:48.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:48.351959+0000) 2023-08-28T01:04:49.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:49 smithi136 ceph-mon[113531]: pgmap v7537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:49.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:49.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:49.352293+0000) 2023-08-28T01:04:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:49 smithi093 ceph-mon[118902]: pgmap v7537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:49 smithi093 ceph-mon[111335]: pgmap v7537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:50.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/907117542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:50.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:50.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:50.352509+0000) 2023-08-28T01:04:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/907117542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/907117542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:51 smithi136 ceph-mon[113531]: pgmap v7538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:51.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:51.352746+0000) 2023-08-28T01:04:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:51 smithi093 ceph-mon[111335]: pgmap v7538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:51 smithi093 ceph-mon[118902]: pgmap v7538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:51.197Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:51.198Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:52.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/22210403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:52.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:52.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:52.352948+0000) 2023-08-28T01:04:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/22210403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/22210403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:52.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:04:52.646Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:04:53.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:53 smithi136 ceph-mon[113531]: pgmap v7539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:53.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:53.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:53.353199+0000) 2023-08-28T01:04:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:53 smithi093 ceph-mon[111335]: pgmap v7539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:53 smithi093 ceph-mon[118902]: pgmap v7539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2119876438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:54.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:54.353437+0000) 2023-08-28T01:04:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2119876438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:04:54.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2119876438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:55.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:55 smithi136 ceph-mon[113531]: pgmap v7540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:55.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:55.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:55.353691+0000) 2023-08-28T01:04:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:55 smithi093 ceph-mon[111335]: pgmap v7540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:55.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:04:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:04:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:04:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:55 smithi093 ceph-mon[118902]: pgmap v7540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:56.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:56.353972+0000) 2023-08-28T01:04:57.354 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:57 smithi136 ceph-mon[113531]: pgmap v7541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:57.354 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1450947907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:57 smithi093 ceph-mon[111335]: pgmap v7541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1450947907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:57 smithi093 ceph-mon[118902]: pgmap v7541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:57.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1450947907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:57.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:57.354296+0000) 2023-08-28T01:04:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:58.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:58.354598+0000) 2023-08-28T01:04:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:59 smithi136 ceph-mon[113531]: pgmap v7542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:04:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3334659606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:04:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:04:59.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:04:59.354867+0000) 2023-08-28T01:04:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:59 smithi093 ceph-mon[111335]: pgmap v7542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:04:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3334659606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:04:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:59 smithi093 ceph-mon[118902]: pgmap v7542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:04:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:04:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3334659606' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:00.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:00.355090+0000) 2023-08-28T01:05:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:01 smithi136 ceph-mon[113531]: pgmap v7543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:01.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:01.355347+0000) 2023-08-28T01:05:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:01 smithi093 ceph-mon[111335]: pgmap v7543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:01 smithi093 ceph-mon[118902]: pgmap v7543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:01.198Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:01.199Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2958199187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:02.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:02.355556+0000) 2023-08-28T01:05:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2958199187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2958199187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:02.645Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:02.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:02.646Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:03 smithi136 ceph-mon[113531]: pgmap v7544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:03.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:03.355878+0000) 2023-08-28T01:05:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:03 smithi093 ceph-mon[111335]: pgmap v7544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:03 smithi093 ceph-mon[118902]: pgmap v7544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3687734579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:04.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:04.356114+0000) 2023-08-28T01:05:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3687734579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3687734579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:05 smithi136 ceph-mon[113531]: pgmap v7545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:05.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:05.356335+0000) 2023-08-28T01:05:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:05 smithi093 ceph-mon[111335]: pgmap v7545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:05 smithi093 ceph-mon[118902]: pgmap v7545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:05:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:05:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:05:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2304226903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:06.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:06.356588+0000) 2023-08-28T01:05:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2304226903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2304226903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:07.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:07 smithi136 ceph-mon[113531]: pgmap v7546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:07.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:07.356837+0000) 2023-08-28T01:05:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:07 smithi093 ceph-mon[111335]: pgmap v7546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:07 smithi093 ceph-mon[118902]: pgmap v7546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:08.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:08.357130+0000) 2023-08-28T01:05:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:09 smithi136 ceph-mon[113531]: pgmap v7547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2833249347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:09.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:09.357336+0000) 2023-08-28T01:05:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:09 smithi093 ceph-mon[111335]: pgmap v7547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2833249347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:09 smithi093 ceph-mon[118902]: pgmap v7547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2833249347' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:09.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:10.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:10.357564+0000) 2023-08-28T01:05:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:11 smithi136 ceph-mon[113531]: pgmap v7548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1752197564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:11.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:11.357791+0000) 2023-08-28T01:05:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:11 smithi093 ceph-mon[111335]: pgmap v7548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1752197564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:11 smithi093 ceph-mon[118902]: pgmap v7548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1752197564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:11.199Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:11.200Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:12.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:12.358010+0000) 2023-08-28T01:05:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:12.646Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:12.647Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:13 smithi136 ceph-mon[113531]: pgmap v7549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:13.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:13.358291+0000) 2023-08-28T01:05:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:13 smithi093 ceph-mon[111335]: pgmap v7549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:13 smithi093 ceph-mon[118902]: pgmap v7549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:14.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1992062796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:14.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:14.358548+0000) 2023-08-28T01:05:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1992062796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1992062796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:15 smithi136 ceph-mon[113531]: pgmap v7550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:15.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:15.358817+0000) 2023-08-28T01:05:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:15 smithi093 ceph-mon[111335]: pgmap v7550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:15 smithi093 ceph-mon[118902]: pgmap v7550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:05:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:05:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:05:16.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1299777237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:16.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:16.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:16.359073+0000) 2023-08-28T01:05:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1299777237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1299777237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:17.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:17 smithi136 ceph-mon[113531]: pgmap v7551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:17.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:17.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:17.359302+0000) 2023-08-28T01:05:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:17 smithi093 ceph-mon[111335]: pgmap v7551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:17 smithi093 ceph-mon[118902]: pgmap v7551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:18.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3715228121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:18.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:18.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:18.359503+0000) 2023-08-28T01:05:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3715228121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3715228121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:19 smithi136 ceph-mon[113531]: pgmap v7552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:19.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:19.359734+0000) 2023-08-28T01:05:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:19 smithi093 ceph-mon[111335]: pgmap v7552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:19 smithi093 ceph-mon[118902]: pgmap v7552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:20.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:20.360001+0000) 2023-08-28T01:05:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:21 smithi136 ceph-mon[113531]: pgmap v7553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:21.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2061732068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:21.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:21.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:21.360215+0000) 2023-08-28T01:05:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:21 smithi093 ceph-mon[111335]: pgmap v7553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2061732068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:21 smithi093 ceph-mon[118902]: pgmap v7553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2061732068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:21.200Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:21.201Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:22.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:22.360402+0000) 2023-08-28T01:05:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:22.647Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:22.648Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:23 smithi136 ceph-mon[113531]: pgmap v7554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:23.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3066138620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:23.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:23.360639+0000) 2023-08-28T01:05:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:23 smithi093 ceph-mon[111335]: pgmap v7554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3066138620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:23 smithi093 ceph-mon[118902]: pgmap v7554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3066138620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:24.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:24.360897+0000) 2023-08-28T01:05:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:25 smithi093 ceph-mon[111335]: pgmap v7555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2717839735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:05:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:05:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:05:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:25 smithi093 ceph-mon[118902]: pgmap v7555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2717839735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:25 smithi136 ceph-mon[113531]: pgmap v7555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2717839735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:25.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:25.361111+0000) 2023-08-28T01:05:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:26.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:26.361322+0000) 2023-08-28T01:05:27.361 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:27 smithi136 ceph-mon[113531]: pgmap v7556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:27 smithi093 ceph-mon[118902]: pgmap v7556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:27 smithi093 ceph-mon[111335]: pgmap v7556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:27.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:27.361570+0000) 2023-08-28T01:05:28.361 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3449174284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3449174284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3449174284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:28.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:28.361770+0000) 2023-08-28T01:05:29.361 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:29 smithi136 ceph-mon[113531]: pgmap v7557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:29 smithi093 ceph-mon[111335]: pgmap v7557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:29 smithi093 ceph-mon[118902]: pgmap v7557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:29.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:29.362061+0000) 2023-08-28T01:05:30.361 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3261064049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3261064049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3261064049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:30.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:30.362262+0000) 2023-08-28T01:05:31.362 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:31 smithi136 ceph-mon[113531]: pgmap v7558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:31 smithi093 ceph-mon[118902]: pgmap v7558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:31 smithi093 ceph-mon[111335]: pgmap v7558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:31.200Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:31.201Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:31.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:31.362449+0000) 2023-08-28T01:05:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:32.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:32.362636+0000) 2023-08-28T01:05:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:32.648Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:32.648Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:33 smithi136 ceph-mon[113531]: pgmap v7559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:33.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3233253003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:33.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:33.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:33.362872+0000) 2023-08-28T01:05:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:33 smithi093 ceph-mon[118902]: pgmap v7559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3233253003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:33 smithi093 ceph-mon[111335]: pgmap v7559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3233253003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:34.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:34.363146+0000) 2023-08-28T01:05:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:35 smithi136 ceph-mon[113531]: pgmap v7560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3878652698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:35.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:35.363358+0000) 2023-08-28T01:05:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:35 smithi093 ceph-mon[111335]: pgmap v7560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3878652698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:05:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:05:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:05:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:35 smithi093 ceph-mon[118902]: pgmap v7560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3878652698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:36.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:36.363638+0000) 2023-08-28T01:05:37.363 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:37 smithi136 ceph-mon[113531]: pgmap v7561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:37.364 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2918085786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:37 smithi093 ceph-mon[111335]: pgmap v7561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2918085786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:37 smithi093 ceph-mon[118902]: pgmap v7561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2918085786' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:37.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:37.363970+0000) 2023-08-28T01:05:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:38.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:38.364249+0000) 2023-08-28T01:05:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:39.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:39.364519+0000) 2023-08-28T01:05:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:39 smithi136 ceph-mon[113531]: pgmap v7562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:39 smithi093 ceph-mon[111335]: pgmap v7562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:39 smithi093 ceph-mon[118902]: pgmap v7562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1696811912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:40.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:40.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:40.364836+0000) 2023-08-28T01:05:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1696811912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1696811912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:41 smithi136 ceph-mon[113531]: pgmap v7563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:41.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:41.365091+0000) 2023-08-28T01:05:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:41 smithi093 ceph-mon[111335]: pgmap v7563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:41 smithi093 ceph-mon[118902]: pgmap v7563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:41.201Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:41.203Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3759573509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:42.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:42.365314+0000) 2023-08-28T01:05:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3759573509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3759573509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:42.649Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:42.650Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:43 smithi136 ceph-mon[113531]: pgmap v7564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:43.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:43.365569+0000) 2023-08-28T01:05:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:43 smithi093 ceph-mon[111335]: pgmap v7564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:43 smithi093 ceph-mon[118902]: pgmap v7564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/622290359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:44.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:44.365810+0000) 2023-08-28T01:05:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/622290359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/622290359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:45 smithi136 ceph-mon[113531]: pgmap v7565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:45.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:05:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:45.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:45.365990+0000) 2023-08-28T01:05:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:05:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:05:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:05:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:45 smithi093 ceph-mon[111335]: pgmap v7565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:05:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:45 smithi093 ceph-mon[118902]: pgmap v7565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:45.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:05:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:46.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:46.366142+0000) 2023-08-28T01:05:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:47.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:47.366289+0000) 2023-08-28T01:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:47 smithi136 ceph-mon[113531]: pgmap v7566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2883551342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[111335]: pgmap v7566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2883551342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[118902]: pgmap v7566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2883551342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:47.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:05:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:05:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:48.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:48.366462+0000) 2023-08-28T01:05:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:05:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:05:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:05:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:05:48.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:05:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:49 smithi136 ceph-mon[113531]: pgmap v7567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/169097083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:49.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:49.366704+0000) 2023-08-28T01:05:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:49 smithi093 ceph-mon[111335]: pgmap v7567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/169097083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:49 smithi093 ceph-mon[118902]: pgmap v7567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/169097083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:50.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:50.366979+0000) 2023-08-28T01:05:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:51.202Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:51.203Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:51 smithi136 ceph-mon[113531]: pgmap v7568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3956427015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:51.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:51.367194+0000) 2023-08-28T01:05:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:51 smithi093 ceph-mon[118902]: pgmap v7568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:51.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3956427015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:51 smithi093 ceph-mon[111335]: pgmap v7568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:51.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3956427015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:52.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:52.367410+0000) 2023-08-28T01:05:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:52.649Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:05:52.651Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:05:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:53 smithi136 ceph-mon[113531]: pgmap v7569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:53.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:53.367713+0000) 2023-08-28T01:05:53.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:53 smithi093 ceph-mon[111335]: pgmap v7569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:53 smithi093 ceph-mon[118902]: pgmap v7569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2267015339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:54.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:54.368003+0000) 2023-08-28T01:05:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2267015339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:54.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:54.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2267015339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:54.991 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:05:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:55 smithi136 ceph-mon[113531]: pgmap v7570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:55.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:55.368337+0000) 2023-08-28T01:05:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:55 smithi093 ceph-mon[111335]: pgmap v7570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:55 smithi093 ceph-mon[118902]: pgmap v7570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:05:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:05:55] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:05:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:56.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:56.368568+0000) 2023-08-28T01:05:57.368 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/109871546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:57.368 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:57 smithi136 ceph-mon[113531]: pgmap v7571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/109871546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:57 smithi093 ceph-mon[111335]: pgmap v7571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/109871546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:57 smithi093 ceph-mon[118902]: pgmap v7571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:57.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:57.368782+0000) 2023-08-28T01:05:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:58.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:58.369067+0000) 2023-08-28T01:05:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:59 smithi136 ceph-mon[113531]: pgmap v7572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:05:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4244429643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:59.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:05:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:05:59.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:05:59.369380+0000) 2023-08-28T01:05:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:59 smithi093 ceph-mon[111335]: pgmap v7572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:05:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4244429643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:05:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:59 smithi093 ceph-mon[118902]: pgmap v7572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:05:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:05:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4244429643' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:00.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:00.369705+0000) 2023-08-28T01:06:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:01 smithi136 ceph-mon[113531]: pgmap v7573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:01.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2681045783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:01.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:01.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:01.370003+0000) 2023-08-28T01:06:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:01 smithi093 ceph-mon[111335]: pgmap v7573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2681045783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:01 smithi093 ceph-mon[118902]: pgmap v7573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2681045783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:01.203Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:01.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:01.204Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:02.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:02.370326+0000) 2023-08-28T01:06:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:02.650Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:02.651Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:03 smithi136 ceph-mon[113531]: pgmap v7574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:03.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:03.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:03.370601+0000) 2023-08-28T01:06:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:03 smithi093 ceph-mon[111335]: pgmap v7574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:03 smithi093 ceph-mon[118902]: pgmap v7574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:04.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/734713360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:04.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:04.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:04.370873+0000) 2023-08-28T01:06:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/734713360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/734713360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:05 smithi136 ceph-mon[113531]: pgmap v7575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:05.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:05.371157+0000) 2023-08-28T01:06:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:05 smithi093 ceph-mon[111335]: pgmap v7575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:05 smithi093 ceph-mon[118902]: pgmap v7575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:06:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:06:05] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:06:06.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3834450178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:06.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:06.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:06.371359+0000) 2023-08-28T01:06:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3834450178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3834450178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:07 smithi136 ceph-mon[113531]: pgmap v7576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:07.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:07.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:07.371626+0000) 2023-08-28T01:06:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:07 smithi093 ceph-mon[111335]: pgmap v7576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:07 smithi093 ceph-mon[118902]: pgmap v7576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:08.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3264859716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:08.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:08.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:08.371899+0000) 2023-08-28T01:06:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3264859716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3264859716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:09 smithi136 ceph-mon[113531]: pgmap v7577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:09.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:09.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:09.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:09.372178+0000) 2023-08-28T01:06:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:09 smithi093 ceph-mon[118902]: pgmap v7577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:09 smithi093 ceph-mon[111335]: pgmap v7577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:09.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:10.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:10.372426+0000) 2023-08-28T01:06:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:11 smithi136 ceph-mon[113531]: pgmap v7578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:11.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4015532448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:11.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:11.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:11.372672+0000) 2023-08-28T01:06:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:11 smithi093 ceph-mon[111335]: pgmap v7578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4015532448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:11 smithi093 ceph-mon[118902]: pgmap v7578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4015532448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:11.204Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:11.205Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:12.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:12.372945+0000) 2023-08-28T01:06:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:12.651Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:12.652Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:13 smithi136 ceph-mon[113531]: pgmap v7579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1267468807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:13.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:13.373212+0000) 2023-08-28T01:06:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:13 smithi093 ceph-mon[111335]: pgmap v7579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1267468807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:13 smithi093 ceph-mon[118902]: pgmap v7579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1267468807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:14.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:14.373420+0000) 2023-08-28T01:06:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:15 smithi136 ceph-mon[113531]: pgmap v7580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3338442284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:15.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:15.373652+0000) 2023-08-28T01:06:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:06:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:06:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:06:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:15 smithi093 ceph-mon[111335]: pgmap v7580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3338442284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:15 smithi093 ceph-mon[118902]: pgmap v7580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3338442284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:16.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:16.373907+0000) 2023-08-28T01:06:17.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:17 smithi136 ceph-mon[113531]: pgmap v7581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:17 smithi093 ceph-mon[111335]: pgmap v7581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:17 smithi093 ceph-mon[118902]: pgmap v7581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:17.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:17.374110+0000) 2023-08-28T01:06:18.374 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/653825399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/653825399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/653825399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:18.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:18.374366+0000) 2023-08-28T01:06:19.374 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:19 smithi136 ceph-mon[113531]: pgmap v7582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:19 smithi093 ceph-mon[111335]: pgmap v7582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:19 smithi093 ceph-mon[118902]: pgmap v7582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:19.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:19.374606+0000) 2023-08-28T01:06:20.374 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2235054921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:20.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2235054921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2235054921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:20.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:20.374853+0000) 2023-08-28T01:06:21.374 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:21 smithi136 ceph-mon[113531]: pgmap v7583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:21 smithi093 ceph-mon[111335]: pgmap v7583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:21 smithi093 ceph-mon[118902]: pgmap v7583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:21.204Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:21.205Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:21.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:21.375124+0000) 2023-08-28T01:06:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:22.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:22.375360+0000) 2023-08-28T01:06:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:22.652Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:22.653Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:23 smithi136 ceph-mon[113531]: pgmap v7584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3925877330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:23.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:23.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:23.375643+0000) 2023-08-28T01:06:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:23 smithi093 ceph-mon[118902]: pgmap v7584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3925877330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:23 smithi093 ceph-mon[111335]: pgmap v7584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3925877330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:24.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:24.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:24.375874+0000) 2023-08-28T01:06:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:25 smithi136 ceph-mon[113531]: pgmap v7585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:25.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/531361490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:25.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:25.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:25.376149+0000) 2023-08-28T01:06:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:25 smithi093 ceph-mon[111335]: pgmap v7585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/531361490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:06:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:06:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:06:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:25 smithi093 ceph-mon[118902]: pgmap v7585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/531361490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:26.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:26.376428+0000) 2023-08-28T01:06:27.376 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:27 smithi136 ceph-mon[113531]: pgmap v7586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:27.376 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1813621615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:27 smithi093 ceph-mon[111335]: pgmap v7586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1813621615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:27 smithi093 ceph-mon[118902]: pgmap v7586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1813621615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:27.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:27.376628+0000) 2023-08-28T01:06:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:28.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:28.376969+0000) 2023-08-28T01:06:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:29 smithi136 ceph-mon[113531]: pgmap v7587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:29.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:29.377245+0000) 2023-08-28T01:06:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:29 smithi093 ceph-mon[111335]: pgmap v7587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:29 smithi093 ceph-mon[118902]: pgmap v7587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:30.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1787583552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:30.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:30.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:30.377513+0000) 2023-08-28T01:06:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1787583552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1787583552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:31.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:31 smithi136 ceph-mon[113531]: pgmap v7588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:31.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:31.377785+0000) 2023-08-28T01:06:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:31 smithi093 ceph-mon[111335]: pgmap v7588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:31 smithi093 ceph-mon[118902]: pgmap v7588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:31.205Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:31.206Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1605109076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:32.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:32.378018+0000) 2023-08-28T01:06:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1605109076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1605109076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:32.653Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:32.654Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:33 smithi136 ceph-mon[113531]: pgmap v7589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:33.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:33.378323+0000) 2023-08-28T01:06:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:33 smithi093 ceph-mon[111335]: pgmap v7589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:33 smithi093 ceph-mon[118902]: pgmap v7589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:34.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:34.378539+0000) 2023-08-28T01:06:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:35 smithi136 ceph-mon[113531]: pgmap v7590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:35.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2745073842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:35.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:35.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:35.378763+0000) 2023-08-28T01:06:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:35 smithi093 ceph-mon[111335]: pgmap v7590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2745073842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:06:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:06:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:06:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:35 smithi093 ceph-mon[118902]: pgmap v7590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:35.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2745073842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:36.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:36.379027+0000) 2023-08-28T01:06:37.378 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:37 smithi136 ceph-mon[113531]: pgmap v7591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:37.379 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/544301522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:37 smithi093 ceph-mon[111335]: pgmap v7591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/544301522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:37 smithi093 ceph-mon[118902]: pgmap v7591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/544301522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:37.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:37.379291+0000) 2023-08-28T01:06:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:38.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:38.379544+0000) 2023-08-28T01:06:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:39 smithi136 ceph-mon[113531]: pgmap v7592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2555166336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:39.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:39.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:39.379807+0000) 2023-08-28T01:06:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:39 smithi093 ceph-mon[111335]: pgmap v7592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2555166336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:39 smithi093 ceph-mon[118902]: pgmap v7592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2555166336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:40.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:40.379994+0000) 2023-08-28T01:06:41.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:41 smithi136 ceph-mon[113531]: pgmap v7593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:41.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:41.380222+0000) 2023-08-28T01:06:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:41 smithi093 ceph-mon[111335]: pgmap v7593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:41 smithi093 ceph-mon[118902]: pgmap v7593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:41.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:41.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2447877044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:42.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:42.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:42.380454+0000) 2023-08-28T01:06:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2447877044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2447877044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:42.654Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:42.654Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:43 smithi136 ceph-mon[113531]: pgmap v7594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:43.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:43.380663+0000) 2023-08-28T01:06:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:43 smithi093 ceph-mon[111335]: pgmap v7594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:43 smithi093 ceph-mon[118902]: pgmap v7594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/122114667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:44.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:44.380815+0000) 2023-08-28T01:06:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/122114667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/122114667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:45 smithi136 ceph-mon[113531]: pgmap v7595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:45.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:45.381103+0000) 2023-08-28T01:06:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:45 smithi093 ceph-mon[118902]: pgmap v7595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:45 smithi093 ceph-mon[111335]: pgmap v7595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:06:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:06:45] "GET /metrics HTTP/1.1" 200 239936 "" "Prometheus/2.43.0" 2023-08-28T01:06:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:46.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:46.381356+0000) 2023-08-28T01:06:47.381 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:47 smithi136 ceph-mon[113531]: pgmap v7596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:47.381 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/202558339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:47 smithi093 ceph-mon[111335]: pgmap v7596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/202558339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:47 smithi093 ceph-mon[118902]: pgmap v7596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/202558339' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:47.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:47.381646+0000) 2023-08-28T01:06:48.381 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:06:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:06:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:06:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:48.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:48.381840+0000) 2023-08-28T01:06:49.381 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:49 smithi136 ceph-mon[113531]: pgmap v7597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:49.381 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1294180086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:49 smithi093 ceph-mon[118902]: pgmap v7597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1294180086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:49 smithi093 ceph-mon[111335]: pgmap v7597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1294180086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:49.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:49.382051+0000) 2023-08-28T01:06:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:50.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:50.382211+0000) 2023-08-28T01:06:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:51.206Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:51.207Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2451M 2023-08-28T01:06:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[118902]: pgmap v7598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3637639955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2451M 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[111335]: pgmap v7598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:51.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3637639955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:51.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:51.382431+0000) 2023-08-28T01:06:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:51 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2451M 2023-08-28T01:06:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:51 smithi136 ceph-mon[113531]: pgmap v7598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:06:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:06:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:06:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3637639955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:52.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:52.382688+0000) 2023-08-28T01:06:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:52.654Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:06:52.655Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:06:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:53 smithi136 ceph-mon[113531]: pgmap v7599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1346700778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:53.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:53.382888+0000) 2023-08-28T01:06:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:53 smithi093 ceph-mon[111335]: pgmap v7599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1346700778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:53 smithi093 ceph-mon[118902]: pgmap v7599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1346700778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:54.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:54.383162+0000) 2023-08-28T01:06:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:06:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:55 smithi136 ceph-mon[113531]: pgmap v7600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:55.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:55.383419+0000) 2023-08-28T01:06:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:55 smithi093 ceph-mon[111335]: pgmap v7600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:55 smithi093 ceph-mon[118902]: pgmap v7600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:55.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:06:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:06:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:06:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/529061852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:56.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:56.383624+0000) 2023-08-28T01:06:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/529061852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/529061852' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:57 smithi136 ceph-mon[113531]: pgmap v7601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:57.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:57.383872+0000) 2023-08-28T01:06:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:57 smithi093 ceph-mon[111335]: pgmap v7601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:57 smithi093 ceph-mon[118902]: pgmap v7601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/422964394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:58.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:58.384110+0000) 2023-08-28T01:06:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/422964394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:58.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/422964394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:06:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:06:59 smithi136 ceph-mon[113531]: pgmap v7602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:06:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:06:59.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:06:59.384298+0000) 2023-08-28T01:06:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:06:59 smithi093 ceph-mon[111335]: pgmap v7602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:06:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:06:59 smithi093 ceph-mon[118902]: pgmap v7602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:00.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:00.384499+0000) 2023-08-28T01:07:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:01 smithi093 ceph-mon[111335]: pgmap v7603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2493462900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:01 smithi093 ceph-mon[118902]: pgmap v7603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2493462900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:01.207Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:01.208Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:01 smithi136 ceph-mon[113531]: pgmap v7603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2493462900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:01.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:01.384701+0000) 2023-08-28T01:07:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:02.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:02.384879+0000) 2023-08-28T01:07:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:02.654Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:02.655Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:03 smithi136 ceph-mon[113531]: pgmap v7604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:03.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/406256849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:03.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:03.385078+0000) 2023-08-28T01:07:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:03 smithi093 ceph-mon[111335]: pgmap v7604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/406256849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:03 smithi093 ceph-mon[118902]: pgmap v7604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/406256849' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:04.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:04.385250+0000) 2023-08-28T01:07:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:05 smithi136 ceph-mon[113531]: pgmap v7605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2249443684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:05.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:05.385414+0000) 2023-08-28T01:07:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:07:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:07:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:07:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:05 smithi093 ceph-mon[118902]: pgmap v7605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2249443684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:05 smithi093 ceph-mon[111335]: pgmap v7605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2249443684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:06.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:06.385611+0000) 2023-08-28T01:07:07.385 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:07 smithi136 ceph-mon[113531]: pgmap v7606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:07 smithi093 ceph-mon[111335]: pgmap v7606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:07 smithi093 ceph-mon[118902]: pgmap v7606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:07.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:07.385824+0000) 2023-08-28T01:07:08.385 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/847510020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/847510020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/847510020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:08.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:08.386072+0000) 2023-08-28T01:07:09.385 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:09 smithi136 ceph-mon[113531]: pgmap v7607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:09.386 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:09 smithi093 ceph-mon[111335]: pgmap v7607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:09 smithi093 ceph-mon[118902]: pgmap v7607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:09.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:09.386339+0000) 2023-08-28T01:07:10.386 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1670550406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1670550406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1670550406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:10.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:10.386598+0000) 2023-08-28T01:07:11.386 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:11 smithi136 ceph-mon[113531]: pgmap v7608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:11 smithi093 ceph-mon[111335]: pgmap v7608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:11 smithi093 ceph-mon[118902]: pgmap v7608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:11.208Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:11.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:11.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:11.386808+0000) 2023-08-28T01:07:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:12.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:12.387078+0000) 2023-08-28T01:07:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:12.654Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:12.656Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:13 smithi136 ceph-mon[113531]: pgmap v7609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:13.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1914718917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:13.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:13.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:13.387302+0000) 2023-08-28T01:07:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:13 smithi093 ceph-mon[111335]: pgmap v7609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1914718917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:13 smithi093 ceph-mon[118902]: pgmap v7609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1914718917' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:14.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:14.387571+0000) 2023-08-28T01:07:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:15 smithi136 ceph-mon[113531]: pgmap v7610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3930468690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:15.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:15.387832+0000) 2023-08-28T01:07:15.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:15 smithi093 ceph-mon[111335]: pgmap v7610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3930468690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:15 smithi093 ceph-mon[118902]: pgmap v7610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3930468690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:07:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:07:15] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:07:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:16.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:16.388108+0000) 2023-08-28T01:07:17.387 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:17 smithi136 ceph-mon[113531]: pgmap v7611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:17 smithi093 ceph-mon[111335]: pgmap v7611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:17 smithi093 ceph-mon[118902]: pgmap v7611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:17.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:17.388363+0000) 2023-08-28T01:07:18.388 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/317138156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/317138156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/317138156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:18.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:18.388609+0000) 2023-08-28T01:07:19.388 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:19 smithi136 ceph-mon[113531]: pgmap v7612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:19 smithi093 ceph-mon[111335]: pgmap v7612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:19 smithi093 ceph-mon[118902]: pgmap v7612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:19.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:19.388805+0000) 2023-08-28T01:07:20.388 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/379866120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:20.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/379866120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:20.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/379866120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:20.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:20.389082+0000) 2023-08-28T01:07:21.388 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:21 smithi136 ceph-mon[113531]: pgmap v7613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:21 smithi093 ceph-mon[111335]: pgmap v7613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:21 smithi093 ceph-mon[118902]: pgmap v7613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:21.209Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:21.209Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:21.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:21.389294+0000) 2023-08-28T01:07:22.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3459830377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3459830377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3459830377' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:22.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:22.389532+0000) 2023-08-28T01:07:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:22.656Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:22.657Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:23.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:23 smithi136 ceph-mon[113531]: pgmap v7614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:23 smithi093 ceph-mon[111335]: pgmap v7614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:23 smithi093 ceph-mon[118902]: pgmap v7614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:23.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:23.389762+0000) 2023-08-28T01:07:24.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:24.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:24.390012+0000) 2023-08-28T01:07:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:25 smithi093 ceph-mon[111335]: pgmap v7615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3367787526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:25 smithi093 ceph-mon[118902]: pgmap v7615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:25.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3367787526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:25.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:25 smithi136 ceph-mon[113531]: pgmap v7615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:25.390 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3367787526' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:25.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:25.390237+0000) 2023-08-28T01:07:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:07:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:07:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:07:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:26.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:26.390480+0000) 2023-08-28T01:07:27.390 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:27 smithi136 ceph-mon[113531]: pgmap v7616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:27.390 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/663921583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:27 smithi093 ceph-mon[111335]: pgmap v7616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/663921583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:27 smithi093 ceph-mon[118902]: pgmap v7616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/663921583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:27.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:27.390744+0000) 2023-08-28T01:07:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:28.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:28.390964+0000) 2023-08-28T01:07:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:29 smithi136 ceph-mon[113531]: pgmap v7617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/827080342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:29.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:29.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:29.391168+0000) 2023-08-28T01:07:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:29 smithi093 ceph-mon[111335]: pgmap v7617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/827080342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:29 smithi093 ceph-mon[118902]: pgmap v7617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/827080342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:30.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:30.391427+0000) 2023-08-28T01:07:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:31 smithi136 ceph-mon[113531]: pgmap v7618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:31.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:31.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:31.391725+0000) 2023-08-28T01:07:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:31 smithi093 ceph-mon[111335]: pgmap v7618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:31 smithi093 ceph-mon[118902]: pgmap v7618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:31.209Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:31.210Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4269615170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:32.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:32.391996+0000) 2023-08-28T01:07:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4269615170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4269615170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:32.657Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:32.658Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:33 smithi136 ceph-mon[113531]: pgmap v7619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:33.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:33.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:33.392295+0000) 2023-08-28T01:07:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:33 smithi093 ceph-mon[111335]: pgmap v7619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:33 smithi093 ceph-mon[118902]: pgmap v7619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2259694818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2259694818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2259694818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:34.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:34.392571+0000) 2023-08-28T01:07:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:35 smithi093 ceph-mon[111335]: pgmap v7620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:35 smithi093 ceph-mon[118902]: pgmap v7620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:07:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:07:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:07:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:35 smithi136 ceph-mon[113531]: pgmap v7620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:35.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:35.392787+0000) 2023-08-28T01:07:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:36.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:36.393084+0000) 2023-08-28T01:07:37.393 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:37 smithi136 ceph-mon[113531]: pgmap v7621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:37.393 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2372633090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:37 smithi093 ceph-mon[111335]: pgmap v7621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2372633090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:37 smithi093 ceph-mon[118902]: pgmap v7621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2372633090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:37.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:37.393387+0000) 2023-08-28T01:07:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:38.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:38.393609+0000) 2023-08-28T01:07:39.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:39.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:39.393798+0000) 2023-08-28T01:07:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:39 smithi136 ceph-mon[113531]: pgmap v7622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2015302750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:39 smithi093 ceph-mon[111335]: pgmap v7622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2015302750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:39 smithi093 ceph-mon[118902]: pgmap v7622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2015302750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:40.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:40.394075+0000) 2023-08-28T01:07:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:41 smithi136 ceph-mon[113531]: pgmap v7623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3915830691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:41.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:41.394302+0000) 2023-08-28T01:07:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:41 smithi093 ceph-mon[111335]: pgmap v7623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3915830691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:41 smithi093 ceph-mon[118902]: pgmap v7623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3915830691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:41.210Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:41.211Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:42.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:42.394540+0000) 2023-08-28T01:07:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:42.658Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:42.659Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:43.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:43 smithi136 ceph-mon[113531]: pgmap v7624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:43.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:43.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:43.394764+0000) 2023-08-28T01:07:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:43 smithi093 ceph-mon[111335]: pgmap v7624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:43 smithi093 ceph-mon[118902]: pgmap v7624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3336801654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:44.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:44.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:44.395058+0000) 2023-08-28T01:07:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3336801654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3336801654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:45 smithi136 ceph-mon[113531]: pgmap v7625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:45.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:45.395314+0000) 2023-08-28T01:07:45.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:45 smithi093 ceph-mon[111335]: pgmap v7625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:45 smithi093 ceph-mon[118902]: pgmap v7625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:07:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:07:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:07:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2778156635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:46.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:46.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:46.395548+0000) 2023-08-28T01:07:46.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2778156635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2778156635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:47 smithi093 ceph-mon[118902]: pgmap v7626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:47 smithi093 ceph-mon[111335]: pgmap v7626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:47 smithi136 ceph-mon[113531]: pgmap v7626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:47.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:47.395808+0000) 2023-08-28T01:07:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1966717865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1966717865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:48.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1966717865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:48.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:48.396054+0000) 2023-08-28T01:07:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:49 smithi093 ceph-mon[111335]: pgmap v7627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:49 smithi093 ceph-mon[118902]: pgmap v7627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:49 smithi136 ceph-mon[113531]: pgmap v7627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:49.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:49.396303+0000) 2023-08-28T01:07:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:50.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:50.396520+0000) 2023-08-28T01:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:51 smithi136 ceph-mon[113531]: pgmap v7628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/458477273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:51.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:07:51.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:51.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:51.396757+0000) 2023-08-28T01:07:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-mon[111335]: pgmap v7628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/458477273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:07:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-mon[118902]: pgmap v7628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/458477273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:51.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:07:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:51.211Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:51.212Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:52.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:52.396898+0000) 2023-08-28T01:07:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:52.658Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:07:52.659Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:53 smithi136 ceph-mon[113531]: pgmap v7629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4163156413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:53.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:53.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:53.397077+0000) 2023-08-28T01:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[111335]: pgmap v7629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4163156413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[118902]: pgmap v7629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4163156413' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:53.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:07:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:07:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:54.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:54.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:54.397326+0000) 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:07:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:07:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:55 smithi136 ceph-mon[113531]: pgmap v7630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:55.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2151120329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:55.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:55.397554+0000) 2023-08-28T01:07:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:07:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:07:55] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:07:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:55 smithi093 ceph-mon[111335]: pgmap v7630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2151120329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:55 smithi093 ceph-mon[118902]: pgmap v7630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:55.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2151120329' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:56.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:56.397831+0000) 2023-08-28T01:07:57.397 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:57 smithi136 ceph-mon[113531]: pgmap v7631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:57 smithi093 ceph-mon[111335]: pgmap v7631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:57 smithi093 ceph-mon[118902]: pgmap v7631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:57.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:57.398105+0000) 2023-08-28T01:07:58.398 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1950871446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:59.318 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1950871446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:59.319 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:58.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:58.398359+0000) 2023-08-28T01:07:59.319 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1950871446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:07:59.518 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:07:59 smithi093 ceph-mon[111335]: pgmap v7632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:59.519 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:07:59 smithi136 ceph-mon[113531]: pgmap v7632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:59.519 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:07:59 smithi093 ceph-mon[118902]: pgmap v7632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:07:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:07:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:07:59.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:07:59.398559+0000) 2023-08-28T01:08:00.398 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3386492370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3386492370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:00.526 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3386492370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:00.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:00.398831+0000) 2023-08-28T01:08:01.398 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:01 smithi136 ceph-mon[113531]: pgmap v7633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:01 smithi093 ceph-mon[111335]: pgmap v7633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:01 smithi093 ceph-mon[118902]: pgmap v7633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:01.212Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:01.213Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:01.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:01.399111+0000) 2023-08-28T01:08:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:02.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:02.399373+0000) 2023-08-28T01:08:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:02.659Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:02.660Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:03.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:03 smithi136 ceph-mon[113531]: pgmap v7634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:03.604 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4031371742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:03.605 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:03.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:03.399622+0000) 2023-08-28T01:08:03.605 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:03 smithi093 ceph-mon[111335]: pgmap v7634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:03.606 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4031371742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:03.606 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:03 smithi093 ceph-mon[118902]: pgmap v7634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:03.606 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4031371742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:04.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:04.399873+0000) 2023-08-28T01:08:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:05 smithi136 ceph-mon[113531]: pgmap v7635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:05.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4211838126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:05.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:05.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:05.400139+0000) 2023-08-28T01:08:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:08:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:08:05] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:08:05.544 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:05 smithi093 ceph-mon[118902]: pgmap v7635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:05.545 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4211838126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:05.545 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:05 smithi093 ceph-mon[111335]: pgmap v7635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:05.545 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4211838126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:06.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:06.400318+0000) 2023-08-28T01:08:07.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:07 smithi136 ceph-mon[113531]: pgmap v7636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:07 smithi093 ceph-mon[111335]: pgmap v7636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:07 smithi093 ceph-mon[118902]: pgmap v7636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:07.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:07.400574+0000) 2023-08-28T01:08:08.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3719469502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3719469502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3719469502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:08.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:08.400828+0000) 2023-08-28T01:08:09.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:09 smithi136 ceph-mon[113531]: pgmap v7637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:09.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:09 smithi093 ceph-mon[111335]: pgmap v7637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:09 smithi093 ceph-mon[118902]: pgmap v7637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:09.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:09.401025+0000) 2023-08-28T01:08:10.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1352764500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1352764500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1352764500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:10.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:10.401283+0000) 2023-08-28T01:08:11.400 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:11 smithi136 ceph-mon[113531]: pgmap v7638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:11 smithi093 ceph-mon[111335]: pgmap v7638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:11 smithi093 ceph-mon[118902]: pgmap v7638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:11.213Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:11.214Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:11.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:11.401521+0000) 2023-08-28T01:08:12.401 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4284556055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4284556055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4284556055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:12.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:12.401769+0000) 2023-08-28T01:08:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:12.660Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:12.662Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:13.401 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:13 smithi136 ceph-mon[113531]: pgmap v7639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:13 smithi093 ceph-mon[111335]: pgmap v7639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:13 smithi093 ceph-mon[118902]: pgmap v7639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:13.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:13.402000+0000) 2023-08-28T01:08:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:14.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:14.402237+0000) 2023-08-28T01:08:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:15 smithi136 ceph-mon[113531]: pgmap v7640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1403248985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:15.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:15.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:15.402479+0000) 2023-08-28T01:08:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:15 smithi093 ceph-mon[111335]: pgmap v7640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1403248985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:08:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:08:15] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:08:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:15 smithi093 ceph-mon[118902]: pgmap v7640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:15.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1403248985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:16.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:16.402752+0000) 2023-08-28T01:08:17.402 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:17 smithi136 ceph-mon[113531]: pgmap v7641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:17.402 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3098768429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:17 smithi093 ceph-mon[111335]: pgmap v7641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3098768429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:17 smithi093 ceph-mon[118902]: pgmap v7641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3098768429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:17.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:17.403027+0000) 2023-08-28T01:08:18.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:18.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:18.403308+0000) 2023-08-28T01:08:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:19 smithi136 ceph-mon[113531]: pgmap v7642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:19.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2067077443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:19.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:19.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:19.403544+0000) 2023-08-28T01:08:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:19 smithi093 ceph-mon[111335]: pgmap v7642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2067077443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:19 smithi093 ceph-mon[118902]: pgmap v7642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2067077443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:20.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:20.403826+0000) 2023-08-28T01:08:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:21 smithi093 ceph-mon[111335]: pgmap v7643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:21 smithi093 ceph-mon[118902]: pgmap v7643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:21.213Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:21.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:21.214Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:21 smithi136 ceph-mon[113531]: pgmap v7643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:21.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:21.404068+0000) 2023-08-28T01:08:22.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1020733864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1020733864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1020733864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:22.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:22.404298+0000) 2023-08-28T01:08:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:22.660Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:22.662Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:23 smithi093 ceph-mon[111335]: pgmap v7644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:23 smithi093 ceph-mon[118902]: pgmap v7644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:23 smithi136 ceph-mon[113531]: pgmap v7644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:23.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:23.404536+0000) 2023-08-28T01:08:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1658181744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1658181744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1658181744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:24.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:24.404731+0000) 2023-08-28T01:08:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:25 smithi093 ceph-mon[111335]: pgmap v7645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:08:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:08:25] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:08:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:25 smithi093 ceph-mon[118902]: pgmap v7645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:25 smithi136 ceph-mon[113531]: pgmap v7645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:25.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:25.404921+0000) 2023-08-28T01:08:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:26.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:26.405141+0000) 2023-08-28T01:08:27.404 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:27 smithi136 ceph-mon[113531]: pgmap v7646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:27.404 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2278291617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:27 smithi093 ceph-mon[111335]: pgmap v7646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2278291617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:27 smithi093 ceph-mon[118902]: pgmap v7646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:27.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2278291617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:27.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:27.405333+0000) 2023-08-28T01:08:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:28.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:28.405572+0000) 2023-08-28T01:08:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:29 smithi136 ceph-mon[113531]: pgmap v7647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/806165727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:29.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:29.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:29.405837+0000) 2023-08-28T01:08:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:29 smithi093 ceph-mon[111335]: pgmap v7647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/806165727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:29 smithi093 ceph-mon[118902]: pgmap v7647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/806165727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:30.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:30.406152+0000) 2023-08-28T01:08:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:31 smithi093 ceph-mon[111335]: pgmap v7648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3706859874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:31 smithi093 ceph-mon[118902]: pgmap v7648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3706859874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:31.213Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:31.214Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:31 smithi136 ceph-mon[113531]: pgmap v7648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3706859874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:31.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:31.406418+0000) 2023-08-28T01:08:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:32.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:32.406630+0000) 2023-08-28T01:08:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:32.662Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:32.663Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:33 smithi093 ceph-mon[111335]: pgmap v7649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:33 smithi093 ceph-mon[118902]: pgmap v7649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:33 smithi136 ceph-mon[113531]: pgmap v7649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:33.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:33.406871+0000) 2023-08-28T01:08:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3651229177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3651229177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3651229177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:34.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:34.407132+0000) 2023-08-28T01:08:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:35 smithi093 ceph-mon[111335]: pgmap v7650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:35 smithi093 ceph-mon[118902]: pgmap v7650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:08:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:08:35] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:08:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:35 smithi136 ceph-mon[113531]: pgmap v7650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:35.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:35.407341+0000) 2023-08-28T01:08:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1525206448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1525206448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1525206448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:36.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:36.407555+0000) 2023-08-28T01:08:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:37 smithi093 ceph-mon[111335]: pgmap v7651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:37 smithi093 ceph-mon[118902]: pgmap v7651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:37 smithi136 ceph-mon[113531]: pgmap v7651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:37.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:37.407793+0000) 2023-08-28T01:08:38.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3757866636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3757866636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:38.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3757866636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:38.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:38.408007+0000) 2023-08-28T01:08:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:39 smithi093 ceph-mon[111335]: pgmap v7652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:39 smithi093 ceph-mon[118902]: pgmap v7652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:39 smithi136 ceph-mon[113531]: pgmap v7652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:39.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:39.408321+0000) 2023-08-28T01:08:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:40.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:40.408591+0000) 2023-08-28T01:08:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:41 smithi093 ceph-mon[111335]: pgmap v7653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:41.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4052055965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:41 smithi093 ceph-mon[118902]: pgmap v7653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:41.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4052055965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:41.214Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:41.491 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:41.215Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:41.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:41.408867+0000) 2023-08-28T01:08:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:41 smithi136 ceph-mon[113531]: pgmap v7653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:41.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4052055965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:42.663 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:42.662Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:42.663 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:42.663Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:42.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:42.409087+0000) 2023-08-28T01:08:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:43 smithi093 ceph-mon[111335]: pgmap v7654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/749076039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:43 smithi093 ceph-mon[118902]: pgmap v7654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/749076039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:43 smithi136 ceph-mon[113531]: pgmap v7654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/749076039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:43.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:43.409326+0000) 2023-08-28T01:08:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:44.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:44.409587+0000) 2023-08-28T01:08:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:45 smithi093 ceph-mon[111335]: pgmap v7655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:45 smithi093 ceph-mon[118902]: pgmap v7655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:08:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:08:45] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:08:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:45 smithi136 ceph-mon[113531]: pgmap v7655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:45.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:45.409895+0000) 2023-08-28T01:08:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/7283755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/7283755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/7283755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:46.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:46.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:46.410159+0000) 2023-08-28T01:08:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:47 smithi093 ceph-mon[111335]: pgmap v7656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:47 smithi093 ceph-mon[118902]: pgmap v7656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:47 smithi136 ceph-mon[113531]: pgmap v7656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:47.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:47.410407+0000) 2023-08-28T01:08:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/502904281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/502904281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/502904281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:48.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:48.410638+0000) 2023-08-28T01:08:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:49 smithi093 ceph-mon[111335]: pgmap v7657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:49 smithi093 ceph-mon[118902]: pgmap v7657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:49 smithi136 ceph-mon[113531]: pgmap v7657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:49.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:49.410869+0000) 2023-08-28T01:08:50.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2256216249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2256216249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2256216249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:50.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:50.411135+0000) 2023-08-28T01:08:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:51 smithi093 ceph-mon[111335]: pgmap v7658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:51 smithi093 ceph-mon[118902]: pgmap v7658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:51.215Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:51.216Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:51 smithi136 ceph-mon[113531]: pgmap v7658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:51.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:51.411419+0000) 2023-08-28T01:08:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:52.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:52.411610+0000) 2023-08-28T01:08:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:52.663Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:08:52.663Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:08:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:53 smithi136 ceph-mon[113531]: pgmap v7659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2965293829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:53.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:53.411879+0000) 2023-08-28T01:08:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:53 smithi093 ceph-mon[111335]: pgmap v7659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2965293829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:53 smithi093 ceph-mon[118902]: pgmap v7659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2965293829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:08:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:08:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:54.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:08:54.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:08:54.549 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:54.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:54.412107+0000) 2023-08-28T01:08:55.412 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:55 smithi136 ceph-mon[113531]: pgmap v7660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:55.412 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3888210049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:55 smithi093 ceph-mon[111335]: pgmap v7660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3888210049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:55.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:08:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:08:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:08:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:55 smithi093 ceph-mon[118902]: pgmap v7660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:55.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3888210049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:55.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:55.412321+0000) 2023-08-28T01:08:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:56.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:56.412513+0000) 2023-08-28T01:08:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:56.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:57.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:57.412760+0000) 2023-08-28T01:08:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:57 smithi136 ceph-mon[113531]: pgmap v7661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:08:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:08:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1676899644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[111335]: pgmap v7661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:08:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:08:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1676899644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[118902]: pgmap v7661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:08:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:08:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:08:57.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1676899644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:08:58.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:58.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:58.413001+0000) 2023-08-28T01:08:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:08:59 smithi136 ceph-mon[113531]: pgmap v7662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:08:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:08:59.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:08:59.413223+0000) 2023-08-28T01:08:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:08:59 smithi093 ceph-mon[111335]: pgmap v7662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:08:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:08:59 smithi093 ceph-mon[118902]: pgmap v7662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1430386894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:00.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:00.413444+0000) 2023-08-28T01:09:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1430386894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:00.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1430386894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:01.486 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:01.216Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:01.486 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:01.217Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:01 smithi136 ceph-mon[113531]: pgmap v7663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:01.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:01.413712+0000) 2023-08-28T01:09:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:01 smithi093 ceph-mon[111335]: pgmap v7663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:01 smithi093 ceph-mon[118902]: pgmap v7663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1477654536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:02.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:02.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:02.413958+0000) 2023-08-28T01:09:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1477654536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1477654536' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:02.663Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:02.664Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:03 smithi136 ceph-mon[113531]: pgmap v7664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:03.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:03.414212+0000) 2023-08-28T01:09:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:03 smithi093 ceph-mon[111335]: pgmap v7664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:03 smithi093 ceph-mon[118902]: pgmap v7664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4000592788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:04.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:04.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:04.414434+0000) 2023-08-28T01:09:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4000592788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4000592788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:05 smithi136 ceph-mon[113531]: pgmap v7665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:05.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:05.414682+0000) 2023-08-28T01:09:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:05 smithi093 ceph-mon[118902]: pgmap v7665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:05 smithi093 ceph-mon[111335]: pgmap v7665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:09:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:09:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:09:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:06.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:06.414903+0000) 2023-08-28T01:09:07.414 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:07 smithi136 ceph-mon[113531]: pgmap v7666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:07.414 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/910692663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:07 smithi093 ceph-mon[111335]: pgmap v7666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/910692663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:07 smithi093 ceph-mon[118902]: pgmap v7666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/910692663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:07.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:07.415145+0000) 2023-08-28T01:09:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:08.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:08.415358+0000) 2023-08-28T01:09:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:09 smithi093 ceph-mon[111335]: pgmap v7667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2834208782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:09 smithi093 ceph-mon[118902]: pgmap v7667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2834208782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:09 smithi136 ceph-mon[113531]: pgmap v7667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2834208782' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:09.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:09.415538+0000) 2023-08-28T01:09:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:10.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:10.415774+0000) 2023-08-28T01:09:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:11 smithi093 ceph-mon[111335]: pgmap v7668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:11 smithi093 ceph-mon[118902]: pgmap v7668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:11.217Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:11.218Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:11 smithi136 ceph-mon[113531]: pgmap v7668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:11.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:11.416003+0000) 2023-08-28T01:09:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1672095707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1672095707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1672095707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:12.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:12.416276+0000) 2023-08-28T01:09:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:12.664Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:12.665Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:13 smithi093 ceph-mon[111335]: pgmap v7669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:13 smithi093 ceph-mon[118902]: pgmap v7669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:13 smithi136 ceph-mon[113531]: pgmap v7669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:13.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:13.416529+0000) 2023-08-28T01:09:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/366872072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/366872072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/366872072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:14.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:14.416818+0000) 2023-08-28T01:09:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:15 smithi093 ceph-mon[111335]: pgmap v7670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:09:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:09:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:09:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:15 smithi093 ceph-mon[118902]: pgmap v7670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:15 smithi136 ceph-mon[113531]: pgmap v7670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:15.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:15.417091+0000) 2023-08-28T01:09:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:16.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:16.417313+0000) 2023-08-28T01:09:17.416 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:17 smithi136 ceph-mon[113531]: pgmap v7671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:17.417 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3068197736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:17 smithi093 ceph-mon[111335]: pgmap v7671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3068197736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:17 smithi093 ceph-mon[118902]: pgmap v7671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3068197736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:17.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:17.417546+0000) 2023-08-28T01:09:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:18.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:18.417802+0000) 2023-08-28T01:09:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:19 smithi093 ceph-mon[111335]: pgmap v7672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3521900079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:19 smithi093 ceph-mon[118902]: pgmap v7672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3521900079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:19.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:19.418059+0000) 2023-08-28T01:09:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:19 smithi136 ceph-mon[113531]: pgmap v7672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:19.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3521900079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:20.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:20.418294+0000) 2023-08-28T01:09:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:21 smithi093 ceph-mon[111335]: pgmap v7673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/684291707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:21 smithi093 ceph-mon[118902]: pgmap v7673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/684291707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:21.217Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:21.218Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:21 smithi136 ceph-mon[113531]: pgmap v7673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/684291707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:21.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:21.418556+0000) 2023-08-28T01:09:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:22.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:22.418761+0000) 2023-08-28T01:09:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:22.665Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:22.666Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:23 smithi093 ceph-mon[111335]: pgmap v7674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:23 smithi093 ceph-mon[118902]: pgmap v7674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:23.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:23 smithi136 ceph-mon[113531]: pgmap v7674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:23.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:23.419025+0000) 2023-08-28T01:09:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3279373934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3279373934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:24.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:24.419294+0000) 2023-08-28T01:09:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3279373934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:24.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:25 smithi093 ceph-mon[111335]: pgmap v7675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:25 smithi093 ceph-mon[118902]: pgmap v7675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:09:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:09:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:09:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:25 smithi136 ceph-mon[113531]: pgmap v7675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:25.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:25.419508+0000) 2023-08-28T01:09:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1915721194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1915721194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1915721194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:26.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:26.419726+0000) 2023-08-28T01:09:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:27 smithi093 ceph-mon[118902]: pgmap v7676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:27 smithi093 ceph-mon[111335]: pgmap v7676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:27 smithi136 ceph-mon[113531]: pgmap v7676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:27.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:27.419990+0000) 2023-08-28T01:09:28.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/680460730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/680460730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:28.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/680460730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:28.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:28.420215+0000) 2023-08-28T01:09:29.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:29 smithi093 ceph-mon[111335]: pgmap v7677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:29 smithi093 ceph-mon[118902]: pgmap v7677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:29 smithi136 ceph-mon[113531]: pgmap v7677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:29.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:29.420421+0000) 2023-08-28T01:09:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:30.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:30.420629+0000) 2023-08-28T01:09:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:31 smithi093 ceph-mon[111335]: pgmap v7678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2201162343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:31 smithi093 ceph-mon[118902]: pgmap v7678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:31.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2201162343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:31.218Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:31.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:31.219Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:31 smithi136 ceph-mon[113531]: pgmap v7678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2201162343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:31.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:31.420898+0000) 2023-08-28T01:09:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:32.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:32.421156+0000) 2023-08-28T01:09:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:32.666Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:32.666Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:33 smithi093 ceph-mon[118902]: pgmap v7679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3804763796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:33 smithi093 ceph-mon[111335]: pgmap v7679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:33.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3804763796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:33 smithi136 ceph-mon[113531]: pgmap v7679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3804763796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:33.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:33.421374+0000) 2023-08-28T01:09:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:34.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:34.421640+0000) 2023-08-28T01:09:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:35 smithi093 ceph-mon[111335]: pgmap v7680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:09:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:09:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:09:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:35 smithi093 ceph-mon[118902]: pgmap v7680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:35 smithi136 ceph-mon[113531]: pgmap v7680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:35.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:35.421866+0000) 2023-08-28T01:09:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2821731361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2821731361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2821731361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:36.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:36.422090+0000) 2023-08-28T01:09:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:37 smithi093 ceph-mon[111335]: pgmap v7681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:37 smithi093 ceph-mon[118902]: pgmap v7681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:37 smithi136 ceph-mon[113531]: pgmap v7681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:37.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:37.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:37.422333+0000) 2023-08-28T01:09:38.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2202825950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:38.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:38.422586+0000) 2023-08-28T01:09:38.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2202825950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:38.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2202825950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:39 smithi136 ceph-mon[113531]: pgmap v7682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:39.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:39.422844+0000) 2023-08-28T01:09:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:39 smithi093 ceph-mon[111335]: pgmap v7682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:39 smithi093 ceph-mon[118902]: pgmap v7682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3372546019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:40.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:40.423134+0000) 2023-08-28T01:09:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3372546019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3372546019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:41 smithi093 ceph-mon[111335]: pgmap v7683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:41 smithi093 ceph-mon[118902]: pgmap v7683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:41.219Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:41.220Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:41 smithi136 ceph-mon[113531]: pgmap v7683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:41.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:41.423388+0000) 2023-08-28T01:09:42.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:42.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:42.423574+0000) 2023-08-28T01:09:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:42.667Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:42.668Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:43 smithi136 ceph-mon[113531]: pgmap v7684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:43.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3526371699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:43.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:43.423861+0000) 2023-08-28T01:09:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:43 smithi093 ceph-mon[111335]: pgmap v7684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3526371699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:43 smithi093 ceph-mon[118902]: pgmap v7684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3526371699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:44.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:44.424141+0000) 2023-08-28T01:09:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:45 smithi136 ceph-mon[113531]: pgmap v7685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/514997555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:45.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:45.424370+0000) 2023-08-28T01:09:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:09:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:09:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:09:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:45 smithi093 ceph-mon[111335]: pgmap v7685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/514997555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:45 smithi093 ceph-mon[118902]: pgmap v7685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/514997555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:46.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:46.424627+0000) 2023-08-28T01:09:47.424 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:47 smithi136 ceph-mon[113531]: pgmap v7686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:47 smithi093 ceph-mon[111335]: pgmap v7686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:47 smithi093 ceph-mon[118902]: pgmap v7686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:47.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:47.424877+0000) 2023-08-28T01:09:48.424 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1714050342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1714050342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1714050342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:48.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:48.425112+0000) 2023-08-28T01:09:49.424 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:49 smithi136 ceph-mon[113531]: pgmap v7687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:49 smithi093 ceph-mon[111335]: pgmap v7687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:49 smithi093 ceph-mon[118902]: pgmap v7687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:49.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:49.425363+0000) 2023-08-28T01:09:50.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/741733007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/741733007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/741733007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:50.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:50.425602+0000) 2023-08-28T01:09:51.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:51 smithi136 ceph-mon[113531]: pgmap v7688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:51 smithi093 ceph-mon[111335]: pgmap v7688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:51 smithi093 ceph-mon[118902]: pgmap v7688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:51.219Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:51.220Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:51.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:51.425884+0000) 2023-08-28T01:09:52.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3452419648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3452419648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:52.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3452419648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:52.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:52.426150+0000) 2023-08-28T01:09:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:52.667Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:09:52.668Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:09:53.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:53 smithi136 ceph-mon[113531]: pgmap v7689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:53 smithi093 ceph-mon[111335]: pgmap v7689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:53 smithi093 ceph-mon[118902]: pgmap v7689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:53.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:53.426406+0000) 2023-08-28T01:09:54.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:09:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:54.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:54.426626+0000) 2023-08-28T01:09:55.426 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:55 smithi136 ceph-mon[113531]: pgmap v7690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:55.426 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1724907862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:55.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:09:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:09:55] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:09:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:55 smithi093 ceph-mon[118902]: pgmap v7690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1724907862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:55 smithi093 ceph-mon[111335]: pgmap v7690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1724907862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:55.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:55.426828+0000) 2023-08-28T01:09:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:56.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:56.427099+0000) 2023-08-28T01:09:57.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:57 smithi136 ceph-mon[113531]: pgmap v7691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:57.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2404371943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:57.415 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:09:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:57 smithi093 ceph-mon[111335]: pgmap v7691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2404371943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:09:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:57 smithi093 ceph-mon[118902]: pgmap v7691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2404371943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:09:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:09:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:57.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:57.427281+0000) 2023-08-28T01:09:58.426 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:58.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:58.427 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:58.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:58.427439+0000) 2023-08-28T01:09:58.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:59.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:59 smithi136 ceph-mon[113531]: pgmap v7692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:59.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:59.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:59 smithi093 ceph-mon[118902]: pgmap v7692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:59 smithi093 ceph-mon[111335]: pgmap v7692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:09:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:09:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:09:59.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:09:59.427619+0000) 2023-08-28T01:10:00.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4014598445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:00.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:10:00.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:10:00.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4014598445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4014598445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:10:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:00.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:00.427800+0000) 2023-08-28T01:10:01.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:01 smithi136 ceph-mon[113531]: pgmap v7693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:01 smithi093 ceph-mon[111335]: pgmap v7693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:01 smithi093 ceph-mon[118902]: pgmap v7693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:01.220Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:01.222Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:01.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:01.428098+0000) 2023-08-28T01:10:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2752698350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:02.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:02.428382+0000) 2023-08-28T01:10:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2752698350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2752698350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:02.668Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:02.669Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:03 smithi136 ceph-mon[113531]: pgmap v7694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:03.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:03.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:03.428600+0000) 2023-08-28T01:10:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:03 smithi093 ceph-mon[111335]: pgmap v7694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:03 smithi093 ceph-mon[118902]: pgmap v7694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/767706777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:04.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:04.428820+0000) 2023-08-28T01:10:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/767706777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/767706777' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:05 smithi093 ceph-mon[111335]: pgmap v7695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:10:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:10:05] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:10:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:05 smithi093 ceph-mon[118902]: pgmap v7695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:05 smithi136 ceph-mon[113531]: pgmap v7695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:05.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:05.429114+0000) 2023-08-28T01:10:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:06.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:06.429373+0000) 2023-08-28T01:10:07.429 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:07 smithi136 ceph-mon[113531]: pgmap v7696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:07.430 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1964814623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:07 smithi093 ceph-mon[111335]: pgmap v7696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1964814623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:07 smithi093 ceph-mon[118902]: pgmap v7696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1964814623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:07.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:07.429638+0000) 2023-08-28T01:10:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:08.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:08.429905+0000) 2023-08-28T01:10:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:09 smithi093 ceph-mon[111335]: pgmap v7697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2110180034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:09 smithi093 ceph-mon[118902]: pgmap v7697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2110180034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:09 smithi136 ceph-mon[113531]: pgmap v7697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2110180034' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:09.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:09.430161+0000) 2023-08-28T01:10:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:10.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:10.430335+0000) 2023-08-28T01:10:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:11 smithi093 ceph-mon[111335]: pgmap v7698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4227666714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:11 smithi093 ceph-mon[118902]: pgmap v7698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4227666714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:11.221Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:11.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:11.222Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:11 smithi136 ceph-mon[113531]: pgmap v7698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4227666714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:11.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:11.430566+0000) 2023-08-28T01:10:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:12.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:12.430833+0000) 2023-08-28T01:10:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:12.668Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:12.669Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:13 smithi093 ceph-mon[111335]: pgmap v7699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:13 smithi093 ceph-mon[118902]: pgmap v7699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:13 smithi136 ceph-mon[113531]: pgmap v7699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:13.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:13.431093+0000) 2023-08-28T01:10:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1326720110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1326720110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1326720110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:14.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:14.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:14.431293+0000) 2023-08-28T01:10:15.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:15 smithi093 ceph-mon[111335]: pgmap v7700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:15 smithi093 ceph-mon[118902]: pgmap v7700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:10:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:10:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:10:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:15 smithi136 ceph-mon[113531]: pgmap v7700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:15.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:15.431530+0000) 2023-08-28T01:10:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/351159954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/351159954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/351159954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:16.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:16.431807+0000) 2023-08-28T01:10:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:17 smithi093 ceph-mon[111335]: pgmap v7701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:17 smithi093 ceph-mon[118902]: pgmap v7701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:17 smithi136 ceph-mon[113531]: pgmap v7701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:17.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:17.432080+0000) 2023-08-28T01:10:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:18.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:18.432344+0000) 2023-08-28T01:10:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:19 smithi093 ceph-mon[111335]: pgmap v7702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3949883608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:19 smithi093 ceph-mon[118902]: pgmap v7702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:19.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3949883608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:19 smithi136 ceph-mon[113531]: pgmap v7702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3949883608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:19.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:19.432611+0000) 2023-08-28T01:10:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:20.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:20.432887+0000) 2023-08-28T01:10:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:21 smithi093 ceph-mon[111335]: pgmap v7703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3777991503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:21 smithi093 ceph-mon[118902]: pgmap v7703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3777991503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:21.222Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:21.223Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:21 smithi136 ceph-mon[113531]: pgmap v7703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3777991503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:21.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:21.433152+0000) 2023-08-28T01:10:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:22.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:22.433364+0000) 2023-08-28T01:10:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:22.669Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:22.670Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:23 smithi093 ceph-mon[111335]: pgmap v7704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/545350451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:23 smithi093 ceph-mon[118902]: pgmap v7704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/545350451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:23 smithi136 ceph-mon[113531]: pgmap v7704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/545350451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:23.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:23.433583+0000) 2023-08-28T01:10:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:24.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:24.433819+0000) 2023-08-28T01:10:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:25 smithi136 ceph-mon[113531]: pgmap v7705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:25.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:25.434106+0000) 2023-08-28T01:10:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:25 smithi093 ceph-mon[111335]: pgmap v7705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:25 smithi093 ceph-mon[118902]: pgmap v7705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:10:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:10:25] "GET /metrics HTTP/1.1" 200 239936 "" "Prometheus/2.43.0" 2023-08-28T01:10:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2403865237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:26.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:26.434368+0000) 2023-08-28T01:10:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2403865237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2403865237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:27 smithi136 ceph-mon[113531]: pgmap v7706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:27.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:27.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:27.434577+0000) 2023-08-28T01:10:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:27 smithi093 ceph-mon[111335]: pgmap v7706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:27 smithi093 ceph-mon[118902]: pgmap v7706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:28.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1896328892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:28.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:28.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:28.434845+0000) 2023-08-28T01:10:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1896328892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1896328892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:29 smithi136 ceph-mon[113531]: pgmap v7707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:29.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:29.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:29.435124+0000) 2023-08-28T01:10:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:29 smithi093 ceph-mon[111335]: pgmap v7707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:29 smithi093 ceph-mon[118902]: pgmap v7707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1834789056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:30.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:30.435354+0000) 2023-08-28T01:10:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1834789056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1834789056' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:31 smithi093 ceph-mon[111335]: pgmap v7708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:31 smithi093 ceph-mon[118902]: pgmap v7708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:31.222Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:31.223Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:31 smithi136 ceph-mon[113531]: pgmap v7708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:31.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:31.435608+0000) 2023-08-28T01:10:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:32.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:32.435921+0000) 2023-08-28T01:10:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:32.669Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:32.670Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:33 smithi136 ceph-mon[113531]: pgmap v7709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:33.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1282718273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:33.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:33.436208+0000) 2023-08-28T01:10:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:33 smithi093 ceph-mon[111335]: pgmap v7709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1282718273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:33 smithi093 ceph-mon[118902]: pgmap v7709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:33.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1282718273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:34.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:34.436448+0000) 2023-08-28T01:10:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:35 smithi136 ceph-mon[113531]: pgmap v7710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:35.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4143337776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:35.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:35.436720+0000) 2023-08-28T01:10:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:35 smithi093 ceph-mon[111335]: pgmap v7710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4143337776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:35 smithi093 ceph-mon[118902]: pgmap v7710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4143337776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:10:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:10:35] "GET /metrics HTTP/1.1" 200 239936 "" "Prometheus/2.43.0" 2023-08-28T01:10:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:36.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:36.436997+0000) 2023-08-28T01:10:37.436 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:37 smithi136 ceph-mon[113531]: pgmap v7711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:37 smithi093 ceph-mon[111335]: pgmap v7711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:37 smithi093 ceph-mon[118902]: pgmap v7711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:37.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:37.437268+0000) 2023-08-28T01:10:38.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3643776894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3643776894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3643776894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:38.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:38.437572+0000) 2023-08-28T01:10:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:39 smithi136 ceph-mon[113531]: pgmap v7712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:39 smithi093 ceph-mon[111335]: pgmap v7712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:39 smithi093 ceph-mon[118902]: pgmap v7712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:39.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:39.437758+0000) 2023-08-28T01:10:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1222983474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1222983474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1222983474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:40.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:40.438028+0000) 2023-08-28T01:10:41.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:41 smithi136 ceph-mon[113531]: pgmap v7713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:41 smithi093 ceph-mon[111335]: pgmap v7713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:41 smithi093 ceph-mon[118902]: pgmap v7713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:41.222Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:41.223Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:41.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:41.438298+0000) 2023-08-28T01:10:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2425006387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2425006387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2425006387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:42.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:42.438549+0000) 2023-08-28T01:10:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:42.670Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:42.671Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:43 smithi136 ceph-mon[113531]: pgmap v7714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:43 smithi093 ceph-mon[111335]: pgmap v7714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:43 smithi093 ceph-mon[118902]: pgmap v7714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:43.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:43.438791+0000) 2023-08-28T01:10:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:44.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:44.439070+0000) 2023-08-28T01:10:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:45 smithi093 ceph-mon[118902]: pgmap v7715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3177444784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:45 smithi093 ceph-mon[111335]: pgmap v7715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:45.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3177444784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:45.490 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:10:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:10:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:10:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:45 smithi136 ceph-mon[113531]: pgmap v7715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3177444784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:45.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:45.439292+0000) 2023-08-28T01:10:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:46.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:46.439529+0000) 2023-08-28T01:10:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:47 smithi136 ceph-mon[113531]: pgmap v7716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3600924619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:47 smithi093 ceph-mon[118902]: pgmap v7716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3600924619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:47 smithi093 ceph-mon[111335]: pgmap v7716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3600924619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:47.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:47.439693+0000) 2023-08-28T01:10:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:48.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:48.439900+0000) 2023-08-28T01:10:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:49 smithi093 ceph-mon[111335]: pgmap v7717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:49 smithi093 ceph-mon[118902]: pgmap v7717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:49 smithi136 ceph-mon[113531]: pgmap v7717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:49.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:49.440174+0000) 2023-08-28T01:10:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2046207215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2046207215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2046207215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:50.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:50.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:50.440454+0000) 2023-08-28T01:10:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:51 smithi093 ceph-mon[111335]: pgmap v7718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:51 smithi093 ceph-mon[118902]: pgmap v7718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:51.223Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:51.224Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:51 smithi136 ceph-mon[113531]: pgmap v7718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:51.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:51.440734+0000) 2023-08-28T01:10:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3351004941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3351004941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:52.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3351004941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:52.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:52.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:52.441009+0000) 2023-08-28T01:10:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:52.670Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:10:52.671Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:10:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:53 smithi136 ceph-mon[113531]: pgmap v7719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:53.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:53.441320+0000) 2023-08-28T01:10:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:53 smithi093 ceph-mon[111335]: pgmap v7719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:53 smithi093 ceph-mon[118902]: pgmap v7719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2699021960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:54.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:54.441608+0000) 2023-08-28T01:10:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2699021960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:10:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2699021960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:55 smithi136 ceph-mon[113531]: pgmap v7720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:55.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:55.441971+0000) 2023-08-28T01:10:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:55 smithi093 ceph-mon[111335]: pgmap v7720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:55 smithi093 ceph-mon[118902]: pgmap v7720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:10:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:10:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:10:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:56.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:56.442260+0000) 2023-08-28T01:10:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:57 smithi136 ceph-mon[113531]: pgmap v7721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:57.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3212689982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:57 smithi093 ceph-mon[111335]: pgmap v7721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3212689982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:57 smithi093 ceph-mon[118902]: pgmap v7721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3212689982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:57.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:57.442501+0000) 2023-08-28T01:10:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:58.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:58.442751+0000) 2023-08-28T01:10:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:59 smithi093 ceph-mon[111335]: pgmap v7722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:10:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4187168485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:59 smithi093 ceph-mon[118902]: pgmap v7722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:10:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4187168485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:59 smithi136 ceph-mon[113531]: pgmap v7722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:10:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:10:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4187168485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:10:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:10:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:10:59.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:10:59.443026+0000) 2023-08-28T01:11:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:11:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:11:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:11:00.706 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:00.706 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:00.707 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:00.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:00.443219+0000) 2023-08-28T01:11:01.392 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:01 smithi093 ceph-mon[111335]: pgmap v7723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:01.635 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3228844723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:01.636 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:01 smithi093 ceph-mon[118902]: pgmap v7723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:01.636 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3228844723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:01.636 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:01.224Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:01.636 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:01.225Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:01.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:01 smithi136 ceph-mon[113531]: pgmap v7723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:01.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3228844723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:01.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:01.443374+0000) 2023-08-28T01:11:02.644 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:02.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:02.443542+0000) 2023-08-28T01:11:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:02 smithi136 ceph-mon[113531]: pgmap v7724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:02.671Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:02.672Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[111335]: pgmap v7724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[118902]: pgmap v7724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:02.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:11:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:11:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:03.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4127888749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:03.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:03.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:03.443781+0000) 2023-08-28T01:11:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:11:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:11:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:03.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4127888749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:11:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:11:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:11:03.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4127888749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:04 smithi136 ceph-mon[113531]: pgmap v7725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:04.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:04.444046+0000) 2023-08-28T01:11:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:04 smithi093 ceph-mon[111335]: pgmap v7725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:04 smithi093 ceph-mon[118902]: pgmap v7725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:05.688 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 01:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T01:11:05.320610561Z level=info msg="Completed cleanup jobs" duration=16.048146ms 2023-08-28T01:11:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:05.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:05.444318+0000) 2023-08-28T01:11:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:11:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:11:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:11:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3170350081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3170350081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3170350081' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:06.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:06.444584+0000) 2023-08-28T01:11:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:07 smithi136 ceph-mon[113531]: pgmap v7726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:07 smithi093 ceph-mon[111335]: pgmap v7726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:07.587 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:07 smithi093 ceph-mon[118902]: pgmap v7726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:07.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:07.444856+0000) 2023-08-28T01:11:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:08.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:08.445118+0000) 2023-08-28T01:11:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:09 smithi093 ceph-mon[111335]: pgmap v7727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4267861118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:09 smithi093 ceph-mon[118902]: pgmap v7727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4267861118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:09 smithi136 ceph-mon[113531]: pgmap v7727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4267861118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:09.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:09.445369+0000) 2023-08-28T01:11:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:10.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:10.445559+0000) 2023-08-28T01:11:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:11 smithi093 ceph-mon[118902]: pgmap v7728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2233293913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:11 smithi093 ceph-mon[111335]: pgmap v7728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2233293913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:11.225Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:11.226Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:11 smithi136 ceph-mon[113531]: pgmap v7728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2233293913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:11.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:11.445773+0000) 2023-08-28T01:11:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:12.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:12.445999+0000) 2023-08-28T01:11:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:12.672Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:12.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:13 smithi093 ceph-mon[118902]: pgmap v7729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/432308006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:13 smithi093 ceph-mon[111335]: pgmap v7729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/432308006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:13 smithi136 ceph-mon[113531]: pgmap v7729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/432308006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:13.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:13.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:13.446246+0000) 2023-08-28T01:11:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:14.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:14.446480+0000) 2023-08-28T01:11:15.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:15 smithi093 ceph-mon[118902]: pgmap v7730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:15 smithi093 ceph-mon[111335]: pgmap v7730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:11:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:11:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:11:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:15 smithi136 ceph-mon[113531]: pgmap v7730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:15.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:15.446700+0000) 2023-08-28T01:11:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3647803184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3647803184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3647803184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:16.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:16.446958+0000) 2023-08-28T01:11:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:17 smithi093 ceph-mon[111335]: pgmap v7731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:17 smithi093 ceph-mon[118902]: pgmap v7731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:17 smithi136 ceph-mon[113531]: pgmap v7731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:17.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:17.447197+0000) 2023-08-28T01:11:18.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/308496325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:18.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:18.447462+0000) 2023-08-28T01:11:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/308496325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/308496325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:19 smithi136 ceph-mon[113531]: pgmap v7732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:19.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:19.447745+0000) 2023-08-28T01:11:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:19 smithi093 ceph-mon[111335]: pgmap v7732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:19 smithi093 ceph-mon[118902]: pgmap v7732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2740113649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:20.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:20.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:20.447990+0000) 2023-08-28T01:11:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2740113649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:20.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2740113649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:21 smithi093 ceph-mon[111335]: pgmap v7733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:21 smithi093 ceph-mon[118902]: pgmap v7733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:21.226Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:21.226Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:21 smithi136 ceph-mon[113531]: pgmap v7733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:21.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:21.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:21.448261+0000) 2023-08-28T01:11:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:22.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:22.448494+0000) 2023-08-28T01:11:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:22.672Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:22.673Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:23 smithi136 ceph-mon[113531]: pgmap v7734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1369189067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:23.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:23.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:23.448692+0000) 2023-08-28T01:11:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:23 smithi093 ceph-mon[111335]: pgmap v7734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1369189067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:23 smithi093 ceph-mon[118902]: pgmap v7734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1369189067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:24.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:24.448927+0000) 2023-08-28T01:11:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:25 smithi136 ceph-mon[113531]: pgmap v7735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2010783423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:25.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:25.449145+0000) 2023-08-28T01:11:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:25 smithi093 ceph-mon[111335]: pgmap v7735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2010783423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:11:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:11:25] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:11:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:25 smithi093 ceph-mon[118902]: pgmap v7735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2010783423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:26.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:26.449415+0000) 2023-08-28T01:11:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:27 smithi136 ceph-mon[113531]: pgmap v7736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:27 smithi093 ceph-mon[111335]: pgmap v7736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:27 smithi093 ceph-mon[118902]: pgmap v7736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:27.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:27.449647+0000) 2023-08-28T01:11:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1229067886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1229067886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1229067886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:28.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:28.449886+0000) 2023-08-28T01:11:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:29 smithi136 ceph-mon[113531]: pgmap v7737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:29 smithi093 ceph-mon[111335]: pgmap v7737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:29 smithi093 ceph-mon[118902]: pgmap v7737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:29.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:29.450162+0000) 2023-08-28T01:11:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1574640397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1574640397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1574640397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:30.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:30.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:30.450366+0000) 2023-08-28T01:11:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:31 smithi136 ceph-mon[113531]: pgmap v7738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:31 smithi093 ceph-mon[111335]: pgmap v7738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:31 smithi093 ceph-mon[118902]: pgmap v7738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:31.226Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:31.227Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:31.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:31.450629+0000) 2023-08-28T01:11:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3562530975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3562530975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3562530975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:32.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:32.450926+0000) 2023-08-28T01:11:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:32.673Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:32.674Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:33 smithi136 ceph-mon[113531]: pgmap v7739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:33 smithi093 ceph-mon[111335]: pgmap v7739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:33 smithi093 ceph-mon[118902]: pgmap v7739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:33.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:33.451213+0000) 2023-08-28T01:11:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:34.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:34.451477+0000) 2023-08-28T01:11:35.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:11:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:11:35] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:11:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:35 smithi093 ceph-mon[118902]: pgmap v7740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2691706535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:35 smithi093 ceph-mon[111335]: pgmap v7740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:35.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2691706535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:35 smithi136 ceph-mon[113531]: pgmap v7740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2691706535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:35.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:35.451737+0000) 2023-08-28T01:11:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:36.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:36.452051+0000) 2023-08-28T01:11:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:37 smithi136 ceph-mon[113531]: pgmap v7741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1502433236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:37 smithi093 ceph-mon[111335]: pgmap v7741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1502433236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:37 smithi093 ceph-mon[118902]: pgmap v7741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1502433236' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:37.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:37.452243+0000) 2023-08-28T01:11:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:38.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:38.452495+0000) 2023-08-28T01:11:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:39 smithi093 ceph-mon[111335]: pgmap v7742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:39 smithi093 ceph-mon[118902]: pgmap v7742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:39 smithi136 ceph-mon[113531]: pgmap v7742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:39.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:39.452750+0000) 2023-08-28T01:11:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3689559629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:40.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:40.453006+0000) 2023-08-28T01:11:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3689559629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3689559629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:41 smithi093 ceph-mon[111335]: pgmap v7743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:41 smithi093 ceph-mon[118902]: pgmap v7743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:41.227Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:41.228Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:41 smithi136 ceph-mon[113531]: pgmap v7743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:41.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:41.453216+0000) 2023-08-28T01:11:42.673 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/881215916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:42.673 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/881215916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:42.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:42.453399+0000) 2023-08-28T01:11:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/881215916' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:42.991 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:42.673Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:42.992 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:42.675Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:43 smithi136 ceph-mon[113531]: pgmap v7744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:43.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:43.453655+0000) 2023-08-28T01:11:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:43 smithi093 ceph-mon[111335]: pgmap v7744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:43 smithi093 ceph-mon[118902]: pgmap v7744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/621930689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:44.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:44.453845+0000) 2023-08-28T01:11:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/621930689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/621930689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:45 smithi136 ceph-mon[113531]: pgmap v7745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:45.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:45.454021+0000) 2023-08-28T01:11:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:45 smithi093 ceph-mon[118902]: pgmap v7745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:45 smithi093 ceph-mon[111335]: pgmap v7745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:11:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:11:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:11:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:46.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:46.454330+0000) 2023-08-28T01:11:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:47 smithi136 ceph-mon[113531]: pgmap v7746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4019382488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:47 smithi093 ceph-mon[111335]: pgmap v7746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4019382488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:47 smithi093 ceph-mon[118902]: pgmap v7746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4019382488' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:47.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:47.454609+0000) 2023-08-28T01:11:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:48.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:48.454844+0000) 2023-08-28T01:11:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:49 smithi136 ceph-mon[113531]: pgmap v7747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1223247541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:49.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:49.455072+0000) 2023-08-28T01:11:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:49 smithi093 ceph-mon[111335]: pgmap v7747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1223247541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:49 smithi093 ceph-mon[118902]: pgmap v7747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1223247541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:50.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:50.455308+0000) 2023-08-28T01:11:51.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:51 smithi093 ceph-mon[118902]: pgmap v7748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:51.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2814900426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:51 smithi093 ceph-mon[111335]: pgmap v7748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2814900426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:51.227Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:51.228Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:51 smithi136 ceph-mon[113531]: pgmap v7748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2814900426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:51.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:51.455507+0000) 2023-08-28T01:11:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:52.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:52.455726+0000) 2023-08-28T01:11:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:52.674Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:11:52.675Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:11:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:53 smithi136 ceph-mon[113531]: pgmap v7749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:53.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:53.456006+0000) 2023-08-28T01:11:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:53 smithi093 ceph-mon[111335]: pgmap v7749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:53 smithi093 ceph-mon[118902]: pgmap v7749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/166121434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:54.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:54.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:54.456275+0000) 2023-08-28T01:11:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/166121434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/166121434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:11:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:55 smithi136 ceph-mon[113531]: pgmap v7750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:55.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:55.456519+0000) 2023-08-28T01:11:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:55 smithi093 ceph-mon[111335]: pgmap v7750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:55 smithi093 ceph-mon[118902]: pgmap v7750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:11:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:11:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:11:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/109039257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:56.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:56.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:56.456778+0000) 2023-08-28T01:11:56.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/109039257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:56.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/109039257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:57 smithi136 ceph-mon[113531]: pgmap v7751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:57.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:57.456987+0000) 2023-08-28T01:11:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:57 smithi093 ceph-mon[111335]: pgmap v7751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:57 smithi093 ceph-mon[118902]: pgmap v7751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:58.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:58.457229+0000) 2023-08-28T01:11:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:59 smithi136 ceph-mon[113531]: pgmap v7752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:11:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3236724545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:59.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:11:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:11:59.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:11:59.457456+0000) 2023-08-28T01:11:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:59 smithi093 ceph-mon[111335]: pgmap v7752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:11:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3236724545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:11:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:59 smithi093 ceph-mon[118902]: pgmap v7752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:11:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:11:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3236724545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:00.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:00.457739+0000) 2023-08-28T01:12:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:01 smithi093 ceph-mon[118902]: pgmap v7753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3998197720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:01 smithi093 ceph-mon[111335]: pgmap v7753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3998197720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:01.228Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:01.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:01.229Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:01 smithi136 ceph-mon[113531]: pgmap v7753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:01.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3998197720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:01.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:01.457995+0000) 2023-08-28T01:12:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:02.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:02.458220+0000) 2023-08-28T01:12:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:02.675Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:02.677Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:03.645 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:03 smithi093 ceph-mon[111335]: pgmap v7754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:03.645 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:12:03.645 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/69112618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:03.645 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:03 smithi093 ceph-mon[118902]: pgmap v7754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:03.646 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:12:03.646 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/69112618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:03.660 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:03 smithi136 ceph-mon[113531]: pgmap v7754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:03.660 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:12:03.661 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/69112618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:03.661 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:03.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:03.458692+0000) 2023-08-28T01:12:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:04.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:04.458888+0000) 2023-08-28T01:12:05.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:05 smithi136 ceph-mon[113531]: pgmap v7755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:05.327 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:05.327 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:05.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:05.459114+0000) 2023-08-28T01:12:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:12:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:12:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:12:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:05 smithi093 ceph-mon[111335]: pgmap v7755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:05 smithi093 ceph-mon[118902]: pgmap v7755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:05.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:06.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:06.459297+0000) 2023-08-28T01:12:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/52957327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:12:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:12:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/52957327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:12:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:12:06.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/52957327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:06.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:12:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:12:06.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:12:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:07 smithi136 ceph-mon[113531]: pgmap v7756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:07.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:07.459551+0000) 2023-08-28T01:12:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:07 smithi093 ceph-mon[111335]: pgmap v7756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:07 smithi093 ceph-mon[118902]: pgmap v7756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2546463892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:08.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:08.459809+0000) 2023-08-28T01:12:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2546463892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2546463892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:09 smithi136 ceph-mon[113531]: pgmap v7757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:09.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:09.460131+0000) 2023-08-28T01:12:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:09 smithi093 ceph-mon[111335]: pgmap v7757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:09 smithi093 ceph-mon[118902]: pgmap v7757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:10.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/990000398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:10.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:10.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:10.460345+0000) 2023-08-28T01:12:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/990000398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/990000398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:11 smithi093 ceph-mon[111335]: pgmap v7758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:11 smithi093 ceph-mon[118902]: pgmap v7758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:11.229Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:11.230Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:11 smithi136 ceph-mon[113531]: pgmap v7758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:11.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:11.460595+0000) 2023-08-28T01:12:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:12.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:12.460842+0000) 2023-08-28T01:12:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:12.676Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:12.677Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:13.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:13 smithi136 ceph-mon[113531]: pgmap v7759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:13.691 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/172136434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:13.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:13.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:13.461108+0000) 2023-08-28T01:12:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:13 smithi093 ceph-mon[111335]: pgmap v7759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:13.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/172136434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:13 smithi093 ceph-mon[118902]: pgmap v7759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/172136434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:14.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:14.461355+0000) 2023-08-28T01:12:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:15 smithi136 ceph-mon[113531]: pgmap v7760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:15.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1803924997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:15.842 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:15.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:15.461641+0000) 2023-08-28T01:12:16.852 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:15 smithi093 ceph-mon[111335]: pgmap v7760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:16.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1803924997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:16.853 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:12:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:12:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:12:16.853 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:15 smithi093 ceph-mon[118902]: pgmap v7760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:16.853 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1803924997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:16.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:16.461891+0000) 2023-08-28T01:12:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:17 smithi136 ceph-mon[113531]: pgmap v7761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:17.680 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:17 smithi093 ceph-mon[111335]: pgmap v7761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:17.844 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:17 smithi093 ceph-mon[118902]: pgmap v7761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:17.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:17.462150+0000) 2023-08-28T01:12:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/475930203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:18.494 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/475930203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:18.495 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/475930203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:18.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:18.462384+0000) 2023-08-28T01:12:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:19 smithi136 ceph-mon[113531]: pgmap v7762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:19.519 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:19 smithi093 ceph-mon[111335]: pgmap v7762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:19.520 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:19 smithi093 ceph-mon[118902]: pgmap v7762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:19.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:19.462647+0000) 2023-08-28T01:12:21.268 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/713571771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:21.268 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/713571771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:21.268 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/713571771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:21.268 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:20.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:20.462921+0000) 2023-08-28T01:12:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:21 smithi136 ceph-mon[113531]: pgmap v7763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:21.935 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:21 smithi093 ceph-mon[111335]: pgmap v7763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:21.936 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:21 smithi093 ceph-mon[118902]: pgmap v7763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:21.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:21.230Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:21.936 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:21.231Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:21.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:21.463190+0000) 2023-08-28T01:12:22.462 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1844524625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:23.003 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1844524625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:23.003 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:22.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:22.463461+0000) 2023-08-28T01:12:23.003 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1844524625' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:23.004 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:22.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:23.004 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:22.678Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:23 smithi136 ceph-mon[113531]: pgmap v7764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:23.827 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:23.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:23.463657+0000) 2023-08-28T01:12:23.828 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:23 smithi093 ceph-mon[111335]: pgmap v7764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:23.828 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:23 smithi093 ceph-mon[118902]: pgmap v7764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:25.126 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:24.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:24.463908+0000) 2023-08-28T01:12:25.128 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:25.128 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:25 smithi136 ceph-mon[113531]: pgmap v7765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:26.075 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/988473137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:26.075 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:25.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:25.464153+0000) 2023-08-28T01:12:26.076 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:25 smithi093 ceph-mon[111335]: pgmap v7765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:26.076 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/988473137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:26.076 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:25 smithi093 ceph-mon[118902]: pgmap v7765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:26.076 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/988473137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:26.077 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:12:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:12:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:12:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:26.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:26.464402+0000) 2023-08-28T01:12:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:27 smithi136 ceph-mon[113531]: pgmap v7766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2234786304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:27 smithi093 ceph-mon[111335]: pgmap v7766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:27.539 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2234786304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:27.540 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:27 smithi093 ceph-mon[118902]: pgmap v7766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:27.541 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2234786304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:27.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:27.464656+0000) 2023-08-28T01:12:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:28.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:28.464919+0000) 2023-08-28T01:12:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:29 smithi136 ceph-mon[113531]: pgmap v7767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:29.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:29.465161+0000) 2023-08-28T01:12:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:29 smithi093 ceph-mon[111335]: pgmap v7767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:29 smithi093 ceph-mon[118902]: pgmap v7767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3325589285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:30.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:30.465408+0000) 2023-08-28T01:12:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3325589285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3325589285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:31 smithi093 ceph-mon[111335]: pgmap v7768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:31 smithi093 ceph-mon[118902]: pgmap v7768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:31.231Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:31.232Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:31 smithi136 ceph-mon[113531]: pgmap v7768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:31.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:31.465662+0000) 2023-08-28T01:12:32.677 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1572395088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:32.677 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1572395088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1572395088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:32.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:32.465904+0000) 2023-08-28T01:12:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:32.677Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:32.678Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:33 smithi136 ceph-mon[113531]: pgmap v7769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:33.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:33.466126+0000) 2023-08-28T01:12:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:33 smithi093 ceph-mon[111335]: pgmap v7769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:33 smithi093 ceph-mon[118902]: pgmap v7769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3762801207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:34.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:34.466325+0000) 2023-08-28T01:12:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3762801207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3762801207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:35 smithi136 ceph-mon[113531]: pgmap v7770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:35.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:35.466513+0000) 2023-08-28T01:12:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:35 smithi093 ceph-mon[111335]: pgmap v7770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:35 smithi093 ceph-mon[118902]: pgmap v7770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:12:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:12:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:12:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:36.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:36.466740+0000) 2023-08-28T01:12:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:37 smithi136 ceph-mon[113531]: pgmap v7771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:37.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2450767836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:37 smithi093 ceph-mon[111335]: pgmap v7771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:37.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2450767836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:37 smithi093 ceph-mon[118902]: pgmap v7771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:37.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2450767836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:37.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:37.466955+0000) 2023-08-28T01:12:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:38.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:38.467209+0000) 2023-08-28T01:12:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:39 smithi136 ceph-mon[113531]: pgmap v7772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3867054406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:39.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:39.467469+0000) 2023-08-28T01:12:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:39 smithi093 ceph-mon[111335]: pgmap v7772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3867054406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:39 smithi093 ceph-mon[118902]: pgmap v7772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3867054406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:40.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:40.467739+0000) 2023-08-28T01:12:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:41 smithi093 ceph-mon[111335]: pgmap v7773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/670242980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:41 smithi093 ceph-mon[118902]: pgmap v7773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/670242980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:41.232Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:41.233Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:41 smithi136 ceph-mon[113531]: pgmap v7773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/670242980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:41.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:41.467985+0000) 2023-08-28T01:12:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:42.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:42.468234+0000) 2023-08-28T01:12:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:42.678Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:42.679Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:43 smithi136 ceph-mon[113531]: pgmap v7774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:43.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:43.468465+0000) 2023-08-28T01:12:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:43 smithi093 ceph-mon[111335]: pgmap v7774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:43 smithi093 ceph-mon[118902]: pgmap v7774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2647742587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:44.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:44.468709+0000) 2023-08-28T01:12:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2647742587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2647742587' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:45 smithi136 ceph-mon[113531]: pgmap v7775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:45.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:45.468864+0000) 2023-08-28T01:12:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:45 smithi093 ceph-mon[111335]: pgmap v7775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:45 smithi093 ceph-mon[118902]: pgmap v7775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:12:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:12:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:12:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/231757647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:46.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:46.469089+0000) 2023-08-28T01:12:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/231757647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:46.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/231757647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:47 smithi136 ceph-mon[113531]: pgmap v7776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:47.810 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:47.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:47.469305+0000) 2023-08-28T01:12:47.811 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:47 smithi093 ceph-mon[111335]: pgmap v7776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:47.811 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:47 smithi093 ceph-mon[118902]: pgmap v7776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:48.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:48.469563+0000) 2023-08-28T01:12:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:49 smithi136 ceph-mon[113531]: pgmap v7777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3493602833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:49.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:49.469825+0000) 2023-08-28T01:12:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:49 smithi093 ceph-mon[111335]: pgmap v7777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3493602833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:49 smithi093 ceph-mon[118902]: pgmap v7777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3493602833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:50.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:50.470061+0000) 2023-08-28T01:12:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:51 smithi093 ceph-mon[111335]: pgmap v7778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/445846436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:51 smithi093 ceph-mon[118902]: pgmap v7778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/445846436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:51.232Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:51.233Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:51.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:51.470312+0000) 2023-08-28T01:12:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:51 smithi136 ceph-mon[113531]: pgmap v7778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:51.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/445846436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:52.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:52.470602+0000) 2023-08-28T01:12:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:52.678Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:12:52.679Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:12:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:53 smithi136 ceph-mon[113531]: pgmap v7779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1300828787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:53.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:53.470848+0000) 2023-08-28T01:12:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:53 smithi093 ceph-mon[111335]: pgmap v7779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1300828787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:53 smithi093 ceph-mon[118902]: pgmap v7779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1300828787' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:54.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:54.471070+0000) 2023-08-28T01:12:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:12:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:55 smithi136 ceph-mon[113531]: pgmap v7780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:55.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:55.471323+0000) 2023-08-28T01:12:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:55 smithi093 ceph-mon[111335]: pgmap v7780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:55 smithi093 ceph-mon[118902]: pgmap v7780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:12:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:12:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:12:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3054170185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:56.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:56.471604+0000) 2023-08-28T01:12:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3054170185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:56.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3054170185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:57.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:57.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:57.471893+0000) 2023-08-28T01:12:57.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:57 smithi136 ceph-mon[113531]: pgmap v7781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:57 smithi093 ceph-mon[111335]: pgmap v7781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:57 smithi093 ceph-mon[118902]: pgmap v7781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1965733674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:58.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:58.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:58.472177+0000) 2023-08-28T01:12:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1965733674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1965733674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:12:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:12:59 smithi136 ceph-mon[113531]: pgmap v7782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:12:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:12:59.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:12:59.472479+0000) 2023-08-28T01:12:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:12:59 smithi093 ceph-mon[111335]: pgmap v7782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:12:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:12:59 smithi093 ceph-mon[118902]: pgmap v7782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/422642489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:00.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:00.472720+0000) 2023-08-28T01:13:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/422642489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/422642489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:01 smithi093 ceph-mon[111335]: pgmap v7783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:01 smithi093 ceph-mon[118902]: pgmap v7783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:01.232Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:01.233Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:01 smithi136 ceph-mon[113531]: pgmap v7783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:01.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:01.473003+0000) 2023-08-28T01:13:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:02.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:02.473281+0000) 2023-08-28T01:13:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:02.679Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:02.680Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:03 smithi136 ceph-mon[113531]: pgmap v7784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/582697985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:03.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:03.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:03.473536+0000) 2023-08-28T01:13:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:03 smithi093 ceph-mon[111335]: pgmap v7784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:03.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/582697985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:03 smithi093 ceph-mon[118902]: pgmap v7784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/582697985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:04.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:04.473776+0000) 2023-08-28T01:13:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:05 smithi136 ceph-mon[113531]: pgmap v7785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1858366869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:05.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:05.474056+0000) 2023-08-28T01:13:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:13:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:13:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:13:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:05 smithi093 ceph-mon[111335]: pgmap v7785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1858366869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:05 smithi093 ceph-mon[118902]: pgmap v7785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1858366869' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:13:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:06.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:06.474227+0000) 2023-08-28T01:13:06.721 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:13:06.722 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:13:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:07 smithi136 ceph-mon[113531]: pgmap v7786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:07.628 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:07 smithi093 ceph-mon[111335]: pgmap v7786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:07.628 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:07 smithi093 ceph-mon[118902]: pgmap v7786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:07.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:07.474397+0000) 2023-08-28T01:13:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/931586435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:08.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:08.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:08.474562+0000) 2023-08-28T01:13:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/931586435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:08.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/931586435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:09.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:09.474772+0000) 2023-08-28T01:13:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-mon[113531]: pgmap v7787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:13:09.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[111335]: pgmap v7787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[118902]: pgmap v7787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:13:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:13:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:13:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3754240382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:10.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:10.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:10.475019+0000) 2023-08-28T01:13:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3754240382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:10.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3754240382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:11.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:11.233Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:11.233Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:11 smithi136 ceph-mon[113531]: pgmap v7788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:11.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:11.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:11.475280+0000) 2023-08-28T01:13:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:11 smithi093 ceph-mon[111335]: pgmap v7788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:11.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:11 smithi093 ceph-mon[118902]: pgmap v7788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1281088240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:12.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:12.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:12.475532+0000) 2023-08-28T01:13:12.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1281088240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:12.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1281088240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:12.680Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:12.680Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:13 smithi136 ceph-mon[113531]: pgmap v7789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:13.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:13.475816+0000) 2023-08-28T01:13:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:13 smithi093 ceph-mon[111335]: pgmap v7789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:13 smithi093 ceph-mon[118902]: pgmap v7789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3580399213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:14.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:14.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:14.476088+0000) 2023-08-28T01:13:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3580399213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3580399213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:15 smithi093 ceph-mon[111335]: pgmap v7790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:15 smithi093 ceph-mon[118902]: pgmap v7790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:13:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:13:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:13:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:15 smithi136 ceph-mon[113531]: pgmap v7790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:15.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:15.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:15.476366+0000) 2023-08-28T01:13:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:16.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:16.476637+0000) 2023-08-28T01:13:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:17 smithi136 ceph-mon[113531]: pgmap v7791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1681159132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:17 smithi093 ceph-mon[111335]: pgmap v7791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1681159132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:17 smithi093 ceph-mon[118902]: pgmap v7791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1681159132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:17.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:17.476943+0000) 2023-08-28T01:13:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:18.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:18.477221+0000) 2023-08-28T01:13:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:19 smithi136 ceph-mon[113531]: pgmap v7792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:19.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:19.477465+0000) 2023-08-28T01:13:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:19 smithi093 ceph-mon[111335]: pgmap v7792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:19 smithi093 ceph-mon[118902]: pgmap v7792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2388250751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:20.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:20.477738+0000) 2023-08-28T01:13:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2388250751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2388250751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:21 smithi093 ceph-mon[111335]: pgmap v7793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:21 smithi093 ceph-mon[118902]: pgmap v7793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:21.233Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:21.234Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:21 smithi136 ceph-mon[113531]: pgmap v7793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:21.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:21.477941+0000) 2023-08-28T01:13:22.679 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1631195649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:22.680 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1631195649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1631195649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:22.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:22.478205+0000) 2023-08-28T01:13:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:22.680Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:22.681Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:23 smithi136 ceph-mon[113531]: pgmap v7794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:23.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:23.478488+0000) 2023-08-28T01:13:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:23 smithi093 ceph-mon[111335]: pgmap v7794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:23 smithi093 ceph-mon[118902]: pgmap v7794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3122636728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:24.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:24.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:24.478791+0000) 2023-08-28T01:13:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3122636728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3122636728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:25 smithi136 ceph-mon[113531]: pgmap v7795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:25.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:25.479040+0000) 2023-08-28T01:13:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:25 smithi093 ceph-mon[111335]: pgmap v7795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:25 smithi093 ceph-mon[118902]: pgmap v7795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:13:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:13:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:13:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:26.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:26.479336+0000) 2023-08-28T01:13:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:27 smithi136 ceph-mon[113531]: pgmap v7796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1977506925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:27 smithi093 ceph-mon[111335]: pgmap v7796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1977506925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:27 smithi093 ceph-mon[118902]: pgmap v7796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1977506925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:27.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:27.479644+0000) 2023-08-28T01:13:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:28.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:28.479941+0000) 2023-08-28T01:13:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:29 smithi136 ceph-mon[113531]: pgmap v7797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2083072294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:29.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:29.480225+0000) 2023-08-28T01:13:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:29 smithi093 ceph-mon[111335]: pgmap v7797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2083072294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:29 smithi093 ceph-mon[118902]: pgmap v7797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2083072294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:30.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:30.480506+0000) 2023-08-28T01:13:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:31 smithi093 ceph-mon[111335]: pgmap v7798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2447905453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:31 smithi093 ceph-mon[118902]: pgmap v7798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2447905453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:31.234Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:31.235Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:31 smithi136 ceph-mon[113531]: pgmap v7798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2447905453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:31.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:31.480746+0000) 2023-08-28T01:13:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:32.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:32.480940+0000) 2023-08-28T01:13:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:32.680Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:32.681Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:33 smithi136 ceph-mon[113531]: pgmap v7799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:33.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:33.481203+0000) 2023-08-28T01:13:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:33 smithi093 ceph-mon[118902]: pgmap v7799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:33 smithi093 ceph-mon[111335]: pgmap v7799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2681737960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:34.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:34.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:34.481445+0000) 2023-08-28T01:13:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2681737960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2681737960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:35 smithi136 ceph-mon[113531]: pgmap v7800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:35.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:35.481661+0000) 2023-08-28T01:13:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:35 smithi093 ceph-mon[111335]: pgmap v7800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:35 smithi093 ceph-mon[118902]: pgmap v7800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:13:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:13:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:13:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1292710101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:36.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:36.481914+0000) 2023-08-28T01:13:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1292710101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1292710101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:37 smithi136 ceph-mon[113531]: pgmap v7801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:37.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:37.482142+0000) 2023-08-28T01:13:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:37 smithi093 ceph-mon[111335]: pgmap v7801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:37 smithi093 ceph-mon[118902]: pgmap v7801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:38.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:38.482396+0000) 2023-08-28T01:13:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:39.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:39.482653+0000) 2023-08-28T01:13:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:39 smithi136 ceph-mon[113531]: pgmap v7802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/197980984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:39 smithi093 ceph-mon[111335]: pgmap v7802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/197980984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:39 smithi093 ceph-mon[118902]: pgmap v7802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/197980984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:40.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:40.482882+0000) 2023-08-28T01:13:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:41 smithi093 ceph-mon[111335]: pgmap v7803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1766871043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:41 smithi093 ceph-mon[118902]: pgmap v7803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1766871043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:41.235Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:41.236Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:41 smithi136 ceph-mon[113531]: pgmap v7803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1766871043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:41.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:41.483157+0000) 2023-08-28T01:13:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:42.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:42.483456+0000) 2023-08-28T01:13:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:42.682Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:42.683Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:43 smithi136 ceph-mon[113531]: pgmap v7804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2418380895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:43.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:43.483670+0000) 2023-08-28T01:13:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:43 smithi093 ceph-mon[111335]: pgmap v7804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2418380895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:43 smithi093 ceph-mon[118902]: pgmap v7804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2418380895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:44.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:44.483922+0000) 2023-08-28T01:13:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:45 smithi136 ceph-mon[113531]: pgmap v7805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:45.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:45.484205+0000) 2023-08-28T01:13:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:45 smithi093 ceph-mon[111335]: pgmap v7805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:45 smithi093 ceph-mon[118902]: pgmap v7805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:13:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:13:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:13:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2235445477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:46.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:46.484423+0000) 2023-08-28T01:13:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2235445477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2235445477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:47 smithi136 ceph-mon[113531]: pgmap v7806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:47.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:47.484680+0000) 2023-08-28T01:13:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:47 smithi093 ceph-mon[111335]: pgmap v7806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:47 smithi093 ceph-mon[118902]: pgmap v7806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3677763258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:48.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:48.484959+0000) 2023-08-28T01:13:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3677763258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3677763258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:49 smithi136 ceph-mon[113531]: pgmap v7807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:49.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:49.485192+0000) 2023-08-28T01:13:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:49 smithi093 ceph-mon[111335]: pgmap v7807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:49 smithi093 ceph-mon[118902]: pgmap v7807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:50.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:50.485445+0000) 2023-08-28T01:13:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:51 smithi093 ceph-mon[111335]: pgmap v7808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2537793778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:51 smithi093 ceph-mon[118902]: pgmap v7808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2537793778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:51.236Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:51.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:51.237Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:51 smithi136 ceph-mon[113531]: pgmap v7808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2537793778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:51.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:51.485729+0000) 2023-08-28T01:13:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:52.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:52.485997+0000) 2023-08-28T01:13:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:52.682Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:13:52.683Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:13:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:53 smithi136 ceph-mon[113531]: pgmap v7809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/577402281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:53.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:53.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:53.486297+0000) 2023-08-28T01:13:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:53 smithi093 ceph-mon[111335]: pgmap v7809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/577402281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:53 smithi093 ceph-mon[118902]: pgmap v7809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/577402281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:54.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:54.486579+0000) 2023-08-28T01:13:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:13:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:55 smithi136 ceph-mon[113531]: pgmap v7810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2694141928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:55.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:55.486859+0000) 2023-08-28T01:13:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:55 smithi093 ceph-mon[111335]: pgmap v7810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2694141928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:55.742 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:13:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:13:55] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:13:55.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:55 smithi093 ceph-mon[118902]: pgmap v7810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:55.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2694141928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:56.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:56.487147+0000) 2023-08-28T01:13:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:57 smithi136 ceph-mon[113531]: pgmap v7811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:57 smithi093 ceph-mon[111335]: pgmap v7811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:57 smithi093 ceph-mon[118902]: pgmap v7811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:57.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:57.487398+0000) 2023-08-28T01:13:58.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3923158870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:58.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3923158870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:58.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3923158870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:13:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:58.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:58.487699+0000) 2023-08-28T01:13:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:13:59 smithi136 ceph-mon[113531]: pgmap v7812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:13:59 smithi093 ceph-mon[111335]: pgmap v7812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:13:59 smithi093 ceph-mon[118902]: pgmap v7812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:13:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:13:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:13:59.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:13:59.487939+0000) 2023-08-28T01:14:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2933864675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2933864675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2933864675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:00.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:00.488187+0000) 2023-08-28T01:14:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:01 smithi136 ceph-mon[113531]: pgmap v7813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:01 smithi093 ceph-mon[111335]: pgmap v7813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:01 smithi093 ceph-mon[118902]: pgmap v7813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:01.237Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:01.238Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:01.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:01.488403+0000) 2023-08-28T01:14:02.682 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4123665226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:02.683 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4123665226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4123665226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:02.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:02.488578+0000) 2023-08-28T01:14:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:02.683Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:02.684Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:03 smithi136 ceph-mon[113531]: pgmap v7814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:03.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:03.488800+0000) 2023-08-28T01:14:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:03 smithi093 ceph-mon[111335]: pgmap v7814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:03 smithi093 ceph-mon[118902]: pgmap v7814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:04.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:04.489084+0000) 2023-08-28T01:14:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:05.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:05.489278+0000) 2023-08-28T01:14:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:05 smithi136 ceph-mon[113531]: pgmap v7815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/679572862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:14:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:14:05] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:14:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:05 smithi093 ceph-mon[111335]: pgmap v7815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/679572862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:05 smithi093 ceph-mon[118902]: pgmap v7815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/679572862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:06.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:06.489470+0000) 2023-08-28T01:14:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:07 smithi136 ceph-mon[113531]: pgmap v7816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2435122608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:07 smithi093 ceph-mon[111335]: pgmap v7816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2435122608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:07 smithi093 ceph-mon[118902]: pgmap v7816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2435122608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:07.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:07.489733+0000) 2023-08-28T01:14:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:08.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:08.489918+0000) 2023-08-28T01:14:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:09.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:09.490106+0000) 2023-08-28T01:14:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:09 smithi136 ceph-mon[113531]: pgmap v7817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:14:09.701 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:09 smithi093 ceph-mon[111335]: pgmap v7817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:09.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:09.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:14:09.702 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:09 smithi093 ceph-mon[118902]: pgmap v7817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:09.703 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:09.704 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:14:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2891143251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:10.555 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2891143251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:10.556 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2891143251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:10.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:10.490243+0000) 2023-08-28T01:14:11.363 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:11 smithi136 ceph-mon[113531]: pgmap v7818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:11.363 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:11.363 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-mon[111335]: pgmap v7818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-mon[118902]: pgmap v7818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:11.237Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:11.239Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:11.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:11.490421+0000) 2023-08-28T01:14:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:12.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:12.490630+0000) 2023-08-28T01:14:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2202759501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:14:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:14:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:12 smithi136 ceph-mon[113531]: pgmap v7819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:12.684Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:12.685Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2202759501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:14:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:14:12.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[111335]: pgmap v7819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2202759501' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:14:12.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:12 smithi093 ceph-mon[118902]: pgmap v7819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:13.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:13.490825+0000) 2023-08-28T01:14:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1836879971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1836879971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:14.490 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1836879971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:14.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:14.491096+0000) 2023-08-28T01:14:15.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:15 smithi093 ceph-mon[111335]: pgmap v7820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:15 smithi093 ceph-mon[118902]: pgmap v7820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:14:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:14:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:14:15.490 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:15 smithi136 ceph-mon[113531]: pgmap v7820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:15.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:15.491323+0000) 2023-08-28T01:14:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:16.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:16.491549+0000) 2023-08-28T01:14:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:17 smithi136 ceph-mon[113531]: pgmap v7821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2449724644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:17 smithi093 ceph-mon[111335]: pgmap v7821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2449724644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:17 smithi093 ceph-mon[118902]: pgmap v7821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2449724644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:17.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:17.491797+0000) 2023-08-28T01:14:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:18.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:18.492070+0000) 2023-08-28T01:14:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:19 smithi136 ceph-mon[113531]: pgmap v7822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/144248976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:19.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:19.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:19.492326+0000) 2023-08-28T01:14:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:19 smithi093 ceph-mon[111335]: pgmap v7822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:19.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/144248976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:19 smithi093 ceph-mon[118902]: pgmap v7822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:19.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/144248976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:20.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:20.492560+0000) 2023-08-28T01:14:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:21 smithi093 ceph-mon[111335]: pgmap v7823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:21 smithi093 ceph-mon[118902]: pgmap v7823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:21.238Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:21.239Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:21 smithi136 ceph-mon[113531]: pgmap v7823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:21.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:21.492815+0000) 2023-08-28T01:14:22.684 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2060970099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:22.685 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2060970099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2060970099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:22.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:22.493079+0000) 2023-08-28T01:14:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:22.685Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:22.686Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:23 smithi136 ceph-mon[113531]: pgmap v7824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:23.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:23.493318+0000) 2023-08-28T01:14:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:23 smithi093 ceph-mon[111335]: pgmap v7824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:23 smithi093 ceph-mon[118902]: pgmap v7824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1526233095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:24.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:24.493568+0000) 2023-08-28T01:14:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1526233095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1526233095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:25 smithi136 ceph-mon[113531]: pgmap v7825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:25.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:25.493785+0000) 2023-08-28T01:14:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:25 smithi093 ceph-mon[111335]: pgmap v7825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:25 smithi093 ceph-mon[118902]: pgmap v7825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:14:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:14:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:14:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3080893735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:26.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:26.493984+0000) 2023-08-28T01:14:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3080893735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:26.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3080893735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:27 smithi136 ceph-mon[113531]: pgmap v7826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:27.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:27.494199+0000) 2023-08-28T01:14:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:27 smithi093 ceph-mon[111335]: pgmap v7826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:27 smithi093 ceph-mon[118902]: pgmap v7826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:28.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:28.494414+0000) 2023-08-28T01:14:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:29 smithi136 ceph-mon[113531]: pgmap v7827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1892010696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:29.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:29.494649+0000) 2023-08-28T01:14:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:29 smithi093 ceph-mon[111335]: pgmap v7827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1892010696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:29 smithi093 ceph-mon[118902]: pgmap v7827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1892010696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:30.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:30.494891+0000) 2023-08-28T01:14:31.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:31 smithi093 ceph-mon[118902]: pgmap v7828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1349063947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:31 smithi093 ceph-mon[111335]: pgmap v7828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1349063947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:31.238Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:31.240Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:31 smithi136 ceph-mon[113531]: pgmap v7828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1349063947' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:31.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:31.495153+0000) 2023-08-28T01:14:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:32.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:32.495398+0000) 2023-08-28T01:14:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:32.686Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:32.687Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:33 smithi136 ceph-mon[113531]: pgmap v7829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3298075605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:33.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:33.495665+0000) 2023-08-28T01:14:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:33 smithi093 ceph-mon[111335]: pgmap v7829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3298075605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:33 smithi093 ceph-mon[118902]: pgmap v7829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3298075605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:34.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:34.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:34.495940+0000) 2023-08-28T01:14:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:35 smithi136 ceph-mon[113531]: pgmap v7830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:35.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:35.496181+0000) 2023-08-28T01:14:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:35 smithi093 ceph-mon[111335]: pgmap v7830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:35 smithi093 ceph-mon[118902]: pgmap v7830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:14:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:14:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:14:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1573555468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:36.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:36.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:36.496449+0000) 2023-08-28T01:14:36.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1573555468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1573555468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:37 smithi136 ceph-mon[113531]: pgmap v7831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:37.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:37.496712+0000) 2023-08-28T01:14:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:37 smithi093 ceph-mon[111335]: pgmap v7831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:37 smithi093 ceph-mon[118902]: pgmap v7831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:38.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/923855871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:38.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:38.496989+0000) 2023-08-28T01:14:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/923855871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/923855871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:39 smithi136 ceph-mon[113531]: pgmap v7832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:39.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:39.497242+0000) 2023-08-28T01:14:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:39 smithi093 ceph-mon[111335]: pgmap v7832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:39 smithi093 ceph-mon[118902]: pgmap v7832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:40.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:40.497464+0000) 2023-08-28T01:14:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:41 smithi093 ceph-mon[111335]: pgmap v7833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3822985589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:41 smithi093 ceph-mon[118902]: pgmap v7833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3822985589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:41.239Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:41.240Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:41 smithi136 ceph-mon[113531]: pgmap v7833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3822985589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:41.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:41.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:41.497724+0000) 2023-08-28T01:14:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:42.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:42.497989+0000) 2023-08-28T01:14:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:42.686Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:42.688Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:43 smithi136 ceph-mon[113531]: pgmap v7834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/627857426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:43.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:43.498257+0000) 2023-08-28T01:14:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:43 smithi093 ceph-mon[111335]: pgmap v7834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/627857426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:43 smithi093 ceph-mon[118902]: pgmap v7834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/627857426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:44.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:44.498504+0000) 2023-08-28T01:14:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:45 smithi136 ceph-mon[113531]: pgmap v7835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2149321918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:45.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:45.498754+0000) 2023-08-28T01:14:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:14:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:14:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:14:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:45 smithi093 ceph-mon[111335]: pgmap v7835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2149321918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:45 smithi093 ceph-mon[118902]: pgmap v7835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2149321918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:46.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:46.499009+0000) 2023-08-28T01:14:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:47 smithi136 ceph-mon[113531]: pgmap v7836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:47 smithi093 ceph-mon[111335]: pgmap v7836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:47 smithi093 ceph-mon[118902]: pgmap v7836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:47.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:47.499294+0000) 2023-08-28T01:14:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1500120988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1500120988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1500120988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:48.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:48.499547+0000) 2023-08-28T01:14:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:49 smithi136 ceph-mon[113531]: pgmap v7837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:49 smithi093 ceph-mon[111335]: pgmap v7837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:49 smithi093 ceph-mon[118902]: pgmap v7837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:49.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:49.499845+0000) 2023-08-28T01:14:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/820377670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/820377670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/820377670' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:50.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:50.500068+0000) 2023-08-28T01:14:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:51 smithi136 ceph-mon[113531]: pgmap v7838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:51 smithi093 ceph-mon[111335]: pgmap v7838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:51.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:51 smithi093 ceph-mon[118902]: pgmap v7838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:51.240Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:51.241Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:51.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:51.500305+0000) 2023-08-28T01:14:52.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:52.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:52.500532+0000) 2023-08-28T01:14:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:52.687Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:14:52.687Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:14:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:53 smithi136 ceph-mon[113531]: pgmap v7839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/159632968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:53.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:53.500757+0000) 2023-08-28T01:14:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:53 smithi093 ceph-mon[111335]: pgmap v7839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/159632968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:53 smithi093 ceph-mon[118902]: pgmap v7839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/159632968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:54.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:54.501007+0000) 2023-08-28T01:14:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:14:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:55 smithi136 ceph-mon[113531]: pgmap v7840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:55.719 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3440393437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:55.719 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:55.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:55.501255+0000) 2023-08-28T01:14:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:55 smithi093 ceph-mon[111335]: pgmap v7840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3440393437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:55 smithi093 ceph-mon[118902]: pgmap v7840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3440393437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:14:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:14:55] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:14:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:56.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:56.501480+0000) 2023-08-28T01:14:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:57 smithi136 ceph-mon[113531]: pgmap v7841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3999729317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:57 smithi093 ceph-mon[111335]: pgmap v7841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3999729317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:57 smithi093 ceph-mon[118902]: pgmap v7841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3999729317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:14:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:57.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:57.501745+0000) 2023-08-28T01:14:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:58.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:58.501983+0000) 2023-08-28T01:14:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:14:59 smithi136 ceph-mon[113531]: pgmap v7842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:14:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:14:59.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:14:59.502233+0000) 2023-08-28T01:14:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:14:59 smithi093 ceph-mon[111335]: pgmap v7842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:14:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:14:59 smithi093 ceph-mon[118902]: pgmap v7842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2414085165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:00.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:00.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:00.502449+0000) 2023-08-28T01:15:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2414085165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2414085165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:01 smithi136 ceph-mon[113531]: pgmap v7843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:01.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:01.502637+0000) 2023-08-28T01:15:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:01 smithi093 ceph-mon[111335]: pgmap v7843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:01 smithi093 ceph-mon[118902]: pgmap v7843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:01.241Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:01.242Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:02.687 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/161932734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:02.687 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/161932734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/161932734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:02.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:02.502879+0000) 2023-08-28T01:15:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:02.687Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:02.689Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:03 smithi136 ceph-mon[113531]: pgmap v7844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:03.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:03.503196+0000) 2023-08-28T01:15:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:03 smithi093 ceph-mon[111335]: pgmap v7844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:03 smithi093 ceph-mon[118902]: pgmap v7844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2370787984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:04.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:04.503483+0000) 2023-08-28T01:15:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2370787984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2370787984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:05 smithi136 ceph-mon[113531]: pgmap v7845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:05.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:05.503740+0000) 2023-08-28T01:15:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:05 smithi093 ceph-mon[111335]: pgmap v7845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:05 smithi093 ceph-mon[118902]: pgmap v7845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:15:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:15:05] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:15:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:06.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:06.504065+0000) 2023-08-28T01:15:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:07 smithi136 ceph-mon[113531]: pgmap v7846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/177298789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:07 smithi093 ceph-mon[111335]: pgmap v7846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/177298789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:07 smithi093 ceph-mon[118902]: pgmap v7846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/177298789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:07.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:07.504317+0000) 2023-08-28T01:15:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:08.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:08.504540+0000) 2023-08-28T01:15:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:09 smithi136 ceph-mon[113531]: pgmap v7847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:09.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3739190585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:09.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:09.504826+0000) 2023-08-28T01:15:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:09 smithi093 ceph-mon[111335]: pgmap v7847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3739190585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:09 smithi093 ceph-mon[118902]: pgmap v7847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3739190585' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:10.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:10.505147+0000) 2023-08-28T01:15:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:11 smithi136 ceph-mon[113531]: pgmap v7848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:11.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:11.505466+0000) 2023-08-28T01:15:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:11 smithi093 ceph-mon[111335]: pgmap v7848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:11 smithi093 ceph-mon[118902]: pgmap v7848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:11.242Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:11.243Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:12.688 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2641638927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:12.688 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:15:12.688 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2641638927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:12.688 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:15:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2641638927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:12.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:15:12.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:12.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:12.505665+0000) 2023-08-28T01:15:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:12.688Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:12.689Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:13.505 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:13 smithi136 ceph-mon[113531]: pgmap v7849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:13.680 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:13 smithi093 ceph-mon[111335]: pgmap v7849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:13.680 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:13 smithi093 ceph-mon[118902]: pgmap v7849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:13.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:13.505805+0000) 2023-08-28T01:15:14.421 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2729083985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:14.421 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:14.421 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:14.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:14.505966+0000) 2023-08-28T01:15:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2729083985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2729083985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.638 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:15:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:15:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:15.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:15.506208+0000) 2023-08-28T01:15:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:15 smithi136 ceph-mon[113531]: pgmap v7850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:15:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:15:15.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[111335]: pgmap v7850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:15:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:15:15.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[118902]: pgmap v7850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:15:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:15:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:15:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/435600675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:16.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:16.506458+0000) 2023-08-28T01:15:16.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/435600675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:16.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/435600675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:17 smithi136 ceph-mon[113531]: pgmap v7851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:17.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:17.506701+0000) 2023-08-28T01:15:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:17 smithi093 ceph-mon[111335]: pgmap v7851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:17 smithi093 ceph-mon[118902]: pgmap v7851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/433356298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:18.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:18.506967+0000) 2023-08-28T01:15:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/433356298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/433356298' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:19 smithi136 ceph-mon[113531]: pgmap v7852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:19.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:19.507249+0000) 2023-08-28T01:15:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:19 smithi093 ceph-mon[111335]: pgmap v7852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:19 smithi093 ceph-mon[118902]: pgmap v7852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:20 smithi136 ceph-mon[113531]: pgmap v7853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:20.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:20.507483+0000) 2023-08-28T01:15:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:20 smithi093 ceph-mon[111335]: pgmap v7853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:20.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:20 smithi093 ceph-mon[118902]: pgmap v7853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:21.653 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:21.243Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:21.653 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:21.244Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3199524382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:21.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:21.507689+0000) 2023-08-28T01:15:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3199524382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3199524382' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:22 smithi136 ceph-mon[113531]: pgmap v7854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:22.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:22.507880+0000) 2023-08-28T01:15:22.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:22 smithi093 ceph-mon[111335]: pgmap v7854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:22 smithi093 ceph-mon[118902]: pgmap v7854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:22.688Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:22.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:22.689Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1791546859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:23.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:23.508048+0000) 2023-08-28T01:15:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1791546859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1791546859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:24 smithi136 ceph-mon[113531]: pgmap v7855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:24.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:24.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:24.508222+0000) 2023-08-28T01:15:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:24 smithi093 ceph-mon[111335]: pgmap v7855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:24 smithi093 ceph-mon[118902]: pgmap v7855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:25.665 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:15:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:15:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:15:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3458263260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:25.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:25.508388+0000) 2023-08-28T01:15:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3458263260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:25.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3458263260' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:26.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:26.508556+0000) 2023-08-28T01:15:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:27 smithi136 ceph-mon[113531]: pgmap v7856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:27 smithi093 ceph-mon[111335]: pgmap v7856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:27 smithi093 ceph-mon[118902]: pgmap v7856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:27.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:27.508764+0000) 2023-08-28T01:15:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002564681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002564681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002564681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:28.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:28.508951+0000) 2023-08-28T01:15:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:29 smithi136 ceph-mon[113531]: pgmap v7857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:29 smithi093 ceph-mon[111335]: pgmap v7857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:29 smithi093 ceph-mon[118902]: pgmap v7857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:29.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:29.509118+0000) 2023-08-28T01:15:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:30.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:30.509264+0000) 2023-08-28T01:15:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:31 smithi136 ceph-mon[113531]: pgmap v7858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:31.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2978799426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:31.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:31.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:31.509433+0000) 2023-08-28T01:15:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:31 smithi093 ceph-mon[111335]: pgmap v7858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2978799426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:31 smithi093 ceph-mon[118902]: pgmap v7858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2978799426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:31.244Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:31.245Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:32.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:32.509606+0000) 2023-08-28T01:15:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:32.689Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:32.690Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:33 smithi136 ceph-mon[113531]: pgmap v7859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3387750489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:33.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:33.509792+0000) 2023-08-28T01:15:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:33 smithi093 ceph-mon[111335]: pgmap v7859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3387750489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:33 smithi093 ceph-mon[118902]: pgmap v7859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3387750489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:34.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:34.509996+0000) 2023-08-28T01:15:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:35 smithi136 ceph-mon[113531]: pgmap v7860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1087401144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:35.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:35.510174+0000) 2023-08-28T01:15:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:35 smithi093 ceph-mon[111335]: pgmap v7860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1087401144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:15:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:15:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:15:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:35 smithi093 ceph-mon[118902]: pgmap v7860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1087401144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:36.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:36.510377+0000) 2023-08-28T01:15:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:37 smithi136 ceph-mon[113531]: pgmap v7861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:37 smithi093 ceph-mon[111335]: pgmap v7861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:37 smithi093 ceph-mon[118902]: pgmap v7861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:37.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:37.510628+0000) 2023-08-28T01:15:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3879796935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3879796935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3879796935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:38.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:38.510839+0000) 2023-08-28T01:15:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:39 smithi136 ceph-mon[113531]: pgmap v7862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:39 smithi093 ceph-mon[111335]: pgmap v7862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:39 smithi093 ceph-mon[118902]: pgmap v7862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:39.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:39.511082+0000) 2023-08-28T01:15:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4209486320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:40.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4209486320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:40.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4209486320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:40.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:40.511232+0000) 2023-08-28T01:15:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:41 smithi136 ceph-mon[113531]: pgmap v7863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:41 smithi093 ceph-mon[111335]: pgmap v7863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:41 smithi093 ceph-mon[118902]: pgmap v7863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:41.245Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:41.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:41.246Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:41.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:41.511423+0000) 2023-08-28T01:15:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:42.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:42.511663+0000) 2023-08-28T01:15:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:42.689Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:42.690Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:43 smithi136 ceph-mon[113531]: pgmap v7864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1735196928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:43.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:43.511917+0000) 2023-08-28T01:15:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:43 smithi093 ceph-mon[111335]: pgmap v7864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1735196928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:43 smithi093 ceph-mon[118902]: pgmap v7864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1735196928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:44.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:44.512115+0000) 2023-08-28T01:15:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:45 smithi136 ceph-mon[113531]: pgmap v7865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:45.743 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4000353370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:45.747 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:45.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:45.512282+0000) 2023-08-28T01:15:45.747 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:15:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:15:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:15:45.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:45 smithi093 ceph-mon[111335]: pgmap v7865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:45.748 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4000353370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:45.748 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:45 smithi093 ceph-mon[118902]: pgmap v7865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:45.748 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4000353370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:46.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:46.512507+0000) 2023-08-28T01:15:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:47 smithi136 ceph-mon[113531]: pgmap v7866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3822564356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:47 smithi093 ceph-mon[111335]: pgmap v7866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3822564356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:47 smithi093 ceph-mon[118902]: pgmap v7866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3822564356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:47.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:47.512743+0000) 2023-08-28T01:15:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:48.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:48.512996+0000) 2023-08-28T01:15:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:49 smithi136 ceph-mon[113531]: pgmap v7867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:49.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:49.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:49.513163+0000) 2023-08-28T01:15:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:49 smithi093 ceph-mon[111335]: pgmap v7867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:49 smithi093 ceph-mon[118902]: pgmap v7867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1832789854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:50.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:50.513329+0000) 2023-08-28T01:15:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1832789854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1832789854' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:51 smithi136 ceph-mon[113531]: pgmap v7868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:51.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:51.513553+0000) 2023-08-28T01:15:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:51 smithi093 ceph-mon[111335]: pgmap v7868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:51 smithi093 ceph-mon[118902]: pgmap v7868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:51.246Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:51.247Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:52.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12416069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:52.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:52.513793+0000) 2023-08-28T01:15:52.689 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12416069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:52.689 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12416069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:52.690Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:15:52.691Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:15:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:53 smithi136 ceph-mon[113531]: pgmap v7869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:53.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:53.514011+0000) 2023-08-28T01:15:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:53 smithi093 ceph-mon[111335]: pgmap v7869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:53 smithi093 ceph-mon[118902]: pgmap v7869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/227264937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:54.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:54.514307+0000) 2023-08-28T01:15:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/227264937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:15:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/227264937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:55 smithi136 ceph-mon[113531]: pgmap v7870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:55.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:55.514537+0000) 2023-08-28T01:15:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:55 smithi093 ceph-mon[111335]: pgmap v7870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:55 smithi093 ceph-mon[118902]: pgmap v7870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:15:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:15:55] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:15:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:56.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:56.514734+0000) 2023-08-28T01:15:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:57 smithi136 ceph-mon[113531]: pgmap v7871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1890853642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:57 smithi093 ceph-mon[118902]: pgmap v7871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1890853642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:57 smithi093 ceph-mon[111335]: pgmap v7871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1890853642' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:57.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:57.515001+0000) 2023-08-28T01:15:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:58.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:58.515263+0000) 2023-08-28T01:15:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:59 smithi136 ceph-mon[113531]: pgmap v7872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:15:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1880486491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:15:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:15:59.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:15:59.515495+0000) 2023-08-28T01:15:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:59 smithi093 ceph-mon[111335]: pgmap v7872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:15:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1880486491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:15:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:59 smithi093 ceph-mon[118902]: pgmap v7872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:15:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:15:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1880486491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:00.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:00.515753+0000) 2023-08-28T01:16:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:01 smithi136 ceph-mon[113531]: pgmap v7873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:01.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:01.515945+0000) 2023-08-28T01:16:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:01 smithi093 ceph-mon[111335]: pgmap v7873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:01 smithi093 ceph-mon[118902]: pgmap v7873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:01.246Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:01.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/778266211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:02.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:02.516186+0000) 2023-08-28T01:16:02.690 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/778266211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:02.690 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/778266211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:02.691Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:02.692Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:03 smithi136 ceph-mon[113531]: pgmap v7874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:03.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:03.516432+0000) 2023-08-28T01:16:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:03 smithi093 ceph-mon[111335]: pgmap v7874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:03 smithi093 ceph-mon[118902]: pgmap v7874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1923374077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:04.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:04.516708+0000) 2023-08-28T01:16:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1923374077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:04.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1923374077' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:05 smithi136 ceph-mon[113531]: pgmap v7875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:05.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:05.516947+0000) 2023-08-28T01:16:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:05 smithi093 ceph-mon[111335]: pgmap v7875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:16:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:16:05] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:16:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:05 smithi093 ceph-mon[118902]: pgmap v7875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1283922757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:06.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:06.517202+0000) 2023-08-28T01:16:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1283922757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1283922757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:07 smithi136 ceph-mon[113531]: pgmap v7876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:07.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:07.517414+0000) 2023-08-28T01:16:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:07 smithi093 ceph-mon[111335]: pgmap v7876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:07 smithi093 ceph-mon[118902]: pgmap v7876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:08.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:08.517656+0000) 2023-08-28T01:16:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:09 smithi136 ceph-mon[113531]: pgmap v7877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/283684006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:09.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:09.517964+0000) 2023-08-28T01:16:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:09 smithi093 ceph-mon[111335]: pgmap v7877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/283684006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:09 smithi093 ceph-mon[118902]: pgmap v7877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/283684006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:10.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:10.518188+0000) 2023-08-28T01:16:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:11 smithi136 ceph-mon[113531]: pgmap v7878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3296213410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:11.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:11.518431+0000) 2023-08-28T01:16:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:11 smithi093 ceph-mon[111335]: pgmap v7878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3296213410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:11 smithi093 ceph-mon[118902]: pgmap v7878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3296213410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:11.247Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:11.248Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:12.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:12.518671+0000) 2023-08-28T01:16:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:12.692Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:12.693Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:13 smithi136 ceph-mon[113531]: pgmap v7879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1344164007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:13.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:13.518889+0000) 2023-08-28T01:16:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:13 smithi093 ceph-mon[111335]: pgmap v7879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1344164007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:13 smithi093 ceph-mon[118902]: pgmap v7879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1344164007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:14.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:14.519096+0000) 2023-08-28T01:16:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:15 smithi136 ceph-mon[113531]: pgmap v7880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:16:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:15.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:15.519263+0000) 2023-08-28T01:16:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:15 smithi093 ceph-mon[118902]: pgmap v7880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:16:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:15 smithi093 ceph-mon[111335]: pgmap v7880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:16:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:16:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:16:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:16:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3246033651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:16.594 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3246033651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:16.595 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3246033651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:16.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:16.519471+0000) 2023-08-28T01:16:17.372 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:17 smithi136 ceph-mon[113531]: pgmap v7881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:17.372 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:17.372 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:17.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:17.519635+0000) 2023-08-28T01:16:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:17 smithi093 ceph-mon[111335]: pgmap v7881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:17 smithi093 ceph-mon[118902]: pgmap v7881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:18.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:18.519881+0000) 2023-08-28T01:16:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:16:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:16:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2226620458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2226620458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:16:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2226620458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:19 smithi136 ceph-mon[113531]: pgmap v7882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:19.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:19.520146+0000) 2023-08-28T01:16:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:19 smithi093 ceph-mon[118902]: pgmap v7882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:19 smithi093 ceph-mon[111335]: pgmap v7882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1174715695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:20.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:20.520380+0000) 2023-08-28T01:16:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1174715695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1174715695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:21.587 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:21.248Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:21.587 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:21.249Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:21 smithi136 ceph-mon[113531]: pgmap v7883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:21.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:21.520634+0000) 2023-08-28T01:16:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:21 smithi093 ceph-mon[111335]: pgmap v7883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:21 smithi093 ceph-mon[118902]: pgmap v7883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:22.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:22.520865+0000) 2023-08-28T01:16:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:22.692Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:22.693Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:23 smithi136 ceph-mon[113531]: pgmap v7884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1309593195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:23.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:23.521130+0000) 2023-08-28T01:16:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:23 smithi093 ceph-mon[111335]: pgmap v7884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1309593195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:23 smithi093 ceph-mon[118902]: pgmap v7884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:23.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1309593195' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:24.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:24.521330+0000) 2023-08-28T01:16:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:25 smithi093 ceph-mon[111335]: pgmap v7885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/540118957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:16:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:16:25] "GET /metrics HTTP/1.1" 200 239934 "" "Prometheus/2.43.0" 2023-08-28T01:16:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:25 smithi093 ceph-mon[118902]: pgmap v7885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/540118957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:25 smithi136 ceph-mon[113531]: pgmap v7885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/540118957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:25.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:25.521589+0000) 2023-08-28T01:16:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:26.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:26.521857+0000) 2023-08-28T01:16:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:27 smithi136 ceph-mon[113531]: pgmap v7886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:27 smithi093 ceph-mon[111335]: pgmap v7886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:27 smithi093 ceph-mon[118902]: pgmap v7886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:27.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:27.522128+0000) 2023-08-28T01:16:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3374935272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3374935272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3374935272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:28.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:28.522369+0000) 2023-08-28T01:16:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:29 smithi136 ceph-mon[113531]: pgmap v7887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:29 smithi093 ceph-mon[111335]: pgmap v7887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:29 smithi093 ceph-mon[118902]: pgmap v7887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:29.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:29.522632+0000) 2023-08-28T01:16:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1016146732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1016146732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:30.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1016146732' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:30.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:30.522899+0000) 2023-08-28T01:16:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:31 smithi136 ceph-mon[113531]: pgmap v7888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:31 smithi093 ceph-mon[111335]: pgmap v7888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:31 smithi093 ceph-mon[118902]: pgmap v7888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:31.249Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:31.251Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:31.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:31.523115+0000) 2023-08-28T01:16:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:32.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:32.523335+0000) 2023-08-28T01:16:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:32.693Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:32.694Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:33 smithi136 ceph-mon[113531]: pgmap v7889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2939262436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:33.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:33.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:33.523572+0000) 2023-08-28T01:16:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:33 smithi093 ceph-mon[111335]: pgmap v7889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2939262436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:33 smithi093 ceph-mon[118902]: pgmap v7889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2939262436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:34.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:34.523840+0000) 2023-08-28T01:16:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:35 smithi136 ceph-mon[113531]: pgmap v7890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2625925429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:35.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:35.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:35.524140+0000) 2023-08-28T01:16:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:35 smithi093 ceph-mon[111335]: pgmap v7890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2625925429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:16:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:16:35] "GET /metrics HTTP/1.1" 200 239934 "" "Prometheus/2.43.0" 2023-08-28T01:16:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:35 smithi093 ceph-mon[118902]: pgmap v7890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2625925429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:36.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:36.524389+0000) 2023-08-28T01:16:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:37 smithi136 ceph-mon[113531]: pgmap v7891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3309370186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:37 smithi093 ceph-mon[111335]: pgmap v7891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3309370186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:37 smithi093 ceph-mon[118902]: pgmap v7891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3309370186' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:37.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:37.524589+0000) 2023-08-28T01:16:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:38.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:38.524804+0000) 2023-08-28T01:16:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:39 smithi136 ceph-mon[113531]: pgmap v7892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:39.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:39.525027+0000) 2023-08-28T01:16:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:39 smithi093 ceph-mon[111335]: pgmap v7892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:39 smithi093 ceph-mon[118902]: pgmap v7892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2374245564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:40.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:40.525294+0000) 2023-08-28T01:16:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2374245564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2374245564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:41 smithi136 ceph-mon[113531]: pgmap v7893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:41.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:41.525538+0000) 2023-08-28T01:16:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:41 smithi093 ceph-mon[111335]: pgmap v7893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:41 smithi093 ceph-mon[118902]: pgmap v7893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:41.250Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:41.251Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2129620357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:42.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:42.525803+0000) 2023-08-28T01:16:42.693 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2129620357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:42.693 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2129620357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:42.694Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:42.695Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:43 smithi136 ceph-mon[113531]: pgmap v7894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:43.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:43.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:43.526085+0000) 2023-08-28T01:16:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:43 smithi093 ceph-mon[111335]: pgmap v7894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:43 smithi093 ceph-mon[118902]: pgmap v7894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2195697335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:44.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:44.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:44.526335+0000) 2023-08-28T01:16:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2195697335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:44.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2195697335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:45 smithi136 ceph-mon[113531]: pgmap v7895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:45.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:45.526558+0000) 2023-08-28T01:16:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:45 smithi093 ceph-mon[111335]: pgmap v7895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:16:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:16:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:16:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:45 smithi093 ceph-mon[118902]: pgmap v7895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:46.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:46.526816+0000) 2023-08-28T01:16:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:47 smithi136 ceph-mon[113531]: pgmap v7896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1587116485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:47 smithi093 ceph-mon[111335]: pgmap v7896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1587116485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:47 smithi093 ceph-mon[118902]: pgmap v7896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1587116485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:47.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:47.527102+0000) 2023-08-28T01:16:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:48.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:48.527335+0000) 2023-08-28T01:16:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:49.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:49.527606+0000) 2023-08-28T01:16:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:49 smithi136 ceph-mon[113531]: pgmap v7897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1501783540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:49 smithi093 ceph-mon[111335]: pgmap v7897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1501783540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:49 smithi093 ceph-mon[118902]: pgmap v7897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1501783540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:50.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:50.527858+0000) 2023-08-28T01:16:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:51 smithi136 ceph-mon[113531]: pgmap v7898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:51.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:51.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:51.528148+0000) 2023-08-28T01:16:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:51 smithi093 ceph-mon[111335]: pgmap v7898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:51 smithi093 ceph-mon[118902]: pgmap v7898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:51.251Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:51.251Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:52.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/874535359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:52.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:52.528382+0000) 2023-08-28T01:16:52.693 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/874535359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:52.694 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/874535359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:52.694Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:16:52.695Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:16:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:53 smithi136 ceph-mon[113531]: pgmap v7899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:53.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:53.528657+0000) 2023-08-28T01:16:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:53 smithi093 ceph-mon[111335]: pgmap v7899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:53 smithi093 ceph-mon[118902]: pgmap v7899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4101882970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:54.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:54.528916+0000) 2023-08-28T01:16:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4101882970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4101882970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:16:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:55 smithi136 ceph-mon[113531]: pgmap v7900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:55.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:55.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:55.529171+0000) 2023-08-28T01:16:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:55 smithi093 ceph-mon[111335]: pgmap v7900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:16:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:16:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:16:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:55 smithi093 ceph-mon[118902]: pgmap v7900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:56.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:56.529423+0000) 2023-08-28T01:16:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3065385702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:57 smithi136 ceph-mon[113531]: pgmap v7901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3065385702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:57 smithi093 ceph-mon[111335]: pgmap v7901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3065385702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:57 smithi093 ceph-mon[118902]: pgmap v7901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:57.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:57.529627+0000) 2023-08-28T01:16:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:58.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:58.529836+0000) 2023-08-28T01:16:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:59 smithi136 ceph-mon[113531]: pgmap v7902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:16:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/961895261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:16:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:16:59.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:16:59.530119+0000) 2023-08-28T01:16:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:59 smithi093 ceph-mon[111335]: pgmap v7902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:16:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/961895261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:16:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:59 smithi093 ceph-mon[118902]: pgmap v7902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:16:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:16:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/961895261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:00.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:00.530368+0000) 2023-08-28T01:17:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:01 smithi136 ceph-mon[113531]: pgmap v7903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3667619259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:01.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:01.530600+0000) 2023-08-28T01:17:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:01 smithi093 ceph-mon[111335]: pgmap v7903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3667619259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:01 smithi093 ceph-mon[118902]: pgmap v7903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3667619259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:01.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:01.251Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:01.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:01.252Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:02.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:02.530842+0000) 2023-08-28T01:17:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:02.695Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:02.696Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:03 smithi136 ceph-mon[113531]: pgmap v7904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:03.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:03.531158+0000) 2023-08-28T01:17:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:03 smithi093 ceph-mon[111335]: pgmap v7904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:03 smithi093 ceph-mon[118902]: pgmap v7904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3793558834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:04.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:04.531421+0000) 2023-08-28T01:17:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3793558834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3793558834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:05 smithi136 ceph-mon[113531]: pgmap v7905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:05.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:05.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:05.531673+0000) 2023-08-28T01:17:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:05 smithi093 ceph-mon[111335]: pgmap v7905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:05 smithi093 ceph-mon[118902]: pgmap v7905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:17:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:17:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:17:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4070665249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:06.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:06.531892+0000) 2023-08-28T01:17:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4070665249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4070665249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:07 smithi136 ceph-mon[113531]: pgmap v7906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:07.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:07.532137+0000) 2023-08-28T01:17:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:07 smithi093 ceph-mon[111335]: pgmap v7906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:07 smithi093 ceph-mon[118902]: pgmap v7906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3405639651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:08.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:08.532355+0000) 2023-08-28T01:17:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3405639651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3405639651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:09 smithi136 ceph-mon[113531]: pgmap v7907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:09.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:09.532598+0000) 2023-08-28T01:17:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:09 smithi093 ceph-mon[111335]: pgmap v7907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:09 smithi093 ceph-mon[118902]: pgmap v7907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:10.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:10.532855+0000) 2023-08-28T01:17:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:11 smithi136 ceph-mon[113531]: pgmap v7908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4151750259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:11.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:11.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:11.533163+0000) 2023-08-28T01:17:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:11 smithi093 ceph-mon[111335]: pgmap v7908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4151750259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:11 smithi093 ceph-mon[118902]: pgmap v7908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4151750259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:11.252Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:11.254Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:12.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:12.533532+0000) 2023-08-28T01:17:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:12.696Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:12.697Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:13 smithi136 ceph-mon[113531]: pgmap v7909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2755384865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:13.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:13.533812+0000) 2023-08-28T01:17:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:13 smithi093 ceph-mon[111335]: pgmap v7909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2755384865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:13 smithi093 ceph-mon[118902]: pgmap v7909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2755384865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:14.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:14.534102+0000) 2023-08-28T01:17:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:15 smithi136 ceph-mon[113531]: pgmap v7910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1286531060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:15.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:15.534347+0000) 2023-08-28T01:17:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:17:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:17:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:17:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:15 smithi093 ceph-mon[111335]: pgmap v7910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1286531060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:15 smithi093 ceph-mon[118902]: pgmap v7910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1286531060' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:16.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:16.534583+0000) 2023-08-28T01:17:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:17 smithi136 ceph-mon[113531]: pgmap v7911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:17 smithi093 ceph-mon[111335]: pgmap v7911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:17 smithi093 ceph-mon[118902]: pgmap v7911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:17.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:17.534837+0000) 2023-08-28T01:17:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2121887975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:17:18.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2121887975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:17:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2121887975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:17:18.805 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:18.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:18.535006+0000) 2023-08-28T01:17:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:19 smithi136 ceph-mon[113531]: pgmap v7912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:19 smithi093 ceph-mon[111335]: pgmap v7912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:19 smithi093 ceph-mon[118902]: pgmap v7912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:19.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:19.535232+0000) 2023-08-28T01:17:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:20.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:20.535405+0000) 2023-08-28T01:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2052397406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: pgmap v7913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2440M 2023-08-28T01:17:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2052397406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: pgmap v7913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2440M 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2052397406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: pgmap v7913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2440M 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:21.253Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:21.254Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:21.925 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:21.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:21.535632+0000) 2023-08-28T01:17:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:17:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:17:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:17:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:17:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:17:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:17:22.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:17:22.929 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:22.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:22.535847+0000) 2023-08-28T01:17:22.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:22 smithi093 ceph-mon[111335]: pgmap v7914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:22.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3365187498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:22.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:22 smithi093 ceph-mon[118902]: pgmap v7914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:22.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3365187498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:22.697Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:22.698Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:22 smithi136 ceph-mon[113531]: pgmap v7914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3365187498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:23.931 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:23.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:23.536094+0000) 2023-08-28T01:17:24.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:24.936 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:24.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:24.536363+0000) 2023-08-28T01:17:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:24 smithi136 ceph-mon[113531]: pgmap v7915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1473727148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:24 smithi093 ceph-mon[111335]: pgmap v7915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1473727148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:24 smithi093 ceph-mon[118902]: pgmap v7915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1473727148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:17:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:17:25] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:17:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:25.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:25.536630+0000) 2023-08-28T01:17:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:26.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:26.536808+0000) 2023-08-28T01:17:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:27 smithi136 ceph-mon[113531]: pgmap v7916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:27 smithi093 ceph-mon[111335]: pgmap v7916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:27 smithi093 ceph-mon[118902]: pgmap v7916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:27.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:27.537097+0000) 2023-08-28T01:17:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/775630259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/775630259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:28.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/775630259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:28.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:28.537379+0000) 2023-08-28T01:17:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:29 smithi136 ceph-mon[113531]: pgmap v7917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:29 smithi093 ceph-mon[111335]: pgmap v7917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:29 smithi093 ceph-mon[118902]: pgmap v7917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:29.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:29.537632+0000) 2023-08-28T01:17:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/573712414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/573712414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/573712414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:30.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:30.537897+0000) 2023-08-28T01:17:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:31 smithi093 ceph-mon[111335]: pgmap v7918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:31 smithi093 ceph-mon[118902]: pgmap v7918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:31.255Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:31.256Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:31.537 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:31 smithi136 ceph-mon[113531]: pgmap v7918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:31.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:31.538087+0000) 2023-08-28T01:17:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3603574429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3603574429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:32.537 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3603574429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:32.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:32.538346+0000) 2023-08-28T01:17:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:32.697Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:32.699Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:33 smithi093 ceph-mon[111335]: pgmap v7919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:33 smithi093 ceph-mon[118902]: pgmap v7919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:33.537 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:33 smithi136 ceph-mon[113531]: pgmap v7919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:33.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:33.538620+0000) 2023-08-28T01:17:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:34.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:34.538836+0000) 2023-08-28T01:17:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:35 smithi136 ceph-mon[113531]: pgmap v7920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3114313167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:35.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:35.539106+0000) 2023-08-28T01:17:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:35 smithi093 ceph-mon[111335]: pgmap v7920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3114313167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:17:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:17:35] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:17:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:35 smithi093 ceph-mon[118902]: pgmap v7920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:35.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3114313167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:36.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:36.539390+0000) 2023-08-28T01:17:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:37 smithi136 ceph-mon[113531]: pgmap v7921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2838862510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:37 smithi093 ceph-mon[111335]: pgmap v7921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2838862510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:37 smithi093 ceph-mon[118902]: pgmap v7921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2838862510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:37.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:37.539648+0000) 2023-08-28T01:17:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:38.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:38.539892+0000) 2023-08-28T01:17:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:39 smithi136 ceph-mon[113531]: pgmap v7922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:39.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1033177222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:39.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:39.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:39.540137+0000) 2023-08-28T01:17:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:39 smithi093 ceph-mon[111335]: pgmap v7922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1033177222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:39 smithi093 ceph-mon[118902]: pgmap v7922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:39.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1033177222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:40.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:40.540313+0000) 2023-08-28T01:17:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:41 smithi136 ceph-mon[113531]: pgmap v7923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:41.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:41.540582+0000) 2023-08-28T01:17:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:41 smithi093 ceph-mon[111335]: pgmap v7923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:41 smithi093 ceph-mon[118902]: pgmap v7923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:41.255Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:41.256Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1406150484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:42.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:42.540818+0000) 2023-08-28T01:17:42.698 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1406150484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:42.698 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1406150484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:42.699Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:42.700Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:43 smithi136 ceph-mon[113531]: pgmap v7924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:43.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:43.541009+0000) 2023-08-28T01:17:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:43 smithi093 ceph-mon[111335]: pgmap v7924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:43 smithi093 ceph-mon[118902]: pgmap v7924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1611525847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:44.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:44.541228+0000) 2023-08-28T01:17:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1611525847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:44.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1611525847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:45 smithi136 ceph-mon[113531]: pgmap v7925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:45.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:45.541484+0000) 2023-08-28T01:17:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:45 smithi093 ceph-mon[111335]: pgmap v7925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:45 smithi093 ceph-mon[118902]: pgmap v7925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:17:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:17:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:17:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:46.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:46.541671+0000) 2023-08-28T01:17:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2551778676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:47 smithi136 ceph-mon[113531]: pgmap v7926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2551778676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:47 smithi093 ceph-mon[111335]: pgmap v7926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2551778676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:47 smithi093 ceph-mon[118902]: pgmap v7926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:47.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:47.541925+0000) 2023-08-28T01:17:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:48.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:48.542125+0000) 2023-08-28T01:17:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:49 smithi136 ceph-mon[113531]: pgmap v7927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2723996372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:49.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:49.542370+0000) 2023-08-28T01:17:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:49 smithi093 ceph-mon[111335]: pgmap v7927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2723996372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:49 smithi093 ceph-mon[118902]: pgmap v7927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2723996372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:50.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:50.542625+0000) 2023-08-28T01:17:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:51 smithi136 ceph-mon[113531]: pgmap v7928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1264686521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:51.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:51.542897+0000) 2023-08-28T01:17:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:51 smithi093 ceph-mon[111335]: pgmap v7928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1264686521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:51 smithi093 ceph-mon[118902]: pgmap v7928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1264686521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:51.256Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:51.257Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:52.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:52.543132+0000) 2023-08-28T01:17:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:52.700Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:17:52.701Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:17:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:53 smithi136 ceph-mon[113531]: pgmap v7929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:53.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:53.543363+0000) 2023-08-28T01:17:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:53 smithi093 ceph-mon[111335]: pgmap v7929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:53 smithi093 ceph-mon[118902]: pgmap v7929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/340076401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:54.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:54.543608+0000) 2023-08-28T01:17:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/340076401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/340076401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:17:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:55 smithi136 ceph-mon[113531]: pgmap v7930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:55.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:55.543882+0000) 2023-08-28T01:17:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:55 smithi093 ceph-mon[111335]: pgmap v7930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:55 smithi093 ceph-mon[118902]: pgmap v7930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:17:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:17:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:17:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/822002968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:56.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:56.544156+0000) 2023-08-28T01:17:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/822002968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:56.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/822002968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:57 smithi136 ceph-mon[113531]: pgmap v7931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:57.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:57.544406+0000) 2023-08-28T01:17:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:57 smithi093 ceph-mon[111335]: pgmap v7931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:57 smithi093 ceph-mon[118902]: pgmap v7931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1097538818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:58.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:58.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:58.544640+0000) 2023-08-28T01:17:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1097538818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1097538818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:17:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:17:59 smithi136 ceph-mon[113531]: pgmap v7932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:59.712 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:17:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:17:59.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:17:59.544905+0000) 2023-08-28T01:17:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:17:59 smithi093 ceph-mon[111335]: pgmap v7932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:17:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:17:59 smithi093 ceph-mon[118902]: pgmap v7932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:00.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:00.545151+0000) 2023-08-28T01:18:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:01 smithi136 ceph-mon[113531]: pgmap v7933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3385619971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:01.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:01.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:01.545357+0000) 2023-08-28T01:18:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:01 smithi093 ceph-mon[111335]: pgmap v7933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3385619971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:01 smithi093 ceph-mon[118902]: pgmap v7933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3385619971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:01.257Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:01.257Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:02.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:02.545551+0000) 2023-08-28T01:18:03.576 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:02.701Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:03.790 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:02.702Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:03.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:03 smithi136 ceph-mon[113531]: pgmap v7934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:03.791 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/971195486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:03.791 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:03 smithi093 ceph-mon[118902]: pgmap v7934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:03.791 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/971195486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:03.791 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:03.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:03.545807+0000) 2023-08-28T01:18:03.792 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:03 smithi093 ceph-mon[111335]: pgmap v7934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:03.792 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/971195486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:04.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:04.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:04.546100+0000) 2023-08-28T01:18:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:05 smithi136 ceph-mon[113531]: pgmap v7935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/893848050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:05.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:05.546348+0000) 2023-08-28T01:18:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:05 smithi093 ceph-mon[111335]: pgmap v7935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/893848050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:05 smithi093 ceph-mon[118902]: pgmap v7935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/893848050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:18:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:18:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:18:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:06.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:06.546589+0000) 2023-08-28T01:18:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:07 smithi136 ceph-mon[113531]: pgmap v7936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:07 smithi093 ceph-mon[111335]: pgmap v7936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:07 smithi093 ceph-mon[118902]: pgmap v7936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:07.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:07.546872+0000) 2023-08-28T01:18:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3955850476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:08.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3955850476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:08.546 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3955850476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:08.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:08.547150+0000) 2023-08-28T01:18:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:09 smithi093 ceph-mon[111335]: pgmap v7937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:09 smithi093 ceph-mon[118902]: pgmap v7937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:09.546 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:09 smithi136 ceph-mon[113531]: pgmap v7937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:09.546 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:09.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:09.547443+0000) 2023-08-28T01:18:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3157854695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:10.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3157854695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:10.546 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3157854695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:10.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:10.547695+0000) 2023-08-28T01:18:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:11 smithi093 ceph-mon[118902]: pgmap v7938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:11 smithi093 ceph-mon[111335]: pgmap v7938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:11.258Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:11.259Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:11.546 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:11 smithi136 ceph-mon[113531]: pgmap v7938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:11.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:11.547931+0000) 2023-08-28T01:18:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:12.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:12.548142+0000) 2023-08-28T01:18:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:12.702Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:12.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:13 smithi136 ceph-mon[113531]: pgmap v7939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2187572252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:13.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:13.548415+0000) 2023-08-28T01:18:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:13 smithi093 ceph-mon[111335]: pgmap v7939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2187572252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:13 smithi093 ceph-mon[118902]: pgmap v7939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2187572252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:14.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:14.548686+0000) 2023-08-28T01:18:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:15 smithi136 ceph-mon[113531]: pgmap v7940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3028901119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:15.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:15.548932+0000) 2023-08-28T01:18:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:18:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:18:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:18:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:15 smithi093 ceph-mon[118902]: pgmap v7940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3028901119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:15 smithi093 ceph-mon[111335]: pgmap v7940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3028901119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:16.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:16.549187+0000) 2023-08-28T01:18:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:17 smithi136 ceph-mon[113531]: pgmap v7941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:17 smithi093 ceph-mon[111335]: pgmap v7941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:17 smithi093 ceph-mon[118902]: pgmap v7941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:17.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:17.549447+0000) 2023-08-28T01:18:18.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4056042795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:18.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4056042795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:18.548 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4056042795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:18.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:18.549709+0000) 2023-08-28T01:18:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:19 smithi093 ceph-mon[111335]: pgmap v7942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:19 smithi093 ceph-mon[118902]: pgmap v7942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:19.548 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:19 smithi136 ceph-mon[113531]: pgmap v7942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:19.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:19.549913+0000) 2023-08-28T01:18:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3536676991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3536676991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:20.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3536676991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:20.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:20.550174+0000) 2023-08-28T01:18:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:21 smithi093 ceph-mon[111335]: pgmap v7943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:18:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:21 smithi093 ceph-mon[118902]: pgmap v7943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:18:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:21.259Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:21.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:21.259Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:21.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:21 smithi136 ceph-mon[113531]: pgmap v7943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:21.549 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:18:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:21.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:21.550419+0000) 2023-08-28T01:18:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3232915145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3232915145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3232915145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:22.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:22.550588+0000) 2023-08-28T01:18:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:22.702Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:22.703Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:23.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:23 smithi136 ceph-mon[113531]: pgmap v7944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:23.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:23.359 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:23 smithi093 ceph-mon[118902]: pgmap v7944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:23 smithi093 ceph-mon[111335]: pgmap v7944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:23.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:23.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:23.550766+0000) 2023-08-28T01:18:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:24.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:24.551029+0000) 2023-08-28T01:18:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:18:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:18:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3975542346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:18:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3975542346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:18:24.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3975542346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:25.610 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:18:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:18:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:18:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:25 smithi136 ceph-mon[113531]: pgmap v7945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:25.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:25.551268+0000) 2023-08-28T01:18:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:25 smithi093 ceph-mon[111335]: pgmap v7945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:25.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:25 smithi093 ceph-mon[118902]: pgmap v7945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:26.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:26.551530+0000) 2023-08-28T01:18:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:27 smithi093 ceph-mon[111335]: pgmap v7946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2691967176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:27 smithi093 ceph-mon[118902]: pgmap v7946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2691967176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:27.550 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:27 smithi136 ceph-mon[113531]: pgmap v7946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:27.550 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2691967176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:27.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:27.551789+0000) 2023-08-28T01:18:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:28.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:28.552066+0000) 2023-08-28T01:18:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:29 smithi136 ceph-mon[113531]: pgmap v7947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/653738744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:29.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:29.552335+0000) 2023-08-28T01:18:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:29 smithi093 ceph-mon[111335]: pgmap v7947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/653738744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:29 smithi093 ceph-mon[118902]: pgmap v7947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/653738744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:30.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:30.552535+0000) 2023-08-28T01:18:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:31 smithi136 ceph-mon[113531]: pgmap v7948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:31.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:31.552784+0000) 2023-08-28T01:18:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:31 smithi093 ceph-mon[111335]: pgmap v7948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:31 smithi093 ceph-mon[118902]: pgmap v7948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:31.259Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:31.260Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1584147966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:32.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:32.553025+0000) 2023-08-28T01:18:32.702 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1584147966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:32.703 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1584147966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:32.704Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:32.705Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:33 smithi136 ceph-mon[113531]: pgmap v7949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:33.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:33.553326+0000) 2023-08-28T01:18:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:33 smithi093 ceph-mon[111335]: pgmap v7949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:33 smithi093 ceph-mon[118902]: pgmap v7949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2982067276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:34.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:34.553591+0000) 2023-08-28T01:18:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2982067276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2982067276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:35 smithi136 ceph-mon[113531]: pgmap v7950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:35.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:35.553851+0000) 2023-08-28T01:18:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:35 smithi093 ceph-mon[111335]: pgmap v7950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:18:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:18:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:18:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:35 smithi093 ceph-mon[118902]: pgmap v7950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:36.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:36.554018+0000) 2023-08-28T01:18:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:37 smithi093 ceph-mon[111335]: pgmap v7951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2951912717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:37 smithi093 ceph-mon[118902]: pgmap v7951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2951912717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:37.553 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:37 smithi136 ceph-mon[113531]: pgmap v7951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:37.553 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2951912717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:37.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:37.554176+0000) 2023-08-28T01:18:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:38.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:38.554427+0000) 2023-08-28T01:18:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:39 smithi136 ceph-mon[113531]: pgmap v7952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2352264024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:39.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:39.554718+0000) 2023-08-28T01:18:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:39 smithi093 ceph-mon[111335]: pgmap v7952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2352264024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:39 smithi093 ceph-mon[118902]: pgmap v7952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2352264024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:40.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:40.555023+0000) 2023-08-28T01:18:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:41 smithi136 ceph-mon[113531]: pgmap v7953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1415573534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:41.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:41.555289+0000) 2023-08-28T01:18:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:41 smithi093 ceph-mon[111335]: pgmap v7953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1415573534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:41 smithi093 ceph-mon[118902]: pgmap v7953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1415573534' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:41.260Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:41.261Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:42.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:42.555525+0000) 2023-08-28T01:18:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:42.704Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:42.705Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:43 smithi136 ceph-mon[113531]: pgmap v7954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:43.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:43.555751+0000) 2023-08-28T01:18:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:43 smithi093 ceph-mon[111335]: pgmap v7954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:43 smithi093 ceph-mon[118902]: pgmap v7954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3610916697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:44.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:44.556097+0000) 2023-08-28T01:18:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3610916697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3610916697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:45.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:45 smithi136 ceph-mon[113531]: pgmap v7955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:45.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:45.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:45.556300+0000) 2023-08-28T01:18:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:45 smithi093 ceph-mon[111335]: pgmap v7955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:45 smithi093 ceph-mon[118902]: pgmap v7955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:18:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:18:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:18:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1780782785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:46.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:46.556515+0000) 2023-08-28T01:18:46.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1780782785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1780782785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:47 smithi136 ceph-mon[113531]: pgmap v7956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:47.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:47.556770+0000) 2023-08-28T01:18:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:47 smithi093 ceph-mon[111335]: pgmap v7956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:47 smithi093 ceph-mon[118902]: pgmap v7956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/942382815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:48.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:48.557063+0000) 2023-08-28T01:18:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/942382815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/942382815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:49 smithi136 ceph-mon[113531]: pgmap v7957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:49.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:49.557262+0000) 2023-08-28T01:18:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:49 smithi093 ceph-mon[111335]: pgmap v7957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:49 smithi093 ceph-mon[118902]: pgmap v7957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:50.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:50.557509+0000) 2023-08-28T01:18:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:51.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:51.557783+0000) 2023-08-28T01:18:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:51 smithi136 ceph-mon[113531]: pgmap v7958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2456388874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:51 smithi093 ceph-mon[118902]: pgmap v7958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2456388874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:51 smithi093 ceph-mon[111335]: pgmap v7958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:51.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2456388874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:51.260Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:51.261Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:52.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:52.557982+0000) 2023-08-28T01:18:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:52.705Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:18:52.706Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:18:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:53.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:53.558217+0000) 2023-08-28T01:18:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:53 smithi136 ceph-mon[113531]: pgmap v7959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2290293011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:53 smithi093 ceph-mon[118902]: pgmap v7959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2290293011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:53 smithi093 ceph-mon[111335]: pgmap v7959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2290293011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:54.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:54.558454+0000) 2023-08-28T01:18:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:18:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:55 smithi136 ceph-mon[113531]: pgmap v7960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3589814901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:55.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:55.558710+0000) 2023-08-28T01:18:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:18:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:18:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:18:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:55 smithi093 ceph-mon[111335]: pgmap v7960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3589814901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:55 smithi093 ceph-mon[118902]: pgmap v7960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3589814901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:56.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:56.558993+0000) 2023-08-28T01:18:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:57 smithi093 ceph-mon[111335]: pgmap v7961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:57 smithi093 ceph-mon[118902]: pgmap v7961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:57.558 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:57 smithi136 ceph-mon[113531]: pgmap v7961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:57.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:57.559256+0000) 2023-08-28T01:18:58.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3562387219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3562387219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:58.558 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3562387219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:18:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:58.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:58.559537+0000) 2023-08-28T01:18:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:18:59 smithi093 ceph-mon[111335]: pgmap v7962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:18:59 smithi093 ceph-mon[118902]: pgmap v7962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:59.558 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:18:59 smithi136 ceph-mon[113531]: pgmap v7962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:18:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:18:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:18:59.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:18:59.559788+0000) 2023-08-28T01:19:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1137655286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1137655286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:00.559 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1137655286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:00.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:00.560077+0000) 2023-08-28T01:19:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:01 smithi093 ceph-mon[111335]: pgmap v7963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:01.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:01 smithi093 ceph-mon[118902]: pgmap v7963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:01.260Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:01.261Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:01.559 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:01 smithi136 ceph-mon[113531]: pgmap v7963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:01.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:01.560322+0000) 2023-08-28T01:19:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:02.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:02.560581+0000) 2023-08-28T01:19:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:02.706Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:02.707Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:03 smithi136 ceph-mon[113531]: pgmap v7964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/998742103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:03.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:03.560837+0000) 2023-08-28T01:19:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:03 smithi093 ceph-mon[111335]: pgmap v7964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/998742103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:03 smithi093 ceph-mon[118902]: pgmap v7964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/998742103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:04.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:04.561110+0000) 2023-08-28T01:19:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:05 smithi136 ceph-mon[113531]: pgmap v7965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1233601110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:05.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:05.561376+0000) 2023-08-28T01:19:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:05 smithi093 ceph-mon[111335]: pgmap v7965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1233601110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:19:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:19:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:19:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:05 smithi093 ceph-mon[118902]: pgmap v7965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1233601110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:06.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:06.561631+0000) 2023-08-28T01:19:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:07 smithi093 ceph-mon[111335]: pgmap v7966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:07 smithi093 ceph-mon[118902]: pgmap v7966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:07.560 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:07 smithi136 ceph-mon[113531]: pgmap v7966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:07.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:07.561860+0000) 2023-08-28T01:19:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/302449613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:08.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/302449613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:08.561 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/302449613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:08.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:08.562077+0000) 2023-08-28T01:19:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:09 smithi093 ceph-mon[111335]: pgmap v7967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:09 smithi093 ceph-mon[118902]: pgmap v7967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:09.561 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:09 smithi136 ceph-mon[113531]: pgmap v7967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:09.562 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:09.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:09.562303+0000) 2023-08-28T01:19:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/996110807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/996110807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:10.561 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/996110807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:10.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:10.562556+0000) 2023-08-28T01:19:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:11 smithi093 ceph-mon[111335]: pgmap v7968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:11 smithi093 ceph-mon[118902]: pgmap v7968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:11.261Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:11.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:11.262Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:11.562 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:11 smithi136 ceph-mon[113531]: pgmap v7968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:11.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:11.562807+0000) 2023-08-28T01:19:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3008477426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3008477426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:12.562 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3008477426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:12.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:12.563011+0000) 2023-08-28T01:19:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:12.707Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:12.708Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:13 smithi093 ceph-mon[111335]: pgmap v7969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:13 smithi093 ceph-mon[118902]: pgmap v7969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:13.562 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:13 smithi136 ceph-mon[113531]: pgmap v7969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:13.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:13.563291+0000) 2023-08-28T01:19:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:14.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:14.563515+0000) 2023-08-28T01:19:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:15 smithi136 ceph-mon[113531]: pgmap v7970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2556386601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:15.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:15.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:15.563724+0000) 2023-08-28T01:19:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:19:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:19:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:19:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:15 smithi093 ceph-mon[111335]: pgmap v7970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:15.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2556386601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:15 smithi093 ceph-mon[118902]: pgmap v7970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2556386601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:16.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:16.563951+0000) 2023-08-28T01:19:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:17 smithi093 ceph-mon[111335]: pgmap v7971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/48228999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:17 smithi093 ceph-mon[118902]: pgmap v7971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/48228999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:17.563 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:17 smithi136 ceph-mon[113531]: pgmap v7971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:17.563 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/48228999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:17.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:17.564202+0000) 2023-08-28T01:19:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:18.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:18.564488+0000) 2023-08-28T01:19:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:19 smithi136 ceph-mon[113531]: pgmap v7972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2195636085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:19.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:19.564779+0000) 2023-08-28T01:19:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:19 smithi093 ceph-mon[111335]: pgmap v7972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2195636085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:19 smithi093 ceph-mon[118902]: pgmap v7972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2195636085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:20.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:20.565068+0000) 2023-08-28T01:19:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:21 smithi136 ceph-mon[113531]: pgmap v7973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:21.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:21.565322+0000) 2023-08-28T01:19:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:21 smithi093 ceph-mon[111335]: pgmap v7973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:21 smithi093 ceph-mon[118902]: pgmap v7973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:21.261Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:21.262Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/403835145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:22.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:22.565593+0000) 2023-08-28T01:19:22.706 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/403835145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:22.706 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/403835145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:22.707Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:22.708Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:23 smithi136 ceph-mon[113531]: pgmap v7974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:23.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:23.565870+0000) 2023-08-28T01:19:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:23 smithi093 ceph-mon[111335]: pgmap v7974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:23 smithi093 ceph-mon[118902]: pgmap v7974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:24.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:24.566052+0000) 2023-08-28T01:19:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1566349035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:19:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1566349035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:19:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1566349035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:19:25.565 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:25 smithi136 ceph-mon[113531]: pgmap v7975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:25.671 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:25 smithi093 ceph-mon[111335]: pgmap v7975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:25.672 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:25 smithi093 ceph-mon[118902]: pgmap v7975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:25.672 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:19:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:19:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:19:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:25.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:25.566194+0000) 2023-08-28T01:19:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:26.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:26.566321+0000) 2023-08-28T01:19:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[111335]: pgmap v7976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1468748004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[118902]: pgmap v7976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1468748004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:26 smithi136 ceph-mon[113531]: pgmap v7976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1468748004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:27.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:27.566503+0000) 2023-08-28T01:19:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:19:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:19:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:19:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:19:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:19:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:19:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:19:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:28.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:28.566738+0000) 2023-08-28T01:19:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:28 smithi093 ceph-mon[111335]: pgmap v7977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3045372564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:28 smithi093 ceph-mon[118902]: pgmap v7977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3045372564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:28 smithi136 ceph-mon[113531]: pgmap v7977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3045372564' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:29.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:29.567004+0000) 2023-08-28T01:19:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:30.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:30.567276+0000) 2023-08-28T01:19:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:31 smithi136 ceph-mon[113531]: pgmap v7978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/771203768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:31.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:31.567520+0000) 2023-08-28T01:19:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:31 smithi093 ceph-mon[111335]: pgmap v7978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/771203768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:31 smithi093 ceph-mon[118902]: pgmap v7978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/771203768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:31.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:31.262Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:31.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:31.263Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:32.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:32.567718+0000) 2023-08-28T01:19:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:32.708Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:32.709Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:33 smithi136 ceph-mon[113531]: pgmap v7979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:33.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:33.567984+0000) 2023-08-28T01:19:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:33 smithi093 ceph-mon[111335]: pgmap v7979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:33 smithi093 ceph-mon[118902]: pgmap v7979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2096131645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:34.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:34.568256+0000) 2023-08-28T01:19:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2096131645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2096131645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:35 smithi136 ceph-mon[113531]: pgmap v7980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:35.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:35.568455+0000) 2023-08-28T01:19:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:35 smithi093 ceph-mon[111335]: pgmap v7980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:35 smithi093 ceph-mon[118902]: pgmap v7980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:19:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:19:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:19:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4043019895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:36.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:36.568732+0000) 2023-08-28T01:19:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4043019895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4043019895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:37 smithi136 ceph-mon[113531]: pgmap v7981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:37.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:37.568991+0000) 2023-08-28T01:19:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:37 smithi093 ceph-mon[111335]: pgmap v7981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:37 smithi093 ceph-mon[118902]: pgmap v7981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:38.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/514919722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:38.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:38.569247+0000) 2023-08-28T01:19:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/514919722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/514919722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:39 smithi136 ceph-mon[113531]: pgmap v7982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:39.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:39.569491+0000) 2023-08-28T01:19:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:39 smithi093 ceph-mon[111335]: pgmap v7982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:39 smithi093 ceph-mon[118902]: pgmap v7982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:40.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:40.569749+0000) 2023-08-28T01:19:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:41.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:41.569987+0000) 2023-08-28T01:19:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:41 smithi136 ceph-mon[113531]: pgmap v7983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1931700345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:41 smithi093 ceph-mon[111335]: pgmap v7983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1931700345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:41 smithi093 ceph-mon[118902]: pgmap v7983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1931700345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:41.263Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:41.264Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:42.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:42.570202+0000) 2023-08-28T01:19:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:42.709Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:42.710Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:43.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:43 smithi136 ceph-mon[113531]: pgmap v7984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:43.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/173077824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:43.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:43.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:43.570458+0000) 2023-08-28T01:19:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:43 smithi093 ceph-mon[118902]: pgmap v7984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/173077824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:43 smithi093 ceph-mon[111335]: pgmap v7984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/173077824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:44.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:44.570623+0000) 2023-08-28T01:19:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:45 smithi136 ceph-mon[113531]: pgmap v7985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:45.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:45.570895+0000) 2023-08-28T01:19:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:45 smithi093 ceph-mon[111335]: pgmap v7985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:19:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:19:45] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:19:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:45 smithi093 ceph-mon[118902]: pgmap v7985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3137430243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:46.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:46.571190+0000) 2023-08-28T01:19:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3137430243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:46.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3137430243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:47 smithi136 ceph-mon[113531]: pgmap v7986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:47.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:47.571460+0000) 2023-08-28T01:19:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:47 smithi093 ceph-mon[111335]: pgmap v7986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:47 smithi093 ceph-mon[118902]: pgmap v7986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1449783325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:48.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:48.571711+0000) 2023-08-28T01:19:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1449783325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1449783325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:49 smithi136 ceph-mon[113531]: pgmap v7987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:49.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:49.571964+0000) 2023-08-28T01:19:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:49 smithi093 ceph-mon[111335]: pgmap v7987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:49 smithi093 ceph-mon[118902]: pgmap v7987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/773864913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:50.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:50.572212+0000) 2023-08-28T01:19:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/773864913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/773864913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:51 smithi136 ceph-mon[113531]: pgmap v7988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:51.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:51.572435+0000) 2023-08-28T01:19:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:51 smithi093 ceph-mon[111335]: pgmap v7988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:51 smithi093 ceph-mon[118902]: pgmap v7988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:51.263Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:51.264Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:52.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:52.572651+0000) 2023-08-28T01:19:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:52.710Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:19:52.711Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:19:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:53 smithi136 ceph-mon[113531]: pgmap v7989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3966148213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:53.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:53.572868+0000) 2023-08-28T01:19:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:53 smithi093 ceph-mon[111335]: pgmap v7989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3966148213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:53 smithi093 ceph-mon[118902]: pgmap v7989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3966148213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:54.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:54.573143+0000) 2023-08-28T01:19:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:19:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:55 smithi136 ceph-mon[113531]: pgmap v7990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/195738215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:55.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:55.573395+0000) 2023-08-28T01:19:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:55 smithi093 ceph-mon[111335]: pgmap v7990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/195738215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:19:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:19:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:19:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:55 smithi093 ceph-mon[118902]: pgmap v7990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/195738215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:56.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:56.573644+0000) 2023-08-28T01:19:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:57 smithi093 ceph-mon[111335]: pgmap v7991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:57 smithi093 ceph-mon[118902]: pgmap v7991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:57.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:57 smithi136 ceph-mon[113531]: pgmap v7991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:57.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:57.573898+0000) 2023-08-28T01:19:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2025485440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2025485440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:58.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2025485440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:19:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:58.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:58.574132+0000) 2023-08-28T01:19:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:19:59 smithi093 ceph-mon[111335]: pgmap v7992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:19:59 smithi093 ceph-mon[118902]: pgmap v7992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:59.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:19:59 smithi136 ceph-mon[113531]: pgmap v7992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:19:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:19:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:19:59.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:19:59.574341+0000) 2023-08-28T01:20:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1283720317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:20:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1283720317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:20:00.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1283720317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:00.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:20:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:00.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:00.574578+0000) 2023-08-28T01:20:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:01 smithi093 ceph-mon[111335]: pgmap v7993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:01.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:01 smithi093 ceph-mon[118902]: pgmap v7993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:01.264Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:01.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:01.265Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:01.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:01 smithi136 ceph-mon[113531]: pgmap v7993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:01.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:01.574823+0000) 2023-08-28T01:20:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/793587925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/793587925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:02.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/793587925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:02.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:02.575061+0000) 2023-08-28T01:20:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:02.711Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:02.712Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:03 smithi093 ceph-mon[111335]: pgmap v7994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:03.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:03 smithi093 ceph-mon[118902]: pgmap v7994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:03.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:03 smithi136 ceph-mon[113531]: pgmap v7994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:03.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:03.575351+0000) 2023-08-28T01:20:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:04.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:04.575623+0000) 2023-08-28T01:20:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:05 smithi136 ceph-mon[113531]: pgmap v7995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1263270227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:05.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:05.575776+0000) 2023-08-28T01:20:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:05 smithi093 ceph-mon[111335]: pgmap v7995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1263270227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:05 smithi093 ceph-mon[118902]: pgmap v7995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1263270227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:20:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:20:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:20:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:06.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:06.575925+0000) 2023-08-28T01:20:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:07 smithi093 ceph-mon[111335]: pgmap v7996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2823040290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:07 smithi093 ceph-mon[118902]: pgmap v7996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2823040290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:07.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:07 smithi136 ceph-mon[113531]: pgmap v7996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:07.575 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2823040290' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:07.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:07.576140+0000) 2023-08-28T01:20:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:08.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:08.576350+0000) 2023-08-28T01:20:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:09 smithi136 ceph-mon[113531]: pgmap v7997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/165524608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:09.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:09.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:09.576599+0000) 2023-08-28T01:20:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:09 smithi093 ceph-mon[111335]: pgmap v7997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/165524608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:09 smithi093 ceph-mon[118902]: pgmap v7997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/165524608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:10.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:10.576855+0000) 2023-08-28T01:20:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:11 smithi136 ceph-mon[113531]: pgmap v7998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:11.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:11.577109+0000) 2023-08-28T01:20:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:11 smithi093 ceph-mon[111335]: pgmap v7998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:11 smithi093 ceph-mon[118902]: pgmap v7998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:11.265Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:11.266Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3643620702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:12.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:12.577330+0000) 2023-08-28T01:20:12.711 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3643620702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:12.711 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3643620702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:12.712Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:12.713Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:13 smithi136 ceph-mon[113531]: pgmap v7999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:13.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:13.577548+0000) 2023-08-28T01:20:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:13 smithi093 ceph-mon[111335]: pgmap v7999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:13 smithi093 ceph-mon[118902]: pgmap v7999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/285230098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:14.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:14.577792+0000) 2023-08-28T01:20:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/285230098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/285230098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:15 smithi136 ceph-mon[113531]: pgmap v8000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:15.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:15.577988+0000) 2023-08-28T01:20:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:15 smithi093 ceph-mon[111335]: pgmap v8000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:15 smithi093 ceph-mon[118902]: pgmap v8000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:20:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:20:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:20:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:16.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:16.578198+0000) 2023-08-28T01:20:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:17 smithi093 ceph-mon[111335]: pgmap v8001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111719437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:17 smithi093 ceph-mon[118902]: pgmap v8001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111719437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:17.577 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:17 smithi136 ceph-mon[113531]: pgmap v8001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:17.577 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111719437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:17.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:17.578396+0000) 2023-08-28T01:20:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:18.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:18.578648+0000) 2023-08-28T01:20:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:19 smithi136 ceph-mon[113531]: pgmap v8002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/998864171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:19.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:19.578921+0000) 2023-08-28T01:20:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:19 smithi093 ceph-mon[111335]: pgmap v8002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/998864171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:19 smithi093 ceph-mon[118902]: pgmap v8002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/998864171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:20.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:20.579170+0000) 2023-08-28T01:20:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:21 smithi136 ceph-mon[113531]: pgmap v8003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1868552414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:21.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:21.579439+0000) 2023-08-28T01:20:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:21 smithi093 ceph-mon[111335]: pgmap v8003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1868552414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:21 smithi093 ceph-mon[118902]: pgmap v8003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1868552414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:21.266Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:21.267Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:22.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:22.579695+0000) 2023-08-28T01:20:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:22.713Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:22.713Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:23 smithi136 ceph-mon[113531]: pgmap v8004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:23.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:23.579942+0000) 2023-08-28T01:20:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:23 smithi093 ceph-mon[111335]: pgmap v8004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:23 smithi093 ceph-mon[118902]: pgmap v8004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1194815826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:24.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:24.580150+0000) 2023-08-28T01:20:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1194815826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1194815826' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:25 smithi136 ceph-mon[113531]: pgmap v8005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:25.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:25.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:25.580388+0000) 2023-08-28T01:20:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:25 smithi093 ceph-mon[111335]: pgmap v8005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:25 smithi093 ceph-mon[118902]: pgmap v8005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:20:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:20:25] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:20:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/635168366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:26.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:26.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:26.580607+0000) 2023-08-28T01:20:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/635168366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/635168366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:27 smithi136 ceph-mon[113531]: pgmap v8006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:20:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:27.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:27.580829+0000) 2023-08-28T01:20:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:27 smithi093 ceph-mon[111335]: pgmap v8006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:20:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:27 smithi093 ceph-mon[118902]: pgmap v8006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:20:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2924144660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:28.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:28.581098+0000) 2023-08-28T01:20:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2924144660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:28.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2924144660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:29 smithi136 ceph-mon[113531]: pgmap v8007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:29.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:29.581339+0000) 2023-08-28T01:20:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:29 smithi093 ceph-mon[111335]: pgmap v8007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:29 smithi093 ceph-mon[118902]: pgmap v8007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:20:30.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:20:30.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:30.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:30.581510+0000) 2023-08-28T01:20:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:20:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:20:30.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:20:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:20:30.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:20:31.698 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:31.267Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:31.698 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:31.268Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:31 smithi136 ceph-mon[113531]: pgmap v8008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2957788278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:31.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:31.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:31.581757+0000) 2023-08-28T01:20:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:31 smithi093 ceph-mon[111335]: pgmap v8008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2957788278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:31 smithi093 ceph-mon[118902]: pgmap v8008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2957788278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:32.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:32 smithi136 ceph-mon[113531]: pgmap v8009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:32.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:32.582048+0000) 2023-08-28T01:20:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:32 smithi093 ceph-mon[111335]: pgmap v8009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:32 smithi093 ceph-mon[118902]: pgmap v8009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:32.713Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:32.714Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3170913690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:33.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:33.582312+0000) 2023-08-28T01:20:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3170913690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3170913690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:34 smithi136 ceph-mon[113531]: pgmap v8010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:34.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:34.582539+0000) 2023-08-28T01:20:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:34 smithi093 ceph-mon[111335]: pgmap v8010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:34 smithi093 ceph-mon[118902]: pgmap v8010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:35.715 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:20:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:20:35] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:20:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3453158067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:35.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:35.582754+0000) 2023-08-28T01:20:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3453158067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3453158067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:36.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:36.582960+0000) 2023-08-28T01:20:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:37 smithi093 ceph-mon[111335]: pgmap v8011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:37.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:37 smithi093 ceph-mon[118902]: pgmap v8011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:37.582 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:37 smithi136 ceph-mon[113531]: pgmap v8011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:37.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:37.583172+0000) 2023-08-28T01:20:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1440878334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:38.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1440878334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:38.582 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1440878334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:38.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:38.583395+0000) 2023-08-28T01:20:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:39 smithi093 ceph-mon[111335]: pgmap v8012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:39.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:39 smithi093 ceph-mon[118902]: pgmap v8012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:39.582 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:39 smithi136 ceph-mon[113531]: pgmap v8012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:39.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:39.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:39.583649+0000) 2023-08-28T01:20:40.582 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/203325149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/203325149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/203325149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:40.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:40.583896+0000) 2023-08-28T01:20:41.583 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:41 smithi136 ceph-mon[113531]: pgmap v8013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:41 smithi093 ceph-mon[111335]: pgmap v8013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:41 smithi093 ceph-mon[118902]: pgmap v8013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:41.268Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:41.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:41.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:41.584129+0000) 2023-08-28T01:20:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:42.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:42.584330+0000) 2023-08-28T01:20:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:42.713Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:42.714Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:43 smithi136 ceph-mon[113531]: pgmap v8014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2943557061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:43.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:43.584586+0000) 2023-08-28T01:20:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:43 smithi093 ceph-mon[118902]: pgmap v8014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2943557061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:43 smithi093 ceph-mon[111335]: pgmap v8014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:43.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2943557061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:44.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:44.584852+0000) 2023-08-28T01:20:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:45 smithi136 ceph-mon[113531]: pgmap v8015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2461004054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:45.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:45.585075+0000) 2023-08-28T01:20:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:45 smithi093 ceph-mon[111335]: pgmap v8015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2461004054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:20:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:20:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:20:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:45 smithi093 ceph-mon[118902]: pgmap v8015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2461004054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:46.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:46.585282+0000) 2023-08-28T01:20:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:47 smithi093 ceph-mon[111335]: pgmap v8016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:47.523 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:47 smithi093 ceph-mon[118902]: pgmap v8016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:47.584 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:47 smithi136 ceph-mon[113531]: pgmap v8016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:47.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:47.585540+0000) 2023-08-28T01:20:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3426379214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3426379214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:48.584 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3426379214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:48.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:48.585765+0000) 2023-08-28T01:20:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:49 smithi093 ceph-mon[111335]: pgmap v8017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:49 smithi093 ceph-mon[118902]: pgmap v8017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:49.584 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:49 smithi136 ceph-mon[113531]: pgmap v8017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:49.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:49.586010+0000) 2023-08-28T01:20:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/263807554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/263807554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:50.585 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/263807554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:50.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:50.586279+0000) 2023-08-28T01:20:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:51 smithi093 ceph-mon[111335]: pgmap v8018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:51 smithi093 ceph-mon[118902]: pgmap v8018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:51.268Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:51.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:51.269Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:51.585 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:51 smithi136 ceph-mon[113531]: pgmap v8018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:51.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:51.586529+0000) 2023-08-28T01:20:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2530683796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2530683796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:52.585 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2530683796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:52.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:52.586787+0000) 2023-08-28T01:20:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:52.714Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:20:52.715Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:20:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:53 smithi093 ceph-mon[111335]: pgmap v8019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:53.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:53 smithi093 ceph-mon[118902]: pgmap v8019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:53.585 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:53 smithi136 ceph-mon[113531]: pgmap v8019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:53.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:53.587051+0000) 2023-08-28T01:20:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:54.586 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:20:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:54.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:54.587288+0000) 2023-08-28T01:20:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:55 smithi093 ceph-mon[111335]: pgmap v8020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:55.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/974273696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:55.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:20:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:20:55] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:20:55.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:55 smithi093 ceph-mon[118902]: pgmap v8020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:55.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/974273696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:55.586 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:55 smithi136 ceph-mon[113531]: pgmap v8020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:55.586 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/974273696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:55.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:55.587486+0000) 2023-08-28T01:20:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:56.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:56.587742+0000) 2023-08-28T01:20:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:57 smithi093 ceph-mon[111335]: pgmap v8021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1106036261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:57 smithi093 ceph-mon[118902]: pgmap v8021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1106036261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:57.587 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:57 smithi136 ceph-mon[113531]: pgmap v8021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:57.587 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1106036261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:57.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:57.588027+0000) 2023-08-28T01:20:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:58.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:58.588285+0000) 2023-08-28T01:20:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:59 smithi136 ceph-mon[113531]: pgmap v8022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:20:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3131169799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:20:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:20:59.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:20:59.588525+0000) 2023-08-28T01:20:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:59 smithi093 ceph-mon[118902]: pgmap v8022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:20:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3131169799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:20:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:59 smithi093 ceph-mon[111335]: pgmap v8022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:20:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:20:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3131169799' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:00.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:00.588771+0000) 2023-08-28T01:21:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:01 smithi136 ceph-mon[113531]: pgmap v8023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:01.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:01.589052+0000) 2023-08-28T01:21:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:01 smithi093 ceph-mon[118902]: pgmap v8023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:01 smithi093 ceph-mon[111335]: pgmap v8023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:01.269Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:01.270Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/867365355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:02.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:02.589299+0000) 2023-08-28T01:21:02.713 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/867365355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:02.713 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/867365355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:02.714Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:02.715Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:03 smithi136 ceph-mon[113531]: pgmap v8024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:03.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:03.589514+0000) 2023-08-28T01:21:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:03 smithi093 ceph-mon[111335]: pgmap v8024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:03 smithi093 ceph-mon[118902]: pgmap v8024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3087030064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:04.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:04.589766+0000) 2023-08-28T01:21:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3087030064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3087030064' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:05.589 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:05 smithi136 ceph-mon[113531]: pgmap v8025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:05.589 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 01:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T01:21:05.317592136Z level=info msg="Completed cleanup jobs" duration=13.014807ms 2023-08-28T01:21:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:05 smithi093 ceph-mon[111335]: pgmap v8025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:21:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:21:05] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:21:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:05 smithi093 ceph-mon[118902]: pgmap v8025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:05.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:05.590005+0000) 2023-08-28T01:21:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:06.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:06.590234+0000) 2023-08-28T01:21:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:07 smithi093 ceph-mon[118902]: pgmap v8026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/521749811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:07 smithi093 ceph-mon[111335]: pgmap v8026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/521749811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:07.589 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:07 smithi136 ceph-mon[113531]: pgmap v8026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:07.589 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/521749811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:07.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:07.590469+0000) 2023-08-28T01:21:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:08.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:08.590712+0000) 2023-08-28T01:21:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:09 smithi136 ceph-mon[113531]: pgmap v8027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1964942299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:09.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:09.590983+0000) 2023-08-28T01:21:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:09 smithi093 ceph-mon[111335]: pgmap v8027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1964942299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:09 smithi093 ceph-mon[118902]: pgmap v8027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1964942299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:10.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:10.591144+0000) 2023-08-28T01:21:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:11.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:11.591368+0000) 2023-08-28T01:21:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:11 smithi136 ceph-mon[113531]: pgmap v8028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1442606598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:11 smithi093 ceph-mon[111335]: pgmap v8028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1442606598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:11 smithi093 ceph-mon[118902]: pgmap v8028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1442606598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:11.270Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:11.271Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:12.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:12.591570+0000) 2023-08-28T01:21:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:12.715Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:12.716Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:13 smithi136 ceph-mon[113531]: pgmap v8029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:13.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:13.591821+0000) 2023-08-28T01:21:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:13 smithi093 ceph-mon[111335]: pgmap v8029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:13 smithi093 ceph-mon[118902]: pgmap v8029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2198997598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:14.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:14.592074+0000) 2023-08-28T01:21:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2198997598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2198997598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:15 smithi136 ceph-mon[113531]: pgmap v8030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:15.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:15.592259+0000) 2023-08-28T01:21:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:15 smithi093 ceph-mon[111335]: pgmap v8030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:21:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:21:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:21:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:15 smithi093 ceph-mon[118902]: pgmap v8030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2984450923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:16.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:16.592535+0000) 2023-08-28T01:21:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2984450923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2984450923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:17 smithi136 ceph-mon[113531]: pgmap v8031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:17.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:17.592783+0000) 2023-08-28T01:21:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:17 smithi093 ceph-mon[111335]: pgmap v8031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:17 smithi093 ceph-mon[118902]: pgmap v8031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:18.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1448525102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:18.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:18.593062+0000) 2023-08-28T01:21:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1448525102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1448525102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:19 smithi136 ceph-mon[113531]: pgmap v8032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:19.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:19.593361+0000) 2023-08-28T01:21:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:19 smithi093 ceph-mon[111335]: pgmap v8032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:19 smithi093 ceph-mon[118902]: pgmap v8032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:20.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:20.593635+0000) 2023-08-28T01:21:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:21 smithi136 ceph-mon[113531]: pgmap v8033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1747653278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:21.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:21.593947+0000) 2023-08-28T01:21:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:21 smithi093 ceph-mon[111335]: pgmap v8033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1747653278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:21 smithi093 ceph-mon[118902]: pgmap v8033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1747653278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:21.271Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:21.272Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:22.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:22.594215+0000) 2023-08-28T01:21:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:22.716Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:22.717Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:23 smithi136 ceph-mon[113531]: pgmap v8034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2748859166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:23.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:23.594447+0000) 2023-08-28T01:21:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:23 smithi093 ceph-mon[111335]: pgmap v8034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2748859166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:23 smithi093 ceph-mon[118902]: pgmap v8034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2748859166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:24.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:24.594709+0000) 2023-08-28T01:21:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:25 smithi136 ceph-mon[113531]: pgmap v8035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:25.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:25.594949+0000) 2023-08-28T01:21:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:25 smithi093 ceph-mon[118902]: pgmap v8035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:25 smithi093 ceph-mon[111335]: pgmap v8035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:21:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:21:25] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:21:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/853189801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:26.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:26.595215+0000) 2023-08-28T01:21:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/853189801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/853189801' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:27 smithi136 ceph-mon[113531]: pgmap v8036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:27.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:27.595484+0000) 2023-08-28T01:21:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:27 smithi093 ceph-mon[111335]: pgmap v8036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:27 smithi093 ceph-mon[118902]: pgmap v8036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:28.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1000313577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:28.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:28.595692+0000) 2023-08-28T01:21:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1000313577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1000313577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:29 smithi136 ceph-mon[113531]: pgmap v8037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:29.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:29.595933+0000) 2023-08-28T01:21:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:29 smithi093 ceph-mon[111335]: pgmap v8037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:29 smithi093 ceph-mon[118902]: pgmap v8037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2490492458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:21:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:30.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:30.596131+0000) 2023-08-28T01:21:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2490492458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:21:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2490492458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:21:31.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:31.272Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:31.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:31.273Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:31.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:31.596282+0000) 2023-08-28T01:21:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:31 smithi136 ceph-mon[113531]: pgmap v8038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:31.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:31.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-mon[111335]: pgmap v8038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:31.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:31.959 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:31.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-mon[118902]: pgmap v8038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:31.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:31.959 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:32.833 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:32.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:32.596463+0000) 2023-08-28T01:21:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:32.717Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:32.718Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:33.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:33 smithi136 ceph-mon[113531]: pgmap v8039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:33.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4097213376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:33.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[118902]: pgmap v8039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4097213376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[111335]: pgmap v8039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4097213376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:33.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:33.596678+0000) 2023-08-28T01:21:34.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:21:34.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:21:34.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:21:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:21:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:21:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:21:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:21:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:34.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:34.596914+0000) 2023-08-28T01:21:35.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:35 smithi136 ceph-mon[113531]: pgmap v8040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:35.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4242257558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:21:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:21:35] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:21:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:35 smithi093 ceph-mon[111335]: pgmap v8040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4242257558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:35 smithi093 ceph-mon[118902]: pgmap v8040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4242257558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:35.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:35.597169+0000) 2023-08-28T01:21:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:36.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:36.597417+0000) 2023-08-28T01:21:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:37 smithi093 ceph-mon[111335]: pgmap v8041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:37.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:37 smithi093 ceph-mon[118902]: pgmap v8041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:37.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:37 smithi136 ceph-mon[113531]: pgmap v8041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:37.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:37.597678+0000) 2023-08-28T01:21:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/315513220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:38.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/315513220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:38.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/315513220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:38.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:38.597897+0000) 2023-08-28T01:21:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:39 smithi093 ceph-mon[111335]: pgmap v8042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:39.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:39 smithi093 ceph-mon[118902]: pgmap v8042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:39.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:39 smithi136 ceph-mon[113531]: pgmap v8042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:39.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:39.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:39.598149+0000) 2023-08-28T01:21:40.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1867556443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:40.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1867556443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:40.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1867556443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:40.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:40.598401+0000) 2023-08-28T01:21:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:41 smithi093 ceph-mon[111335]: pgmap v8043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:41.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:41 smithi093 ceph-mon[118902]: pgmap v8043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:41.273Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:41.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:41.274Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:41.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:41 smithi136 ceph-mon[113531]: pgmap v8043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:41.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:41.598620+0000) 2023-08-28T01:21:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2627013046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2627013046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:42.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2627013046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:42.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:42.598859+0000) 2023-08-28T01:21:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:42.718Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:42.719Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:43 smithi093 ceph-mon[111335]: pgmap v8044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:43 smithi093 ceph-mon[118902]: pgmap v8044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:43.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:43 smithi136 ceph-mon[113531]: pgmap v8044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:43.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:43.599126+0000) 2023-08-28T01:21:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:44.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:44.599379+0000) 2023-08-28T01:21:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:45 smithi136 ceph-mon[113531]: pgmap v8045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4191942739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:45.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:45.599582+0000) 2023-08-28T01:21:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:45 smithi093 ceph-mon[111335]: pgmap v8045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4191942739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:21:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:21:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:21:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:45 smithi093 ceph-mon[118902]: pgmap v8045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4191942739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:46.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:46.599844+0000) 2023-08-28T01:21:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:47 smithi093 ceph-mon[111335]: pgmap v8046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1454374769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:47 smithi093 ceph-mon[118902]: pgmap v8046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1454374769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:47.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:47 smithi136 ceph-mon[113531]: pgmap v8046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:47.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1454374769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:47.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:47.600127+0000) 2023-08-28T01:21:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:48.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:48.600345+0000) 2023-08-28T01:21:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:49 smithi136 ceph-mon[113531]: pgmap v8047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1486814091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:49.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:49.600568+0000) 2023-08-28T01:21:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:49 smithi093 ceph-mon[118902]: pgmap v8047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1486814091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:49 smithi093 ceph-mon[111335]: pgmap v8047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:49.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1486814091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:50.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:50.600803+0000) 2023-08-28T01:21:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:51 smithi136 ceph-mon[113531]: pgmap v8048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:51.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:51.601078+0000) 2023-08-28T01:21:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:51 smithi093 ceph-mon[111335]: pgmap v8048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:51 smithi093 ceph-mon[118902]: pgmap v8048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:51.274Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:51.275Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:52.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3647684435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:52.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:52.601305+0000) 2023-08-28T01:21:52.718 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3647684435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:52.718 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3647684435' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:52.719Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:21:52.720Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:21:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:53 smithi136 ceph-mon[113531]: pgmap v8049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:53.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:53.601531+0000) 2023-08-28T01:21:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:53 smithi093 ceph-mon[111335]: pgmap v8049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:53 smithi093 ceph-mon[118902]: pgmap v8049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/80057345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:54.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:54.601714+0000) 2023-08-28T01:21:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/80057345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:21:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/80057345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:55 smithi136 ceph-mon[113531]: pgmap v8050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:55.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:55.601964+0000) 2023-08-28T01:21:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:55 smithi093 ceph-mon[111335]: pgmap v8050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:21:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:21:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:21:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:55 smithi093 ceph-mon[118902]: pgmap v8050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:56.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:56.602274+0000) 2023-08-28T01:21:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:57 smithi093 ceph-mon[111335]: pgmap v8051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1937827682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:57 smithi093 ceph-mon[118902]: pgmap v8051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1937827682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:57.601 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:57 smithi136 ceph-mon[113531]: pgmap v8051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:57.601 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1937827682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:57.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:57.602488+0000) 2023-08-28T01:21:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:58.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:58.602748+0000) 2023-08-28T01:21:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:59 smithi136 ceph-mon[113531]: pgmap v8052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:21:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2545258392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:21:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:21:59.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:21:59.602912+0000) 2023-08-28T01:21:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:59 smithi093 ceph-mon[111335]: pgmap v8052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:21:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2545258392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:21:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:59 smithi093 ceph-mon[118902]: pgmap v8052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:21:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:21:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2545258392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:00.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:00.603124+0000) 2023-08-28T01:22:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:01 smithi136 ceph-mon[113531]: pgmap v8053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1577054485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:01.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:01.603394+0000) 2023-08-28T01:22:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:01 smithi093 ceph-mon[111335]: pgmap v8053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1577054485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:01 smithi093 ceph-mon[118902]: pgmap v8053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1577054485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:01.275Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:01.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:02.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:02.603666+0000) 2023-08-28T01:22:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:02.720Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:02.721Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:03 smithi136 ceph-mon[113531]: pgmap v8054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:03.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:03.603914+0000) 2023-08-28T01:22:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:03 smithi093 ceph-mon[111335]: pgmap v8054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:03 smithi093 ceph-mon[118902]: pgmap v8054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/903199088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:04.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:04.604170+0000) 2023-08-28T01:22:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/903199088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/903199088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:05 smithi136 ceph-mon[113531]: pgmap v8055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:05.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:05.604341+0000) 2023-08-28T01:22:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:05 smithi093 ceph-mon[111335]: pgmap v8055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:05 smithi093 ceph-mon[118902]: pgmap v8055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:22:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:22:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:22:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1906238251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:06.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:06.604546+0000) 2023-08-28T01:22:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1906238251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1906238251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:07 smithi136 ceph-mon[113531]: pgmap v8056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:07.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:07.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:07.604828+0000) 2023-08-28T01:22:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:07 smithi093 ceph-mon[111335]: pgmap v8056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:07 smithi093 ceph-mon[118902]: pgmap v8056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1254390117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:08.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:08.605103+0000) 2023-08-28T01:22:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1254390117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1254390117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:09.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:09.605369+0000) 2023-08-28T01:22:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:09 smithi136 ceph-mon[113531]: pgmap v8057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:09 smithi093 ceph-mon[118902]: pgmap v8057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:09 smithi093 ceph-mon[111335]: pgmap v8057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:10.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:10.605615+0000) 2023-08-28T01:22:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:11.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:11.605873+0000) 2023-08-28T01:22:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:11 smithi136 ceph-mon[113531]: pgmap v8058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/291061335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:11 smithi093 ceph-mon[111335]: pgmap v8058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/291061335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:11 smithi093 ceph-mon[118902]: pgmap v8058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/291061335' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:11.276Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:11.276Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:12.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:12.606135+0000) 2023-08-28T01:22:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:12.721Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:12.722Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:13.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:13.606383+0000) 2023-08-28T01:22:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:13 smithi136 ceph-mon[113531]: pgmap v8059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1281016097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:13 smithi093 ceph-mon[111335]: pgmap v8059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1281016097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:13 smithi093 ceph-mon[118902]: pgmap v8059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1281016097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:14.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:14.606585+0000) 2023-08-28T01:22:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:15 smithi136 ceph-mon[113531]: pgmap v8060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:15.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:15.606863+0000) 2023-08-28T01:22:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:15 smithi093 ceph-mon[111335]: pgmap v8060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:15 smithi093 ceph-mon[118902]: pgmap v8060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:22:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:22:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:22:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3037322546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:16.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:16.607099+0000) 2023-08-28T01:22:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3037322546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3037322546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:17 smithi136 ceph-mon[113531]: pgmap v8061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:17.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:17.607328+0000) 2023-08-28T01:22:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:17 smithi093 ceph-mon[111335]: pgmap v8061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:17 smithi093 ceph-mon[118902]: pgmap v8061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1245178029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1245178029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1245178029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:18.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:18.607582+0000) 2023-08-28T01:22:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:19 smithi093 ceph-mon[111335]: pgmap v8062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:19 smithi093 ceph-mon[118902]: pgmap v8062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:19 smithi136 ceph-mon[113531]: pgmap v8062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:19.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:19.607843+0000) 2023-08-28T01:22:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2040469464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2040469464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:20.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2040469464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:20.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:20.608090+0000) 2023-08-28T01:22:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:21 smithi093 ceph-mon[111335]: pgmap v8063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:21 smithi093 ceph-mon[118902]: pgmap v8063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:21.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:21.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:21 smithi136 ceph-mon[113531]: pgmap v8063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:21.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:21.608352+0000) 2023-08-28T01:22:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:22.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:22.608584+0000) 2023-08-28T01:22:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:22.722Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:22.723Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:23 smithi093 ceph-mon[118902]: pgmap v8064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3959122603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:23 smithi093 ceph-mon[111335]: pgmap v8064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3959122603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:23 smithi136 ceph-mon[113531]: pgmap v8064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3959122603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:23.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:23.608857+0000) 2023-08-28T01:22:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:24.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:24.609063+0000) 2023-08-28T01:22:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:25 smithi093 ceph-mon[111335]: pgmap v8065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2616234715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:25 smithi093 ceph-mon[118902]: pgmap v8065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2616234715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:25.740 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:22:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:22:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:22:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:25 smithi136 ceph-mon[113531]: pgmap v8065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2616234715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:25.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:25.609258+0000) 2023-08-28T01:22:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:26.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:26.609493+0000) 2023-08-28T01:22:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:27 smithi093 ceph-mon[111335]: pgmap v8066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:27 smithi093 ceph-mon[118902]: pgmap v8066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:27.608 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:27 smithi136 ceph-mon[113531]: pgmap v8066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:27.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:27.609721+0000) 2023-08-28T01:22:28.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/554017943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/554017943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:28.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/554017943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:28.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:28.609985+0000) 2023-08-28T01:22:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:29 smithi093 ceph-mon[111335]: pgmap v8067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:29 smithi093 ceph-mon[118902]: pgmap v8067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:29.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:29 smithi136 ceph-mon[113531]: pgmap v8067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:29.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:29.610245+0000) 2023-08-28T01:22:30.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2965476043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2965476043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2965476043' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:30.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:30.610493+0000) 2023-08-28T01:22:31.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:31 smithi136 ceph-mon[113531]: pgmap v8068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:31 smithi093 ceph-mon[111335]: pgmap v8068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:31 smithi093 ceph-mon[118902]: pgmap v8068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:31.277Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:31.278Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:31.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:31.610733+0000) 2023-08-28T01:22:32.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020708289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:32.721 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020708289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:32.721 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020708289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:32.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:32.610969+0000) 2023-08-28T01:22:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:32.722Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:32.723Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:33.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:33 smithi136 ceph-mon[113531]: pgmap v8069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:33 smithi093 ceph-mon[111335]: pgmap v8069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:33 smithi093 ceph-mon[118902]: pgmap v8069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:33.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:33.611116+0000) 2023-08-28T01:22:34.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:22:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:22:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:22:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:34.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:34.611291+0000) 2023-08-28T01:22:35.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:35 smithi136 ceph-mon[113531]: pgmap v8070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:35.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2523953859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:35 smithi093 ceph-mon[111335]: pgmap v8070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2523953859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:35 smithi093 ceph-mon[118902]: pgmap v8070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2523953859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:22:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:22:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:22:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:35.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:35.611457+0000) 2023-08-28T01:22:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:36.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:36.611633+0000) 2023-08-28T01:22:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:37.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:37.611858+0000) 2023-08-28T01:22:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:37 smithi136 ceph-mon[113531]: pgmap v8071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:22:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:22:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:37.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1115983595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[111335]: pgmap v8071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:22:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:22:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1115983595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[118902]: pgmap v8071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:22:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:22:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:22:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1115983595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:38.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:38.612154+0000) 2023-08-28T01:22:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:39 smithi136 ceph-mon[113531]: pgmap v8072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/529948744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:39.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:39.612421+0000) 2023-08-28T01:22:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:39 smithi093 ceph-mon[111335]: pgmap v8072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/529948744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:39 smithi093 ceph-mon[118902]: pgmap v8072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/529948744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:40.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:40.612643+0000) 2023-08-28T01:22:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:41 smithi093 ceph-mon[111335]: pgmap v8073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:41 smithi093 ceph-mon[118902]: pgmap v8073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:41.278Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:41.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:41 smithi136 ceph-mon[113531]: pgmap v8073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:41.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:41.612825+0000) 2023-08-28T01:22:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2161838076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:42.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:42.613018+0000) 2023-08-28T01:22:42.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2161838076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2161838076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:42.723Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:42.723Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:43 smithi136 ceph-mon[113531]: pgmap v8074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:43.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:43.613287+0000) 2023-08-28T01:22:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:43 smithi093 ceph-mon[111335]: pgmap v8074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:43 smithi093 ceph-mon[118902]: pgmap v8074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2779229934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:44.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:44.613540+0000) 2023-08-28T01:22:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2779229934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2779229934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:45 smithi093 ceph-mon[111335]: pgmap v8075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:22:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:22:45] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:22:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:45 smithi093 ceph-mon[118902]: pgmap v8075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:45 smithi136 ceph-mon[113531]: pgmap v8075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:45.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:45.613736+0000) 2023-08-28T01:22:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:46.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:46.614011+0000) 2023-08-28T01:22:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:47 smithi093 ceph-mon[118902]: pgmap v8076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:47.539 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/919429844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:47.539 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:47 smithi093 ceph-mon[111335]: pgmap v8076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:47.539 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/919429844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:47.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:47 smithi136 ceph-mon[113531]: pgmap v8076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:47.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/919429844' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:47.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:47.614274+0000) 2023-08-28T01:22:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:48.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:48.614528+0000) 2023-08-28T01:22:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:49 smithi136 ceph-mon[113531]: pgmap v8077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/509465503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:49.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:49.614811+0000) 2023-08-28T01:22:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:49 smithi093 ceph-mon[111335]: pgmap v8077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/509465503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:49 smithi093 ceph-mon[118902]: pgmap v8077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/509465503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:50.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:50.615062+0000) 2023-08-28T01:22:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:51 smithi136 ceph-mon[113531]: pgmap v8078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2635711341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:51.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:51.615301+0000) 2023-08-28T01:22:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:51 smithi093 ceph-mon[111335]: pgmap v8078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2635711341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:51 smithi093 ceph-mon[118902]: pgmap v8078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2635711341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:51.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:51.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:52.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:52.615534+0000) 2023-08-28T01:22:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:52.723Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:22:52.724Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:22:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:53 smithi136 ceph-mon[113531]: pgmap v8079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:53.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:53.615779+0000) 2023-08-28T01:22:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:53 smithi093 ceph-mon[111335]: pgmap v8079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:53 smithi093 ceph-mon[118902]: pgmap v8079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1332372900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:54.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:54.616025+0000) 2023-08-28T01:22:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1332372900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1332372900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:22:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:55 smithi136 ceph-mon[113531]: pgmap v8080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:55.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:55.616293+0000) 2023-08-28T01:22:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:55 smithi093 ceph-mon[111335]: pgmap v8080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:55 smithi093 ceph-mon[118902]: pgmap v8080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:22:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:22:55] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:22:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2253167256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:56.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:56.616542+0000) 2023-08-28T01:22:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2253167256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:56.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2253167256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:57 smithi136 ceph-mon[113531]: pgmap v8081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:57.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:57.616728+0000) 2023-08-28T01:22:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:57 smithi093 ceph-mon[111335]: pgmap v8081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:57 smithi093 ceph-mon[118902]: pgmap v8081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:58.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:58.616908+0000) 2023-08-28T01:22:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:59 smithi136 ceph-mon[113531]: pgmap v8082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:22:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3335948100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:22:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:22:59.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:22:59.617141+0000) 2023-08-28T01:22:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:59 smithi093 ceph-mon[118902]: pgmap v8082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:22:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3335948100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:22:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:59 smithi093 ceph-mon[111335]: pgmap v8082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:22:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:22:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3335948100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:00.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:00.617373+0000) 2023-08-28T01:23:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:01 smithi136 ceph-mon[113531]: pgmap v8083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4141033044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:01.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:01.617601+0000) 2023-08-28T01:23:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:01 smithi093 ceph-mon[111335]: pgmap v8083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4141033044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:01 smithi093 ceph-mon[118902]: pgmap v8083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4141033044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:01.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:01.279Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:02.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:02.617818+0000) 2023-08-28T01:23:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:02.724Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:02.726Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:03 smithi136 ceph-mon[113531]: pgmap v8084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3287830420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:03.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:03.618108+0000) 2023-08-28T01:23:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:03 smithi093 ceph-mon[111335]: pgmap v8084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3287830420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:03 smithi093 ceph-mon[118902]: pgmap v8084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3287830420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:04.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:04.618274+0000) 2023-08-28T01:23:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:05 smithi093 ceph-mon[111335]: pgmap v8085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:05 smithi093 ceph-mon[118902]: pgmap v8085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:23:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:23:05] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:23:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:05 smithi136 ceph-mon[113531]: pgmap v8085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:05.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:05.618506+0000) 2023-08-28T01:23:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2949148540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2949148540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:06.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2949148540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:06.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:06.618761+0000) 2023-08-28T01:23:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:07 smithi093 ceph-mon[111335]: pgmap v8086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:07 smithi093 ceph-mon[118902]: pgmap v8086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:07 smithi136 ceph-mon[113531]: pgmap v8086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:07.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:07.618990+0000) 2023-08-28T01:23:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/239727954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:08.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/239727954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/239727954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:08.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:08.619235+0000) 2023-08-28T01:23:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:09 smithi093 ceph-mon[111335]: pgmap v8087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:09 smithi093 ceph-mon[118902]: pgmap v8087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:09 smithi136 ceph-mon[113531]: pgmap v8087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:09.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:09.619505+0000) 2023-08-28T01:23:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2717283103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2717283103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2717283103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:10.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:10.619771+0000) 2023-08-28T01:23:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:11 smithi093 ceph-mon[111335]: pgmap v8088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:11 smithi093 ceph-mon[118902]: pgmap v8088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:11.279Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:11.280Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:11 smithi136 ceph-mon[113531]: pgmap v8088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:11.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:11.620018+0000) 2023-08-28T01:23:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:12.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:12.620291+0000) 2023-08-28T01:23:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:12.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:12.726Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:13 smithi093 ceph-mon[111335]: pgmap v8089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4181360351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:13 smithi093 ceph-mon[118902]: pgmap v8089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4181360351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:13 smithi136 ceph-mon[113531]: pgmap v8089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4181360351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:13.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:13.620543+0000) 2023-08-28T01:23:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:14.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:14.620749+0000) 2023-08-28T01:23:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:23:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:23:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:23:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:15 smithi093 ceph-mon[111335]: pgmap v8090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3158276779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:15 smithi093 ceph-mon[118902]: pgmap v8090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3158276779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:15 smithi136 ceph-mon[113531]: pgmap v8090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3158276779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:15.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:15.620986+0000) 2023-08-28T01:23:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:16.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:16.621243+0000) 2023-08-28T01:23:17.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:17 smithi136 ceph-mon[113531]: pgmap v8091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:17 smithi093 ceph-mon[111335]: pgmap v8091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:17 smithi093 ceph-mon[118902]: pgmap v8091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:17.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:17.621527+0000) 2023-08-28T01:23:18.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4208960112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4208960112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4208960112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:18.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:18.621785+0000) 2023-08-28T01:23:19.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:19 smithi136 ceph-mon[113531]: pgmap v8092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:19 smithi093 ceph-mon[111335]: pgmap v8092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:19 smithi093 ceph-mon[118902]: pgmap v8092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:19.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:19.622070+0000) 2023-08-28T01:23:20.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3077453950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3077453950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3077453950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:20.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:20.622287+0000) 2023-08-28T01:23:21.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:21 smithi136 ceph-mon[113531]: pgmap v8093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:21 smithi093 ceph-mon[118902]: pgmap v8093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:21 smithi093 ceph-mon[111335]: pgmap v8093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:21.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:21.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:21.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:21.622531+0000) 2023-08-28T01:23:22.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/948129390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:22.724 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/948129390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:22.724 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/948129390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:22.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:22.622706+0000) 2023-08-28T01:23:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:22.725Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:22.726Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:23.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:23 smithi136 ceph-mon[113531]: pgmap v8094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:23 smithi093 ceph-mon[111335]: pgmap v8094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:23 smithi093 ceph-mon[118902]: pgmap v8094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:23.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:23.622941+0000) 2023-08-28T01:23:24.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:24.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:24.623167+0000) 2023-08-28T01:23:25.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:25 smithi136 ceph-mon[113531]: pgmap v8095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:25.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2698675046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:25 smithi093 ceph-mon[111335]: pgmap v8095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2698675046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:23:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:23:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:23:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:25 smithi093 ceph-mon[118902]: pgmap v8095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2698675046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:25.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:25.623342+0000) 2023-08-28T01:23:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:26.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:26.623582+0000) 2023-08-28T01:23:27.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:27 smithi136 ceph-mon[113531]: pgmap v8096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:27.623 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2947938875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:27 smithi093 ceph-mon[111335]: pgmap v8096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2947938875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:27 smithi093 ceph-mon[118902]: pgmap v8096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2947938875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:27.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:27.623868+0000) 2023-08-28T01:23:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:28.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:28.624129+0000) 2023-08-28T01:23:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:29 smithi136 ceph-mon[113531]: pgmap v8097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:29.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:29.624372+0000) 2023-08-28T01:23:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:29 smithi093 ceph-mon[111335]: pgmap v8097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:29 smithi093 ceph-mon[118902]: pgmap v8097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1422457362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:30.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:30.624607+0000) 2023-08-28T01:23:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1422457362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1422457362' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:31 smithi136 ceph-mon[113531]: pgmap v8098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:31.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:31.624755+0000) 2023-08-28T01:23:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:31 smithi093 ceph-mon[111335]: pgmap v8098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:31 smithi093 ceph-mon[118902]: pgmap v8098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:31.280Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:31.281Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/148152940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:32.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:32.624926+0000) 2023-08-28T01:23:32.725 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/148152940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:32.725 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/148152940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:32.726Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:32.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:32.727Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:33 smithi136 ceph-mon[113531]: pgmap v8099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:33.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:33.625150+0000) 2023-08-28T01:23:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:33 smithi093 ceph-mon[111335]: pgmap v8099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:33 smithi093 ceph-mon[118902]: pgmap v8099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/265490268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:34.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:34.625382+0000) 2023-08-28T01:23:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/265490268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/265490268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:35 smithi136 ceph-mon[113531]: pgmap v8100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:35.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:35.625594+0000) 2023-08-28T01:23:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:35 smithi093 ceph-mon[111335]: pgmap v8100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:23:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:23:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:23:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:35 smithi093 ceph-mon[118902]: pgmap v8100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:36.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:36.625775+0000) 2023-08-28T01:23:37.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:37 smithi136 ceph-mon[113531]: pgmap v8101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:37.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3468506122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:37.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:23:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:37 smithi093 ceph-mon[111335]: pgmap v8101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3468506122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:23:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:37 smithi093 ceph-mon[118902]: pgmap v8101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3468506122' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:23:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:37.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:37.625997+0000) 2023-08-28T01:23:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:38.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:38.626159+0000) 2023-08-28T01:23:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[111335]: pgmap v8102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2181135788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[118902]: pgmap v8102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2181135788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:39.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:39.626329+0000) 2023-08-28T01:23:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-mon[113531]: pgmap v8102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2181135788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:39.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:40.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:40.626599+0000) 2023-08-28T01:23:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:41.281Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:41.282Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[118902]: pgmap v8103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3144580546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[111335]: pgmap v8103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:23:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:23:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3144580546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:41.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:41.626848+0000) 2023-08-28T01:23:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-mon[113531]: pgmap v8103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:23:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:23:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:23:41.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3144580546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:42.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:42.627144+0000) 2023-08-28T01:23:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:42.727Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:42.729Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:43 smithi093 ceph-mon[111335]: pgmap v8104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:43 smithi093 ceph-mon[118902]: pgmap v8104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:43 smithi136 ceph-mon[113531]: pgmap v8104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:43.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:43.627401+0000) 2023-08-28T01:23:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/378033350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/378033350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/378033350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:44.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:44.627654+0000) 2023-08-28T01:23:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:45 smithi093 ceph-mon[111335]: pgmap v8105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:45 smithi093 ceph-mon[118902]: pgmap v8105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:23:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:23:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:23:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:45 smithi136 ceph-mon[113531]: pgmap v8105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:45.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:45.627901+0000) 2023-08-28T01:23:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3783875610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:46.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3783875610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3783875610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:46.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:46.628146+0000) 2023-08-28T01:23:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:47 smithi093 ceph-mon[111335]: pgmap v8106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:47 smithi093 ceph-mon[118902]: pgmap v8106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:47 smithi136 ceph-mon[113531]: pgmap v8106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:47.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:47.628392+0000) 2023-08-28T01:23:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:48.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:48.628644+0000) 2023-08-28T01:23:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:49 smithi093 ceph-mon[111335]: pgmap v8107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1329163778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:49 smithi093 ceph-mon[118902]: pgmap v8107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1329163778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:49 smithi136 ceph-mon[113531]: pgmap v8107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1329163778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:49.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:49.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:49.628899+0000) 2023-08-28T01:23:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:50.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:50.629168+0000) 2023-08-28T01:23:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:51 smithi093 ceph-mon[111335]: pgmap v8108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1240831467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:51 smithi093 ceph-mon[118902]: pgmap v8108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1240831467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:51.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:51.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:51 smithi136 ceph-mon[113531]: pgmap v8108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1240831467' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:51.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:51.629357+0000) 2023-08-28T01:23:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:52.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:52.629594+0000) 2023-08-28T01:23:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:52.728Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:23:52.730Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:23:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:53 smithi093 ceph-mon[111335]: pgmap v8109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2567705872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:53 smithi093 ceph-mon[118902]: pgmap v8109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2567705872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:53 smithi136 ceph-mon[113531]: pgmap v8109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2567705872' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:53.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:53.629869+0000) 2023-08-28T01:23:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:23:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:54.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:54.630110+0000) 2023-08-28T01:23:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:55 smithi093 ceph-mon[111335]: pgmap v8110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:23:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:23:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:23:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:55 smithi093 ceph-mon[118902]: pgmap v8110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:55 smithi136 ceph-mon[113531]: pgmap v8110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:55.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:55.630359+0000) 2023-08-28T01:23:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2221207344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:56.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2221207344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2221207344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:56.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:56.630642+0000) 2023-08-28T01:23:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:57 smithi093 ceph-mon[111335]: pgmap v8111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:57 smithi093 ceph-mon[118902]: pgmap v8111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:57 smithi136 ceph-mon[113531]: pgmap v8111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:57.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:57.630913+0000) 2023-08-28T01:23:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3772401312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3772401312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3772401312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:23:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:58.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:58.631193+0000) 2023-08-28T01:23:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:23:59 smithi093 ceph-mon[111335]: pgmap v8112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:23:59 smithi093 ceph-mon[118902]: pgmap v8112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:23:59 smithi136 ceph-mon[113531]: pgmap v8112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:23:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:23:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:23:59.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:23:59.631464+0000) 2023-08-28T01:24:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/291284275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/291284275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/291284275' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:00.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:00.631675+0000) 2023-08-28T01:24:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:01 smithi093 ceph-mon[111335]: pgmap v8113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:01 smithi093 ceph-mon[118902]: pgmap v8113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:01.282Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:01.283Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:01 smithi136 ceph-mon[113531]: pgmap v8113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:01.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:01.631916+0000) 2023-08-28T01:24:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:02.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:02.632143+0000) 2023-08-28T01:24:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:02.729Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:02.730Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:03 smithi093 ceph-mon[111335]: pgmap v8114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1619133455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:03 smithi093 ceph-mon[118902]: pgmap v8114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1619133455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:03 smithi136 ceph-mon[113531]: pgmap v8114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1619133455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:03.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:03.632433+0000) 2023-08-28T01:24:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:04.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:04.632650+0000) 2023-08-28T01:24:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:24:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:24:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:24:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:05 smithi093 ceph-mon[111335]: pgmap v8115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2027408819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:05 smithi093 ceph-mon[118902]: pgmap v8115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2027408819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:05 smithi136 ceph-mon[113531]: pgmap v8115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2027408819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:05.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:05.632920+0000) 2023-08-28T01:24:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:06.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:06.633171+0000) 2023-08-28T01:24:07.632 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:07 smithi136 ceph-mon[113531]: pgmap v8116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:07 smithi093 ceph-mon[111335]: pgmap v8116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:07 smithi093 ceph-mon[118902]: pgmap v8116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:07.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:07.633425+0000) 2023-08-28T01:24:08.632 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1256996619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1256996619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1256996619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:08.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:08.633677+0000) 2023-08-28T01:24:09.632 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:09 smithi136 ceph-mon[113531]: pgmap v8117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:09.632 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:09 smithi093 ceph-mon[111335]: pgmap v8117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:09 smithi093 ceph-mon[118902]: pgmap v8117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:09.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:09.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:09.633859+0000) 2023-08-28T01:24:10.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1580916266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1580916266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1580916266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:10.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:10.634155+0000) 2023-08-28T01:24:11.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:11 smithi136 ceph-mon[113531]: pgmap v8118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:11 smithi093 ceph-mon[111335]: pgmap v8118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:11 smithi093 ceph-mon[118902]: pgmap v8118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:11.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:11.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:11.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:11.634410+0000) 2023-08-28T01:24:12.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3786621021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:12.729 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3786621021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:12.729 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3786621021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:12.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:12.634651+0000) 2023-08-28T01:24:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:12.730Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:12.731Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:13.633 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:13 smithi136 ceph-mon[113531]: pgmap v8119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:13 smithi093 ceph-mon[111335]: pgmap v8119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:13 smithi093 ceph-mon[118902]: pgmap v8119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:13.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:13.634909+0000) 2023-08-28T01:24:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:14.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:14.635163+0000) 2023-08-28T01:24:15.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:15 smithi136 ceph-mon[113531]: pgmap v8120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:15.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2117807784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:15 smithi093 ceph-mon[111335]: pgmap v8120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2117807784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:15 smithi093 ceph-mon[118902]: pgmap v8120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2117807784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:24:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:24:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:24:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:15.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:15.635358+0000) 2023-08-28T01:24:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:16.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:16.635595+0000) 2023-08-28T01:24:17.634 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:17 smithi136 ceph-mon[113531]: pgmap v8121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:17.635 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2015838921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:17 smithi093 ceph-mon[111335]: pgmap v8121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2015838921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:17 smithi093 ceph-mon[118902]: pgmap v8121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2015838921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:17.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:17.635837+0000) 2023-08-28T01:24:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:18.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:18.636121+0000) 2023-08-28T01:24:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:19 smithi136 ceph-mon[113531]: pgmap v8122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:19.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:19.636347+0000) 2023-08-28T01:24:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:19 smithi093 ceph-mon[111335]: pgmap v8122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:19 smithi093 ceph-mon[118902]: pgmap v8122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3809478326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:20.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:20.636601+0000) 2023-08-28T01:24:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3809478326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3809478326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:21 smithi093 ceph-mon[111335]: pgmap v8123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:21 smithi093 ceph-mon[118902]: pgmap v8123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:21.283Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:21.284Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:21 smithi136 ceph-mon[113531]: pgmap v8123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:21.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:21.636862+0000) 2023-08-28T01:24:22.730 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1319445750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:22.730 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1319445750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1319445750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:22.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:22.637124+0000) 2023-08-28T01:24:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:22.731Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:22.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:23 smithi093 ceph-mon[111335]: pgmap v8124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:23 smithi093 ceph-mon[118902]: pgmap v8124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:23 smithi136 ceph-mon[113531]: pgmap v8124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:23.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:23.637377+0000) 2023-08-28T01:24:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2114564956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:24.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2114564956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2114564956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:24.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:24.637632+0000) 2023-08-28T01:24:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:25 smithi093 ceph-mon[111335]: pgmap v8125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:25 smithi093 ceph-mon[118902]: pgmap v8125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:24:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:24:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:24:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:25 smithi136 ceph-mon[113531]: pgmap v8125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:25.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:25.637881+0000) 2023-08-28T01:24:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:26.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:26.638182+0000) 2023-08-28T01:24:27.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:27 smithi136 ceph-mon[113531]: pgmap v8126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:27.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/141217430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:27 smithi093 ceph-mon[111335]: pgmap v8126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/141217430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:27 smithi093 ceph-mon[118902]: pgmap v8126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/141217430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:27.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:27.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:27.638413+0000) 2023-08-28T01:24:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:28.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:28.638657+0000) 2023-08-28T01:24:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:29 smithi136 ceph-mon[113531]: pgmap v8127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1777185791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:29.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:29.638902+0000) 2023-08-28T01:24:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:29 smithi093 ceph-mon[111335]: pgmap v8127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1777185791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:29 smithi093 ceph-mon[118902]: pgmap v8127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1777185791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:30.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:30.639160+0000) 2023-08-28T01:24:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:31 smithi136 ceph-mon[113531]: pgmap v8128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/131956433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:31.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:31.639392+0000) 2023-08-28T01:24:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:31 smithi093 ceph-mon[111335]: pgmap v8128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/131956433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:31 smithi093 ceph-mon[118902]: pgmap v8128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/131956433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:31.284Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:31.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:32.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:32.639600+0000) 2023-08-28T01:24:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:32.731Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:32.732Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:33 smithi093 ceph-mon[111335]: pgmap v8129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:33 smithi093 ceph-mon[118902]: pgmap v8129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:33 smithi136 ceph-mon[113531]: pgmap v8129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:33.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:33.639867+0000) 2023-08-28T01:24:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2081378092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2081378092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2081378092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:34.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:34.640103+0000) 2023-08-28T01:24:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:35 smithi093 ceph-mon[111335]: pgmap v8130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:24:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:24:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:24:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:35 smithi093 ceph-mon[118902]: pgmap v8130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:35 smithi136 ceph-mon[113531]: pgmap v8130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:35.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:35.640357+0000) 2023-08-28T01:24:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4200959157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4200959157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4200959157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:36.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:36.640570+0000) 2023-08-28T01:24:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:37 smithi093 ceph-mon[111335]: pgmap v8131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:37 smithi093 ceph-mon[118902]: pgmap v8131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:37 smithi136 ceph-mon[113531]: pgmap v8131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:37.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:37.640784+0000) 2023-08-28T01:24:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:38.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:38.641062+0000) 2023-08-28T01:24:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:39 smithi093 ceph-mon[111335]: pgmap v8132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2928769979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:39 smithi093 ceph-mon[118902]: pgmap v8132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2928769979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:39 smithi136 ceph-mon[113531]: pgmap v8132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2928769979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:39.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:39.641320+0000) 2023-08-28T01:24:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:40.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:40.641579+0000) 2023-08-28T01:24:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-mon[111335]: pgmap v8133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3489720381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:24:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-mon[118902]: pgmap v8133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3489720381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:24:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:41.284Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:41.285Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:41.850 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:41.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:41.641750+0000) 2023-08-28T01:24:41.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:41 smithi136 ceph-mon[113531]: pgmap v8133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:41.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3489720381' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:41.850 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:24:42.641 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:42.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:42.641899+0000) 2023-08-28T01:24:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:42.732Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:42.734Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:43.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:43.642081+0000) 2023-08-28T01:24:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:43 smithi136 ceph-mon[113531]: pgmap v8134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/330853293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[111335]: pgmap v8134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/330853293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[118902]: pgmap v8134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/330853293' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:43.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:24:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:24:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:44.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:44.642292+0000) 2023-08-28T01:24:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:24:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:24:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:24:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:24:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:24:45.657 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:24:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:24:45] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:24:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:45 smithi136 ceph-mon[113531]: pgmap v8135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:45.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:45.642468+0000) 2023-08-28T01:24:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:45 smithi093 ceph-mon[111335]: pgmap v8135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:45 smithi093 ceph-mon[118902]: pgmap v8135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:46.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:46.642691+0000) 2023-08-28T01:24:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111746577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111746577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:46.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111746577' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:47 smithi136 ceph-mon[113531]: pgmap v8136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:47.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:47.642926+0000) 2023-08-28T01:24:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:47 smithi093 ceph-mon[111335]: pgmap v8136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:47 smithi093 ceph-mon[118902]: pgmap v8136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:48.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2120930655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:48.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:48.643184+0000) 2023-08-28T01:24:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2120930655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2120930655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:49 smithi136 ceph-mon[113531]: pgmap v8137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:49.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:49.643431+0000) 2023-08-28T01:24:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:49 smithi093 ceph-mon[111335]: pgmap v8137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:49 smithi093 ceph-mon[118902]: pgmap v8137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2470287179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:50.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:50.643697+0000) 2023-08-28T01:24:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2470287179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2470287179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:51.672 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:51.285Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:51.672 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:51.286Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:51 smithi136 ceph-mon[113531]: pgmap v8138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:51.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:51.643945+0000) 2023-08-28T01:24:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:51 smithi093 ceph-mon[111335]: pgmap v8138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:51 smithi093 ceph-mon[118902]: pgmap v8138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:52.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:52.644178+0000) 2023-08-28T01:24:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:52.734Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:24:52.735Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:24:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:53 smithi136 ceph-mon[113531]: pgmap v8139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2839601249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:53.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:53.644436+0000) 2023-08-28T01:24:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:53 smithi093 ceph-mon[111335]: pgmap v8139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2839601249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:53 smithi093 ceph-mon[118902]: pgmap v8139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2839601249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:54.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:54.644684+0000) 2023-08-28T01:24:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:24:55.680 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:24:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:24:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:24:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:55 smithi136 ceph-mon[113531]: pgmap v8140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1284468981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:55.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:55.644944+0000) 2023-08-28T01:24:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:55 smithi093 ceph-mon[111335]: pgmap v8140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1284468981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:55 smithi093 ceph-mon[118902]: pgmap v8140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:55.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1284468981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:56.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:56.645188+0000) 2023-08-28T01:24:57.644 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:57 smithi136 ceph-mon[113531]: pgmap v8141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:57 smithi093 ceph-mon[111335]: pgmap v8141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:57 smithi093 ceph-mon[118902]: pgmap v8141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:57.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:57.645443+0000) 2023-08-28T01:24:58.644 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/241562182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/241562182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/241562182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:24:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:58.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:58.645659+0000) 2023-08-28T01:24:59.644 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:24:59 smithi136 ceph-mon[113531]: pgmap v8142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:24:59 smithi093 ceph-mon[111335]: pgmap v8142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:24:59 smithi093 ceph-mon[118902]: pgmap v8142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:24:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:24:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:24:59.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:24:59.645922+0000) 2023-08-28T01:25:00.645 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1951620507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1951620507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1951620507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:00.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:00.646191+0000) 2023-08-28T01:25:01.645 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:01 smithi136 ceph-mon[113531]: pgmap v8143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:01 smithi093 ceph-mon[111335]: pgmap v8143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:01 smithi093 ceph-mon[118902]: pgmap v8143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:01.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:01.287Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:01.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:01.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:01.646433+0000) 2023-08-28T01:25:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/88542409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:02.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:02.646666+0000) 2023-08-28T01:25:02.733 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/88542409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:02.734 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/88542409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:02.735Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:02.735Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:03 smithi093 ceph-mon[111335]: pgmap v8144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:03 smithi093 ceph-mon[118902]: pgmap v8144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:03 smithi136 ceph-mon[113531]: pgmap v8144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:03.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:03.646900+0000) 2023-08-28T01:25:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:04.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:04.647160+0000) 2023-08-28T01:25:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:05 smithi093 ceph-mon[111335]: pgmap v8145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/668864047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:05 smithi093 ceph-mon[118902]: pgmap v8145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/668864047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:25:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:25:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:25:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:05 smithi136 ceph-mon[113531]: pgmap v8145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/668864047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:05.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:05.647391+0000) 2023-08-28T01:25:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:06.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:06.647594+0000) 2023-08-28T01:25:07.646 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:07 smithi136 ceph-mon[113531]: pgmap v8146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:07.647 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1347682962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:07 smithi093 ceph-mon[111335]: pgmap v8146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1347682962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:07 smithi093 ceph-mon[118902]: pgmap v8146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1347682962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:07.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:07.647815+0000) 2023-08-28T01:25:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:08.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:08.648070+0000) 2023-08-28T01:25:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:09 smithi093 ceph-mon[111335]: pgmap v8147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:09 smithi093 ceph-mon[118902]: pgmap v8147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:09 smithi136 ceph-mon[113531]: pgmap v8147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:09.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:09.648308+0000) 2023-08-28T01:25:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2463053883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2463053883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2463053883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:10.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:10.648554+0000) 2023-08-28T01:25:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:11 smithi093 ceph-mon[111335]: pgmap v8148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:11 smithi093 ceph-mon[118902]: pgmap v8148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:11.286Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:11.287Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:11 smithi136 ceph-mon[113531]: pgmap v8148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:11.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:11.648798+0000) 2023-08-28T01:25:12.734 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1841366758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:12.734 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1841366758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1841366758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:12.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:12.649088+0000) 2023-08-28T01:25:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:12.735Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:12.736Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:13 smithi093 ceph-mon[111335]: pgmap v8149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:13 smithi093 ceph-mon[118902]: pgmap v8149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:13 smithi136 ceph-mon[113531]: pgmap v8149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:13.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:13.649313+0000) 2023-08-28T01:25:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3598462656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3598462656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3598462656' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:14.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:14.649556+0000) 2023-08-28T01:25:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:15 smithi093 ceph-mon[111335]: pgmap v8150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:25:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:25:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:25:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:15 smithi093 ceph-mon[118902]: pgmap v8150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:15 smithi136 ceph-mon[113531]: pgmap v8150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:15.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:15.649804+0000) 2023-08-28T01:25:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:16.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:16.650005+0000) 2023-08-28T01:25:17.649 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:17 smithi136 ceph-mon[113531]: pgmap v8151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:17.649 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/876530306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:17 smithi093 ceph-mon[111335]: pgmap v8151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/876530306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:17 smithi093 ceph-mon[118902]: pgmap v8151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/876530306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:17.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:17.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:17.650268+0000) 2023-08-28T01:25:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:18.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:18.650544+0000) 2023-08-28T01:25:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:19 smithi093 ceph-mon[118902]: pgmap v8152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3152911750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:19 smithi093 ceph-mon[111335]: pgmap v8152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:19.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3152911750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:19 smithi136 ceph-mon[113531]: pgmap v8152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3152911750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:19.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:19.650778+0000) 2023-08-28T01:25:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:20.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:20.651014+0000) 2023-08-28T01:25:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:21 smithi093 ceph-mon[111335]: pgmap v8153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:21.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/451252100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:21 smithi093 ceph-mon[118902]: pgmap v8153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/451252100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:21.287Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:21.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:21 smithi136 ceph-mon[113531]: pgmap v8153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/451252100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:21.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:21.651306+0000) 2023-08-28T01:25:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:22.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:22.651554+0000) 2023-08-28T01:25:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:22.735Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:22.737Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:23 smithi093 ceph-mon[111335]: pgmap v8154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:23 smithi093 ceph-mon[118902]: pgmap v8154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:23 smithi136 ceph-mon[113531]: pgmap v8154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:23.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:23.651794+0000) 2023-08-28T01:25:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1220333836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1220333836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1220333836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:24.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:24.652069+0000) 2023-08-28T01:25:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:25 smithi093 ceph-mon[111335]: pgmap v8155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:25 smithi093 ceph-mon[118902]: pgmap v8155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:25:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:25:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:25:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:25 smithi136 ceph-mon[113531]: pgmap v8155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:25.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:25.652295+0000) 2023-08-28T01:25:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3104637897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3104637897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3104637897' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:26.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:26.652510+0000) 2023-08-28T01:25:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:27 smithi093 ceph-mon[118902]: pgmap v8156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:27 smithi093 ceph-mon[111335]: pgmap v8156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:27 smithi136 ceph-mon[113531]: pgmap v8156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:27.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:27.652738+0000) 2023-08-28T01:25:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:28.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:28.652992+0000) 2023-08-28T01:25:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:29 smithi093 ceph-mon[111335]: pgmap v8157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/443291360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:29 smithi093 ceph-mon[118902]: pgmap v8157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/443291360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:29 smithi136 ceph-mon[113531]: pgmap v8157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/443291360' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:29.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:29.653275+0000) 2023-08-28T01:25:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:30.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:30.653517+0000) 2023-08-28T01:25:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:31 smithi093 ceph-mon[111335]: pgmap v8158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2745931833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:31 smithi093 ceph-mon[118902]: pgmap v8158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2745931833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:31.288Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:31.288Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:31 smithi136 ceph-mon[113531]: pgmap v8158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2745931833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:31.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:31.653733+0000) 2023-08-28T01:25:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:32.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:32.653957+0000) 2023-08-28T01:25:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:32.736Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:32.737Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:33 smithi093 ceph-mon[111335]: pgmap v8159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3720495981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:33 smithi093 ceph-mon[118902]: pgmap v8159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3720495981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:33 smithi136 ceph-mon[113531]: pgmap v8159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3720495981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:33.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:33.654222+0000) 2023-08-28T01:25:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:34.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:34.654516+0000) 2023-08-28T01:25:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:35 smithi093 ceph-mon[111335]: pgmap v8160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:35 smithi093 ceph-mon[118902]: pgmap v8160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:25:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:25:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:25:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:35 smithi136 ceph-mon[113531]: pgmap v8160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:35.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:35.654694+0000) 2023-08-28T01:25:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/420372680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/420372680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/420372680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:36.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:36.654899+0000) 2023-08-28T01:25:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:37 smithi093 ceph-mon[111335]: pgmap v8161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:37 smithi093 ceph-mon[118902]: pgmap v8161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:37 smithi136 ceph-mon[113531]: pgmap v8161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:37.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:37.655095+0000) 2023-08-28T01:25:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3803884135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3803884135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:38.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3803884135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:38.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:38.655353+0000) 2023-08-28T01:25:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:39 smithi093 ceph-mon[111335]: pgmap v8162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:39 smithi093 ceph-mon[118902]: pgmap v8162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:39 smithi136 ceph-mon[113531]: pgmap v8162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:39.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:39.655612+0000) 2023-08-28T01:25:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:40.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:40.655860+0000) 2023-08-28T01:25:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:41 smithi093 ceph-mon[111335]: pgmap v8163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1251925592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:41 smithi093 ceph-mon[118902]: pgmap v8163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1251925592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:41.288Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:41.289Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:41 smithi136 ceph-mon[113531]: pgmap v8163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1251925592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:41.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:41.656147+0000) 2023-08-28T01:25:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:42.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:42.656406+0000) 2023-08-28T01:25:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:42.737Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:42.738Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:43 smithi136 ceph-mon[113531]: pgmap v8164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2285703722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:43.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:43.656680+0000) 2023-08-28T01:25:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:43 smithi093 ceph-mon[111335]: pgmap v8164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2285703722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:43 smithi093 ceph-mon[118902]: pgmap v8164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2285703722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:25:44.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:25:44.821 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:25:44.821 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:44.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:44.656898+0000) 2023-08-28T01:25:45.656 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:45 smithi136 ceph-mon[113531]: pgmap v8165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:45.656 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/186529640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:45.656 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:45.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:45.657093+0000) 2023-08-28T01:25:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:45 smithi093 ceph-mon[111335]: pgmap v8165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/186529640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:45 smithi093 ceph-mon[118902]: pgmap v8165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/186529640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:25:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:25:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:25:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:46.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:46.657236+0000) 2023-08-28T01:25:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:46.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:47 smithi136 ceph-mon[113531]: pgmap v8166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:25:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:25:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:47.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:47.657438+0000) 2023-08-28T01:25:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[111335]: pgmap v8166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:25:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:25:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[118902]: pgmap v8166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:25:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:25:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:25:48.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/820670448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:48.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:48.657707+0000) 2023-08-28T01:25:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/820670448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/820670448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:49 smithi136 ceph-mon[113531]: pgmap v8167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:49.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:49.657963+0000) 2023-08-28T01:25:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:49 smithi093 ceph-mon[111335]: pgmap v8167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:49 smithi093 ceph-mon[118902]: pgmap v8167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1072806050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:50.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:50.658199+0000) 2023-08-28T01:25:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1072806050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1072806050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:51.609 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:51.288Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:51.609 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:51.290Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:51 smithi136 ceph-mon[113531]: pgmap v8168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:51.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:51.658471+0000) 2023-08-28T01:25:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:51 smithi093 ceph-mon[111335]: pgmap v8168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:51 smithi093 ceph-mon[118902]: pgmap v8168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4138492507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:52.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:52.658679+0000) 2023-08-28T01:25:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4138492507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4138492507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:52.738Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:25:52.739Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:25:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:53 smithi136 ceph-mon[113531]: pgmap v8169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:53.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:53.658917+0000) 2023-08-28T01:25:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:53 smithi093 ceph-mon[111335]: pgmap v8169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:53 smithi093 ceph-mon[118902]: pgmap v8169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:54.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:54.659179+0000) 2023-08-28T01:25:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:25:55.620 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:25:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:25:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:25:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:55 smithi136 ceph-mon[113531]: pgmap v8170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1238087156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:55.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:55.659404+0000) 2023-08-28T01:25:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:55 smithi093 ceph-mon[118902]: pgmap v8170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1238087156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:55 smithi093 ceph-mon[111335]: pgmap v8170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1238087156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:56.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:56.659628+0000) 2023-08-28T01:25:57.658 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:57 smithi136 ceph-mon[113531]: pgmap v8171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:57.659 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2696695730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:57 smithi093 ceph-mon[111335]: pgmap v8171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2696695730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:57 smithi093 ceph-mon[118902]: pgmap v8171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2696695730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:25:57.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:57.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:57.659852+0000) 2023-08-28T01:25:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:58.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:58.660057+0000) 2023-08-28T01:25:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:25:59 smithi093 ceph-mon[111335]: pgmap v8172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:25:59 smithi093 ceph-mon[118902]: pgmap v8172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:25:59 smithi136 ceph-mon[113531]: pgmap v8172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:25:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:25:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:25:59.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:25:59.660273+0000) 2023-08-28T01:26:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2457263580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2457263580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2457263580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:00.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:00.660499+0000) 2023-08-28T01:26:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:01 smithi093 ceph-mon[111335]: pgmap v8173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:01 smithi093 ceph-mon[118902]: pgmap v8173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:01.289Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:01.291Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:01 smithi136 ceph-mon[113531]: pgmap v8173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:01.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:01.660750+0000) 2023-08-28T01:26:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1461567911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1461567911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1461567911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:02.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:02.660985+0000) 2023-08-28T01:26:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:02.739Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:02.740Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:03 smithi093 ceph-mon[111335]: pgmap v8174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:03 smithi093 ceph-mon[118902]: pgmap v8174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:03 smithi136 ceph-mon[113531]: pgmap v8174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:03.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:03.661199+0000) 2023-08-28T01:26:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2779351621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2779351621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2779351621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:04.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:04.661370+0000) 2023-08-28T01:26:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:05 smithi093 ceph-mon[118902]: pgmap v8175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:05 smithi093 ceph-mon[111335]: pgmap v8175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:26:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:26:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:26:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:05 smithi136 ceph-mon[113531]: pgmap v8175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:05.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:05.661603+0000) 2023-08-28T01:26:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:06.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:06.661868+0000) 2023-08-28T01:26:07.661 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:07 smithi136 ceph-mon[113531]: pgmap v8176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:07.661 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778689156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:07 smithi093 ceph-mon[118902]: pgmap v8176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778689156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:07 smithi093 ceph-mon[111335]: pgmap v8176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778689156' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:07.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:07.662146+0000) 2023-08-28T01:26:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:08.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:08.662329+0000) 2023-08-28T01:26:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:09 smithi093 ceph-mon[111335]: pgmap v8177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1692182492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:09 smithi093 ceph-mon[118902]: pgmap v8177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1692182492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:09 smithi136 ceph-mon[113531]: pgmap v8177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1692182492' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:09.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:09.662587+0000) 2023-08-28T01:26:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:10.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:10.662844+0000) 2023-08-28T01:26:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:11 smithi093 ceph-mon[111335]: pgmap v8178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:11 smithi093 ceph-mon[118902]: pgmap v8178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:11.291Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:11.292Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:11 smithi136 ceph-mon[113531]: pgmap v8178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:11.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:11.663120+0000) 2023-08-28T01:26:12.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2588113438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:12.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2588113438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2588113438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:12.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:12.663294+0000) 2023-08-28T01:26:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:12.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:12.740Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:13 smithi093 ceph-mon[111335]: pgmap v8179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:13 smithi093 ceph-mon[118902]: pgmap v8179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:13 smithi136 ceph-mon[113531]: pgmap v8179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:13.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:13.663576+0000) 2023-08-28T01:26:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3238514560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3238514560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3238514560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:14.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:14.663835+0000) 2023-08-28T01:26:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:15 smithi093 ceph-mon[118902]: pgmap v8180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:15 smithi093 ceph-mon[111335]: pgmap v8180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:26:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:26:15] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:26:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:15 smithi136 ceph-mon[113531]: pgmap v8180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:15.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:15.664110+0000) 2023-08-28T01:26:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1671409141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1671409141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1671409141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:16.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:16.664310+0000) 2023-08-28T01:26:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:17 smithi093 ceph-mon[118902]: pgmap v8181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:17 smithi093 ceph-mon[111335]: pgmap v8181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:17 smithi136 ceph-mon[113531]: pgmap v8181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:17.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:17.664574+0000) 2023-08-28T01:26:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:18.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:18.664850+0000) 2023-08-28T01:26:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:19 smithi093 ceph-mon[111335]: pgmap v8182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4276868966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:19 smithi093 ceph-mon[118902]: pgmap v8182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4276868966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:19 smithi136 ceph-mon[113531]: pgmap v8182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4276868966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:19.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:19.665133+0000) 2023-08-28T01:26:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:20.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:20.665402+0000) 2023-08-28T01:26:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:21 smithi093 ceph-mon[111335]: pgmap v8183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2486140959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:21 smithi093 ceph-mon[118902]: pgmap v8183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:21.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2486140959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:21.292Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:21.293Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:21 smithi136 ceph-mon[113531]: pgmap v8183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2486140959' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:21.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:21.665641+0000) 2023-08-28T01:26:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:22.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:22.665874+0000) 2023-08-28T01:26:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:22.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:22.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:23 smithi093 ceph-mon[111335]: pgmap v8184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1708535129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:23 smithi093 ceph-mon[118902]: pgmap v8184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1708535129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:23 smithi136 ceph-mon[113531]: pgmap v8184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1708535129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:23.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:23.666141+0000) 2023-08-28T01:26:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:24.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:24.666399+0000) 2023-08-28T01:26:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:25 smithi093 ceph-mon[111335]: pgmap v8185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:25 smithi093 ceph-mon[118902]: pgmap v8185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:26:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:26:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:26:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:25 smithi136 ceph-mon[113531]: pgmap v8185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:25.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:25.666660+0000) 2023-08-28T01:26:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/501043416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:26.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:26.666914+0000) 2023-08-28T01:26:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/501043416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:26.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/501043416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:27 smithi136 ceph-mon[113531]: pgmap v8186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:27.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:27.667177+0000) 2023-08-28T01:26:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:27 smithi093 ceph-mon[111335]: pgmap v8186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:27 smithi093 ceph-mon[118902]: pgmap v8186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/870374793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:28.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:28.667444+0000) 2023-08-28T01:26:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/870374793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/870374793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:29 smithi136 ceph-mon[113531]: pgmap v8187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:29.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:29.667704+0000) 2023-08-28T01:26:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:29 smithi093 ceph-mon[111335]: pgmap v8187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:29 smithi093 ceph-mon[118902]: pgmap v8187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:30.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:30.667967+0000) 2023-08-28T01:26:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:31 smithi093 ceph-mon[111335]: pgmap v8188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2732382408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:31 smithi093 ceph-mon[118902]: pgmap v8188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2732382408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:31.293Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:31.294Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:31 smithi136 ceph-mon[113531]: pgmap v8188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2732382408' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:31.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:31.668174+0000) 2023-08-28T01:26:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:32.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:32.668401+0000) 2023-08-28T01:26:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:32.740Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:32.741Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:33 smithi136 ceph-mon[113531]: pgmap v8189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3073681173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:33.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:33.668687+0000) 2023-08-28T01:26:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:33 smithi093 ceph-mon[111335]: pgmap v8189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3073681173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:33 smithi093 ceph-mon[118902]: pgmap v8189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3073681173' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:34.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:34.668927+0000) 2023-08-28T01:26:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:35 smithi093 ceph-mon[111335]: pgmap v8190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3149010153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:35 smithi093 ceph-mon[118902]: pgmap v8190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3149010153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:26:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:26:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:26:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:35 smithi136 ceph-mon[113531]: pgmap v8190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3149010153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:35.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:35.669150+0000) 2023-08-28T01:26:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:36.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:36.669393+0000) 2023-08-28T01:26:37.668 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:37 smithi136 ceph-mon[113531]: pgmap v8191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:37 smithi093 ceph-mon[111335]: pgmap v8191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:37 smithi093 ceph-mon[118902]: pgmap v8191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:37.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:37.669600+0000) 2023-08-28T01:26:38.668 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3242841279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3242841279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3242841279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:38.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:38.669844+0000) 2023-08-28T01:26:39.668 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:39 smithi136 ceph-mon[113531]: pgmap v8192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:39.669 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:39 smithi093 ceph-mon[111335]: pgmap v8192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:39 smithi093 ceph-mon[118902]: pgmap v8192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:39.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:39.670120+0000) 2023-08-28T01:26:40.669 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4285392450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4285392450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4285392450' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:40.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:40.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:40.670345+0000) 2023-08-28T01:26:41.669 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:41 smithi136 ceph-mon[113531]: pgmap v8193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:41 smithi093 ceph-mon[111335]: pgmap v8193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:41 smithi093 ceph-mon[118902]: pgmap v8193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:41.293Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:41.294Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:41.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:41.670603+0000) 2023-08-28T01:26:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3077980415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:42.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:42.670868+0000) 2023-08-28T01:26:42.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3077980415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3077980415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:42.741Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:42.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:43 smithi136 ceph-mon[113531]: pgmap v8194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:43.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:43.671104+0000) 2023-08-28T01:26:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:43 smithi093 ceph-mon[111335]: pgmap v8194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:43 smithi093 ceph-mon[118902]: pgmap v8194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:44.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:44.671373+0000) 2023-08-28T01:26:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:45 smithi093 ceph-mon[111335]: pgmap v8195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4183836956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:26:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:26:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:26:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:45 smithi093 ceph-mon[118902]: pgmap v8195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4183836956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:45 smithi136 ceph-mon[113531]: pgmap v8195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4183836956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:45.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:45.671632+0000) 2023-08-28T01:26:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:46.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:46.671879+0000) 2023-08-28T01:26:47.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:47 smithi136 ceph-mon[113531]: pgmap v8196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:47.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:26:47.671 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1569081328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:47 smithi093 ceph-mon[111335]: pgmap v8196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:26:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1569081328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:47 smithi093 ceph-mon[118902]: pgmap v8196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:26:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1569081328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:47.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:47.672052+0000) 2023-08-28T01:26:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:48.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:48.672200+0000) 2023-08-28T01:26:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:49.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:49.672348+0000) 2023-08-28T01:26:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:49 smithi136 ceph-mon[113531]: pgmap v8197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/425661106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[111335]: pgmap v8197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/425661106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[118902]: pgmap v8197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:49.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/425661106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:50.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:50.672550+0000) 2023-08-28T01:26:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:26:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:26:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:26:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:26:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:26:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:26:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:26:51.685 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:51.294Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:51.685 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:51.295Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:51 smithi136 ceph-mon[113531]: pgmap v8198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:51.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:51.672795+0000) 2023-08-28T01:26:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:51 smithi093 ceph-mon[111335]: pgmap v8198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:51 smithi093 ceph-mon[118902]: pgmap v8198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1618755338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:52.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:52.673059+0000) 2023-08-28T01:26:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1618755338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1618755338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:52.742Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:26:52.742Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:26:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:53 smithi136 ceph-mon[113531]: pgmap v8199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:53.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:53.673288+0000) 2023-08-28T01:26:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:53 smithi093 ceph-mon[111335]: pgmap v8199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:53 smithi093 ceph-mon[118902]: pgmap v8199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/503993128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:54.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:54.673571+0000) 2023-08-28T01:26:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/503993128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:26:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/503993128' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:55.698 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:26:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:26:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:26:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:55 smithi136 ceph-mon[113531]: pgmap v8200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:55.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:55.673776+0000) 2023-08-28T01:26:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:55 smithi093 ceph-mon[111335]: pgmap v8200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:55 smithi093 ceph-mon[118902]: pgmap v8200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:56.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:56.674016+0000) 2023-08-28T01:26:57.673 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:57 smithi136 ceph-mon[113531]: pgmap v8201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:57.673 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3385507291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:57 smithi093 ceph-mon[111335]: pgmap v8201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3385507291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:57 smithi093 ceph-mon[118902]: pgmap v8201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3385507291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:57.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:57.674292+0000) 2023-08-28T01:26:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:58.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:58.674518+0000) 2023-08-28T01:26:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:59 smithi093 ceph-mon[111335]: pgmap v8202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:26:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1327382163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:59 smithi093 ceph-mon[118902]: pgmap v8202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:26:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1327382163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:59 smithi136 ceph-mon[113531]: pgmap v8202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:26:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:26:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1327382163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:26:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:26:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:26:59.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:26:59.674733+0000) 2023-08-28T01:27:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:00.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:00.674990+0000) 2023-08-28T01:27:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:01 smithi093 ceph-mon[111335]: pgmap v8203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:01 smithi093 ceph-mon[118902]: pgmap v8203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:01.295Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:01.296Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:01 smithi136 ceph-mon[113531]: pgmap v8203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:01.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:01.675210+0000) 2023-08-28T01:27:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3476905441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3476905441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3476905441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:02.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:02.675449+0000) 2023-08-28T01:27:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:02.742Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:02.743Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:03 smithi093 ceph-mon[111335]: pgmap v8204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:03 smithi093 ceph-mon[118902]: pgmap v8204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:03 smithi136 ceph-mon[113531]: pgmap v8204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:03.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:03.675710+0000) 2023-08-28T01:27:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/23464476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/23464476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/23464476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:04.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:04.675974+0000) 2023-08-28T01:27:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:05 smithi093 ceph-mon[111335]: pgmap v8205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:05 smithi093 ceph-mon[118902]: pgmap v8205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:27:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:27:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:27:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:05 smithi136 ceph-mon[113531]: pgmap v8205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:05.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:05.676239+0000) 2023-08-28T01:27:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:06.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:06.676505+0000) 2023-08-28T01:27:07.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3857872490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:07.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:07 smithi136 ceph-mon[113531]: pgmap v8206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3857872490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:07 smithi093 ceph-mon[111335]: pgmap v8206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3857872490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:07 smithi093 ceph-mon[118902]: pgmap v8206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:07.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:07.676730+0000) 2023-08-28T01:27:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:08.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:08.676971+0000) 2023-08-28T01:27:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:09 smithi093 ceph-mon[111335]: pgmap v8207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2328713483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:09 smithi093 ceph-mon[118902]: pgmap v8207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2328713483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:09 smithi136 ceph-mon[113531]: pgmap v8207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2328713483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:09.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:09.677217+0000) 2023-08-28T01:27:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:10.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:10.677454+0000) 2023-08-28T01:27:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:11 smithi093 ceph-mon[111335]: pgmap v8208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2962286024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:11 smithi093 ceph-mon[118902]: pgmap v8208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2962286024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:11.295Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:11.296Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:11 smithi136 ceph-mon[113531]: pgmap v8208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2962286024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:11.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:11.677729+0000) 2023-08-28T01:27:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:12.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:12.677966+0000) 2023-08-28T01:27:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:12.743Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:12.744Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:13 smithi093 ceph-mon[111335]: pgmap v8209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/212611684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:13 smithi093 ceph-mon[118902]: pgmap v8209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/212611684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:13 smithi136 ceph-mon[113531]: pgmap v8209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/212611684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:13.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:13.678228+0000) 2023-08-28T01:27:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:14.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:14.678491+0000) 2023-08-28T01:27:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:15 smithi093 ceph-mon[111335]: pgmap v8210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:15 smithi093 ceph-mon[118902]: pgmap v8210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:27:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:27:15] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:27:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:15 smithi136 ceph-mon[113531]: pgmap v8210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:15.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:15.678730+0000) 2023-08-28T01:27:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1634056612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1634056612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1634056612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:16.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:16.678962+0000) 2023-08-28T01:27:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:17 smithi093 ceph-mon[111335]: pgmap v8211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:17 smithi093 ceph-mon[118902]: pgmap v8211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:17 smithi136 ceph-mon[113531]: pgmap v8211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:17.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:17.679211+0000) 2023-08-28T01:27:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2675334858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:18.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2675334858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:18.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2675334858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:18.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:18.679462+0000) 2023-08-28T01:27:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:19 smithi093 ceph-mon[111335]: pgmap v8212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:19 smithi093 ceph-mon[118902]: pgmap v8212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:19 smithi136 ceph-mon[113531]: pgmap v8212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:19.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:19.679695+0000) 2023-08-28T01:27:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:20.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:20.679935+0000) 2023-08-28T01:27:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:21 smithi093 ceph-mon[111335]: pgmap v8213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1001008969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:21 smithi093 ceph-mon[118902]: pgmap v8213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1001008969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:21.296Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:21.297Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:21 smithi136 ceph-mon[113531]: pgmap v8213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1001008969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:21.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:21.680190+0000) 2023-08-28T01:27:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:22.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:22.680426+0000) 2023-08-28T01:27:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:22.744Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:22.745Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:23 smithi136 ceph-mon[113531]: pgmap v8214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/756100276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:23.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:23.680676+0000) 2023-08-28T01:27:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:23 smithi093 ceph-mon[111335]: pgmap v8214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/756100276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:23 smithi093 ceph-mon[118902]: pgmap v8214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/756100276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:24.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:24.680940+0000) 2023-08-28T01:27:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:27:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:27:25] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:27:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:25 smithi093 ceph-mon[111335]: pgmap v8215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1840527211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:25 smithi093 ceph-mon[118902]: pgmap v8215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1840527211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:25 smithi136 ceph-mon[113531]: pgmap v8215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1840527211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:25.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:25.681191+0000) 2023-08-28T01:27:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:26.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:26.681448+0000) 2023-08-28T01:27:27.680 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:27 smithi136 ceph-mon[113531]: pgmap v8216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:27 smithi093 ceph-mon[111335]: pgmap v8216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:27 smithi093 ceph-mon[118902]: pgmap v8216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:27.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:27.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:27.681735+0000) 2023-08-28T01:27:28.680 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/588631657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/588631657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/588631657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:28.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:28.682006+0000) 2023-08-28T01:27:29.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:29 smithi136 ceph-mon[113531]: pgmap v8217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:29 smithi093 ceph-mon[111335]: pgmap v8217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:29 smithi093 ceph-mon[118902]: pgmap v8217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:29.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:29.682239+0000) 2023-08-28T01:27:30.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/912807105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/912807105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/912807105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:30.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:30.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:30.682473+0000) 2023-08-28T01:27:31.681 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:31 smithi136 ceph-mon[113531]: pgmap v8218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:31 smithi093 ceph-mon[111335]: pgmap v8218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:31 smithi093 ceph-mon[118902]: pgmap v8218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:31.297Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:31.298Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:31.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:31.682732+0000) 2023-08-28T01:27:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:32.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:32.683006+0000) 2023-08-28T01:27:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:32.745Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:32.746Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:33 smithi093 ceph-mon[118902]: pgmap v8219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/463352017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:33 smithi093 ceph-mon[111335]: pgmap v8219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/463352017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:33.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:33.683203+0000) 2023-08-28T01:27:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:33 smithi136 ceph-mon[113531]: pgmap v8219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/463352017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:34.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:34.683444+0000) 2023-08-28T01:27:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:27:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:27:35] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:27:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:35 smithi093 ceph-mon[118902]: pgmap v8220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2968519522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:35 smithi093 ceph-mon[111335]: pgmap v8220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2968519522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:35.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:35.683707+0000) 2023-08-28T01:27:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:35 smithi136 ceph-mon[113531]: pgmap v8220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2968519522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:36.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:36.683964+0000) 2023-08-28T01:27:37.682 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:37 smithi136 ceph-mon[113531]: pgmap v8221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:37.683 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2662162197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:37 smithi093 ceph-mon[111335]: pgmap v8221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2662162197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:37 smithi093 ceph-mon[118902]: pgmap v8221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2662162197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:37.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:37.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:37.684226+0000) 2023-08-28T01:27:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:38.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:38.684458+0000) 2023-08-28T01:27:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:39 smithi093 ceph-mon[118902]: pgmap v8222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:39 smithi093 ceph-mon[111335]: pgmap v8222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:39 smithi136 ceph-mon[113531]: pgmap v8222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:39.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:39.684704+0000) 2023-08-28T01:27:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/154518301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/154518301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:40.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/154518301' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:40.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:40.684914+0000) 2023-08-28T01:27:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:41 smithi093 ceph-mon[111335]: pgmap v8223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:41 smithi093 ceph-mon[118902]: pgmap v8223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:41.297Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:41.298Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:41 smithi136 ceph-mon[113531]: pgmap v8223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:41.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:41.685180+0000) 2023-08-28T01:27:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/553753030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:42.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/553753030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/553753030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:42.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:42.685433+0000) 2023-08-28T01:27:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:42.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:42.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:43 smithi136 ceph-mon[113531]: pgmap v8224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:43.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:43.685703+0000) 2023-08-28T01:27:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:43 smithi093 ceph-mon[111335]: pgmap v8224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:43 smithi093 ceph-mon[118902]: pgmap v8224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:44.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3370562895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:44.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:44.685887+0000) 2023-08-28T01:27:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3370562895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3370562895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:45 smithi093 ceph-mon[111335]: pgmap v8225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:27:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:27:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:27:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:45 smithi093 ceph-mon[118902]: pgmap v8225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:45 smithi136 ceph-mon[113531]: pgmap v8225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:45.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:45.686165+0000) 2023-08-28T01:27:46.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:46.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:46.686412+0000) 2023-08-28T01:27:47.685 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:47 smithi136 ceph-mon[113531]: pgmap v8226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:47.685 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3397592843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:47 smithi093 ceph-mon[118902]: pgmap v8226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3397592843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:47 smithi093 ceph-mon[111335]: pgmap v8226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3397592843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:47.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:47.686622+0000) 2023-08-28T01:27:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:48.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:48.686812+0000) 2023-08-28T01:27:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:49 smithi093 ceph-mon[111335]: pgmap v8227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4112959791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:49 smithi093 ceph-mon[118902]: pgmap v8227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4112959791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:49 smithi136 ceph-mon[113531]: pgmap v8227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4112959791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:49.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:49.687016+0000) 2023-08-28T01:27:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:27:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:27:50.843 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:27:50.843 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:50.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:50.687245+0000) 2023-08-28T01:27:51.418 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:51.298Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:51.418 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:51.299Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:51.686 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:51 smithi136 ceph-mon[113531]: pgmap v8228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:51 smithi093 ceph-mon[111335]: pgmap v8228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:51 smithi093 ceph-mon[118902]: pgmap v8228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:51.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:51.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:51.687423+0000) 2023-08-28T01:27:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1836395376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1836395376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:52.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:52.687581+0000) 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1836395376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:27:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:52.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:52.746Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:27:52.747Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:27:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:53.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:53.687849+0000) 2023-08-28T01:27:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:53 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2429M 2023-08-28T01:27:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:53 smithi136 ceph-mon[113531]: pgmap v8229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:27:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:27:53.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2429M 2023-08-28T01:27:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[111335]: pgmap v8229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:27:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:27:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2429M 2023-08-28T01:27:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[118902]: pgmap v8229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:27:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:27:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:27:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3724483403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:54.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:54.688162+0000) 2023-08-28T01:27:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3724483403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:27:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3724483403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:55 smithi093 ceph-mon[118902]: pgmap v8230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:55 smithi093 ceph-mon[111335]: pgmap v8230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:27:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:27:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:27:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:55 smithi136 ceph-mon[113531]: pgmap v8230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:55.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:55.688405+0000) 2023-08-28T01:27:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:56.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:56.688665+0000) 2023-08-28T01:27:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3726804115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:57 smithi136 ceph-mon[113531]: pgmap v8231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3726804115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:57 smithi093 ceph-mon[111335]: pgmap v8231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3726804115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:57 smithi093 ceph-mon[118902]: pgmap v8231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:57.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:57.688935+0000) 2023-08-28T01:27:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:58.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:58.689248+0000) 2023-08-28T01:27:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:59 smithi093 ceph-mon[118902]: pgmap v8232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:27:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4196741556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:59 smithi093 ceph-mon[111335]: pgmap v8232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:59.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:27:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4196741556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:59 smithi136 ceph-mon[113531]: pgmap v8232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:27:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:27:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4196741556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:27:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:27:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:27:59.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:27:59.689503+0000) 2023-08-28T01:28:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:00.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:00.689789+0000) 2023-08-28T01:28:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:01 smithi093 ceph-mon[111335]: pgmap v8233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/710274615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:01 smithi093 ceph-mon[118902]: pgmap v8233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/710274615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:01.299Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:01.300Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:01 smithi136 ceph-mon[113531]: pgmap v8233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/710274615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:01.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:01.690045+0000) 2023-08-28T01:28:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:02.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:02.690316+0000) 2023-08-28T01:28:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:02.747Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:02.748Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:03 smithi136 ceph-mon[113531]: pgmap v8234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:03.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:03.690601+0000) 2023-08-28T01:28:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:03 smithi093 ceph-mon[111335]: pgmap v8234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:03 smithi093 ceph-mon[118902]: pgmap v8234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2507833915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:04.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:04.690875+0000) 2023-08-28T01:28:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2507833915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2507833915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:05 smithi093 ceph-mon[111335]: pgmap v8235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:28:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:28:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:28:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:05 smithi093 ceph-mon[118902]: pgmap v8235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:05 smithi136 ceph-mon[113531]: pgmap v8235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:05.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:05.691082+0000) 2023-08-28T01:28:06.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1486276648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:06.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:06.691346+0000) 2023-08-28T01:28:06.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1486276648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:06.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1486276648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:07 smithi136 ceph-mon[113531]: pgmap v8236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:07.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:07.691600+0000) 2023-08-28T01:28:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:07 smithi093 ceph-mon[111335]: pgmap v8236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:07 smithi093 ceph-mon[118902]: pgmap v8236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/220960698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:08.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:08.691856+0000) 2023-08-28T01:28:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/220960698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/220960698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:09 smithi136 ceph-mon[113531]: pgmap v8237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:09.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:09.692162+0000) 2023-08-28T01:28:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:09 smithi093 ceph-mon[118902]: pgmap v8237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:09 smithi093 ceph-mon[111335]: pgmap v8237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:11.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:10.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:10.692402+0000) 2023-08-28T01:28:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:11 smithi093 ceph-mon[118902]: pgmap v8238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3140096785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:11 smithi093 ceph-mon[111335]: pgmap v8238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:11.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3140096785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:11.300Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:11.301Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:11 smithi136 ceph-mon[113531]: pgmap v8238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3140096785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:11.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:11.692659+0000) 2023-08-28T01:28:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:12.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:12.692956+0000) 2023-08-28T01:28:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:12.747Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:12.748Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:13 smithi136 ceph-mon[113531]: pgmap v8239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3321064862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:13.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:13.693160+0000) 2023-08-28T01:28:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:13 smithi093 ceph-mon[118902]: pgmap v8239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3321064862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:13 smithi093 ceph-mon[111335]: pgmap v8239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3321064862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:14.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:14.693402+0000) 2023-08-28T01:28:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:15 smithi093 ceph-mon[111335]: pgmap v8240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2986990127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:28:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:28:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:28:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:15 smithi093 ceph-mon[118902]: pgmap v8240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2986990127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:15.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:15 smithi136 ceph-mon[113531]: pgmap v8240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2986990127' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:15.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:15.693701+0000) 2023-08-28T01:28:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:16.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:16.693991+0000) 2023-08-28T01:28:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:17 smithi136 ceph-mon[113531]: pgmap v8241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:17 smithi093 ceph-mon[111335]: pgmap v8241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:17 smithi093 ceph-mon[118902]: pgmap v8241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:17.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:17.694246+0000) 2023-08-28T01:28:18.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/860276914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/860276914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/860276914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:18.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:18.694487+0000) 2023-08-28T01:28:19.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:19 smithi136 ceph-mon[113531]: pgmap v8242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:19 smithi093 ceph-mon[111335]: pgmap v8242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:19 smithi093 ceph-mon[118902]: pgmap v8242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:19.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:19.694768+0000) 2023-08-28T01:28:20.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3451714648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3451714648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3451714648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:20.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:20.695012+0000) 2023-08-28T01:28:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:21 smithi136 ceph-mon[113531]: pgmap v8243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:21 smithi093 ceph-mon[111335]: pgmap v8243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:21 smithi093 ceph-mon[118902]: pgmap v8243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:21.301Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:21.302Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:21.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:21.695249+0000) 2023-08-28T01:28:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:22.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:22.695517+0000) 2023-08-28T01:28:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:22.748Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:22.749Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:23 smithi136 ceph-mon[113531]: pgmap v8244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1027609203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:23.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:23.695852+0000) 2023-08-28T01:28:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:23 smithi093 ceph-mon[118902]: pgmap v8244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1027609203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:23 smithi093 ceph-mon[111335]: pgmap v8244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1027609203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:24.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:24.696117+0000) 2023-08-28T01:28:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:28:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:28:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:28:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:25 smithi093 ceph-mon[111335]: pgmap v8245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2644555210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:25 smithi093 ceph-mon[118902]: pgmap v8245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2644555210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:25 smithi136 ceph-mon[113531]: pgmap v8245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2644555210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:25.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:25.696383+0000) 2023-08-28T01:28:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:26.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:26.696632+0000) 2023-08-28T01:28:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:27 smithi136 ceph-mon[113531]: pgmap v8246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:27 smithi093 ceph-mon[111335]: pgmap v8246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:27 smithi093 ceph-mon[118902]: pgmap v8246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:27.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:27.696893+0000) 2023-08-28T01:28:28.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2306140976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2306140976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2306140976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:28.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:28.697179+0000) 2023-08-28T01:28:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:29 smithi136 ceph-mon[113531]: pgmap v8247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:29 smithi093 ceph-mon[111335]: pgmap v8247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:29 smithi093 ceph-mon[118902]: pgmap v8247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:29.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:29.697403+0000) 2023-08-28T01:28:30.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4008984559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4008984559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4008984559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:30.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:30.697651+0000) 2023-08-28T01:28:31.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:31 smithi136 ceph-mon[113531]: pgmap v8248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:31 smithi093 ceph-mon[111335]: pgmap v8248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:31 smithi093 ceph-mon[118902]: pgmap v8248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:31.302Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:31.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:31.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:31.697906+0000) 2023-08-28T01:28:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3831545907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3831545907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:32.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3831545907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:32.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:32.698186+0000) 2023-08-28T01:28:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:32.748Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:32.749Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:33.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:33 smithi136 ceph-mon[113531]: pgmap v8249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:33 smithi093 ceph-mon[111335]: pgmap v8249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:33 smithi093 ceph-mon[118902]: pgmap v8249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:33.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:33.698463+0000) 2023-08-28T01:28:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3080871227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:34.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:34.698707+0000) 2023-08-28T01:28:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3080871227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3080871227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:35 smithi093 ceph-mon[111335]: pgmap v8250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:35 smithi093 ceph-mon[118902]: pgmap v8250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:28:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:28:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:28:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:35 smithi136 ceph-mon[113531]: pgmap v8250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:35.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:35.698967+0000) 2023-08-28T01:28:37.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:36.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:36.699210+0000) 2023-08-28T01:28:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:37 smithi136 ceph-mon[113531]: pgmap v8251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1766858140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:37 smithi093 ceph-mon[111335]: pgmap v8251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1766858140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:37 smithi093 ceph-mon[118902]: pgmap v8251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1766858140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:37.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:37.699427+0000) 2023-08-28T01:28:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:38.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:38.699633+0000) 2023-08-28T01:28:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:39 smithi136 ceph-mon[113531]: pgmap v8252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3310208355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:39.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:39.699892+0000) 2023-08-28T01:28:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:39 smithi093 ceph-mon[111335]: pgmap v8252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3310208355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:39 smithi093 ceph-mon[118902]: pgmap v8252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3310208355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:40.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:40.700172+0000) 2023-08-28T01:28:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:41 smithi093 ceph-mon[111335]: pgmap v8253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:41 smithi093 ceph-mon[118902]: pgmap v8253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:41.302Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:41.303Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:41 smithi136 ceph-mon[113531]: pgmap v8253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:41.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:41.700492+0000) 2023-08-28T01:28:42.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2679413108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:42.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:42.700730+0000) 2023-08-28T01:28:42.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2679413108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2679413108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:42.749Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:42.750Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:43 smithi136 ceph-mon[113531]: pgmap v8254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:43.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:43.700978+0000) 2023-08-28T01:28:43.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:43 smithi093 ceph-mon[111335]: pgmap v8254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:43 smithi093 ceph-mon[118902]: pgmap v8254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4001007740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:44.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:44.701255+0000) 2023-08-28T01:28:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4001007740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4001007740' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:45 smithi093 ceph-mon[111335]: pgmap v8255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:45 smithi093 ceph-mon[118902]: pgmap v8255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:28:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:28:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:28:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:45 smithi136 ceph-mon[113531]: pgmap v8255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:45.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:45.701500+0000) 2023-08-28T01:28:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:46.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:46.701756+0000) 2023-08-28T01:28:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2123549744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:47 smithi136 ceph-mon[113531]: pgmap v8256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2123549744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:47 smithi093 ceph-mon[111335]: pgmap v8256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2123549744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:47 smithi093 ceph-mon[118902]: pgmap v8256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:47.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:47.702029+0000) 2023-08-28T01:28:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:48.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:48.702308+0000) 2023-08-28T01:28:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:49 smithi136 ceph-mon[113531]: pgmap v8257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3874036312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:49.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:49.702519+0000) 2023-08-28T01:28:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:49 smithi093 ceph-mon[118902]: pgmap v8257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3874036312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:49 smithi093 ceph-mon[111335]: pgmap v8257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3874036312' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:50.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:50.702711+0000) 2023-08-28T01:28:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:51 smithi093 ceph-mon[111335]: pgmap v8258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2182260812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:51 smithi093 ceph-mon[118902]: pgmap v8258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2182260812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:51.303Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:51.304Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:51 smithi136 ceph-mon[113531]: pgmap v8258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2182260812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:51.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:51.702906+0000) 2023-08-28T01:28:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:52.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:52.703169+0000) 2023-08-28T01:28:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:52.750Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:28:52.751Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:28:53.879 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:53 smithi136 ceph-mon[113531]: pgmap v8259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:53.879 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:28:53.879 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:53.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:53.703467+0000) 2023-08-28T01:28:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:53 smithi093 ceph-mon[111335]: pgmap v8259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:28:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:53 smithi093 ceph-mon[118902]: pgmap v8259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:53.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:28:54.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3624808914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3624808914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3624808914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:28:55.117 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:54.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:54.703662+0000) 2023-08-28T01:28:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:28:55] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:28:56.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:55.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:55.703852+0000) 2023-08-28T01:28:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-mon[113531]: pgmap v8260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[118902]: pgmap v8260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[111335]: pgmap v8260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.922 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4172607238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:56.922 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.922 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:56.923 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:56.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:56.704121+0000) 2023-08-28T01:28:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4172607238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4172607238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:57.752 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:57.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:57.704285+0000) 2023-08-28T01:28:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:57 smithi136 ceph-mon[113531]: pgmap v8261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:57 smithi093 ceph-mon[111335]: pgmap v8261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:57 smithi093 ceph-mon[118902]: pgmap v8261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:58.974 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:58.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:58.704556+0000) 2023-08-28T01:28:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:28:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:28:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/755171132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[111335]: pgmap v8262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/755171132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:28:58 smithi093 ceph-mon[118902]: pgmap v8262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:28:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:28:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:28:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:28:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/755171132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:28:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:28:58 smithi136 ceph-mon[113531]: pgmap v8262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:28:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:28:59.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:28:59.704807+0000) 2023-08-28T01:29:01.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:00.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:00.705068+0000) 2023-08-28T01:29:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:01 smithi093 ceph-mon[111335]: pgmap v8263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/886636713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:01 smithi093 ceph-mon[118902]: pgmap v8263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/886636713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:01.303Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:01.304Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:01.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:01.705331+0000) 2023-08-28T01:29:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:01 smithi136 ceph-mon[113531]: pgmap v8263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/886636713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:02.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:02.705572+0000) 2023-08-28T01:29:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:02.751Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:02.752Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:03 smithi136 ceph-mon[113531]: pgmap v8264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/94716436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:03.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:03.705848+0000) 2023-08-28T01:29:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:03 smithi093 ceph-mon[111335]: pgmap v8264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/94716436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:03 smithi093 ceph-mon[118902]: pgmap v8264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/94716436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:04.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:04.706163+0000) 2023-08-28T01:29:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:05 smithi093 ceph-mon[111335]: pgmap v8265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/601278052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:29:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:29:05] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:29:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:05 smithi093 ceph-mon[118902]: pgmap v8265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/601278052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:05 smithi136 ceph-mon[113531]: pgmap v8265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/601278052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:05.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:05.706388+0000) 2023-08-28T01:29:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:06.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:06.706609+0000) 2023-08-28T01:29:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:07 smithi136 ceph-mon[113531]: pgmap v8266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:07 smithi093 ceph-mon[111335]: pgmap v8266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:07 smithi093 ceph-mon[118902]: pgmap v8266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:07.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:07.706899+0000) 2023-08-28T01:29:08.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2596314702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2596314702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2596314702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:08.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:08.707138+0000) 2023-08-28T01:29:09.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:09 smithi136 ceph-mon[113531]: pgmap v8267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:09 smithi093 ceph-mon[111335]: pgmap v8267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:09 smithi093 ceph-mon[118902]: pgmap v8267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:09.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:09.707348+0000) 2023-08-28T01:29:10.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3183615193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3183615193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3183615193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:11.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:10.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:10.707652+0000) 2023-08-28T01:29:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:11 smithi136 ceph-mon[113531]: pgmap v8268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:11 smithi093 ceph-mon[111335]: pgmap v8268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:11 smithi093 ceph-mon[118902]: pgmap v8268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:11.304Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:11.305Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:11.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:11.707946+0000) 2023-08-28T01:29:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:12.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:12.708146+0000) 2023-08-28T01:29:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:12.751Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:12.752Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:13.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:13 smithi136 ceph-mon[113531]: pgmap v8269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/996778294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:13.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:13.708391+0000) 2023-08-28T01:29:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:13 smithi093 ceph-mon[111335]: pgmap v8269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/996778294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:13 smithi093 ceph-mon[118902]: pgmap v8269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/996778294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:14.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:14.708554+0000) 2023-08-28T01:29:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:15 smithi093 ceph-mon[111335]: pgmap v8270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1259202682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:29:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:29:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:29:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:15 smithi093 ceph-mon[118902]: pgmap v8270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1259202682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:15 smithi136 ceph-mon[113531]: pgmap v8270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1259202682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:15.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:15.708827+0000) 2023-08-28T01:29:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:16.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:16.709023+0000) 2023-08-28T01:29:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:17 smithi136 ceph-mon[113531]: pgmap v8271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:17 smithi093 ceph-mon[111335]: pgmap v8271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:17 smithi093 ceph-mon[118902]: pgmap v8271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:18.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:17.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:17.709292+0000) 2023-08-28T01:29:18.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2549964608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2549964608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2549964608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:18.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:18.709537+0000) 2023-08-28T01:29:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:19 smithi136 ceph-mon[113531]: pgmap v8272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:19 smithi093 ceph-mon[111335]: pgmap v8272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:19 smithi093 ceph-mon[118902]: pgmap v8272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:19.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:19.709791+0000) 2023-08-28T01:29:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1574611054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1574611054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1574611054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:20.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:20.710068+0000) 2023-08-28T01:29:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:21 smithi136 ceph-mon[113531]: pgmap v8273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:21 smithi093 ceph-mon[111335]: pgmap v8273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:21 smithi093 ceph-mon[118902]: pgmap v8273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:21.305Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:21.306Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:21.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:21.710259+0000) 2023-08-28T01:29:22.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2350906162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2350906162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:22.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2350906162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:22.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:22.710497+0000) 2023-08-28T01:29:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:22.752Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:22.753Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:23 smithi136 ceph-mon[113531]: pgmap v8274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:23 smithi093 ceph-mon[111335]: pgmap v8274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:23 smithi093 ceph-mon[118902]: pgmap v8274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:23.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:23.710695+0000) 2023-08-28T01:29:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:24.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:24.710968+0000) 2023-08-28T01:29:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:25 smithi136 ceph-mon[113531]: pgmap v8275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1902271473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:25 smithi093 ceph-mon[111335]: pgmap v8275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1902271473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:25 smithi093 ceph-mon[118902]: pgmap v8275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1902271473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:29:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:29:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:29:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:25.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:25.711219+0000) 2023-08-28T01:29:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:26.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:26.711440+0000) 2023-08-28T01:29:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:27 smithi136 ceph-mon[113531]: pgmap v8276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3448358981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:27 smithi093 ceph-mon[111335]: pgmap v8276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3448358981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:27 smithi093 ceph-mon[118902]: pgmap v8276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3448358981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:27.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:27.711650+0000) 2023-08-28T01:29:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:28.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:28.711887+0000) 2023-08-28T01:29:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:29 smithi136 ceph-mon[113531]: pgmap v8277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2333539672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:29.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:29.712114+0000) 2023-08-28T01:29:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:29 smithi093 ceph-mon[118902]: pgmap v8277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2333539672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:29 smithi093 ceph-mon[111335]: pgmap v8277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2333539672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:30.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:30.712371+0000) 2023-08-28T01:29:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:31 smithi093 ceph-mon[111335]: pgmap v8278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:31 smithi093 ceph-mon[118902]: pgmap v8278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:31.306Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:31.307Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:31 smithi136 ceph-mon[113531]: pgmap v8278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:31.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:31.712619+0000) 2023-08-28T01:29:32.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3390382779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:32.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:32.712880+0000) 2023-08-28T01:29:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3390382779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3390382779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:32.753Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:32.754Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:33 smithi136 ceph-mon[113531]: pgmap v8279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:33.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:33.713165+0000) 2023-08-28T01:29:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:33 smithi093 ceph-mon[111335]: pgmap v8279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:33.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:33 smithi093 ceph-mon[118902]: pgmap v8279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2907776541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:34.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:34.713331+0000) 2023-08-28T01:29:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2907776541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2907776541' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:35 smithi093 ceph-mon[111335]: pgmap v8280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:35 smithi093 ceph-mon[118902]: pgmap v8280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:29:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:29:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:29:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:35 smithi136 ceph-mon[113531]: pgmap v8280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:35.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:35.713592+0000) 2023-08-28T01:29:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:36.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:36.713864+0000) 2023-08-28T01:29:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1606931018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:37 smithi136 ceph-mon[113531]: pgmap v8281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1606931018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:37 smithi093 ceph-mon[111335]: pgmap v8281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1606931018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:37 smithi093 ceph-mon[118902]: pgmap v8281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:37.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:37.714142+0000) 2023-08-28T01:29:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:38.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:38.714357+0000) 2023-08-28T01:29:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:39 smithi136 ceph-mon[113531]: pgmap v8282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2763550835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:39.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:39.714581+0000) 2023-08-28T01:29:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:39 smithi093 ceph-mon[111335]: pgmap v8282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2763550835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:39 smithi093 ceph-mon[118902]: pgmap v8282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2763550835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:40.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:40.714833+0000) 2023-08-28T01:29:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:41 smithi093 ceph-mon[111335]: pgmap v8283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/422057086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:41 smithi093 ceph-mon[118902]: pgmap v8283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/422057086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:41.307Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:41.308Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:41 smithi136 ceph-mon[113531]: pgmap v8283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/422057086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:41.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:41.715091+0000) 2023-08-28T01:29:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:42.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:42.715339+0000) 2023-08-28T01:29:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:42.754Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:42.755Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:43 smithi136 ceph-mon[113531]: pgmap v8284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:43.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:43.715567+0000) 2023-08-28T01:29:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:43 smithi093 ceph-mon[111335]: pgmap v8284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:43 smithi093 ceph-mon[118902]: pgmap v8284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/523102474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:44.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:44.715839+0000) 2023-08-28T01:29:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/523102474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/523102474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:45 smithi093 ceph-mon[111335]: pgmap v8285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:45 smithi093 ceph-mon[118902]: pgmap v8285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:29:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:29:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:29:45.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:45 smithi136 ceph-mon[113531]: pgmap v8285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:45.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:45.716153+0000) 2023-08-28T01:29:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2833606198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:46.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:46.716422+0000) 2023-08-28T01:29:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2833606198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:46.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2833606198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:47 smithi136 ceph-mon[113531]: pgmap v8286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:47.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:47.716629+0000) 2023-08-28T01:29:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:47 smithi093 ceph-mon[111335]: pgmap v8286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:47 smithi093 ceph-mon[118902]: pgmap v8286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:48.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1938316084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:48.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:48.716883+0000) 2023-08-28T01:29:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1938316084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1938316084' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:49 smithi136 ceph-mon[113531]: pgmap v8287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:49.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:49.717154+0000) 2023-08-28T01:29:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:49 smithi093 ceph-mon[111335]: pgmap v8287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:49 smithi093 ceph-mon[118902]: pgmap v8287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:50.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:50.717403+0000) 2023-08-28T01:29:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:51 smithi093 ceph-mon[111335]: pgmap v8288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2140975019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:51 smithi093 ceph-mon[118902]: pgmap v8288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2140975019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:51.308Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:51.309Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:51 smithi136 ceph-mon[113531]: pgmap v8288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2140975019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:51.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:51.717610+0000) 2023-08-28T01:29:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:52.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:52.717823+0000) 2023-08-28T01:29:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:52.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:29:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:29:52.756Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:29:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:53 smithi136 ceph-mon[113531]: pgmap v8289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/585637961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:53.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:53.718071+0000) 2023-08-28T01:29:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:53 smithi093 ceph-mon[111335]: pgmap v8289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/585637961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:53 smithi093 ceph-mon[118902]: pgmap v8289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/585637961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:54.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:54.718328+0000) 2023-08-28T01:29:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:29:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:55 smithi093 ceph-mon[111335]: pgmap v8290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2142236498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:29:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:29:55] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:29:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:55 smithi093 ceph-mon[118902]: pgmap v8290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2142236498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:55 smithi136 ceph-mon[113531]: pgmap v8290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2142236498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:55.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:55.718593+0000) 2023-08-28T01:29:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:56.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:56.718849+0000) 2023-08-28T01:29:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:57 smithi136 ceph-mon[113531]: pgmap v8291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:57 smithi093 ceph-mon[111335]: pgmap v8291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:57 smithi093 ceph-mon[118902]: pgmap v8291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:57.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:57.719148+0000) 2023-08-28T01:29:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3217177846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:58 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:29:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3217177846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:58 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:29:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3217177846' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:29:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:58 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:29:59.112 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:58.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:58.719305+0000) 2023-08-28T01:29:59.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:29:59 smithi136 ceph-mon[113531]: pgmap v8292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:59.674 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:29:59 smithi093 ceph-mon[111335]: pgmap v8292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:59.674 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:29:59 smithi093 ceph-mon[118902]: pgmap v8292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:29:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:29:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:29:59.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:29:59.719513+0000) 2023-08-28T01:30:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:00.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:00.719690+0000) 2023-08-28T01:30:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:30:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1421374757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[111335]: pgmap v8293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:30:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1421374757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[118902]: pgmap v8293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1421374757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-mon[113531]: pgmap v8293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:01.308Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:01.310Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:30:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:30:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:01.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:01.719916+0000) 2023-08-28T01:30:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:30:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:30:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:30:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:30:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:30:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:02 smithi136 ceph-mon[113531]: pgmap v8294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1828279882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:02.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:02.720193+0000) 2023-08-28T01:30:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:02 smithi093 ceph-mon[111335]: pgmap v8294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1828279882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:02 smithi093 ceph-mon[118902]: pgmap v8294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1828279882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:02.755Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:02.756Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:04.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:03.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:03.720447+0000) 2023-08-28T01:30:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:04.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:04.720678+0000) 2023-08-28T01:30:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:30:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:30:05] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:30:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:05 smithi093 ceph-mon[111335]: pgmap v8295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2451270902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:05 smithi093 ceph-mon[118902]: pgmap v8295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2451270902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:05 smithi136 ceph-mon[113531]: pgmap v8295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2451270902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:05.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:05.720929+0000) 2023-08-28T01:30:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:06.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:06.721159+0000) 2023-08-28T01:30:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:07 smithi136 ceph-mon[113531]: pgmap v8296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:07 smithi093 ceph-mon[111335]: pgmap v8296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:07 smithi093 ceph-mon[118902]: pgmap v8296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:07.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:07.721400+0000) 2023-08-28T01:30:08.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2021573880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2021573880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2021573880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:08.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:08.721651+0000) 2023-08-28T01:30:09.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:09 smithi136 ceph-mon[113531]: pgmap v8297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:09 smithi093 ceph-mon[111335]: pgmap v8297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:09 smithi093 ceph-mon[118902]: pgmap v8297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:09.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:09.721855+0000) 2023-08-28T01:30:10.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3311972201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3311972201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3311972201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:10.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:10.722126+0000) 2023-08-28T01:30:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:11 smithi136 ceph-mon[113531]: pgmap v8298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:11 smithi093 ceph-mon[111335]: pgmap v8298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:11 smithi093 ceph-mon[118902]: pgmap v8298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:11.309Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:11.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:11.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:11.722391+0000) 2023-08-28T01:30:12.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4010414879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:12.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4010414879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:12.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4010414879' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:12.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:12.722628+0000) 2023-08-28T01:30:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:12.756Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:12.758Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:13.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:13 smithi136 ceph-mon[113531]: pgmap v8299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:13 smithi093 ceph-mon[111335]: pgmap v8299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:13 smithi093 ceph-mon[118902]: pgmap v8299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:13.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:13.722880+0000) 2023-08-28T01:30:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:14.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:14.723156+0000) 2023-08-28T01:30:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:30:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:30:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:30:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:15 smithi093 ceph-mon[111335]: pgmap v8300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2708661881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:15 smithi093 ceph-mon[118902]: pgmap v8300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2708661881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:15 smithi136 ceph-mon[113531]: pgmap v8300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2708661881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:15.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:15.723395+0000) 2023-08-28T01:30:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:16.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:16.723639+0000) 2023-08-28T01:30:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:17 smithi136 ceph-mon[113531]: pgmap v8301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/875207944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:17 smithi093 ceph-mon[118902]: pgmap v8301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/875207944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:17 smithi093 ceph-mon[111335]: pgmap v8301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/875207944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:17.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:17.723896+0000) 2023-08-28T01:30:19.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:18.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:18.724170+0000) 2023-08-28T01:30:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:19 smithi136 ceph-mon[113531]: pgmap v8302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1525569013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:19.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:19.724414+0000) 2023-08-28T01:30:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:19 smithi093 ceph-mon[111335]: pgmap v8302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1525569013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:19 smithi093 ceph-mon[118902]: pgmap v8302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1525569013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:20.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:20.724646+0000) 2023-08-28T01:30:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:21 smithi093 ceph-mon[111335]: pgmap v8303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:21 smithi093 ceph-mon[118902]: pgmap v8303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:21.310Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:21.311Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:21 smithi136 ceph-mon[113531]: pgmap v8303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:21.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:21.724919+0000) 2023-08-28T01:30:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1573348984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:22.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:22.725182+0000) 2023-08-28T01:30:22.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1573348984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:22.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1573348984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:22.758Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:22.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:22.759Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:23 smithi136 ceph-mon[113531]: pgmap v8304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:23.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:23.725439+0000) 2023-08-28T01:30:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:23 smithi093 ceph-mon[111335]: pgmap v8304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:23 smithi093 ceph-mon[118902]: pgmap v8304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1131895004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:24.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:24.725670+0000) 2023-08-28T01:30:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1131895004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1131895004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:25 smithi093 ceph-mon[111335]: pgmap v8305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:30:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:30:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:30:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:25 smithi093 ceph-mon[118902]: pgmap v8305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:25.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:25 smithi136 ceph-mon[113531]: pgmap v8305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:25.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:25.725933+0000) 2023-08-28T01:30:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:26.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:26.726185+0000) 2023-08-28T01:30:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:27 smithi136 ceph-mon[113531]: pgmap v8306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/216719148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:27 smithi093 ceph-mon[118902]: pgmap v8306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/216719148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:27 smithi093 ceph-mon[111335]: pgmap v8306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/216719148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:27.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:27.726443+0000) 2023-08-28T01:30:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:28.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:28.726657+0000) 2023-08-28T01:30:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:29 smithi136 ceph-mon[113531]: pgmap v8307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3870118913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:29.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:29.726903+0000) 2023-08-28T01:30:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:29 smithi093 ceph-mon[111335]: pgmap v8307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3870118913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:29 smithi093 ceph-mon[118902]: pgmap v8307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3870118913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:30.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:30.727152+0000) 2023-08-28T01:30:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:31 smithi093 ceph-mon[111335]: pgmap v8308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/356184433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:31 smithi093 ceph-mon[118902]: pgmap v8308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/356184433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:31.311Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:31.312Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:31 smithi136 ceph-mon[113531]: pgmap v8308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/356184433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:31.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:31.727417+0000) 2023-08-28T01:30:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:32.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:32.727670+0000) 2023-08-28T01:30:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:32.758Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:32.759Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:33 smithi136 ceph-mon[113531]: pgmap v8309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:33.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:33.727931+0000) 2023-08-28T01:30:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:33 smithi093 ceph-mon[111335]: pgmap v8309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:33 smithi093 ceph-mon[118902]: pgmap v8309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2971171906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:34.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:34.728199+0000) 2023-08-28T01:30:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2971171906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2971171906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:35 smithi093 ceph-mon[111335]: pgmap v8310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:35 smithi093 ceph-mon[118902]: pgmap v8310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:30:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:30:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:30:35.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:35 smithi136 ceph-mon[113531]: pgmap v8310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:35.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:35.728437+0000) 2023-08-28T01:30:36.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2023343645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:36.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:36.728684+0000) 2023-08-28T01:30:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2023343645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:36.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2023343645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:37 smithi136 ceph-mon[113531]: pgmap v8311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:37.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:37.728950+0000) 2023-08-28T01:30:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:37 smithi093 ceph-mon[111335]: pgmap v8311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:37 smithi093 ceph-mon[118902]: pgmap v8311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:38.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3566382791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:38.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:38.729172+0000) 2023-08-28T01:30:38.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3566382791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3566382791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:39 smithi136 ceph-mon[113531]: pgmap v8312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:39.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:39.729425+0000) 2023-08-28T01:30:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:39 smithi093 ceph-mon[111335]: pgmap v8312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:39 smithi093 ceph-mon[118902]: pgmap v8312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:40.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:40.729684+0000) 2023-08-28T01:30:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:41 smithi093 ceph-mon[111335]: pgmap v8313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1231277461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:41 smithi093 ceph-mon[118902]: pgmap v8313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1231277461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:41.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:41.312Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:41.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:41.313Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:41 smithi136 ceph-mon[113531]: pgmap v8313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1231277461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:41.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:41.729923+0000) 2023-08-28T01:30:43.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:42.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:42.730162+0000) 2023-08-28T01:30:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:42.758Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:42.759Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:43 smithi136 ceph-mon[113531]: pgmap v8314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1576882280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:43.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:43.730404+0000) 2023-08-28T01:30:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:43 smithi093 ceph-mon[111335]: pgmap v8314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1576882280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:43 smithi093 ceph-mon[118902]: pgmap v8314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1576882280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:45.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:44.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:44.730690+0000) 2023-08-28T01:30:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:45 smithi093 ceph-mon[111335]: pgmap v8315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3641860201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:30:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:30:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:30:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:45 smithi093 ceph-mon[118902]: pgmap v8315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3641860201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:45.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:45 smithi136 ceph-mon[113531]: pgmap v8315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3641860201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:45.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:45.730932+0000) 2023-08-28T01:30:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:46.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:46.731203+0000) 2023-08-28T01:30:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:47 smithi136 ceph-mon[113531]: pgmap v8316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:47 smithi093 ceph-mon[111335]: pgmap v8316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:47 smithi093 ceph-mon[118902]: pgmap v8316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:47.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:47.731473+0000) 2023-08-28T01:30:48.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1733120525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1733120525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1733120525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:48.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:48.731730+0000) 2023-08-28T01:30:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:49 smithi136 ceph-mon[113531]: pgmap v8317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:49 smithi093 ceph-mon[111335]: pgmap v8317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:49 smithi093 ceph-mon[118902]: pgmap v8317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:50.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:49.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:49.731967+0000) 2023-08-28T01:30:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2740147201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2740147201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2740147201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:50.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:50.732237+0000) 2023-08-28T01:30:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:51 smithi136 ceph-mon[113531]: pgmap v8318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:51 smithi093 ceph-mon[111335]: pgmap v8318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:51 smithi093 ceph-mon[118902]: pgmap v8318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:51.313Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:51.314Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:51.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:51.732470+0000) 2023-08-28T01:30:53.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:52.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:52.732722+0000) 2023-08-28T01:30:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:52.759Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:30:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:30:52.760Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:30:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:53 smithi136 ceph-mon[113531]: pgmap v8319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2798176591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:53.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:53.733002+0000) 2023-08-28T01:30:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:53 smithi093 ceph-mon[111335]: pgmap v8319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2798176591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:53 smithi093 ceph-mon[118902]: pgmap v8319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2798176591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:54.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:54.733271+0000) 2023-08-28T01:30:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:30:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:55 smithi093 ceph-mon[111335]: pgmap v8320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2685818834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:55 smithi093 ceph-mon[118902]: pgmap v8320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2685818834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:30:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:30:55] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:30:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:55 smithi136 ceph-mon[113531]: pgmap v8320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2685818834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:55.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:55.733547+0000) 2023-08-28T01:30:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:56.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:56.733798+0000) 2023-08-28T01:30:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:57 smithi136 ceph-mon[113531]: pgmap v8321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:57 smithi093 ceph-mon[111335]: pgmap v8321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:57 smithi093 ceph-mon[118902]: pgmap v8321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:57.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:57.734052+0000) 2023-08-28T01:30:58.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1100502494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1100502494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1100502494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:30:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:58.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:58.734291+0000) 2023-08-28T01:30:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:30:59 smithi136 ceph-mon[113531]: pgmap v8322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:30:59 smithi093 ceph-mon[111335]: pgmap v8322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:30:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:30:59 smithi093 ceph-mon[118902]: pgmap v8322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:30:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:30:59.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:30:59.734493+0000) 2023-08-28T01:31:00.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2601104372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2601104372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2601104372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:00.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:00.734753+0000) 2023-08-28T01:31:01.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:01 smithi136 ceph-mon[113531]: pgmap v8323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:01 smithi093 ceph-mon[111335]: pgmap v8323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:01 smithi093 ceph-mon[118902]: pgmap v8323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:01.314Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:01.315Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:01.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:01.734922+0000) 2023-08-28T01:31:02.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:31:02.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1880015456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1880015456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1880015456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:02.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:02.735087+0000) 2023-08-28T01:31:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:02.759Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:02.761Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:03.682 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:03 smithi136 ceph-mon[113531]: pgmap v8324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:03 smithi093 ceph-mon[111335]: pgmap v8324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:03 smithi093 ceph-mon[118902]: pgmap v8324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:03.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:03.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:03.735299+0000) 2023-08-28T01:31:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:04.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:04.735492+0000) 2023-08-28T01:31:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:31:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:31:04.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:31:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:31:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:31:04.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:31:05.688 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 01:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T01:31:05.320194809Z level=info msg="Completed cleanup jobs" duration=15.656228ms 2023-08-28T01:31:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:05 smithi136 ceph-mon[113531]: pgmap v8325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:05.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2050787937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:31:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:31:05] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:31:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:05 smithi093 ceph-mon[111335]: pgmap v8325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2050787937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:05 smithi093 ceph-mon[118902]: pgmap v8325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2050787937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:05.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:05.735740+0000) 2023-08-28T01:31:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:06.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:06.736026+0000) 2023-08-28T01:31:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:07 smithi136 ceph-mon[113531]: pgmap v8326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/522367605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:07 smithi093 ceph-mon[111335]: pgmap v8326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:07.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/522367605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:07 smithi093 ceph-mon[118902]: pgmap v8326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/522367605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:07.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:07.736285+0000) 2023-08-28T01:31:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:08.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:08.736520+0000) 2023-08-28T01:31:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:09 smithi136 ceph-mon[113531]: pgmap v8327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/961520214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:09.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:09.736720+0000) 2023-08-28T01:31:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:09 smithi093 ceph-mon[111335]: pgmap v8327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/961520214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:09 smithi093 ceph-mon[118902]: pgmap v8327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/961520214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:11.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:10.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:10.736954+0000) 2023-08-28T01:31:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:11 smithi093 ceph-mon[111335]: pgmap v8328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:11 smithi093 ceph-mon[118902]: pgmap v8328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:11.315Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:11.316Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:11 smithi136 ceph-mon[113531]: pgmap v8328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:11.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:11.737236+0000) 2023-08-28T01:31:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1552973149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:12.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:12.737435+0000) 2023-08-28T01:31:12.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1552973149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:12.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1552973149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:12.760Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:12.761Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:13 smithi136 ceph-mon[113531]: pgmap v8329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:13.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:13.737674+0000) 2023-08-28T01:31:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:13 smithi093 ceph-mon[111335]: pgmap v8329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:13 smithi093 ceph-mon[118902]: pgmap v8329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3798406054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:14.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:14.737876+0000) 2023-08-28T01:31:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3798406054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3798406054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:15 smithi093 ceph-mon[111335]: pgmap v8330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:31:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:31:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:31:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:15 smithi093 ceph-mon[118902]: pgmap v8330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:15 smithi136 ceph-mon[113531]: pgmap v8330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:15.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:15.738163+0000) 2023-08-28T01:31:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:16.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:16.738402+0000) 2023-08-28T01:31:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:17 smithi136 ceph-mon[113531]: pgmap v8331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340970721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:17 smithi093 ceph-mon[111335]: pgmap v8331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340970721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:17 smithi093 ceph-mon[118902]: pgmap v8331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340970721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:17.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:17.738652+0000) 2023-08-28T01:31:19.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:18.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:18.738898+0000) 2023-08-28T01:31:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:19.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:19.739149+0000) 2023-08-28T01:31:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:19 smithi136 ceph-mon[113531]: pgmap v8332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2288427728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:19 smithi093 ceph-mon[111335]: pgmap v8332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2288427728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:19 smithi093 ceph-mon[118902]: pgmap v8332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2288427728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:20.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:20.739434+0000) 2023-08-28T01:31:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:21 smithi093 ceph-mon[118902]: pgmap v8333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2324348808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:21 smithi093 ceph-mon[111335]: pgmap v8333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2324348808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:21.315Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:21.316Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:21 smithi136 ceph-mon[113531]: pgmap v8333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2324348808' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:21.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:21.739660+0000) 2023-08-28T01:31:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:22.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:22.739859+0000) 2023-08-28T01:31:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:22.761Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:22.762Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:23 smithi136 ceph-mon[113531]: pgmap v8334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:23.951 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:23.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:23.740105+0000) 2023-08-28T01:31:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:23 smithi093 ceph-mon[111335]: pgmap v8334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:23 smithi093 ceph-mon[118902]: pgmap v8334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3026722598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:24.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:24.740338+0000) 2023-08-28T01:31:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3026722598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3026722598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:25 smithi093 ceph-mon[111335]: pgmap v8335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:25 smithi093 ceph-mon[118902]: pgmap v8335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:31:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:31:25] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:31:25.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:25 smithi136 ceph-mon[113531]: pgmap v8335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:25.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:25.740572+0000) 2023-08-28T01:31:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2126277359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:26.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:26.740816+0000) 2023-08-28T01:31:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2126277359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:26.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2126277359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:27 smithi136 ceph-mon[113531]: pgmap v8336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:27.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:27.741108+0000) 2023-08-28T01:31:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:27 smithi093 ceph-mon[111335]: pgmap v8336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:27 smithi093 ceph-mon[118902]: pgmap v8336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1439950738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:28.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:28.741377+0000) 2023-08-28T01:31:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1439950738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1439950738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:29 smithi136 ceph-mon[113531]: pgmap v8337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:29.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:29.741573+0000) 2023-08-28T01:31:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:29 smithi093 ceph-mon[111335]: pgmap v8337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:29 smithi093 ceph-mon[118902]: pgmap v8337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:30.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:30.741807+0000) 2023-08-28T01:31:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:31 smithi093 ceph-mon[111335]: pgmap v8338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2367786697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:31 smithi093 ceph-mon[118902]: pgmap v8338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2367786697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:31.316Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:31.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:31.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:31.742089+0000) 2023-08-28T01:31:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:31 smithi136 ceph-mon[113531]: pgmap v8338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2367786697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:32.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:32.742328+0000) 2023-08-28T01:31:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:32.761Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:32.762Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:33 smithi136 ceph-mon[113531]: pgmap v8339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1861039353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:33.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:33.742600+0000) 2023-08-28T01:31:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:33 smithi093 ceph-mon[118902]: pgmap v8339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1861039353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:33 smithi093 ceph-mon[111335]: pgmap v8339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1861039353' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:34.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:34.742820+0000) 2023-08-28T01:31:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:35 smithi093 ceph-mon[111335]: pgmap v8340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:35 smithi093 ceph-mon[118902]: pgmap v8340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:31:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:31:35] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:31:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:35 smithi136 ceph-mon[113531]: pgmap v8340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:35.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:35.743072+0000) 2023-08-28T01:31:36.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3578085061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:36.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:36.743300+0000) 2023-08-28T01:31:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3578085061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:36.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3578085061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:37 smithi136 ceph-mon[113531]: pgmap v8341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:37.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:37.743576+0000) 2023-08-28T01:31:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:37 smithi093 ceph-mon[111335]: pgmap v8341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:37 smithi093 ceph-mon[118902]: pgmap v8341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:38.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1121742756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:38.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:38.743817+0000) 2023-08-28T01:31:38.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1121742756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1121742756' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:39 smithi136 ceph-mon[113531]: pgmap v8342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:39.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:39.744079+0000) 2023-08-28T01:31:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:39 smithi093 ceph-mon[111335]: pgmap v8342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:39 smithi093 ceph-mon[118902]: pgmap v8342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:40.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3538777232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:40.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:40.744309+0000) 2023-08-28T01:31:40.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3538777232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:40.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3538777232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:41.584 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:41.316Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:41.584 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:41.317Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:41 smithi136 ceph-mon[113531]: pgmap v8343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:41.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:41.744547+0000) 2023-08-28T01:31:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:41 smithi093 ceph-mon[111335]: pgmap v8343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:41 smithi093 ceph-mon[118902]: pgmap v8343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:42.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:42.744813+0000) 2023-08-28T01:31:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:42.762Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:42.763Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:43 smithi136 ceph-mon[113531]: pgmap v8344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2933980653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:43.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:43.745087+0000) 2023-08-28T01:31:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:43 smithi093 ceph-mon[111335]: pgmap v8344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2933980653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:43 smithi093 ceph-mon[118902]: pgmap v8344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2933980653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:45.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:44.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:44.745380+0000) 2023-08-28T01:31:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:31:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:31:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:31:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:45 smithi093 ceph-mon[118902]: pgmap v8345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/263152638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:45 smithi093 ceph-mon[111335]: pgmap v8345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/263152638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:45 smithi136 ceph-mon[113531]: pgmap v8345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/263152638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:45.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:45.745604+0000) 2023-08-28T01:31:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:46.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:46.745800+0000) 2023-08-28T01:31:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:47 smithi136 ceph-mon[113531]: pgmap v8346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:47.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:47 smithi093 ceph-mon[111335]: pgmap v8346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:47 smithi093 ceph-mon[118902]: pgmap v8346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:47.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:47.746102+0000) 2023-08-28T01:31:48.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2212805592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2212805592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2212805592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:48.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:48.746348+0000) 2023-08-28T01:31:49.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:49 smithi136 ceph-mon[113531]: pgmap v8347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:49.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:49 smithi093 ceph-mon[111335]: pgmap v8347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:49 smithi093 ceph-mon[118902]: pgmap v8347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:50.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:49.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:49.746616+0000) 2023-08-28T01:31:50.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/194953480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/194953480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/194953480' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:50.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:50.746884+0000) 2023-08-28T01:31:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:51 smithi136 ceph-mon[113531]: pgmap v8348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:51 smithi093 ceph-mon[111335]: pgmap v8348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:51 smithi093 ceph-mon[118902]: pgmap v8348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:51.317Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:51.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:52.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:51.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:51.747153+0000) 2023-08-28T01:31:52.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2752090395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:52.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2752090395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2752090395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:52.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:52.747413+0000) 2023-08-28T01:31:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:52.762Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:31:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:31:52.763Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:31:53.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:53 smithi136 ceph-mon[113531]: pgmap v8349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:53 smithi093 ceph-mon[111335]: pgmap v8349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:53 smithi093 ceph-mon[118902]: pgmap v8349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:54.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:53.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:53.747696+0000) 2023-08-28T01:31:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:31:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:54.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:54.747973+0000) 2023-08-28T01:31:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:55 smithi136 ceph-mon[113531]: pgmap v8350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1342129769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:55 smithi093 ceph-mon[111335]: pgmap v8350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1342129769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:31:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:31:55] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:31:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:55 smithi093 ceph-mon[118902]: pgmap v8350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1342129769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:56.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:55.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:55.748207+0000) 2023-08-28T01:31:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:56.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:56.748464+0000) 2023-08-28T01:31:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:57 smithi136 ceph-mon[113531]: pgmap v8351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3437541632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:57 smithi093 ceph-mon[111335]: pgmap v8351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3437541632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:57 smithi093 ceph-mon[118902]: pgmap v8351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3437541632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:57.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:57.748711+0000) 2023-08-28T01:31:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:58.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:58.748924+0000) 2023-08-28T01:31:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:59 smithi136 ceph-mon[113531]: pgmap v8352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:31:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2818964709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:31:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:31:59.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:31:59.749194+0000) 2023-08-28T01:31:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:59 smithi093 ceph-mon[111335]: pgmap v8352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:31:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2818964709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:31:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:59 smithi093 ceph-mon[118902]: pgmap v8352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:31:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:31:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2818964709' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:00.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:00.749422+0000) 2023-08-28T01:32:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:01 smithi093 ceph-mon[111335]: pgmap v8353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:01 smithi093 ceph-mon[118902]: pgmap v8353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:01.317Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:01.318Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:01 smithi136 ceph-mon[113531]: pgmap v8353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:01.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:01.749673+0000) 2023-08-28T01:32:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/765288730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:02.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:02.749937+0000) 2023-08-28T01:32:02.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/765288730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:02.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/765288730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:02.763Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:02.764Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:03 smithi136 ceph-mon[113531]: pgmap v8354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:03.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:03.750143+0000) 2023-08-28T01:32:03.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:03 smithi093 ceph-mon[111335]: pgmap v8354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:03 smithi093 ceph-mon[118902]: pgmap v8354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2337526041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:32:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:04.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:04.750283+0000) 2023-08-28T01:32:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2337526041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:32:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2337526041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:32:05.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:32:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:32:05] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:32:05.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:05 smithi136 ceph-mon[113531]: pgmap v8355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:05 smithi093 ceph-mon[111335]: pgmap v8355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:05 smithi093 ceph-mon[118902]: pgmap v8355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:05.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:06.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:05.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:05.750408+0000) 2023-08-28T01:32:06.989 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:06.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:06.750598+0000) 2023-08-28T01:32:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:07 smithi136 ceph-mon[113531]: pgmap v8356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/176282845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[111335]: pgmap v8356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/176282845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[118902]: pgmap v8356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/176282845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:07.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:07.750747+0000) 2023-08-28T01:32:08.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:32:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:32:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:32:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:32:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:32:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:32:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:32:09.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:08.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:08.750945+0000) 2023-08-28T01:32:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:09 smithi136 ceph-mon[113531]: pgmap v8357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3176146579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:09 smithi093 ceph-mon[111335]: pgmap v8357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3176146579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:09 smithi093 ceph-mon[118902]: pgmap v8357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3176146579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:09.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:09.751204+0000) 2023-08-28T01:32:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:10.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:10.751432+0000) 2023-08-28T01:32:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:11 smithi093 ceph-mon[111335]: pgmap v8358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/710669491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:11 smithi093 ceph-mon[118902]: pgmap v8358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/710669491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:11.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:11.319Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:11 smithi136 ceph-mon[113531]: pgmap v8358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/710669491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:11.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:11.751661+0000) 2023-08-28T01:32:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:12.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:12.751925+0000) 2023-08-28T01:32:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:12.764Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:12.765Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:13 smithi136 ceph-mon[113531]: pgmap v8359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:13.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:13.752140+0000) 2023-08-28T01:32:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:13 smithi093 ceph-mon[111335]: pgmap v8359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:13 smithi093 ceph-mon[118902]: pgmap v8359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/591029668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:14.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:14.752406+0000) 2023-08-28T01:32:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/591029668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/591029668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:15 smithi093 ceph-mon[111335]: pgmap v8360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:15 smithi093 ceph-mon[118902]: pgmap v8360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:32:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:32:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:32:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:15 smithi136 ceph-mon[113531]: pgmap v8360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:15.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:15.752645+0000) 2023-08-28T01:32:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1401375322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:16.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:16.752894+0000) 2023-08-28T01:32:16.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1401375322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:16.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1401375322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:17 smithi136 ceph-mon[113531]: pgmap v8361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:17.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:17.753159+0000) 2023-08-28T01:32:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:17 smithi093 ceph-mon[111335]: pgmap v8361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:17 smithi093 ceph-mon[118902]: pgmap v8361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1163190243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:18.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:18.753370+0000) 2023-08-28T01:32:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1163190243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1163190243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:19 smithi136 ceph-mon[113531]: pgmap v8362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:19.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:19.753619+0000) 2023-08-28T01:32:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:19 smithi093 ceph-mon[111335]: pgmap v8362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:19 smithi093 ceph-mon[118902]: pgmap v8362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:20.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:20.753874+0000) 2023-08-28T01:32:21.573 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:21.318Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:21.574 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:21.319Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:21 smithi136 ceph-mon[113531]: pgmap v8363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2964952734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:21.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:21.754143+0000) 2023-08-28T01:32:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:21 smithi093 ceph-mon[111335]: pgmap v8363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2964952734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:21 smithi093 ceph-mon[118902]: pgmap v8363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2964952734' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:22.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:22.754386+0000) 2023-08-28T01:32:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:22.764Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:22.765Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:23 smithi136 ceph-mon[113531]: pgmap v8364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2368514855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:23.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:23.754641+0000) 2023-08-28T01:32:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:23 smithi093 ceph-mon[118902]: pgmap v8364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2368514855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:23 smithi093 ceph-mon[111335]: pgmap v8364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2368514855' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:24.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:24.754946+0000) 2023-08-28T01:32:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:25 smithi093 ceph-mon[111335]: pgmap v8365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:25 smithi093 ceph-mon[118902]: pgmap v8365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:32:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:32:25] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:32:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:25 smithi136 ceph-mon[113531]: pgmap v8365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:25.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:25.755214+0000) 2023-08-28T01:32:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2471028795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:26.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:26.755484+0000) 2023-08-28T01:32:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2471028795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:26.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2471028795' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:27 smithi136 ceph-mon[113531]: pgmap v8366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:27.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:27.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:27.755779+0000) 2023-08-28T01:32:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:27 smithi093 ceph-mon[111335]: pgmap v8366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:27 smithi093 ceph-mon[118902]: pgmap v8366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/322083496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:28.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:28.755985+0000) 2023-08-28T01:32:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/322083496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/322083496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:29 smithi136 ceph-mon[113531]: pgmap v8367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:29.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:29.756214+0000) 2023-08-28T01:32:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:29 smithi093 ceph-mon[111335]: pgmap v8367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:29 smithi093 ceph-mon[118902]: pgmap v8367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:30.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3106440245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:30.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:30.756460+0000) 2023-08-28T01:32:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3106440245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3106440245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:31.601 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:31.319Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:31.602 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:31.320Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:31 smithi136 ceph-mon[113531]: pgmap v8368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:31.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:31.756725+0000) 2023-08-28T01:32:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:31 smithi093 ceph-mon[111335]: pgmap v8368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:31 smithi093 ceph-mon[118902]: pgmap v8368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:32.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:32.756953+0000) 2023-08-28T01:32:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:32.765Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:32.766Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:33.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:33.757211+0000) 2023-08-28T01:32:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:33 smithi136 ceph-mon[113531]: pgmap v8369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1171184981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:33 smithi093 ceph-mon[111335]: pgmap v8369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1171184981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:33 smithi093 ceph-mon[118902]: pgmap v8369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1171184981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:34.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:34.757412+0000) 2023-08-28T01:32:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:32:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:32:35] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:32:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:35 smithi093 ceph-mon[111335]: pgmap v8370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3301911238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:35 smithi093 ceph-mon[118902]: pgmap v8370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3301911238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:35 smithi136 ceph-mon[113531]: pgmap v8370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3301911238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:35.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:35.757669+0000) 2023-08-28T01:32:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:36.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:36.757947+0000) 2023-08-28T01:32:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:37 smithi136 ceph-mon[113531]: pgmap v8371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:37 smithi093 ceph-mon[111335]: pgmap v8371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:37 smithi093 ceph-mon[118902]: pgmap v8371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:37.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:37.758164+0000) 2023-08-28T01:32:38.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3827952677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3827952677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3827952677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:38.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:38.758370+0000) 2023-08-28T01:32:39.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:39 smithi136 ceph-mon[113531]: pgmap v8372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:39 smithi093 ceph-mon[118902]: pgmap v8372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:39 smithi093 ceph-mon[111335]: pgmap v8372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:39.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:39.758609+0000) 2023-08-28T01:32:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3392341212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3392341212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3392341212' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:40.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:40.758889+0000) 2023-08-28T01:32:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:41 smithi136 ceph-mon[113531]: pgmap v8373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:41 smithi093 ceph-mon[111335]: pgmap v8373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:41 smithi093 ceph-mon[118902]: pgmap v8373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:41.320Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:41.321Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:42.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:41.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:41.759134+0000) 2023-08-28T01:32:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2315465229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2315465229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:42.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2315465229' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:42.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:42.759340+0000) 2023-08-28T01:32:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:42.766Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:42.766Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:43.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:43 smithi136 ceph-mon[113531]: pgmap v8374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:43 smithi093 ceph-mon[111335]: pgmap v8374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:43 smithi093 ceph-mon[118902]: pgmap v8374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:43.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:43.759595+0000) 2023-08-28T01:32:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:44.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:44.759859+0000) 2023-08-28T01:32:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:32:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:32:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:32:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:45 smithi093 ceph-mon[111335]: pgmap v8375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1361543018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:45 smithi093 ceph-mon[118902]: pgmap v8375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1361543018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:45 smithi136 ceph-mon[113531]: pgmap v8375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1361543018' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:45.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:45.760146+0000) 2023-08-28T01:32:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:46.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:46.760413+0000) 2023-08-28T01:32:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:47 smithi136 ceph-mon[113531]: pgmap v8376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2588814076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:47 smithi093 ceph-mon[118902]: pgmap v8376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2588814076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:47 smithi093 ceph-mon[111335]: pgmap v8376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:47.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2588814076' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:47.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:47.760709+0000) 2023-08-28T01:32:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:48.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:48.760977+0000) 2023-08-28T01:32:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:49 smithi136 ceph-mon[113531]: pgmap v8377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1108630025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:49.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:49.761212+0000) 2023-08-28T01:32:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:49 smithi093 ceph-mon[118902]: pgmap v8377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1108630025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:49 smithi093 ceph-mon[111335]: pgmap v8377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1108630025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:50.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:50.761366+0000) 2023-08-28T01:32:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:51 smithi093 ceph-mon[111335]: pgmap v8378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:51 smithi093 ceph-mon[118902]: pgmap v8378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:51.321Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:51.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:51 smithi136 ceph-mon[113531]: pgmap v8378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:51.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:51.761593+0000) 2023-08-28T01:32:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2822127482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:52.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:52.761874+0000) 2023-08-28T01:32:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2822127482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2822127482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:52.766Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:32:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:32:52.767Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:32:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:53 smithi136 ceph-mon[113531]: pgmap v8379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:53.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:53.762160+0000) 2023-08-28T01:32:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:53 smithi093 ceph-mon[111335]: pgmap v8379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:53 smithi093 ceph-mon[118902]: pgmap v8379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2595868673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:54.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:54.762445+0000) 2023-08-28T01:32:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2595868673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:32:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2595868673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:55 smithi093 ceph-mon[111335]: pgmap v8380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:32:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:32:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:32:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:55 smithi093 ceph-mon[118902]: pgmap v8380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:55 smithi136 ceph-mon[113531]: pgmap v8380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:55.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:55.762725+0000) 2023-08-28T01:32:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:56.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:56.762959+0000) 2023-08-28T01:32:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:57 smithi136 ceph-mon[113531]: pgmap v8381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2122830636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:57 smithi093 ceph-mon[111335]: pgmap v8381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2122830636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:57 smithi093 ceph-mon[118902]: pgmap v8381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2122830636' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:57.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:57.763272+0000) 2023-08-28T01:32:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:58.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:58.763476+0000) 2023-08-28T01:32:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:59 smithi136 ceph-mon[113531]: pgmap v8382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:32:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/105092998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:32:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:32:59.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:32:59.763711+0000) 2023-08-28T01:32:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:59 smithi093 ceph-mon[111335]: pgmap v8382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:32:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/105092998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:32:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:59 smithi093 ceph-mon[118902]: pgmap v8382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:32:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:32:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/105092998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:00.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:00.763975+0000) 2023-08-28T01:33:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:01 smithi093 ceph-mon[111335]: pgmap v8383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2784853886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:01 smithi093 ceph-mon[118902]: pgmap v8383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2784853886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:01.321Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:01.322Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:01 smithi136 ceph-mon[113531]: pgmap v8383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2784853886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:01.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:01.764193+0000) 2023-08-28T01:33:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:02.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:02.764376+0000) 2023-08-28T01:33:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:02.766Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:02.767Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:03 smithi136 ceph-mon[113531]: pgmap v8384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:03.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:03.764620+0000) 2023-08-28T01:33:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:03 smithi093 ceph-mon[111335]: pgmap v8384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:03 smithi093 ceph-mon[118902]: pgmap v8384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:04.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3344893664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:04.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:04.764890+0000) 2023-08-28T01:33:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3344893664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3344893664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:05 smithi093 ceph-mon[111335]: pgmap v8385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:33:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:33:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:33:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:05 smithi093 ceph-mon[118902]: pgmap v8385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:05 smithi136 ceph-mon[113531]: pgmap v8385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:05.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:05.765149+0000) 2023-08-28T01:33:06.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2809139405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:06.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:06.765401+0000) 2023-08-28T01:33:06.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2809139405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:06.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2809139405' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:07 smithi136 ceph-mon[113531]: pgmap v8386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:08.160 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:07.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:07.765694+0000) 2023-08-28T01:33:08.160 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:07 smithi093 ceph-mon[111335]: pgmap v8386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:08.161 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:07 smithi093 ceph-mon[118902]: pgmap v8386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:33:08.768 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3283103702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:08.770 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:33:08.770 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3283103702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:08.770 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:33:08.771 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3283103702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:09.185 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:08.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:08.765846+0000) 2023-08-28T01:33:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:09 smithi093 ceph-mon[111335]: pgmap v8387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:09.831 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:09.831 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:09 smithi093 ceph-mon[118902]: pgmap v8387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:09.831 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:09 smithi136 ceph-mon[113531]: pgmap v8387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:09.991 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:09.991 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:09.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:09.766040+0000) 2023-08-28T01:33:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:10.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:10.766238+0000) 2023-08-28T01:33:10.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.958 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:11.635 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:11.322Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:11.744 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:11.323Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:11.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:11.766512+0000) 2023-08-28T01:33:12.075 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:11 smithi136 ceph-mon[113531]: pgmap v8388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:12.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:33:12.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:33:12.076 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:12.235 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3184716394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:12.235 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[111335]: pgmap v8388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3184716394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[118902]: pgmap v8388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:33:12.236 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:33:12.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3184716394' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:12.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:12.766814+0000) 2023-08-28T01:33:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:12.767Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:12.768Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:13 smithi136 ceph-mon[113531]: pgmap v8389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3306138571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:13.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:13.767117+0000) 2023-08-28T01:33:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:13 smithi093 ceph-mon[118902]: pgmap v8389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3306138571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:13 smithi093 ceph-mon[111335]: pgmap v8389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3306138571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:14.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:14.767432+0000) 2023-08-28T01:33:15.641 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:33:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:33:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:33:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:15 smithi136 ceph-mon[113531]: pgmap v8390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:15.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:15.767620+0000) 2023-08-28T01:33:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:15 smithi093 ceph-mon[111335]: pgmap v8390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:15 smithi093 ceph-mon[118902]: pgmap v8390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1552321784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:16.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:16.767895+0000) 2023-08-28T01:33:16.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1552321784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:16.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1552321784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:17 smithi136 ceph-mon[113531]: pgmap v8391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:17.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:17.768179+0000) 2023-08-28T01:33:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:17 smithi093 ceph-mon[111335]: pgmap v8391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:17 smithi093 ceph-mon[118902]: pgmap v8391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1909410247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:18.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:18.768452+0000) 2023-08-28T01:33:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1909410247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1909410247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:19 smithi136 ceph-mon[113531]: pgmap v8392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:19.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:19.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:19.768793+0000) 2023-08-28T01:33:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:19 smithi093 ceph-mon[111335]: pgmap v8392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:19 smithi093 ceph-mon[118902]: pgmap v8392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:20.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4123930816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:20.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:20.769103+0000) 2023-08-28T01:33:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4123930816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:20.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4123930816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:21.661 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:21.323Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:21.661 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:21.324Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:21 smithi136 ceph-mon[113531]: pgmap v8393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:21.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:21.769376+0000) 2023-08-28T01:33:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:21 smithi093 ceph-mon[111335]: pgmap v8393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:21 smithi093 ceph-mon[118902]: pgmap v8393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:22.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:22.769611+0000) 2023-08-28T01:33:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:22.768Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:22.768Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:23 smithi136 ceph-mon[113531]: pgmap v8394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2678748309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:23.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:23.769856+0000) 2023-08-28T01:33:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:23 smithi093 ceph-mon[118902]: pgmap v8394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2678748309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:23 smithi093 ceph-mon[111335]: pgmap v8394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2678748309' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:24.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:24.770094+0000) 2023-08-28T01:33:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:25.671 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:33:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:33:25] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:33:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:25 smithi136 ceph-mon[113531]: pgmap v8395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/632471276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:25.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:25.770372+0000) 2023-08-28T01:33:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:25 smithi093 ceph-mon[111335]: pgmap v8395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/632471276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:25.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:25 smithi093 ceph-mon[118902]: pgmap v8395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:25.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/632471276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:26.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:26.770653+0000) 2023-08-28T01:33:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:27 smithi136 ceph-mon[113531]: pgmap v8396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:27 smithi093 ceph-mon[111335]: pgmap v8396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:27 smithi093 ceph-mon[118902]: pgmap v8396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:27.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:27.770954+0000) 2023-08-28T01:33:28.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/574071194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/574071194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/574071194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:28.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:28.771235+0000) 2023-08-28T01:33:29.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:29 smithi136 ceph-mon[113531]: pgmap v8397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:29.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:29 smithi093 ceph-mon[111335]: pgmap v8397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:29 smithi093 ceph-mon[118902]: pgmap v8397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:29.770+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:29.771555+0000) 2023-08-28T01:33:30.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2679316446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2679316446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2679316446' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:30.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:30.771815+0000) 2023-08-28T01:33:31.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:31 smithi136 ceph-mon[113531]: pgmap v8398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:31 smithi093 ceph-mon[111335]: pgmap v8398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:31 smithi093 ceph-mon[118902]: pgmap v8398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:31.324Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:31.325Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:32.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:31.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:31.772120+0000) 2023-08-28T01:33:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2350781763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2350781763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:32.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2350781763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:32.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:32.772403+0000) 2023-08-28T01:33:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:32.768Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:32.769Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:33.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:33 smithi136 ceph-mon[113531]: pgmap v8399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:33 smithi093 ceph-mon[111335]: pgmap v8399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:33 smithi093 ceph-mon[118902]: pgmap v8399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:33.771+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:33.772704+0000) 2023-08-28T01:33:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:34.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:34.772989+0000) 2023-08-28T01:33:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:35 smithi093 ceph-mon[111335]: pgmap v8400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:35.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3935240975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:33:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:33:35] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:33:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:35 smithi093 ceph-mon[118902]: pgmap v8400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3935240975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:35 smithi136 ceph-mon[113531]: pgmap v8400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3935240975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:35.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:35.773284+0000) 2023-08-28T01:33:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:36.772+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:36.773504+0000) 2023-08-28T01:33:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:37 smithi136 ceph-mon[113531]: pgmap v8401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/265689978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:37 smithi093 ceph-mon[111335]: pgmap v8401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/265689978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:37 smithi093 ceph-mon[118902]: pgmap v8401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/265689978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:37.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:37.773794+0000) 2023-08-28T01:33:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:38.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:38.774085+0000) 2023-08-28T01:33:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:39 smithi136 ceph-mon[113531]: pgmap v8402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3513059682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:39.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:39.774316+0000) 2023-08-28T01:33:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:39 smithi093 ceph-mon[111335]: pgmap v8402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3513059682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:39 smithi093 ceph-mon[118902]: pgmap v8402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3513059682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:40.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:40.774477+0000) 2023-08-28T01:33:41.575 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:41.324Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:41.575 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:41.325Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:41 smithi136 ceph-mon[113531]: pgmap v8403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:41.773+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:41.774732+0000) 2023-08-28T01:33:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:41 smithi093 ceph-mon[111335]: pgmap v8403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:41 smithi093 ceph-mon[118902]: pgmap v8403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:42.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2103125417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:42.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:42.774941+0000) 2023-08-28T01:33:42.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2103125417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2103125417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:42.769Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:42.770Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:43 smithi136 ceph-mon[113531]: pgmap v8404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:43.774+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:43.775203+0000) 2023-08-28T01:33:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:43 smithi093 ceph-mon[111335]: pgmap v8404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:43 smithi093 ceph-mon[118902]: pgmap v8404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:44.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1727319540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:44.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:44.775466+0000) 2023-08-28T01:33:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1727319540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1727319540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:45 smithi093 ceph-mon[111335]: pgmap v8405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:45 smithi093 ceph-mon[118902]: pgmap v8405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:33:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:33:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:33:45.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:45 smithi136 ceph-mon[113531]: pgmap v8405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:45.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:45.775757+0000) 2023-08-28T01:33:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:46.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:46.776070+0000) 2023-08-28T01:33:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:47 smithi136 ceph-mon[113531]: pgmap v8406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3720689598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:47 smithi093 ceph-mon[111335]: pgmap v8406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3720689598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:47 smithi093 ceph-mon[118902]: pgmap v8406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3720689598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:47.775+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:47.776292+0000) 2023-08-28T01:33:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:48.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:48.776521+0000) 2023-08-28T01:33:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:49.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:49.776773+0000) 2023-08-28T01:33:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:49 smithi136 ceph-mon[113531]: pgmap v8407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1694140444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:49 smithi093 ceph-mon[111335]: pgmap v8407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1694140444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:49 smithi093 ceph-mon[118902]: pgmap v8407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1694140444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:50.776+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:50.777092+0000) 2023-08-28T01:33:51.577 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:51.325Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:51.577 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:51.326Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:51 smithi136 ceph-mon[113531]: pgmap v8408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/258091233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:51.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:51.777374+0000) 2023-08-28T01:33:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:51 smithi093 ceph-mon[118902]: pgmap v8408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/258091233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:51 smithi093 ceph-mon[111335]: pgmap v8408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/258091233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:52.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:52.777642+0000) 2023-08-28T01:33:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:52.770Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:33:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:33:52.771Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:33:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:53 smithi136 ceph-mon[113531]: pgmap v8409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:53.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:53.777928+0000) 2023-08-28T01:33:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:53 smithi093 ceph-mon[111335]: pgmap v8409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:53 smithi093 ceph-mon[118902]: pgmap v8409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2501020791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:54.777+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:54.778191+0000) 2023-08-28T01:33:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2501020791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2501020791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:33:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:55 smithi093 ceph-mon[111335]: pgmap v8410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:55 smithi093 ceph-mon[118902]: pgmap v8410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:33:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:33:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:33:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:55 smithi136 ceph-mon[113531]: pgmap v8410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:55.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:55.778434+0000) 2023-08-28T01:33:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/424138945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:56.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:56.778724+0000) 2023-08-28T01:33:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/424138945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:56.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/424138945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:57 smithi136 ceph-mon[113531]: pgmap v8411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:57.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:57.779008+0000) 2023-08-28T01:33:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:57 smithi093 ceph-mon[111335]: pgmap v8411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:57 smithi093 ceph-mon[118902]: pgmap v8411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2875715567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:58.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:58.779287+0000) 2023-08-28T01:33:58.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2875715567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2875715567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:33:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:33:59 smithi136 ceph-mon[113531]: pgmap v8412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:33:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:33:59.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:33:59.779504+0000) 2023-08-28T01:33:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:33:59 smithi093 ceph-mon[111335]: pgmap v8412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:33:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:33:59 smithi093 ceph-mon[118902]: pgmap v8412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:00.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:00.779751+0000) 2023-08-28T01:34:01.598 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:01.326Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:01.598 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:01.327Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:01 smithi136 ceph-mon[113531]: pgmap v8413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1284243862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:01.778+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:01.779976+0000) 2023-08-28T01:34:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:01 smithi093 ceph-mon[111335]: pgmap v8413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1284243862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:01 smithi093 ceph-mon[118902]: pgmap v8413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1284243862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:02.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:02.780221+0000) 2023-08-28T01:34:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:02.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:02.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:03 smithi136 ceph-mon[113531]: pgmap v8414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2750386251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:03.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:03.780495+0000) 2023-08-28T01:34:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:03 smithi093 ceph-mon[118902]: pgmap v8414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2750386251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:03 smithi093 ceph-mon[111335]: pgmap v8414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2750386251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:04.779+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:04.780822+0000) 2023-08-28T01:34:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:05 smithi093 ceph-mon[111335]: pgmap v8415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:05 smithi093 ceph-mon[118902]: pgmap v8415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:34:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:34:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:34:05.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:05 smithi136 ceph-mon[113531]: pgmap v8415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:05.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:05.781132+0000) 2023-08-28T01:34:06.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/923262557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:06.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:06.781348+0000) 2023-08-28T01:34:06.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/923262557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:06.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/923262557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:07.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:07 smithi136 ceph-mon[113531]: pgmap v8416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:07.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:07.781626+0000) 2023-08-28T01:34:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:07 smithi093 ceph-mon[111335]: pgmap v8416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:07 smithi093 ceph-mon[118902]: pgmap v8416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4201481514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:08.780+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:08.781885+0000) 2023-08-28T01:34:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4201481514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4201481514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:09 smithi136 ceph-mon[113531]: pgmap v8417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:09.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:09.782180+0000) 2023-08-28T01:34:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:09 smithi093 ceph-mon[111335]: pgmap v8417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:09 smithi093 ceph-mon[118902]: pgmap v8417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:10.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2647460052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:10.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:10.782424+0000) 2023-08-28T01:34:10.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2647460052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2647460052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:11.617 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:11.327Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:11.617 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:11.328Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:11 smithi136 ceph-mon[113531]: pgmap v8418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:34:11.952 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:11 smithi093 ceph-mon[111335]: pgmap v8418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:11.953 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:34:11.953 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:11 smithi093 ceph-mon[118902]: pgmap v8418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:11.953 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:34:12.139 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:11.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:11.782598+0000) 2023-08-28T01:34:12.897 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:12.781+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:12.782757+0000) 2023-08-28T01:34:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:12.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:12.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:13.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.781 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:13 smithi136 ceph-mon[113531]: pgmap v8419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:13.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3382468834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:13.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[118902]: pgmap v8419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3382468834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[111335]: pgmap v8419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3382468834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:13.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:13.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:13.782956+0000) 2023-08-28T01:34:14.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:34:14.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:34:14.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:34:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:34:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:34:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:34:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:34:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:14.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:14.783216+0000) 2023-08-28T01:34:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:34:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:34:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:34:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:15 smithi093 ceph-mon[111335]: pgmap v8420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2868976386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:15 smithi093 ceph-mon[118902]: pgmap v8420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2868976386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:15.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:15 smithi136 ceph-mon[113531]: pgmap v8420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:15.782 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2868976386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:16.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:15.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:15.783465+0000) 2023-08-28T01:34:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:16.782+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:16.783734+0000) 2023-08-28T01:34:17.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:17 smithi136 ceph-mon[113531]: pgmap v8421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:17 smithi093 ceph-mon[111335]: pgmap v8421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:17 smithi093 ceph-mon[118902]: pgmap v8421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:18.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:17.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:17.784056+0000) 2023-08-28T01:34:18.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1819200548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1819200548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1819200548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:19.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:18.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:18.784323+0000) 2023-08-28T01:34:19.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:19 smithi136 ceph-mon[113531]: pgmap v8422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:19 smithi093 ceph-mon[111335]: pgmap v8422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:19 smithi093 ceph-mon[118902]: pgmap v8422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:19.783+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:19.784579+0000) 2023-08-28T01:34:20.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/758541231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/758541231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/758541231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:20.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:20.784770+0000) 2023-08-28T01:34:21.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:21 smithi136 ceph-mon[113531]: pgmap v8423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:21 smithi093 ceph-mon[111335]: pgmap v8423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:21 smithi093 ceph-mon[118902]: pgmap v8423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:21.328Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:21.329Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:21.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:21.785027+0000) 2023-08-28T01:34:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2812479355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:22.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2812479355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:22.783 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2812479355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:22.784+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:22.785311+0000) 2023-08-28T01:34:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:22.771Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:22.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:23 smithi093 ceph-mon[111335]: pgmap v8424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:23 smithi093 ceph-mon[118902]: pgmap v8424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:23.783 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:23 smithi136 ceph-mon[113531]: pgmap v8424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:24.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:23.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:23.785530+0000) 2023-08-28T01:34:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:24.784 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:25.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:24.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:24.785796+0000) 2023-08-28T01:34:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:34:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:34:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:34:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:25 smithi093 ceph-mon[111335]: pgmap v8425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3264294648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:25 smithi093 ceph-mon[118902]: pgmap v8425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3264294648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:25.784 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:25 smithi136 ceph-mon[113531]: pgmap v8425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:25.784 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3264294648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:25.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:25.786111+0000) 2023-08-28T01:34:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:26.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:26.786393+0000) 2023-08-28T01:34:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:27 smithi136 ceph-mon[113531]: pgmap v8426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2630766721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:27 smithi093 ceph-mon[111335]: pgmap v8426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2630766721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:27 smithi093 ceph-mon[118902]: pgmap v8426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2630766721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:27.785+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:27.786550+0000) 2023-08-28T01:34:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:28.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:28.786843+0000) 2023-08-28T01:34:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:29 smithi136 ceph-mon[113531]: pgmap v8427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3879497365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:29.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:29.787113+0000) 2023-08-28T01:34:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:29 smithi093 ceph-mon[111335]: pgmap v8427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3879497365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:29 smithi093 ceph-mon[118902]: pgmap v8427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3879497365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:30.786+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:30.787336+0000) 2023-08-28T01:34:31.585 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:31.329Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:31.586 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:31.330Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:31 smithi136 ceph-mon[113531]: pgmap v8428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:31.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:31.787587+0000) 2023-08-28T01:34:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:31 smithi093 ceph-mon[111335]: pgmap v8428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:31 smithi093 ceph-mon[118902]: pgmap v8428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:32.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1309012193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:32.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:32.787813+0000) 2023-08-28T01:34:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1309012193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1309012193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:32.772Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:32.772Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:33 smithi136 ceph-mon[113531]: pgmap v8429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:33.787+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:33.788075+0000) 2023-08-28T01:34:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:33 smithi093 ceph-mon[111335]: pgmap v8429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:33 smithi093 ceph-mon[118902]: pgmap v8429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/338073349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:34.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:34.788313+0000) 2023-08-28T01:34:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/338073349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/338073349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:35 smithi093 ceph-mon[111335]: pgmap v8430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:34:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:34:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:34:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:35 smithi093 ceph-mon[118902]: pgmap v8430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:35.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:35 smithi136 ceph-mon[113531]: pgmap v8430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:35.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:35.788522+0000) 2023-08-28T01:34:37.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:36.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:36.788792+0000) 2023-08-28T01:34:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:37 smithi136 ceph-mon[113531]: pgmap v8431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2339629607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:37 smithi093 ceph-mon[111335]: pgmap v8431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2339629607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:37 smithi093 ceph-mon[118902]: pgmap v8431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2339629607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:37.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:37.789062+0000) 2023-08-28T01:34:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:38.788+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:38.789356+0000) 2023-08-28T01:34:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:39 smithi136 ceph-mon[113531]: pgmap v8432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1190105107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:39.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:39.789631+0000) 2023-08-28T01:34:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:39 smithi093 ceph-mon[118902]: pgmap v8432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1190105107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:39 smithi093 ceph-mon[111335]: pgmap v8432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:39.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1190105107' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:40.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:40.789867+0000) 2023-08-28T01:34:41.588 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:41.330Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:41.588 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:41.331Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:41 smithi136 ceph-mon[113531]: pgmap v8433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1764422936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:41.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:41.790140+0000) 2023-08-28T01:34:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:41 smithi093 ceph-mon[111335]: pgmap v8433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1764422936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:41 smithi093 ceph-mon[118902]: pgmap v8433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1764422936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:42.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:42.790389+0000) 2023-08-28T01:34:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:42.773Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:42.774Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:43 smithi136 ceph-mon[113531]: pgmap v8434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:43.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:43.790655+0000) 2023-08-28T01:34:43.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:43 smithi093 ceph-mon[111335]: pgmap v8434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:43 smithi093 ceph-mon[118902]: pgmap v8434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:44.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3368373395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:44.789+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:44.790915+0000) 2023-08-28T01:34:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3368373395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3368373395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:45 smithi093 ceph-mon[111335]: pgmap v8435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:34:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:34:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:34:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:45 smithi093 ceph-mon[118902]: pgmap v8435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:45.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:45 smithi136 ceph-mon[113531]: pgmap v8435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:45.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:45.791165+0000) 2023-08-28T01:34:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2607818464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:46.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:46.791335+0000) 2023-08-28T01:34:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2607818464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:46.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2607818464' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:47 smithi136 ceph-mon[113531]: pgmap v8436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:47.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:47.791539+0000) 2023-08-28T01:34:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:47 smithi093 ceph-mon[111335]: pgmap v8436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:47 smithi093 ceph-mon[118902]: pgmap v8436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:48.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:48.791763+0000) 2023-08-28T01:34:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:49 smithi136 ceph-mon[113531]: pgmap v8437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1034775966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:49.790+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:49.792007+0000) 2023-08-28T01:34:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:49 smithi093 ceph-mon[111335]: pgmap v8437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1034775966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:49 smithi093 ceph-mon[118902]: pgmap v8437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1034775966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:50.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:50.792264+0000) 2023-08-28T01:34:51.608 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:51.331Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:51.608 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:51.332Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:51 smithi136 ceph-mon[113531]: pgmap v8438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/542428556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:51.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:51.792520+0000) 2023-08-28T01:34:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:51 smithi093 ceph-mon[111335]: pgmap v8438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/542428556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:51 smithi093 ceph-mon[118902]: pgmap v8438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/542428556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:52.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:52.792715+0000) 2023-08-28T01:34:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:52.773Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:34:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:34:52.774Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:34:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:53 smithi136 ceph-mon[113531]: pgmap v8439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1726541453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:53.791+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:53.792988+0000) 2023-08-28T01:34:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:53 smithi093 ceph-mon[111335]: pgmap v8439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1726541453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:53 smithi093 ceph-mon[118902]: pgmap v8439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1726541453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:54.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:54.793207+0000) 2023-08-28T01:34:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:34:55.617 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:34:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:34:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:34:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:55 smithi136 ceph-mon[113531]: pgmap v8440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:55.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:55.793478+0000) 2023-08-28T01:34:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:55 smithi093 ceph-mon[111335]: pgmap v8440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:55 smithi093 ceph-mon[118902]: pgmap v8440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4139151065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:56.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:56.793721+0000) 2023-08-28T01:34:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4139151065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:56.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4139151065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:57 smithi136 ceph-mon[113531]: pgmap v8441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:57.792+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:57.793983+0000) 2023-08-28T01:34:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:57 smithi093 ceph-mon[111335]: pgmap v8441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:57 smithi093 ceph-mon[118902]: pgmap v8441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/296598592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:58.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:58.794260+0000) 2023-08-28T01:34:58.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/296598592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/296598592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:34:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:34:59 smithi136 ceph-mon[113531]: pgmap v8442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:34:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:34:59.793+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:34:59.794547+0000) 2023-08-28T01:34:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:34:59 smithi093 ceph-mon[111335]: pgmap v8442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:34:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:34:59 smithi093 ceph-mon[118902]: pgmap v8442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4233293250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:00.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:00.794802+0000) 2023-08-28T01:35:00.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4233293250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4233293250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:01.636 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:01.332Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:01.637 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:01.333Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:01 smithi136 ceph-mon[113531]: pgmap v8443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:01.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:01.795082+0000) 2023-08-28T01:35:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:01 smithi093 ceph-mon[111335]: pgmap v8443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:01 smithi093 ceph-mon[118902]: pgmap v8443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:02.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:02.795273+0000) 2023-08-28T01:35:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:02.774Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:02.775Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:03.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:03 smithi136 ceph-mon[113531]: pgmap v8444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2779993171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:03.794+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:03.795466+0000) 2023-08-28T01:35:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:03 smithi093 ceph-mon[111335]: pgmap v8444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2779993171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:03 smithi093 ceph-mon[118902]: pgmap v8444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2779993171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:04.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:04.795719+0000) 2023-08-28T01:35:05.642 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:35:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:35:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:35:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:05 smithi136 ceph-mon[113531]: pgmap v8445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1513021103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:05.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:05.795946+0000) 2023-08-28T01:35:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:05 smithi093 ceph-mon[111335]: pgmap v8445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1513021103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:05 smithi093 ceph-mon[118902]: pgmap v8445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1513021103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:07.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:06.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:06.796216+0000) 2023-08-28T01:35:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:07 smithi136 ceph-mon[113531]: pgmap v8446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:07 smithi093 ceph-mon[111335]: pgmap v8446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:07 smithi093 ceph-mon[118902]: pgmap v8446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:07.795+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:07.796486+0000) 2023-08-28T01:35:08.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1315931328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1315931328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1315931328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:08.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:08.796702+0000) 2023-08-28T01:35:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:09 smithi136 ceph-mon[113531]: pgmap v8447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:09 smithi093 ceph-mon[118902]: pgmap v8447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:09 smithi093 ceph-mon[111335]: pgmap v8447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:09.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:09.796961+0000) 2023-08-28T01:35:10.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3138155167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3138155167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3138155167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:11.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:10.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:10.797121+0000) 2023-08-28T01:35:11.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:11 smithi136 ceph-mon[113531]: pgmap v8448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:11 smithi093 ceph-mon[111335]: pgmap v8448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:11 smithi093 ceph-mon[118902]: pgmap v8448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:11.333Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:11.334Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:12.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:11.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:11.797332+0000) 2023-08-28T01:35:12.774 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2026621701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:12.774 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2026621701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:12.796 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2026621701' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:13.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:12.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:12.797579+0000) 2023-08-28T01:35:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:12.775Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:12.776Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:13.796 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:13 smithi136 ceph-mon[113531]: pgmap v8449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:13.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:13 smithi093 ceph-mon[111335]: pgmap v8449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:13 smithi093 ceph-mon[118902]: pgmap v8449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:13.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:13.797778+0000) 2023-08-28T01:35:14.616 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:35:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:35:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:35:14.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:14.796+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:14.797924+0000) 2023-08-28T01:35:15.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:35:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:35:15.796 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:15 smithi136 ceph-mon[113531]: pgmap v8450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:15.797 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/552502276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:15.797 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:15.797 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[111335]: pgmap v8450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/552502276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:15.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[118902]: pgmap v8450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/552502276' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:15.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:16.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:15.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:15.798062+0000) 2023-08-28T01:35:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:35:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:35:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:17 smithi136 ceph-mon[113531]: pgmap v8451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:16.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:16.798283+0000) 2023-08-28T01:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[111335]: pgmap v8451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:17 smithi093 ceph-mon[118902]: pgmap v8451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1281438730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:17.797+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:17.798492+0000) 2023-08-28T01:35:18.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1281438730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:18.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1281438730' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:19 smithi136 ceph-mon[113531]: pgmap v8452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:18.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:18.798735+0000) 2023-08-28T01:35:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:19 smithi093 ceph-mon[111335]: pgmap v8452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:19 smithi093 ceph-mon[118902]: pgmap v8452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3193533057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:19.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:19.798979+0000) 2023-08-28T01:35:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3193533057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3193533057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:21 smithi136 ceph-mon[113531]: pgmap v8453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:20.798+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:20.799236+0000) 2023-08-28T01:35:21.332 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:21 smithi093 ceph-mon[111335]: pgmap v8453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:21.333 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:21 smithi093 ceph-mon[118902]: pgmap v8453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:21.333Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:21.334Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:21.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:21.799442+0000) 2023-08-28T01:35:22.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/189283070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/189283070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:22.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/189283070' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:22.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:22.799674+0000) 2023-08-28T01:35:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:22.776Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:22.777Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:23.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:23 smithi136 ceph-mon[113531]: pgmap v8454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:23 smithi093 ceph-mon[111335]: pgmap v8454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:23 smithi093 ceph-mon[118902]: pgmap v8454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:24.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:23.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:23.799957+0000) 2023-08-28T01:35:24.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:24.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:24.800221+0000) 2023-08-28T01:35:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3433642447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:25 smithi136 ceph-mon[113531]: pgmap v8455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3433642447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:25 smithi093 ceph-mon[111335]: pgmap v8455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3433642447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:25 smithi093 ceph-mon[118902]: pgmap v8455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:35:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:35:25] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:35:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:25.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:25.800460+0000) 2023-08-28T01:35:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:26.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:26.800665+0000) 2023-08-28T01:35:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:27 smithi136 ceph-mon[113531]: pgmap v8456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/412891310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:27 smithi093 ceph-mon[111335]: pgmap v8456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/412891310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:27 smithi093 ceph-mon[118902]: pgmap v8456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/412891310' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:27.799+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:27.800905+0000) 2023-08-28T01:35:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:28.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:28.801162+0000) 2023-08-28T01:35:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:29 smithi136 ceph-mon[113531]: pgmap v8457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4246012775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:29.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:29.801413+0000) 2023-08-28T01:35:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:29 smithi093 ceph-mon[111335]: pgmap v8457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4246012775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:29 smithi093 ceph-mon[118902]: pgmap v8457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4246012775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:30.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:30.801670+0000) 2023-08-28T01:35:31.601 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:31.335Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:31.601 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:31.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:31 smithi136 ceph-mon[113531]: pgmap v8458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1875530237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:31.800+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:31.801922+0000) 2023-08-28T01:35:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:31 smithi093 ceph-mon[118902]: pgmap v8458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1875530237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:31 smithi093 ceph-mon[111335]: pgmap v8458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1875530237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:32.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:32.802177+0000) 2023-08-28T01:35:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:32.777Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:32.778Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:33 smithi136 ceph-mon[113531]: pgmap v8459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:33.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:33.802407+0000) 2023-08-28T01:35:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:33 smithi093 ceph-mon[111335]: pgmap v8459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:33 smithi093 ceph-mon[118902]: pgmap v8459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:34.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2088926274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:34.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:34.802639+0000) 2023-08-28T01:35:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2088926274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2088926274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:35.609 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:35:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:35:35] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:35:35.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:35 smithi136 ceph-mon[113531]: pgmap v8460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:35.801+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:35.802885+0000) 2023-08-28T01:35:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:35 smithi093 ceph-mon[111335]: pgmap v8460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:35 smithi093 ceph-mon[118902]: pgmap v8460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/816703966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:36.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:36.803175+0000) 2023-08-28T01:35:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/816703966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:36.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/816703966' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:37 smithi136 ceph-mon[113531]: pgmap v8461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:37.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:37.803462+0000) 2023-08-28T01:35:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:37 smithi093 ceph-mon[111335]: pgmap v8461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:37 smithi093 ceph-mon[118902]: pgmap v8461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:38.802+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:38.803721+0000) 2023-08-28T01:35:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:39.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:39.803948+0000) 2023-08-28T01:35:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:39 smithi136 ceph-mon[113531]: pgmap v8462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/591530067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:39 smithi093 ceph-mon[111335]: pgmap v8462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/591530067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:39 smithi093 ceph-mon[118902]: pgmap v8462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/591530067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:39.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:40.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:40.804200+0000) 2023-08-28T01:35:41.618 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:41.335Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:41.618 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:41.336Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:41.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:41.804400+0000) 2023-08-28T01:35:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:41 smithi136 ceph-mon[113531]: pgmap v8463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1862955437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:41 smithi093 ceph-mon[111335]: pgmap v8463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1862955437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:41 smithi093 ceph-mon[118902]: pgmap v8463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1862955437' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:42.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:42.804618+0000) 2023-08-28T01:35:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:42.778Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:42.779Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:43 smithi136 ceph-mon[113531]: pgmap v8464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/999630539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:43.803+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:43.804854+0000) 2023-08-28T01:35:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:43 smithi093 ceph-mon[111335]: pgmap v8464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/999630539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:43 smithi093 ceph-mon[118902]: pgmap v8464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/999630539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:44.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:44.805062+0000) 2023-08-28T01:35:45.624 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:35:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:35:45] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:35:45.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:45 smithi136 ceph-mon[113531]: pgmap v8465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:45.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:45.805267+0000) 2023-08-28T01:35:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:45 smithi093 ceph-mon[111335]: pgmap v8465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:45 smithi093 ceph-mon[118902]: pgmap v8465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:46.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/408455171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:46.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:46.805488+0000) 2023-08-28T01:35:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/408455171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:46.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/408455171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:47 smithi136 ceph-mon[113531]: pgmap v8466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:47.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:47.805724+0000) 2023-08-28T01:35:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:47 smithi093 ceph-mon[111335]: pgmap v8466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:47 smithi093 ceph-mon[118902]: pgmap v8466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2243994225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:48.804+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:48.806007+0000) 2023-08-28T01:35:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2243994225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2243994225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:49 smithi136 ceph-mon[113531]: pgmap v8467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:49.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:49.806228+0000) 2023-08-28T01:35:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:49 smithi093 ceph-mon[111335]: pgmap v8467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:49 smithi093 ceph-mon[118902]: pgmap v8467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/314474088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:50.805+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:50.806472+0000) 2023-08-28T01:35:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/314474088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/314474088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:51.646 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:51.336Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:51.646 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:51.337Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:51 smithi136 ceph-mon[113531]: pgmap v8468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:51.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:51.806670+0000) 2023-08-28T01:35:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:51 smithi093 ceph-mon[111335]: pgmap v8468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:51 smithi093 ceph-mon[118902]: pgmap v8468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:52.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:52.806890+0000) 2023-08-28T01:35:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:52.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:35:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:35:52.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:35:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:53 smithi136 ceph-mon[113531]: pgmap v8469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1667070570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:53.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:53.807162+0000) 2023-08-28T01:35:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:53 smithi093 ceph-mon[111335]: pgmap v8469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1667070570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:53 smithi093 ceph-mon[118902]: pgmap v8469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1667070570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:54.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:54.807396+0000) 2023-08-28T01:35:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:35:55.657 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:35:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:35:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:35:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:55 smithi136 ceph-mon[113531]: pgmap v8470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3743014835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:55.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:55.807640+0000) 2023-08-28T01:35:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:55 smithi093 ceph-mon[111335]: pgmap v8470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3743014835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:55 smithi093 ceph-mon[118902]: pgmap v8470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3743014835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:56.806+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:56.807875+0000) 2023-08-28T01:35:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:57 smithi136 ceph-mon[113531]: pgmap v8471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:57 smithi093 ceph-mon[111335]: pgmap v8471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:57 smithi093 ceph-mon[118902]: pgmap v8471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:57.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:57.808157+0000) 2023-08-28T01:35:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3215589278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3215589278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3215589278' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:35:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:58.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:58.808378+0000) 2023-08-28T01:35:59.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:35:59 smithi136 ceph-mon[113531]: pgmap v8472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:35:59 smithi093 ceph-mon[111335]: pgmap v8472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:35:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:35:59 smithi093 ceph-mon[118902]: pgmap v8472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:00.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:35:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:35:59.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:35:59.808608+0000) 2023-08-28T01:36:00.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2620949791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2620949791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2620949791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:00.807+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:00.808860+0000) 2023-08-28T01:36:01.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:01 smithi136 ceph-mon[113531]: pgmap v8473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:01 smithi093 ceph-mon[111335]: pgmap v8473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:01 smithi093 ceph-mon[118902]: pgmap v8473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:01.336Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:01.337Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:02.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:01.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:01.809135+0000) 2023-08-28T01:36:02.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4085457285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:02.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:02.809311+0000) 2023-08-28T01:36:02.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4085457285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:02.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4085457285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:02.779Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:02.780Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:03.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:03 smithi136 ceph-mon[113531]: pgmap v8474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:03.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:03.809573+0000) 2023-08-28T01:36:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:03 smithi093 ceph-mon[111335]: pgmap v8474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:03 smithi093 ceph-mon[118902]: pgmap v8474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:04.808+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:04.809856+0000) 2023-08-28T01:36:05.610 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:36:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:36:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:36:05.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:05 smithi136 ceph-mon[113531]: pgmap v8475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/866209469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:05.809+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:05.810132+0000) 2023-08-28T01:36:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:05 smithi093 ceph-mon[111335]: pgmap v8475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/866209469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:05 smithi093 ceph-mon[118902]: pgmap v8475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/866209469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:07.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:06.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:06.810367+0000) 2023-08-28T01:36:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:07 smithi136 ceph-mon[113531]: pgmap v8476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4275722025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:07 smithi093 ceph-mon[111335]: pgmap v8476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4275722025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:07 smithi093 ceph-mon[118902]: pgmap v8476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4275722025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:07.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:07.810614+0000) 2023-08-28T01:36:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:08.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:08.810833+0000) 2023-08-28T01:36:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:09 smithi136 ceph-mon[113531]: pgmap v8477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:09.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:09.811117+0000) 2023-08-28T01:36:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:09 smithi093 ceph-mon[111335]: pgmap v8477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:09 smithi093 ceph-mon[118902]: pgmap v8477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1674588051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:10.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:10.811367+0000) 2023-08-28T01:36:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1674588051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1674588051' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:11.615 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:11.337Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:11.615 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:11.338Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:11 smithi136 ceph-mon[113531]: pgmap v8478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:11.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:11.811636+0000) 2023-08-28T01:36:11.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:11 smithi093 ceph-mon[111335]: pgmap v8478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:11 smithi093 ceph-mon[118902]: pgmap v8478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/832924443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:12.810+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:12.811894+0000) 2023-08-28T01:36:12.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/832924443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:12.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/832924443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:12.780Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:12.781Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:13 smithi136 ceph-mon[113531]: pgmap v8479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:13.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:13.812103+0000) 2023-08-28T01:36:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:13 smithi093 ceph-mon[111335]: pgmap v8479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:13 smithi093 ceph-mon[118902]: pgmap v8479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1649706812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:14.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:14.812313+0000) 2023-08-28T01:36:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1649706812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1649706812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:15.627 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:36:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:36:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:36:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:15 smithi136 ceph-mon[113531]: pgmap v8480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:15.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:15.812572+0000) 2023-08-28T01:36:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:15 smithi093 ceph-mon[111335]: pgmap v8480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:15 smithi093 ceph-mon[118902]: pgmap v8480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:17.182 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:16.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:16.812767+0000) 2023-08-28T01:36:17.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:17 smithi136 ceph-mon[113531]: pgmap v8481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:36:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1031220010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:17 smithi093 ceph-mon[111335]: pgmap v8481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:36:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1031220010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:17 smithi093 ceph-mon[118902]: pgmap v8481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:36:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1031220010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:17.812 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:17.811+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:17.812909+0000) 2023-08-28T01:36:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:18.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:18.813146+0000) 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[111335]: pgmap v8482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[118902]: pgmap v8482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:19 smithi136 ceph-mon[113531]: pgmap v8482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:19.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:19.813375+0000) 2023-08-28T01:36:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1945130605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:36:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:36:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1945130605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:36:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:36:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1945130605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:36:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:36:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:36:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:20.812+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:20.813628+0000) 2023-08-28T01:36:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:21 smithi093 ceph-mon[111335]: pgmap v8483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:21 smithi093 ceph-mon[118902]: pgmap v8483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:21.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:21.338Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:21.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:21.339Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:21.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:21 smithi136 ceph-mon[113531]: pgmap v8483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:21.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:21.813853+0000) 2023-08-28T01:36:22.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1900458208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1900458208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:22.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1900458208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:22.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:22.814139+0000) 2023-08-28T01:36:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:22.781Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:22.782Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:23.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:23 smithi136 ceph-mon[113531]: pgmap v8484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:23 smithi093 ceph-mon[111335]: pgmap v8484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:23 smithi093 ceph-mon[118902]: pgmap v8484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:24.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:23.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:23.814360+0000) 2023-08-28T01:36:24.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3796448421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3796448421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3796448421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:24.813+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:24.814585+0000) 2023-08-28T01:36:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:25 smithi136 ceph-mon[113531]: pgmap v8485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:25 smithi093 ceph-mon[111335]: pgmap v8485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:25 smithi093 ceph-mon[118902]: pgmap v8485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:36:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:36:25] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:36:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:25.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:25.814839+0000) 2023-08-28T01:36:26.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2206036791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2206036791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2206036791' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:26.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:26.815126+0000) 2023-08-28T01:36:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:27 smithi136 ceph-mon[113531]: pgmap v8486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:27 smithi093 ceph-mon[111335]: pgmap v8486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:27 smithi093 ceph-mon[118902]: pgmap v8486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:27.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:27.815338+0000) 2023-08-28T01:36:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:28.814+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:28.815589+0000) 2023-08-28T01:36:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:29 smithi136 ceph-mon[113531]: pgmap v8487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2564328472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:29.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:29.815826+0000) 2023-08-28T01:36:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:29 smithi093 ceph-mon[111335]: pgmap v8487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2564328472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:29 smithi093 ceph-mon[118902]: pgmap v8487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2564328472' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:30.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:30.816008+0000) 2023-08-28T01:36:31.616 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:31.338Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:31.616 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:31.339Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:31 smithi136 ceph-mon[113531]: pgmap v8488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2191129495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:31.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:31.816284+0000) 2023-08-28T01:36:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:31 smithi093 ceph-mon[118902]: pgmap v8488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2191129495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:31 smithi093 ceph-mon[111335]: pgmap v8488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2191129495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:32.815+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:32.816533+0000) 2023-08-28T01:36:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:32.782Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:32.783Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:33 smithi136 ceph-mon[113531]: pgmap v8489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/169618639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:33.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:33.816784+0000) 2023-08-28T01:36:33.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:33 smithi093 ceph-mon[111335]: pgmap v8489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:33.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/169618639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:33 smithi093 ceph-mon[118902]: pgmap v8489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/169618639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:34.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:34.817054+0000) 2023-08-28T01:36:35.622 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:36:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:36:35] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:36:35.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:35 smithi136 ceph-mon[113531]: pgmap v8490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:35.816+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:35.817288+0000) 2023-08-28T01:36:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:35 smithi093 ceph-mon[111335]: pgmap v8490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:35 smithi093 ceph-mon[118902]: pgmap v8490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1163817788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:36.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:36.817542+0000) 2023-08-28T01:36:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1163817788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:36.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1163817788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:37.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:37 smithi136 ceph-mon[113531]: pgmap v8491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:37.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:37.817788+0000) 2023-08-28T01:36:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:37 smithi093 ceph-mon[111335]: pgmap v8491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:37 smithi093 ceph-mon[118902]: pgmap v8491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/81541255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:38.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:38.818062+0000) 2023-08-28T01:36:38.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/81541255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/81541255' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:39 smithi136 ceph-mon[113531]: pgmap v8492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:39.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:39.818291+0000) 2023-08-28T01:36:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:39 smithi093 ceph-mon[111335]: pgmap v8492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:39 smithi093 ceph-mon[118902]: pgmap v8492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:40.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:40.818513+0000) 2023-08-28T01:36:41.635 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:41.339Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:41.635 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:41.340Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:41 smithi136 ceph-mon[113531]: pgmap v8493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2167142788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:41.817+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:41.818782+0000) 2023-08-28T01:36:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:41 smithi093 ceph-mon[118902]: pgmap v8493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2167142788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:41 smithi093 ceph-mon[111335]: pgmap v8493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2167142788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:42.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:42.819054+0000) 2023-08-28T01:36:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:42.782Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:42.783Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:43 smithi136 ceph-mon[113531]: pgmap v8494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:43.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1455843148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:43.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:43.819293+0000) 2023-08-28T01:36:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:43 smithi093 ceph-mon[111335]: pgmap v8494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1455843148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:43 smithi093 ceph-mon[118902]: pgmap v8494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1455843148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:44.818+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:44.819509+0000) 2023-08-28T01:36:45.641 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:36:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:36:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:36:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:45 smithi136 ceph-mon[113531]: pgmap v8495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3616417210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:45.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:45.819783+0000) 2023-08-28T01:36:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:45 smithi093 ceph-mon[118902]: pgmap v8495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3616417210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:45 smithi093 ceph-mon[111335]: pgmap v8495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3616417210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:46.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:46.820084+0000) 2023-08-28T01:36:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:47 smithi136 ceph-mon[113531]: pgmap v8496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:47 smithi093 ceph-mon[111335]: pgmap v8496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:47 smithi093 ceph-mon[118902]: pgmap v8496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:47.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:47.820289+0000) 2023-08-28T01:36:48.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2399638605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2399638605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2399638605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:48.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:48.820517+0000) 2023-08-28T01:36:49.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:49 smithi136 ceph-mon[113531]: pgmap v8497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:49 smithi093 ceph-mon[111335]: pgmap v8497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:49 smithi093 ceph-mon[118902]: pgmap v8497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:50.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:49.820+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:49.820754+0000) 2023-08-28T01:36:50.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3097664138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3097664138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3097664138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:50.819+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:50.820983+0000) 2023-08-28T01:36:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:51 smithi136 ceph-mon[113531]: pgmap v8498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:51 smithi093 ceph-mon[111335]: pgmap v8498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:51 smithi093 ceph-mon[118902]: pgmap v8498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:51.340Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:51.341Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:52.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:51.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:51.821186+0000) 2023-08-28T01:36:52.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1790107059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:52.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:52.821423+0000) 2023-08-28T01:36:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1790107059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1790107059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:52.783Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:36:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:36:52.784Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:36:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:53 smithi136 ceph-mon[113531]: pgmap v8499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:53.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:53.821600+0000) 2023-08-28T01:36:53.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:53 smithi093 ceph-mon[111335]: pgmap v8499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:53 smithi093 ceph-mon[118902]: pgmap v8499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:54.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:54.821833+0000) 2023-08-28T01:36:54.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:36:55.632 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:36:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:36:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:36:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:55 smithi136 ceph-mon[113531]: pgmap v8500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1704095816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:55.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:55.822102+0000) 2023-08-28T01:36:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:55 smithi093 ceph-mon[111335]: pgmap v8500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1704095816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:55 smithi093 ceph-mon[118902]: pgmap v8500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1704095816' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:56.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:56.822283+0000) 2023-08-28T01:36:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:57 smithi136 ceph-mon[113531]: pgmap v8501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3956112553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:57 smithi093 ceph-mon[111335]: pgmap v8501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3956112553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:57 smithi093 ceph-mon[118902]: pgmap v8501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3956112553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:36:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:57.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:57.822525+0000) 2023-08-28T01:36:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:58.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:58.822777+0000) 2023-08-28T01:36:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:36:59 smithi136 ceph-mon[113531]: pgmap v8502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:36:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:36:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:36:59.821+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:36:59.822993+0000) 2023-08-28T01:36:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:36:59 smithi093 ceph-mon[111335]: pgmap v8502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:00.041 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:36:59 smithi093 ceph-mon[118902]: pgmap v8502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:00.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1526065583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:00.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:00.823253+0000) 2023-08-28T01:37:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1526065583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1526065583' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:01.629 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:01.341Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:01.629 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:01.342Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:01 smithi136 ceph-mon[113531]: pgmap v8503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:01.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:01.823515+0000) 2023-08-28T01:37:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:01 smithi093 ceph-mon[111335]: pgmap v8503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:01 smithi093 ceph-mon[118902]: pgmap v8503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:02.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/114255710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:02.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:02.823713+0000) 2023-08-28T01:37:02.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/114255710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:02.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/114255710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:02.784Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:02.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:02.785Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:03 smithi136 ceph-mon[113531]: pgmap v8504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:03.822+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:03.823968+0000) 2023-08-28T01:37:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:03 smithi093 ceph-mon[111335]: pgmap v8504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:03 smithi093 ceph-mon[118902]: pgmap v8504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1806527225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:04.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:04.824188+0000) 2023-08-28T01:37:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1806527225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1806527225' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:05.640 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:37:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:37:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:37:05.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:05 smithi136 ceph-mon[113531]: pgmap v8505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:05.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:05.824376+0000) 2023-08-28T01:37:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:05 smithi093 ceph-mon[111335]: pgmap v8505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:05 smithi093 ceph-mon[118902]: pgmap v8505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:07.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:06.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:06.824576+0000) 2023-08-28T01:37:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:07 smithi136 ceph-mon[113531]: pgmap v8506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:07.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1929255050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:07 smithi093 ceph-mon[111335]: pgmap v8506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1929255050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:07 smithi093 ceph-mon[118902]: pgmap v8506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1929255050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:07.823+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:07.824804+0000) 2023-08-28T01:37:09.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:08.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:08.825094+0000) 2023-08-28T01:37:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:09 smithi136 ceph-mon[113531]: pgmap v8507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2003805349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:09.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:09.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:09.825298+0000) 2023-08-28T01:37:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:09 smithi093 ceph-mon[111335]: pgmap v8507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2003805349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:09 smithi093 ceph-mon[118902]: pgmap v8507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2003805349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:10.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:10.825548+0000) 2023-08-28T01:37:11.630 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:11.342Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:11.630 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:11.343Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:11 smithi136 ceph-mon[113531]: pgmap v8508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:11.824+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:11.825815+0000) 2023-08-28T01:37:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:11 smithi093 ceph-mon[111335]: pgmap v8508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:11 smithi093 ceph-mon[118902]: pgmap v8508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4090732568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:12.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:12.826081+0000) 2023-08-28T01:37:12.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4090732568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:12.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4090732568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:12.785Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:12.786Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:13.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:13 smithi136 ceph-mon[113531]: pgmap v8509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:13.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:13.826321+0000) 2023-08-28T01:37:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:13 smithi093 ceph-mon[111335]: pgmap v8509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:13 smithi093 ceph-mon[118902]: pgmap v8509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:14.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1768353641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:14.825+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:14.826556+0000) 2023-08-28T01:37:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1768353641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1768353641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:15.645 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:37:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:37:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:37:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:15 smithi136 ceph-mon[113531]: pgmap v8510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:15.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:15.826824+0000) 2023-08-28T01:37:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:15 smithi093 ceph-mon[111335]: pgmap v8510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:15 smithi093 ceph-mon[118902]: pgmap v8510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:16.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:16.827101+0000) 2023-08-28T01:37:17.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2922943182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:17 smithi136 ceph-mon[113531]: pgmap v8511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2922943182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:17 smithi093 ceph-mon[111335]: pgmap v8511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2922943182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:17 smithi093 ceph-mon[118902]: pgmap v8511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:18.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:17.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:17.827314+0000) 2023-08-28T01:37:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:18.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:18.827523+0000) 2023-08-28T01:37:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:19 smithi136 ceph-mon[113531]: pgmap v8512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3546667315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:19.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:37:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:19.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:19.827700+0000) 2023-08-28T01:37:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:19 smithi093 ceph-mon[111335]: pgmap v8512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3546667315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:37:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:19 smithi093 ceph-mon[118902]: pgmap v8512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3546667315' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:37:20.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:20.826+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:20.827866+0000) 2023-08-28T01:37:21.342 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:21.342Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:21.342 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:21.344Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:21.826 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:21 smithi136 ceph-mon[113531]: pgmap v8513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:21.826 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2963545302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:21.826 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:21.827 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[111335]: pgmap v8513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2963545302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[118902]: pgmap v8513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2963545302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:21.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:21.828100+0000) 2023-08-28T01:37:23.091 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:22.786Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:23.092 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:22.787Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:23.092 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:22.827+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:22.828279+0000) 2023-08-28T01:37:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:37:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:37:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:23 smithi136 ceph-mon[113531]: pgmap v8514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:23.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[111335]: pgmap v8514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:37:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:37:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:23 smithi093 ceph-mon[118902]: pgmap v8514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:24.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:23.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:23.828533+0000) 2023-08-28T01:37:24.827 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3991192861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:24.827 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3991192861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3991192861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:25.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:24.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:24.828776+0000) 2023-08-28T01:37:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:25 smithi093 ceph-mon[111335]: pgmap v8515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:25 smithi093 ceph-mon[118902]: pgmap v8515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:37:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:37:25] "GET /metrics HTTP/1.1" 200 239935 "" "Prometheus/2.43.0" 2023-08-28T01:37:25.827 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:25 smithi136 ceph-mon[113531]: pgmap v8515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:25.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:25.829003+0000) 2023-08-28T01:37:26.827 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2034193974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2034193974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:26.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2034193974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:26.828+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:26.829240+0000) 2023-08-28T01:37:27.828 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:27 smithi136 ceph-mon[113531]: pgmap v8516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:27 smithi093 ceph-mon[111335]: pgmap v8516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:27 smithi093 ceph-mon[118902]: pgmap v8516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:27.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:27.829506+0000) 2023-08-28T01:37:28.828 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2758262349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2758262349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2758262349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:28.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:28.829750+0000) 2023-08-28T01:37:29.828 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:29 smithi136 ceph-mon[113531]: pgmap v8517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:29 smithi093 ceph-mon[111335]: pgmap v8517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:29 smithi093 ceph-mon[118902]: pgmap v8517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:29.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:29.830026+0000) 2023-08-28T01:37:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:30.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:30.830184+0000) 2023-08-28T01:37:31.630 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:31.344Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:31.631 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:31.345Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:31.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:31.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:31.830387+0000) 2023-08-28T01:37:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:31 smithi136 ceph-mon[113531]: pgmap v8518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:31.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/379884611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:31 smithi093 ceph-mon[111335]: pgmap v8518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/379884611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:31 smithi093 ceph-mon[118902]: pgmap v8518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/379884611' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:32.829+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:32.830649+0000) 2023-08-28T01:37:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:32.787Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:32.788Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:33 smithi136 ceph-mon[113531]: pgmap v8519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3737194936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:33.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:33.830870+0000) 2023-08-28T01:37:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:33 smithi093 ceph-mon[111335]: pgmap v8519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3737194936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:33 smithi093 ceph-mon[118902]: pgmap v8519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3737194936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:34.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:34.831135+0000) 2023-08-28T01:37:35.636 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:37:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:37:35] "GET /metrics HTTP/1.1" 200 239935 "" "Prometheus/2.43.0" 2023-08-28T01:37:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:35.830+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:35.831343+0000) 2023-08-28T01:37:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:35 smithi136 ceph-mon[113531]: pgmap v8520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1684614805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:35 smithi093 ceph-mon[118902]: pgmap v8520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1684614805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:35 smithi093 ceph-mon[111335]: pgmap v8520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1684614805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:36.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:36.831569+0000) 2023-08-28T01:37:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:37 smithi136 ceph-mon[113531]: pgmap v8521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:37 smithi093 ceph-mon[111335]: pgmap v8521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:37 smithi093 ceph-mon[118902]: pgmap v8521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:37.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:37.831808+0000) 2023-08-28T01:37:38.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2757091438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2757091438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2757091438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:38.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:38.831980+0000) 2023-08-28T01:37:39.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:39 smithi136 ceph-mon[113531]: pgmap v8522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:39 smithi093 ceph-mon[111335]: pgmap v8522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:39 smithi093 ceph-mon[118902]: pgmap v8522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:39.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:39.832169+0000) 2023-08-28T01:37:40.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2819089993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:40.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2819089993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2819089993' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:40.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:40.832364+0000) 2023-08-28T01:37:41.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:41 smithi136 ceph-mon[113531]: pgmap v8523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:41 smithi093 ceph-mon[111335]: pgmap v8523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:41 smithi093 ceph-mon[118902]: pgmap v8523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:41.345Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:41.346Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:42.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:41.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:41.832570+0000) 2023-08-28T01:37:42.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/679984429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:42.831+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:42.832816+0000) 2023-08-28T01:37:42.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/679984429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/679984429' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:42.788Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:42.789Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:43 smithi136 ceph-mon[113531]: pgmap v8524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:43.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:43.833084+0000) 2023-08-28T01:37:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:43 smithi093 ceph-mon[111335]: pgmap v8524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:43 smithi093 ceph-mon[118902]: pgmap v8524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:44.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:44.833284+0000) 2023-08-28T01:37:45.664 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:37:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:37:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:37:45.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:45 smithi136 ceph-mon[113531]: pgmap v8525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/201792857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:45.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:45.833487+0000) 2023-08-28T01:37:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:45 smithi093 ceph-mon[111335]: pgmap v8525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/201792857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:45 smithi093 ceph-mon[118902]: pgmap v8525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/201792857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:46.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:46.833675+0000) 2023-08-28T01:37:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:47 smithi136 ceph-mon[113531]: pgmap v8526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/255106703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:47 smithi093 ceph-mon[111335]: pgmap v8526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/255106703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:47 smithi093 ceph-mon[118902]: pgmap v8526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/255106703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:47.832+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:47.833860+0000) 2023-08-28T01:37:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:48.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:48.834086+0000) 2023-08-28T01:37:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:49 smithi136 ceph-mon[113531]: pgmap v8527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:49.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:49.834318+0000) 2023-08-28T01:37:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:49 smithi093 ceph-mon[111335]: pgmap v8527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:49 smithi093 ceph-mon[118902]: pgmap v8527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1323007458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:50.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:50.834536+0000) 2023-08-28T01:37:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1323007458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1323007458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:51.642 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:51.346Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:51.642 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:51.347Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:51 smithi136 ceph-mon[113531]: pgmap v8528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:51.833+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:51.834754+0000) 2023-08-28T01:37:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:51 smithi093 ceph-mon[111335]: pgmap v8528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:51 smithi093 ceph-mon[118902]: pgmap v8528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:52.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3450036145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:52.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:52.834995+0000) 2023-08-28T01:37:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3450036145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3450036145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:52.789Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:37:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:37:52.790Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:37:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:53 smithi136 ceph-mon[113531]: pgmap v8529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:53.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:53.835171+0000) 2023-08-28T01:37:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:53 smithi093 ceph-mon[111335]: pgmap v8529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:53 smithi093 ceph-mon[118902]: pgmap v8529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3210475898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:54.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:54.835438+0000) 2023-08-28T01:37:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3210475898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:37:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3210475898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:55.654 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:37:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:37:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:37:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:55 smithi136 ceph-mon[113531]: pgmap v8530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:55.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:55.835708+0000) 2023-08-28T01:37:55.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:55 smithi093 ceph-mon[111335]: pgmap v8530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:55 smithi093 ceph-mon[118902]: pgmap v8530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:56.834+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:56.835918+0000) 2023-08-28T01:37:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:57 smithi136 ceph-mon[113531]: pgmap v8531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3085409868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:57 smithi093 ceph-mon[111335]: pgmap v8531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3085409868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:57 smithi093 ceph-mon[118902]: pgmap v8531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3085409868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:57.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:57.836176+0000) 2023-08-28T01:37:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:58.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:58.836420+0000) 2023-08-28T01:37:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:59 smithi136 ceph-mon[113531]: pgmap v8532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:37:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2313402326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:37:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:37:59.835+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:37:59.836606+0000) 2023-08-28T01:37:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:59 smithi093 ceph-mon[111335]: pgmap v8532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:37:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2313402326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:37:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:59 smithi093 ceph-mon[118902]: pgmap v8532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:37:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:37:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2313402326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:00.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:00.836862+0000) 2023-08-28T01:38:01.641 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:01.347Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:01.641 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:01.348Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:01 smithi136 ceph-mon[113531]: pgmap v8533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:01.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:01.837176+0000) 2023-08-28T01:38:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:01 smithi093 ceph-mon[111335]: pgmap v8533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:01 smithi093 ceph-mon[118902]: pgmap v8533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:02.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/888767546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:02.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:02.837363+0000) 2023-08-28T01:38:02.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/888767546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:02.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/888767546' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:02.790Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:02.791Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:03 smithi136 ceph-mon[113531]: pgmap v8534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:03.837+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:03.837616+0000) 2023-08-28T01:38:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:03 smithi093 ceph-mon[111335]: pgmap v8534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:03 smithi093 ceph-mon[118902]: pgmap v8534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:04.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3329110498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:04.836+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:04.837915+0000) 2023-08-28T01:38:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3329110498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3329110498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:05.662 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:38:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:38:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:38:05.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:05 smithi136 ceph-mon[113531]: pgmap v8535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:05.838+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:05.839374+0000) 2023-08-28T01:38:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:05 smithi093 ceph-mon[111335]: pgmap v8535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:05 smithi093 ceph-mon[118902]: pgmap v8535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:06.838+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:06.839612+0000) 2023-08-28T01:38:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/73010068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:07 smithi136 ceph-mon[113531]: pgmap v8536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/73010068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:07 smithi093 ceph-mon[111335]: pgmap v8536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/73010068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:07 smithi093 ceph-mon[118902]: pgmap v8536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:07.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:07.839881+0000) 2023-08-28T01:38:09.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:08.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:08.840163+0000) 2023-08-28T01:38:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:09 smithi136 ceph-mon[113531]: pgmap v8537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2820582981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:09.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:09.840463+0000) 2023-08-28T01:38:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:09 smithi093 ceph-mon[111335]: pgmap v8537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2820582981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:09 smithi093 ceph-mon[118902]: pgmap v8537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2820582981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:10.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:10.840659+0000) 2023-08-28T01:38:11.643 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:11.348Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:11.644 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:11.349Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:11 smithi136 ceph-mon[113531]: pgmap v8538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2896441809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:11.839+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:11.840909+0000) 2023-08-28T01:38:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:11 smithi093 ceph-mon[118902]: pgmap v8538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2896441809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:11 smithi093 ceph-mon[111335]: pgmap v8538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2896441809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:12.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:12.841219+0000) 2023-08-28T01:38:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:12.791Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:12.792Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:13.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:13 smithi136 ceph-mon[113531]: pgmap v8539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/201139336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:13.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:13.841482+0000) 2023-08-28T01:38:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:13 smithi093 ceph-mon[111335]: pgmap v8539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/201139336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:13 smithi093 ceph-mon[118902]: pgmap v8539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/201139336' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:14.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:14.841745+0000) 2023-08-28T01:38:15.649 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:38:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:38:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:38:15.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:15 smithi136 ceph-mon[113531]: pgmap v8540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:15.840+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:15.841966+0000) 2023-08-28T01:38:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:15 smithi093 ceph-mon[111335]: pgmap v8540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:15 smithi093 ceph-mon[118902]: pgmap v8540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:16.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/475338922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:16.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:16.842223+0000) 2023-08-28T01:38:16.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/475338922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:16.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/475338922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:17 smithi136 ceph-mon[113531]: pgmap v8541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:17.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:17.842499+0000) 2023-08-28T01:38:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:17 smithi093 ceph-mon[111335]: pgmap v8541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:17 smithi093 ceph-mon[118902]: pgmap v8541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2043200597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:18.841+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:18.842681+0000) 2023-08-28T01:38:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2043200597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2043200597' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:19 smithi136 ceph-mon[113531]: pgmap v8542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:19.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:19.842925+0000) 2023-08-28T01:38:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:19 smithi093 ceph-mon[111335]: pgmap v8542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:19 smithi093 ceph-mon[118902]: pgmap v8542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:20.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:20.843134+0000) 2023-08-28T01:38:21.661 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:21.348Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:21.661 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:21.349Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:21 smithi136 ceph-mon[113531]: pgmap v8543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:21.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2166339969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:21.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:21.843353+0000) 2023-08-28T01:38:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:21 smithi093 ceph-mon[111335]: pgmap v8543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2166339969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:21 smithi093 ceph-mon[118902]: pgmap v8543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2166339969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:22.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:38:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:22.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:22.843513+0000) 2023-08-28T01:38:22.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:38:22.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:38:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:22.792Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:22.793Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:23 smithi136 ceph-mon[113531]: pgmap v8544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3055498494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:23.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:23.843666+0000) 2023-08-28T01:38:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:23 smithi093 ceph-mon[111335]: pgmap v8544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3055498494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:23 smithi093 ceph-mon[118902]: pgmap v8544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3055498494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:24.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:24.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:24.842 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:24.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:24.842+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:24.843882+0000) 2023-08-28T01:38:25.669 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:38:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:38:25.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:25.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:25.844078+0000) 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: pgmap v8545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2418M 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:38:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:38:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3898271052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: pgmap v8545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2418M 2023-08-28T01:38:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3898271052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: pgmap v8545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2418M 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:38:25.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3898271052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:26.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:26.844284+0000) 2023-08-28T01:38:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:27 smithi136 ceph-mon[113531]: pgmap v8546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:27 smithi093 ceph-mon[111335]: pgmap v8546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:27 smithi093 ceph-mon[118902]: pgmap v8546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:27.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:27.844517+0000) 2023-08-28T01:38:28.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3226967044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3226967044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3226967044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:28.843+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:28.844766+0000) 2023-08-28T01:38:29.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:29 smithi136 ceph-mon[113531]: pgmap v8547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:29 smithi093 ceph-mon[111335]: pgmap v8547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:29 smithi093 ceph-mon[118902]: pgmap v8547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:29.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:29.845006+0000) 2023-08-28T01:38:30.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2433396423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2433396423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2433396423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:30.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:30.845247+0000) 2023-08-28T01:38:31.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:31 smithi136 ceph-mon[113531]: pgmap v8548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:31 smithi093 ceph-mon[111335]: pgmap v8548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:31 smithi093 ceph-mon[118902]: pgmap v8548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:31.349Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:31.350Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:32.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:31.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:31.845461+0000) 2023-08-28T01:38:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:32.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:32.845721+0000) 2023-08-28T01:38:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:32.792Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:32.793Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:33 smithi136 ceph-mon[113531]: pgmap v8549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:33.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3359289280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:33.844+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:33.845990+0000) 2023-08-28T01:38:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:33 smithi093 ceph-mon[111335]: pgmap v8549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3359289280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:33 smithi093 ceph-mon[118902]: pgmap v8549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3359289280' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:34.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:34.846229+0000) 2023-08-28T01:38:35.649 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:38:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:38:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:38:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:35 smithi136 ceph-mon[113531]: pgmap v8550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:35.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3988173483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:35.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:35.846438+0000) 2023-08-28T01:38:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:35 smithi093 ceph-mon[111335]: pgmap v8550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3988173483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:35 smithi093 ceph-mon[118902]: pgmap v8550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3988173483' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:36.845+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:36.846638+0000) 2023-08-28T01:38:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:37 smithi136 ceph-mon[113531]: pgmap v8551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:37 smithi093 ceph-mon[111335]: pgmap v8551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:37 smithi093 ceph-mon[118902]: pgmap v8551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:37.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:37.846889+0000) 2023-08-28T01:38:38.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3065136425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3065136425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3065136425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:38.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:38.847152+0000) 2023-08-28T01:38:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:39 smithi136 ceph-mon[113531]: pgmap v8552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:39 smithi093 ceph-mon[111335]: pgmap v8552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:39 smithi093 ceph-mon[118902]: pgmap v8552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:40.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:39.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:39.847381+0000) 2023-08-28T01:38:40.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2039841648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:40.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2039841648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2039841648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:40.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:40.847621+0000) 2023-08-28T01:38:41.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:41 smithi136 ceph-mon[113531]: pgmap v8553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:41 smithi093 ceph-mon[111335]: pgmap v8553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:41 smithi093 ceph-mon[118902]: pgmap v8553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:41.350Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:41.352Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:41.846+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:41.847792+0000) 2023-08-28T01:38:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/802317762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/802317762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:42.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/802317762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:43.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:42.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:42.848020+0000) 2023-08-28T01:38:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:42.793Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:42.794Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:43 smithi136 ceph-mon[113531]: pgmap v8554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:43 smithi093 ceph-mon[111335]: pgmap v8554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:43 smithi093 ceph-mon[118902]: pgmap v8554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:44.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:43.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:43.848203+0000) 2023-08-28T01:38:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3288404905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:44.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:44.848437+0000) 2023-08-28T01:38:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3288404905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3288404905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:45.649 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:38:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:38:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:38:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:45 smithi136 ceph-mon[113531]: pgmap v8555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:45.847+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:45.848627+0000) 2023-08-28T01:38:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:45 smithi093 ceph-mon[111335]: pgmap v8555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:45 smithi093 ceph-mon[118902]: pgmap v8555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:46.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:46.848881+0000) 2023-08-28T01:38:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:47 smithi136 ceph-mon[113531]: pgmap v8556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/367579461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:47 smithi093 ceph-mon[111335]: pgmap v8556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/367579461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:47 smithi093 ceph-mon[118902]: pgmap v8556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/367579461' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:47.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:47.849152+0000) 2023-08-28T01:38:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:48.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:48.849389+0000) 2023-08-28T01:38:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:49 smithi136 ceph-mon[113531]: pgmap v8557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1403057272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:49.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:49.849669+0000) 2023-08-28T01:38:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:49 smithi093 ceph-mon[111335]: pgmap v8557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1403057272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:49 smithi093 ceph-mon[118902]: pgmap v8557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1403057272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:50.848+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:50.849935+0000) 2023-08-28T01:38:51.653 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:51.351Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:51.653 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:51.352Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:51 smithi136 ceph-mon[113531]: pgmap v8558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:51.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:51.850156+0000) 2023-08-28T01:38:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:51 smithi093 ceph-mon[111335]: pgmap v8558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:51 smithi093 ceph-mon[118902]: pgmap v8558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:52.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1339292438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:52.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:52.850365+0000) 2023-08-28T01:38:52.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1339292438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:52.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1339292438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:52.794Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:52.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:38:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:38:52.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:38:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:53 smithi136 ceph-mon[113531]: pgmap v8559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:53.849+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:53.850558+0000) 2023-08-28T01:38:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:53 smithi093 ceph-mon[111335]: pgmap v8559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:53 smithi093 ceph-mon[118902]: pgmap v8559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4241716877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:54.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:54.850758+0000) 2023-08-28T01:38:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4241716877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:38:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4241716877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:55.666 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:38:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:38:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:38:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:55 smithi136 ceph-mon[113531]: pgmap v8560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:55.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:55.851028+0000) 2023-08-28T01:38:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:55 smithi093 ceph-mon[111335]: pgmap v8560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:55 smithi093 ceph-mon[118902]: pgmap v8560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:56.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:56.851250+0000) 2023-08-28T01:38:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/729401204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:57 smithi136 ceph-mon[113531]: pgmap v8561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/729401204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:57 smithi093 ceph-mon[111335]: pgmap v8561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/729401204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:57 smithi093 ceph-mon[118902]: pgmap v8561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:57.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:57.851483+0000) 2023-08-28T01:38:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:58.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:58.851753+0000) 2023-08-28T01:38:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:59 smithi136 ceph-mon[113531]: pgmap v8562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:38:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/543930368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:38:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:38:59.850+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:38:59.851965+0000) 2023-08-28T01:38:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:59 smithi093 ceph-mon[111335]: pgmap v8562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:38:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/543930368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:38:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:59 smithi093 ceph-mon[118902]: pgmap v8562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:38:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:38:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/543930368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:00.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:00.852221+0000) 2023-08-28T01:39:01.657 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:01.352Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:01.657 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:01.353Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:01 smithi136 ceph-mon[113531]: pgmap v8563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:01.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2231500703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:01.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:01.852474+0000) 2023-08-28T01:39:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:01 smithi093 ceph-mon[111335]: pgmap v8563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2231500703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:01 smithi093 ceph-mon[118902]: pgmap v8563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2231500703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:02.851+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:02.852760+0000) 2023-08-28T01:39:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:02.794Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:02.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:03.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:03 smithi136 ceph-mon[113531]: pgmap v8564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:03.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:03.852942+0000) 2023-08-28T01:39:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:03 smithi093 ceph-mon[111335]: pgmap v8564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:03 smithi093 ceph-mon[118902]: pgmap v8564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:04.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/469565902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:04.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:04.853132+0000) 2023-08-28T01:39:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/469565902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:04.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/469565902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:05.670 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:39:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:39:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:39:05.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:05 smithi136 ceph-mon[113531]: pgmap v8565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:05.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:05.853290+0000) 2023-08-28T01:39:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:05 smithi093 ceph-mon[111335]: pgmap v8565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:05 smithi093 ceph-mon[118902]: pgmap v8565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:06.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1337059632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:06.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:06.853512+0000) 2023-08-28T01:39:06.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1337059632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:06.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1337059632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:07 smithi136 ceph-mon[113531]: pgmap v8566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:07.852+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:07.853792+0000) 2023-08-28T01:39:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:07 smithi093 ceph-mon[111335]: pgmap v8566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:07 smithi093 ceph-mon[118902]: pgmap v8566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1978475817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:08.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:08.854062+0000) 2023-08-28T01:39:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1978475817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1978475817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:09 smithi136 ceph-mon[113531]: pgmap v8567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:09.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:09.854281+0000) 2023-08-28T01:39:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:09 smithi093 ceph-mon[111335]: pgmap v8567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:09 smithi093 ceph-mon[118902]: pgmap v8567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:09.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:11.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:10.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:10.854531+0000) 2023-08-28T01:39:11.686 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:11.353Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:11.687 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:11.355Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:11 smithi136 ceph-mon[113531]: pgmap v8568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:11.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4240969952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:11.853+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:11.854795+0000) 2023-08-28T01:39:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:11 smithi093 ceph-mon[118902]: pgmap v8568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4240969952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:11 smithi093 ceph-mon[111335]: pgmap v8568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4240969952' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:13.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:12.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:12.855097+0000) 2023-08-28T01:39:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:12.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:12.795Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:13 smithi093 ceph-mon[118902]: pgmap v8569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3943408682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:13 smithi093 ceph-mon[111335]: pgmap v8569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3943408682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:13 smithi136 ceph-mon[113531]: pgmap v8569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:14.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3943408682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:13.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:13.855388+0000) 2023-08-28T01:39:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:14.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:14.855654+0000) 2023-08-28T01:39:15.693 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:39:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:39:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:39:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:15 smithi093 ceph-mon[111335]: pgmap v8570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/542278308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:15 smithi093 ceph-mon[118902]: pgmap v8570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/542278308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:15 smithi136 ceph-mon[113531]: pgmap v8570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/542278308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:15.854+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:15.855904+0000) 2023-08-28T01:39:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:16.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:16.856113+0000) 2023-08-28T01:39:17.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:17 smithi136 ceph-mon[113531]: pgmap v8571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:17 smithi093 ceph-mon[111335]: pgmap v8571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:17 smithi093 ceph-mon[118902]: pgmap v8571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:18.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:17.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:17.856385+0000) 2023-08-28T01:39:18.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2067610653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2067610653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2067610653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:18.855+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:18.856626+0000) 2023-08-28T01:39:19.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:19 smithi136 ceph-mon[113531]: pgmap v8572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:19 smithi093 ceph-mon[111335]: pgmap v8572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:19 smithi093 ceph-mon[118902]: pgmap v8572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:19.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:19.856825+0000) 2023-08-28T01:39:20.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/345984206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/345984206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/345984206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:20.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:20.857093+0000) 2023-08-28T01:39:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:21 smithi136 ceph-mon[113531]: pgmap v8573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:21 smithi093 ceph-mon[111335]: pgmap v8573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:21 smithi093 ceph-mon[118902]: pgmap v8573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:21.355Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:21.356Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:21.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:21.857326+0000) 2023-08-28T01:39:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:22.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:22.857564+0000) 2023-08-28T01:39:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:22.795Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:22.796Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:23.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:23 smithi136 ceph-mon[113531]: pgmap v8574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:23.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3116347444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:23.856+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:23.857822+0000) 2023-08-28T01:39:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:23 smithi093 ceph-mon[111335]: pgmap v8574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3116347444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:23 smithi093 ceph-mon[118902]: pgmap v8574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3116347444' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:24.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:24.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:24.858103+0000) 2023-08-28T01:39:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:25.665 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:39:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:39:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:39:25.937 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:25.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:25.858263+0000) 2023-08-28T01:39:26.107 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:25 smithi136 ceph-mon[113531]: pgmap v8575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:26.107 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4064460080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:26.107 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:39:26.108 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:25 smithi093 ceph-mon[111335]: pgmap v8575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:26.108 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4064460080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:26.108 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:39:26.108 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:25 smithi093 ceph-mon[118902]: pgmap v8575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:26.109 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4064460080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:26.109 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:39:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:26.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:26.858421+0000) 2023-08-28T01:39:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:27 smithi136 ceph-mon[113531]: pgmap v8576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:27 smithi093 ceph-mon[111335]: pgmap v8576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:27 smithi093 ceph-mon[118902]: pgmap v8576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:27.857+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:27.858576+0000) 2023-08-28T01:39:28.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2174483618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:28.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:28.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2174483618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2174483618' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:28.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:28.858734+0000) 2023-08-28T01:39:29.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:39:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:39:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:29 smithi136 ceph-mon[113531]: pgmap v8577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:39:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:39:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[118902]: pgmap v8577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:39:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:39:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:39:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:29 smithi093 ceph-mon[111335]: pgmap v8577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:29.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:29.858978+0000) 2023-08-28T01:39:30.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/783715737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/783715737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/783715737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:30.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:30.859316+0000) 2023-08-28T01:39:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:31 smithi093 ceph-mon[111335]: pgmap v8578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:31 smithi093 ceph-mon[118902]: pgmap v8578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:31.356Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:31.357Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:31.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:31 smithi136 ceph-mon[113531]: pgmap v8578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:32.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:31.858+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:31.859557+0000) 2023-08-28T01:39:32.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/533157766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/533157766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/533157766' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:32.796Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:32.797Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:32.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:32.859830+0000) 2023-08-28T01:39:33.858 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:33 smithi136 ceph-mon[113531]: pgmap v8579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:33 smithi093 ceph-mon[111335]: pgmap v8579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:33 smithi093 ceph-mon[118902]: pgmap v8579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:33.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:33.860056+0000) 2023-08-28T01:39:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2980822789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2980822789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:35.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2980822789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:34.859+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:34.860256+0000) 2023-08-28T01:39:35.701 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:39:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:39:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:39:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:35 smithi093 ceph-mon[111335]: pgmap v8580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:35 smithi093 ceph-mon[118902]: pgmap v8580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:35 smithi136 ceph-mon[113531]: pgmap v8580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:35.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:35.860531+0000) 2023-08-28T01:39:37.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:36.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:36.860697+0000) 2023-08-28T01:39:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:37 smithi136 ceph-mon[113531]: pgmap v8581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/555159338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:37 smithi093 ceph-mon[111335]: pgmap v8581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/555159338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:37 smithi093 ceph-mon[118902]: pgmap v8581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/555159338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:37.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:37.860982+0000) 2023-08-28T01:39:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:38.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:38.861279+0000) 2023-08-28T01:39:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:39 smithi136 ceph-mon[113531]: pgmap v8582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2208756199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:39.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:39.861527+0000) 2023-08-28T01:39:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:39 smithi093 ceph-mon[111335]: pgmap v8582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2208756199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:39 smithi093 ceph-mon[118902]: pgmap v8582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2208756199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:40.860+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:40.861724+0000) 2023-08-28T01:39:41.665 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:41.356Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:41.666 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:41.357Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:41 smithi136 ceph-mon[113531]: pgmap v8583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:41.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:41.861963+0000) 2023-08-28T01:39:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:41 smithi093 ceph-mon[111335]: pgmap v8583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:41 smithi093 ceph-mon[118902]: pgmap v8583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:42.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3386538704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:42.861+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:42.862184+0000) 2023-08-28T01:39:42.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3386538704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3386538704' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:42.797Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:42.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:42.798Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:43 smithi136 ceph-mon[113531]: pgmap v8584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:43.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:43.862411+0000) 2023-08-28T01:39:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:43 smithi093 ceph-mon[111335]: pgmap v8584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:43 smithi093 ceph-mon[118902]: pgmap v8584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2243997184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:44.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:44.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:44.862596+0000) 2023-08-28T01:39:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2243997184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2243997184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:45.677 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:39:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:39:45] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:39:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:45 smithi136 ceph-mon[113531]: pgmap v8585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:45.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:45.862786+0000) 2023-08-28T01:39:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:45 smithi093 ceph-mon[111335]: pgmap v8585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:45 smithi093 ceph-mon[118902]: pgmap v8585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:46.862+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:46.862991+0000) 2023-08-28T01:39:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1480739811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:47 smithi136 ceph-mon[113531]: pgmap v8586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1480739811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:47 smithi093 ceph-mon[111335]: pgmap v8586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1480739811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:47 smithi093 ceph-mon[118902]: pgmap v8586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:47.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:47.863304+0000) 2023-08-28T01:39:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:48.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:48.863553+0000) 2023-08-28T01:39:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:49 smithi136 ceph-mon[113531]: pgmap v8587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3265533509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:49.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:49.863794+0000) 2023-08-28T01:39:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:49 smithi093 ceph-mon[111335]: pgmap v8587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3265533509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:49 smithi093 ceph-mon[118902]: pgmap v8587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3265533509' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:50.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:50.864012+0000) 2023-08-28T01:39:51.668 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:51.357Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:51.668 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:51.357Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:51 smithi136 ceph-mon[113531]: pgmap v8588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:51.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2560135878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:51.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:51.864295+0000) 2023-08-28T01:39:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:51 smithi093 ceph-mon[118902]: pgmap v8588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2560135878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:51 smithi093 ceph-mon[111335]: pgmap v8588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2560135878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:52.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:52.864587+0000) 2023-08-28T01:39:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:52.798Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:39:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:39:52.799Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:39:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:53 smithi136 ceph-mon[113531]: pgmap v8589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:53.863+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:53.864877+0000) 2023-08-28T01:39:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:53 smithi093 ceph-mon[118902]: pgmap v8589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:53 smithi093 ceph-mon[111335]: pgmap v8589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/229681247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:54.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:54.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:54.865153+0000) 2023-08-28T01:39:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/229681247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/229681247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:39:55.677 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:39:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:39:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:39:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:55 smithi136 ceph-mon[113531]: pgmap v8590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:55.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:55.865429+0000) 2023-08-28T01:39:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:55 smithi093 ceph-mon[111335]: pgmap v8590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:55 smithi093 ceph-mon[118902]: pgmap v8590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:56.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/436275883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:56.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:56.865705+0000) 2023-08-28T01:39:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/436275883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:56.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/436275883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:57 smithi136 ceph-mon[113531]: pgmap v8591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:57.864+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:57.865978+0000) 2023-08-28T01:39:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:57 smithi093 ceph-mon[111335]: pgmap v8591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:57 smithi093 ceph-mon[118902]: pgmap v8591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1492984857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:58.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:58.866279+0000) 2023-08-28T01:39:58.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1492984857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1492984857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:39:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:39:59 smithi136 ceph-mon[113531]: pgmap v8592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:39:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:39:59.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:39:59.866500+0000) 2023-08-28T01:39:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:39:59 smithi093 ceph-mon[111335]: pgmap v8592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:39:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:39:59 smithi093 ceph-mon[118902]: pgmap v8592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:40:00.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:40:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:40:01.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:00.865+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:00.866754+0000) 2023-08-28T01:40:01.694 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:01.358Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:01.694 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:01.359Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:01 smithi093 ceph-mon[111335]: pgmap v8593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/8545548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:01 smithi093 ceph-mon[118902]: pgmap v8593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:01.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/8545548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:01 smithi136 ceph-mon[113531]: pgmap v8593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/8545548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:01.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:01.866983+0000) 2023-08-28T01:40:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:02.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:02.867222+0000) 2023-08-28T01:40:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:02.799Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:02.800Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:03 smithi093 ceph-mon[118902]: pgmap v8594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2132270824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:03 smithi093 ceph-mon[111335]: pgmap v8594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2132270824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:03.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:03.867464+0000) 2023-08-28T01:40:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:03 smithi136 ceph-mon[113531]: pgmap v8594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2132270824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:05.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:04.866+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:04.867702+0000) 2023-08-28T01:40:05.701 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:40:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:40:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:40:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:05 smithi093 ceph-mon[111335]: pgmap v8595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/93414951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:05 smithi093 ceph-mon[118902]: pgmap v8595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/93414951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:05 smithi136 ceph-mon[113531]: pgmap v8595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/93414951' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:05.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:05.867954+0000) 2023-08-28T01:40:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:06.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:06.868140+0000) 2023-08-28T01:40:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:07 smithi136 ceph-mon[113531]: pgmap v8596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:07 smithi093 ceph-mon[111335]: pgmap v8596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:07 smithi093 ceph-mon[118902]: pgmap v8596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:07.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:07.868374+0000) 2023-08-28T01:40:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020040399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020040399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020040399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:08.867+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:08.868625+0000) 2023-08-28T01:40:09.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:09 smithi136 ceph-mon[113531]: pgmap v8597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:09.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:09 smithi093 ceph-mon[111335]: pgmap v8597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:09 smithi093 ceph-mon[118902]: pgmap v8597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:09.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:09.868866+0000) 2023-08-28T01:40:10.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2402334386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2402334386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2402334386' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:10.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:10.869130+0000) 2023-08-28T01:40:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:11 smithi136 ceph-mon[113531]: pgmap v8598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:11 smithi093 ceph-mon[111335]: pgmap v8598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:11 smithi093 ceph-mon[118902]: pgmap v8598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:11.358Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:11.359Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:12.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:11.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:11.869449+0000) 2023-08-28T01:40:13.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:12.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:12.869679+0000) 2023-08-28T01:40:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:12.799Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:12.800Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:13 smithi136 ceph-mon[113531]: pgmap v8599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/276860648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:13.868+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:13.869943+0000) 2023-08-28T01:40:13.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:13 smithi093 ceph-mon[111335]: pgmap v8599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/276860648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:13 smithi093 ceph-mon[118902]: pgmap v8599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/276860648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:14.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:14.870160+0000) 2023-08-28T01:40:15.674 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:40:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:40:15] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:40:15.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:15 smithi136 ceph-mon[113531]: pgmap v8600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:15.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2625653194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:15.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:15.870411+0000) 2023-08-28T01:40:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:15 smithi093 ceph-mon[111335]: pgmap v8600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2625653194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:15 smithi093 ceph-mon[118902]: pgmap v8600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2625653194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:16.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:16.870676+0000) 2023-08-28T01:40:17.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:17 smithi136 ceph-mon[113531]: pgmap v8601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:17 smithi093 ceph-mon[111335]: pgmap v8601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:17 smithi093 ceph-mon[118902]: pgmap v8601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:18.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:17.869+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:17.870965+0000) 2023-08-28T01:40:18.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1640837365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1640837365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1640837365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:19.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:18.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:18.871244+0000) 2023-08-28T01:40:19.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:19 smithi136 ceph-mon[113531]: pgmap v8602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:19 smithi093 ceph-mon[111335]: pgmap v8602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:19 smithi093 ceph-mon[118902]: pgmap v8602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:19.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:19.871524+0000) 2023-08-28T01:40:20.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/668782341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/668782341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/668782341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:20.870+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:20.871776+0000) 2023-08-28T01:40:21.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:21 smithi136 ceph-mon[113531]: pgmap v8603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:21 smithi093 ceph-mon[111335]: pgmap v8603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:21 smithi093 ceph-mon[118902]: pgmap v8603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:21.359Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:21.360Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:21.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:21.871994+0000) 2023-08-28T01:40:22.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1881403619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1881403619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:22.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1881403619' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:22.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:22.872181+0000) 2023-08-28T01:40:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:22.800Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:22.801Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:23.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:23 smithi136 ceph-mon[113531]: pgmap v8604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:23 smithi093 ceph-mon[111335]: pgmap v8604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:23 smithi093 ceph-mon[118902]: pgmap v8604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:24.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:23.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:23.872421+0000) 2023-08-28T01:40:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:25.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:24.871+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:24.872686+0000) 2023-08-28T01:40:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:25 smithi093 ceph-mon[111335]: pgmap v8605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3127452616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:40:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:40:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:40:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:25 smithi093 ceph-mon[118902]: pgmap v8605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3127452616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:25.871 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:25 smithi136 ceph-mon[113531]: pgmap v8605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:25.871 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3127452616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:25.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:25.872964+0000) 2023-08-28T01:40:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:26.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:26.873277+0000) 2023-08-28T01:40:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:27 smithi136 ceph-mon[113531]: pgmap v8606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4100082486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:27 smithi093 ceph-mon[111335]: pgmap v8606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4100082486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:27 smithi093 ceph-mon[118902]: pgmap v8606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4100082486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:27.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:27.873553+0000) 2023-08-28T01:40:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:28.872+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:28.873739+0000) 2023-08-28T01:40:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:29 smithi136 ceph-mon[113531]: pgmap v8607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:40:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2907134271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:29.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:29.873874+0000) 2023-08-28T01:40:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:29 smithi093 ceph-mon[111335]: pgmap v8607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:40:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2907134271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:29 smithi093 ceph-mon[118902]: pgmap v8607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:40:29.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2907134271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:30.873+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:30.874144+0000) 2023-08-28T01:40:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[118902]: pgmap v8608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[111335]: pgmap v8608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:31.360Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:31.362Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:31.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:31 smithi136 ceph-mon[113531]: pgmap v8608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:31.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:31.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:32.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:31.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:31.874398+0000) 2023-08-28T01:40:32.799 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:40:32.799 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:40:32.799 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:32.799 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2269717103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:32.800 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:40:32.800 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:40:32.800 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:32.800 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2269717103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:32.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:40:32.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:40:32.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:40:32.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2269717103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:32.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:32.874644+0000) 2023-08-28T01:40:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:32.800Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:32.801Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:33.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:33 smithi136 ceph-mon[113531]: pgmap v8609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:33 smithi093 ceph-mon[111335]: pgmap v8609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:33 smithi093 ceph-mon[118902]: pgmap v8609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:33.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:33.874880+0000) 2023-08-28T01:40:34.873 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2821855864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2821855864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2821855864' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:34.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:34.875135+0000) 2023-08-28T01:40:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:35 smithi093 ceph-mon[111335]: pgmap v8610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:40:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:40:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:40:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:35 smithi093 ceph-mon[118902]: pgmap v8610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:35.874 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:35 smithi136 ceph-mon[113531]: pgmap v8610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:35.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:35.875389+0000) 2023-08-28T01:40:37.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:36.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:36.875638+0000) 2023-08-28T01:40:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2087659949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:37.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:37 smithi136 ceph-mon[113531]: pgmap v8611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2087659949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:37 smithi093 ceph-mon[111335]: pgmap v8611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2087659949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:37 smithi093 ceph-mon[118902]: pgmap v8611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:37.874+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:37.875891+0000) 2023-08-28T01:40:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:38.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:38.876165+0000) 2023-08-28T01:40:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:39 smithi136 ceph-mon[113531]: pgmap v8612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/244821929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:39.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:39.876358+0000) 2023-08-28T01:40:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:39 smithi093 ceph-mon[111335]: pgmap v8612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/244821929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:39 smithi093 ceph-mon[118902]: pgmap v8612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/244821929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:40.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:40.876624+0000) 2023-08-28T01:40:41.681 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:41.362Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:41.681 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:41.363Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:41 smithi136 ceph-mon[113531]: pgmap v8613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3802462404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:41.875+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:41.876885+0000) 2023-08-28T01:40:41.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:41 smithi093 ceph-mon[111335]: pgmap v8613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3802462404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:41 smithi093 ceph-mon[118902]: pgmap v8613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3802462404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:42.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:42.877174+0000) 2023-08-28T01:40:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:42.801Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:42.802Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:43 smithi136 ceph-mon[113531]: pgmap v8614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:43.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:43.877435+0000) 2023-08-28T01:40:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:43 smithi093 ceph-mon[111335]: pgmap v8614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:43 smithi093 ceph-mon[118902]: pgmap v8614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/412326834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/412326834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:45.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/412326834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:45.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:44.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:44.877684+0000) 2023-08-28T01:40:45.690 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:40:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:40:45] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:40:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:45 smithi093 ceph-mon[111335]: pgmap v8615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:45.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:45 smithi093 ceph-mon[118902]: pgmap v8615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:45 smithi136 ceph-mon[113531]: pgmap v8615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:45.876+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:45.877991+0000) 2023-08-28T01:40:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2500350832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:46.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2500350832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:47.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2500350832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:46.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:46.878243+0000) 2023-08-28T01:40:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:47 smithi093 ceph-mon[111335]: pgmap v8616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:47 smithi093 ceph-mon[118902]: pgmap v8616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:47 smithi136 ceph-mon[113531]: pgmap v8616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:47.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:47.878563+0000) 2023-08-28T01:40:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1656063922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1656063922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:49.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1656063922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:48.877+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:48.878797+0000) 2023-08-28T01:40:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:49 smithi093 ceph-mon[118902]: pgmap v8617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:49 smithi093 ceph-mon[111335]: pgmap v8617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:49 smithi136 ceph-mon[113531]: pgmap v8617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:49.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:49.879070+0000) 2023-08-28T01:40:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:50.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:50.879306+0000) 2023-08-28T01:40:51.702 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:51.363Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:51.702 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:51.364Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:51 smithi093 ceph-mon[111335]: pgmap v8618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3216032364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:51 smithi093 ceph-mon[118902]: pgmap v8618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3216032364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:51 smithi136 ceph-mon[113531]: pgmap v8618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3216032364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:51.878+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:51.879534+0000) 2023-08-28T01:40:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:52.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:52.879795+0000) 2023-08-28T01:40:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:52.801Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:40:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:40:52.802Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:40:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:53 smithi093 ceph-mon[111335]: pgmap v8619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/542344868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:53 smithi093 ceph-mon[118902]: pgmap v8619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/542344868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:53 smithi136 ceph-mon[113531]: pgmap v8619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/542344868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:53.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:53.880075+0000) 2023-08-28T01:40:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:40:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:54.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:54.880308+0000) 2023-08-28T01:40:55.711 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:40:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:40:55] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:40:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:55 smithi093 ceph-mon[111335]: pgmap v8620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1237264261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:55 smithi093 ceph-mon[118902]: pgmap v8620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1237264261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:55 smithi136 ceph-mon[113531]: pgmap v8620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1237264261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:55.879+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:55.880511+0000) 2023-08-28T01:40:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:56.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:56.880765+0000) 2023-08-28T01:40:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:57 smithi136 ceph-mon[113531]: pgmap v8621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:57 smithi093 ceph-mon[111335]: pgmap v8621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:57 smithi093 ceph-mon[118902]: pgmap v8621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:57.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:57.880997+0000) 2023-08-28T01:40:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4268665929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4268665929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4268665929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:40:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:58.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:58.881163+0000) 2023-08-28T01:40:59.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:40:59 smithi136 ceph-mon[113531]: pgmap v8622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:40:59 smithi093 ceph-mon[111335]: pgmap v8622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:40:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:40:59 smithi093 ceph-mon[118902]: pgmap v8622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:00.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:40:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:40:59.880+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:40:59.881406+0000) 2023-08-28T01:41:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3835727458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3835727458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3835727458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:00.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:00.881659+0000) 2023-08-28T01:41:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:01 smithi093 ceph-mon[111335]: pgmap v8623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:01 smithi093 ceph-mon[118902]: pgmap v8623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:01.363Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:01.364Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:01.880 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:01 smithi136 ceph-mon[113531]: pgmap v8623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:02.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:01.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:01.881902+0000) 2023-08-28T01:41:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:02.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:02.882168+0000) 2023-08-28T01:41:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:02.802Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:02.803Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:03 smithi136 ceph-mon[113531]: pgmap v8624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:03.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1449519890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:03.881+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:03.882414+0000) 2023-08-28T01:41:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:03 smithi093 ceph-mon[111335]: pgmap v8624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1449519890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:03 smithi093 ceph-mon[118902]: pgmap v8624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1449519890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:05.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:04.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:04.882633+0000) 2023-08-28T01:41:05.687 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:41:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:41:05] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:41:05.687 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 01:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T01:41:05.317416918Z level=info msg="Completed cleanup jobs" duration=12.399619ms 2023-08-28T01:41:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:05 smithi093 ceph-mon[111335]: pgmap v8625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3237355919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:05 smithi093 ceph-mon[118902]: pgmap v8625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3237355919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:05 smithi136 ceph-mon[113531]: pgmap v8625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3237355919' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:05.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:05.882885+0000) 2023-08-28T01:41:07.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:06.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:06.883100+0000) 2023-08-28T01:41:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:07 smithi136 ceph-mon[113531]: pgmap v8626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:07 smithi093 ceph-mon[111335]: pgmap v8626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:07 smithi093 ceph-mon[118902]: pgmap v8626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:07.882+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:07.883313+0000) 2023-08-28T01:41:08.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3738925778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3738925778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3738925778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:09.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:08.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:08.883584+0000) 2023-08-28T01:41:09.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:09 smithi136 ceph-mon[113531]: pgmap v8627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:09 smithi093 ceph-mon[118902]: pgmap v8627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:09 smithi093 ceph-mon[111335]: pgmap v8627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:09.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:09.883804+0000) 2023-08-28T01:41:10.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2272250049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2272250049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2272250049' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:10.883+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:10.884083+0000) 2023-08-28T01:41:11.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:11 smithi136 ceph-mon[113531]: pgmap v8628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:11 smithi093 ceph-mon[111335]: pgmap v8628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:11 smithi093 ceph-mon[118902]: pgmap v8628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:11.364Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:11.365Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:12.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:11.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:11.884375+0000) 2023-08-28T01:41:12.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1354375410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:12.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1354375410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:12.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1354375410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:13.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:12.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:12.884649+0000) 2023-08-28T01:41:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:12.802Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:12.803Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:13 smithi093 ceph-mon[111335]: pgmap v8629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:13 smithi093 ceph-mon[118902]: pgmap v8629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:13.883 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:13 smithi136 ceph-mon[113531]: pgmap v8629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:13.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:13.884906+0000) 2023-08-28T01:41:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:14.884+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:14.885133+0000) 2023-08-28T01:41:15.687 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:41:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:41:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:41:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:15 smithi093 ceph-mon[111335]: pgmap v8630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/579019094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:15 smithi093 ceph-mon[118902]: pgmap v8630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/579019094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:16.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:15.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:15.885412+0000) 2023-08-28T01:41:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:15 smithi136 ceph-mon[113531]: pgmap v8630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/579019094' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:16.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:16.885692+0000) 2023-08-28T01:41:17.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:17 smithi136 ceph-mon[113531]: pgmap v8631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1645743313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:17 smithi093 ceph-mon[118902]: pgmap v8631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1645743313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:17 smithi093 ceph-mon[111335]: pgmap v8631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1645743313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:18.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:17.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:17.885923+0000) 2023-08-28T01:41:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:18.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:18.886201+0000) 2023-08-28T01:41:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:19 smithi093 ceph-mon[111335]: pgmap v8632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2015008237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:19 smithi093 ceph-mon[118902]: pgmap v8632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2015008237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:19 smithi136 ceph-mon[113531]: pgmap v8632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2015008237' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:19.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:19.886440+0000) 2023-08-28T01:41:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:20.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:20.886648+0000) 2023-08-28T01:41:21.693 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:21.365Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:21.694 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:21.366Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:21 smithi093 ceph-mon[111335]: pgmap v8633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:21 smithi093 ceph-mon[118902]: pgmap v8633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:21 smithi136 ceph-mon[113531]: pgmap v8633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:21.885+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:21.886904+0000) 2023-08-28T01:41:22.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2181642470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:22.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2181642470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:22.803Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:22.804Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:23.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2181642470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:22.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:22.887100+0000) 2023-08-28T01:41:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:23 smithi093 ceph-mon[111335]: pgmap v8634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:23 smithi093 ceph-mon[118902]: pgmap v8634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:23 smithi136 ceph-mon[113531]: pgmap v8634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:23.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:23.887380+0000) 2023-08-28T01:41:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1580546594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1580546594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1580546594' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:24.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:24.887601+0000) 2023-08-28T01:41:25.712 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:41:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:41:25] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:41:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:25 smithi093 ceph-mon[111335]: pgmap v8635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:25.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:25 smithi093 ceph-mon[118902]: pgmap v8635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:25 smithi136 ceph-mon[113531]: pgmap v8635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:25.886+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:25.887873+0000) 2023-08-28T01:41:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:26.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:26.888156+0000) 2023-08-28T01:41:27.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2793595085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:27 smithi136 ceph-mon[113531]: pgmap v8636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2793595085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:27 smithi093 ceph-mon[111335]: pgmap v8636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2793595085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:27 smithi093 ceph-mon[118902]: pgmap v8636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:27.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:27.888429+0000) 2023-08-28T01:41:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:28.887+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:28.888663+0000) 2023-08-28T01:41:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:29 smithi093 ceph-mon[111335]: pgmap v8637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1160202978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:29 smithi093 ceph-mon[118902]: pgmap v8637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1160202978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:29 smithi136 ceph-mon[113531]: pgmap v8637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1160202978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:29.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:29.888950+0000) 2023-08-28T01:41:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:30.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:30.889223+0000) 2023-08-28T01:41:31.694 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:31.366Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:31.694 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:31.367Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:31 smithi093 ceph-mon[111335]: pgmap v8638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3581129640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:31 smithi093 ceph-mon[118902]: pgmap v8638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3581129640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:31 smithi136 ceph-mon[113531]: pgmap v8638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3581129640' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:31.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:31.889409+0000) 2023-08-28T01:41:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:41:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:41:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:32.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:32.803Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:32.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:32.804Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:33.186 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:41:33.186 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:33.186 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:33.186 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:32.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:32.889585+0000) 2023-08-28T01:41:33.908 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:33 smithi136 ceph-mon[113531]: pgmap v8639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:33.908 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:33.888+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:33.889751+0000) 2023-08-28T01:41:33.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:33 smithi093 ceph-mon[111335]: pgmap v8639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:33 smithi093 ceph-mon[118902]: pgmap v8639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:34.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:34.889964+0000) 2023-08-28T01:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/946026652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:35.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:41:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/946026652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:41:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/946026652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:41:35.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:41:35.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:41:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:41:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:41:35] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:41:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:35 smithi136 ceph-mon[113531]: pgmap v8640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:35.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:35.890220+0000) 2023-08-28T01:41:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:35 smithi093 ceph-mon[111335]: pgmap v8640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:35 smithi093 ceph-mon[118902]: pgmap v8640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:37.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4173815025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:36.889+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:36.890496+0000) 2023-08-28T01:41:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4173815025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4173815025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:37 smithi136 ceph-mon[113531]: pgmap v8641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:37.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:37.890766+0000) 2023-08-28T01:41:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:37 smithi093 ceph-mon[111335]: pgmap v8641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:37 smithi093 ceph-mon[118902]: pgmap v8641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3558389133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:38.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:38.890949+0000) 2023-08-28T01:41:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3558389133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3558389133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:40.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:39.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:39.891160+0000) 2023-08-28T01:41:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:39 smithi136 ceph-mon[113531]: pgmap v8642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:39 smithi093 ceph-mon[118902]: pgmap v8642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:39 smithi093 ceph-mon[111335]: pgmap v8642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:40.890+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:40.891395+0000) 2023-08-28T01:41:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:41.366Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:41.367Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:41 smithi136 ceph-mon[113531]: pgmap v8643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4194455416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:41.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:41.891675+0000) 2023-08-28T01:41:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:41 smithi093 ceph-mon[111335]: pgmap v8643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4194455416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:41 smithi093 ceph-mon[118902]: pgmap v8643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4194455416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:42.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:42.891951+0000) 2023-08-28T01:41:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:42.804Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:42.805Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:43 smithi136 ceph-mon[113531]: pgmap v8644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2522847834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:43.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:43.892122+0000) 2023-08-28T01:41:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:43 smithi093 ceph-mon[111335]: pgmap v8644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2522847834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:43 smithi093 ceph-mon[118902]: pgmap v8644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2522847834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:44.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:44.892367+0000) 2023-08-28T01:41:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:41:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:41:45] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:41:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:45 smithi136 ceph-mon[113531]: pgmap v8645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2284039226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:45.891+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:45.892589+0000) 2023-08-28T01:41:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:45 smithi093 ceph-mon[111335]: pgmap v8645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2284039226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:45 smithi093 ceph-mon[118902]: pgmap v8645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2284039226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:46.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:46.892833+0000) 2023-08-28T01:41:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:47 smithi136 ceph-mon[113531]: pgmap v8646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:47 smithi093 ceph-mon[111335]: pgmap v8646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:47 smithi093 ceph-mon[118902]: pgmap v8646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:47.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:47.893111+0000) 2023-08-28T01:41:48.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3452391679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3452391679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3452391679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:48.892+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:48.893353+0000) 2023-08-28T01:41:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:49 smithi093 ceph-mon[111335]: pgmap v8647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:49 smithi093 ceph-mon[118902]: pgmap v8647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:49.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:49 smithi136 ceph-mon[113531]: pgmap v8647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:50.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:49.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:49.893598+0000) 2023-08-28T01:41:50.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1316375767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1316375767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1316375767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:50.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:50.893859+0000) 2023-08-28T01:41:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:51 smithi093 ceph-mon[111335]: pgmap v8648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:51 smithi093 ceph-mon[118902]: pgmap v8648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:51.367Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:51.368Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:51.892 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:51 smithi136 ceph-mon[113531]: pgmap v8648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:52.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:51.893+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:51.894128+0000) 2023-08-28T01:41:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:52.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:52.894386+0000) 2023-08-28T01:41:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:52.805Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:41:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:41:52.806Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:41:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:53 smithi093 ceph-mon[111335]: pgmap v8649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3899550913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:53 smithi093 ceph-mon[118902]: pgmap v8649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3899550913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:53 smithi136 ceph-mon[113531]: pgmap v8649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3899550913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:53.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:53.894667+0000) 2023-08-28T01:41:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:41:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:54.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:54.894914+0000) 2023-08-28T01:41:55.703 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:41:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:41:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:41:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:55 smithi093 ceph-mon[118902]: pgmap v8650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3285461626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:55 smithi093 ceph-mon[111335]: pgmap v8650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:55.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3285461626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:55 smithi136 ceph-mon[113531]: pgmap v8650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3285461626' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:55.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:55.895157+0000) 2023-08-28T01:41:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:56.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:56.895405+0000) 2023-08-28T01:41:57.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:57 smithi136 ceph-mon[113531]: pgmap v8651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:57 smithi093 ceph-mon[111335]: pgmap v8651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:57 smithi093 ceph-mon[118902]: pgmap v8651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:57.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:57.895675+0000) 2023-08-28T01:41:58.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/510708407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/510708407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/510708407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:41:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:58.894+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:58.895948+0000) 2023-08-28T01:41:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:41:59 smithi093 ceph-mon[111335]: pgmap v8652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:41:59 smithi093 ceph-mon[118902]: pgmap v8652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:41:59.894 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:41:59 smithi136 ceph-mon[113531]: pgmap v8652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:00.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:41:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:41:59.895+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:41:59.896177+0000) 2023-08-28T01:42:00.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/921005614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:00.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/921005614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/921005614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:00.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:00.896420+0000) 2023-08-28T01:42:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:01 smithi093 ceph-mon[111335]: pgmap v8653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:01 smithi093 ceph-mon[118902]: pgmap v8653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:01.367Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:01.368Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:01.895 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:01 smithi136 ceph-mon[113531]: pgmap v8653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:02.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:01.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:01.896693+0000) 2023-08-28T01:42:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/396326863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/396326863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:02.895 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/396326863' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:02.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:02.896963+0000) 2023-08-28T01:42:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:02.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:02.806Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:03 smithi093 ceph-mon[118902]: pgmap v8654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:03 smithi093 ceph-mon[111335]: pgmap v8654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:03.895 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:03 smithi136 ceph-mon[113531]: pgmap v8654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:04.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:03.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:03.897215+0000) 2023-08-28T01:42:05.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:04.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:04.897470+0000) 2023-08-28T01:42:05.699 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:42:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:42:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:42:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:05 smithi093 ceph-mon[111335]: pgmap v8655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1470720591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:05 smithi093 ceph-mon[118902]: pgmap v8655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1470720591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:05 smithi136 ceph-mon[113531]: pgmap v8655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1470720591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:05.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:05.897693+0000) 2023-08-28T01:42:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:06.896+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:06.897848+0000) 2023-08-28T01:42:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:07 smithi136 ceph-mon[113531]: pgmap v8656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:07.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3035659531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:07 smithi093 ceph-mon[111335]: pgmap v8656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3035659531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:07 smithi093 ceph-mon[118902]: pgmap v8656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3035659531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:07.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:07.898130+0000) 2023-08-28T01:42:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:08.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:08.898405+0000) 2023-08-28T01:42:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:09 smithi093 ceph-mon[111335]: pgmap v8657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1532296796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:09 smithi093 ceph-mon[118902]: pgmap v8657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1532296796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:09 smithi136 ceph-mon[113531]: pgmap v8657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1532296796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:09.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:09.898658+0000) 2023-08-28T01:42:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:10.897+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:10.898907+0000) 2023-08-28T01:42:11.705 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:11.368Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:11.705 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:11.369Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:11 smithi093 ceph-mon[111335]: pgmap v8658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:11 smithi093 ceph-mon[118902]: pgmap v8658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:11 smithi136 ceph-mon[113531]: pgmap v8658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:11.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:11.899183+0000) 2023-08-28T01:42:12.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3616246968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:12.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3616246968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:12.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:12.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:13.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3616246968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:12.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:12.899471+0000) 2023-08-28T01:42:13.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:13 smithi093 ceph-mon[111335]: pgmap v8659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:13 smithi093 ceph-mon[118902]: pgmap v8659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:13 smithi136 ceph-mon[113531]: pgmap v8659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:13.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:13.899704+0000) 2023-08-28T01:42:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1355265944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1355265944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:15.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1355265944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:14.898+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:14.899956+0000) 2023-08-28T01:42:15.719 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:42:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:42:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:42:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:15 smithi093 ceph-mon[111335]: pgmap v8660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:15 smithi093 ceph-mon[118902]: pgmap v8660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:15 smithi136 ceph-mon[113531]: pgmap v8660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:15.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:15.900192+0000) 2023-08-28T01:42:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:16.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:16.900446+0000) 2023-08-28T01:42:17.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:17 smithi136 ceph-mon[113531]: pgmap v8661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3843253510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:17 smithi093 ceph-mon[111335]: pgmap v8661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3843253510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:17 smithi093 ceph-mon[118902]: pgmap v8661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3843253510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:18.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:17.899+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:17.900740+0000) 2023-08-28T01:42:19.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:18.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:18.900987+0000) 2023-08-28T01:42:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:19 smithi093 ceph-mon[111335]: pgmap v8662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3972788749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:19 smithi093 ceph-mon[118902]: pgmap v8662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3972788749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:19 smithi136 ceph-mon[113531]: pgmap v8662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3972788749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:19.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:19.901187+0000) 2023-08-28T01:42:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:20.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:20.901445+0000) 2023-08-28T01:42:21.706 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:21.368Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:21.706 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:21.369Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:21 smithi093 ceph-mon[111335]: pgmap v8663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2693406025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:21 smithi093 ceph-mon[118902]: pgmap v8663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2693406025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:21 smithi136 ceph-mon[113531]: pgmap v8663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2693406025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:21.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:21.901697+0000) 2023-08-28T01:42:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:22.900+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:22.901929+0000) 2023-08-28T01:42:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:22.806Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:22.807Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:23.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:23 smithi093 ceph-mon[111335]: pgmap v8664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:23 smithi093 ceph-mon[118902]: pgmap v8664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:23 smithi136 ceph-mon[113531]: pgmap v8664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:23.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:23.902201+0000) 2023-08-28T01:42:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/638815794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:24.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/638815794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/638815794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:24.901+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:24.902445+0000) 2023-08-28T01:42:25.715 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:42:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:42:25] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:42:25.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:25 smithi093 ceph-mon[111335]: pgmap v8665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:25.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:25 smithi093 ceph-mon[118902]: pgmap v8665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:25 smithi136 ceph-mon[113531]: pgmap v8665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:25.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:25.902685+0000) 2023-08-28T01:42:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3151210523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3151210523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:27.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3151210523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:26.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:26.902946+0000) 2023-08-28T01:42:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:27 smithi093 ceph-mon[111335]: pgmap v8666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:27 smithi093 ceph-mon[118902]: pgmap v8666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:27 smithi136 ceph-mon[113531]: pgmap v8666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:27.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:27.903152+0000) 2023-08-28T01:42:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/968065144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/968065144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/968065144' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:28.902+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:28.903424+0000) 2023-08-28T01:42:29.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:29 smithi093 ceph-mon[111335]: pgmap v8667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:29.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:29 smithi093 ceph-mon[118902]: pgmap v8667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:29 smithi136 ceph-mon[113531]: pgmap v8667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:29.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:29.903692+0000) 2023-08-28T01:42:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:30.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:30.903959+0000) 2023-08-28T01:42:31.726 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:31.368Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:31.726 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:31.369Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:31 smithi093 ceph-mon[111335]: pgmap v8668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:31.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2868208471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:31 smithi093 ceph-mon[118902]: pgmap v8668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:31.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2868208471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:31 smithi136 ceph-mon[113531]: pgmap v8668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2868208471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:31.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:31.904205+0000) 2023-08-28T01:42:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:32.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:32.904448+0000) 2023-08-28T01:42:33.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:32.808Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:32.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:33 smithi093 ceph-mon[111335]: pgmap v8669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/461439955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:33 smithi093 ceph-mon[118902]: pgmap v8669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/461439955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:33 smithi136 ceph-mon[113531]: pgmap v8669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/461439955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:33.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:33.904701+0000) 2023-08-28T01:42:34.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:42:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:42:35.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:42:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:34.903+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:34.904900+0000) 2023-08-28T01:42:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:35 smithi093 ceph-mon[111335]: pgmap v8670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:42:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:42:35] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:42:35.903 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:35 smithi136 ceph-mon[113531]: pgmap v8670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:35.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:35.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:35.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:35 smithi093 ceph-mon[118902]: pgmap v8670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:35.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:35.905065+0000) 2023-08-28T01:42:36.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4272646669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:36.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:36.904 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:37.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:36.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:36.905237+0000) 2023-08-28T01:42:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4272646669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4272646669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:37 smithi136 ceph-mon[113531]: pgmap v8671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:37.904+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:37.905449+0000) 2023-08-28T01:42:38.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:37 smithi093 ceph-mon[111335]: pgmap v8671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:37 smithi093 ceph-mon[118902]: pgmap v8671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:38.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:38.905663+0000) 2023-08-28T01:42:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:42:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:42:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/707958669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:42:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:42:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/707958669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:42:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:42:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:42:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/707958669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:40.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:39.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:39.905906+0000) 2023-08-28T01:42:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:39 smithi136 ceph-mon[113531]: pgmap v8672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:39 smithi093 ceph-mon[111335]: pgmap v8672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:39 smithi093 ceph-mon[118902]: pgmap v8672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:41.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3042592458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:40.905+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:40.906127+0000) 2023-08-28T01:42:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3042592458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3042592458' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:41.368Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:41.369Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:41 smithi136 ceph-mon[113531]: pgmap v8673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:41.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:41.906356+0000) 2023-08-28T01:42:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:41 smithi093 ceph-mon[111335]: pgmap v8673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:41 smithi093 ceph-mon[118902]: pgmap v8673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:42.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:42.906618+0000) 2023-08-28T01:42:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:42.809Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:42.809Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:43 smithi136 ceph-mon[113531]: pgmap v8674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3550874651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:43.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:43.906865+0000) 2023-08-28T01:42:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:43 smithi093 ceph-mon[111335]: pgmap v8674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3550874651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:43 smithi093 ceph-mon[118902]: pgmap v8674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3550874651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:44.906+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:44.907136+0000) 2023-08-28T01:42:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:42:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:42:45] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:42:46.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:45.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:45.907374+0000) 2023-08-28T01:42:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:45 smithi136 ceph-mon[113531]: pgmap v8675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2256091368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:45 smithi093 ceph-mon[111335]: pgmap v8675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2256091368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:45 smithi093 ceph-mon[118902]: pgmap v8675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2256091368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:46.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:46.907594+0000) 2023-08-28T01:42:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:47 smithi093 ceph-mon[111335]: pgmap v8676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:47 smithi093 ceph-mon[118902]: pgmap v8676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:47.906 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:47 smithi136 ceph-mon[113531]: pgmap v8676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:47.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:47.907853+0000) 2023-08-28T01:42:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/887824422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/887824422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:48.906 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/887824422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:48.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:48.908129+0000) 2023-08-28T01:42:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:49 smithi093 ceph-mon[111335]: pgmap v8677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:49 smithi093 ceph-mon[118902]: pgmap v8677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:49.906 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:49 smithi136 ceph-mon[113531]: pgmap v8677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:50.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:49.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:49.908382+0000) 2023-08-28T01:42:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3294428304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3294428304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:50.907 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3294428304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:50.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:50.908652+0000) 2023-08-28T01:42:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:51 smithi093 ceph-mon[111335]: pgmap v8678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:51 smithi093 ceph-mon[118902]: pgmap v8678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:51.370Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:51.371Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:51.907 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:51 smithi136 ceph-mon[113531]: pgmap v8678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:52.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:51.907+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:51.908899+0000) 2023-08-28T01:42:52.908 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3140645524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3140645524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3140645524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:52.809Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:42:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:42:52.810Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:42:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:52.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:52.909098+0000) 2023-08-28T01:42:53.907 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:53 smithi136 ceph-mon[113531]: pgmap v8679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:53 smithi093 ceph-mon[111335]: pgmap v8679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:53 smithi093 ceph-mon[118902]: pgmap v8679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:54.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:53.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:53.909361+0000) 2023-08-28T01:42:54.908 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:42:55.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:54.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:54.909613+0000) 2023-08-28T01:42:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:55 smithi093 ceph-mon[111335]: pgmap v8680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4266599219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:42:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:42:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:42:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:55 smithi093 ceph-mon[118902]: pgmap v8680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4266599219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:55.908 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:55 smithi136 ceph-mon[113531]: pgmap v8680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:55.908 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4266599219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:56.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:55.908+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:55.909867+0000) 2023-08-28T01:42:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:56.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:56.910141+0000) 2023-08-28T01:42:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:57 smithi093 ceph-mon[111335]: pgmap v8681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3247110416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:57 smithi093 ceph-mon[118902]: pgmap v8681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3247110416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:57.908 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:57 smithi136 ceph-mon[113531]: pgmap v8681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:57.909 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3247110416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:57.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:57.910401+0000) 2023-08-28T01:42:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:58.909+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:58.910658+0000) 2023-08-28T01:42:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:59 smithi093 ceph-mon[111335]: pgmap v8682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:42:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/845594062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:42:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:59 smithi093 ceph-mon[118902]: pgmap v8682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:42:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:42:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/845594062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:59 smithi136 ceph-mon[113531]: pgmap v8682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:42:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/845594062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:42:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:42:59.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:42:59.910920+0000) 2023-08-28T01:43:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:00.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:00.911176+0000) 2023-08-28T01:43:01.718 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:01.370Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:01.718 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:01.371Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:01 smithi093 ceph-mon[111335]: pgmap v8683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:01 smithi093 ceph-mon[118902]: pgmap v8683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:01 smithi136 ceph-mon[113531]: pgmap v8683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:01.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:01.911436+0000) 2023-08-28T01:43:02.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3926694851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:02.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3926694851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:02.810Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:02.810Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3926694851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:02.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:02.911645+0000) 2023-08-28T01:43:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:03 smithi093 ceph-mon[111335]: pgmap v8684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:03 smithi093 ceph-mon[118902]: pgmap v8684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:03 smithi136 ceph-mon[113531]: pgmap v8684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:03.910+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:03.911886+0000) 2023-08-28T01:43:04.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3233638653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3233638653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:05.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3233638653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:04.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:04.912165+0000) 2023-08-28T01:43:05.731 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:43:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:43:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:43:05.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:05 smithi093 ceph-mon[111335]: pgmap v8685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:05.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:05 smithi093 ceph-mon[118902]: pgmap v8685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:05 smithi136 ceph-mon[113531]: pgmap v8685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:05.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:05.912400+0000) 2023-08-28T01:43:07.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:06.911+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:06.912647+0000) 2023-08-28T01:43:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:07 smithi093 ceph-mon[111335]: pgmap v8686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4010425504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:07 smithi093 ceph-mon[118902]: pgmap v8686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4010425504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:07.911 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:07 smithi136 ceph-mon[113531]: pgmap v8686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:07.911 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4010425504' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:07.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:07.912897+0000) 2023-08-28T01:43:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:08.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:08.913154+0000) 2023-08-28T01:43:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:09 smithi093 ceph-mon[111335]: pgmap v8687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1369015990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:09 smithi093 ceph-mon[118902]: pgmap v8687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1369015990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:09 smithi136 ceph-mon[113531]: pgmap v8687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1369015990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:09.912+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:09.913381+0000) 2023-08-28T01:43:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:10.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:10.913617+0000) 2023-08-28T01:43:11.719 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:11.371Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:11.719 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:11.372Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:11 smithi093 ceph-mon[111335]: pgmap v8688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:11.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/812063333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:11 smithi093 ceph-mon[118902]: pgmap v8688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:11.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/812063333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:11 smithi136 ceph-mon[113531]: pgmap v8688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:12.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/812063333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:11.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:11.913884+0000) 2023-08-28T01:43:13.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:12.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:12.914139+0000) 2023-08-28T01:43:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:12.811Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:12.812Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:13.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:13 smithi093 ceph-mon[111335]: pgmap v8689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:13 smithi093 ceph-mon[118902]: pgmap v8689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:13 smithi136 ceph-mon[113531]: pgmap v8689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:13.913+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:13.914405+0000) 2023-08-28T01:43:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1673593582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1673593582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:15.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1673593582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:14.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:14.914648+0000) 2023-08-28T01:43:15.728 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:43:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:43:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:43:15.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:15 smithi093 ceph-mon[118902]: pgmap v8690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:15.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:15 smithi093 ceph-mon[111335]: pgmap v8690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:15 smithi136 ceph-mon[113531]: pgmap v8690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:15.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:15.914896+0000) 2023-08-28T01:43:16.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/920488968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:16.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/920488968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:17.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/920488968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:16.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:16.915142+0000) 2023-08-28T01:43:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:17 smithi093 ceph-mon[111335]: pgmap v8691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:17 smithi093 ceph-mon[118902]: pgmap v8691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:17 smithi136 ceph-mon[113531]: pgmap v8691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:17.914+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:17.915412+0000) 2023-08-28T01:43:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:18.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:18.915649+0000) 2023-08-28T01:43:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:19 smithi093 ceph-mon[111335]: pgmap v8692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2539392693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:19 smithi093 ceph-mon[118902]: pgmap v8692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2539392693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:19 smithi136 ceph-mon[113531]: pgmap v8692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2539392693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:19.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:19.915887+0000) 2023-08-28T01:43:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:20.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:20.916180+0000) 2023-08-28T01:43:21.736 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:21.372Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:21.736 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:21.373Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:21 smithi093 ceph-mon[111335]: pgmap v8693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1101860211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:21 smithi093 ceph-mon[118902]: pgmap v8693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1101860211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:21 smithi136 ceph-mon[113531]: pgmap v8693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1101860211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:21.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:21.916476+0000) 2023-08-28T01:43:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:22.915+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:22.916767+0000) 2023-08-28T01:43:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:22.811Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:22.812Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:23 smithi136 ceph-mon[113531]: pgmap v8694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3340593449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:23.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:23.917027+0000) 2023-08-28T01:43:24.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:23 smithi093 ceph-mon[111335]: pgmap v8694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3340593449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:23 smithi093 ceph-mon[118902]: pgmap v8694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3340593449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:24.916+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:24.917284+0000) 2023-08-28T01:43:25.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:43:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:43:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:43:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:25 smithi136 ceph-mon[113531]: pgmap v8695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:25.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:25.917527+0000) 2023-08-28T01:43:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:25 smithi093 ceph-mon[111335]: pgmap v8695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:25 smithi093 ceph-mon[118902]: pgmap v8695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:27.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3828739926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:26.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:26.917785+0000) 2023-08-28T01:43:27.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3828739926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:27.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3828739926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:27 smithi136 ceph-mon[113531]: pgmap v8696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:27.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:27.918091+0000) 2023-08-28T01:43:28.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:27 smithi093 ceph-mon[111335]: pgmap v8696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:27 smithi093 ceph-mon[118902]: pgmap v8696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2391154428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:28.917+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:28.918316+0000) 2023-08-28T01:43:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2391154428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:29.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2391154428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:29 smithi136 ceph-mon[113531]: pgmap v8697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:29.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:29.918597+0000) 2023-08-28T01:43:30.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:29 smithi093 ceph-mon[111335]: pgmap v8697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:29 smithi093 ceph-mon[118902]: pgmap v8697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:31.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3728068343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:30.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:30.918786+0000) 2023-08-28T01:43:31.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3728068343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3728068343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:31.373Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:31.374Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:31 smithi136 ceph-mon[113531]: pgmap v8698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:31.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:31.919022+0000) 2023-08-28T01:43:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:31 smithi093 ceph-mon[111335]: pgmap v8698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:31 smithi093 ceph-mon[118902]: pgmap v8698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:32.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:32.919296+0000) 2023-08-28T01:43:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:32.812Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:32.812Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:33 smithi136 ceph-mon[113531]: pgmap v8699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/120492545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:33.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:33.919559+0000) 2023-08-28T01:43:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:33 smithi093 ceph-mon[111335]: pgmap v8699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/120492545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:33 smithi093 ceph-mon[118902]: pgmap v8699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/120492545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:34.918+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:34.919841+0000) 2023-08-28T01:43:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:43:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:43:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:43:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:35 smithi136 ceph-mon[113531]: pgmap v8700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/619561282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:35.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:35.920106+0000) 2023-08-28T01:43:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:35 smithi093 ceph-mon[111335]: pgmap v8700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/619561282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:35 smithi093 ceph-mon[118902]: pgmap v8700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/619561282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:37.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:36.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:36.920382+0000) 2023-08-28T01:43:37.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:37 smithi093 ceph-mon[111335]: pgmap v8701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:37 smithi093 ceph-mon[118902]: pgmap v8701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:37.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:37 smithi136 ceph-mon[113531]: pgmap v8701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:37.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:37.920626+0000) 2023-08-28T01:43:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3104793604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:43:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3104793604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:43:38.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3104793604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:38.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:43:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:38.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:38.920830+0000) 2023-08-28T01:43:39.609 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:39 smithi093 ceph-mon[118902]: pgmap v8702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:39.609 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:39.609 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:39 smithi093 ceph-mon[111335]: pgmap v8702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:39.609 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:39.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:39 smithi136 ceph-mon[113531]: pgmap v8702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:39.919 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:40.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:39.919+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:39.920986+0000) 2023-08-28T01:43:41.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:40.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:40.921141+0000) 2023-08-28T01:43:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4015751750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4015751750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4015751750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:41.373Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:41.374Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:42.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:41.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:41.921381+0000) 2023-08-28T01:43:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:41 smithi136 ceph-mon[113531]: pgmap v8703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:43:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:43:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[111335]: pgmap v8703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:43:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:43:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[118902]: pgmap v8703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:43:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:43:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:43:43.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3398601999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:43.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:42.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:42.921644+0000) 2023-08-28T01:43:43.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3398601999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3398601999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:42.812Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:42.813Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:43 smithi136 ceph-mon[113531]: pgmap v8704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:43.920+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:43.921920+0000) 2023-08-28T01:43:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:43 smithi093 ceph-mon[111335]: pgmap v8704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:43 smithi093 ceph-mon[118902]: pgmap v8704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:44.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:44.922114+0000) 2023-08-28T01:43:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:43:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:43:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:43:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:45 smithi136 ceph-mon[113531]: pgmap v8705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2227332292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:45.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:45.922378+0000) 2023-08-28T01:43:46.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:45 smithi093 ceph-mon[118902]: pgmap v8705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2227332292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:45 smithi093 ceph-mon[111335]: pgmap v8705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2227332292' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:46.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:46.922590+0000) 2023-08-28T01:43:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:47 smithi093 ceph-mon[118902]: pgmap v8706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1234852050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:47 smithi093 ceph-mon[111335]: pgmap v8706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1234852050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:47.921 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:47 smithi136 ceph-mon[113531]: pgmap v8706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:47.921 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1234852050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:47.921+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:47.922884+0000) 2023-08-28T01:43:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:48.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:48.923149+0000) 2023-08-28T01:43:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:49 smithi093 ceph-mon[111335]: pgmap v8707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:49 smithi093 ceph-mon[118902]: pgmap v8707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:49 smithi136 ceph-mon[113531]: pgmap v8707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:49.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:49.923398+0000) 2023-08-28T01:43:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/985627469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/985627469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/985627469' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:50.922+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:50.923639+0000) 2023-08-28T01:43:51.733 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:51.374Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:51.733 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:51.375Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:51.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:51 smithi093 ceph-mon[111335]: pgmap v8708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:51.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:51 smithi093 ceph-mon[118902]: pgmap v8708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:51 smithi136 ceph-mon[113531]: pgmap v8708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:51.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:51.923883+0000) 2023-08-28T01:43:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4149671762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4149671762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:52.813Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:43:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:43:52.814Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:43:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4149671762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:52.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:52.924149+0000) 2023-08-28T01:43:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:53 smithi136 ceph-mon[113531]: pgmap v8709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:53.923+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:53.924403+0000) 2023-08-28T01:43:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:53 smithi093 ceph-mon[111335]: pgmap v8709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:53 smithi093 ceph-mon[118902]: pgmap v8709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2930438048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:54.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:54.924613+0000) 2023-08-28T01:43:55.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2930438048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:43:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2930438048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:43:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:43:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:43:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:55 smithi136 ceph-mon[113531]: pgmap v8710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:55.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:55.924860+0000) 2023-08-28T01:43:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:55 smithi093 ceph-mon[111335]: pgmap v8710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:55 smithi093 ceph-mon[118902]: pgmap v8710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:56.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:56.925182+0000) 2023-08-28T01:43:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:57 smithi093 ceph-mon[111335]: pgmap v8711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1143874543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:57 smithi093 ceph-mon[118902]: pgmap v8711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1143874543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:57.923 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:57 smithi136 ceph-mon[113531]: pgmap v8711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:57.924 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1143874543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:57.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:57.925452+0000) 2023-08-28T01:43:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:58.924+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:58.925699+0000) 2023-08-28T01:43:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:59 smithi093 ceph-mon[118902]: pgmap v8712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:43:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1293090140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:43:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:59 smithi093 ceph-mon[111335]: pgmap v8712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:43:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:43:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1293090140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:59 smithi136 ceph-mon[113531]: pgmap v8712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:43:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1293090140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:43:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:43:59.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:43:59.925947+0000) 2023-08-28T01:44:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:00.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:00.926174+0000) 2023-08-28T01:44:01.733 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:01.375Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:01.733 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:01.376Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:01 smithi093 ceph-mon[111335]: pgmap v8713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:01.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1241796720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:01 smithi093 ceph-mon[118902]: pgmap v8713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:01.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1241796720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:02.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:01.925+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:01.926409+0000) 2023-08-28T01:44:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:01 smithi136 ceph-mon[113531]: pgmap v8713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1241796720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:02.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:02.926653+0000) 2023-08-28T01:44:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:02.814Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:02.815Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:03 smithi093 ceph-mon[111335]: pgmap v8714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:03 smithi093 ceph-mon[118902]: pgmap v8714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:03 smithi136 ceph-mon[113531]: pgmap v8714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:03.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:03.926901+0000) 2023-08-28T01:44:05.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1655389518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:04.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:04.927136+0000) 2023-08-28T01:44:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1655389518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1655389518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:44:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:44:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:44:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:05 smithi136 ceph-mon[113531]: pgmap v8715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:05.926+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:05.927370+0000) 2023-08-28T01:44:06.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:05 smithi093 ceph-mon[111335]: pgmap v8715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:05 smithi093 ceph-mon[118902]: pgmap v8715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:07.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/569358167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:06.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:06.927629+0000) 2023-08-28T01:44:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/569358167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/569358167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:07 smithi136 ceph-mon[113531]: pgmap v8716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:07.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:07.927859+0000) 2023-08-28T01:44:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:07 smithi093 ceph-mon[111335]: pgmap v8716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:07 smithi093 ceph-mon[118902]: pgmap v8716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:08.927+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:08.928142+0000) 2023-08-28T01:44:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:09.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:09.928365+0000) 2023-08-28T01:44:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:09 smithi136 ceph-mon[113531]: pgmap v8717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1213057717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:09 smithi093 ceph-mon[111335]: pgmap v8717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1213057717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:09 smithi093 ceph-mon[118902]: pgmap v8717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1213057717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:10.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:10.928640+0000) 2023-08-28T01:44:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:11.375Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:11.376Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:11 smithi136 ceph-mon[113531]: pgmap v8718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/6642245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:11.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:11.928942+0000) 2023-08-28T01:44:12.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:11 smithi093 ceph-mon[118902]: pgmap v8718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/6642245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:11 smithi093 ceph-mon[111335]: pgmap v8718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/6642245' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:13.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:12.928+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:12.929161+0000) 2023-08-28T01:44:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:12.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:12.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:13 smithi136 ceph-mon[113531]: pgmap v8719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:14.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/296117510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:13.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:13.929365+0000) 2023-08-28T01:44:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:13 smithi093 ceph-mon[111335]: pgmap v8719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/296117510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:13 smithi093 ceph-mon[118902]: pgmap v8719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/296117510' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:14.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:14.929594+0000) 2023-08-28T01:44:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:44:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:44:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:44:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:15 smithi136 ceph-mon[113531]: pgmap v8720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:15.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:15.929821+0000) 2023-08-28T01:44:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:15 smithi093 ceph-mon[111335]: pgmap v8720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:15 smithi093 ceph-mon[118902]: pgmap v8720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:17.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2660789521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:16.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:16.930073+0000) 2023-08-28T01:44:17.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2660789521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:17.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2660789521' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:17 smithi136 ceph-mon[113531]: pgmap v8721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:17.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:17.930299+0000) 2023-08-28T01:44:18.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:17 smithi093 ceph-mon[111335]: pgmap v8721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:17 smithi093 ceph-mon[118902]: pgmap v8721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3807034957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:18.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:18.930559+0000) 2023-08-28T01:44:19.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3807034957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3807034957' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:19 smithi136 ceph-mon[113531]: pgmap v8722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:19.929+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:19.930806+0000) 2023-08-28T01:44:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:19 smithi093 ceph-mon[111335]: pgmap v8722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:19 smithi093 ceph-mon[118902]: pgmap v8722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2009084466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:20.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:20.931079+0000) 2023-08-28T01:44:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2009084466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2009084466' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:21.376Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:21.376Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:21 smithi136 ceph-mon[113531]: pgmap v8723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:21.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:21.931323+0000) 2023-08-28T01:44:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:21 smithi093 ceph-mon[111335]: pgmap v8723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:21 smithi093 ceph-mon[118902]: pgmap v8723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:22.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:22.931592+0000) 2023-08-28T01:44:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:22.815Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:22.816Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:23 smithi136 ceph-mon[113531]: pgmap v8724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1026508274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:23.930+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:23.931888+0000) 2023-08-28T01:44:24.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:23 smithi093 ceph-mon[111335]: pgmap v8724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:24.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1026508274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:23 smithi093 ceph-mon[118902]: pgmap v8724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1026508274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:25.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:24.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:24.932159+0000) 2023-08-28T01:44:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:44:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:44:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:44:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:25 smithi136 ceph-mon[113531]: pgmap v8725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3277158922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:25.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:25.932431+0000) 2023-08-28T01:44:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:25 smithi093 ceph-mon[111335]: pgmap v8725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:26.248 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3277158922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:26.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:25 smithi093 ceph-mon[118902]: pgmap v8725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:26.248 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3277158922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:26.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:26.932720+0000) 2023-08-28T01:44:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:27 smithi093 ceph-mon[111335]: pgmap v8726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:27 smithi093 ceph-mon[118902]: pgmap v8726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:27.931 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:27 smithi136 ceph-mon[113531]: pgmap v8726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:27.931+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:27.932992+0000) 2023-08-28T01:44:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/572811682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/572811682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:28.931 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/572811682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:29.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:28.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:28.933261+0000) 2023-08-28T01:44:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:29 smithi093 ceph-mon[111335]: pgmap v8727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:29 smithi093 ceph-mon[118902]: pgmap v8727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:29.932 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:29 smithi136 ceph-mon[113531]: pgmap v8727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:29.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:29.933530+0000) 2023-08-28T01:44:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2477134110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2477134110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:30.932 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2477134110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:31.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:30.932+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:30.933775+0000) 2023-08-28T01:44:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:31 smithi093 ceph-mon[111335]: pgmap v8728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:31 smithi093 ceph-mon[118902]: pgmap v8728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:31.376Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:31.377Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:31.932 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:31 smithi136 ceph-mon[113531]: pgmap v8728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:32.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:31.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:31.934017+0000) 2023-08-28T01:44:32.932 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2006348785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2006348785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2006348785' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:32.816Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:32.817Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:33.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:32.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:32.934299+0000) 2023-08-28T01:44:33.934 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:33 smithi136 ceph-mon[113531]: pgmap v8729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:33 smithi093 ceph-mon[111335]: pgmap v8729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:33 smithi093 ceph-mon[118902]: pgmap v8729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:33.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:33.934536+0000) 2023-08-28T01:44:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:34.933+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:34.934817+0000) 2023-08-28T01:44:35.736 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:44:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:44:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:44:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:35 smithi093 ceph-mon[118902]: pgmap v8730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3900108728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:35 smithi093 ceph-mon[111335]: pgmap v8730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3900108728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:35 smithi136 ceph-mon[113531]: pgmap v8730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:36.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3900108728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:36.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:35.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:35.935096+0000) 2023-08-28T01:44:37.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:36.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:36.935327+0000) 2023-08-28T01:44:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:37 smithi093 ceph-mon[118902]: pgmap v8731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/380537794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:37 smithi093 ceph-mon[111335]: pgmap v8731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/380537794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:37.934 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:37 smithi136 ceph-mon[113531]: pgmap v8731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:37.934 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/380537794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:37.934+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:37.935580+0000) 2023-08-28T01:44:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:38.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:38.935827+0000) 2023-08-28T01:44:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:39 smithi093 ceph-mon[111335]: pgmap v8732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:39 smithi093 ceph-mon[118902]: pgmap v8732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:39 smithi136 ceph-mon[113531]: pgmap v8732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:39.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:39.936126+0000) 2023-08-28T01:44:41.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/806787963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:40.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:40.936364+0000) 2023-08-28T01:44:41.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/806787963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/806787963' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:41.377Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:41.377Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:42.117 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:41 smithi136 ceph-mon[113531]: pgmap v8733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:42.118 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:44:42.118 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:41.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:41.936654+0000) 2023-08-28T01:44:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:41 smithi093 ceph-mon[111335]: pgmap v8733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:44:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:41 smithi093 ceph-mon[118902]: pgmap v8733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:44:42.935 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/93236376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:42.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/93236376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/93236376' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:42.817Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:42.818Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:42.935+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:42.936821+0000) 2023-08-28T01:44:44.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:43.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:43.937072+0000) 2023-08-28T01:44:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:43 smithi136 ceph-mon[113531]: pgmap v8734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:43 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[111335]: pgmap v8734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[118902]: pgmap v8734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:43 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:44.936+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:44.937257+0000) 2023-08-28T01:44:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:44:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:44:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2156669351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2156669351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:44:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2156669351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:44:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:44:45] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:44:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:45 smithi136 ceph-mon[113531]: pgmap v8735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:45.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:45.937449+0000) 2023-08-28T01:44:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:45 smithi093 ceph-mon[111335]: pgmap v8735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:45 smithi093 ceph-mon[118902]: pgmap v8735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:46.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:46.937680+0000) 2023-08-28T01:44:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:47 smithi093 ceph-mon[118902]: pgmap v8736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2243915058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:47 smithi093 ceph-mon[111335]: pgmap v8736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2243915058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:47.936 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:47 smithi136 ceph-mon[113531]: pgmap v8736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:47.936 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2243915058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:47.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:47.937960+0000) 2023-08-28T01:44:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:48.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:48.938240+0000) 2023-08-28T01:44:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:49 smithi136 ceph-mon[113531]: pgmap v8737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/156814403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:49.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:49.938484+0000) 2023-08-28T01:44:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:49 smithi093 ceph-mon[111335]: pgmap v8737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/156814403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:49 smithi093 ceph-mon[118902]: pgmap v8737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/156814403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:50.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:50.938724+0000) 2023-08-28T01:44:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:51.377Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:51.379Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:51 smithi136 ceph-mon[113531]: pgmap v8738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2143321137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:51.937+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:51.938979+0000) 2023-08-28T01:44:52.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:51 smithi093 ceph-mon[111335]: pgmap v8738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2143321137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:51 smithi093 ceph-mon[118902]: pgmap v8738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2143321137' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:52.818Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:44:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:44:52.819Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:44:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:52.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:52.939191+0000) 2023-08-28T01:44:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:53 smithi136 ceph-mon[113531]: pgmap v8739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:53.938+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:53.939400+0000) 2023-08-28T01:44:54.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:53 smithi093 ceph-mon[111335]: pgmap v8739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:53 smithi093 ceph-mon[118902]: pgmap v8739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2708458130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:54.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:54.939668+0000) 2023-08-28T01:44:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2708458130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:44:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2708458130' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:44:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:44:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:44:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:55 smithi136 ceph-mon[113531]: pgmap v8740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:55.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:55.939906+0000) 2023-08-28T01:44:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:55 smithi093 ceph-mon[111335]: pgmap v8740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:55 smithi093 ceph-mon[118902]: pgmap v8740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:56.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:56.940172+0000) 2023-08-28T01:44:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1706181862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:57 smithi093 ceph-mon[111335]: pgmap v8741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1706181862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:57 smithi093 ceph-mon[118902]: pgmap v8741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1706181862' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:44:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:57 smithi136 ceph-mon[113531]: pgmap v8741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:44:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:57.939+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:57.940394+0000) 2023-08-28T01:44:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:58.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:58.940621+0000) 2023-08-28T01:45:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:59 smithi136 ceph-mon[113531]: pgmap v8742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:44:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2616298940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:44:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:44:59.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:44:59.940870+0000) 2023-08-28T01:45:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:59 smithi093 ceph-mon[111335]: pgmap v8742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:44:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2616298940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:59 smithi093 ceph-mon[118902]: pgmap v8742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:44:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2616298940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:00.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:00.941100+0000) 2023-08-28T01:45:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:01.378Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:01.379Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:01 smithi136 ceph-mon[113531]: pgmap v8743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4246777729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:01.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:01.941334+0000) 2023-08-28T01:45:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:01 smithi093 ceph-mon[111335]: pgmap v8743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4246777729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:01 smithi093 ceph-mon[118902]: pgmap v8743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4246777729' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:02.818Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:02.819Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:02.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:02.941586+0000) 2023-08-28T01:45:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:03 smithi136 ceph-mon[113531]: pgmap v8744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/386750138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:03.940+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:03.941786+0000) 2023-08-28T01:45:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:03 smithi093 ceph-mon[111335]: pgmap v8744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/386750138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:03 smithi093 ceph-mon[118902]: pgmap v8744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/386750138' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:04.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:04.942073+0000) 2023-08-28T01:45:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:45:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:45:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:45:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:05 smithi136 ceph-mon[113531]: pgmap v8745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:05.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:05.942340+0000) 2023-08-28T01:45:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:05 smithi093 ceph-mon[111335]: pgmap v8745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:05 smithi093 ceph-mon[118902]: pgmap v8745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2792278266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:07.193 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:06.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:06.942548+0000) 2023-08-28T01:45:07.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2792278266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2792278266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:07 smithi136 ceph-mon[113531]: pgmap v8746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:07.941+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:07.942810+0000) 2023-08-28T01:45:08.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:07 smithi093 ceph-mon[111335]: pgmap v8746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:07 smithi093 ceph-mon[118902]: pgmap v8746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1069891203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:08.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:08.943080+0000) 2023-08-28T01:45:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1069891203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1069891203' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:09 smithi136 ceph-mon[113531]: pgmap v8747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:09.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:09.943290+0000) 2023-08-28T01:45:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:09 smithi093 ceph-mon[111335]: pgmap v8747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:09 smithi093 ceph-mon[118902]: pgmap v8747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:10.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:10.943528+0000) 2023-08-28T01:45:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:11.379Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:11.380Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:11 smithi136 ceph-mon[113531]: pgmap v8748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:12.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3571039378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:11.942+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:11.943800+0000) 2023-08-28T01:45:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:11 smithi093 ceph-mon[111335]: pgmap v8748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3571039378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:11 smithi093 ceph-mon[118902]: pgmap v8748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3571039378' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:12.819Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:12.819Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:12.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:12.944085+0000) 2023-08-28T01:45:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:13 smithi136 ceph-mon[113531]: pgmap v8749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3771388902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:13.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:13.944351+0000) 2023-08-28T01:45:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:13 smithi093 ceph-mon[111335]: pgmap v8749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3771388902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:13 smithi093 ceph-mon[118902]: pgmap v8749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3771388902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:14.943+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:14.944581+0000) 2023-08-28T01:45:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:45:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:45:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:45:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:15 smithi136 ceph-mon[113531]: pgmap v8750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/687098471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:15.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:15.944826+0000) 2023-08-28T01:45:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:15 smithi093 ceph-mon[111335]: pgmap v8750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/687098471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:15 smithi093 ceph-mon[118902]: pgmap v8750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/687098471' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:16.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:16.945105+0000) 2023-08-28T01:45:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:17 smithi093 ceph-mon[111335]: pgmap v8751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:17 smithi093 ceph-mon[118902]: pgmap v8751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:17 smithi136 ceph-mon[113531]: pgmap v8751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:17.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:17.945343+0000) 2023-08-28T01:45:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2515534015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2515534015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2515534015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:18.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:18.945517+0000) 2023-08-28T01:45:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:19 smithi093 ceph-mon[111335]: pgmap v8752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:19 smithi093 ceph-mon[118902]: pgmap v8752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:19 smithi136 ceph-mon[113531]: pgmap v8752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:20.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:19.944+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:19.945802+0000) 2023-08-28T01:45:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1283525366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1283525366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:20.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1283525366' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:20.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:20.946087+0000) 2023-08-28T01:45:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:21 smithi093 ceph-mon[118902]: pgmap v8753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:21 smithi093 ceph-mon[111335]: pgmap v8753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:21.380Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:21.381Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:21 smithi136 ceph-mon[113531]: pgmap v8753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:21.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:21.946271+0000) 2023-08-28T01:45:23.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2028874010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:22.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:22.946478+0000) 2023-08-28T01:45:23.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2028874010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2028874010' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:22.820Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:22.821Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:23 smithi136 ceph-mon[113531]: pgmap v8754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:23.945+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:23.946672+0000) 2023-08-28T01:45:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:23 smithi093 ceph-mon[111335]: pgmap v8754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:24.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:23 smithi093 ceph-mon[118902]: pgmap v8754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:24.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:24.946889+0000) 2023-08-28T01:45:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:45:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:45:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:45:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:25 smithi136 ceph-mon[113531]: pgmap v8755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2164098129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:25.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:25.947157+0000) 2023-08-28T01:45:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:25 smithi093 ceph-mon[111335]: pgmap v8755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2164098129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:25 smithi093 ceph-mon[118902]: pgmap v8755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2164098129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:26.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:26.947365+0000) 2023-08-28T01:45:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:27 smithi093 ceph-mon[111335]: pgmap v8756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:27 smithi093 ceph-mon[118902]: pgmap v8756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:27 smithi136 ceph-mon[113531]: pgmap v8756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:27.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:27.947568+0000) 2023-08-28T01:45:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/99872848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/99872848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/99872848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:28.946+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:28.947788+0000) 2023-08-28T01:45:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:29 smithi093 ceph-mon[111335]: pgmap v8757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:29 smithi093 ceph-mon[118902]: pgmap v8757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:29 smithi136 ceph-mon[113531]: pgmap v8757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:29.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:29.948095+0000) 2023-08-28T01:45:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1442438659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1442438659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:30.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1442438659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:31.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:30.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:30.948286+0000) 2023-08-28T01:45:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:31 smithi093 ceph-mon[111335]: pgmap v8758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:31 smithi093 ceph-mon[118902]: pgmap v8758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:31.381Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:31.382Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:31 smithi136 ceph-mon[113531]: pgmap v8758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:31.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:31.948469+0000) 2023-08-28T01:45:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/479481885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:32.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/479481885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:32.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/479481885' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:32.820Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:32.821Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:32.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:32.948728+0000) 2023-08-28T01:45:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:33 smithi093 ceph-mon[111335]: pgmap v8759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:33 smithi093 ceph-mon[118902]: pgmap v8759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:33 smithi136 ceph-mon[113531]: pgmap v8759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:33.947+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:33.949000+0000) 2023-08-28T01:45:34.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1769193696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:34.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1769193696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:34.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1769193696' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:34.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:34.949266+0000) 2023-08-28T01:45:35.690 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:45:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:45:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:45:35.948 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:35 smithi136 ceph-mon[113531]: pgmap v8760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:35 smithi093 ceph-mon[111335]: pgmap v8760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:35.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:35 smithi093 ceph-mon[118902]: pgmap v8760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:35.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:35.949507+0000) 2023-08-28T01:45:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:36.948+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:36.949757+0000) 2023-08-28T01:45:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:37 smithi093 ceph-mon[111335]: pgmap v8761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2024727109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:37 smithi093 ceph-mon[118902]: pgmap v8761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2024727109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:37 smithi136 ceph-mon[113531]: pgmap v8761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2024727109' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:37.949+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:37.950077+0000) 2023-08-28T01:45:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:38.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:38.950293+0000) 2023-08-28T01:45:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:39 smithi136 ceph-mon[113531]: pgmap v8762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3478249092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:39.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:39.950540+0000) 2023-08-28T01:45:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:39 smithi093 ceph-mon[111335]: pgmap v8762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3478249092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:39 smithi093 ceph-mon[118902]: pgmap v8762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3478249092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:40.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:40.950832+0000) 2023-08-28T01:45:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:41.382Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:41.383Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:41 smithi136 ceph-mon[113531]: pgmap v8763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:41.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:41.951136+0000) 2023-08-28T01:45:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:41 smithi093 ceph-mon[111335]: pgmap v8763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:41 smithi093 ceph-mon[118902]: pgmap v8763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:43.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3744553198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:43.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:42.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:42.951393+0000) 2023-08-28T01:45:43.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3744553198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3744553198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:42.821Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:42.822Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:43 smithi136 ceph-mon[113531]: pgmap v8764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:43.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:43.951667+0000) 2023-08-28T01:45:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:43 smithi093 ceph-mon[111335]: pgmap v8764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:44.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:43 smithi093 ceph-mon[118902]: pgmap v8764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:44.950 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:45:44.951 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/675866702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:44.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:45:44.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/675866702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:45:44.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/675866702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:44.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:44.951836+0000) 2023-08-28T01:45:45.487 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:45:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:45:45] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:45:45.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:45 smithi136 ceph-mon[113531]: pgmap v8765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:45 smithi093 ceph-mon[111335]: pgmap v8765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:45 smithi093 ceph-mon[118902]: pgmap v8765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:46.293 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:45.950+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:45.951970+0000) 2023-08-28T01:45:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:46.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:46.952128+0000) 2023-08-28T01:45:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:46 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3197105257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:47.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3197105257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3197105257' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:47 smithi136 ceph-mon[113531]: pgmap v8766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:45:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:45:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:47.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:47.952307+0000) 2023-08-28T01:45:48.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[111335]: pgmap v8766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:45:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:45:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[118902]: pgmap v8766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:45:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:45:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:45:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:48.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:48.952548+0000) 2023-08-28T01:45:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:49 smithi136 ceph-mon[113531]: pgmap v8767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2998762026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:49.951+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:49.952789+0000) 2023-08-28T01:45:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:49 smithi093 ceph-mon[111335]: pgmap v8767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2998762026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:49 smithi093 ceph-mon[118902]: pgmap v8767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2998762026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:50.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:50.953065+0000) 2023-08-28T01:45:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:51.383Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:51.384Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:51 smithi136 ceph-mon[113531]: pgmap v8768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2347809457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:51.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:51.953326+0000) 2023-08-28T01:45:52.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:51 smithi093 ceph-mon[111335]: pgmap v8768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2347809457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:51 smithi093 ceph-mon[118902]: pgmap v8768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2347809457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:52.821Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:45:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:45:52.822Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:45:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:52.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:52.953566+0000) 2023-08-28T01:45:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:53 smithi136 ceph-mon[113531]: pgmap v8769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1492016474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:53.952+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:53.953832+0000) 2023-08-28T01:45:54.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:53 smithi093 ceph-mon[118902]: pgmap v8769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1492016474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:53 smithi093 ceph-mon[111335]: pgmap v8769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1492016474' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:54.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:54.954109+0000) 2023-08-28T01:45:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:45:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:45:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:45:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:45:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:55 smithi136 ceph-mon[113531]: pgmap v8770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:55.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:55.954351+0000) 2023-08-28T01:45:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:55 smithi093 ceph-mon[111335]: pgmap v8770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:55 smithi093 ceph-mon[118902]: pgmap v8770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:57.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2861825299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:56.953+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:56.954637+0000) 2023-08-28T01:45:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2861825299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2861825299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:57 smithi136 ceph-mon[113531]: pgmap v8771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:57.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:57.954891+0000) 2023-08-28T01:45:58.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:57 smithi093 ceph-mon[111335]: pgmap v8771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:57 smithi093 ceph-mon[118902]: pgmap v8771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:45:59.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3921736482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:58.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:58.955154+0000) 2023-08-28T01:45:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3921736482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:45:59.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3921736482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:45:59 smithi136 ceph-mon[113531]: pgmap v8772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:45:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:45:59.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:45:59.955335+0000) 2023-08-28T01:46:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:45:59 smithi093 ceph-mon[111335]: pgmap v8772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:45:59 smithi093 ceph-mon[118902]: pgmap v8772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:00.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:00.955481+0000) 2023-08-28T01:46:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:01.383Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:01.384Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:01 smithi136 ceph-mon[113531]: pgmap v8773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/205399097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:01.954+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:01.955710+0000) 2023-08-28T01:46:02.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:01 smithi093 ceph-mon[118902]: pgmap v8773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/205399097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:01 smithi093 ceph-mon[111335]: pgmap v8773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/205399097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:02.822Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:02.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:02.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:02.955966+0000) 2023-08-28T01:46:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:03 smithi136 ceph-mon[113531]: pgmap v8774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2289797311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:03.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:03.956233+0000) 2023-08-28T01:46:04.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:03 smithi093 ceph-mon[111335]: pgmap v8774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2289797311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:03 smithi093 ceph-mon[118902]: pgmap v8774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2289797311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:04.955+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:04.956478+0000) 2023-08-28T01:46:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:46:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:46:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:46:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:05 smithi136 ceph-mon[113531]: pgmap v8775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3476044487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:05.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:05.956681+0000) 2023-08-28T01:46:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:05 smithi093 ceph-mon[111335]: pgmap v8775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3476044487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:05 smithi093 ceph-mon[118902]: pgmap v8775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3476044487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:06.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:06.956956+0000) 2023-08-28T01:46:07.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:07 smithi093 ceph-mon[111335]: pgmap v8776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:07 smithi093 ceph-mon[118902]: pgmap v8776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:07 smithi136 ceph-mon[113531]: pgmap v8776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:07.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:07.957214+0000) 2023-08-28T01:46:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3896538687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3896538687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3896538687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:08.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:08.957438+0000) 2023-08-28T01:46:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:09 smithi093 ceph-mon[111335]: pgmap v8777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:09 smithi093 ceph-mon[118902]: pgmap v8777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:09.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:09 smithi136 ceph-mon[113531]: pgmap v8777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:09.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:09.957619+0000) 2023-08-28T01:46:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2401683143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2401683143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:10.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2401683143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:10.956+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:10.957873+0000) 2023-08-28T01:46:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:11 smithi093 ceph-mon[111335]: pgmap v8778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:11 smithi093 ceph-mon[118902]: pgmap v8778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:11.383Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:11.385Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:11 smithi136 ceph-mon[113531]: pgmap v8778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:11.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:11.958156+0000) 2023-08-28T01:46:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:12.823Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:12.823Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:12.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:12.958307+0000) 2023-08-28T01:46:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:13 smithi136 ceph-mon[113531]: pgmap v8779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1657896607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:13.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:13.958535+0000) 2023-08-28T01:46:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:13 smithi093 ceph-mon[111335]: pgmap v8779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1657896607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:13 smithi093 ceph-mon[118902]: pgmap v8779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1657896607' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:14.957+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:14.958809+0000) 2023-08-28T01:46:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:46:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:46:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:46:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:15 smithi136 ceph-mon[113531]: pgmap v8780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/389721954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:15.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:15.959028+0000) 2023-08-28T01:46:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:15 smithi093 ceph-mon[111335]: pgmap v8780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/389721954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:15 smithi093 ceph-mon[118902]: pgmap v8780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/389721954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:16.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:16.959288+0000) 2023-08-28T01:46:17.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:17 smithi093 ceph-mon[111335]: pgmap v8781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:17 smithi093 ceph-mon[118902]: pgmap v8781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:17 smithi136 ceph-mon[113531]: pgmap v8781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:17.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:17.959556+0000) 2023-08-28T01:46:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/480008491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:18.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/480008491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/480008491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:18.958+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:18.959865+0000) 2023-08-28T01:46:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:19 smithi093 ceph-mon[111335]: pgmap v8782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:19 smithi093 ceph-mon[118902]: pgmap v8782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:19 smithi136 ceph-mon[113531]: pgmap v8782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:20.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:19.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:19.960142+0000) 2023-08-28T01:46:20.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1019076746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1019076746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:20.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1019076746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:20.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:20.960430+0000) 2023-08-28T01:46:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:21 smithi093 ceph-mon[111335]: pgmap v8783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:21 smithi093 ceph-mon[118902]: pgmap v8783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:21.385Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:21.386Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:21 smithi136 ceph-mon[113531]: pgmap v8783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:21.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:21.960719+0000) 2023-08-28T01:46:22.822 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1305259809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:22.822 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1305259809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:22.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1305259809' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:22.824Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:22.825Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:22.959+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:22.960999+0000) 2023-08-28T01:46:23.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:23 smithi136 ceph-mon[113531]: pgmap v8784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:23.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:23 smithi093 ceph-mon[111335]: pgmap v8784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:23 smithi093 ceph-mon[118902]: pgmap v8784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:23.960+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:23.961249+0000) 2023-08-28T01:46:24.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:24.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:24.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:24.961479+0000) 2023-08-28T01:46:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:46:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:46:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:46:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1601442798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:25 smithi093 ceph-mon[111335]: pgmap v8785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1601442798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:25 smithi093 ceph-mon[118902]: pgmap v8785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:25.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1601442798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:25.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:25 smithi136 ceph-mon[113531]: pgmap v8785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:25.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:25.961736+0000) 2023-08-28T01:46:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:26.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:26.962003+0000) 2023-08-28T01:46:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:27 smithi093 ceph-mon[111335]: pgmap v8786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/838443759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:27 smithi093 ceph-mon[118902]: pgmap v8786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/838443759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:27 smithi136 ceph-mon[113531]: pgmap v8786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/838443759' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:27.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:27.962309+0000) 2023-08-28T01:46:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:28.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:28.962579+0000) 2023-08-28T01:46:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:29 smithi136 ceph-mon[113531]: pgmap v8787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/995707217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:29.961+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:29.962843+0000) 2023-08-28T01:46:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:29 smithi093 ceph-mon[111335]: pgmap v8787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/995707217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:29 smithi093 ceph-mon[118902]: pgmap v8787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/995707217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:30.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:30.963113+0000) 2023-08-28T01:46:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:31.385Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:31.386Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:31 smithi136 ceph-mon[113531]: pgmap v8788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:31.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:31.963379+0000) 2023-08-28T01:46:32.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:31 smithi093 ceph-mon[111335]: pgmap v8788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:31 smithi093 ceph-mon[118902]: pgmap v8788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:33.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3920255870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:32.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:32.963605+0000) 2023-08-28T01:46:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3920255870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3920255870' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:32.825Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:32.826Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:33 smithi136 ceph-mon[113531]: pgmap v8789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:33.962+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:33.963862+0000) 2023-08-28T01:46:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:33 smithi093 ceph-mon[111335]: pgmap v8789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:33 smithi093 ceph-mon[118902]: pgmap v8789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:35.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3085321099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:34.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:34.964169+0000) 2023-08-28T01:46:35.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3085321099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3085321099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:46:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:46:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:46:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:35 smithi136 ceph-mon[113531]: pgmap v8790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:35.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:35.964459+0000) 2023-08-28T01:46:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:35 smithi093 ceph-mon[111335]: pgmap v8790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:35 smithi093 ceph-mon[118902]: pgmap v8790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:36.963+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:36.964734+0000) 2023-08-28T01:46:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2683378319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:37 smithi093 ceph-mon[111335]: pgmap v8791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2683378319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:37 smithi093 ceph-mon[118902]: pgmap v8791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2683378319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:37 smithi136 ceph-mon[113531]: pgmap v8791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:37.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:37.964982+0000) 2023-08-28T01:46:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:38.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:38.965233+0000) 2023-08-28T01:46:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:39 smithi136 ceph-mon[113531]: pgmap v8792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:40.550 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:40.550 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1623256911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:40.550 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:39.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:39.965458+0000) 2023-08-28T01:46:40.551 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:39 smithi093 ceph-mon[111335]: pgmap v8792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:40.551 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:40.551 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1623256911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:40.551 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:39 smithi093 ceph-mon[118902]: pgmap v8792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:40.552 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:40.552 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1623256911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:40.964+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:40.965711+0000) 2023-08-28T01:46:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:41.385Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:41.773 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:41.386Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:41 smithi136 ceph-mon[113531]: pgmap v8793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4022464764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:41.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:41.965934+0000) 2023-08-28T01:46:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:41 smithi093 ceph-mon[111335]: pgmap v8793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4022464764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:41 smithi093 ceph-mon[118902]: pgmap v8793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4022464764' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:42.825Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:42.826Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:42.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:42.966181+0000) 2023-08-28T01:46:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:43 smithi136 ceph-mon[113531]: pgmap v8794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2582383177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:43.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:43.966423+0000) 2023-08-28T01:46:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:43 smithi093 ceph-mon[111335]: pgmap v8794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2582383177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:43 smithi093 ceph-mon[118902]: pgmap v8794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2582383177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:44.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:44.966698+0000) 2023-08-28T01:46:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:46:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:46:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:46:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:45 smithi136 ceph-mon[113531]: pgmap v8795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:45.965+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:45.966976+0000) 2023-08-28T01:46:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:45 smithi093 ceph-mon[111335]: pgmap v8795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:45 smithi093 ceph-mon[118902]: pgmap v8795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:47.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1215350392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:46.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:46.967213+0000) 2023-08-28T01:46:47.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1215350392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1215350392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:47.966 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:47 smithi136 ceph-mon[113531]: pgmap v8796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:47.966 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:46:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:47 smithi093 ceph-mon[111335]: pgmap v8796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:46:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:47 smithi093 ceph-mon[118902]: pgmap v8796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:46:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:47.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:47.967388+0000) 2023-08-28T01:46:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2123701600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:48.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2123701600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2123701600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:49.347 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:48.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:48.967570+0000) 2023-08-28T01:46:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:49 smithi136 ceph-mon[113531]: pgmap v8797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:49.966+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:49.967768+0000) 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[111335]: pgmap v8797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[118902]: pgmap v8797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:50.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:50.967955+0000) 2023-08-28T01:46:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:46:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:46:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:46:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:46:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:46:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:46:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:46:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:51.385Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:51.387Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:51.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:51.968179+0000) 2023-08-28T01:46:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:51 smithi136 ceph-mon[113531]: pgmap v8798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3187175079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:51 smithi093 ceph-mon[111335]: pgmap v8798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3187175079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:51 smithi093 ceph-mon[118902]: pgmap v8798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3187175079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:52.826Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:46:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:46:52.827Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:46:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:52.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:52.968414+0000) 2023-08-28T01:46:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:53 smithi136 ceph-mon[113531]: pgmap v8799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1114455068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:53.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:53.968645+0000) 2023-08-28T01:46:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:53 smithi093 ceph-mon[111335]: pgmap v8799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1114455068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:53 smithi093 ceph-mon[118902]: pgmap v8799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1114455068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:54.967+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:54.968907+0000) 2023-08-28T01:46:55.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:46:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:46:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:46:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:46:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:55 smithi136 ceph-mon[113531]: pgmap v8800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2998628744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:55.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:55.969165+0000) 2023-08-28T01:46:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:55 smithi093 ceph-mon[111335]: pgmap v8800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2998628744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:55 smithi093 ceph-mon[118902]: pgmap v8800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2998628744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:57.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:56.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:56.969404+0000) 2023-08-28T01:46:57.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:57 smithi093 ceph-mon[111335]: pgmap v8801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:57 smithi093 ceph-mon[118902]: pgmap v8801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:57 smithi136 ceph-mon[113531]: pgmap v8801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:58.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:57.968+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:57.969671+0000) 2023-08-28T01:46:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1236511768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1236511768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1236511768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:46:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:58.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:58.969893+0000) 2023-08-28T01:46:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:46:59 smithi093 ceph-mon[111335]: pgmap v8802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:46:59 smithi093 ceph-mon[118902]: pgmap v8802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:46:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:46:59 smithi136 ceph-mon[113531]: pgmap v8802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:46:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:46:59.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:46:59.970158+0000) 2023-08-28T01:47:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1261841047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1261841047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1261841047' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:00.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:00.970363+0000) 2023-08-28T01:47:01.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:01 smithi093 ceph-mon[111335]: pgmap v8803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:01 smithi093 ceph-mon[118902]: pgmap v8803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:01.386Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:01.388Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:01 smithi136 ceph-mon[113531]: pgmap v8803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:01.969+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:01.970609+0000) 2023-08-28T01:47:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:02.827Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:02.828Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:02.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:02.970852+0000) 2023-08-28T01:47:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:03 smithi136 ceph-mon[113531]: pgmap v8804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3319401972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:03.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:03.971110+0000) 2023-08-28T01:47:04.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:03 smithi093 ceph-mon[118902]: pgmap v8804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3319401972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:03 smithi093 ceph-mon[111335]: pgmap v8804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3319401972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:04.970+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:04.971340+0000) 2023-08-28T01:47:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:47:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:47:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:47:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:05 smithi136 ceph-mon[113531]: pgmap v8805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:06.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/144097055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:06.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:05.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:05.971597+0000) 2023-08-28T01:47:06.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:05 smithi093 ceph-mon[118902]: pgmap v8805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/144097055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:05 smithi093 ceph-mon[111335]: pgmap v8805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/144097055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:06.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:06.971856+0000) 2023-08-28T01:47:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:07 smithi136 ceph-mon[113531]: pgmap v8806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:07 smithi093 ceph-mon[111335]: pgmap v8806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:07 smithi093 ceph-mon[118902]: pgmap v8806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:07.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:07.972137+0000) 2023-08-28T01:47:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4009671847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4009671847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4009671847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:08.971+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:08.972390+0000) 2023-08-28T01:47:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:09 smithi136 ceph-mon[113531]: pgmap v8807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:09 smithi093 ceph-mon[111335]: pgmap v8807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:09 smithi093 ceph-mon[118902]: pgmap v8807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:09.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:09.972595+0000) 2023-08-28T01:47:10.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/504525728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:10.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/504525728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/504525728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:10.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:10.972874+0000) 2023-08-28T01:47:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:11 smithi093 ceph-mon[118902]: pgmap v8808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:11 smithi093 ceph-mon[111335]: pgmap v8808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:11.387Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:11.388Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:11 smithi136 ceph-mon[113531]: pgmap v8808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:11.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:11.973168+0000) 2023-08-28T01:47:12.826 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/989020735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:12.827 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/989020735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:12.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/989020735' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:12.828Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:12.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:12.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:12.973403+0000) 2023-08-28T01:47:13.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:13 smithi136 ceph-mon[113531]: pgmap v8809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:13 smithi093 ceph-mon[111335]: pgmap v8809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:13 smithi093 ceph-mon[118902]: pgmap v8809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:13.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:13.973625+0000) 2023-08-28T01:47:15.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2932817738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:14.972+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:14.973815+0000) 2023-08-28T01:47:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2932817738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2932817738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:47:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:47:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:47:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:15 smithi136 ceph-mon[113531]: pgmap v8810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:16.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:15.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:15.974113+0000) 2023-08-28T01:47:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:15 smithi093 ceph-mon[111335]: pgmap v8810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:15 smithi093 ceph-mon[118902]: pgmap v8810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:16.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:16.974369+0000) 2023-08-28T01:47:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:17 smithi093 ceph-mon[111335]: pgmap v8811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1326050110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:17 smithi093 ceph-mon[118902]: pgmap v8811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:17.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1326050110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:17 smithi136 ceph-mon[113531]: pgmap v8811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1326050110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:17.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:17.974649+0000) 2023-08-28T01:47:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:18.973+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:18.974883+0000) 2023-08-28T01:47:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:19 smithi136 ceph-mon[113531]: pgmap v8812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/666991017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:19.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:19.975141+0000) 2023-08-28T01:47:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:19 smithi093 ceph-mon[111335]: pgmap v8812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/666991017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:19 smithi093 ceph-mon[118902]: pgmap v8812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/666991017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:20.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:20.975399+0000) 2023-08-28T01:47:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:21.387Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:21.388Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:21 smithi136 ceph-mon[113531]: pgmap v8813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:21.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:21.975687+0000) 2023-08-28T01:47:22.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:21 smithi093 ceph-mon[118902]: pgmap v8813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:21 smithi093 ceph-mon[111335]: pgmap v8813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:23.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1118931063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:22.974+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:22.975940+0000) 2023-08-28T01:47:23.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1118931063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1118931063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:22.829Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:22.829Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:23 smithi136 ceph-mon[113531]: pgmap v8814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:23.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:23.976195+0000) 2023-08-28T01:47:24.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:23 smithi093 ceph-mon[111335]: pgmap v8814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:24.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:23 smithi093 ceph-mon[118902]: pgmap v8814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3483518114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:24.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:24.976390+0000) 2023-08-28T01:47:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3483518114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3483518114' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:47:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:47:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:47:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:25 smithi136 ceph-mon[113531]: pgmap v8815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:25.975+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:25.976668+0000) 2023-08-28T01:47:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:25 smithi093 ceph-mon[111335]: pgmap v8815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:25 smithi093 ceph-mon[118902]: pgmap v8815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:26.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:26.976920+0000) 2023-08-28T01:47:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3833353121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:27 smithi136 ceph-mon[113531]: pgmap v8816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3833353121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:27 smithi093 ceph-mon[118902]: pgmap v8816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3833353121' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:27 smithi093 ceph-mon[111335]: pgmap v8816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:27.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:27.977151+0000) 2023-08-28T01:47:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:28.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:28.977375+0000) 2023-08-28T01:47:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:29 smithi136 ceph-mon[113531]: pgmap v8817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/546822342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:29.976+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:29.977639+0000) 2023-08-28T01:47:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:29 smithi093 ceph-mon[111335]: pgmap v8817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/546822342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:29 smithi093 ceph-mon[118902]: pgmap v8817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/546822342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:30.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:30.977903+0000) 2023-08-28T01:47:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:31.388Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:31.389Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:31 smithi136 ceph-mon[113531]: pgmap v8818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3907051576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:31.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:31.978173+0000) 2023-08-28T01:47:32.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:31 smithi093 ceph-mon[111335]: pgmap v8818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3907051576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:31 smithi093 ceph-mon[118902]: pgmap v8818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3907051576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:32.830Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:32.831Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:32.977+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:32.978425+0000) 2023-08-28T01:47:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:33 smithi136 ceph-mon[113531]: pgmap v8819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:33.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:33.978674+0000) 2023-08-28T01:47:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:33 smithi093 ceph-mon[111335]: pgmap v8819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:33 smithi093 ceph-mon[118902]: pgmap v8819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:35.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/915301393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:34.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:34.978905+0000) 2023-08-28T01:47:35.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/915301393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/915301393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:47:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:47:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:47:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:35 smithi136 ceph-mon[113531]: pgmap v8820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:35.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:35.979160+0000) 2023-08-28T01:47:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:35 smithi093 ceph-mon[111335]: pgmap v8820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:35 smithi093 ceph-mon[118902]: pgmap v8820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:37.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1237772166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:36.978+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:36.979394+0000) 2023-08-28T01:47:37.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1237772166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1237772166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:37 smithi136 ceph-mon[113531]: pgmap v8821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:37.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:37.979649+0000) 2023-08-28T01:47:38.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:37 smithi093 ceph-mon[111335]: pgmap v8821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:37 smithi093 ceph-mon[118902]: pgmap v8821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2612712653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:38.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:38.979871+0000) 2023-08-28T01:47:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2612712653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2612712653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:39 smithi136 ceph-mon[113531]: pgmap v8822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:39.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:39.980154+0000) 2023-08-28T01:47:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:39 smithi093 ceph-mon[111335]: pgmap v8822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:39 smithi093 ceph-mon[118902]: pgmap v8822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:40.979+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:40.980441+0000) 2023-08-28T01:47:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:41.388Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:41.389Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:41 smithi136 ceph-mon[113531]: pgmap v8823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3101821890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:41.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:41.980662+0000) 2023-08-28T01:47:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:41 smithi093 ceph-mon[111335]: pgmap v8823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3101821890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:41 smithi093 ceph-mon[118902]: pgmap v8823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3101821890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:42.831Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:42.832Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:42.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:42.980877+0000) 2023-08-28T01:47:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:43 smithi136 ceph-mon[113531]: pgmap v8824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1288528760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:43.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:43.981150+0000) 2023-08-28T01:47:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:43 smithi093 ceph-mon[111335]: pgmap v8824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1288528760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:43 smithi093 ceph-mon[118902]: pgmap v8824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1288528760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:44.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:44.981407+0000) 2023-08-28T01:47:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:47:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:47:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:47:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:45 smithi136 ceph-mon[113531]: pgmap v8825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/532479320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:45.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:45.981662+0000) 2023-08-28T01:47:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:45 smithi093 ceph-mon[111335]: pgmap v8825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/532479320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:45 smithi093 ceph-mon[118902]: pgmap v8825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/532479320' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:46.980+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:46.981922+0000) 2023-08-28T01:47:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:47 smithi136 ceph-mon[113531]: pgmap v8826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:47 smithi093 ceph-mon[111335]: pgmap v8826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:47 smithi093 ceph-mon[118902]: pgmap v8826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:47.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:47.982184+0000) 2023-08-28T01:47:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2154069873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2154069873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2154069873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:48.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:48.982391+0000) 2023-08-28T01:47:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:49 smithi136 ceph-mon[113531]: pgmap v8827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:49 smithi093 ceph-mon[111335]: pgmap v8827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:49 smithi093 ceph-mon[118902]: pgmap v8827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:49.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:49.982644+0000) 2023-08-28T01:47:50.755 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:47:50.756 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:47:50.917 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:47:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:50.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:50.982826+0000) 2023-08-28T01:47:51.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/616969589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:51.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:51 smithi136 ceph-mon[113531]: pgmap v8828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:51.713 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/616969589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:51.713 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:51 smithi093 ceph-mon[118902]: pgmap v8828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:51.714 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/616969589' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:51.714 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:51 smithi093 ceph-mon[111335]: pgmap v8828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:51.714 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:51.389Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:51.714 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:51.390Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:52.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:51.981+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:51.983022+0000) 2023-08-28T01:47:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:52 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/371056988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:52.982+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:52.983182+0000) 2023-08-28T01:47:53.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:53.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/371056988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/371056988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:52.831Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:47:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:47:52.832Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:47:54.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:53.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:53.983386+0000) 2023-08-28T01:47:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-mon[113531]: pgmap v8829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:47:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:47:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:54.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[118902]: pgmap v8829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[111335]: pgmap v8829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:47:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:47:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:47:54.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:47:55.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:54.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:54.983612+0000) 2023-08-28T01:47:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:47:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:47:55] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:47:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:55 smithi136 ceph-mon[113531]: pgmap v8830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3062475533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:55.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:55.983872+0000) 2023-08-28T01:47:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:55 smithi093 ceph-mon[118902]: pgmap v8830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3062475533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:55 smithi093 ceph-mon[111335]: pgmap v8830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3062475533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:57.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:56.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:56.984142+0000) 2023-08-28T01:47:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:57 smithi136 ceph-mon[113531]: pgmap v8831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:57 smithi093 ceph-mon[111335]: pgmap v8831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:57 smithi093 ceph-mon[118902]: pgmap v8831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:57.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:57.984384+0000) 2023-08-28T01:47:58.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3426607507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:58.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3426607507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3426607507' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:47:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:58.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:58.984608+0000) 2023-08-28T01:47:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:47:59 smithi136 ceph-mon[113531]: pgmap v8832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:59.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:47:59 smithi093 ceph-mon[111335]: pgmap v8832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:47:59.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:47:59 smithi093 ceph-mon[118902]: pgmap v8832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:47:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:47:59.983+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:47:59.984850+0000) 2023-08-28T01:48:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/901676569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/901676569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/901676569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:00.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:00.985084+0000) 2023-08-28T01:48:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:01 smithi093 ceph-mon[111335]: pgmap v8833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:01 smithi093 ceph-mon[118902]: pgmap v8833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:01.389Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:01.391Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:01 smithi136 ceph-mon[113531]: pgmap v8833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:01.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:01.985358+0000) 2023-08-28T01:48:02.831 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2308814142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:02.831 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2308814142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:02.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2308814142' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:02.832Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:02.833Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:02.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:02.985612+0000) 2023-08-28T01:48:03.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:03 smithi136 ceph-mon[113531]: pgmap v8834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:03.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:03 smithi093 ceph-mon[111335]: pgmap v8834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:03 smithi093 ceph-mon[118902]: pgmap v8834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:03.984+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:03.985868+0000) 2023-08-28T01:48:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:04.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:04.986142+0000) 2023-08-28T01:48:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:48:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:48:05] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:48:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:05 smithi136 ceph-mon[113531]: pgmap v8835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2735257565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:05.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:05.986418+0000) 2023-08-28T01:48:06.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:05 smithi093 ceph-mon[118902]: pgmap v8835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2735257565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:05 smithi093 ceph-mon[111335]: pgmap v8835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2735257565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:06.985+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:06.986674+0000) 2023-08-28T01:48:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:07 smithi136 ceph-mon[113531]: pgmap v8836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/454651901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:07 smithi093 ceph-mon[111335]: pgmap v8836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/454651901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:07 smithi093 ceph-mon[118902]: pgmap v8836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/454651901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:07.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:07.986923+0000) 2023-08-28T01:48:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:08.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:08.987180+0000) 2023-08-28T01:48:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:09.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:09.987438+0000) 2023-08-28T01:48:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:09 smithi136 ceph-mon[113531]: pgmap v8837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2628131090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:10.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:09 smithi093 ceph-mon[118902]: pgmap v8837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:10.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2628131090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:09 smithi093 ceph-mon[111335]: pgmap v8837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2628131090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:10.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:10.987713+0000) 2023-08-28T01:48:11.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:11.390Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:11.391Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:11 smithi136 ceph-mon[113531]: pgmap v8838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:11.986+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:11.987965+0000) 2023-08-28T01:48:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:11 smithi093 ceph-mon[111335]: pgmap v8838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:11 smithi093 ceph-mon[118902]: pgmap v8838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:13.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1003270654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:12.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:12.988213+0000) 2023-08-28T01:48:13.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1003270654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1003270654' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:12.832Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:12.833Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:13 smithi136 ceph-mon[113531]: pgmap v8839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:13.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:13.988437+0000) 2023-08-28T01:48:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:13 smithi093 ceph-mon[111335]: pgmap v8839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:13 smithi093 ceph-mon[118902]: pgmap v8839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:15.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/689924439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:14.987+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:14.988690+0000) 2023-08-28T01:48:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/689924439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/689924439' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:48:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:48:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:48:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:15 smithi136 ceph-mon[113531]: pgmap v8840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:16.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:15.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:15.988903+0000) 2023-08-28T01:48:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:15 smithi093 ceph-mon[111335]: pgmap v8840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:15 smithi093 ceph-mon[118902]: pgmap v8840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:16.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:16.989179+0000) 2023-08-28T01:48:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1166978718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:17 smithi136 ceph-mon[113531]: pgmap v8841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1166978718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:17 smithi093 ceph-mon[111335]: pgmap v8841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1166978718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:17 smithi093 ceph-mon[118902]: pgmap v8841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:17.988+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:17.989439+0000) 2023-08-28T01:48:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:18.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:18.989710+0000) 2023-08-28T01:48:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:19 smithi136 ceph-mon[113531]: pgmap v8842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1402913351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:19.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:19.989932+0000) 2023-08-28T01:48:20.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:19 smithi093 ceph-mon[111335]: pgmap v8842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1402913351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:19 smithi093 ceph-mon[118902]: pgmap v8842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1402913351' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:20.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:20.990174+0000) 2023-08-28T01:48:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:21.390Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:21.391Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:21 smithi136 ceph-mon[113531]: pgmap v8843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2970991745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:21.989+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:21.990412+0000) 2023-08-28T01:48:22.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:21 smithi093 ceph-mon[118902]: pgmap v8843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:22.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2970991745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:21 smithi093 ceph-mon[111335]: pgmap v8843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2970991745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:22.833Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:22.834Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:22.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:22.990665+0000) 2023-08-28T01:48:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:23 smithi136 ceph-mon[113531]: pgmap v8844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:23.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:23.990895+0000) 2023-08-28T01:48:24.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:23 smithi093 ceph-mon[111335]: pgmap v8844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:23 smithi093 ceph-mon[118902]: pgmap v8844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3175992163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:24.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:24.991118+0000) 2023-08-28T01:48:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3175992163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3175992163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:48:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:48:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:48:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:25 smithi136 ceph-mon[113531]: pgmap v8845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:25.990+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:25.991337+0000) 2023-08-28T01:48:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:25 smithi093 ceph-mon[111335]: pgmap v8845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:25 smithi093 ceph-mon[118902]: pgmap v8845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:27.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2572574221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:26.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:26.991539+0000) 2023-08-28T01:48:27.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2572574221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2572574221' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:27 smithi136 ceph-mon[113531]: pgmap v8846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:28.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:27.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:27.991789+0000) 2023-08-28T01:48:28.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:27 smithi093 ceph-mon[111335]: pgmap v8846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:27 smithi093 ceph-mon[118902]: pgmap v8846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4250274254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:28.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:28.992079+0000) 2023-08-28T01:48:29.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4250274254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:29.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4250274254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:29 smithi136 ceph-mon[113531]: pgmap v8847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:29.991+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:29.992344+0000) 2023-08-28T01:48:30.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:29 smithi093 ceph-mon[111335]: pgmap v8847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:29 smithi093 ceph-mon[118902]: pgmap v8847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:30.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:30.992611+0000) 2023-08-28T01:48:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:31.391Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:31.392Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:31 smithi136 ceph-mon[113531]: pgmap v8848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3460118731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:31.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:31.992877+0000) 2023-08-28T01:48:32.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:31 smithi093 ceph-mon[118902]: pgmap v8848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3460118731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:31 smithi093 ceph-mon[111335]: pgmap v8848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3460118731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:33.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:32.833Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:32.834Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:32.992+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:32.993158+0000) 2023-08-28T01:48:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:33 smithi136 ceph-mon[113531]: pgmap v8849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/307995251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:33.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:33.993396+0000) 2023-08-28T01:48:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:33 smithi093 ceph-mon[111335]: pgmap v8849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/307995251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:33 smithi093 ceph-mon[118902]: pgmap v8849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/307995251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:34.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:34.993616+0000) 2023-08-28T01:48:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:48:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:48:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:48:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:35 smithi136 ceph-mon[113531]: pgmap v8850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/198247250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:35.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:35.993841+0000) 2023-08-28T01:48:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:35 smithi093 ceph-mon[111335]: pgmap v8850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/198247250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:35 smithi093 ceph-mon[118902]: pgmap v8850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/198247250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:36.993+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:36.994074+0000) 2023-08-28T01:48:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:37 smithi136 ceph-mon[113531]: pgmap v8851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:37 smithi093 ceph-mon[111335]: pgmap v8851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:37 smithi093 ceph-mon[118902]: pgmap v8851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:37.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:37.994306+0000) 2023-08-28T01:48:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/827306157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:38.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/827306157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/827306157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:38.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:38.994544+0000) 2023-08-28T01:48:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:39 smithi136 ceph-mon[113531]: pgmap v8852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:39 smithi093 ceph-mon[111335]: pgmap v8852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:39 smithi093 ceph-mon[118902]: pgmap v8852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:39.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:39.994812+0000) 2023-08-28T01:48:40.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2049181851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:40.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2049181851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:40.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2049181851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:40.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:40.995088+0000) 2023-08-28T01:48:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:41 smithi093 ceph-mon[111335]: pgmap v8853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:41 smithi093 ceph-mon[118902]: pgmap v8853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:41.391Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:41.392Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:41 smithi136 ceph-mon[113531]: pgmap v8853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:41.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:41.995332+0000) 2023-08-28T01:48:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:42.834Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:42.835Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:42.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:42.995589+0000) 2023-08-28T01:48:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:43 smithi136 ceph-mon[113531]: pgmap v8854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2211486545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:43.994+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:43.995850+0000) 2023-08-28T01:48:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:43 smithi093 ceph-mon[111335]: pgmap v8854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2211486545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:43 smithi093 ceph-mon[118902]: pgmap v8854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2211486545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:44.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:44.996123+0000) 2023-08-28T01:48:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:48:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:48:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:48:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:45 smithi136 ceph-mon[113531]: pgmap v8855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2010825016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:45.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:45.996351+0000) 2023-08-28T01:48:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:45 smithi093 ceph-mon[111335]: pgmap v8855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2010825016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:45 smithi093 ceph-mon[118902]: pgmap v8855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2010825016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:46.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:46.996612+0000) 2023-08-28T01:48:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:47 smithi136 ceph-mon[113531]: pgmap v8856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:47 smithi093 ceph-mon[111335]: pgmap v8856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:47 smithi093 ceph-mon[118902]: pgmap v8856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:47.995+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:47.996877+0000) 2023-08-28T01:48:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2248920923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:48.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2248920923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2248920923' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:48.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:48.997149+0000) 2023-08-28T01:48:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:49 smithi136 ceph-mon[113531]: pgmap v8857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:49.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:49 smithi093 ceph-mon[111335]: pgmap v8857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:49 smithi093 ceph-mon[118902]: pgmap v8857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:49.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:49.997390+0000) 2023-08-28T01:48:50.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/847547274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/847547274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/847547274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:50.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:50.997577+0000) 2023-08-28T01:48:51.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:51 smithi093 ceph-mon[111335]: pgmap v8858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:51 smithi093 ceph-mon[118902]: pgmap v8858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:51.392Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:51.393Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:51 smithi136 ceph-mon[113531]: pgmap v8858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:51.996+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:51.997793+0000) 2023-08-28T01:48:52.833 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1502817867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:52.834 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1502817867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:52.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1502817867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:52.835Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:48:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:48:52.836Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:48:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:52.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:52.998065+0000) 2023-08-28T01:48:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:53 smithi136 ceph-mon[113531]: pgmap v8859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:48:53.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:53 smithi093 ceph-mon[111335]: pgmap v8859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:53.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:48:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:53 smithi093 ceph-mon[118902]: pgmap v8859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:48:54.410 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:53.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:53.998228+0000) 2023-08-28T01:48:54.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:54.976 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:54.976 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:48:55.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:54.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:54.998361+0000) 2023-08-28T01:48:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[118902]: pgmap v8860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2971403037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[111335]: pgmap v8860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2971403037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:48:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:48:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:48:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:55 smithi136 ceph-mon[113531]: pgmap v8860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2971403037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:55 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:55.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:55.998510+0000) 2023-08-28T01:48:56.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2407M 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:48:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2407M 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:48:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2407M 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:48:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:48:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:56.997+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:56.998752+0000) 2023-08-28T01:48:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:57 smithi136 ceph-mon[113531]: pgmap v8861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2638662091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:57 smithi093 ceph-mon[111335]: pgmap v8861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2638662091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:57 smithi093 ceph-mon[118902]: pgmap v8861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:48:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2638662091' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:48:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:57.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:57.999060+0000) 2023-08-28T01:48:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:58.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:58.999291+0000) 2023-08-28T01:49:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:59 smithi136 ceph-mon[113531]: pgmap v8862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:48:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3759799517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:48:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:48:59.998+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:48:59.999560+0000) 2023-08-28T01:49:00.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:59 smithi093 ceph-mon[111335]: pgmap v8862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:48:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3759799517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:59 smithi093 ceph-mon[118902]: pgmap v8862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:48:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3759799517' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:00.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:00.999817+0000) 2023-08-28T01:49:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:01.393Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:01.394Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:01 smithi136 ceph-mon[113531]: pgmap v8863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:01.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:02.000019+0000) 2023-08-28T01:49:02.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:01 smithi093 ceph-mon[111335]: pgmap v8863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:01 smithi093 ceph-mon[118902]: pgmap v8863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/776405746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:02.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:03.000271+0000) 2023-08-28T01:49:03.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/776405746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/776405746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:02.836Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:02.837Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:03 smithi136 ceph-mon[113531]: pgmap v8864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:03.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:04.000518+0000) 2023-08-28T01:49:04.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:03 smithi093 ceph-mon[111335]: pgmap v8864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:03 smithi093 ceph-mon[118902]: pgmap v8864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:05.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2910644528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:04.999+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:05.000727+0000) 2023-08-28T01:49:05.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2910644528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2910644528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:49:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:49:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:49:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:05 smithi136 ceph-mon[113531]: pgmap v8865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:06.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:06.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:06.000988+0000) 2023-08-28T01:49:06.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:05 smithi093 ceph-mon[111335]: pgmap v8865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:05 smithi093 ceph-mon[118902]: pgmap v8865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:07.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:07.001251+0000) 2023-08-28T01:49:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:07 smithi136 ceph-mon[113531]: pgmap v8866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3104261641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:07 smithi093 ceph-mon[111335]: pgmap v8866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3104261641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:07 smithi093 ceph-mon[118902]: pgmap v8866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3104261641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:08.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:08.001493+0000) 2023-08-28T01:49:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:09.000+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:09.001751+0000) 2023-08-28T01:49:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:09 smithi136 ceph-mon[113531]: pgmap v8867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/478404544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:10.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:10.002004+0000) 2023-08-28T01:49:10.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:09 smithi093 ceph-mon[118902]: pgmap v8867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/478404544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:09 smithi093 ceph-mon[111335]: pgmap v8867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/478404544' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:11.001+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:11.002237+0000) 2023-08-28T01:49:11.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:11.394Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:11.395Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:11 smithi136 ceph-mon[113531]: pgmap v8868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1419057133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:12.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:12.002530+0000) 2023-08-28T01:49:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:11 smithi093 ceph-mon[118902]: pgmap v8868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1419057133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:11 smithi093 ceph-mon[111335]: pgmap v8868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1419057133' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:12.837Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:12.838Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:13.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:13.002774+0000) 2023-08-28T01:49:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:13 smithi136 ceph-mon[113531]: pgmap v8869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:14.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:14.003088+0000) 2023-08-28T01:49:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:13 smithi093 ceph-mon[111335]: pgmap v8869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:13 smithi093 ceph-mon[118902]: pgmap v8869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:15.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3942996566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:15.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:15.002+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:15.003353+0000) 2023-08-28T01:49:15.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3942996566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3942996566' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:49:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:49:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:49:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:15 smithi136 ceph-mon[113531]: pgmap v8870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:16.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:16.003580+0000) 2023-08-28T01:49:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:15 smithi093 ceph-mon[111335]: pgmap v8870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:15 smithi093 ceph-mon[118902]: pgmap v8870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:17.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/422517603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:17.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:17.003826+0000) 2023-08-28T01:49:17.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/422517603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:17.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/422517603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:17 smithi136 ceph-mon[113531]: pgmap v8871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:18.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:18.004126+0000) 2023-08-28T01:49:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:17 smithi093 ceph-mon[111335]: pgmap v8871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:17 smithi093 ceph-mon[118902]: pgmap v8871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2777841984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:19.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:19.004383+0000) 2023-08-28T01:49:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2777841984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2777841984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:19 smithi136 ceph-mon[113531]: pgmap v8872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:20.003+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:20.004602+0000) 2023-08-28T01:49:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:19 smithi093 ceph-mon[111335]: pgmap v8872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:19 smithi093 ceph-mon[118902]: pgmap v8872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:21.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:21.004849+0000) 2023-08-28T01:49:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:21.395Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:21.396Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:21 smithi136 ceph-mon[113531]: pgmap v8873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/791529233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:22.004+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:22.005133+0000) 2023-08-28T01:49:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:21 smithi093 ceph-mon[111335]: pgmap v8873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/791529233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:21 smithi093 ceph-mon[118902]: pgmap v8873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/791529233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:22.838Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:22.839Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:23.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:23.005372+0000) 2023-08-28T01:49:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:23 smithi136 ceph-mon[113531]: pgmap v8874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2201708163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:24.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:24.005607+0000) 2023-08-28T01:49:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:23 smithi093 ceph-mon[111335]: pgmap v8874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2201708163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:23 smithi093 ceph-mon[118902]: pgmap v8874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2201708163' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:25.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:25.005850+0000) 2023-08-28T01:49:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:49:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:49:25] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:49:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:25 smithi136 ceph-mon[113531]: pgmap v8875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:26.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:26.006142+0000) 2023-08-28T01:49:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:25 smithi093 ceph-mon[111335]: pgmap v8875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:25 smithi093 ceph-mon[118902]: pgmap v8875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:27.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3967409249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:27.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:27.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:27.006395+0000) 2023-08-28T01:49:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3967409249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:27.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3967409249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:27 smithi136 ceph-mon[113531]: pgmap v8876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:28.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:28.006651+0000) 2023-08-28T01:49:28.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:27 smithi093 ceph-mon[111335]: pgmap v8876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:27 smithi093 ceph-mon[118902]: pgmap v8876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:29.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1710386217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:29.005+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:29.006885+0000) 2023-08-28T01:49:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1710386217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:29.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1710386217' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:29 smithi136 ceph-mon[113531]: pgmap v8877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:30.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:30.007155+0000) 2023-08-28T01:49:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:29 smithi093 ceph-mon[111335]: pgmap v8877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:29 smithi093 ceph-mon[118902]: pgmap v8877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:31.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/330654178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:31.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:31.007355+0000) 2023-08-28T01:49:31.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/330654178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/330654178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:31.396Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:31.397Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:31 smithi136 ceph-mon[113531]: pgmap v8878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:32.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:32.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:32.007612+0000) 2023-08-28T01:49:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:31 smithi093 ceph-mon[111335]: pgmap v8878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:31 smithi093 ceph-mon[118902]: pgmap v8878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:32.839Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:32.840Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:33.006+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:33.007833+0000) 2023-08-28T01:49:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:33 smithi136 ceph-mon[113531]: pgmap v8879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4116246494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:34.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:34.008091+0000) 2023-08-28T01:49:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:33 smithi093 ceph-mon[111335]: pgmap v8879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4116246494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:33 smithi093 ceph-mon[118902]: pgmap v8879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4116246494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:35.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:35.008313+0000) 2023-08-28T01:49:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:49:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:49:35] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:49:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:35 smithi136 ceph-mon[113531]: pgmap v8880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3506445592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:36.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:36.008541+0000) 2023-08-28T01:49:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:35 smithi093 ceph-mon[111335]: pgmap v8880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3506445592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:35 smithi093 ceph-mon[118902]: pgmap v8880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3506445592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:37.007+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:37.008801+0000) 2023-08-28T01:49:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:37 smithi136 ceph-mon[113531]: pgmap v8881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:37.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:37 smithi093 ceph-mon[111335]: pgmap v8881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:37 smithi093 ceph-mon[118902]: pgmap v8881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:38.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:38.009106+0000) 2023-08-28T01:49:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3691826052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:38.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3691826052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3691826052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:39.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:39.009339+0000) 2023-08-28T01:49:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:39 smithi136 ceph-mon[113531]: pgmap v8882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:39 smithi093 ceph-mon[111335]: pgmap v8882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:39 smithi093 ceph-mon[118902]: pgmap v8882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:40.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:40.009542+0000) 2023-08-28T01:49:40.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1662736736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:40.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1662736736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:40.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1662736736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:41.008+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:41.009819+0000) 2023-08-28T01:49:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:41 smithi093 ceph-mon[111335]: pgmap v8883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:41 smithi093 ceph-mon[118902]: pgmap v8883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:41.397Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:41.398Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:41 smithi136 ceph-mon[113531]: pgmap v8883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:42.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:42.010097+0000) 2023-08-28T01:49:42.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2138027447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:42.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2138027447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2138027447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:42.840Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:42.841Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:43.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:43.010348+0000) 2023-08-28T01:49:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:43 smithi136 ceph-mon[113531]: pgmap v8884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:43.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:43 smithi093 ceph-mon[111335]: pgmap v8884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:43 smithi093 ceph-mon[118902]: pgmap v8884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:44.009+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:44.010608+0000) 2023-08-28T01:49:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:45.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:45.010842+0000) 2023-08-28T01:49:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:49:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:49:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:49:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:45 smithi136 ceph-mon[113531]: pgmap v8885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1678741508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:46.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:46.011115+0000) 2023-08-28T01:49:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:45 smithi093 ceph-mon[118902]: pgmap v8885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1678741508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:45 smithi093 ceph-mon[111335]: pgmap v8885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1678741508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:47.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:47.011325+0000) 2023-08-28T01:49:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:47 smithi136 ceph-mon[113531]: pgmap v8886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2737915582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:47 smithi093 ceph-mon[111335]: pgmap v8886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2737915582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:47 smithi093 ceph-mon[118902]: pgmap v8886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2737915582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:48.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:48.011593+0000) 2023-08-28T01:49:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:49.010+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:49.011831+0000) 2023-08-28T01:49:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:49 smithi136 ceph-mon[113531]: pgmap v8887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2419215982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:50.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:50.012096+0000) 2023-08-28T01:49:50.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:49 smithi093 ceph-mon[111335]: pgmap v8887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:50.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2419215982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:49 smithi093 ceph-mon[118902]: pgmap v8887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:50.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2419215982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:51.011+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:51.012384+0000) 2023-08-28T01:49:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:51.398Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:51.399Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:51 smithi136 ceph-mon[113531]: pgmap v8888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:52.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:52.012611+0000) 2023-08-28T01:49:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:51 smithi093 ceph-mon[111335]: pgmap v8888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:51 smithi093 ceph-mon[118902]: pgmap v8888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2698350473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:53.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:53.012854+0000) 2023-08-28T01:49:53.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2698350473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2698350473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:52.840Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:49:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:49:52.841Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:49:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:53 smithi136 ceph-mon[113531]: pgmap v8889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:54.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:54.013132+0000) 2023-08-28T01:49:54.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:53 smithi093 ceph-mon[111335]: pgmap v8889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:53 smithi093 ceph-mon[118902]: pgmap v8889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1191233811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:55.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:55.013385+0000) 2023-08-28T01:49:55.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1191233811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:49:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1191233811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:49:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:49:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:49:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:55 smithi136 ceph-mon[113531]: pgmap v8890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:56.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:56.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:56.013653+0000) 2023-08-28T01:49:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:55 smithi093 ceph-mon[111335]: pgmap v8890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:55 smithi093 ceph-mon[118902]: pgmap v8890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:57.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:49:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:57.012+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:57.013880+0000) 2023-08-28T01:49:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:49:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:49:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:57 smithi136 ceph-mon[113531]: pgmap v8891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1834027663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:58.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:58.014071+0000) 2023-08-28T01:49:58.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:57 smithi093 ceph-mon[111335]: pgmap v8891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1834027663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:57 smithi093 ceph-mon[118902]: pgmap v8891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1834027663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:49:59.013+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:49:59.014209+0000) 2023-08-28T01:49:59.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: pgmap v8892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3457310893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: pgmap v8892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:49:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3457310893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:49:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: pgmap v8892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:49:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:49:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:49:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:49:59.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:49:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3457310893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:00.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:00.014434+0000) 2023-08-28T01:50:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:50:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:50:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T01:50:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:01.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:01.014680+0000) 2023-08-28T01:50:01.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:01 smithi093 ceph-mon[111335]: pgmap v8893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:01 smithi093 ceph-mon[118902]: pgmap v8893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:01.399Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:01.401Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:01 smithi136 ceph-mon[113531]: pgmap v8893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:02.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:02.014947+0000) 2023-08-28T01:50:02.839 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1078021017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:02.840 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1078021017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:02.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1078021017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:02.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:02.842Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:03.014+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:03.015200+0000) 2023-08-28T01:50:03.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:03 smithi136 ceph-mon[113531]: pgmap v8894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:03.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:03 smithi093 ceph-mon[111335]: pgmap v8894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:03 smithi093 ceph-mon[118902]: pgmap v8894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:04.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:04.015472+0000) 2023-08-28T01:50:04.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2328730343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:04.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2328730343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:04.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2328730343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:05.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:05.015771+0000) 2023-08-28T01:50:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:05 smithi093 ceph-mon[111335]: pgmap v8895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:05 smithi093 ceph-mon[118902]: pgmap v8895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:50:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:50:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:50:05.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:05 smithi136 ceph-mon[113531]: pgmap v8895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:06.015+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:06.016052+0000) 2023-08-28T01:50:06.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2086825758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:06.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2086825758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:06.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2086825758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:07.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:07.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:07.016316+0000) 2023-08-28T01:50:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:07 smithi136 ceph-mon[113531]: pgmap v8896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:07 smithi093 ceph-mon[111335]: pgmap v8896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:07 smithi093 ceph-mon[118902]: pgmap v8896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:08.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:08.016615+0000) 2023-08-28T01:50:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/98526513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:09.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:09.016821+0000) 2023-08-28T01:50:09.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/98526513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/98526513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:09 smithi136 ceph-mon[113531]: pgmap v8897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:10.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:10.017111+0000) 2023-08-28T01:50:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:09 smithi093 ceph-mon[111335]: pgmap v8897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:09 smithi093 ceph-mon[118902]: pgmap v8897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:11.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:11.017388+0000) 2023-08-28T01:50:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:11.401Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:11.401Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:12.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:12.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:12.017670+0000) 2023-08-28T01:50:12.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:11 smithi136 ceph-mon[113531]: pgmap v8898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:12.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3454016995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:11 smithi093 ceph-mon[111335]: pgmap v8898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3454016995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:11 smithi093 ceph-mon[118902]: pgmap v8898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3454016995' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:12.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:12.842Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:13.016+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:13.017897+0000) 2023-08-28T01:50:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:13 smithi136 ceph-mon[113531]: pgmap v8899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2520385911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:14.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:14.018122+0000) 2023-08-28T01:50:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:13 smithi093 ceph-mon[111335]: pgmap v8899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2520385911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:13 smithi093 ceph-mon[118902]: pgmap v8899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2520385911' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:15.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:15.018379+0000) 2023-08-28T01:50:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:50:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:50:15] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:50:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:15 smithi136 ceph-mon[113531]: pgmap v8900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:16.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:16.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:16.018637+0000) 2023-08-28T01:50:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:15 smithi093 ceph-mon[111335]: pgmap v8900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:15 smithi093 ceph-mon[118902]: pgmap v8900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:17.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4165144848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:17.017+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:17.018898+0000) 2023-08-28T01:50:17.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4165144848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:17.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4165144848' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:17 smithi136 ceph-mon[113531]: pgmap v8901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:18.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:18.019167+0000) 2023-08-28T01:50:18.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:17 smithi093 ceph-mon[111335]: pgmap v8901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:17 smithi093 ceph-mon[118902]: pgmap v8901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1575232317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:19.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:19.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:19.019361+0000) 2023-08-28T01:50:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1575232317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1575232317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:19 smithi136 ceph-mon[113531]: pgmap v8902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:20.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:20.019635+0000) 2023-08-28T01:50:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:19 smithi093 ceph-mon[111335]: pgmap v8902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:19 smithi093 ceph-mon[118902]: pgmap v8902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2265152420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:21.018+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:21.019871+0000) 2023-08-28T01:50:21.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2265152420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2265152420' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:21.401Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:21.402Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:21 smithi136 ceph-mon[113531]: pgmap v8903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:22.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:22.020138+0000) 2023-08-28T01:50:22.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:21 smithi093 ceph-mon[111335]: pgmap v8903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:21 smithi093 ceph-mon[118902]: pgmap v8903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:22.841Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:22.842Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:23.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:23.020408+0000) 2023-08-28T01:50:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:23 smithi136 ceph-mon[113531]: pgmap v8904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1298034508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:24.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:24.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:24.020669+0000) 2023-08-28T01:50:24.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:23 smithi093 ceph-mon[118902]: pgmap v8904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1298034508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:23 smithi093 ceph-mon[111335]: pgmap v8904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1298034508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:25.751 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:25.019+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:25.020915+0000) 2023-08-28T01:50:25.752 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:25.752 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:25.752 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:50:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:50:25] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:50:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:25 smithi136 ceph-mon[113531]: pgmap v8905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3908158323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:26.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:26.021166+0000) 2023-08-28T01:50:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:25 smithi093 ceph-mon[111335]: pgmap v8905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3908158323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:25 smithi093 ceph-mon[118902]: pgmap v8905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3908158323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:27.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:27.021399+0000) 2023-08-28T01:50:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:27 smithi136 ceph-mon[113531]: pgmap v8906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:27 smithi093 ceph-mon[118902]: pgmap v8906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:27 smithi093 ceph-mon[111335]: pgmap v8906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:28.020+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:28.021647+0000) 2023-08-28T01:50:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1757632218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1757632218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1757632218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:29.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:29.021920+0000) 2023-08-28T01:50:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:29 smithi136 ceph-mon[113531]: pgmap v8907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:29.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:29 smithi093 ceph-mon[111335]: pgmap v8907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:29 smithi093 ceph-mon[118902]: pgmap v8907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:30.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:30.022209+0000) 2023-08-28T01:50:30.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2945273243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2945273243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2945273243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:31.021+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:31.022432+0000) 2023-08-28T01:50:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:31 smithi093 ceph-mon[111335]: pgmap v8908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:31 smithi093 ceph-mon[118902]: pgmap v8908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:31.402Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:31.403Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:31 smithi136 ceph-mon[113531]: pgmap v8908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:32.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:32.022687+0000) 2023-08-28T01:50:32.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1695662455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:32.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1695662455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1695662455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:32.842Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:32.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:33.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:33.022948+0000) 2023-08-28T01:50:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:33 smithi093 ceph-mon[111335]: pgmap v8909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:33 smithi093 ceph-mon[118902]: pgmap v8909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:34.021 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:33 smithi136 ceph-mon[113531]: pgmap v8909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:34.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:34.023221+0000) 2023-08-28T01:50:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:35.022+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:35.023471+0000) 2023-08-28T01:50:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:50:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:50:35] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:50:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:36.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:36.023730+0000) 2023-08-28T01:50:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:35 smithi136 ceph-mon[113531]: pgmap v8910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3255639972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:35 smithi093 ceph-mon[111335]: pgmap v8910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3255639972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:35 smithi093 ceph-mon[118902]: pgmap v8910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3255639972' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:37.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:37.023976+0000) 2023-08-28T01:50:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:37 smithi136 ceph-mon[113531]: pgmap v8911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/112276033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:37 smithi093 ceph-mon[111335]: pgmap v8911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/112276033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:37 smithi093 ceph-mon[118902]: pgmap v8911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/112276033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:38.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:38.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:38.024198+0000) 2023-08-28T01:50:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:39.023+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:39.024448+0000) 2023-08-28T01:50:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:39 smithi136 ceph-mon[113531]: pgmap v8912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1905165608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:40.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:40.024754+0000) 2023-08-28T01:50:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:39 smithi093 ceph-mon[111335]: pgmap v8912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1905165608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:39 smithi093 ceph-mon[118902]: pgmap v8912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1905165608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:41.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:41.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:41.025051+0000) 2023-08-28T01:50:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:41.402Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:41.403Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:41 smithi136 ceph-mon[113531]: pgmap v8913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:42.024+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:42.025273+0000) 2023-08-28T01:50:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:41 smithi093 ceph-mon[111335]: pgmap v8913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:41 smithi093 ceph-mon[118902]: pgmap v8913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:43.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2673054600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:43.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:43.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:43.025515+0000) 2023-08-28T01:50:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2673054600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2673054600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:42.842Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:42.843Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:43 smithi136 ceph-mon[113531]: pgmap v8914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:44.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:44.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:44.025770+0000) 2023-08-28T01:50:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:43 smithi093 ceph-mon[111335]: pgmap v8914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:43 smithi093 ceph-mon[118902]: pgmap v8914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:45.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4225308495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:45.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:45.025+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:45.026056+0000) 2023-08-28T01:50:45.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4225308495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4225308495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:50:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:50:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:50:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:45 smithi136 ceph-mon[113531]: pgmap v8915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:46.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:46.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:46.026340+0000) 2023-08-28T01:50:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:45 smithi093 ceph-mon[111335]: pgmap v8915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:45 smithi093 ceph-mon[118902]: pgmap v8915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:47.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:47.026585+0000) 2023-08-28T01:50:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:47 smithi136 ceph-mon[113531]: pgmap v8916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4273363999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:47 smithi093 ceph-mon[111335]: pgmap v8916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4273363999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:47 smithi093 ceph-mon[118902]: pgmap v8916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4273363999' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:48.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:48.026847+0000) 2023-08-28T01:50:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:49.026+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:49.027134+0000) 2023-08-28T01:50:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:49 smithi136 ceph-mon[113531]: pgmap v8917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/959842134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:50.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:50.027397+0000) 2023-08-28T01:50:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:49 smithi093 ceph-mon[111335]: pgmap v8917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/959842134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:49 smithi093 ceph-mon[118902]: pgmap v8917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/959842134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:51.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:51.027629+0000) 2023-08-28T01:50:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:51.403Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:51.404Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:51 smithi136 ceph-mon[113531]: pgmap v8918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4014893673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:52.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:52.027825+0000) 2023-08-28T01:50:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:51 smithi093 ceph-mon[111335]: pgmap v8918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4014893673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:51 smithi093 ceph-mon[118902]: pgmap v8918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4014893673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:52.843Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:50:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:50:52.844Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:50:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:53.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:53.028088+0000) 2023-08-28T01:50:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:53 smithi136 ceph-mon[113531]: pgmap v8919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:54.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:54.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:54.028313+0000) 2023-08-28T01:50:54.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:53 smithi093 ceph-mon[111335]: pgmap v8919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:53 smithi093 ceph-mon[118902]: pgmap v8919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3807873760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:55.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:55.028571+0000) 2023-08-28T01:50:55.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3807873760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:50:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3807873760' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:50:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:50:55] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:50:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:55 smithi136 ceph-mon[113531]: pgmap v8920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:56.027+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:56.028843+0000) 2023-08-28T01:50:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:55 smithi093 ceph-mon[111335]: pgmap v8920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:55 smithi093 ceph-mon[118902]: pgmap v8920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:57.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2300948261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:57.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:57.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:57.029153+0000) 2023-08-28T01:50:57.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2300948261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2300948261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:57 smithi136 ceph-mon[113531]: pgmap v8921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:58.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:58.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:58.029354+0000) 2023-08-28T01:50:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:57 smithi093 ceph-mon[118902]: pgmap v8921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:57 smithi093 ceph-mon[111335]: pgmap v8921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:50:59.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2467528524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:50:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:50:59.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:50:59.029584+0000) 2023-08-28T01:50:59.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2467528524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:50:59.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2467528524' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:59 smithi136 ceph-mon[113531]: pgmap v8922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:50:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:51:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:00.028+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:00.029904+0000) 2023-08-28T01:51:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:59 smithi093 ceph-mon[111335]: pgmap v8922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:50:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:51:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:59 smithi093 ceph-mon[118902]: pgmap v8922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:50:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:51:01.029 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:01.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:01.030070+0000) 2023-08-28T01:51:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:01.403Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:01.404Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:02.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:02.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:02.030295+0000) 2023-08-28T01:51:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:01 smithi136 ceph-mon[113531]: pgmap v8923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1698535350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[111335]: pgmap v8923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1698535350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[118902]: pgmap v8923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1698535350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:51:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:51:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:03.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:03.030481+0000) 2023-08-28T01:51:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:51:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:51:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:51:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:51:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:51:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:02.844Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:02.845Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:03 smithi136 ceph-mon[113531]: pgmap v8924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3676831997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:04.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:04.030714+0000) 2023-08-28T01:51:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:03 smithi093 ceph-mon[111335]: pgmap v8924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3676831997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:03 smithi093 ceph-mon[118902]: pgmap v8924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3676831997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:05.317 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:05.029+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:05.030884+0000) 2023-08-28T01:51:05.687 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 01:51:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T01:51:05.318681313Z level=info msg="Completed cleanup jobs" duration=13.439291ms 2023-08-28T01:51:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:51:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:51:05] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:51:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:05 smithi136 ceph-mon[113531]: pgmap v8925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:06.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:06.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:06.031085+0000) 2023-08-28T01:51:06.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:05 smithi093 ceph-mon[111335]: pgmap v8925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:05 smithi093 ceph-mon[118902]: pgmap v8925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:07.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1965448514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:07.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:07.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:07.031333+0000) 2023-08-28T01:51:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1965448514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1965448514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:07 smithi136 ceph-mon[113531]: pgmap v8926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:08.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:08.031579+0000) 2023-08-28T01:51:08.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:07 smithi093 ceph-mon[111335]: pgmap v8926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:07 smithi093 ceph-mon[118902]: pgmap v8926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2788339286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:09.030+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:09.031850+0000) 2023-08-28T01:51:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2788339286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2788339286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:09 smithi136 ceph-mon[113531]: pgmap v8927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:10.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:10.032075+0000) 2023-08-28T01:51:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:09 smithi093 ceph-mon[111335]: pgmap v8927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:09 smithi093 ceph-mon[118902]: pgmap v8927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:11.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2294156317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:11.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:11.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:11.032319+0000) 2023-08-28T01:51:11.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2294156317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2294156317' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:11.404Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:11.405Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:11 smithi136 ceph-mon[113531]: pgmap v8928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:12.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:12.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:12.032605+0000) 2023-08-28T01:51:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:11 smithi093 ceph-mon[111335]: pgmap v8928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:11 smithi093 ceph-mon[118902]: pgmap v8928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:12.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:12.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:13.031+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:13.032860+0000) 2023-08-28T01:51:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:13 smithi136 ceph-mon[113531]: pgmap v8929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/870739054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:14.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:14.033135+0000) 2023-08-28T01:51:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:13 smithi093 ceph-mon[111335]: pgmap v8929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/870739054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:13 smithi093 ceph-mon[118902]: pgmap v8929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/870739054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:15.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:15.033356+0000) 2023-08-28T01:51:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:51:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:51:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:51:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:15 smithi136 ceph-mon[113531]: pgmap v8930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3722455113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:16.032+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:16.033556+0000) 2023-08-28T01:51:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:15 smithi093 ceph-mon[111335]: pgmap v8930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3722455113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:15 smithi093 ceph-mon[118902]: pgmap v8930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3722455113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:17.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:17.033842+0000) 2023-08-28T01:51:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:17 smithi136 ceph-mon[113531]: pgmap v8931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:17 smithi093 ceph-mon[111335]: pgmap v8931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:17 smithi093 ceph-mon[118902]: pgmap v8931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:18.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:18.034119+0000) 2023-08-28T01:51:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/505921672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/505921672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/505921672' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:19.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:19.034376+0000) 2023-08-28T01:51:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:19 smithi136 ceph-mon[113531]: pgmap v8932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:19.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:19 smithi093 ceph-mon[111335]: pgmap v8932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:19 smithi093 ceph-mon[118902]: pgmap v8932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:20.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:20.033+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:20.034575+0000) 2023-08-28T01:51:20.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/892998177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/892998177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:20.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/892998177' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:21.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:21.034829+0000) 2023-08-28T01:51:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:21 smithi093 ceph-mon[111335]: pgmap v8933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:21 smithi093 ceph-mon[118902]: pgmap v8933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:21.404Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:21.405Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:21 smithi136 ceph-mon[113531]: pgmap v8933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:22.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:22.035092+0000) 2023-08-28T01:51:22.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1747636201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:22.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1747636201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:22.845Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:22.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:23.033 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1747636201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:23.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:23.035345+0000) 2023-08-28T01:51:23.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:23 smithi093 ceph-mon[111335]: pgmap v8934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:23 smithi093 ceph-mon[118902]: pgmap v8934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:24.034 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:23 smithi136 ceph-mon[113531]: pgmap v8934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:24.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:24.035604+0000) 2023-08-28T01:51:24.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:25.034 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:25.034+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:25.035885+0000) 2023-08-28T01:51:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:51:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:51:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:51:26.034 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:25 smithi136 ceph-mon[113531]: pgmap v8935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:26.034 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3947434545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:25 smithi093 ceph-mon[111335]: pgmap v8935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3947434545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:25 smithi093 ceph-mon[118902]: pgmap v8935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3947434545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:26.035+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:26.036174+0000) 2023-08-28T01:51:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:27.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:27.036383+0000) 2023-08-28T01:51:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:27 smithi136 ceph-mon[113531]: pgmap v8936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3775270865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:27 smithi093 ceph-mon[111335]: pgmap v8936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3775270865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:27 smithi093 ceph-mon[118902]: pgmap v8936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3775270865' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:28.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:28.036594+0000) 2023-08-28T01:51:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:29.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:29.036861+0000) 2023-08-28T01:51:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:29 smithi136 ceph-mon[113531]: pgmap v8937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3240825617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:30.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:30.037142+0000) 2023-08-28T01:51:30.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:29 smithi093 ceph-mon[118902]: pgmap v8937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3240825617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:29 smithi093 ceph-mon[111335]: pgmap v8937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3240825617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:31.036+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:31.037373+0000) 2023-08-28T01:51:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:31.405Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:31.406Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:31 smithi136 ceph-mon[113531]: pgmap v8938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:32.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:32.037628+0000) 2023-08-28T01:51:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:31 smithi093 ceph-mon[111335]: pgmap v8938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:31 smithi093 ceph-mon[118902]: pgmap v8938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:33.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3611310430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:33.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:33.037883+0000) 2023-08-28T01:51:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3611310430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3611310430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:32.846Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:32.846Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:33 smithi136 ceph-mon[113531]: pgmap v8939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:34.037+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:34.038111+0000) 2023-08-28T01:51:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:33 smithi093 ceph-mon[111335]: pgmap v8939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:33 smithi093 ceph-mon[118902]: pgmap v8939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:35.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3847789313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:35.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:35.038317+0000) 2023-08-28T01:51:35.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3847789313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3847789313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:51:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:51:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:51:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:35 smithi136 ceph-mon[113531]: pgmap v8940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:36.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:36.038576+0000) 2023-08-28T01:51:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:35 smithi093 ceph-mon[111335]: pgmap v8940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:35 smithi093 ceph-mon[118902]: pgmap v8940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:37.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:37.038799+0000) 2023-08-28T01:51:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:37 smithi136 ceph-mon[113531]: pgmap v8941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4069688102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:37.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:37 smithi093 ceph-mon[118902]: pgmap v8941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4069688102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:37 smithi093 ceph-mon[111335]: pgmap v8941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4069688102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:38.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:38.039101+0000) 2023-08-28T01:51:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:39.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:39.039382+0000) 2023-08-28T01:51:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:39 smithi136 ceph-mon[113531]: pgmap v8942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3672837319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:40.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:40.039665+0000) 2023-08-28T01:51:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:39 smithi093 ceph-mon[111335]: pgmap v8942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3672837319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:39 smithi093 ceph-mon[118902]: pgmap v8942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3672837319' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:41.038+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:41.039921+0000) 2023-08-28T01:51:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:41.406Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:41.407Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:41 smithi136 ceph-mon[113531]: pgmap v8943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1145399857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:42.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:42.040202+0000) 2023-08-28T01:51:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:41 smithi093 ceph-mon[111335]: pgmap v8943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1145399857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:41 smithi093 ceph-mon[118902]: pgmap v8943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1145399857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:42.846Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:42.847Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:43.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:43.040424+0000) 2023-08-28T01:51:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:43 smithi136 ceph-mon[113531]: pgmap v8944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:44.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:44.040701+0000) 2023-08-28T01:51:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:43 smithi093 ceph-mon[111335]: pgmap v8944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:43 smithi093 ceph-mon[118902]: pgmap v8944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:45.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3857633449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:45.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:45.039+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:45.040963+0000) 2023-08-28T01:51:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3857633449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3857633449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:51:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:51:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:51:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:45 smithi136 ceph-mon[113531]: pgmap v8945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:46.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:46.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:46.041225+0000) 2023-08-28T01:51:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:45 smithi093 ceph-mon[111335]: pgmap v8945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:45 smithi093 ceph-mon[118902]: pgmap v8945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:47.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/963828967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:47.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:47.041474+0000) 2023-08-28T01:51:47.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/963828967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/963828967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:47 smithi136 ceph-mon[113531]: pgmap v8946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:48.040+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:48.041728+0000) 2023-08-28T01:51:48.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:47 smithi093 ceph-mon[111335]: pgmap v8946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:47 smithi093 ceph-mon[118902]: pgmap v8946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:49.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:49.041984+0000) 2023-08-28T01:51:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:49 smithi136 ceph-mon[113531]: pgmap v8947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4020241563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:50.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:50.042238+0000) 2023-08-28T01:51:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:49 smithi093 ceph-mon[111335]: pgmap v8947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4020241563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:49 smithi093 ceph-mon[118902]: pgmap v8947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4020241563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:51.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:51.042491+0000) 2023-08-28T01:51:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:51.407Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:51.408Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:51 smithi136 ceph-mon[113531]: pgmap v8948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1967533992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:52.041+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:52.042744+0000) 2023-08-28T01:51:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:51 smithi093 ceph-mon[111335]: pgmap v8948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1967533992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:51 smithi093 ceph-mon[118902]: pgmap v8948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1967533992' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:52.847Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:51:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:51:52.848Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:51:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:53.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:53.043000+0000) 2023-08-28T01:51:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:53 smithi136 ceph-mon[113531]: pgmap v8949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/325466772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:54.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:54.043288+0000) 2023-08-28T01:51:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:53 smithi093 ceph-mon[111335]: pgmap v8949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/325466772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:53 smithi093 ceph-mon[118902]: pgmap v8949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/325466772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:55.042+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:55.043537+0000) 2023-08-28T01:51:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:51:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:51:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:51:55] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:51:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:55 smithi136 ceph-mon[113531]: pgmap v8950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:56.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:56.043820+0000) 2023-08-28T01:51:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:55 smithi093 ceph-mon[111335]: pgmap v8950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:55 smithi093 ceph-mon[118902]: pgmap v8950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:57.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1260849537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:57.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:57.044058+0000) 2023-08-28T01:51:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1260849537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1260849537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:57 smithi136 ceph-mon[113531]: pgmap v8951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:58.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:58.044267+0000) 2023-08-28T01:51:58.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:57 smithi093 ceph-mon[111335]: pgmap v8951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:57 smithi093 ceph-mon[118902]: pgmap v8951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:51:59.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2344257797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:59.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:51:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:51:59.043+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:51:59.044527+0000) 2023-08-28T01:51:59.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2344257797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:51:59.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2344257797' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:51:59 smithi136 ceph-mon[113531]: pgmap v8952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:00.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:00.044777+0000) 2023-08-28T01:52:00.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:51:59 smithi093 ceph-mon[111335]: pgmap v8952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:51:59 smithi093 ceph-mon[118902]: pgmap v8952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:01.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/457304017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:01.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:01.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:01.045053+0000) 2023-08-28T01:52:01.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/457304017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/457304017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:01.407Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:01.408Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:01 smithi136 ceph-mon[113531]: pgmap v8953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:02.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:02.045365+0000) 2023-08-28T01:52:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:01 smithi093 ceph-mon[111335]: pgmap v8953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:01 smithi093 ceph-mon[118902]: pgmap v8953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:52:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:03.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:03.045721+0000) 2023-08-28T01:52:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:52:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:52:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:02.847Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:02.848Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:03 smithi136 ceph-mon[113531]: pgmap v8954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/945676059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:04.044+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:04.045862+0000) 2023-08-28T01:52:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:03 smithi093 ceph-mon[111335]: pgmap v8954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/945676059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:03 smithi093 ceph-mon[118902]: pgmap v8954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/945676059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:05.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:05.046044+0000) 2023-08-28T01:52:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:52:05] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:52:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: pgmap v8955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:52:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: pgmap v8955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:52:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: pgmap v8955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:52:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:52:05.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:52:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:06.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:06.046277+0000) 2023-08-28T01:52:06.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2578050022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:06.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2578050022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:06.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2578050022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:07.045+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:07.046539+0000) 2023-08-28T01:52:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:07 smithi136 ceph-mon[113531]: pgmap v8956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:07 smithi093 ceph-mon[111335]: pgmap v8956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:07 smithi093 ceph-mon[118902]: pgmap v8956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:08.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:08.046823+0000) 2023-08-28T01:52:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2548423026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:08.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2548423026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2548423026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:09.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:09.047070+0000) 2023-08-28T01:52:09.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:09 smithi136 ceph-mon[113531]: pgmap v8957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:10.321 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:10.321 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:09 smithi093 ceph-mon[111335]: pgmap v8957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:10.322 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:10.322 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:09 smithi093 ceph-mon[118902]: pgmap v8957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:10.322 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:10.046+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:10.047315+0000) 2023-08-28T01:52:10.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3175444333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3175444333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3175444333' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:11.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:11.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:11.047530+0000) 2023-08-28T01:52:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:11 smithi093 ceph-mon[111335]: pgmap v8958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:11 smithi093 ceph-mon[118902]: pgmap v8958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:11.408Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:11.409Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:11 smithi136 ceph-mon[113531]: pgmap v8958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:12.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:12.047753+0000) 2023-08-28T01:52:13.046 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/137085321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/137085321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/137085321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:12.848Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:12.848Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:13.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:13.047983+0000) 2023-08-28T01:52:14.046 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:13 smithi136 ceph-mon[113531]: pgmap v8959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:13 smithi093 ceph-mon[111335]: pgmap v8959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:13 smithi093 ceph-mon[118902]: pgmap v8959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:14.047+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:14.048245+0000) 2023-08-28T01:52:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:15.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:15.048497+0000) 2023-08-28T01:52:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:52:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:52:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:52:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:15 smithi136 ceph-mon[113531]: pgmap v8960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/997720871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:16.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:16.048769+0000) 2023-08-28T01:52:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:15 smithi093 ceph-mon[111335]: pgmap v8960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/997720871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:15 smithi093 ceph-mon[118902]: pgmap v8960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/997720871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:17.048+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:17.048986+0000) 2023-08-28T01:52:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:17 smithi136 ceph-mon[113531]: pgmap v8961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2891696560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:17 smithi093 ceph-mon[111335]: pgmap v8961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2891696560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:17 smithi093 ceph-mon[118902]: pgmap v8961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2891696560' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:18.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:18.049256+0000) 2023-08-28T01:52:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:19.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:19.049518+0000) 2023-08-28T01:52:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:19 smithi136 ceph-mon[113531]: pgmap v8962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:20.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:20.049786+0000) 2023-08-28T01:52:20.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:19 smithi093 ceph-mon[111335]: pgmap v8962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:19 smithi093 ceph-mon[118902]: pgmap v8962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2502813623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:21.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:21.050021+0000) 2023-08-28T01:52:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2502813623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2502813623' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:21.409Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:21.410Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:21 smithi136 ceph-mon[113531]: pgmap v8963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:22.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:22.050280+0000) 2023-08-28T01:52:22.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:21 smithi093 ceph-mon[111335]: pgmap v8963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:21 smithi093 ceph-mon[118902]: pgmap v8963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:23.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/27032659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:23.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:23.050462+0000) 2023-08-28T01:52:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/27032659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/27032659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:22.848Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:22.849Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:23 smithi136 ceph-mon[113531]: pgmap v8964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:24.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:24.050703+0000) 2023-08-28T01:52:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:23 smithi093 ceph-mon[111335]: pgmap v8964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:23 smithi093 ceph-mon[118902]: pgmap v8964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/13138433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:25.049+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:25.050992+0000) 2023-08-28T01:52:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/13138433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/13138433' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:52:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:52:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:52:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:25 smithi136 ceph-mon[113531]: pgmap v8965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:26.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:26.051249+0000) 2023-08-28T01:52:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:25 smithi093 ceph-mon[111335]: pgmap v8965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:25 smithi093 ceph-mon[118902]: pgmap v8965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:27.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:27.051508+0000) 2023-08-28T01:52:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:27 smithi136 ceph-mon[113531]: pgmap v8966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4026343406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:27 smithi093 ceph-mon[111335]: pgmap v8966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4026343406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:27 smithi093 ceph-mon[118902]: pgmap v8966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4026343406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:28.050+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:28.051800+0000) 2023-08-28T01:52:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:29.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:29.052079+0000) 2023-08-28T01:52:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:29 smithi136 ceph-mon[113531]: pgmap v8967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2195339716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:30.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:30.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:30.052366+0000) 2023-08-28T01:52:30.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:29 smithi093 ceph-mon[111335]: pgmap v8967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2195339716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:29 smithi093 ceph-mon[118902]: pgmap v8967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2195339716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:31.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:31.052632+0000) 2023-08-28T01:52:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:31.410Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:31.411Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:31 smithi136 ceph-mon[113531]: pgmap v8968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2688394191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:32.051+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:32.052903+0000) 2023-08-28T01:52:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:31 smithi093 ceph-mon[111335]: pgmap v8968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2688394191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:31 smithi093 ceph-mon[118902]: pgmap v8968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2688394191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:32.849Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:32.850Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:33.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:33.053191+0000) 2023-08-28T01:52:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:33 smithi136 ceph-mon[113531]: pgmap v8969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:34.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:34.053436+0000) 2023-08-28T01:52:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:33 smithi093 ceph-mon[111335]: pgmap v8969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:33 smithi093 ceph-mon[118902]: pgmap v8969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:35.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2209178682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:35.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:35.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:35.053706+0000) 2023-08-28T01:52:35.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2209178682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2209178682' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:52:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:52:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:52:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:35 smithi136 ceph-mon[113531]: pgmap v8970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:36.052+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:36.054000+0000) 2023-08-28T01:52:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:35 smithi093 ceph-mon[111335]: pgmap v8970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:36.281 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:35 smithi093 ceph-mon[118902]: pgmap v8970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:37.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:37.054270+0000) 2023-08-28T01:52:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2436178030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:37 smithi136 ceph-mon[113531]: pgmap v8971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2436178030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:37.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:37 smithi093 ceph-mon[111335]: pgmap v8971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:37.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2436178030' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:37.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:37 smithi093 ceph-mon[118902]: pgmap v8971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:38.053+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:38.054513+0000) 2023-08-28T01:52:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:39.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:39.054811+0000) 2023-08-28T01:52:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:39 smithi136 ceph-mon[113531]: pgmap v8972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3661488135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:40.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:40.055119+0000) 2023-08-28T01:52:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:39 smithi093 ceph-mon[111335]: pgmap v8972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3661488135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:39 smithi093 ceph-mon[118902]: pgmap v8972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3661488135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:41.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:41.055352+0000) 2023-08-28T01:52:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:41.410Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:41.411Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:41 smithi136 ceph-mon[113531]: pgmap v8973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3579877468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:42.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:42.055595+0000) 2023-08-28T01:52:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:41 smithi093 ceph-mon[111335]: pgmap v8973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3579877468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:41 smithi093 ceph-mon[118902]: pgmap v8973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3579877468' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:42.849Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:42.850Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:43.054+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:43.055871+0000) 2023-08-28T01:52:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:43 smithi136 ceph-mon[113531]: pgmap v8974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4260926776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:44.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:44.056139+0000) 2023-08-28T01:52:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:43 smithi093 ceph-mon[118902]: pgmap v8974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4260926776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:43 smithi093 ceph-mon[111335]: pgmap v8974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4260926776' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:45.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:45.056357+0000) 2023-08-28T01:52:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:52:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:52:45] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:52:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:45 smithi136 ceph-mon[113531]: pgmap v8975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:46.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:46.055+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:46.056561+0000) 2023-08-28T01:52:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:45 smithi093 ceph-mon[111335]: pgmap v8975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:45 smithi093 ceph-mon[118902]: pgmap v8975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:47.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1559045860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:47.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:47.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:47.056804+0000) 2023-08-28T01:52:47.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1559045860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1559045860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:47 smithi136 ceph-mon[113531]: pgmap v8976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:48.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:48.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:48.057074+0000) 2023-08-28T01:52:48.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:47 smithi093 ceph-mon[111335]: pgmap v8976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:47 smithi093 ceph-mon[118902]: pgmap v8976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:49.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2895953193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:49.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:49.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:49.057282+0000) 2023-08-28T01:52:49.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2895953193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2895953193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:49 smithi136 ceph-mon[113531]: pgmap v8977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:50.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:50.056+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:50.057489+0000) 2023-08-28T01:52:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:49 smithi093 ceph-mon[111335]: pgmap v8977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:49 smithi093 ceph-mon[118902]: pgmap v8977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:51.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3221178092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:51.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:51.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:51.057711+0000) 2023-08-28T01:52:51.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3221178092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3221178092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:51.410Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:51.411Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:51 smithi136 ceph-mon[113531]: pgmap v8978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:52.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:52.057947+0000) 2023-08-28T01:52:52.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:51 smithi093 ceph-mon[111335]: pgmap v8978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:51 smithi093 ceph-mon[118902]: pgmap v8978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:52.850Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:52:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:52:52.851Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:52:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:53.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:53.058211+0000) 2023-08-28T01:52:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:53 smithi136 ceph-mon[113531]: pgmap v8979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/136651924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:54.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:54.058447+0000) 2023-08-28T01:52:54.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:53 smithi093 ceph-mon[111335]: pgmap v8979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/136651924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:53 smithi093 ceph-mon[118902]: pgmap v8979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/136651924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:55.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:55.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:55.058689+0000) 2023-08-28T01:52:55.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:52:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:52:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:52:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:52:56.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:56.057+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:56.058949+0000) 2023-08-28T01:52:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:55 smithi136 ceph-mon[113531]: pgmap v8980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/976849924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:55 smithi093 ceph-mon[111335]: pgmap v8980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/976849924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:55 smithi093 ceph-mon[118902]: pgmap v8980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/976849924' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:57.058+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:57.059176+0000) 2023-08-28T01:52:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:57 smithi136 ceph-mon[113531]: pgmap v8981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:57 smithi093 ceph-mon[111335]: pgmap v8981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:57 smithi093 ceph-mon[118902]: pgmap v8981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:58.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:58.059404+0000) 2023-08-28T01:52:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/506179964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:58.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/506179964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/506179964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:52:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:52:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:52:59.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:52:59.059620+0000) 2023-08-28T01:52:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:52:59 smithi136 ceph-mon[113531]: pgmap v8982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:52:59 smithi093 ceph-mon[111335]: pgmap v8982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:52:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:52:59 smithi093 ceph-mon[118902]: pgmap v8982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:00.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:00.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:00.059872+0000) 2023-08-28T01:53:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/679268749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/679268749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/679268749' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:01.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:01.060148+0000) 2023-08-28T01:53:01.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:01 smithi093 ceph-mon[111335]: pgmap v8983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:01 smithi093 ceph-mon[118902]: pgmap v8983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:01.411Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:01.412Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:01 smithi136 ceph-mon[113531]: pgmap v8983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:02.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:02.060382+0000) 2023-08-28T01:53:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2725810438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:03.059+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:03.060615+0000) 2023-08-28T01:53:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2725810438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2725810438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:02.851Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:02.852Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:03 smithi136 ceph-mon[113531]: pgmap v8984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:04.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:04.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:04.060857+0000) 2023-08-28T01:53:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:03 smithi093 ceph-mon[111335]: pgmap v8984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:03 smithi093 ceph-mon[118902]: pgmap v8984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:05.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:05.061163+0000) 2023-08-28T01:53:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:53:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:53:06.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:06.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:06.061316+0000) 2023-08-28T01:53:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:05 smithi136 ceph-mon[113531]: pgmap v8985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1196186788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:53:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:06.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[111335]: pgmap v8985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:06.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1196186788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[118902]: pgmap v8985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1196186788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:07.386 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:07.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:07.061525+0000) 2023-08-28T01:53:07.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:07 smithi136 ceph-mon[113531]: pgmap v8986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:07.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:07.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:07 smithi093 ceph-mon[111335]: pgmap v8986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:07 smithi093 ceph-mon[118902]: pgmap v8986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:08.060+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:08.061746+0000) 2023-08-28T01:53:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3634343921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:53:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:53:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3634343921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3634343921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:53:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:53:08.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:53:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:09.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:09.062051+0000) 2023-08-28T01:53:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:09 smithi136 ceph-mon[113531]: pgmap v8987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:09 smithi093 ceph-mon[111335]: pgmap v8987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:09 smithi093 ceph-mon[118902]: pgmap v8987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:10.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:10.062320+0000) 2023-08-28T01:53:10.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/339259662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:10.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/339259662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/339259662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:11.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:11.062549+0000) 2023-08-28T01:53:11.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:11 smithi093 ceph-mon[111335]: pgmap v8988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:11 smithi093 ceph-mon[118902]: pgmap v8988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:11.413Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:11.414Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:11 smithi136 ceph-mon[113531]: pgmap v8988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:12.061+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:12.062816+0000) 2023-08-28T01:53:12.850 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2887961840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:12.851 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2887961840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:12.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2887961840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:12.851Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:12.852Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:13.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:13.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:13.063081+0000) 2023-08-28T01:53:13.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:13 smithi136 ceph-mon[113531]: pgmap v8989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:13 smithi093 ceph-mon[111335]: pgmap v8989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:13 smithi093 ceph-mon[118902]: pgmap v8989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:14.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:14.063314+0000) 2023-08-28T01:53:15.061 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/96587899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:15.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/96587899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/96587899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:15.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:15.063542+0000) 2023-08-28T01:53:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:53:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:53:15] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T01:53:16.062 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:15 smithi136 ceph-mon[113531]: pgmap v8990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:15 smithi093 ceph-mon[111335]: pgmap v8990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:15 smithi093 ceph-mon[118902]: pgmap v8990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:16.062+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:16.063794+0000) 2023-08-28T01:53:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:17.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:17.064019+0000) 2023-08-28T01:53:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:17 smithi136 ceph-mon[113531]: pgmap v8991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3380412617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:17 smithi093 ceph-mon[111335]: pgmap v8991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3380412617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:17 smithi093 ceph-mon[118902]: pgmap v8991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3380412617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:18.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:18.064315+0000) 2023-08-28T01:53:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:19.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:19.064564+0000) 2023-08-28T01:53:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:19 smithi136 ceph-mon[113531]: pgmap v8992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1202331599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:20.063+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:20.064811+0000) 2023-08-28T01:53:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:19 smithi093 ceph-mon[111335]: pgmap v8992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1202331599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:19 smithi093 ceph-mon[118902]: pgmap v8992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1202331599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:21.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:21.065124+0000) 2023-08-28T01:53:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:21.413Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:21.414Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:21 smithi136 ceph-mon[113531]: pgmap v8993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3434976258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:22.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:22.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:22.065321+0000) 2023-08-28T01:53:22.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:21 smithi093 ceph-mon[111335]: pgmap v8993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3434976258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:21 smithi093 ceph-mon[118902]: pgmap v8993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3434976258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:22.852Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:22.853Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:23.064+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:23.065566+0000) 2023-08-28T01:53:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:23 smithi136 ceph-mon[113531]: pgmap v8994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:24.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:24.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:24.065791+0000) 2023-08-28T01:53:24.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:23 smithi093 ceph-mon[111335]: pgmap v8994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:23 smithi093 ceph-mon[118902]: pgmap v8994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3678733295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:25.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:25.066078+0000) 2023-08-28T01:53:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3678733295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3678733295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:53:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:53:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:53:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:25 smithi136 ceph-mon[113531]: pgmap v8995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:26.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:26.066357+0000) 2023-08-28T01:53:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:25 smithi093 ceph-mon[111335]: pgmap v8995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:25 smithi093 ceph-mon[118902]: pgmap v8995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:27.065+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:27.066631+0000) 2023-08-28T01:53:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2790947101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:27 smithi136 ceph-mon[113531]: pgmap v8996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2790947101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:27 smithi093 ceph-mon[111335]: pgmap v8996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2790947101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:27 smithi093 ceph-mon[118902]: pgmap v8996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:28.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:28.066845+0000) 2023-08-28T01:53:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:29.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:29.067090+0000) 2023-08-28T01:53:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:29 smithi136 ceph-mon[113531]: pgmap v8997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2944108825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:30.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:30.067321+0000) 2023-08-28T01:53:30.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:29 smithi093 ceph-mon[111335]: pgmap v8997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2944108825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:29 smithi093 ceph-mon[118902]: pgmap v8997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2944108825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:31.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:31.067554+0000) 2023-08-28T01:53:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:31.414Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:31.415Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:31 smithi136 ceph-mon[113531]: pgmap v8998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/182861710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:32.066+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:32.067759+0000) 2023-08-28T01:53:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:31 smithi093 ceph-mon[111335]: pgmap v8998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/182861710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:31 smithi093 ceph-mon[118902]: pgmap v8998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/182861710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:32.852Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:32.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:33.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:33.067992+0000) 2023-08-28T01:53:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:33 smithi136 ceph-mon[113531]: pgmap v8999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1717578918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:34.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:34.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:34.068232+0000) 2023-08-28T01:53:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:33 smithi093 ceph-mon[111335]: pgmap v8999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1717578918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:33 smithi093 ceph-mon[118902]: pgmap v8999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1717578918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:35.067+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:35.068484+0000) 2023-08-28T01:53:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:53:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:53:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T01:53:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:35 smithi136 ceph-mon[113531]: pgmap v9000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:36.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:36.068762+0000) 2023-08-28T01:53:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:35 smithi093 ceph-mon[111335]: pgmap v9000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:35 smithi093 ceph-mon[118902]: pgmap v9000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:37.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/955193231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:37.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:37.069095+0000) 2023-08-28T01:53:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/955193231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/955193231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:37 smithi136 ceph-mon[113531]: pgmap v9001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:38.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:38.068+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:38.069387+0000) 2023-08-28T01:53:38.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:37 smithi093 ceph-mon[111335]: pgmap v9001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:37 smithi093 ceph-mon[118902]: pgmap v9001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/16523406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:39.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:39.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:39.069640+0000) 2023-08-28T01:53:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/16523406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/16523406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:39 smithi136 ceph-mon[113531]: pgmap v9002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:40.770 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:40.793 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:40.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:40.069897+0000) 2023-08-28T01:53:40.794 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:39 smithi093 ceph-mon[111335]: pgmap v9002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:40.794 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:40.794 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:39 smithi093 ceph-mon[118902]: pgmap v9002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:40.794 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:41.069+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:41.070146+0000) 2023-08-28T01:53:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:41.415Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:41.416Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:41 smithi136 ceph-mon[113531]: pgmap v9003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/767101191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:42.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:42.070402+0000) 2023-08-28T01:53:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:41 smithi093 ceph-mon[111335]: pgmap v9003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/767101191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:41 smithi093 ceph-mon[118902]: pgmap v9003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/767101191' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:42.853Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:42.854Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:43.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:43.070656+0000) 2023-08-28T01:53:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:43 smithi136 ceph-mon[113531]: pgmap v9004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1606338570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:44.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:44.070938+0000) 2023-08-28T01:53:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:43 smithi093 ceph-mon[111335]: pgmap v9004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1606338570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:43 smithi093 ceph-mon[118902]: pgmap v9004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1606338570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:45.070+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:45.071186+0000) 2023-08-28T01:53:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:53:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:53:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:53:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:45 smithi136 ceph-mon[113531]: pgmap v9005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1253794286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:46.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:46.071463+0000) 2023-08-28T01:53:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:45 smithi093 ceph-mon[111335]: pgmap v9005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1253794286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:45 smithi093 ceph-mon[118902]: pgmap v9005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1253794286' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:47.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:47.071676+0000) 2023-08-28T01:53:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:47 smithi136 ceph-mon[113531]: pgmap v9006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:47 smithi093 ceph-mon[111335]: pgmap v9006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:47 smithi093 ceph-mon[118902]: pgmap v9006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:48.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:48.071908+0000) 2023-08-28T01:53:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3160185241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:48.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3160185241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3160185241' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:49.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:49.072171+0000) 2023-08-28T01:53:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:49 smithi136 ceph-mon[113531]: pgmap v9007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:49 smithi093 ceph-mon[111335]: pgmap v9007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:49 smithi093 ceph-mon[118902]: pgmap v9007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:50.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:50.072460+0000) 2023-08-28T01:53:50.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1889308274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1889308274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1889308274' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:51.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:51.072740+0000) 2023-08-28T01:53:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:51 smithi093 ceph-mon[111335]: pgmap v9008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:51 smithi093 ceph-mon[118902]: pgmap v9008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:51.415Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:51.416Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:51 smithi136 ceph-mon[113531]: pgmap v9008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:52.071+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:52.073007+0000) 2023-08-28T01:53:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/565967166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:53.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:53.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:53.073214+0000) 2023-08-28T01:53:53.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/565967166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/565967166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:52.854Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:53:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:53:52.855Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:53:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:53 smithi136 ceph-mon[113531]: pgmap v9009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:54.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:54.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:54.073480+0000) 2023-08-28T01:53:54.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:53 smithi093 ceph-mon[111335]: pgmap v9009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:53 smithi093 ceph-mon[118902]: pgmap v9009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:55.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:55.072+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:55.073766+0000) 2023-08-28T01:53:55.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:53:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:53:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:53:55] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:53:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:55 smithi136 ceph-mon[113531]: pgmap v9010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1972295677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:56.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:56.073993+0000) 2023-08-28T01:53:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:55 smithi093 ceph-mon[111335]: pgmap v9010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1972295677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:55 smithi093 ceph-mon[118902]: pgmap v9010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1972295677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:57.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:57.074192+0000) 2023-08-28T01:53:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:57 smithi136 ceph-mon[113531]: pgmap v9011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:57 smithi093 ceph-mon[111335]: pgmap v9011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:57 smithi093 ceph-mon[118902]: pgmap v9011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:58.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:58.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:58.074454+0000) 2023-08-28T01:53:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/110923881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:58.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/110923881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/110923881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:53:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:53:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:53:59.073+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:53:59.074745+0000) 2023-08-28T01:53:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:53:59 smithi136 ceph-mon[113531]: pgmap v9012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:59.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:53:59 smithi093 ceph-mon[111335]: pgmap v9012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:53:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:53:59 smithi093 ceph-mon[118902]: pgmap v9012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:00.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:00.075005+0000) 2023-08-28T01:54:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3949612813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3949612813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3949612813' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:01.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:01.075227+0000) 2023-08-28T01:54:01.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:01 smithi093 ceph-mon[111335]: pgmap v9013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:01 smithi093 ceph-mon[118902]: pgmap v9013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:01.415Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:01.416Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:01 smithi136 ceph-mon[113531]: pgmap v9013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:02.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:02.075507+0000) 2023-08-28T01:54:02.853 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3503669400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:02.853 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3503669400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:02.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3503669400' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:02.855Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:02.856Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:03.074+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:03.075736+0000) 2023-08-28T01:54:03.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:03 smithi136 ceph-mon[113531]: pgmap v9014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:03.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:03 smithi093 ceph-mon[111335]: pgmap v9014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:03 smithi093 ceph-mon[118902]: pgmap v9014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:04.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:04.075948+0000) 2023-08-28T01:54:05.074 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3759957741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:05.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3759957741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3759957741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:05.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:05.076229+0000) 2023-08-28T01:54:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:54:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:54:05] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:54:06.074 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:05 smithi136 ceph-mon[113531]: pgmap v9015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:06.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:05 smithi093 ceph-mon[111335]: pgmap v9015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:05 smithi093 ceph-mon[118902]: pgmap v9015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:06.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:06.076526+0000) 2023-08-28T01:54:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:07.075+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:07.076830+0000) 2023-08-28T01:54:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:07 smithi136 ceph-mon[113531]: pgmap v9016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2042129456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:07 smithi093 ceph-mon[111335]: pgmap v9016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2042129456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:07 smithi093 ceph-mon[118902]: pgmap v9016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2042129456' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:08.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:08.077144+0000) 2023-08-28T01:54:08.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:54:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:54:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:54:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:09.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:09.077486+0000) 2023-08-28T01:54:09.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:09 smithi136 ceph-mon[113531]: pgmap v9017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:09.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:09.898 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3404888110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:09.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:09 smithi093 ceph-mon[111335]: pgmap v9017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:09.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:09.961 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3404888110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:09.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:09 smithi093 ceph-mon[118902]: pgmap v9017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:09.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:09.961 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3404888110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:10.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:10.077675+0000) 2023-08-28T01:54:11.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[111335]: pgmap v9018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:11.416 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.417 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.417 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.417 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.417 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[118902]: pgmap v9018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:11.417 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.417 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:11 smithi136 ceph-mon[113531]: pgmap v9018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:11.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:11.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:11.076+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:11.077858+0000) 2023-08-28T01:54:11.667 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:11.416Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:11.667 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:11.418Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:12.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:12.078117+0000) 2023-08-28T01:54:12.856 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3530315649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:12.856 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3530315649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:12.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3530315649' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:12.856Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:12.857Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:13.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:13.078395+0000) 2023-08-28T01:54:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:54:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:54:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:13 smithi136 ceph-mon[113531]: pgmap v9019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:13.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:54:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:54:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[118902]: pgmap v9019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:54:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:54:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:54:13.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:13 smithi093 ceph-mon[111335]: pgmap v9019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:14.077+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:14.078687+0000) 2023-08-28T01:54:14.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2526163920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:14.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2526163920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:14.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2526163920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:15.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:15.078923+0000) 2023-08-28T01:54:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:15 smithi093 ceph-mon[111335]: pgmap v9020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:15 smithi093 ceph-mon[118902]: pgmap v9020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:54:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:54:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:54:15.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:15 smithi136 ceph-mon[113531]: pgmap v9020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:16.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:16.079169+0000) 2023-08-28T01:54:17.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:17.078+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:17.079430+0000) 2023-08-28T01:54:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2906916641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:17 smithi136 ceph-mon[113531]: pgmap v9021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2906916641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:17 smithi093 ceph-mon[111335]: pgmap v9021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2906916641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:17 smithi093 ceph-mon[118902]: pgmap v9021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:18.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:18.079725+0000) 2023-08-28T01:54:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:19.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:19.079958+0000) 2023-08-28T01:54:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:20.079+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:20.080210+0000) 2023-08-28T01:54:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:19 smithi136 ceph-mon[113531]: pgmap v9022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2136457092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:20.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:19 smithi093 ceph-mon[118902]: pgmap v9022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2136457092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:19 smithi093 ceph-mon[111335]: pgmap v9022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2136457092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:21.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:21.080436+0000) 2023-08-28T01:54:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:21.417Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:21.418Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:21 smithi136 ceph-mon[113531]: pgmap v9023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1892994189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:22.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:22.080681+0000) 2023-08-28T01:54:22.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:21 smithi093 ceph-mon[111335]: pgmap v9023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1892994189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:21 smithi093 ceph-mon[118902]: pgmap v9023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1892994189' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:22.857Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:22.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:23.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:23.080959+0000) 2023-08-28T01:54:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:23 smithi136 ceph-mon[113531]: pgmap v9024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1054290463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:24.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:24.081239+0000) 2023-08-28T01:54:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:23 smithi093 ceph-mon[111335]: pgmap v9024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1054290463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:23 smithi093 ceph-mon[118902]: pgmap v9024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1054290463' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:25.080+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:25.081414+0000) 2023-08-28T01:54:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:54:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:54:25] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:54:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:25 smithi136 ceph-mon[113531]: pgmap v9025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:26.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:26.081680+0000) 2023-08-28T01:54:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:25 smithi093 ceph-mon[111335]: pgmap v9025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:25 smithi093 ceph-mon[118902]: pgmap v9025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:27.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/305107035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:27.249 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:27.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:27.081929+0000) 2023-08-28T01:54:27.249 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/305107035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:27.250 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/305107035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:27 smithi136 ceph-mon[113531]: pgmap v9026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:28.229 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:28.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:28.082190+0000) 2023-08-28T01:54:28.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:27 smithi093 ceph-mon[111335]: pgmap v9026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:27 smithi093 ceph-mon[118902]: pgmap v9026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/229673252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:29.364 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:29.081+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:29.082395+0000) 2023-08-28T01:54:29.365 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/229673252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:29.365 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/229673252' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:29 smithi136 ceph-mon[113531]: pgmap v9027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:30.495 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:30.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:30.082646+0000) 2023-08-28T01:54:30.496 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:29 smithi093 ceph-mon[111335]: pgmap v9027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:30.496 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:29 smithi093 ceph-mon[118902]: pgmap v9027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:31.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:31.082883+0000) 2023-08-28T01:54:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:31.418Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:31.419Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:31 smithi136 ceph-mon[113531]: pgmap v9028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3409697987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:32.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:32.083132+0000) 2023-08-28T01:54:32.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:31 smithi093 ceph-mon[111335]: pgmap v9028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3409697987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:31 smithi093 ceph-mon[118902]: pgmap v9028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3409697987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:33.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:32.857Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:32.858Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:33.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:33.083335+0000) 2023-08-28T01:54:34.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:33 smithi136 ceph-mon[113531]: pgmap v9029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:34.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/961230887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:34.191 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:34.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:34.083609+0000) 2023-08-28T01:54:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:33 smithi093 ceph-mon[111335]: pgmap v9029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/961230887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:33 smithi093 ceph-mon[118902]: pgmap v9029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/961230887' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:35.082+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:35.083834+0000) 2023-08-28T01:54:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:54:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:54:35] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:54:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:35 smithi136 ceph-mon[113531]: pgmap v9030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/269020242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:36.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:36.084110+0000) 2023-08-28T01:54:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:35 smithi093 ceph-mon[111335]: pgmap v9030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/269020242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:35 smithi093 ceph-mon[118902]: pgmap v9030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/269020242' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:37.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:37.084357+0000) 2023-08-28T01:54:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:37 smithi136 ceph-mon[113531]: pgmap v9031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:37 smithi093 ceph-mon[111335]: pgmap v9031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:37 smithi093 ceph-mon[118902]: pgmap v9031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:38.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:38.084612+0000) 2023-08-28T01:54:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1372840135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:38.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1372840135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1372840135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:39.083+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:39.084863+0000) 2023-08-28T01:54:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:39 smithi136 ceph-mon[113531]: pgmap v9032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:39 smithi093 ceph-mon[111335]: pgmap v9032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:39 smithi093 ceph-mon[118902]: pgmap v9032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:40.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:40.085151+0000) 2023-08-28T01:54:40.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2996505692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:41.001 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2996505692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:41.001 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2996505692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:41.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:41.085373+0000) 2023-08-28T01:54:41.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:41 smithi093 ceph-mon[111335]: pgmap v9033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:41 smithi093 ceph-mon[118902]: pgmap v9033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:41.419Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:41.420Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:41 smithi136 ceph-mon[113531]: pgmap v9033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:42.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:42.085626+0000) 2023-08-28T01:54:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:42.858Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:42.859Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:43.084+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:43.085867+0000) 2023-08-28T01:54:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:43 smithi136 ceph-mon[113531]: pgmap v9034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2209462148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:44.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:44.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:44.086074+0000) 2023-08-28T01:54:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:43 smithi093 ceph-mon[111335]: pgmap v9034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2209462148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:43 smithi093 ceph-mon[118902]: pgmap v9034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2209462148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:45.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:45.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:45.086296+0000) 2023-08-28T01:54:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:54:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:54:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:54:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:45 smithi136 ceph-mon[113531]: pgmap v9035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3541488159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:46.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:46.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:46.086552+0000) 2023-08-28T01:54:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:45 smithi093 ceph-mon[111335]: pgmap v9035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3541488159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:45 smithi093 ceph-mon[118902]: pgmap v9035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3541488159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:47.085+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:47.086830+0000) 2023-08-28T01:54:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:47 smithi136 ceph-mon[113531]: pgmap v9036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:47 smithi093 ceph-mon[111335]: pgmap v9036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:47 smithi093 ceph-mon[118902]: pgmap v9036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:48.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:48.087088+0000) 2023-08-28T01:54:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/327006746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:49.132 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/327006746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:49.132 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/327006746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:49.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:49.087309+0000) 2023-08-28T01:54:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:49 smithi136 ceph-mon[113531]: pgmap v9037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:49.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:49 smithi093 ceph-mon[111335]: pgmap v9037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:49 smithi093 ceph-mon[118902]: pgmap v9037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:50.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:50.087518+0000) 2023-08-28T01:54:50.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/408351083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/408351083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/408351083' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:51.086+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:51.087780+0000) 2023-08-28T01:54:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:51 smithi093 ceph-mon[111335]: pgmap v9038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:51 smithi093 ceph-mon[118902]: pgmap v9038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:51.420Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:51.421Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:51 smithi136 ceph-mon[113531]: pgmap v9038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:52.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:52.088024+0000) 2023-08-28T01:54:52.858 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/486406834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:52.858 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/486406834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:52.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/486406834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:52.859Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:54:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:54:52.860Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:54:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:53.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:53.088268+0000) 2023-08-28T01:54:53.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:53 smithi136 ceph-mon[113531]: pgmap v9039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:53.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:53 smithi093 ceph-mon[111335]: pgmap v9039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:53 smithi093 ceph-mon[118902]: pgmap v9039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:54.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:54.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:54.088495+0000) 2023-08-28T01:54:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:54.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:54:55.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:55.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:55.088724+0000) 2023-08-28T01:54:55.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3902924843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:55 smithi093 ceph-mon[111335]: pgmap v9040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:54:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:54:55] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:54:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3902924843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:55 smithi093 ceph-mon[118902]: pgmap v9040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3902924843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:55 smithi136 ceph-mon[113531]: pgmap v9040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:56.087+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:56.088993+0000) 2023-08-28T01:54:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:57.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:57.089224+0000) 2023-08-28T01:54:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:57 smithi136 ceph-mon[113531]: pgmap v9041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2678780104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:57 smithi093 ceph-mon[111335]: pgmap v9041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2678780104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:57 smithi093 ceph-mon[118902]: pgmap v9041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:54:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2678780104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:54:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:58.088+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:58.089541+0000) 2023-08-28T01:54:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:54:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:54:59.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:54:59.089762+0000) 2023-08-28T01:55:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:59 smithi136 ceph-mon[113531]: pgmap v9042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:54:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/606277567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:00.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:00.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:00.090053+0000) 2023-08-28T01:55:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:59 smithi093 ceph-mon[111335]: pgmap v9042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:54:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/606277567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:59 smithi093 ceph-mon[118902]: pgmap v9042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:54:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/606277567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:01.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:01.090276+0000) 2023-08-28T01:55:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:01.421Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:01.422Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:01 smithi136 ceph-mon[113531]: pgmap v9043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:02.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:02.090485+0000) 2023-08-28T01:55:02.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:01 smithi093 ceph-mon[111335]: pgmap v9043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:01 smithi093 ceph-mon[118902]: pgmap v9043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1096310197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:03.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:03.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:03.090737+0000) 2023-08-28T01:55:03.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1096310197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1096310197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:02.859Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:02.860Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:03 smithi136 ceph-mon[113531]: pgmap v9044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:04.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:04.089+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:04.090981+0000) 2023-08-28T01:55:04.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:03 smithi093 ceph-mon[111335]: pgmap v9044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:03 smithi093 ceph-mon[118902]: pgmap v9044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:05.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2536384062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:05.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:05.090+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:05.091211+0000) 2023-08-28T01:55:05.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2536384062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2536384062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:55:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:55:05] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:55:06.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:05 smithi136 ceph-mon[113531]: pgmap v9045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:06.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:06.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:06.091510+0000) 2023-08-28T01:55:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:05 smithi093 ceph-mon[111335]: pgmap v9045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:05 smithi093 ceph-mon[118902]: pgmap v9045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:07.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:07.091775+0000) 2023-08-28T01:55:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4148739617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:07 smithi136 ceph-mon[113531]: pgmap v9046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4148739617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:07 smithi093 ceph-mon[111335]: pgmap v9046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4148739617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:07 smithi093 ceph-mon[118902]: pgmap v9046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:08.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:08.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:08.092021+0000) 2023-08-28T01:55:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:09.091+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:09.092270+0000) 2023-08-28T01:55:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:09 smithi136 ceph-mon[113531]: pgmap v9047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2564852162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:10.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:10.092501+0000) 2023-08-28T01:55:10.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:09 smithi093 ceph-mon[111335]: pgmap v9047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2564852162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:09 smithi093 ceph-mon[118902]: pgmap v9047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2564852162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:11.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:11.092758+0000) 2023-08-28T01:55:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:11.422Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:11.422Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:11 smithi136 ceph-mon[113531]: pgmap v9048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2898138314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:12.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:12.093021+0000) 2023-08-28T01:55:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:11 smithi093 ceph-mon[111335]: pgmap v9048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2898138314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:11 smithi093 ceph-mon[118902]: pgmap v9048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2898138314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:13.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:55:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:13.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:13.093231+0000) 2023-08-28T01:55:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:55:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:55:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:12.860Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:12.861Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:13.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:13 smithi136 ceph-mon[113531]: pgmap v9049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:13 smithi093 ceph-mon[111335]: pgmap v9049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:13 smithi093 ceph-mon[118902]: pgmap v9049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:14.092+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:14.093377+0000) 2023-08-28T01:55:15.092 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1951983985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:15.092 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:15.092 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:15.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1951983985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1951983985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:15.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:15.093591+0000) 2023-08-28T01:55:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:55:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:55:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:16.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:16.093804+0000) 2023-08-28T01:55:16.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:15 smithi136 ceph-mon[113531]: pgmap v9050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:16.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:55:16.374 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:55:16.374 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.374 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[118902]: pgmap v9050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:16.374 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.374 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.374 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[111335]: pgmap v9050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:55:16.375 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:55:17.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/10020914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:17.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:17.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:17.094119+0000) 2023-08-28T01:55:17.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/10020914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/10020914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:17 smithi136 ceph-mon[113531]: pgmap v9051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:18.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:18.094349+0000) 2023-08-28T01:55:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:17 smithi093 ceph-mon[111335]: pgmap v9051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:17 smithi093 ceph-mon[118902]: pgmap v9051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1259182438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:19.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:19.094567+0000) 2023-08-28T01:55:19.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1259182438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1259182438' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:19 smithi136 ceph-mon[113531]: pgmap v9052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:20.093+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:20.094745+0000) 2023-08-28T01:55:20.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:19 smithi093 ceph-mon[118902]: pgmap v9052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:19 smithi093 ceph-mon[111335]: pgmap v9052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:21.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:21.095010+0000) 2023-08-28T01:55:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:21.422Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:21.423Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:21 smithi136 ceph-mon[113531]: pgmap v9053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1880183858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:22.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:22.095260+0000) 2023-08-28T01:55:22.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:21 smithi093 ceph-mon[111335]: pgmap v9053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1880183858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:21 smithi093 ceph-mon[118902]: pgmap v9053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1880183858' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:22.861Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:22.861Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:23.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:23.095495+0000) 2023-08-28T01:55:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:23 smithi136 ceph-mon[113531]: pgmap v9054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3428544426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:24.094+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:24.095756+0000) 2023-08-28T01:55:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:23 smithi093 ceph-mon[118902]: pgmap v9054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3428544426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:23 smithi093 ceph-mon[111335]: pgmap v9054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3428544426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:25.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:25.096000+0000) 2023-08-28T01:55:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:55:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:55:25] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:55:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:25 smithi136 ceph-mon[113531]: pgmap v9055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1786307817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:26.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:26.096247+0000) 2023-08-28T01:55:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:25 smithi093 ceph-mon[111335]: pgmap v9055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1786307817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:25 smithi093 ceph-mon[118902]: pgmap v9055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1786307817' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:27.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:27.096503+0000) 2023-08-28T01:55:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:27 smithi136 ceph-mon[113531]: pgmap v9056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:27 smithi093 ceph-mon[111335]: pgmap v9056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:27 smithi093 ceph-mon[118902]: pgmap v9056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:28.095+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:28.096764+0000) 2023-08-28T01:55:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/206053616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:28.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/206053616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/206053616' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:29.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:29.097019+0000) 2023-08-28T01:55:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:29 smithi136 ceph-mon[113531]: pgmap v9057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:29 smithi093 ceph-mon[111335]: pgmap v9057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:29 smithi093 ceph-mon[118902]: pgmap v9057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:30.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:30.097314+0000) 2023-08-28T01:55:30.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3575682738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:30.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3575682738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3575682738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:31.096+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:31.097574+0000) 2023-08-28T01:55:31.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:31 smithi093 ceph-mon[111335]: pgmap v9058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:31 smithi093 ceph-mon[118902]: pgmap v9058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:31.423Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:31.424Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:31 smithi136 ceph-mon[113531]: pgmap v9058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:32.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:32.097823+0000) 2023-08-28T01:55:33.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:32.861Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:32.862Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:33.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:33.098075+0000) 2023-08-28T01:55:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:33 smithi136 ceph-mon[113531]: pgmap v9059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/691673859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:34.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:34.098349+0000) 2023-08-28T01:55:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:33 smithi093 ceph-mon[118902]: pgmap v9059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/691673859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:33 smithi093 ceph-mon[111335]: pgmap v9059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/691673859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:35.097+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:35.098602+0000) 2023-08-28T01:55:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:55:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:55:35] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T01:55:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:35 smithi136 ceph-mon[113531]: pgmap v9060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/911979722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:36.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:36.098869+0000) 2023-08-28T01:55:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:35 smithi093 ceph-mon[111335]: pgmap v9060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/911979722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:35 smithi093 ceph-mon[118902]: pgmap v9060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:36.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/911979722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:37.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:37.099150+0000) 2023-08-28T01:55:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:37 smithi136 ceph-mon[113531]: pgmap v9061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:37 smithi093 ceph-mon[111335]: pgmap v9061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:37 smithi093 ceph-mon[118902]: pgmap v9061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:38.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:38.099435+0000) 2023-08-28T01:55:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1177999158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:38.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1177999158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1177999158' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:39.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:39.099690+0000) 2023-08-28T01:55:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:39 smithi136 ceph-mon[113531]: pgmap v9062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:39.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:39 smithi093 ceph-mon[111335]: pgmap v9062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:39 smithi093 ceph-mon[118902]: pgmap v9062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:40.098+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:40.099911+0000) 2023-08-28T01:55:40.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/17966633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:40.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/17966633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:40.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/17966633' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:41.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:41.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:41.100167+0000) 2023-08-28T01:55:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:41 smithi093 ceph-mon[111335]: pgmap v9063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:41 smithi093 ceph-mon[118902]: pgmap v9063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:41.423Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:41.424Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:41.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:41 smithi136 ceph-mon[113531]: pgmap v9063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:42.099+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:42.100453+0000) 2023-08-28T01:55:42.861 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3499009940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:42.862 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3499009940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:42.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3499009940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:42.862Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:42.863Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:43.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:43.100725+0000) 2023-08-28T01:55:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:43 smithi136 ceph-mon[113531]: pgmap v9064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:43 smithi093 ceph-mon[111335]: pgmap v9064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:43 smithi093 ceph-mon[118902]: pgmap v9064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:44.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:44.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:44.101014+0000) 2023-08-28T01:55:45.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3553759097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3553759097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3553759097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:45.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:45.101265+0000) 2023-08-28T01:55:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:55:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:55:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:55:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:45 smithi093 ceph-mon[111335]: pgmap v9065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:45 smithi093 ceph-mon[118902]: pgmap v9065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:45 smithi136 ceph-mon[113531]: pgmap v9065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:46.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:46.101516+0000) 2023-08-28T01:55:47.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:47.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:47.101765+0000) 2023-08-28T01:55:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:47 smithi136 ceph-mon[113531]: pgmap v9066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1177359046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:47 smithi093 ceph-mon[111335]: pgmap v9066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1177359046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:47 smithi093 ceph-mon[118902]: pgmap v9066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:47.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1177359046' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:48.100+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:48.101995+0000) 2023-08-28T01:55:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:49.101+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:49.102254+0000) 2023-08-28T01:55:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:49 smithi136 ceph-mon[113531]: pgmap v9067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3222302927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:50.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:50.102486+0000) 2023-08-28T01:55:50.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:49 smithi093 ceph-mon[111335]: pgmap v9067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3222302927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:49 smithi093 ceph-mon[118902]: pgmap v9067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3222302927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:51.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:51.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:51.102742+0000) 2023-08-28T01:55:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:51.424Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:51.425Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:51 smithi136 ceph-mon[113531]: pgmap v9068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:52.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:52.102990+0000) 2023-08-28T01:55:52.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:51 smithi093 ceph-mon[111335]: pgmap v9068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:51 smithi093 ceph-mon[118902]: pgmap v9068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1076410703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:53.102+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:53.103223+0000) 2023-08-28T01:55:53.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1076410703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1076410703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:52.862Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:55:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:55:52.863Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:55:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:53 smithi136 ceph-mon[113531]: pgmap v9069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:54.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:54.103489+0000) 2023-08-28T01:55:54.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:53 smithi093 ceph-mon[118902]: pgmap v9069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:54.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:53 smithi093 ceph-mon[111335]: pgmap v9069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:55:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/359700512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:55.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:55.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:55.103750+0000) 2023-08-28T01:55:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/359700512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/359700512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:55:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:55:55] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:55:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:55 smithi136 ceph-mon[113531]: pgmap v9070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:56.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:56.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:56.103946+0000) 2023-08-28T01:55:56.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:55 smithi093 ceph-mon[118902]: pgmap v9070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:55 smithi093 ceph-mon[111335]: pgmap v9070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:57.103+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:57.104217+0000) 2023-08-28T01:55:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1640833580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:57.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:57 smithi136 ceph-mon[113531]: pgmap v9071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1640833580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:57 smithi093 ceph-mon[111335]: pgmap v9071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1640833580' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:55:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:57 smithi093 ceph-mon[118902]: pgmap v9071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:55:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:58.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:58.104463+0000) 2023-08-28T01:55:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:55:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:55:59.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:55:59.104727+0000) 2023-08-28T01:56:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:59 smithi136 ceph-mon[113531]: pgmap v9072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:00.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:55:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2832565080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:00.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:00.104977+0000) 2023-08-28T01:56:00.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:59 smithi093 ceph-mon[111335]: pgmap v9072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:55:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2832565080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:59 smithi093 ceph-mon[118902]: pgmap v9072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:55:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2832565080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:01.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:01.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:01.105200+0000) 2023-08-28T01:56:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:01.425Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:01.426Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:01 smithi136 ceph-mon[113531]: pgmap v9073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1491718663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:02.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:02.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:02.105453+0000) 2023-08-28T01:56:02.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:01 smithi093 ceph-mon[111335]: pgmap v9073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1491718663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:01 smithi093 ceph-mon[118902]: pgmap v9073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1491718663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:02.863Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:02.864Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:03.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:03.105723+0000) 2023-08-28T01:56:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:03 smithi136 ceph-mon[113531]: pgmap v9074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:04.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:04.104+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:04.105949+0000) 2023-08-28T01:56:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:03 smithi093 ceph-mon[111335]: pgmap v9074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:03 smithi093 ceph-mon[118902]: pgmap v9074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2130024648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:05.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:05.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:05.106157+0000) 2023-08-28T01:56:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2130024648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2130024648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:56:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:56:05] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:56:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:05 smithi136 ceph-mon[113531]: pgmap v9075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:06.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:06.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:06.106384+0000) 2023-08-28T01:56:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:05 smithi093 ceph-mon[111335]: pgmap v9075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:05 smithi093 ceph-mon[118902]: pgmap v9075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:07.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3601919831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:07.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:07.106636+0000) 2023-08-28T01:56:07.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3601919831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3601919831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:07 smithi136 ceph-mon[113531]: pgmap v9076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:08.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:08.105+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:08.106912+0000) 2023-08-28T01:56:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:07 smithi093 ceph-mon[111335]: pgmap v9076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:07 smithi093 ceph-mon[118902]: pgmap v9076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1325256554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:09.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:09.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:09.107181+0000) 2023-08-28T01:56:09.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1325256554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:09.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1325256554' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:09 smithi136 ceph-mon[113531]: pgmap v9077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:10.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:10.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:10.107411+0000) 2023-08-28T01:56:10.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:09 smithi093 ceph-mon[111335]: pgmap v9077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:09 smithi093 ceph-mon[118902]: pgmap v9077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:11.106+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:11.107666+0000) 2023-08-28T01:56:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:11.426Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:11.427Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:11 smithi136 ceph-mon[113531]: pgmap v9078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3372458624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:12.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:12.107919+0000) 2023-08-28T01:56:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:11 smithi093 ceph-mon[111335]: pgmap v9078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3372458624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:11 smithi093 ceph-mon[118902]: pgmap v9078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3372458624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:12.864Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:12.865Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:13.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:13.108171+0000) 2023-08-28T01:56:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:13 smithi136 ceph-mon[113531]: pgmap v9079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1984557944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:14.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:14.108369+0000) 2023-08-28T01:56:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:13 smithi093 ceph-mon[111335]: pgmap v9079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1984557944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:13 smithi093 ceph-mon[118902]: pgmap v9079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1984557944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:15.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:15.108624+0000) 2023-08-28T01:56:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:56:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:56:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:56:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:15 smithi136 ceph-mon[113531]: pgmap v9080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:15 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:56:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3669657155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:16.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:16.108781+0000) 2023-08-28T01:56:16.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:15 smithi093 ceph-mon[118902]: pgmap v9080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:15 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:56:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3669657155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:15 smithi093 ceph-mon[111335]: pgmap v9080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:15 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:56:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3669657155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:17.107+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:17.108984+0000) 2023-08-28T01:56:17.696 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:17 smithi136 ceph-mon[113531]: pgmap v9081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:17.696 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:17.696 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:17.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:17 smithi093 ceph-mon[118902]: pgmap v9081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:17 smithi093 ceph-mon[111335]: pgmap v9081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:18.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:18.109165+0000) 2023-08-28T01:56:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1713821118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:56:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:56:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:19.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:19.109354+0000) 2023-08-28T01:56:19.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1713821118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1713821118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:56:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:56:19.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:56:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:19 smithi136 ceph-mon[113531]: pgmap v9082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:20.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:20.109627+0000) 2023-08-28T01:56:20.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:19 smithi093 ceph-mon[111335]: pgmap v9082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:19 smithi093 ceph-mon[118902]: pgmap v9082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/673296802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:21.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:21.108+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:21.109863+0000) 2023-08-28T01:56:21.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/673296802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/673296802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:21.427Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:21.428Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:21 smithi093 ceph-mon[111335]: pgmap v9083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:21 smithi093 ceph-mon[118902]: pgmap v9083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:21 smithi136 ceph-mon[113531]: pgmap v9083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:22.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:22.110101+0000) 2023-08-28T01:56:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:22.864Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:22.865Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:23.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:23.110320+0000) 2023-08-28T01:56:24.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:23 smithi093 ceph-mon[118902]: pgmap v9084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3514255470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:23 smithi093 ceph-mon[111335]: pgmap v9084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3514255470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:23 smithi136 ceph-mon[113531]: pgmap v9084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3514255470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:24.109+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:24.110558+0000) 2023-08-28T01:56:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:25.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:25.110836+0000) 2023-08-28T01:56:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:56:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:56:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:56:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:25 smithi093 ceph-mon[111335]: pgmap v9085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1844526465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:25 smithi093 ceph-mon[118902]: pgmap v9085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1844526465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:25 smithi136 ceph-mon[113531]: pgmap v9085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1844526465' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:26.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:26.111109+0000) 2023-08-28T01:56:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:27.110+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:27.111336+0000) 2023-08-28T01:56:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:27 smithi136 ceph-mon[113531]: pgmap v9086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:27 smithi093 ceph-mon[111335]: pgmap v9086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:27.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:27 smithi093 ceph-mon[118902]: pgmap v9086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:28.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:28.111593+0000) 2023-08-28T01:56:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1031033545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1031033545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1031033545' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:29.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:29.111814+0000) 2023-08-28T01:56:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:29 smithi136 ceph-mon[113531]: pgmap v9087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:30.062 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:29 smithi093 ceph-mon[111335]: pgmap v9087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:30.062 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:29 smithi093 ceph-mon[118902]: pgmap v9087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:30.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:30.112111+0000) 2023-08-28T01:56:30.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2360157363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2360157363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2360157363' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:31.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:31.112371+0000) 2023-08-28T01:56:31.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:31 smithi093 ceph-mon[111335]: pgmap v9088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:31 smithi093 ceph-mon[118902]: pgmap v9088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:31.428Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:31.429Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:31 smithi136 ceph-mon[113531]: pgmap v9088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:32.111+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:32.112584+0000) 2023-08-28T01:56:32.863 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1032213238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:32.864 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1032213238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:32.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1032213238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:33.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:32.865Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:32.866Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:33.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:33.112860+0000) 2023-08-28T01:56:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:33 smithi136 ceph-mon[113531]: pgmap v9089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:33 smithi093 ceph-mon[111335]: pgmap v9089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:33 smithi093 ceph-mon[118902]: pgmap v9089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:34.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:34.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:34.113169+0000) 2023-08-28T01:56:35.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:35.112+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:35.113397+0000) 2023-08-28T01:56:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:56:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:56:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:56:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:35 smithi136 ceph-mon[113531]: pgmap v9090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1069668683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:36.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:36.113657+0000) 2023-08-28T01:56:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:35 smithi093 ceph-mon[111335]: pgmap v9090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1069668683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:35 smithi093 ceph-mon[118902]: pgmap v9090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1069668683' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:37.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:37.113913+0000) 2023-08-28T01:56:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:37 smithi136 ceph-mon[113531]: pgmap v9091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2531795556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:37 smithi093 ceph-mon[111335]: pgmap v9091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2531795556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:37 smithi093 ceph-mon[118902]: pgmap v9091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2531795556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:38.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:38.114137+0000) 2023-08-28T01:56:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:39.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:39.114375+0000) 2023-08-28T01:56:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:39 smithi093 ceph-mon[111335]: pgmap v9092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2665601306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:39 smithi093 ceph-mon[118902]: pgmap v9092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2665601306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:39 smithi136 ceph-mon[113531]: pgmap v9092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2665601306' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:40.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:40.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:40.114648+0000) 2023-08-28T01:56:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:41.113+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:41.114898+0000) 2023-08-28T01:56:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:41.428Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:41.429Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:41 smithi093 ceph-mon[111335]: pgmap v9093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:41 smithi093 ceph-mon[118902]: pgmap v9093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:41 smithi136 ceph-mon[113531]: pgmap v9093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:42.114+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:42.115177+0000) 2023-08-28T01:56:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2216145542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2216145542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:42.866Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:42.867Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:43.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2216145542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:43.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:43.115412+0000) 2023-08-28T01:56:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:43 smithi093 ceph-mon[111335]: pgmap v9094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:43 smithi093 ceph-mon[118902]: pgmap v9094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:43 smithi136 ceph-mon[113531]: pgmap v9094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:44.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:44.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:44.115613+0000) 2023-08-28T01:56:45.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/602682395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/602682395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/602682395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:45.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:45.115871+0000) 2023-08-28T01:56:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:56:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:56:45] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:56:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:45 smithi093 ceph-mon[111335]: pgmap v9095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:45 smithi093 ceph-mon[118902]: pgmap v9095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:45 smithi136 ceph-mon[113531]: pgmap v9095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:46.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:46.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:46.116120+0000) 2023-08-28T01:56:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:47.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:47.116375+0000) 2023-08-28T01:56:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:47 smithi136 ceph-mon[113531]: pgmap v9096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1649629891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:47 smithi093 ceph-mon[111335]: pgmap v9096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1649629891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:47 smithi093 ceph-mon[118902]: pgmap v9096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1649629891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:48.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:48.116639+0000) 2023-08-28T01:56:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:49.115+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:49.116895+0000) 2023-08-28T01:56:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:49 smithi136 ceph-mon[113531]: pgmap v9097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/668725246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:50.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:50.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:50.117160+0000) 2023-08-28T01:56:50.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:49 smithi093 ceph-mon[111335]: pgmap v9097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/668725246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:49 smithi093 ceph-mon[118902]: pgmap v9097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/668725246' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:51.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:51.117411+0000) 2023-08-28T01:56:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:51.429Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:51.430Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:51 smithi136 ceph-mon[113531]: pgmap v9098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4134022304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:52.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:52.117654+0000) 2023-08-28T01:56:52.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:51 smithi093 ceph-mon[111335]: pgmap v9098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4134022304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:51 smithi093 ceph-mon[118902]: pgmap v9098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4134022304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:52.867Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:56:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:56:52.868Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:56:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:53.116+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:53.117890+0000) 2023-08-28T01:56:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:53 smithi136 ceph-mon[113531]: pgmap v9099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:54.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:54.118127+0000) 2023-08-28T01:56:54.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:53 smithi093 ceph-mon[111335]: pgmap v9099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:53 smithi093 ceph-mon[118902]: pgmap v9099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:56:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/238063243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:55.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:55.118386+0000) 2023-08-28T01:56:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/238063243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/238063243' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:56:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:56:55] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:56:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:55 smithi136 ceph-mon[113531]: pgmap v9100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:56.189 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:56.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:56.118656+0000) 2023-08-28T01:56:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:55 smithi093 ceph-mon[111335]: pgmap v9100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:55 smithi093 ceph-mon[118902]: pgmap v9100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:57.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2170823073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:57.117+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:57.118942+0000) 2023-08-28T01:56:57.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2170823073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2170823073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:57 smithi093 ceph-mon[111335]: pgmap v9101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:57 smithi093 ceph-mon[118902]: pgmap v9101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:58.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:57 smithi136 ceph-mon[113531]: pgmap v9101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:56:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:58.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:58.119207+0000) 2023-08-28T01:56:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2491789374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:59.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2491789374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:59.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2491789374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:56:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:56:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:56:59.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:56:59.119462+0000) 2023-08-28T01:57:00.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:56:59 smithi093 ceph-mon[111335]: pgmap v9102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:56:59 smithi093 ceph-mon[118902]: pgmap v9102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:56:59 smithi136 ceph-mon[113531]: pgmap v9102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:00.118+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:00.119729+0000) 2023-08-28T01:57:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:01.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:01.119979+0000) 2023-08-28T01:57:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:01.430Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:01.431Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:02.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:01 smithi093 ceph-mon[111335]: pgmap v9103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3335870204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:01 smithi093 ceph-mon[118902]: pgmap v9103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3335870204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:01 smithi136 ceph-mon[113531]: pgmap v9103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3335870204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:02.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:02.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:02.120200+0000) 2023-08-28T01:57:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:02.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:02.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:03.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:03.120423+0000) 2023-08-28T01:57:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:03 smithi093 ceph-mon[111335]: pgmap v9104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3215036543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:03 smithi093 ceph-mon[118902]: pgmap v9104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3215036543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:03 smithi136 ceph-mon[113531]: pgmap v9104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3215036543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:04.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:04.120684+0000) 2023-08-28T01:57:05.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:05.119+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:05.120911+0000) 2023-08-28T01:57:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:57:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:57:05] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:57:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:05 smithi093 ceph-mon[111335]: pgmap v9105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/723541002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:05 smithi093 ceph-mon[118902]: pgmap v9105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/723541002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:05 smithi136 ceph-mon[113531]: pgmap v9105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/723541002' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:06.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:06.121174+0000) 2023-08-28T01:57:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:07.120+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:07.121431+0000) 2023-08-28T01:57:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:07 smithi136 ceph-mon[113531]: pgmap v9106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:07 smithi093 ceph-mon[111335]: pgmap v9106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:07 smithi093 ceph-mon[118902]: pgmap v9106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:08.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:08.121679+0000) 2023-08-28T01:57:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/125475775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:08.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/125475775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/125475775' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:09.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:09.121934+0000) 2023-08-28T01:57:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:09 smithi136 ceph-mon[113531]: pgmap v9107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:09 smithi093 ceph-mon[111335]: pgmap v9107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:09 smithi093 ceph-mon[118902]: pgmap v9107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:10.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:10.122193+0000) 2023-08-28T01:57:11.120 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/447585208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:11.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/447585208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/447585208' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:11.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:11.122421+0000) 2023-08-28T01:57:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:11.431Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:11.432Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:12.120 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:11 smithi136 ceph-mon[113531]: pgmap v9108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:11 smithi093 ceph-mon[111335]: pgmap v9108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:11 smithi093 ceph-mon[118902]: pgmap v9108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:12.121+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:12.122644+0000) 2023-08-28T01:57:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:12.868Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:12.869Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:13.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:13.122897+0000) 2023-08-28T01:57:14.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:14.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:14.123138+0000) 2023-08-28T01:57:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:13 smithi136 ceph-mon[113531]: pgmap v9109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:14.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1738213850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:13 smithi093 ceph-mon[111335]: pgmap v9109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1738213850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:13 smithi093 ceph-mon[118902]: pgmap v9109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1738213850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:15.122+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:15.123377+0000) 2023-08-28T01:57:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:57:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:57:15] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:57:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:15 smithi136 ceph-mon[113531]: pgmap v9110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2428834194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:16.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:16.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:16.123576+0000) 2023-08-28T01:57:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:15 smithi093 ceph-mon[111335]: pgmap v9110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2428834194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:15 smithi093 ceph-mon[118902]: pgmap v9110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2428834194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:17.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:17.123814+0000) 2023-08-28T01:57:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:17 smithi136 ceph-mon[113531]: pgmap v9111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:17 smithi093 ceph-mon[111335]: pgmap v9111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:17 smithi093 ceph-mon[118902]: pgmap v9111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:18.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:18.124096+0000) 2023-08-28T01:57:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1384196404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:18 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:57:18.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1384196404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:18.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:18 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:57:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1384196404' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:18 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:57:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:19.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:19.124298+0000) 2023-08-28T01:57:19.894 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:19 smithi136 ceph-mon[113531]: pgmap v9112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:19.963 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:19 smithi093 ceph-mon[111335]: pgmap v9112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:19.963 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:19 smithi093 ceph-mon[118902]: pgmap v9112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:20.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:20.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:20.124478+0000) 2023-08-28T01:57:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:21.123+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:21.124659+0000) 2023-08-28T01:57:21.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3634821703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:21 smithi136 ceph-mon[113531]: pgmap v9113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3634821703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[118902]: pgmap v9113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:21.432Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:21.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:21.433Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3634821703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[111335]: pgmap v9113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:22.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:22.124879+0000) 2023-08-28T01:57:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:57:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:57:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:57:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:57:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:57:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:57:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:57:23.228 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:22.869Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:23.229 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:22.870Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:23.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:23.125089+0000) 2023-08-28T01:57:23.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2470050977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:23 smithi136 ceph-mon[113531]: pgmap v9114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:23.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2470050977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:23 smithi093 ceph-mon[111335]: pgmap v9114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2470050977' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:23 smithi093 ceph-mon[118902]: pgmap v9114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:24.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:24.125352+0000) 2023-08-28T01:57:24.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:25.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:25.125582+0000) 2023-08-28T01:57:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:25 smithi136 ceph-mon[113531]: pgmap v9115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3524395259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:25.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:25 smithi093 ceph-mon[111335]: pgmap v9115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:25.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3524395259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:25.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:57:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:57:25] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:57:25.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:25 smithi093 ceph-mon[118902]: pgmap v9115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:25.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3524395259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:26.124+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:26.125825+0000) 2023-08-28T01:57:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:27.125+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:27.126108+0000) 2023-08-28T01:57:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:27 smithi136 ceph-mon[113531]: pgmap v9116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2531187918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:27 smithi093 ceph-mon[111335]: pgmap v9116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2531187918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:27 smithi093 ceph-mon[118902]: pgmap v9116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2531187918' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:28.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:28.126377+0000) 2023-08-28T01:57:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:29.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:29.126602+0000) 2023-08-28T01:57:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:29 smithi136 ceph-mon[113531]: pgmap v9117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:30.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3066849261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:30.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:30.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:30.126876+0000) 2023-08-28T01:57:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:29 smithi093 ceph-mon[111335]: pgmap v9117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3066849261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:29 smithi093 ceph-mon[118902]: pgmap v9117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3066849261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:31.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:31.127093+0000) 2023-08-28T01:57:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:31.433Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:31.434Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:31 smithi136 ceph-mon[113531]: pgmap v9118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:32.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:32.127327+0000) 2023-08-28T01:57:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:31 smithi093 ceph-mon[111335]: pgmap v9118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:31 smithi093 ceph-mon[118902]: pgmap v9118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:33.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/327510974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:33.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:33.127575+0000) 2023-08-28T01:57:33.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/327510974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/327510974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:32.870Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:32.871Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:33 smithi093 ceph-mon[111335]: pgmap v9119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:33 smithi093 ceph-mon[118902]: pgmap v9119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:33 smithi136 ceph-mon[113531]: pgmap v9119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:34.126+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:34.127889+0000) 2023-08-28T01:57:35.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1479200029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1479200029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:35.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1479200029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:35.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:35.128113+0000) 2023-08-28T01:57:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:57:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:57:35] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T01:57:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:35 smithi093 ceph-mon[111335]: pgmap v9120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:35 smithi093 ceph-mon[118902]: pgmap v9120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:36.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:35 smithi136 ceph-mon[113531]: pgmap v9120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:36.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:36.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:36.128356+0000) 2023-08-28T01:57:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:37.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:37.128601+0000) 2023-08-28T01:57:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:37 smithi136 ceph-mon[113531]: pgmap v9121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1422298058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:37 smithi093 ceph-mon[111335]: pgmap v9121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1422298058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:37 smithi093 ceph-mon[118902]: pgmap v9121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1422298058' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:38.127+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:38.128849+0000) 2023-08-28T01:57:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:39.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:39.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:39.129131+0000) 2023-08-28T01:57:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:39 smithi093 ceph-mon[111335]: pgmap v9122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3319941511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:39 smithi093 ceph-mon[118902]: pgmap v9122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3319941511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:39 smithi136 ceph-mon[113531]: pgmap v9122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3319941511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:40.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:40.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:40.129404+0000) 2023-08-28T01:57:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:41.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:41.129679+0000) 2023-08-28T01:57:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:41.434Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:41.435Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:42.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:41 smithi093 ceph-mon[118902]: pgmap v9123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:42.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/164300302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:41 smithi093 ceph-mon[111335]: pgmap v9123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/164300302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:41 smithi136 ceph-mon[113531]: pgmap v9123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/164300302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:42.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:42.128+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:42.129931+0000) 2023-08-28T01:57:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:42.870Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:42.871Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:43.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:43.130207+0000) 2023-08-28T01:57:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:43 smithi093 ceph-mon[111335]: pgmap v9124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:43 smithi093 ceph-mon[118902]: pgmap v9124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:43 smithi136 ceph-mon[113531]: pgmap v9124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:44.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:44.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:44.130467+0000) 2023-08-28T01:57:45.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/86937105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/86937105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/86937105' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:45.129+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:45.130725+0000) 2023-08-28T01:57:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:57:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:57:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:57:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:45 smithi093 ceph-mon[111335]: pgmap v9125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:45 smithi093 ceph-mon[118902]: pgmap v9125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:45 smithi136 ceph-mon[113531]: pgmap v9125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:46.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:46.131011+0000) 2023-08-28T01:57:47.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/493263982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:47.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/493263982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:47.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/493263982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:47.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:47.131318+0000) 2023-08-28T01:57:48.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:47 smithi093 ceph-mon[111335]: pgmap v9126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:47 smithi093 ceph-mon[118902]: pgmap v9126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:48.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:47 smithi136 ceph-mon[113531]: pgmap v9126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:48.130+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:48.131531+0000) 2023-08-28T01:57:49.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/716064074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:49.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/716064074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:49.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/716064074' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:49.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:49.131792+0000) 2023-08-28T01:57:50.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:49 smithi093 ceph-mon[111335]: pgmap v9127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:49 smithi093 ceph-mon[118902]: pgmap v9127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:49 smithi136 ceph-mon[113531]: pgmap v9127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:50.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:50.132088+0000) 2023-08-28T01:57:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:51.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:51.132370+0000) 2023-08-28T01:57:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:51.435Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:51.435Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:51 smithi093 ceph-mon[111335]: pgmap v9128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2346152574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:51 smithi093 ceph-mon[118902]: pgmap v9128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2346152574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:51 smithi136 ceph-mon[113531]: pgmap v9128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2346152574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:52.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:52.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:52.132683+0000) 2023-08-28T01:57:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:52.871Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:57:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:57:52.873Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:57:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:53.131+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:53.132932+0000) 2023-08-28T01:57:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:53 smithi136 ceph-mon[113531]: pgmap v9129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3848078981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:54.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:54.133212+0000) 2023-08-28T01:57:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:53 smithi093 ceph-mon[111335]: pgmap v9129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3848078981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:53 smithi093 ceph-mon[118902]: pgmap v9129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3848078981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:57:55.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:55.132+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:55.133390+0000) 2023-08-28T01:57:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:57:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:57:55] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:57:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:55 smithi136 ceph-mon[113531]: pgmap v9130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12541967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:56.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:56.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:56.133673+0000) 2023-08-28T01:57:56.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:55 smithi093 ceph-mon[111335]: pgmap v9130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12541967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:55 smithi093 ceph-mon[118902]: pgmap v9130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12541967' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:57.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:57.133934+0000) 2023-08-28T01:57:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:57 smithi136 ceph-mon[113531]: pgmap v9131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:57 smithi093 ceph-mon[111335]: pgmap v9131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:57 smithi093 ceph-mon[118902]: pgmap v9131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:58.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:58.134184+0000) 2023-08-28T01:57:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2104105968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:58.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2104105968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2104105968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:57:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:57:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:57:59.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:57:59.134437+0000) 2023-08-28T01:57:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:57:59 smithi136 ceph-mon[113531]: pgmap v9132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:59.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:57:59 smithi093 ceph-mon[111335]: pgmap v9132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:57:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:57:59 smithi093 ceph-mon[118902]: pgmap v9132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:00.133+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:00.134698+0000) 2023-08-28T01:58:01.133 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1381124898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1381124898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1381124898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:01.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:01.134891+0000) 2023-08-28T01:58:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:01.436Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:01.437Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:02.133 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:01 smithi136 ceph-mon[113531]: pgmap v9133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:01 smithi093 ceph-mon[111335]: pgmap v9133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:01 smithi093 ceph-mon[118902]: pgmap v9133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:02.134+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:02.135164+0000) 2023-08-28T01:58:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:02.872Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:02.873Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:03.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:03.135392+0000) 2023-08-28T01:58:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:03 smithi136 ceph-mon[113531]: pgmap v9134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2377559409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:04.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:04.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:04.135648+0000) 2023-08-28T01:58:04.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:03 smithi093 ceph-mon[111335]: pgmap v9134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:04.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2377559409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:03 smithi093 ceph-mon[118902]: pgmap v9134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2377559409' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:05.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:05.135898+0000) 2023-08-28T01:58:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:58:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:58:05] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T01:58:06.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:05 smithi093 ceph-mon[118902]: pgmap v9135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4101595041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:05 smithi093 ceph-mon[111335]: pgmap v9135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4101595041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:05 smithi136 ceph-mon[113531]: pgmap v9135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4101595041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:06.135+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:06.136155+0000) 2023-08-28T01:58:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:07.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:07.136407+0000) 2023-08-28T01:58:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:07 smithi136 ceph-mon[113531]: pgmap v9136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:07 smithi093 ceph-mon[111335]: pgmap v9136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:07 smithi093 ceph-mon[118902]: pgmap v9136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:08.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:08.136671+0000) 2023-08-28T01:58:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/618461411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:08.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/618461411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/618461411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:09.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:09.136945+0000) 2023-08-28T01:58:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:09 smithi136 ceph-mon[113531]: pgmap v9137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:09 smithi093 ceph-mon[111335]: pgmap v9137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:09 smithi093 ceph-mon[118902]: pgmap v9137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:10.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:10.137196+0000) 2023-08-28T01:58:10.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1464569962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:10.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1464569962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1464569962' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:11.136+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:11.137433+0000) 2023-08-28T01:58:11.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:11 smithi093 ceph-mon[111335]: pgmap v9138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:11 smithi093 ceph-mon[118902]: pgmap v9138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:11.436Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:11.437Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:11 smithi136 ceph-mon[113531]: pgmap v9138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:12.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:12.137674+0000) 2023-08-28T01:58:12.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1918153055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:12.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1918153055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:12.872Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:12.873Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:13.135 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1918153055' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:13.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:13.137922+0000) 2023-08-28T01:58:13.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:13 smithi093 ceph-mon[111335]: pgmap v9139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:13 smithi093 ceph-mon[118902]: pgmap v9139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:14.136 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:13 smithi136 ceph-mon[113531]: pgmap v9139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:14.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:14.138128+0000) 2023-08-28T01:58:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:15.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:15.138333+0000) 2023-08-28T01:58:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:58:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:58:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:58:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:15 smithi093 ceph-mon[111335]: pgmap v9140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1109039160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:15 smithi093 ceph-mon[118902]: pgmap v9140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1109039160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:16.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:16.138557+0000) 2023-08-28T01:58:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:15 smithi136 ceph-mon[113531]: pgmap v9140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1109039160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:17.137+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:17.138783+0000) 2023-08-28T01:58:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:17 smithi136 ceph-mon[113531]: pgmap v9141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/591852016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:17 smithi093 ceph-mon[111335]: pgmap v9141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/591852016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:17 smithi093 ceph-mon[118902]: pgmap v9141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/591852016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:18.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:18.139078+0000) 2023-08-28T01:58:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:19.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:19.139321+0000) 2023-08-28T01:58:20.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:19 smithi093 ceph-mon[111335]: pgmap v9142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4124067417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:19 smithi093 ceph-mon[118902]: pgmap v9142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4124067417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:19 smithi136 ceph-mon[113531]: pgmap v9142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4124067417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:20.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:20.139585+0000) 2023-08-28T01:58:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:21.138+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:21.139798+0000) 2023-08-28T01:58:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:21.437Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:21.438Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:21 smithi093 ceph-mon[118902]: pgmap v9143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:21 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:58:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:21 smithi093 ceph-mon[111335]: pgmap v9143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:21 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:58:22.423 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:21 smithi136 ceph-mon[113531]: pgmap v9143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:22.423 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:21 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:58:22.424 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:22.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:22.140056+0000) 2023-08-28T01:58:22.949 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:22.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:22.949 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:22.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:23.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2862862207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:23.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:23.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:23.140231+0000) 2023-08-28T01:58:23.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2862862207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:23.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2862862207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:24.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[118902]: pgmap v9144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:24.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[111335]: pgmap v9144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:24.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:24.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:24.140413+0000) 2023-08-28T01:58:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:23 smithi136 ceph-mon[113531]: pgmap v9144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:58:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:58:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/666519496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:58:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:58:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/666519496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:58:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:58:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:58:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/666519496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:25.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:25.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:25.140674+0000) 2023-08-28T01:58:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:58:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:58:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:58:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:25 smithi093 ceph-mon[111335]: pgmap v9145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:25 smithi093 ceph-mon[118902]: pgmap v9145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:25 smithi136 ceph-mon[113531]: pgmap v9145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:26.139+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:26.140975+0000) 2023-08-28T01:58:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:27.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:27.141224+0000) 2023-08-28T01:58:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:27 smithi136 ceph-mon[113531]: pgmap v9146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1865031136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:27 smithi093 ceph-mon[111335]: pgmap v9146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1865031136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:27 smithi093 ceph-mon[118902]: pgmap v9146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1865031136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:28.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:28.141484+0000) 2023-08-28T01:58:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:29.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:29.141727+0000) 2023-08-28T01:58:30.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:29 smithi093 ceph-mon[111335]: pgmap v9147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2931324540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:29 smithi093 ceph-mon[118902]: pgmap v9147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2931324540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:29 smithi136 ceph-mon[113531]: pgmap v9147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2931324540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:30.140+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:30.141984+0000) 2023-08-28T01:58:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:31.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:31.142281+0000) 2023-08-28T01:58:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:31.437Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:31.439Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:32.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:31 smithi093 ceph-mon[111335]: pgmap v9148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:32.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1338737815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:31 smithi093 ceph-mon[118902]: pgmap v9148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1338737815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:31 smithi136 ceph-mon[113531]: pgmap v9148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1338737815' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:32.141+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:32.142532+0000) 2023-08-28T01:58:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:32.873Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:32.874Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:33.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:33.142774+0000) 2023-08-28T01:58:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:33 smithi093 ceph-mon[111335]: pgmap v9149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:33 smithi093 ceph-mon[118902]: pgmap v9149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:33 smithi136 ceph-mon[113531]: pgmap v9149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:34.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:34.143092+0000) 2023-08-28T01:58:35.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/28985529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/28985529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:35.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/28985529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:35.142+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:35.143383+0000) 2023-08-28T01:58:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:58:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:58:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:58:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:35 smithi093 ceph-mon[111335]: pgmap v9150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:35 smithi093 ceph-mon[118902]: pgmap v9150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:35 smithi136 ceph-mon[113531]: pgmap v9150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:36.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:36.143675+0000) 2023-08-28T01:58:37.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/920312973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/920312973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:37.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/920312973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:37.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:37.143951+0000) 2023-08-28T01:58:38.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:37 smithi093 ceph-mon[111335]: pgmap v9151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:37 smithi093 ceph-mon[118902]: pgmap v9151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:37 smithi136 ceph-mon[113531]: pgmap v9151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:38.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:38.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:38.144201+0000) 2023-08-28T01:58:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3419265788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3419265788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3419265788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:39.143+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:39.144448+0000) 2023-08-28T01:58:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:39 smithi093 ceph-mon[111335]: pgmap v9152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:39 smithi093 ceph-mon[118902]: pgmap v9152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:39 smithi136 ceph-mon[113531]: pgmap v9152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:40.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:40.144701+0000) 2023-08-28T01:58:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:41.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:41.144882+0000) 2023-08-28T01:58:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:41.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:41.439Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:41 smithi093 ceph-mon[111335]: pgmap v9153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3827382253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:41 smithi093 ceph-mon[118902]: pgmap v9153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3827382253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:41 smithi136 ceph-mon[113531]: pgmap v9153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3827382253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:42.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:42.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:42.145181+0000) 2023-08-28T01:58:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:42.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:42.875Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:43.144+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:43.145432+0000) 2023-08-28T01:58:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:43 smithi093 ceph-mon[111335]: pgmap v9154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/783668473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:43 smithi093 ceph-mon[118902]: pgmap v9154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/783668473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:43 smithi136 ceph-mon[113531]: pgmap v9154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/783668473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:44.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:44.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:44.145698+0000) 2023-08-28T01:58:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:45.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:45.145920+0000) 2023-08-28T01:58:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:58:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:58:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:58:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:45 smithi093 ceph-mon[111335]: pgmap v9155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:45 smithi093 ceph-mon[118902]: pgmap v9155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:45 smithi136 ceph-mon[113531]: pgmap v9155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:46.145+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:46.146198+0000) 2023-08-28T01:58:47.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2645630349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2645630349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:47.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2645630349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:47.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:47.146453+0000) 2023-08-28T01:58:48.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:47 smithi093 ceph-mon[111335]: pgmap v9156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:47 smithi093 ceph-mon[118902]: pgmap v9156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:48.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:47 smithi136 ceph-mon[113531]: pgmap v9156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:48.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:48.146699+0000) 2023-08-28T01:58:49.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/184104763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/184104763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:49.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/184104763' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:49.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:49.146952+0000) 2023-08-28T01:58:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:49 smithi093 ceph-mon[111335]: pgmap v9157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:49 smithi093 ceph-mon[118902]: pgmap v9157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:49 smithi136 ceph-mon[113531]: pgmap v9157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:50.146+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:50.147219+0000) 2023-08-28T01:58:51.437 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2713443154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:51.437 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2713443154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2713443154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:51.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:51.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:51.147515+0000) 2023-08-28T01:58:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:51.438Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:51.439Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:51 smithi136 ceph-mon[113531]: pgmap v9158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:52.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:52.147775+0000) 2023-08-28T01:58:52.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:51 smithi093 ceph-mon[111335]: pgmap v9158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:51 smithi093 ceph-mon[118902]: pgmap v9158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:52.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:58:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:58:52.875Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:58:53.442 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:53.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:53.148064+0000) 2023-08-28T01:58:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:53 smithi136 ceph-mon[113531]: pgmap v9159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4250290101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:54.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:54.148346+0000) 2023-08-28T01:58:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:53 smithi093 ceph-mon[111335]: pgmap v9159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4250290101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:54 smithi093 ceph-mon[118902]: pgmap v9159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4250290101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:58:55.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:55.147+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:55.148630+0000) 2023-08-28T01:58:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:58:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:58:55] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:58:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:56 smithi136 ceph-mon[113531]: pgmap v9160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3337610355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:56.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:56.148891+0000) 2023-08-28T01:58:56.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:56 smithi093 ceph-mon[118902]: pgmap v9160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3337610355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:56 smithi093 ceph-mon[111335]: pgmap v9160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3337610355' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:57.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:57.149173+0000) 2023-08-28T01:58:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:57 smithi136 ceph-mon[113531]: pgmap v9161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:57 smithi093 ceph-mon[111335]: pgmap v9161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:57 smithi093 ceph-mon[118902]: pgmap v9161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:58.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:58.149396+0000) 2023-08-28T01:58:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2271394421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:58.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2271394421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2271394421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:58:59.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:58:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:58:59.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:58:59.149628+0000) 2023-08-28T01:58:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:58:59 smithi136 ceph-mon[113531]: pgmap v9162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:59.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:58:59 smithi093 ceph-mon[111335]: pgmap v9162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:58:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:58:59 smithi093 ceph-mon[118902]: pgmap v9162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:00.148+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:00.149902+0000) 2023-08-28T01:59:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1446575976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:00.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1446575976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1446575976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:01.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:01.150156+0000) 2023-08-28T01:59:01.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:01 smithi093 ceph-mon[111335]: pgmap v9163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:01 smithi093 ceph-mon[118902]: pgmap v9163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:01.439Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:01.440Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:01 smithi136 ceph-mon[113531]: pgmap v9163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:02.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:02.150389+0000) 2023-08-28T01:59:03.148 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3411282135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:03.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3411282135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3411282135' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:02.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:02.875Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:03.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:03.150659+0000) 2023-08-28T01:59:04.149 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:03 smithi136 ceph-mon[113531]: pgmap v9164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:04.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:03 smithi093 ceph-mon[111335]: pgmap v9164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:03 smithi093 ceph-mon[118902]: pgmap v9164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:04.149+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:04.150909+0000) 2023-08-28T01:59:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:05.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:05.151168+0000) 2023-08-28T01:59:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:59:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:59:05] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T01:59:06.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:05 smithi093 ceph-mon[111335]: pgmap v9165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1690112819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:05 smithi093 ceph-mon[118902]: pgmap v9165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1690112819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:05 smithi136 ceph-mon[113531]: pgmap v9165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1690112819' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:06.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:06.151416+0000) 2023-08-28T01:59:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:07.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:07.151613+0000) 2023-08-28T01:59:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:07 smithi136 ceph-mon[113531]: pgmap v9166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2469129089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:07 smithi093 ceph-mon[111335]: pgmap v9166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2469129089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:07.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:07 smithi093 ceph-mon[118902]: pgmap v9166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2469129089' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:08.150+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:08.151838+0000) 2023-08-28T01:59:09.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:09.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:09.152128+0000) 2023-08-28T01:59:10.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:09 smithi093 ceph-mon[118902]: pgmap v9167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2922333998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:09 smithi093 ceph-mon[111335]: pgmap v9167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2922333998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:09 smithi136 ceph-mon[113531]: pgmap v9167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2922333998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:10.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:10.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:10.152381+0000) 2023-08-28T01:59:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:11.151+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:11.152616+0000) 2023-08-28T01:59:11.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:11.440Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:11.441Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:11 smithi093 ceph-mon[111335]: pgmap v9168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:11 smithi093 ceph-mon[118902]: pgmap v9168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:11 smithi136 ceph-mon[113531]: pgmap v9168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:12.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:12.152873+0000) 2023-08-28T01:59:13.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/608941997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/608941997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:12.874Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:12.875Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:13.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/608941997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:13.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:13.153140+0000) 2023-08-28T01:59:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:13 smithi093 ceph-mon[111335]: pgmap v9169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:13 smithi093 ceph-mon[118902]: pgmap v9169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:13 smithi136 ceph-mon[113531]: pgmap v9169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:14.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:14.153391+0000) 2023-08-28T01:59:15.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2065685856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2065685856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:15.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2065685856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:15.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:15.153644+0000) 2023-08-28T01:59:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:59:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:59:15] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T01:59:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:15 smithi093 ceph-mon[111335]: pgmap v9170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:15 smithi093 ceph-mon[118902]: pgmap v9170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:15 smithi136 ceph-mon[113531]: pgmap v9170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:16.152+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:16.153911+0000) 2023-08-28T01:59:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:17.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:17.154175+0000) 2023-08-28T01:59:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:17 smithi136 ceph-mon[113531]: pgmap v9171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4064046124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:17 smithi093 ceph-mon[111335]: pgmap v9171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4064046124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:17 smithi093 ceph-mon[118902]: pgmap v9171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4064046124' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:18.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:18.154396+0000) 2023-08-28T01:59:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:19.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:19.154670+0000) 2023-08-28T01:59:20.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:19 smithi093 ceph-mon[111335]: pgmap v9172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2669654039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:19 smithi093 ceph-mon[118902]: pgmap v9172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2669654039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:19 smithi136 ceph-mon[113531]: pgmap v9172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2669654039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:20.153+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:20.154873+0000) 2023-08-28T01:59:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:21.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:21.155151+0000) 2023-08-28T01:59:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:21.441Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:21.442Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:22.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:21 smithi093 ceph-mon[118902]: pgmap v9173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2523637496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:21 smithi093 ceph-mon[111335]: pgmap v9173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2523637496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:21 smithi136 ceph-mon[113531]: pgmap v9173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2523637496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:22.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:22.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:22.155427+0000) 2023-08-28T01:59:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:22.875Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:22.877Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:23.154+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:23.155668+0000) 2023-08-28T01:59:24.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:23 smithi093 ceph-mon[111335]: pgmap v9174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:23 smithi093 ceph-mon[118902]: pgmap v9174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:23 smithi136 ceph-mon[113531]: pgmap v9174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:24.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:24.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:24.155945+0000) 2023-08-28T01:59:25.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3099583052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:59:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3099583052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:59:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3099583052' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T01:59:25.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:25.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:25.156212+0000) 2023-08-28T01:59:25.641 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:59:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:59:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:59:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:25 smithi136 ceph-mon[113531]: pgmap v9175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:26.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:26.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:26.156379+0000) 2023-08-28T01:59:26.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:25 smithi093 ceph-mon[111335]: pgmap v9175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:26.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:25 smithi093 ceph-mon[118902]: pgmap v9175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:27.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:27.156541+0000) 2023-08-28T01:59:27.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/151358805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:27.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: pgmap v9176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2396M 2023-08-28T01:59:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/151358805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: pgmap v9176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2396M 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/151358805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: pgmap v9176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T01:59:27.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2396M 2023-08-28T01:59:27.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:28.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:28.156682+0000) 2023-08-28T01:59:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:28.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:28.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.292 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:29.155+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:29.156866+0000) 2023-08-28T01:59:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-mon[113531]: pgmap v9177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1591858668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:59:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:59:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[118902]: pgmap v9177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:29.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1591858668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[111335]: pgmap v9177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1591858668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T01:59:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T01:59:29.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T01:59:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:30.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:30.157127+0000) 2023-08-28T01:59:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:31.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:31.157378+0000) 2023-08-28T01:59:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:31.441Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:31.925 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:31.442Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:31 smithi093 ceph-mon[111335]: pgmap v9178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1230647692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:31 smithi093 ceph-mon[118902]: pgmap v9178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1230647692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:32.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:32.157608+0000) 2023-08-28T01:59:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:31 smithi136 ceph-mon[113531]: pgmap v9178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1230647692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:33.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:32.876Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:32.877Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:33.156+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:33.157903+0000) 2023-08-28T01:59:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:33 smithi093 ceph-mon[111335]: pgmap v9179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:34.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1830742754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:33 smithi093 ceph-mon[118902]: pgmap v9179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1830742754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:33 smithi136 ceph-mon[113531]: pgmap v9179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1830742754' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:34.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:34.157+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:34.158173+0000) 2023-08-28T01:59:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:35.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:35.158399+0000) 2023-08-28T01:59:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:59:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:59:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T01:59:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:35 smithi093 ceph-mon[111335]: pgmap v9180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:35 smithi093 ceph-mon[118902]: pgmap v9180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:35 smithi136 ceph-mon[113531]: pgmap v9180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:36.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:36.158638+0000) 2023-08-28T01:59:37.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2940061571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2940061571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:37.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2940061571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:37.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:37.158898+0000) 2023-08-28T01:59:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:37 smithi093 ceph-mon[111335]: pgmap v9181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:37 smithi093 ceph-mon[118902]: pgmap v9181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:38.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:37 smithi136 ceph-mon[113531]: pgmap v9181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:38.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:38.159157+0000) 2023-08-28T01:59:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1756584308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1756584308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1756584308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:39.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:39.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:39.159432+0000) 2023-08-28T01:59:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:39 smithi093 ceph-mon[111335]: pgmap v9182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:39 smithi093 ceph-mon[118902]: pgmap v9182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:39 smithi136 ceph-mon[113531]: pgmap v9182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:40.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:40.159667+0000) 2023-08-28T01:59:41.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1419172181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1419172181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:41.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1419172181' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:41.158+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:41.159910+0000) 2023-08-28T01:59:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:41.443Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:41.443Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:41 smithi093 ceph-mon[111335]: pgmap v9183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:41 smithi093 ceph-mon[118902]: pgmap v9183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:41 smithi136 ceph-mon[113531]: pgmap v9183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:42.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:42.160158+0000) 2023-08-28T01:59:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:42.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:42.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:43.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:43.160405+0000) 2023-08-28T01:59:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:43 smithi136 ceph-mon[113531]: pgmap v9184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/434062586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:44.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:44.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:44.160668+0000) 2023-08-28T01:59:44.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:43 smithi093 ceph-mon[111335]: pgmap v9184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:44.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/434062586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:43 smithi093 ceph-mon[118902]: pgmap v9184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/434062586' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:45.159+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:45.160924+0000) 2023-08-28T01:59:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:59:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:59:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T01:59:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:45 smithi136 ceph-mon[113531]: pgmap v9185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3675635449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:46.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:46.161217+0000) 2023-08-28T01:59:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:45 smithi093 ceph-mon[111335]: pgmap v9185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3675635449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:45 smithi093 ceph-mon[118902]: pgmap v9185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3675635449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:47.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:47.161453+0000) 2023-08-28T01:59:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:47 smithi136 ceph-mon[113531]: pgmap v9186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:47 smithi093 ceph-mon[111335]: pgmap v9186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:47.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:47 smithi093 ceph-mon[118902]: pgmap v9186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:48.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:48.161645+0000) 2023-08-28T01:59:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3938049090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:48.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3938049090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3938049090' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:49.160+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:49.161870+0000) 2023-08-28T01:59:49.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:49 smithi136 ceph-mon[113531]: pgmap v9187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:49.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:49 smithi093 ceph-mon[111335]: pgmap v9187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:49 smithi093 ceph-mon[118902]: pgmap v9187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:50.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:50.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:50.162080+0000) 2023-08-28T01:59:50.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1117284232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:50.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1117284232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:50.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1117284232' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:51.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:51.162320+0000) 2023-08-28T01:59:51.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:51 smithi093 ceph-mon[111335]: pgmap v9188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:51 smithi093 ceph-mon[118902]: pgmap v9188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:51.443Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:51.444Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:51.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:51 smithi136 ceph-mon[113531]: pgmap v9188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:52.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:52.162541+0000) 2023-08-28T01:59:53.160 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3929522198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:53.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3929522198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3929522198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:52.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 01:59:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T01:59:52.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T01:59:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:53.161+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:53.162761+0000) 2023-08-28T01:59:54.161 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:53 smithi136 ceph-mon[113531]: pgmap v9189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:54.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:53 smithi093 ceph-mon[111335]: pgmap v9189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:54.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:53 smithi093 ceph-mon[118902]: pgmap v9189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:54.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:54.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:54.163020+0000) 2023-08-28T01:59:55.161 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:55.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T01:59:55.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:55.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:55.163300+0000) 2023-08-28T01:59:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 01:59:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:01:59:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T01:59:56.161 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:55 smithi136 ceph-mon[113531]: pgmap v9190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:56.161 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1991631281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:56.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:55 smithi093 ceph-mon[118902]: pgmap v9190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:56.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1991631281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:55 smithi093 ceph-mon[111335]: pgmap v9190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1991631281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:56.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:56.163537+0000) 2023-08-28T01:59:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:57.162+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:57.163789+0000) 2023-08-28T01:59:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:57 smithi136 ceph-mon[113531]: pgmap v9191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2983876689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:57 smithi093 ceph-mon[111335]: pgmap v9191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2983876689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:57 smithi093 ceph-mon[118902]: pgmap v9191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T01:59:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2983876689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T01:59:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:58.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:58.163987+0000) 2023-08-28T01:59:59.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 01:59:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T01:59:59.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T00:59:59.164237+0000) 2023-08-28T02:00:00.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:59 smithi093 ceph-mon[111335]: pgmap v9192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 01:59:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1078828304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:59 smithi093 ceph-mon[118902]: pgmap v9192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 01:59:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1078828304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:59 smithi136 ceph-mon[113531]: pgmap v9192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 01:59:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1078828304' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:00.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:00.164484+0000) 2023-08-28T02:00:01.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:00:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:00:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:00:01.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:01.163+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:01.164697+0000) 2023-08-28T02:00:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:01.445Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:01.446Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:02.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:01 smithi093 ceph-mon[111335]: pgmap v9193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:02.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:01 smithi093 ceph-mon[118902]: pgmap v9193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:01 smithi136 ceph-mon[113531]: pgmap v9193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:02.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:02.164934+0000) 2023-08-28T02:00:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1706801284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1706801284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:02.878Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:02.879Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:03.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1706801284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:03.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:03.165166+0000) 2023-08-28T02:00:04.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:03 smithi093 ceph-mon[111335]: pgmap v9194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:03 smithi093 ceph-mon[118902]: pgmap v9194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:03 smithi136 ceph-mon[113531]: pgmap v9194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:04.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:04.165382+0000) 2023-08-28T02:00:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2665591215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:05.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2665591215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:05.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2665591215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:05.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:05.165618+0000) 2023-08-28T02:00:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:00:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:00:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:00:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:05 smithi136 ceph-mon[113531]: pgmap v9195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:06.164+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:06.165849+0000) 2023-08-28T02:00:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:05 smithi093 ceph-mon[111335]: pgmap v9195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:06.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:05 smithi093 ceph-mon[118902]: pgmap v9195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:07.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:07.166122+0000) 2023-08-28T02:00:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:07 smithi136 ceph-mon[113531]: pgmap v9196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/412441898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:07 smithi093 ceph-mon[111335]: pgmap v9196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/412441898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:07 smithi093 ceph-mon[118902]: pgmap v9196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/412441898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:08.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:08.166377+0000) 2023-08-28T02:00:09.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:09.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:09.166606+0000) 2023-08-28T02:00:10.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:09 smithi093 ceph-mon[111335]: pgmap v9197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/905892199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:09 smithi093 ceph-mon[118902]: pgmap v9197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/905892199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:09 smithi136 ceph-mon[113531]: pgmap v9197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/905892199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:10.165+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:10.166857+0000) 2023-08-28T02:00:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:11.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:11.167130+0000) 2023-08-28T02:00:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:11.445Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:11.446Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:12.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:11 smithi093 ceph-mon[111335]: pgmap v9198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1012782662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:11 smithi093 ceph-mon[118902]: pgmap v9198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1012782662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:11 smithi136 ceph-mon[113531]: pgmap v9198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1012782662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:12.166+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:12.167389+0000) 2023-08-28T02:00:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:12.879Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:12.880Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:13.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:13.167594+0000) 2023-08-28T02:00:14.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:13 smithi093 ceph-mon[111335]: pgmap v9199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:13 smithi093 ceph-mon[118902]: pgmap v9199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:13 smithi136 ceph-mon[113531]: pgmap v9199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:14.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:14.167840+0000) 2023-08-28T02:00:15.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4196585996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4196585996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:15.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4196585996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:15.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:15.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:15.168071+0000) 2023-08-28T02:00:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:00:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:00:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:00:16.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:15 smithi093 ceph-mon[111335]: pgmap v9200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:15 smithi093 ceph-mon[118902]: pgmap v9200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:15 smithi136 ceph-mon[113531]: pgmap v9200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:16.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:16.168341+0000) 2023-08-28T02:00:17.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4253759629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:17.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4253759629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:17.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4253759629' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:17.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:17.168553+0000) 2023-08-28T02:00:18.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:17 smithi093 ceph-mon[111335]: pgmap v9201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:17 smithi093 ceph-mon[118902]: pgmap v9201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:18.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:17 smithi136 ceph-mon[113531]: pgmap v9201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:18.167+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:18.168809+0000) 2023-08-28T02:00:19.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3562785352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:19.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:19.169077+0000) 2023-08-28T02:00:19.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3562785352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:19.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3562785352' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:19 smithi136 ceph-mon[113531]: pgmap v9202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:20.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:20.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:20.169316+0000) 2023-08-28T02:00:20.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:19 smithi093 ceph-mon[118902]: pgmap v9202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:20.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:19 smithi093 ceph-mon[111335]: pgmap v9202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:21.168+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:21.169572+0000) 2023-08-28T02:00:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:21.446Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:21.447Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:21 smithi136 ceph-mon[113531]: pgmap v9203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/722608496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:22.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:22.169806+0000) 2023-08-28T02:00:22.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:21 smithi093 ceph-mon[111335]: pgmap v9203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/722608496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:21 smithi093 ceph-mon[118902]: pgmap v9203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/722608496' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:22.880Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:22.881Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:23.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:23.170051+0000) 2023-08-28T02:00:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:23 smithi136 ceph-mon[113531]: pgmap v9204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/818072071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:24.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:24.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:24.170295+0000) 2023-08-28T02:00:24.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:23 smithi093 ceph-mon[111335]: pgmap v9204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/818072071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:23 smithi093 ceph-mon[118902]: pgmap v9204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/818072071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:25.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:25.170504+0000) 2023-08-28T02:00:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:00:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:00:25] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T02:00:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:26 smithi136 ceph-mon[113531]: pgmap v9205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:26.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:26.170745+0000) 2023-08-28T02:00:26.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:26 smithi093 ceph-mon[111335]: pgmap v9205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:26 smithi093 ceph-mon[118902]: pgmap v9205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:27.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3083106484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:27.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:27.169+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:27.170994+0000) 2023-08-28T02:00:27.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3083106484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3083106484' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:28.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:28 smithi136 ceph-mon[113531]: pgmap v9206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:28.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:28.171199+0000) 2023-08-28T02:00:28.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:28 smithi093 ceph-mon[111335]: pgmap v9206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:28.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:28 smithi093 ceph-mon[118902]: pgmap v9206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:29.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2861069318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:29.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:29.171456+0000) 2023-08-28T02:00:29.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2861069318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2861069318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:30.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:30 smithi093 ceph-mon[111335]: pgmap v9207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:30.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:30 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:00:30.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:30 smithi093 ceph-mon[118902]: pgmap v9207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:30.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:30 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:00:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:30 smithi136 ceph-mon[113531]: pgmap v9207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:30 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:00:30.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:30.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:30.171638+0000) 2023-08-28T02:00:31.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/155552356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/155552356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:31.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/155552356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:31.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:31.171828+0000) 2023-08-28T02:00:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:31.447Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:31.448Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:32.170 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:32 smithi136 ceph-mon[113531]: pgmap v9208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:32.170 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:32.170 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:32.170+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:32.172011+0000) 2023-08-28T02:00:32.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-mon[111335]: pgmap v9208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:32.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-mon[118902]: pgmap v9208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:00:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:00:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:32.881Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:32.882Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:00:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:00:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:33 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:33.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:33.172215+0000) 2023-08-28T02:00:33.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:00:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:00:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:33 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:00:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:34 smithi136 ceph-mon[113531]: pgmap v9209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4011076126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:34.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:34.172493+0000) 2023-08-28T02:00:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:34 smithi093 ceph-mon[111335]: pgmap v9209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4011076126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:34 smithi093 ceph-mon[118902]: pgmap v9209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4011076126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:35.171+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:35.172694+0000) 2023-08-28T02:00:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:00:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:00:35] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T02:00:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:36 smithi136 ceph-mon[113531]: pgmap v9210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2644363369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:36.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:36.172923+0000) 2023-08-28T02:00:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:36 smithi093 ceph-mon[111335]: pgmap v9210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2644363369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:36 smithi093 ceph-mon[118902]: pgmap v9210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2644363369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:37.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:37.173178+0000) 2023-08-28T02:00:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:37 smithi136 ceph-mon[113531]: pgmap v9211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:37 smithi093 ceph-mon[111335]: pgmap v9211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:37 smithi093 ceph-mon[118902]: pgmap v9211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:38.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:38.173476+0000) 2023-08-28T02:00:38.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3479465358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:38.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3479465358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3479465358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:39.172+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:39.173749+0000) 2023-08-28T02:00:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:39 smithi136 ceph-mon[113531]: pgmap v9212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:39 smithi093 ceph-mon[111335]: pgmap v9212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:39 smithi093 ceph-mon[118902]: pgmap v9212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:40.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:40.174013+0000) 2023-08-28T02:00:40.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/904616227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:40.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/904616227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:40.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/904616227' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:41.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:41.174257+0000) 2023-08-28T02:00:41.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:41 smithi093 ceph-mon[111335]: pgmap v9213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:41 smithi093 ceph-mon[118902]: pgmap v9213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:41.448Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:41.449Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:41 smithi136 ceph-mon[113531]: pgmap v9213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:42.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:42.174464+0000) 2023-08-28T02:00:43.172 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/11387451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:43.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/11387451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/11387451' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:42.882Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:42.883Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:43.173+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:43.174673+0000) 2023-08-28T02:00:44.173 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:43 smithi136 ceph-mon[113531]: pgmap v9214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:44.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:43 smithi093 ceph-mon[111335]: pgmap v9214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:44.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:43 smithi093 ceph-mon[118902]: pgmap v9214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:44.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:44.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:44.174942+0000) 2023-08-28T02:00:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:45.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:45.175165+0000) 2023-08-28T02:00:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:00:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:00:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:00:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:45 smithi093 ceph-mon[111335]: pgmap v9215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2402608692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:45 smithi093 ceph-mon[118902]: pgmap v9215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2402608692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:46.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:46.175365+0000) 2023-08-28T02:00:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:45 smithi136 ceph-mon[113531]: pgmap v9215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2402608692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:47.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:47.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:47.175554+0000) 2023-08-28T02:00:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:47 smithi136 ceph-mon[113531]: pgmap v9216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/159900615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:47 smithi093 ceph-mon[111335]: pgmap v9216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/159900615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:47 smithi093 ceph-mon[118902]: pgmap v9216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/159900615' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:48.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:48.174+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:48.175812+0000) 2023-08-28T02:00:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:49.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:49.176084+0000) 2023-08-28T02:00:50.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:49 smithi093 ceph-mon[111335]: pgmap v9217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/241588922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:49 smithi093 ceph-mon[118902]: pgmap v9217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/241588922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:49 smithi136 ceph-mon[113531]: pgmap v9217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/241588922' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:50.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:50.176317+0000) 2023-08-28T02:00:51.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:51.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:51.176573+0000) 2023-08-28T02:00:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:51.449Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:51.450Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:51 smithi093 ceph-mon[111335]: pgmap v9218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:51 smithi093 ceph-mon[118902]: pgmap v9218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:51 smithi136 ceph-mon[113531]: pgmap v9218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:52.175+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:52.176736+0000) 2023-08-28T02:00:53.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1389795288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1389795288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:52.883Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:00:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:00:52.884Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:00:53.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1389795288' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:53.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:53.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:53.177001+0000) 2023-08-28T02:00:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:53 smithi136 ceph-mon[113531]: pgmap v9219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:54.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:54.176+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:54.177264+0000) 2023-08-28T02:00:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:53 smithi093 ceph-mon[111335]: pgmap v9219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:53 smithi093 ceph-mon[118902]: pgmap v9219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:00:55.356 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1641063989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:55.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1641063989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:55.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1641063989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:55.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:55.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:55.177502+0000) 2023-08-28T02:00:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:00:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:00:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:00:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:55 smithi136 ceph-mon[113531]: pgmap v9220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:56.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:56.177753+0000) 2023-08-28T02:00:56.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:55 smithi093 ceph-mon[118902]: pgmap v9220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:55 smithi093 ceph-mon[111335]: pgmap v9220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:57.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:57.178016+0000) 2023-08-28T02:00:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:57 smithi136 ceph-mon[113531]: pgmap v9221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/224602942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:57 smithi093 ceph-mon[111335]: pgmap v9221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/224602942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:57 smithi093 ceph-mon[118902]: pgmap v9221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:00:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/224602942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:00:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:58.177+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:58.178321+0000) 2023-08-28T02:00:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:00:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:00:59.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:00:59.178552+0000) 2023-08-28T02:01:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:59 smithi093 ceph-mon[111335]: pgmap v9222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:00:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1057279143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:59 smithi093 ceph-mon[118902]: pgmap v9222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:00:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1057279143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:59 smithi136 ceph-mon[113531]: pgmap v9222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:00:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1057279143' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:00.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:00.178801+0000) 2023-08-28T02:01:01.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:01.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:01.179102+0000) 2023-08-28T02:01:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:01.449Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:01.451Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:02.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:01 smithi093 ceph-mon[111335]: pgmap v9223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2493999224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:01 smithi093 ceph-mon[118902]: pgmap v9223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2493999224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:01 smithi136 ceph-mon[113531]: pgmap v9223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2493999224' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:02.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:02.179356+0000) 2023-08-28T02:01:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:02.884Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:02.885Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:03.178+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:03.179554+0000) 2023-08-28T02:01:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:03 smithi136 ceph-mon[113531]: pgmap v9224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:04.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:04.179814+0000) 2023-08-28T02:01:04.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:03 smithi093 ceph-mon[111335]: pgmap v9224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:04.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:03 smithi093 ceph-mon[118902]: pgmap v9224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:05.316 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3184627226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:05.316 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:05.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:05.180085+0000) 2023-08-28T02:01:05.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3184627226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3184627226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:05.687 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 02:01:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T02:01:05.318168538Z level=info msg="Completed cleanup jobs" duration=12.952283ms 2023-08-28T02:01:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:01:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:01:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:01:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:05 smithi136 ceph-mon[113531]: pgmap v9225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:06.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:06.179+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:06.180294+0000) 2023-08-28T02:01:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:05 smithi093 ceph-mon[111335]: pgmap v9225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:05 smithi093 ceph-mon[118902]: pgmap v9225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:07.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:07.180561+0000) 2023-08-28T02:01:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1605523691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:07 smithi136 ceph-mon[113531]: pgmap v9226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1605523691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:07 smithi093 ceph-mon[111335]: pgmap v9226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1605523691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:07 smithi093 ceph-mon[118902]: pgmap v9226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:08.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:08.180780+0000) 2023-08-28T02:01:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:09.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:09.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:09.181011+0000) 2023-08-28T02:01:10.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:09 smithi093 ceph-mon[111335]: pgmap v9227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/823814774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:09 smithi093 ceph-mon[118902]: pgmap v9227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/823814774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:09 smithi136 ceph-mon[113531]: pgmap v9227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/823814774' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:10.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:10.181273+0000) 2023-08-28T02:01:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:11.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:11.181513+0000) 2023-08-28T02:01:11.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:11.451Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:11.452Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:11 smithi136 ceph-mon[113531]: pgmap v9228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2405668900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:12.180+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:12.181791+0000) 2023-08-28T02:01:12.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:11 smithi093 ceph-mon[111335]: pgmap v9228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:12.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2405668900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:11 smithi093 ceph-mon[118902]: pgmap v9228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2405668900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:12.884Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:12.885Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:13.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:13.182024+0000) 2023-08-28T02:01:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:13 smithi136 ceph-mon[113531]: pgmap v9229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3225379110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:14.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:14.182309+0000) 2023-08-28T02:01:14.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:13 smithi093 ceph-mon[111335]: pgmap v9229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:14.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3225379110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:13 smithi093 ceph-mon[118902]: pgmap v9229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3225379110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:15.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:15.182509+0000) 2023-08-28T02:01:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:01:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:01:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:01:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:15 smithi136 ceph-mon[113531]: pgmap v9230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:16.181+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:16.182751+0000) 2023-08-28T02:01:16.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:15 smithi093 ceph-mon[111335]: pgmap v9230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:15 smithi093 ceph-mon[118902]: pgmap v9230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:17.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2480776736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:17.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:17.182969+0000) 2023-08-28T02:01:17.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2480776736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:17.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2480776736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:18.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:18 smithi136 ceph-mon[113531]: pgmap v9231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:18.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:18.183196+0000) 2023-08-28T02:01:18.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:17 smithi093 ceph-mon[111335]: pgmap v9231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:18.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:18 smithi093 ceph-mon[118902]: pgmap v9231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:19.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/441750563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:19.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:19.183411+0000) 2023-08-28T02:01:19.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/441750563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:19.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/441750563' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:20 smithi136 ceph-mon[113531]: pgmap v9232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:20.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:20.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:20.183662+0000) 2023-08-28T02:01:20.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:20 smithi093 ceph-mon[111335]: pgmap v9232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:20 smithi093 ceph-mon[118902]: pgmap v9232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:21.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1550074695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:21.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:21.182+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:21.183931+0000) 2023-08-28T02:01:21.450 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1550074695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:21.450 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1550074695' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:21.452Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:21.453Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:22 smithi136 ceph-mon[113531]: pgmap v9233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:22.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:22.184189+0000) 2023-08-28T02:01:22.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:22 smithi093 ceph-mon[111335]: pgmap v9233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:22.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:22 smithi093 ceph-mon[118902]: pgmap v9233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:22.885Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:23.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:22.885Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:23.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:23.184442+0000) 2023-08-28T02:01:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:24.183+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:24.184707+0000) 2023-08-28T02:01:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:24 smithi136 ceph-mon[113531]: pgmap v9234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4291323632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:24.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:24 smithi093 ceph-mon[111335]: pgmap v9234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:24.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4291323632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:24.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:24 smithi093 ceph-mon[118902]: pgmap v9234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4291323632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:25.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:25.184934+0000) 2023-08-28T02:01:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:01:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:01:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:01:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:26 smithi136 ceph-mon[113531]: pgmap v9235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1105998266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:26.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:26.185219+0000) 2023-08-28T02:01:26.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:26 smithi093 ceph-mon[111335]: pgmap v9235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:26.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1105998266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:26 smithi093 ceph-mon[118902]: pgmap v9235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1105998266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:27.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:27.185420+0000) 2023-08-28T02:01:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:27 smithi136 ceph-mon[113531]: pgmap v9236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:27 smithi093 ceph-mon[111335]: pgmap v9236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:27.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:27 smithi093 ceph-mon[118902]: pgmap v9236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:28.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:28.184+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:28.185687+0000) 2023-08-28T02:01:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/928599406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:28.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/928599406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/928599406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:29.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:29.185962+0000) 2023-08-28T02:01:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:29 smithi136 ceph-mon[113531]: pgmap v9237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:29.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:29 smithi093 ceph-mon[111335]: pgmap v9237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:29 smithi093 ceph-mon[118902]: pgmap v9237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:30.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:30.186239+0000) 2023-08-28T02:01:30.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2763007428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:30.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2763007428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2763007428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:31.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:31.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:31.186541+0000) 2023-08-28T02:01:31.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:31 smithi093 ceph-mon[111335]: pgmap v9238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:31 smithi093 ceph-mon[118902]: pgmap v9238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:31.452Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:31.453Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:31 smithi136 ceph-mon[113531]: pgmap v9238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:32.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:32.186820+0000) 2023-08-28T02:01:32.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:01:32.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:01:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:01:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:32.885Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:32.886Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:33.185+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:33.186973+0000) 2023-08-28T02:01:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/897432016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:33 smithi136 ceph-mon[113531]: pgmap v9239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/897432016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:33.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:33 smithi093 ceph-mon[111335]: pgmap v9239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/897432016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:33.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:33 smithi093 ceph-mon[118902]: pgmap v9239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:34.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:34.187134+0000) 2023-08-28T02:01:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:35.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:35.187284+0000) 2023-08-28T02:01:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:35 smithi136 ceph-mon[113531]: pgmap v9240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3550836955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:01:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:01:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[111335]: pgmap v9240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3550836955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[118902]: pgmap v9240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3550836955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:01:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:01:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:36.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:36.187541+0000) 2023-08-28T02:01:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:01:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:01:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:01:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:01:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:01:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:37.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:37.187694+0000) 2023-08-28T02:01:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:37 smithi136 ceph-mon[113531]: pgmap v9241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:37 smithi093 ceph-mon[111335]: pgmap v9241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:37 smithi093 ceph-mon[118902]: pgmap v9241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:38.186+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:38.188004+0000) 2023-08-28T02:01:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/337283773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:38.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/337283773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/337283773' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:39.187+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:39.188256+0000) 2023-08-28T02:01:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:39 smithi136 ceph-mon[113531]: pgmap v9242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:39 smithi093 ceph-mon[111335]: pgmap v9242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:39.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:39 smithi093 ceph-mon[118902]: pgmap v9242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:40.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:40.188514+0000) 2023-08-28T02:01:40.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3075675364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:40.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3075675364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:40.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3075675364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:41.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:41.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:41.188782+0000) 2023-08-28T02:01:41.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:41 smithi093 ceph-mon[111335]: pgmap v9243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:41 smithi093 ceph-mon[118902]: pgmap v9243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:41.453Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:41.454Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:41.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:41 smithi136 ceph-mon[113531]: pgmap v9243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:42.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:42.189061+0000) 2023-08-28T02:01:42.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3908319956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:42.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3908319956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:42.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3908319956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:42.885Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:42.886Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:43.649 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:43.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:43.189247+0000) 2023-08-28T02:01:43.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:43 smithi136 ceph-mon[113531]: pgmap v9244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:43.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:43 smithi093 ceph-mon[111335]: pgmap v9244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:43 smithi093 ceph-mon[118902]: pgmap v9244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:44.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:44.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:44.189460+0000) 2023-08-28T02:01:45.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1923041442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:45.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1923041442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1923041442' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:45.188+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:45.189669+0000) 2023-08-28T02:01:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:01:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:01:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:01:46.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:45 smithi136 ceph-mon[113531]: pgmap v9245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:46.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:45 smithi093 ceph-mon[111335]: pgmap v9245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:45 smithi093 ceph-mon[118902]: pgmap v9245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:46.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:46.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:46.189844+0000) 2023-08-28T02:01:47.629 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:47.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:47.190093+0000) 2023-08-28T02:01:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:47 smithi136 ceph-mon[113531]: pgmap v9246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4199472323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:47 smithi093 ceph-mon[111335]: pgmap v9246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4199472323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:47 smithi093 ceph-mon[118902]: pgmap v9246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4199472323' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:48.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:48.190348+0000) 2023-08-28T02:01:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:49.189+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:49.190592+0000) 2023-08-28T02:01:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:49 smithi136 ceph-mon[113531]: pgmap v9247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4032155372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:50.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:50.190839+0000) 2023-08-28T02:01:50.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:49 smithi093 ceph-mon[111335]: pgmap v9247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:50.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4032155372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:50.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:49 smithi093 ceph-mon[118902]: pgmap v9247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4032155372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:51.190+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:51.191122+0000) 2023-08-28T02:01:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:51.454Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:51.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:51 smithi136 ceph-mon[113531]: pgmap v9248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2846263045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:52.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:52.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:52.191425+0000) 2023-08-28T02:01:52.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:51 smithi093 ceph-mon[111335]: pgmap v9248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:52.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2846263045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:52.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:52 smithi093 ceph-mon[118902]: pgmap v9248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2846263045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:52.886Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:01:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:01:52.887Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:01:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:53.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:53.191716+0000) 2023-08-28T02:01:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:54 smithi136 ceph-mon[113531]: pgmap v9249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:54.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:54.191954+0000) 2023-08-28T02:01:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:54 smithi093 ceph-mon[111335]: pgmap v9249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:54 smithi093 ceph-mon[118902]: pgmap v9249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:01:55.356 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4118502214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:55.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4118502214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:55.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4118502214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:55.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:55.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:55.192170+0000) 2023-08-28T02:01:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:01:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:01:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:01:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:56 smithi136 ceph-mon[113531]: pgmap v9250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:56.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:56.192420+0000) 2023-08-28T02:01:56.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:56 smithi093 ceph-mon[111335]: pgmap v9250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:56.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:56 smithi093 ceph-mon[118902]: pgmap v9250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:57.633 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:57.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:57.192693+0000) 2023-08-28T02:01:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1781007798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:57 smithi136 ceph-mon[113531]: pgmap v9251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1781007798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:57 smithi093 ceph-mon[111335]: pgmap v9251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1781007798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:01:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:57 smithi093 ceph-mon[118902]: pgmap v9251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:01:58.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:58.191+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:58.192986+0000) 2023-08-28T02:01:59.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:01:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:01:59.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:01:59.193281+0000) 2023-08-28T02:02:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:59 smithi136 ceph-mon[113531]: pgmap v9252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:01:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2092886620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:00.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:00.193545+0000) 2023-08-28T02:02:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:59 smithi093 ceph-mon[111335]: pgmap v9252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:01:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2092886620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:59 smithi093 ceph-mon[118902]: pgmap v9252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:01:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2092886620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:01.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:01.192+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:01.193821+0000) 2023-08-28T02:02:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:01.454Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:01.455Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:02 smithi136 ceph-mon[113531]: pgmap v9253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1715703479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:02.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:02.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:02.194106+0000) 2023-08-28T02:02:02.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:02 smithi093 ceph-mon[111335]: pgmap v9253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:02.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1715703479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:02 smithi093 ceph-mon[118902]: pgmap v9253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1715703479' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:02.887 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:02.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:02.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:03.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:03.194413+0000) 2023-08-28T02:02:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:04 smithi136 ceph-mon[113531]: pgmap v9254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1868118297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:04.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:04.194678+0000) 2023-08-28T02:02:04.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:04 smithi093 ceph-mon[111335]: pgmap v9254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1868118297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:04 smithi093 ceph-mon[118902]: pgmap v9254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1868118297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:05.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:05.193+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:05.194930+0000) 2023-08-28T02:02:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:02:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:02:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:02:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:06 smithi136 ceph-mon[113531]: pgmap v9255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:06.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:06.195226+0000) 2023-08-28T02:02:06.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:06 smithi093 ceph-mon[111335]: pgmap v9255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:06 smithi093 ceph-mon[118902]: pgmap v9255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:07.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1913530575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:07.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:07.195492+0000) 2023-08-28T02:02:07.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1913530575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1913530575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:08.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:08 smithi136 ceph-mon[113531]: pgmap v9256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:08.194+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:08.195770+0000) 2023-08-28T02:02:08.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:08 smithi093 ceph-mon[111335]: pgmap v9256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:08.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:08 smithi093 ceph-mon[118902]: pgmap v9256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3989001119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:09.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:09.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:09.196063+0000) 2023-08-28T02:02:09.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3989001119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:09.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3989001119' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:10 smithi136 ceph-mon[113531]: pgmap v9257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:10.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:10.196328+0000) 2023-08-28T02:02:10.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:10 smithi093 ceph-mon[111335]: pgmap v9257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:10.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:10 smithi093 ceph-mon[118902]: pgmap v9257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:11.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:11.195+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:11.196600+0000) 2023-08-28T02:02:11.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:11.455Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:11.456Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:12 smithi136 ceph-mon[113531]: pgmap v9258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1490000769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:12.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:12.196840+0000) 2023-08-28T02:02:12.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:12 smithi093 ceph-mon[111335]: pgmap v9258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1490000769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:12 smithi093 ceph-mon[118902]: pgmap v9258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1490000769' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:12.887Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:12.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:13.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:13.197072+0000) 2023-08-28T02:02:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:14 smithi136 ceph-mon[113531]: pgmap v9259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/254115171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:14.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:14.197308+0000) 2023-08-28T02:02:14.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:14 smithi093 ceph-mon[111335]: pgmap v9259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/254115171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:14 smithi093 ceph-mon[118902]: pgmap v9259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/254115171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:15.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:15.196+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:15.197574+0000) 2023-08-28T02:02:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:02:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:02:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:02:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:16 smithi136 ceph-mon[113531]: pgmap v9260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2293063599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:16.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:16.197812+0000) 2023-08-28T02:02:16.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:16 smithi093 ceph-mon[111335]: pgmap v9260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:16.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2293063599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:16 smithi093 ceph-mon[118902]: pgmap v9260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2293063599' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:17.631 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:17.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:17.198085+0000) 2023-08-28T02:02:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:17 smithi136 ceph-mon[113531]: pgmap v9261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:17 smithi093 ceph-mon[111335]: pgmap v9261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:17 smithi093 ceph-mon[118902]: pgmap v9261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:18.633 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:18.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:18.198341+0000) 2023-08-28T02:02:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/55902088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:18.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/55902088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:18.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/55902088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:19.637 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:19.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:19.198642+0000) 2023-08-28T02:02:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:19 smithi136 ceph-mon[113531]: pgmap v9262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:19 smithi093 ceph-mon[118902]: pgmap v9262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:19 smithi093 ceph-mon[111335]: pgmap v9262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:20.639 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:20.197+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:20.198892+0000) 2023-08-28T02:02:20.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1984577003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:20.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1984577003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:20.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1984577003' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:21.646 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:21.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:21.199169+0000) 2023-08-28T02:02:21.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:21 smithi093 ceph-mon[111335]: pgmap v9263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:21 smithi093 ceph-mon[118902]: pgmap v9263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:21.455Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:21.456Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:21 smithi136 ceph-mon[113531]: pgmap v9263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:22.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:22.198+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:22.199396+0000) 2023-08-28T02:02:23.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3875121434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3875121434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:22.888Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:22.888Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:23.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3875121434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:23.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:23.199668+0000) 2023-08-28T02:02:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:24 smithi136 ceph-mon[113531]: pgmap v9264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:24.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:24.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:24.199974+0000) 2023-08-28T02:02:24.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:24 smithi093 ceph-mon[111335]: pgmap v9264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:24 smithi093 ceph-mon[118902]: pgmap v9264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:25.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:25.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:25.200254+0000) 2023-08-28T02:02:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:02:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:02:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:02:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:26 smithi136 ceph-mon[113531]: pgmap v9265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3371439738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:26.199+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:26.200440+0000) 2023-08-28T02:02:26.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:26 smithi093 ceph-mon[111335]: pgmap v9265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:26.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3371439738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:26 smithi093 ceph-mon[118902]: pgmap v9265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3371439738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:27.633 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:27.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:27.200691+0000) 2023-08-28T02:02:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:27 smithi136 ceph-mon[113531]: pgmap v9266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:27 smithi093 ceph-mon[111335]: pgmap v9266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:27 smithi093 ceph-mon[118902]: pgmap v9266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:28.634 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:28.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:28.200944+0000) 2023-08-28T02:02:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4253396679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:28.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4253396679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4253396679' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:29.641 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:29.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:29.201205+0000) 2023-08-28T02:02:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:29 smithi136 ceph-mon[113531]: pgmap v9267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:29.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:29 smithi093 ceph-mon[111335]: pgmap v9267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:29.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:29 smithi093 ceph-mon[118902]: pgmap v9267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:30.642 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:30.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:30.201460+0000) 2023-08-28T02:02:30.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/241332282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:30.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/241332282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/241332282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:31.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:31.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:31.201702+0000) 2023-08-28T02:02:31.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:31 smithi093 ceph-mon[111335]: pgmap v9268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:31 smithi093 ceph-mon[118902]: pgmap v9268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:31.456Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:31.457Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:31.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:31 smithi136 ceph-mon[113531]: pgmap v9268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:32.655 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:32.200+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:32.201965+0000) 2023-08-28T02:02:32.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2669570324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:32.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2669570324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:32.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2669570324' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:32.889Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:32.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:32.889Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:33.660 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:33.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:33.202233+0000) 2023-08-28T02:02:33.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:33 smithi136 ceph-mon[113531]: pgmap v9269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:33.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:33 smithi093 ceph-mon[111335]: pgmap v9269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:33.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:33 smithi093 ceph-mon[118902]: pgmap v9269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:34.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:34.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:34.202451+0000) 2023-08-28T02:02:35.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2912239761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:35.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2912239761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2912239761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:35.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:35.202710+0000) 2023-08-28T02:02:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:02:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:02:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:35 smithi136 ceph-mon[113531]: pgmap v9270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:02:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:36.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[111335]: pgmap v9270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:02:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:36.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[118902]: pgmap v9270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:02:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:36.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:36.201+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:36.202871+0000) 2023-08-28T02:02:37.635 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:37.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:37.203027+0000) 2023-08-28T02:02:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:37 smithi136 ceph-mon[113531]: pgmap v9271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3604457684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[111335]: pgmap v9271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3604457684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[118902]: pgmap v9271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3604457684' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:38.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:38.203210+0000) 2023-08-28T02:02:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:02:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:02:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:02:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:02:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:02:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:39.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:39.203395+0000) 2023-08-28T02:02:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:39 smithi136 ceph-mon[113531]: pgmap v9272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1846853345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:39 smithi093 ceph-mon[111335]: pgmap v9272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1846853345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:39 smithi093 ceph-mon[118902]: pgmap v9272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1846853345' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:40.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:40.203652+0000) 2023-08-28T02:02:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:41.202+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:41.203887+0000) 2023-08-28T02:02:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:41.457Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:41.459Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:42 smithi136 ceph-mon[113531]: pgmap v9273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2766413197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:42.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:42.204160+0000) 2023-08-28T02:02:42.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:42 smithi093 ceph-mon[111335]: pgmap v9273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2766413197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:42 smithi093 ceph-mon[118902]: pgmap v9273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2766413197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:42.889Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:42.890Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:43.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:43.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:43.204439+0000) 2023-08-28T02:02:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:44 smithi136 ceph-mon[113531]: pgmap v9274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:44.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:44.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:44.204695+0000) 2023-08-28T02:02:44.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:44 smithi093 ceph-mon[111335]: pgmap v9274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:44 smithi093 ceph-mon[118902]: pgmap v9274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:45.356 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2378192898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2378192898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2378192898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:45.203+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:45.204950+0000) 2023-08-28T02:02:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:02:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:02:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:02:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:46 smithi136 ceph-mon[113531]: pgmap v9275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:46.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:46.205182+0000) 2023-08-28T02:02:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:46 smithi093 ceph-mon[111335]: pgmap v9275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:46.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:46 smithi093 ceph-mon[118902]: pgmap v9275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:47.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:47.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:47.205397+0000) 2023-08-28T02:02:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1361546085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:47 smithi136 ceph-mon[113531]: pgmap v9276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1361546085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:47 smithi093 ceph-mon[111335]: pgmap v9276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1361546085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:47 smithi093 ceph-mon[118902]: pgmap v9276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:48.204+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:48.205658+0000) 2023-08-28T02:02:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:49.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:49.205855+0000) 2023-08-28T02:02:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:50 smithi136 ceph-mon[113531]: pgmap v9277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1028075839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:50.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:50.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:50.206152+0000) 2023-08-28T02:02:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:50 smithi093 ceph-mon[111335]: pgmap v9277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1028075839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:50 smithi093 ceph-mon[118902]: pgmap v9277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1028075839' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:51.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:51.206369+0000) 2023-08-28T02:02:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:51.458Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:51.459Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:52 smithi136 ceph-mon[113531]: pgmap v9278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3040292610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:52.205+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:52.206625+0000) 2023-08-28T02:02:52.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:52 smithi093 ceph-mon[111335]: pgmap v9278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:52.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3040292610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:52 smithi093 ceph-mon[118902]: pgmap v9278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3040292610' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:52.890Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:02:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:02:52.891Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:02:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:53.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:53.206877+0000) 2023-08-28T02:02:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:54 smithi136 ceph-mon[113531]: pgmap v9279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/218763668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:54.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:54.207125+0000) 2023-08-28T02:02:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:54 smithi093 ceph-mon[111335]: pgmap v9279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/218763668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:54 smithi093 ceph-mon[118902]: pgmap v9279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:02:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/218763668' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:55.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:55.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:55.207352+0000) 2023-08-28T02:02:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:02:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:02:55] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T02:02:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:56 smithi136 ceph-mon[113531]: pgmap v9280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:56.206+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:56.207609+0000) 2023-08-28T02:02:56.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:56 smithi093 ceph-mon[111335]: pgmap v9280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:56 smithi093 ceph-mon[118902]: pgmap v9280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:57.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/761782473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:57.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:57.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:57.207888+0000) 2023-08-28T02:02:57.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/761782473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:57.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/761782473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:58.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:58 smithi136 ceph-mon[113531]: pgmap v9281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:58.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:58.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:58.208107+0000) 2023-08-28T02:02:58.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:58 smithi093 ceph-mon[111335]: pgmap v9281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:58 smithi093 ceph-mon[118902]: pgmap v9281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:02:59.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:02:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/45105748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:59.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:02:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:02:59.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:02:59.208333+0000) 2023-08-28T02:02:59.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:02:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/45105748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:02:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:02:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/45105748' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:00 smithi136 ceph-mon[113531]: pgmap v9282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:00.207+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:00.208582+0000) 2023-08-28T02:03:00.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:00 smithi093 ceph-mon[111335]: pgmap v9282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:00.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:00 smithi093 ceph-mon[118902]: pgmap v9282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:01.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:01.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:01.208881+0000) 2023-08-28T02:03:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:01.459Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:01.460Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:02 smithi136 ceph-mon[113531]: pgmap v9283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4154454540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:02.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:02.209160+0000) 2023-08-28T02:03:02.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:02 smithi093 ceph-mon[111335]: pgmap v9283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:02.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4154454540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:02 smithi093 ceph-mon[118902]: pgmap v9283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4154454540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:02.891Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:03.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:02.892Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:03.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:03.209406+0000) 2023-08-28T02:03:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:04 smithi136 ceph-mon[113531]: pgmap v9284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2393849721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:04.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:04.209665+0000) 2023-08-28T02:03:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:04 smithi093 ceph-mon[111335]: pgmap v9284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2393849721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:04 smithi093 ceph-mon[118902]: pgmap v9284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2393849721' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:05.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:05.208+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:05.209938+0000) 2023-08-28T02:03:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:03:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:03:05] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T02:03:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:06 smithi136 ceph-mon[113531]: pgmap v9285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3532151425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:06.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:06.210179+0000) 2023-08-28T02:03:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:06 smithi093 ceph-mon[111335]: pgmap v9285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3532151425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:06 smithi093 ceph-mon[118902]: pgmap v9285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3532151425' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:07.640 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:07.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:07.210434+0000) 2023-08-28T02:03:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:07 smithi136 ceph-mon[113531]: pgmap v9286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:07 smithi093 ceph-mon[111335]: pgmap v9286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:07 smithi093 ceph-mon[118902]: pgmap v9286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:08.642 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:08.209+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:08.210708+0000) 2023-08-28T02:03:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1956153247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:08.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1956153247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1956153247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:09.646 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:09.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:09.210932+0000) 2023-08-28T02:03:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:09 smithi136 ceph-mon[113531]: pgmap v9287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:09 smithi093 ceph-mon[111335]: pgmap v9287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:09 smithi093 ceph-mon[118902]: pgmap v9287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:10.648 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:10.210+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:10.211201+0000) 2023-08-28T02:03:10.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/966060987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:10.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/966060987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/966060987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:11.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:11.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:11.211465+0000) 2023-08-28T02:03:11.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:11 smithi093 ceph-mon[111335]: pgmap v9288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:11 smithi093 ceph-mon[118902]: pgmap v9288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:11.460Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:11.461Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:11 smithi136 ceph-mon[113531]: pgmap v9288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:12.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:12.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:12.211698+0000) 2023-08-28T02:03:13.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/234906266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/234906266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:12.892Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:12.892Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:13.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/234906266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:13.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:13.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:13.211952+0000) 2023-08-28T02:03:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:14 smithi136 ceph-mon[113531]: pgmap v9289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:14.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:14.212198+0000) 2023-08-28T02:03:14.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:14 smithi093 ceph-mon[111335]: pgmap v9289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:14.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:14 smithi093 ceph-mon[118902]: pgmap v9289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:15.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:15.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:15.212463+0000) 2023-08-28T02:03:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:03:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:03:15] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T02:03:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:16 smithi136 ceph-mon[113531]: pgmap v9290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3734389890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:16.211+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:16.212730+0000) 2023-08-28T02:03:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:16 smithi093 ceph-mon[111335]: pgmap v9290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3734389890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:16 smithi093 ceph-mon[118902]: pgmap v9290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3734389890' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:17.643 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:17.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:17.212952+0000) 2023-08-28T02:03:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:17 smithi136 ceph-mon[113531]: pgmap v9291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:17 smithi093 ceph-mon[111335]: pgmap v9291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:17 smithi093 ceph-mon[118902]: pgmap v9291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:18.644 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:18.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:18.213189+0000) 2023-08-28T02:03:18.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1084621314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:18.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1084621314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1084621314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:19.651 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:19.212+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:19.213387+0000) 2023-08-28T02:03:19.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:19 smithi136 ceph-mon[113531]: pgmap v9292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:19.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:19 smithi093 ceph-mon[111335]: pgmap v9292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:19.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:19 smithi093 ceph-mon[118902]: pgmap v9292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:20.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:20.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:20.213639+0000) 2023-08-28T02:03:20.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1416624705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:20.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1416624705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:20.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1416624705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:21.661 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:21.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:21.213889+0000) 2023-08-28T02:03:21.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:21 smithi093 ceph-mon[111335]: pgmap v9293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:21 smithi093 ceph-mon[118902]: pgmap v9293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:21.461Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:21.462Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:21.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:21 smithi136 ceph-mon[113531]: pgmap v9293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:22.663 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:22.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:22.214159+0000) 2023-08-28T02:03:22.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3006156803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:22.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3006156803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:22.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3006156803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:22.893Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:22.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:22.893Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:23.667 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:23.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:23.214420+0000) 2023-08-28T02:03:23.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:23 smithi136 ceph-mon[113531]: pgmap v9294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:23.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:23 smithi093 ceph-mon[111335]: pgmap v9294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:23.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:23 smithi093 ceph-mon[118902]: pgmap v9294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:24.668 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:24.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:24.214670+0000) 2023-08-28T02:03:24.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:24.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:24.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:25.672 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:03:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:03:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:03:25.673 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:25.213+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:25.214831+0000) 2023-08-28T02:03:25.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3376039944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:25.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:25 smithi136 ceph-mon[113531]: pgmap v9295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:25.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3376039944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:25.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:25 smithi093 ceph-mon[111335]: pgmap v9295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:25.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3376039944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:25.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:25 smithi093 ceph-mon[118902]: pgmap v9295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:26.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:26.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:26.215107+0000) 2023-08-28T02:03:27.643 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:27.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:27.215313+0000) 2023-08-28T02:03:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:27 smithi136 ceph-mon[113531]: pgmap v9296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3139612719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:27.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:27 smithi093 ceph-mon[118902]: pgmap v9296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3139612719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:27 smithi093 ceph-mon[111335]: pgmap v9296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3139612719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:28.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:28.215575+0000) 2023-08-28T02:03:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:29.214+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:29.215840+0000) 2023-08-28T02:03:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:30 smithi136 ceph-mon[113531]: pgmap v9297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3100146282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:30.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:30.216133+0000) 2023-08-28T02:03:30.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:30 smithi093 ceph-mon[111335]: pgmap v9297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:30.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3100146282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:30 smithi093 ceph-mon[118902]: pgmap v9297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3100146282' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:31.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:31.216399+0000) 2023-08-28T02:03:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:31.461Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:32.115 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:31.462Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:32 smithi136 ceph-mon[113531]: pgmap v9298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:32.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:32.215+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:32.216673+0000) 2023-08-28T02:03:32.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:32 smithi093 ceph-mon[111335]: pgmap v9298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:32 smithi093 ceph-mon[118902]: pgmap v9298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:33.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3090905828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3090905828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:32.893Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:32.895Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:33.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3090905828' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:33.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:33.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:33.216894+0000) 2023-08-28T02:03:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:34 smithi136 ceph-mon[113531]: pgmap v9299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:34.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:34.217071+0000) 2023-08-28T02:03:34.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:34 smithi093 ceph-mon[111335]: pgmap v9299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:34 smithi093 ceph-mon[118902]: pgmap v9299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:35.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3980922099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:35.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3980922099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:35.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3980922099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:35.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:35.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:35.217330+0000) 2023-08-28T02:03:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:03:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:03:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:03:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:36 smithi136 ceph-mon[113531]: pgmap v9300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:36.216+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:36.217589+0000) 2023-08-28T02:03:36.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:36 smithi093 ceph-mon[111335]: pgmap v9300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:36 smithi093 ceph-mon[118902]: pgmap v9300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:37.644 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:37.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:37.217840+0000) 2023-08-28T02:03:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2925015647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:37 smithi136 ceph-mon[113531]: pgmap v9301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2925015647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:37 smithi093 ceph-mon[111335]: pgmap v9301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2925015647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:37 smithi093 ceph-mon[118902]: pgmap v9301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:38.645 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:38.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:38.218130+0000) 2023-08-28T02:03:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:03:38.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:03:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:03:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:39.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:39.218367+0000) 2023-08-28T02:03:39.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:39.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:39.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:39.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[111335]: pgmap v9302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:39.983 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2958428984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:39.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:39.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:39.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:39.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[118902]: pgmap v9302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:39.984 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2958428984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:39 smithi136 ceph-mon[113531]: pgmap v9302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2958428984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:40.621 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:40.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:40.218579+0000) 2023-08-28T02:03:41.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:41 smithi136 ceph-mon[113531]: pgmap v9303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:41.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:41.217+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:41.218779+0000) 2023-08-28T02:03:41.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[111335]: pgmap v9303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:41.462Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:41.463Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[118902]: pgmap v9303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:42.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:42.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:42.218967+0000) 2023-08-28T02:03:42.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:03:42.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:03:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3128960514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:42.737 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:03:42.737 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:03:42.737 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:42.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3128960514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:03:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:03:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:03:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3128960514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:42.894Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:42.895Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:43.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:43 smithi136 ceph-mon[113531]: pgmap v9304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:43.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:43.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:43.219235+0000) 2023-08-28T02:03:43.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:43 smithi093 ceph-mon[111335]: pgmap v9304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:43.737 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:43 smithi093 ceph-mon[118902]: pgmap v9304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:44.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1624692226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:44.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:44.218+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:44.219416+0000) 2023-08-28T02:03:44.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1624692226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:44.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1624692226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:45.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:45 smithi136 ceph-mon[113531]: pgmap v9305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:45.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:45.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:45.219674+0000) 2023-08-28T02:03:45.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:45 smithi093 ceph-mon[111335]: pgmap v9305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:45 smithi093 ceph-mon[118902]: pgmap v9305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:03:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:03:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:03:46.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3970233866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:46.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:46.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:46.219920+0000) 2023-08-28T02:03:46.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3970233866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:46.737 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3970233866' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:47.647 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:47.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:47.220194+0000) 2023-08-28T02:03:47.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:47 smithi136 ceph-mon[113531]: pgmap v9306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:47 smithi093 ceph-mon[111335]: pgmap v9306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:47 smithi093 ceph-mon[118902]: pgmap v9306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:48.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:48.220426+0000) 2023-08-28T02:03:49.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3783338884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:49.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3783338884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3783338884' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:49.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:49.220618+0000) 2023-08-28T02:03:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:49 smithi136 ceph-mon[113531]: pgmap v9307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:50.237 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:49 smithi093 ceph-mon[118902]: pgmap v9307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:49 smithi093 ceph-mon[111335]: pgmap v9307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:50.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:50.219+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:50.220917+0000) 2023-08-28T02:03:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:51.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:51.221456+0000) 2023-08-28T02:03:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:51.463Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:51.464Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:52 smithi136 ceph-mon[113531]: pgmap v9308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2271655761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:52.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:52.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:52.221735+0000) 2023-08-28T02:03:52.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:52 smithi093 ceph-mon[111335]: pgmap v9308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:52.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2271655761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:52 smithi093 ceph-mon[118902]: pgmap v9308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2271655761' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:52.895Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:03:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:03:52.896Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:03:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:53.220+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:53.222001+0000) 2023-08-28T02:03:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:54 smithi136 ceph-mon[113531]: pgmap v9309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1521635873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:54.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:54.222277+0000) 2023-08-28T02:03:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:54 smithi093 ceph-mon[111335]: pgmap v9309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1521635873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:54 smithi093 ceph-mon[118902]: pgmap v9309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1521635873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:03:55.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:55.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:55.222520+0000) 2023-08-28T02:03:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:03:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:03:55] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T02:03:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:56 smithi136 ceph-mon[113531]: pgmap v9310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/310758263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:56.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:56.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:56.222759+0000) 2023-08-28T02:03:56.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:56 smithi093 ceph-mon[111335]: pgmap v9310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/310758263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:56 smithi093 ceph-mon[118902]: pgmap v9310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/310758263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:57.648 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:57.221+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:57.222986+0000) 2023-08-28T02:03:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:57 smithi136 ceph-mon[113531]: pgmap v9311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:57 smithi093 ceph-mon[111335]: pgmap v9311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:57 smithi093 ceph-mon[118902]: pgmap v9311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:58.649 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:58.222+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:58.223261+0000) 2023-08-28T02:03:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1451030015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:58.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1451030015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1451030015' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:03:59.656 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:03:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:03:59.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:03:59.223513+0000) 2023-08-28T02:03:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:03:59 smithi136 ceph-mon[113531]: pgmap v9312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:59.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:03:59 smithi093 ceph-mon[111335]: pgmap v9312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:03:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:03:59 smithi093 ceph-mon[118902]: pgmap v9312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:00.658 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:00.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:00.223793+0000) 2023-08-28T02:04:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/727146860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/727146860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/727146860' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:01.667 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:01.223+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:01.224065+0000) 2023-08-28T02:04:01.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:01 smithi093 ceph-mon[111335]: pgmap v9313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:01 smithi093 ceph-mon[118902]: pgmap v9313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:01.464Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:01.465Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:01 smithi136 ceph-mon[113531]: pgmap v9313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:02.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:02.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:02.224331+0000) 2023-08-28T02:04:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:02.896Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:02.897Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:03.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:03.224615+0000) 2023-08-28T02:04:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:04 smithi136 ceph-mon[113531]: pgmap v9314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/864337200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:04.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:04.224860+0000) 2023-08-28T02:04:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:04 smithi093 ceph-mon[111335]: pgmap v9314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/864337200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:04 smithi093 ceph-mon[118902]: pgmap v9314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/864337200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:05.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:05.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:05.225149+0000) 2023-08-28T02:04:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:04:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:04:05] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T02:04:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:06 smithi136 ceph-mon[113531]: pgmap v9315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3955139240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:06.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:06.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:06.225415+0000) 2023-08-28T02:04:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:06 smithi093 ceph-mon[111335]: pgmap v9315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3955139240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:06 smithi093 ceph-mon[118902]: pgmap v9315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3955139240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:07.663 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:07.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:07.225655+0000) 2023-08-28T02:04:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:07 smithi136 ceph-mon[113531]: pgmap v9316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:07 smithi093 ceph-mon[111335]: pgmap v9316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:07 smithi093 ceph-mon[118902]: pgmap v9316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:08.664 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:08.224+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:08.225921+0000) 2023-08-28T02:04:08.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3639191987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:08.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3639191987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3639191987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:09.670 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:09.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:09.226172+0000) 2023-08-28T02:04:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:09.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:09 smithi136 ceph-mon[113531]: pgmap v9317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:09.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:09 smithi093 ceph-mon[111335]: pgmap v9317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:09 smithi093 ceph-mon[118902]: pgmap v9317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:10.671 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:10.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:10.226431+0000) 2023-08-28T02:04:10.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778383539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:10.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778383539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:10.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778383539' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:11.677 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:11.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:11.226686+0000) 2023-08-28T02:04:11.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:11 smithi093 ceph-mon[111335]: pgmap v9318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:11 smithi093 ceph-mon[118902]: pgmap v9318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:11.464Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:11.465Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:11.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:11 smithi136 ceph-mon[113531]: pgmap v9318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:12.679 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:12.225+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:12.226949+0000) 2023-08-28T02:04:12.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1475194093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:12.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1475194093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:12.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1475194093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:12.897Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:12.897Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:13.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:13.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:13.227151+0000) 2023-08-28T02:04:13.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:13 smithi136 ceph-mon[113531]: pgmap v9319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:13.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:13 smithi093 ceph-mon[111335]: pgmap v9319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:13 smithi093 ceph-mon[118902]: pgmap v9319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:14.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:14.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:14.227406+0000) 2023-08-28T02:04:15.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2495608525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:15.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2495608525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:15.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2495608525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:15.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:15.227651+0000) 2023-08-28T02:04:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:04:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:04:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:04:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:16 smithi136 ceph-mon[113531]: pgmap v9320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:16.226+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:16.227912+0000) 2023-08-28T02:04:16.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:16 smithi093 ceph-mon[118902]: pgmap v9320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:16 smithi093 ceph-mon[111335]: pgmap v9320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:17.652 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:17.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:17.228174+0000) 2023-08-28T02:04:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:17 smithi136 ceph-mon[113531]: pgmap v9321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/293628932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:17 smithi093 ceph-mon[111335]: pgmap v9321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/293628932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:17 smithi093 ceph-mon[118902]: pgmap v9321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/293628932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:18.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:18.228405+0000) 2023-08-28T02:04:19.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:19.227+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:19.228652+0000) 2023-08-28T02:04:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:20 smithi136 ceph-mon[113531]: pgmap v9322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1360611867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:20.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:20.228902+0000) 2023-08-28T02:04:20.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:20 smithi093 ceph-mon[111335]: pgmap v9322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:20.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1360611867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:20 smithi093 ceph-mon[118902]: pgmap v9322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1360611867' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:21.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:21.229159+0000) 2023-08-28T02:04:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:21.465Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:21.466Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:22 smithi136 ceph-mon[113531]: pgmap v9323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:22.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:22.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:22.229346+0000) 2023-08-28T02:04:22.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:22 smithi093 ceph-mon[111335]: pgmap v9323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:22 smithi093 ceph-mon[118902]: pgmap v9323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:23.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4136933914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4136933914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:22.897Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:22.898Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:23.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4136933914' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:23.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:23.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:23.229566+0000) 2023-08-28T02:04:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:24 smithi136 ceph-mon[113531]: pgmap v9324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:24.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:24.228+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:24.229842+0000) 2023-08-28T02:04:24.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:24 smithi093 ceph-mon[111335]: pgmap v9324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:24 smithi093 ceph-mon[118902]: pgmap v9324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3987663861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:25.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3987663861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3987663861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:25.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:25.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:25.230069+0000) 2023-08-28T02:04:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:04:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:04:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:04:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:26 smithi136 ceph-mon[113531]: pgmap v9325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:26.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:26.230311+0000) 2023-08-28T02:04:26.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:26 smithi093 ceph-mon[111335]: pgmap v9325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:26 smithi093 ceph-mon[118902]: pgmap v9325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:27.654 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:27.229+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:27.230568+0000) 2023-08-28T02:04:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/826440016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:27.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:27 smithi136 ceph-mon[113531]: pgmap v9326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/826440016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:27 smithi093 ceph-mon[111335]: pgmap v9326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/826440016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:27 smithi093 ceph-mon[118902]: pgmap v9326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:28.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:28.230820+0000) 2023-08-28T02:04:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:29.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:29.231104+0000) 2023-08-28T02:04:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:30 smithi136 ceph-mon[113531]: pgmap v9327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1553382537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:30.230+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:30.231357+0000) 2023-08-28T02:04:30.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:30 smithi093 ceph-mon[111335]: pgmap v9327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:30.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1553382537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:30 smithi093 ceph-mon[118902]: pgmap v9327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1553382537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:31.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:31.231599+0000) 2023-08-28T02:04:31.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:31.466Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:31.467Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:32 smithi136 ceph-mon[113531]: pgmap v9328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2228452426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:32.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:32.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:32.231854+0000) 2023-08-28T02:04:32.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:32 smithi093 ceph-mon[111335]: pgmap v9328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:32.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2228452426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:32 smithi093 ceph-mon[118902]: pgmap v9328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2228452426' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:33.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:32.898Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:32.899Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:33.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:33.232125+0000) 2023-08-28T02:04:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:34 smithi136 ceph-mon[113531]: pgmap v9329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12467267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:34.231+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:34.232367+0000) 2023-08-28T02:04:34.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:34 smithi093 ceph-mon[111335]: pgmap v9329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12467267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:34 smithi093 ceph-mon[118902]: pgmap v9329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12467267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:35.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:35.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:35.232637+0000) 2023-08-28T02:04:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:04:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:04:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:04:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:36 smithi136 ceph-mon[113531]: pgmap v9330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:36.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:36.232890+0000) 2023-08-28T02:04:36.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:36 smithi093 ceph-mon[111335]: pgmap v9330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:36 smithi093 ceph-mon[118902]: pgmap v9330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:37.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/914204888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:37.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:37.232+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:37.233165+0000) 2023-08-28T02:04:37.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/914204888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:37.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/914204888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:38.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:38 smithi136 ceph-mon[113531]: pgmap v9331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:38.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:38.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:38.233397+0000) 2023-08-28T02:04:38.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:38 smithi093 ceph-mon[111335]: pgmap v9331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:38.487 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:38 smithi093 ceph-mon[118902]: pgmap v9331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/25741179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:39.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:39.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:39.233652+0000) 2023-08-28T02:04:39.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/25741179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:39.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/25741179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:39.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:40 smithi136 ceph-mon[113531]: pgmap v9332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:40.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:40.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:40.233910+0000) 2023-08-28T02:04:40.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:40 smithi093 ceph-mon[111335]: pgmap v9332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:40.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:40 smithi093 ceph-mon[118902]: pgmap v9332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:41.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:41.234147+0000) 2023-08-28T02:04:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:41.467Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:41.468Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:42.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:42.234325+0000) 2023-08-28T02:04:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:42 smithi136 ceph-mon[113531]: pgmap v9333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2868009882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:04:42.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-mon[111335]: pgmap v9333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:42.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2868009882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:42.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:04:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-mon[118902]: pgmap v9333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2868009882' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:04:43.157 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:42.898Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:43.157 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:42.899Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:43.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:43.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:43.234456+0000) 2023-08-28T02:04:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:44 smithi136 ceph-mon[113531]: pgmap v9334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3862928031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:44.233+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:44.234632+0000) 2023-08-28T02:04:44.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[111335]: pgmap v9334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3862928031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[118902]: pgmap v9334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3862928031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:04:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:04:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:04:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:04:45.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:04:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:04:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:04:45.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:45.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:45.234871+0000) 2023-08-28T02:04:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:04:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:04:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:04:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:46 smithi136 ceph-mon[113531]: pgmap v9335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3537489689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:46.234+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:46.235104+0000) 2023-08-28T02:04:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:46 smithi093 ceph-mon[111335]: pgmap v9335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3537489689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:46 smithi093 ceph-mon[118902]: pgmap v9335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3537489689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:47.661 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:47.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:47.235357+0000) 2023-08-28T02:04:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:47 smithi136 ceph-mon[113531]: pgmap v9336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:47.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:47 smithi093 ceph-mon[111335]: pgmap v9336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:47 smithi093 ceph-mon[118902]: pgmap v9336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:48.663 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:48.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:48.235599+0000) 2023-08-28T02:04:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2283599440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:48.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2283599440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2283599440' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:49.668 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:49.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:49.235823+0000) 2023-08-28T02:04:49.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:49 smithi136 ceph-mon[113531]: pgmap v9337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:49 smithi093 ceph-mon[111335]: pgmap v9337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:49 smithi093 ceph-mon[118902]: pgmap v9337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:50.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:50.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:50.236099+0000) 2023-08-28T02:04:51.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/495599455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:51.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/495599455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/495599455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:51.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:51.236356+0000) 2023-08-28T02:04:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:51.467Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:51.468Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:51 smithi136 ceph-mon[113531]: pgmap v9338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:52.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:51 smithi093 ceph-mon[111335]: pgmap v9338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:51 smithi093 ceph-mon[118902]: pgmap v9338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:52.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:52.236641+0000) 2023-08-28T02:04:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:52.899Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:53.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:04:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:04:52.900Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:04:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:53.235+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:53.236942+0000) 2023-08-28T02:04:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:54 smithi136 ceph-mon[113531]: pgmap v9339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/443170330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:54.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:54.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:54.237209+0000) 2023-08-28T02:04:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:54 smithi093 ceph-mon[111335]: pgmap v9339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:54.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/443170330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:54 smithi093 ceph-mon[118902]: pgmap v9339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/443170330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:04:55.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:55.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:55.237461+0000) 2023-08-28T02:04:55.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:04:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:04:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:04:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:56 smithi136 ceph-mon[113531]: pgmap v9340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2281624843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:56.236+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:56.237722+0000) 2023-08-28T02:04:56.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:56 smithi093 ceph-mon[111335]: pgmap v9340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:56.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2281624843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:56 smithi093 ceph-mon[118902]: pgmap v9340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2281624843' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:57.658 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:57.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:57.237972+0000) 2023-08-28T02:04:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:57 smithi136 ceph-mon[113531]: pgmap v9341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:57 smithi093 ceph-mon[111335]: pgmap v9341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:57 smithi093 ceph-mon[118902]: pgmap v9341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:58.659 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:58.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:58.238252+0000) 2023-08-28T02:04:58.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3531776841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:58.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3531776841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3531776841' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:04:59.664 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:04:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:04:59.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:04:59.238497+0000) 2023-08-28T02:04:59.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:04:59 smithi136 ceph-mon[113531]: pgmap v9342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:59.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:04:59 smithi093 ceph-mon[111335]: pgmap v9342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:04:59.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:04:59 smithi093 ceph-mon[118902]: pgmap v9342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:00.666 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:00.237+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:00.238696+0000) 2023-08-28T02:05:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2066126871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2066126871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:00.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2066126871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:01.675 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:01.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:01.238957+0000) 2023-08-28T02:05:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:01 smithi093 ceph-mon[111335]: pgmap v9343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:01 smithi093 ceph-mon[118902]: pgmap v9343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:01.468Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:01.469Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:01.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:01 smithi136 ceph-mon[113531]: pgmap v9343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:02.678 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:02.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:02.239232+0000) 2023-08-28T02:05:02.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2632900982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:02.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2632900982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:02.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2632900982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:02.899Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:02.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:02.900Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:03.683 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:03.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:03.239494+0000) 2023-08-28T02:05:03.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:03 smithi136 ceph-mon[113531]: pgmap v9344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:03.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:03 smithi093 ceph-mon[111335]: pgmap v9344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:03.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:03 smithi093 ceph-mon[118902]: pgmap v9344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:04.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:04.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:04.239749+0000) 2023-08-28T02:05:05.356 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/589257551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:05.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/589257551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:05.238+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:05.239965+0000) 2023-08-28T02:05:05.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/589257551' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:05:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:05:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:05:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:06.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:06.240167+0000) 2023-08-28T02:05:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:06 smithi136 ceph-mon[113531]: pgmap v9345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:06 smithi093 ceph-mon[111335]: pgmap v9345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:06 smithi093 ceph-mon[118902]: pgmap v9345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:07.664 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:07.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:07.240416+0000) 2023-08-28T02:05:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:07 smithi136 ceph-mon[113531]: pgmap v9346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3618332699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:07 smithi093 ceph-mon[111335]: pgmap v9346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3618332699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:07 smithi093 ceph-mon[118902]: pgmap v9346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3618332699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:08.239+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:08.240632+0000) 2023-08-28T02:05:09.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:09.238 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:09.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:09.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:09.240889+0000) 2023-08-28T02:05:10.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:09 smithi093 ceph-mon[111335]: pgmap v9347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3009776455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:09 smithi093 ceph-mon[118902]: pgmap v9347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3009776455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:10.239 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:09 smithi136 ceph-mon[113531]: pgmap v9347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:10.239 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3009776455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:10.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:10.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:10.241154+0000) 2023-08-28T02:05:11.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:11.240+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:11.241414+0000) 2023-08-28T02:05:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:11.468Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:11.469Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:12 smithi136 ceph-mon[113531]: pgmap v9348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:12.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:12.241664+0000) 2023-08-28T02:05:12.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:12 smithi093 ceph-mon[111335]: pgmap v9348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:12 smithi093 ceph-mon[118902]: pgmap v9348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/306011553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/306011553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:12.900Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:12.901Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:13.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/306011553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:13.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:13.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:13.241925+0000) 2023-08-28T02:05:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:14 smithi136 ceph-mon[113531]: pgmap v9349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:14.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:14.242150+0000) 2023-08-28T02:05:14.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:14 smithi093 ceph-mon[111335]: pgmap v9349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:14 smithi093 ceph-mon[118902]: pgmap v9349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:15.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3375678072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:15.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3375678072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:15.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3375678072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:15.241+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:15.242410+0000) 2023-08-28T02:05:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:05:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:05:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:05:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:16 smithi136 ceph-mon[113531]: pgmap v9350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:16.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:16.242669+0000) 2023-08-28T02:05:16.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:16 smithi093 ceph-mon[111335]: pgmap v9350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:16 smithi093 ceph-mon[118902]: pgmap v9350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:17.661 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:17.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:17.242928+0000) 2023-08-28T02:05:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1578787401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:17.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:17 smithi136 ceph-mon[113531]: pgmap v9351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:17.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1578787401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:17 smithi093 ceph-mon[118902]: pgmap v9351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1578787401' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:17 smithi093 ceph-mon[111335]: pgmap v9351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:18.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:18.243201+0000) 2023-08-28T02:05:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:19.242+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:19.243471+0000) 2023-08-28T02:05:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:20 smithi136 ceph-mon[113531]: pgmap v9352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/716368421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:20.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:20.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:20.243727+0000) 2023-08-28T02:05:20.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:20 smithi093 ceph-mon[111335]: pgmap v9352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/716368421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:20 smithi093 ceph-mon[118902]: pgmap v9352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/716368421' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:21.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:21.243990+0000) 2023-08-28T02:05:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:21.469Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:21.470Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:22 smithi136 ceph-mon[113531]: pgmap v9353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/281289638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:22.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:22.243+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:22.244297+0000) 2023-08-28T02:05:22.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:22 smithi093 ceph-mon[111335]: pgmap v9353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/281289638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:22 smithi093 ceph-mon[118902]: pgmap v9353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/281289638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:22.901Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:22.902Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:23.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:23.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:23.244551+0000) 2023-08-28T02:05:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:24 smithi136 ceph-mon[113531]: pgmap v9354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:24.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4247435736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:24.440 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:24.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:24.244780+0000) 2023-08-28T02:05:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:24 smithi093 ceph-mon[111335]: pgmap v9354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4247435736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:24 smithi093 ceph-mon[118902]: pgmap v9354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4247435736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:25.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:25.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:25.244995+0000) 2023-08-28T02:05:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:05:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:05:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:05:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:26 smithi136 ceph-mon[113531]: pgmap v9355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:26.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:26.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:26.245264+0000) 2023-08-28T02:05:26.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:26 smithi093 ceph-mon[111335]: pgmap v9355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:26 smithi093 ceph-mon[118902]: pgmap v9355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:27.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1343605856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:27.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:27.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:27.245543+0000) 2023-08-28T02:05:27.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1343605856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1343605856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:28.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:28 smithi136 ceph-mon[113531]: pgmap v9356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:28.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:28.244+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:28.245795+0000) 2023-08-28T02:05:28.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:28 smithi093 ceph-mon[111335]: pgmap v9356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:28.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:28 smithi093 ceph-mon[118902]: pgmap v9356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:29.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2158590424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:29.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:29.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:29.246083+0000) 2023-08-28T02:05:29.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2158590424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2158590424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:30 smithi136 ceph-mon[113531]: pgmap v9357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:30.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:30.245+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:30.246330+0000) 2023-08-28T02:05:30.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:30 smithi093 ceph-mon[111335]: pgmap v9357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:30 smithi093 ceph-mon[118902]: pgmap v9357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:31.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:31.246553+0000) 2023-08-28T02:05:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:31.470Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:31.471Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:32 smithi136 ceph-mon[113531]: pgmap v9358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2926842747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:32.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:32.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:32.246829+0000) 2023-08-28T02:05:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:32 smithi093 ceph-mon[111335]: pgmap v9358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:32.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2926842747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:32 smithi093 ceph-mon[118902]: pgmap v9358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2926842747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:32.902Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:32.902Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:33.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:33.247156+0000) 2023-08-28T02:05:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:34 smithi136 ceph-mon[113531]: pgmap v9359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2346624772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:34.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:34.247358+0000) 2023-08-28T02:05:34.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:34 smithi093 ceph-mon[111335]: pgmap v9359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2346624772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:34 smithi093 ceph-mon[118902]: pgmap v9359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2346624772' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:35.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:35.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:35.247576+0000) 2023-08-28T02:05:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:05:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:05:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:05:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:36 smithi136 ceph-mon[113531]: pgmap v9360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1085065522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:36.246+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:36.247863+0000) 2023-08-28T02:05:36.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:36 smithi093 ceph-mon[111335]: pgmap v9360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1085065522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:36 smithi093 ceph-mon[118902]: pgmap v9360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1085065522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:37.669 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:37.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:37.248098+0000) 2023-08-28T02:05:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:37 smithi136 ceph-mon[113531]: pgmap v9361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:37 smithi093 ceph-mon[111335]: pgmap v9361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:37 smithi093 ceph-mon[118902]: pgmap v9361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:38.670 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:38.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:38.248335+0000) 2023-08-28T02:05:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3003378961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:38.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3003378961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3003378961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:39.675 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:39.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:39.248589+0000) 2023-08-28T02:05:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:39.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:39 smithi136 ceph-mon[113531]: pgmap v9362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:39 smithi093 ceph-mon[111335]: pgmap v9362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:39 smithi093 ceph-mon[118902]: pgmap v9362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:40.247+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:40.248836+0000) 2023-08-28T02:05:41.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1683990349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:41.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1683990349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1683990349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:41.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:41.249094+0000) 2023-08-28T02:05:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:41.470Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:41.472Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:41 smithi136 ceph-mon[113531]: pgmap v9363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:42.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:41 smithi093 ceph-mon[111335]: pgmap v9363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:41 smithi093 ceph-mon[118902]: pgmap v9363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:42.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:42.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:42.249373+0000) 2023-08-28T02:05:43.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:42.902Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:42.903Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:43.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:43.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:43.249641+0000) 2023-08-28T02:05:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:44 smithi136 ceph-mon[113531]: pgmap v9364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/467456901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:44.248+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:44.249906+0000) 2023-08-28T02:05:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:44 smithi093 ceph-mon[111335]: pgmap v9364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/467456901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:44 smithi093 ceph-mon[118902]: pgmap v9364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/467456901' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:05:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:45.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:45.250096+0000) 2023-08-28T02:05:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:05:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:05:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:05:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:46 smithi136 ceph-mon[113531]: pgmap v9365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2055461741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:46.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:46.250324+0000) 2023-08-28T02:05:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:46 smithi093 ceph-mon[111335]: pgmap v9365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2055461741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:46 smithi093 ceph-mon[118902]: pgmap v9365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2055461741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:47.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:47.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:47.250523+0000) 2023-08-28T02:05:47.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:47 smithi136 ceph-mon[113531]: pgmap v9366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[111335]: pgmap v9366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[118902]: pgmap v9366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:48.249+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:48.250758+0000) 2023-08-28T02:05:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:05:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/273609063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:05:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/273609063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:05:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:05:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:05:48.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/273609063' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:49.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:49 smithi136 ceph-mon[113531]: pgmap v9367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:49.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:49.250976+0000) 2023-08-28T02:05:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:49 smithi093 ceph-mon[111335]: pgmap v9367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:49 smithi093 ceph-mon[118902]: pgmap v9367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:50.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2772413111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:50.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:50.251213+0000) 2023-08-28T02:05:50.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2772413111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2772413111' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:51.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:51 smithi136 ceph-mon[113531]: pgmap v9368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:51.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:51.251455+0000) 2023-08-28T02:05:51.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:51 smithi093 ceph-mon[111335]: pgmap v9368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:51 smithi093 ceph-mon[118902]: pgmap v9368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:51.471Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:51.472Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:52.665 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:52.250+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:52.251672+0000) 2023-08-28T02:05:52.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/94052399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:52.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/94052399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/94052399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:52.903Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:05:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:05:52.904Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:05:53.672 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:53.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:53.251934+0000) 2023-08-28T02:05:53.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:53 smithi136 ceph-mon[113531]: pgmap v9369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:53.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:53 smithi093 ceph-mon[111335]: pgmap v9369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:53 smithi093 ceph-mon[118902]: pgmap v9369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:54.671 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:54.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:54.252186+0000) 2023-08-28T02:05:54.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:54.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:05:55.677 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:05:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:05:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:05:55.678 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:55.251+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:55.252386+0000) 2023-08-28T02:05:55.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:55 smithi136 ceph-mon[113531]: pgmap v9370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:55.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/597172768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:55.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:55 smithi093 ceph-mon[111335]: pgmap v9370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:55.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/597172768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:55 smithi093 ceph-mon[118902]: pgmap v9370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/597172768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:56.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:56.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:56.252648+0000) 2023-08-28T02:05:57.670 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:57.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:57.252901+0000) 2023-08-28T02:05:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:57 smithi136 ceph-mon[113531]: pgmap v9371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:05:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1708109644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:57.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:57 smithi093 ceph-mon[111335]: pgmap v9371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:05:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1708109644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:57 smithi093 ceph-mon[118902]: pgmap v9371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:05:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:05:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1708109644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:05:58.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:58.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:58.253134+0000) 2023-08-28T02:05:59.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:05:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:05:59.252+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:05:59.253386+0000) 2023-08-28T02:06:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:00 smithi136 ceph-mon[113531]: pgmap v9372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/703529447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:00.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:00.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:00.253658+0000) 2023-08-28T02:06:00.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:00 smithi093 ceph-mon[111335]: pgmap v9372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/703529447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:00 smithi093 ceph-mon[118902]: pgmap v9372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/703529447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:01.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:01.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:01.253918+0000) 2023-08-28T02:06:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:01.472Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:01.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:01.473Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:02 smithi136 ceph-mon[113531]: pgmap v9373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:02.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:02.254183+0000) 2023-08-28T02:06:02.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:02 smithi093 ceph-mon[111335]: pgmap v9373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:02 smithi093 ceph-mon[118902]: pgmap v9373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:03.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1048169029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1048169029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:02.904Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:02.905Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:03.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1048169029' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:03.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:03.253+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:03.254371+0000) 2023-08-28T02:06:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:04 smithi136 ceph-mon[113531]: pgmap v9374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:04.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:04.254610+0000) 2023-08-28T02:06:04.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:04 smithi093 ceph-mon[111335]: pgmap v9374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:04 smithi093 ceph-mon[118902]: pgmap v9374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/7440392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:05.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/7440392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:05.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/7440392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:05.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:05.254844+0000) 2023-08-28T02:06:05.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:06:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:06:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:06:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:06 smithi136 ceph-mon[113531]: pgmap v9375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:06.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:06.255129+0000) 2023-08-28T02:06:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:06 smithi093 ceph-mon[111335]: pgmap v9375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:06 smithi093 ceph-mon[118902]: pgmap v9375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:07.676 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:07.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:07.255402+0000) 2023-08-28T02:06:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4155499024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:07.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:07 smithi136 ceph-mon[113531]: pgmap v9376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:07.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4155499024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:07 smithi093 ceph-mon[111335]: pgmap v9376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4155499024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:07 smithi093 ceph-mon[118902]: pgmap v9376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:08.254+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:08.255689+0000) 2023-08-28T02:06:09.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:09.254 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:09.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:09.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:09.255949+0000) 2023-08-28T02:06:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:09 smithi093 ceph-mon[111335]: pgmap v9377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1756442097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:09 smithi093 ceph-mon[118902]: pgmap v9377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1756442097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:10.254 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:09 smithi136 ceph-mon[113531]: pgmap v9377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:10.254 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1756442097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:10.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:10.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:10.256149+0000) 2023-08-28T02:06:11.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:11.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:11.256390+0000) 2023-08-28T02:06:11.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:11.473Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:11.473Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:12.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:12.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:12.256647+0000) 2023-08-28T02:06:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:12 smithi136 ceph-mon[113531]: pgmap v9378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/119157807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:12 smithi093 ceph-mon[111335]: pgmap v9378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/119157807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:12 smithi093 ceph-mon[118902]: pgmap v9378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/119157807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:13.237 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:12.904Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:12.905Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:13.255+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:13.256915+0000) 2023-08-28T02:06:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:14 smithi136 ceph-mon[113531]: pgmap v9379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:14.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:14.256+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:14.257129+0000) 2023-08-28T02:06:14.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:14 smithi093 ceph-mon[111335]: pgmap v9379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:14 smithi093 ceph-mon[118902]: pgmap v9379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:15.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3131164825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:15.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3131164825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:15.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3131164825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:15.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:15.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:15.257328+0000) 2023-08-28T02:06:15.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:06:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:06:15] "GET /metrics HTTP/1.1" 200 239935 "" "Prometheus/2.43.0" 2023-08-28T02:06:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:16 smithi136 ceph-mon[113531]: pgmap v9380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:16.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:16.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:16.257579+0000) 2023-08-28T02:06:16.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:16 smithi093 ceph-mon[111335]: pgmap v9380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:16 smithi093 ceph-mon[118902]: pgmap v9380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:17.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2803258489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:17.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:17.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:17.257873+0000) 2023-08-28T02:06:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2803258489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2803258489' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:18.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:18 smithi136 ceph-mon[113531]: pgmap v9381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:18.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:18.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:18.258176+0000) 2023-08-28T02:06:18.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:18 smithi093 ceph-mon[111335]: pgmap v9381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:18.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:18 smithi093 ceph-mon[118902]: pgmap v9381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:19.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/525468452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:19.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:19.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:19.258442+0000) 2023-08-28T02:06:19.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/525468452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/525468452' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:20 smithi136 ceph-mon[113531]: pgmap v9382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:20.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:20.258686+0000) 2023-08-28T02:06:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:20 smithi093 ceph-mon[111335]: pgmap v9382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:20 smithi093 ceph-mon[118902]: pgmap v9382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:21.257+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:21.258913+0000) 2023-08-28T02:06:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:21.474Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:21.475Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:22 smithi136 ceph-mon[113531]: pgmap v9383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3136739928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:22.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:22.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:22.259187+0000) 2023-08-28T02:06:22.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:22 smithi093 ceph-mon[111335]: pgmap v9383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3136739928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:22 smithi093 ceph-mon[118902]: pgmap v9383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3136739928' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:22.905Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:22.906Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:23.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:23.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:23.259426+0000) 2023-08-28T02:06:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:24 smithi136 ceph-mon[113531]: pgmap v9384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1273373691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:24.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:24.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:24.259679+0000) 2023-08-28T02:06:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:24 smithi093 ceph-mon[111335]: pgmap v9384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1273373691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:24 smithi093 ceph-mon[118902]: pgmap v9384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1273373691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:25.258+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:25.259915+0000) 2023-08-28T02:06:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:06:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:06:25] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T02:06:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:26 smithi136 ceph-mon[113531]: pgmap v9385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/113258973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:26.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:26.260203+0000) 2023-08-28T02:06:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:26 smithi093 ceph-mon[111335]: pgmap v9385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/113258973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:26 smithi093 ceph-mon[118902]: pgmap v9385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/113258973' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:27.674 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:27.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:27.260428+0000) 2023-08-28T02:06:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:27 smithi136 ceph-mon[113531]: pgmap v9386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:27.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:27 smithi093 ceph-mon[111335]: pgmap v9386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:27 smithi093 ceph-mon[118902]: pgmap v9386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:28.675 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:28.259+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:28.260699+0000) 2023-08-28T02:06:28.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1268363930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1268363930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1268363930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:29.681 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:29.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:29.260944+0000) 2023-08-28T02:06:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:29 smithi136 ceph-mon[113531]: pgmap v9387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:29.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:29 smithi093 ceph-mon[111335]: pgmap v9387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:29 smithi093 ceph-mon[118902]: pgmap v9387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:30.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:30.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:30.261189+0000) 2023-08-28T02:06:31.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/828504044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:31.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/828504044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/828504044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:31.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:31.261413+0000) 2023-08-28T02:06:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:31.475Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:31.476Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:31 smithi136 ceph-mon[113531]: pgmap v9388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:31 smithi093 ceph-mon[111335]: pgmap v9388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:31 smithi093 ceph-mon[118902]: pgmap v9388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:32.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:32.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:32.261636+0000) 2023-08-28T02:06:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:32.905Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:32.906Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:33.260+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:33.261859+0000) 2023-08-28T02:06:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:34 smithi136 ceph-mon[113531]: pgmap v9389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1930249567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:34.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:34.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:34.262130+0000) 2023-08-28T02:06:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:34 smithi093 ceph-mon[111335]: pgmap v9389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:34.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1930249567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:34 smithi093 ceph-mon[118902]: pgmap v9389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1930249567' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:35.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:35.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:35.262297+0000) 2023-08-28T02:06:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:06:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:06:35] "GET /metrics HTTP/1.1" 200 239938 "" "Prometheus/2.43.0" 2023-08-28T02:06:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:36.261+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:36.262542+0000) 2023-08-28T02:06:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:36 smithi136 ceph-mon[113531]: pgmap v9390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/549205739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:36.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:36 smithi093 ceph-mon[111335]: pgmap v9390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/549205739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:36 smithi093 ceph-mon[118902]: pgmap v9390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/549205739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:37.679 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:37.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:37.262806+0000) 2023-08-28T02:06:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:37 smithi136 ceph-mon[113531]: pgmap v9391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:37 smithi093 ceph-mon[111335]: pgmap v9391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:37 smithi093 ceph-mon[118902]: pgmap v9391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:38.680 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:38.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:38.263065+0000) 2023-08-28T02:06:38.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3355794151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:38.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3355794151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3355794151' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:39.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:39.263309+0000) 2023-08-28T02:06:39.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:39 smithi093 ceph-mon[111335]: pgmap v9392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:39 smithi093 ceph-mon[118902]: pgmap v9392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:39 smithi136 ceph-mon[113531]: pgmap v9392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:40.262+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:40.263561+0000) 2023-08-28T02:06:40.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3259136713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:40.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3259136713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:41.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3259136713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:41.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:41.263772+0000) 2023-08-28T02:06:41.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:41 smithi093 ceph-mon[111335]: pgmap v9393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:41 smithi093 ceph-mon[118902]: pgmap v9393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:41.475Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:41.476Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:41 smithi136 ceph-mon[113531]: pgmap v9393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:42.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:42.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:42.263974+0000) 2023-08-28T02:06:42.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2038429240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:42.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2038429240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:42.906Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:42.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:42.906Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:43.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2038429240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:43.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:43.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:43.264212+0000) 2023-08-28T02:06:43.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:43 smithi093 ceph-mon[111335]: pgmap v9394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:43.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:43 smithi093 ceph-mon[118902]: pgmap v9394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:44.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:43 smithi136 ceph-mon[113531]: pgmap v9394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:44.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:44.263+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:44.264481+0000) 2023-08-28T02:06:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2578260502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2578260502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2578260502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:45.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:45.264762+0000) 2023-08-28T02:06:45.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:06:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:06:45] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T02:06:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:46 smithi136 ceph-mon[113531]: pgmap v9395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:46.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:46.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:46.265009+0000) 2023-08-28T02:06:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:46 smithi093 ceph-mon[111335]: pgmap v9395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:46 smithi093 ceph-mon[118902]: pgmap v9395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:47.683 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:47.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:47.265259+0000) 2023-08-28T02:06:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:47 smithi136 ceph-mon[113531]: pgmap v9396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:47.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3295248850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:47.987 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:47 smithi093 ceph-mon[118902]: pgmap v9396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3295248850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:47 smithi093 ceph-mon[111335]: pgmap v9396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3295248850' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:48.679 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:48.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:48.265460+0000) 2023-08-28T02:06:48.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:06:49.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:49.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:49.265631+0000) 2023-08-28T02:06:49.856 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:49 smithi136 ceph-mon[113531]: pgmap v9397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:49.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:49.857 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:49 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:49.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:49 smithi093 ceph-mon[111335]: pgmap v9397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:49 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:49 smithi093 ceph-mon[118902]: pgmap v9397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:49 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:50.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:50.264+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:50.265809+0000) 2023-08-28T02:06:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1635431574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1635431574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:50.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1635431574' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:51.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:51.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:51.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:06:51.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:06:51.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:06:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:51.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:51.266050+0000) 2023-08-28T02:06:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:51 smithi093 ceph-mon[111335]: pgmap v9398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:51 smithi093 ceph-mon[118902]: pgmap v9398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:51.475Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:51.476Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:51 smithi136 ceph-mon[113531]: pgmap v9398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:52.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:52.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:52.266311+0000) 2023-08-28T02:06:52.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2296503638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:52.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2296503638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:52.906Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:52.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:06:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:06:52.907Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:06:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2296503638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:53.265+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:53.266578+0000) 2023-08-28T02:06:53.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:53 smithi093 ceph-mon[111335]: pgmap v9399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:53.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:53 smithi093 ceph-mon[118902]: pgmap v9399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:53 smithi136 ceph-mon[113531]: pgmap v9399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:54.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:54.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:54.266847+0000) 2023-08-28T02:06:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:54.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2979226219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:54.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2979226219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:06:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2979226219' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:55.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:55.266+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:55.267115+0000) 2023-08-28T02:06:55.708 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:06:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:06:55] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:06:55.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:55 smithi093 ceph-mon[111335]: pgmap v9400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:55.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:55 smithi093 ceph-mon[118902]: pgmap v9400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:55 smithi136 ceph-mon[113531]: pgmap v9400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:56.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:56.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:56.267378+0000) 2023-08-28T02:06:57.682 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:57.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:57.267614+0000) 2023-08-28T02:06:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2314758624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:57.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:06:57 smithi136 ceph-mon[113531]: pgmap v9401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2314758624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:06:57 smithi093 ceph-mon[118902]: pgmap v9401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2314758624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:06:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:06:57 smithi093 ceph-mon[111335]: pgmap v9401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:06:58.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:58.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:58.267892+0000) 2023-08-28T02:06:59.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:06:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:06:59.267+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:06:59.268155+0000) 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:00 smithi136 ceph-mon[113531]: pgmap v9402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/207378984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:00.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:00.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:00.268389+0000) 2023-08-28T02:07:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:00 smithi093 ceph-mon[111335]: pgmap v9402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/207378984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:00 smithi093 ceph-mon[118902]: pgmap v9402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/207378984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:01.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:01.268654+0000) 2023-08-28T02:07:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:01.476Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:01.478Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:02 smithi136 ceph-mon[113531]: pgmap v9403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3385404582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:02.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:02.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:02.268916+0000) 2023-08-28T02:07:02.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:02 smithi093 ceph-mon[111335]: pgmap v9403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3385404582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:02 smithi093 ceph-mon[118902]: pgmap v9403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3385404582' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:02.906Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:02.907Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:03.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:03.269175+0000) 2023-08-28T02:07:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:04 smithi136 ceph-mon[113531]: pgmap v9404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:04.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:04.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:04.269379+0000) 2023-08-28T02:07:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:04 smithi093 ceph-mon[111335]: pgmap v9404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:04 smithi093 ceph-mon[118902]: pgmap v9404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2190550354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:05.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2190550354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:05.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2190550354' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:05.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:05.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:05.269617+0000) 2023-08-28T02:07:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:07:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:07:05] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:07:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:06 smithi136 ceph-mon[113531]: pgmap v9405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:06.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:06.268+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:06.269910+0000) 2023-08-28T02:07:06.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:06 smithi093 ceph-mon[111335]: pgmap v9405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:06 smithi093 ceph-mon[118902]: pgmap v9405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:07.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2309175374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:07.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:07.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:07.270187+0000) 2023-08-28T02:07:07.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2309175374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2309175374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:08.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:08 smithi136 ceph-mon[113531]: pgmap v9406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:08.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:08.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:08.270387+0000) 2023-08-28T02:07:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:08 smithi093 ceph-mon[111335]: pgmap v9406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:08.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:08 smithi093 ceph-mon[118902]: pgmap v9406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2462868555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:09.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:09.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:09.270662+0000) 2023-08-28T02:07:09.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2462868555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2462868555' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:10 smithi136 ceph-mon[113531]: pgmap v9407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:10.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:10.269+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:10.270923+0000) 2023-08-28T02:07:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:10 smithi093 ceph-mon[111335]: pgmap v9407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:10.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:10 smithi093 ceph-mon[118902]: pgmap v9407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:11.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:11.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:11.271161+0000) 2023-08-28T02:07:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:11.477Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:11.478Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:12 smithi136 ceph-mon[113531]: pgmap v9408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2522628344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:12.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:12.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:12.271321+0000) 2023-08-28T02:07:12.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:12 smithi093 ceph-mon[111335]: pgmap v9408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2522628344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:12 smithi093 ceph-mon[118902]: pgmap v9408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2522628344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:12.907Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:12.908Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:13.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:13.271520+0000) 2023-08-28T02:07:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:14 smithi136 ceph-mon[113531]: pgmap v9409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/267478638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:14.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:14.271747+0000) 2023-08-28T02:07:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:14 smithi093 ceph-mon[111335]: pgmap v9409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/267478638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:14 smithi093 ceph-mon[118902]: pgmap v9409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/267478638' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:15.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:15.270+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:15.271990+0000) 2023-08-28T02:07:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:07:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:07:15] "GET /metrics HTTP/1.1" 200 239936 "" "Prometheus/2.43.0" 2023-08-28T02:07:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:16 smithi136 ceph-mon[113531]: pgmap v9410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3852496674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:16.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:16.272197+0000) 2023-08-28T02:07:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:16 smithi093 ceph-mon[111335]: pgmap v9410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3852496674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:16 smithi093 ceph-mon[118902]: pgmap v9410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3852496674' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:17.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:17.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:17.272462+0000) 2023-08-28T02:07:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:17 smithi093 ceph-mon[111335]: pgmap v9411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:17 smithi093 ceph-mon[118902]: pgmap v9411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:17 smithi136 ceph-mon[113531]: pgmap v9411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:18.271+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:18.272760+0000) 2023-08-28T02:07:18.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2585404522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:18.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2585404522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2585404522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:19.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:19.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:19.272986+0000) 2023-08-28T02:07:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:19 smithi093 ceph-mon[111335]: pgmap v9412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:19 smithi093 ceph-mon[118902]: pgmap v9412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:19 smithi136 ceph-mon[113531]: pgmap v9412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:20.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:20.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:20.273221+0000) 2023-08-28T02:07:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1278060592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:21.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1278060592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1278060592' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:21.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:21.273478+0000) 2023-08-28T02:07:21.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:21.478Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:21.479Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:21 smithi136 ceph-mon[113531]: pgmap v9413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:21 smithi093 ceph-mon[111335]: pgmap v9413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:21 smithi093 ceph-mon[118902]: pgmap v9413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:22.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:22.272+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:22.273780+0000) 2023-08-28T02:07:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:22.908Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:22.909Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:23.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:23.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:23.274069+0000) 2023-08-28T02:07:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:24 smithi136 ceph-mon[113531]: pgmap v9414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/284819990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:24.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:24.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:24.274282+0000) 2023-08-28T02:07:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:24 smithi093 ceph-mon[111335]: pgmap v9414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/284819990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:24 smithi093 ceph-mon[118902]: pgmap v9414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/284819990' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:25.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:25.273+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:25.274516+0000) 2023-08-28T02:07:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:07:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:07:25] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:07:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:26 smithi136 ceph-mon[113531]: pgmap v9415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2966033861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:26.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:26.274771+0000) 2023-08-28T02:07:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:26 smithi093 ceph-mon[111335]: pgmap v9415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2966033861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:26 smithi093 ceph-mon[118902]: pgmap v9415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2966033861' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:27.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:27.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:27.274994+0000) 2023-08-28T02:07:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:27 smithi093 ceph-mon[111335]: pgmap v9416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:27 smithi093 ceph-mon[118902]: pgmap v9416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:27 smithi136 ceph-mon[113531]: pgmap v9416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:28.274+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:28.275255+0000) 2023-08-28T02:07:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2563152165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2563152165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2563152165' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:29.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:29.275508+0000) 2023-08-28T02:07:29.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:29 smithi093 ceph-mon[111335]: pgmap v9417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:29.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:29 smithi093 ceph-mon[118902]: pgmap v9417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:29 smithi136 ceph-mon[113531]: pgmap v9417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:30.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:30.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:30.275744+0000) 2023-08-28T02:07:30.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778983518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:30.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778983518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:31.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778983518' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:31.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:31.275991+0000) 2023-08-28T02:07:31.737 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:31 smithi093 ceph-mon[111335]: pgmap v9418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:31 smithi093 ceph-mon[118902]: pgmap v9418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:31.478Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:31.479Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:31 smithi136 ceph-mon[113531]: pgmap v9418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:32.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:32.275+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:32.276245+0000) 2023-08-28T02:07:33.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2726623033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2726623033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2726623033' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:32.908Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:32.909Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:33.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:33.276486+0000) 2023-08-28T02:07:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:33 smithi136 ceph-mon[113531]: pgmap v9419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:33 smithi093 ceph-mon[111335]: pgmap v9419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:33 smithi093 ceph-mon[118902]: pgmap v9419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:34.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:34.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:34.276725+0000) 2023-08-28T02:07:35.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:35.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:35.277000+0000) 2023-08-28T02:07:35.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:07:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:07:35] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:07:36.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:36.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:36.277242+0000) 2023-08-28T02:07:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:36 smithi136 ceph-mon[113531]: pgmap v9420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/379609013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:36.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:36 smithi093 ceph-mon[111335]: pgmap v9420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/379609013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:36 smithi093 ceph-mon[118902]: pgmap v9420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/379609013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:37.686 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:37.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:37.277454+0000) 2023-08-28T02:07:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:37 smithi136 ceph-mon[113531]: pgmap v9421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:37.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/719289931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:37.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:37 smithi093 ceph-mon[111335]: pgmap v9421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/719289931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:37 smithi093 ceph-mon[118902]: pgmap v9421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/719289931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:38.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:38.277672+0000) 2023-08-28T02:07:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:39.276 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:39.276+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:39.277931+0000) 2023-08-28T02:07:40.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:39 smithi093 ceph-mon[111335]: pgmap v9422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2155307529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:39 smithi093 ceph-mon[118902]: pgmap v9422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2155307529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:40.276 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:39 smithi136 ceph-mon[113531]: pgmap v9422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:40.276 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2155307529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:40.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:40.278149+0000) 2023-08-28T02:07:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:41.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:41.278380+0000) 2023-08-28T02:07:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:41.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:41.479Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:42 smithi136 ceph-mon[113531]: pgmap v9423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:42.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:42.277+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:42.278557+0000) 2023-08-28T02:07:42.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:42 smithi093 ceph-mon[111335]: pgmap v9423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:42 smithi093 ceph-mon[118902]: pgmap v9423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2785227881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2785227881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:42.909Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:42.910Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:43.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2785227881' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:43.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:43.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:43.278707+0000) 2023-08-28T02:07:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:44 smithi136 ceph-mon[113531]: pgmap v9424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:44.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:44.278944+0000) 2023-08-28T02:07:44.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:44 smithi093 ceph-mon[111335]: pgmap v9424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:44 smithi093 ceph-mon[118902]: pgmap v9424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/377334279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:45.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/377334279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/377334279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:45.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:45.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:45.279199+0000) 2023-08-28T02:07:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:07:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:07:45] "GET /metrics HTTP/1.1" 200 239936 "" "Prometheus/2.43.0" 2023-08-28T02:07:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:46 smithi136 ceph-mon[113531]: pgmap v9425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:46.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:46.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:46.279418+0000) 2023-08-28T02:07:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:46 smithi093 ceph-mon[111335]: pgmap v9425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:46 smithi093 ceph-mon[118902]: pgmap v9425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:47.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:47.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:47.279662+0000) 2023-08-28T02:07:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2152198155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:47 smithi093 ceph-mon[111335]: pgmap v9426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2152198155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:47 smithi093 ceph-mon[118902]: pgmap v9426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2152198155' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:48.191 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:47 smithi136 ceph-mon[113531]: pgmap v9426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:48.278+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:48.279908+0000) 2023-08-28T02:07:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:49.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:49.280178+0000) 2023-08-28T02:07:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:50 smithi136 ceph-mon[113531]: pgmap v9427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3358208302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:50.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:50.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:50.280393+0000) 2023-08-28T02:07:50.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:50 smithi093 ceph-mon[111335]: pgmap v9427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3358208302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:50 smithi093 ceph-mon[118902]: pgmap v9427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3358208302' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:51.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:51.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:51.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:51.280639+0000) 2023-08-28T02:07:51.477 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:51.478 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:07:51.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:51.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:51.480Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:52.182 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:52 smithi093 ceph-mon[111335]: pgmap v9428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:52.182 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3795127601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:52.182 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:52 smithi093 ceph-mon[118902]: pgmap v9428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:52.182 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3795127601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:52.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:52 smithi136 ceph-mon[113531]: pgmap v9428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:52.279 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3795127601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:52.650 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:52.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:52.280814+0000) 2023-08-28T02:07:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:52.910Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:07:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:07:52.911Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:07:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:53.279+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:53.281020+0000) 2023-08-28T02:07:53.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: pgmap v9429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: pgmap v9429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: pgmap v9429: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:07:53.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:07:54.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:54.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:54.281261+0000) 2023-08-28T02:07:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/405599116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/405599116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:07:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/405599116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:55.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:55.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:55.281476+0000) 2023-08-28T02:07:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:07:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:07:55] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T02:07:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:55 smithi136 ceph-mon[113531]: pgmap v9430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:56.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:55 smithi093 ceph-mon[111335]: pgmap v9430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:55 smithi093 ceph-mon[118902]: pgmap v9430: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:56.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:56.280+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:56.281660+0000) 2023-08-28T02:07:57.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2359407535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2359407535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2359407535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:57.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:57.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:57.281897+0000) 2023-08-28T02:07:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:57 smithi136 ceph-mon[113531]: pgmap v9431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:57 smithi093 ceph-mon[111335]: pgmap v9431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:57 smithi093 ceph-mon[118902]: pgmap v9431: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:07:58.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:58.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:58.282154+0000) 2023-08-28T02:07:59.280 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:07:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/430702788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:07:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/430702788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:07:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/430702788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:07:59.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:07:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:07:59.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:07:59.282305+0000) 2023-08-28T02:08:00.281 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:00 smithi136 ceph-mon[113531]: pgmap v9432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:00.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:00 smithi093 ceph-mon[111335]: pgmap v9432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:00 smithi093 ceph-mon[118902]: pgmap v9432: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:00.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:00.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:00.282503+0000) 2023-08-28T02:08:01.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:01.281+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:01.282779+0000) 2023-08-28T02:08:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:01.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:01.480Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:02.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:02.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:02.283044+0000) 2023-08-28T02:08:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:02 smithi136 ceph-mon[113531]: pgmap v9433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/59823211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:02 smithi093 ceph-mon[111335]: pgmap v9433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/59823211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:02 smithi093 ceph-mon[118902]: pgmap v9433: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/59823211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:02.910Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:02.911Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:03.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:03.283252+0000) 2023-08-28T02:08:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:04 smithi136 ceph-mon[113531]: pgmap v9434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/573105666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:04.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:04.283455+0000) 2023-08-28T02:08:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:04 smithi093 ceph-mon[111335]: pgmap v9434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/573105666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:04 smithi093 ceph-mon[118902]: pgmap v9434: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/573105666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:05.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:05.282+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:05.283716+0000) 2023-08-28T02:08:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:08:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:08:05] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T02:08:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:06 smithi136 ceph-mon[113531]: pgmap v9435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:06.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:06.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:06.283973+0000) 2023-08-28T02:08:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:06 smithi093 ceph-mon[111335]: pgmap v9435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:06 smithi093 ceph-mon[118902]: pgmap v9435: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:07.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2157145739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:07.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:07.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:07.284224+0000) 2023-08-28T02:08:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2157145739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2157145739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:08.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:08 smithi136 ceph-mon[113531]: pgmap v9436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:08.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:08.283+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:08.284495+0000) 2023-08-28T02:08:08.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:08 smithi093 ceph-mon[111335]: pgmap v9436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:08.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:08 smithi093 ceph-mon[118902]: pgmap v9436: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4285942927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:09.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:09.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:09.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:09.284753+0000) 2023-08-28T02:08:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4285942927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4285942927' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:10 smithi136 ceph-mon[113531]: pgmap v9437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:10.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:10.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:10.285008+0000) 2023-08-28T02:08:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:10 smithi093 ceph-mon[111335]: pgmap v9437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:10.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:10 smithi093 ceph-mon[118902]: pgmap v9437: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:11.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2008941579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:11.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:11.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:11.285276+0000) 2023-08-28T02:08:11.478 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2008941579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:11.478 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2008941579' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:11.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:11.479Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:11.480Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:12 smithi136 ceph-mon[113531]: pgmap v9438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:12.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:12.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:12.285523+0000) 2023-08-28T02:08:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:12 smithi093 ceph-mon[111335]: pgmap v9438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:12 smithi093 ceph-mon[118902]: pgmap v9438: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:12.911Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:12.912Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:13.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:13.285760+0000) 2023-08-28T02:08:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:14 smithi136 ceph-mon[113531]: pgmap v9439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1553882762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:14.284+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:14.285983+0000) 2023-08-28T02:08:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:14 smithi093 ceph-mon[111335]: pgmap v9439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1553882762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:14 smithi093 ceph-mon[118902]: pgmap v9439: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1553882762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:15.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:15.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:15.286226+0000) 2023-08-28T02:08:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:08:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:08:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T02:08:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:16 smithi136 ceph-mon[113531]: pgmap v9440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/892919331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:16.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:16.286489+0000) 2023-08-28T02:08:16.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:16 smithi093 ceph-mon[111335]: pgmap v9440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/892919331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:16 smithi093 ceph-mon[118902]: pgmap v9440: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/892919331' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:17.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:17.285+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:17.286754+0000) 2023-08-28T02:08:17.987 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:17 smithi093 ceph-mon[111335]: pgmap v9441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:17 smithi093 ceph-mon[118902]: pgmap v9441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:17 smithi136 ceph-mon[113531]: pgmap v9441: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:18.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:18.287071+0000) 2023-08-28T02:08:18.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2223921635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:18.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2223921635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:19.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2223921635' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:19.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:19.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:19.287304+0000) 2023-08-28T02:08:19.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:19 smithi093 ceph-mon[111335]: pgmap v9442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:19 smithi093 ceph-mon[118902]: pgmap v9442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:19 smithi136 ceph-mon[113531]: pgmap v9442: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:20.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:20.287529+0000) 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1857399562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:20.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1857399562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1857399562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:21.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:21.287737+0000) 2023-08-28T02:08:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:21 smithi093 ceph-mon[111335]: pgmap v9443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:21 smithi093 ceph-mon[118902]: pgmap v9443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:21.480Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:21.481Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:21 smithi136 ceph-mon[113531]: pgmap v9443: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:22.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:22.286+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:22.287985+0000) 2023-08-28T02:08:23.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/767632455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/767632455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/767632455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:22.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:22.912Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:23.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:23.287+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:23.288243+0000) 2023-08-28T02:08:24.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:23 smithi136 ceph-mon[113531]: pgmap v9444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:23 smithi093 ceph-mon[111335]: pgmap v9444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:23 smithi093 ceph-mon[118902]: pgmap v9444: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:24.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:24.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:24.288501+0000) 2023-08-28T02:08:25.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:25.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:25.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:25.288756+0000) 2023-08-28T02:08:25.737 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:08:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:08:25] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T02:08:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:25 smithi136 ceph-mon[113531]: pgmap v9445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3773288757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:25 smithi093 ceph-mon[111335]: pgmap v9445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3773288757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:25 smithi093 ceph-mon[118902]: pgmap v9445: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3773288757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:26.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:26.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:26.288986+0000) 2023-08-28T02:08:27.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:27.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:27.289198+0000) 2023-08-28T02:08:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:27 smithi093 ceph-mon[111335]: pgmap v9446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2983250838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:27 smithi093 ceph-mon[118902]: pgmap v9446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2983250838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:27 smithi136 ceph-mon[113531]: pgmap v9446: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2983250838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:28.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:28.289487+0000) 2023-08-28T02:08:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:29.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:29.289756+0000) 2023-08-28T02:08:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:30 smithi136 ceph-mon[113531]: pgmap v9447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12286553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:30.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:30.288+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:30.289980+0000) 2023-08-28T02:08:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:30 smithi093 ceph-mon[111335]: pgmap v9447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12286553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:30 smithi093 ceph-mon[118902]: pgmap v9447: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12286553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:31.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:31.290213+0000) 2023-08-28T02:08:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:31.481Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:31.483Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:32 smithi136 ceph-mon[113531]: pgmap v9448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:32.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:32.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:32.290456+0000) 2023-08-28T02:08:32.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:32 smithi093 ceph-mon[111335]: pgmap v9448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:32 smithi093 ceph-mon[118902]: pgmap v9448: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:33.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2794573714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2794573714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:32.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:32.913Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:33.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2794573714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:33.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:33.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:33.290689+0000) 2023-08-28T02:08:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:34 smithi136 ceph-mon[113531]: pgmap v9449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:34.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:34.289+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:34.290910+0000) 2023-08-28T02:08:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:34 smithi093 ceph-mon[111335]: pgmap v9449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:34 smithi093 ceph-mon[118902]: pgmap v9449: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:35.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3356827909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:35.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:35.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:35.291154+0000) 2023-08-28T02:08:35.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3356827909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:35.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3356827909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:35.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:08:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:08:35] "GET /metrics HTTP/1.1" 200 239937 "" "Prometheus/2.43.0" 2023-08-28T02:08:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:36 smithi136 ceph-mon[113531]: pgmap v9450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:36.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:36.291410+0000) 2023-08-28T02:08:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:36 smithi093 ceph-mon[111335]: pgmap v9450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:36 smithi093 ceph-mon[118902]: pgmap v9450: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:37.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:37.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:37.291601+0000) 2023-08-28T02:08:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:37 smithi093 ceph-mon[111335]: pgmap v9451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1646175929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:37 smithi093 ceph-mon[118902]: pgmap v9451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1646175929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:37 smithi136 ceph-mon[113531]: pgmap v9451: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1646175929' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:38.290+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:38.291847+0000) 2023-08-28T02:08:39.237 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:39.290 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:39.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:39.292085+0000) 2023-08-28T02:08:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:39 smithi093 ceph-mon[118902]: pgmap v9452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1492455287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:39 smithi093 ceph-mon[111335]: pgmap v9452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1492455287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:40.290 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:39 smithi136 ceph-mon[113531]: pgmap v9452: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:40.291 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1492455287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:40.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:40.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:40.292289+0000) 2023-08-28T02:08:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:41.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:41.292502+0000) 2023-08-28T02:08:41.737 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:41.482Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:41.483Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:42 smithi136 ceph-mon[113531]: pgmap v9453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2838215035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:42.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:42.291+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:42.292753+0000) 2023-08-28T02:08:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:42 smithi093 ceph-mon[111335]: pgmap v9453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2838215035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:42 smithi093 ceph-mon[118902]: pgmap v9453: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2838215035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:42.912Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:42.914Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:43.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:43.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:43.292972+0000) 2023-08-28T02:08:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:44 smithi136 ceph-mon[113531]: pgmap v9454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:44.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:44.293218+0000) 2023-08-28T02:08:44.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:44 smithi093 ceph-mon[111335]: pgmap v9454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:44 smithi093 ceph-mon[118902]: pgmap v9454: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2666101537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:45.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2666101537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:45.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2666101537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:45.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:45.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:45.293456+0000) 2023-08-28T02:08:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:08:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:08:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:08:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:46 smithi136 ceph-mon[113531]: pgmap v9455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:46.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:46.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:46.293707+0000) 2023-08-28T02:08:46.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:46 smithi093 ceph-mon[111335]: pgmap v9455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:46 smithi093 ceph-mon[118902]: pgmap v9455: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3565094591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:47.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:47.292+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:47.293964+0000) 2023-08-28T02:08:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3565094591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3565094591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:48.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:48 smithi136 ceph-mon[113531]: pgmap v9456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:48.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:48.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:48.294281+0000) 2023-08-28T02:08:48.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:48 smithi093 ceph-mon[111335]: pgmap v9456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:48 smithi093 ceph-mon[118902]: pgmap v9456: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:49.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/960598647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:49.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:49.293+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:49.294537+0000) 2023-08-28T02:08:49.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/960598647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/960598647' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:50 smithi136 ceph-mon[113531]: pgmap v9457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:50.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:50.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:50.294815+0000) 2023-08-28T02:08:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:50 smithi093 ceph-mon[111335]: pgmap v9457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:50 smithi093 ceph-mon[118902]: pgmap v9457: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:51.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:51.295086+0000) 2023-08-28T02:08:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:51.483Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:51.484Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:52 smithi136 ceph-mon[113531]: pgmap v9458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4080417905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:52.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:52.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:52.295352+0000) 2023-08-28T02:08:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:52 smithi093 ceph-mon[111335]: pgmap v9458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4080417905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:52 smithi093 ceph-mon[118902]: pgmap v9458: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4080417905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:52.913Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:08:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:08:52.914Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:08:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:53.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:53.295627+0000) 2023-08-28T02:08:54.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:54.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:54.295781+0000) 2023-08-28T02:08:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:54 smithi136 ceph-mon[113531]: pgmap v9459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4124987714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[118902]: pgmap v9459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4124987714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[111335]: pgmap v9459: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4124987714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:08:55.629 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:55.294+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:55.295958+0000) 2023-08-28T02:08:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:08:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:08:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:08:56.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:56.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:56.296184+0000) 2023-08-28T02:08:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:56 smithi136 ceph-mon[113531]: pgmap v9460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[111335]: pgmap v9460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[118902]: pgmap v9460: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:57.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3011644935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:57.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:57.295+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:57.296441+0000) 2023-08-28T02:08:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3011644935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3011644935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:08:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:08:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:08:58.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:58 smithi136 ceph-mon[113531]: pgmap v9461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:58.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:58.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:58.296730+0000) 2023-08-28T02:08:58.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:58 smithi093 ceph-mon[111335]: pgmap v9461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:58 smithi093 ceph-mon[118902]: pgmap v9461: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:08:59.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:08:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3972759745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:59.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:08:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:08:59.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:08:59.297030+0000) 2023-08-28T02:08:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:08:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3972759745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:08:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:08:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3972759745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:00 smithi136 ceph-mon[113531]: pgmap v9462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:00.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:00.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:00.297365+0000) 2023-08-28T02:09:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:00 smithi093 ceph-mon[111335]: pgmap v9462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:00 smithi093 ceph-mon[118902]: pgmap v9462: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/677521935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:01.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:01.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:01.297614+0000) 2023-08-28T02:09:01.482 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/677521935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:01.482 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/677521935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:01.483Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:01.484Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:02 smithi136 ceph-mon[113531]: pgmap v9463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:02.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:02.296+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:02.297771+0000) 2023-08-28T02:09:02.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:02 smithi093 ceph-mon[111335]: pgmap v9463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:02 smithi093 ceph-mon[118902]: pgmap v9463: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:02.914Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:02.915Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:03.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:03.297990+0000) 2023-08-28T02:09:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:04 smithi136 ceph-mon[113531]: pgmap v9464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4052783473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:04.297+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:04.298235+0000) 2023-08-28T02:09:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:04 smithi093 ceph-mon[111335]: pgmap v9464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4052783473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:04 smithi093 ceph-mon[118902]: pgmap v9464: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4052783473' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:05.487 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:05 smithi093 ceph-mon[111335]: pgmap v9465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:05.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:05 smithi093 ceph-mon[118902]: pgmap v9465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:05.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:09:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:09:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:09:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:05 smithi136 ceph-mon[113531]: pgmap v9465: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:05.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:05.298517+0000) 2023-08-28T02:09:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1959224664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1959224664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1959224664' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:06.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:06.298738+0000) 2023-08-28T02:09:07.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:07.298+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:07.298973+0000) 2023-08-28T02:09:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:07 smithi093 ceph-mon[111335]: pgmap v9466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:07.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:07 smithi093 ceph-mon[118902]: pgmap v9466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:07 smithi136 ceph-mon[113531]: pgmap v9466: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:08.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:08.299276+0000) 2023-08-28T02:09:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2436671912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2436671912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2436671912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:09.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:09.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:09.299566+0000) 2023-08-28T02:09:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:09 smithi093 ceph-mon[111335]: pgmap v9467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:09 smithi093 ceph-mon[118902]: pgmap v9467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:10.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:09 smithi136 ceph-mon[113531]: pgmap v9467: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:10.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:10.299833+0000) 2023-08-28T02:09:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3513007915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3513007915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:11.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3513007915' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:11.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:11.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:11.300089+0000) 2023-08-28T02:09:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:11 smithi093 ceph-mon[111335]: pgmap v9468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:11 smithi093 ceph-mon[118902]: pgmap v9468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:11.483Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:11.484Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:11 smithi136 ceph-mon[113531]: pgmap v9468: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:12.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:12.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:12.300363+0000) 2023-08-28T02:09:13.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3355917765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3355917765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3355917765' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:12.915Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:12.916Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:13.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:13.300612+0000) 2023-08-28T02:09:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:13 smithi136 ceph-mon[113531]: pgmap v9469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:13 smithi093 ceph-mon[111335]: pgmap v9469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:13 smithi093 ceph-mon[118902]: pgmap v9469: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:14.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:14.299+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:14.300881+0000) 2023-08-28T02:09:15.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:15.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:15.301105+0000) 2023-08-28T02:09:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:09:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:09:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:09:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:16 smithi136 ceph-mon[113531]: pgmap v9470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3673423313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:16.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:16.301264+0000) 2023-08-28T02:09:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:16 smithi093 ceph-mon[111335]: pgmap v9470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3673423313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:16 smithi093 ceph-mon[118902]: pgmap v9470: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3673423313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:17.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:17.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:17.301439+0000) 2023-08-28T02:09:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:17 smithi093 ceph-mon[111335]: pgmap v9471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3247924747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:17 smithi093 ceph-mon[118902]: pgmap v9471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3247924747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:17 smithi136 ceph-mon[113531]: pgmap v9471: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3247924747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:18.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:18.301662+0000) 2023-08-28T02:09:19.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:19.300+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:19.301930+0000) 2023-08-28T02:09:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:20 smithi136 ceph-mon[113531]: pgmap v9472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2418633136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:20.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:20.302126+0000) 2023-08-28T02:09:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:20 smithi093 ceph-mon[118902]: pgmap v9472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2418633136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:20 smithi093 ceph-mon[111335]: pgmap v9472: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2418633136' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:21.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:21.302361+0000) 2023-08-28T02:09:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:21.484Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:21.485Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:22 smithi136 ceph-mon[113531]: pgmap v9473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:22.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:22.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:22.302607+0000) 2023-08-28T02:09:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:22 smithi093 ceph-mon[111335]: pgmap v9473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:22 smithi093 ceph-mon[118902]: pgmap v9473: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2851559910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2851559910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:22.915Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:22.916Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:23.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2851559910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:23.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:23.301+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:23.302789+0000) 2023-08-28T02:09:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:24.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:24.303022+0000) 2023-08-28T02:09:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:24 smithi136 ceph-mon[113531]: pgmap v9474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:24 smithi093 ceph-mon[111335]: pgmap v9474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:24 smithi093 ceph-mon[118902]: pgmap v9474: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/311580374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:25.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:25.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:25.303251+0000) 2023-08-28T02:09:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/311580374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:25.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/311580374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:25.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:09:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:09:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:09:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:26 smithi136 ceph-mon[113531]: pgmap v9475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:26.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:26.303506+0000) 2023-08-28T02:09:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:26 smithi093 ceph-mon[111335]: pgmap v9475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:26 smithi093 ceph-mon[118902]: pgmap v9475: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:27.302+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:27.303753+0000) 2023-08-28T02:09:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:27 smithi093 ceph-mon[111335]: pgmap v9476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:27.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2639850687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:27 smithi093 ceph-mon[118902]: pgmap v9476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2639850687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:27 smithi136 ceph-mon[113531]: pgmap v9476: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2639850687' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:28.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:28.304021+0000) 2023-08-28T02:09:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:29.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:29.304255+0000) 2023-08-28T02:09:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:30 smithi136 ceph-mon[113531]: pgmap v9477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1621527637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:30.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:30.303+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:30.304501+0000) 2023-08-28T02:09:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:30 smithi093 ceph-mon[111335]: pgmap v9477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1621527637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:30 smithi093 ceph-mon[118902]: pgmap v9477: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1621527637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:31.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:31.304785+0000) 2023-08-28T02:09:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:31.485Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:31.486Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:32 smithi136 ceph-mon[113531]: pgmap v9478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1705990171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:32.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:32.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:32.305006+0000) 2023-08-28T02:09:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:32 smithi093 ceph-mon[111335]: pgmap v9478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1705990171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:32 smithi093 ceph-mon[118902]: pgmap v9478: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1705990171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:32.916Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:32.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:33.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:33.305267+0000) 2023-08-28T02:09:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:34 smithi136 ceph-mon[113531]: pgmap v9479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:34.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:34.304+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:34.305516+0000) 2023-08-28T02:09:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:34 smithi093 ceph-mon[111335]: pgmap v9479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:34 smithi093 ceph-mon[118902]: pgmap v9479: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:35.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3637594383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:35.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:35.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:35.305759+0000) 2023-08-28T02:09:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3637594383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:35.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:09:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:09:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:09:35.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3637594383' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:36 smithi136 ceph-mon[113531]: pgmap v9480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:36.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:36.306083+0000) 2023-08-28T02:09:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:36 smithi093 ceph-mon[111335]: pgmap v9480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:36 smithi093 ceph-mon[118902]: pgmap v9480: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:37.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:37.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:37.306292+0000) 2023-08-28T02:09:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3917172878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:37 smithi093 ceph-mon[111335]: pgmap v9481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3917172878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:37 smithi093 ceph-mon[118902]: pgmap v9481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3917172878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:37 smithi136 ceph-mon[113531]: pgmap v9481: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:38.305+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:38.306494+0000) 2023-08-28T02:09:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:39.305 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:39.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:39.306748+0000) 2023-08-28T02:09:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:39 smithi093 ceph-mon[111335]: pgmap v9482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1458072955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:39 smithi093 ceph-mon[118902]: pgmap v9482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1458072955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:40.305 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:39 smithi136 ceph-mon[113531]: pgmap v9482: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:40.305 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1458072955' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:40.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:40.307008+0000) 2023-08-28T02:09:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:41.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:41.307235+0000) 2023-08-28T02:09:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:41.485Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:41.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:41.487Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:42 smithi136 ceph-mon[113531]: pgmap v9483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1997653411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:42.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:42.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:42.307517+0000) 2023-08-28T02:09:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:42 smithi093 ceph-mon[111335]: pgmap v9483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1997653411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:42 smithi093 ceph-mon[118902]: pgmap v9483: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1997653411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:42.916Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:42.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:43.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:43.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:43.307737+0000) 2023-08-28T02:09:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:44 smithi136 ceph-mon[113531]: pgmap v9484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/101619131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:44.306+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:44.308004+0000) 2023-08-28T02:09:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:44 smithi093 ceph-mon[111335]: pgmap v9484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/101619131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:44 smithi093 ceph-mon[118902]: pgmap v9484: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/101619131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:45.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:45.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:45.308219+0000) 2023-08-28T02:09:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:09:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:09:45] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T02:09:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:46 smithi136 ceph-mon[113531]: pgmap v9485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:46.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:46.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:46.308413+0000) 2023-08-28T02:09:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:46 smithi093 ceph-mon[111335]: pgmap v9485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:46 smithi093 ceph-mon[118902]: pgmap v9485: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:47.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1592572803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:47.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:47.307+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:47.308675+0000) 2023-08-28T02:09:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1592572803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1592572803' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:48.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:48 smithi136 ceph-mon[113531]: pgmap v9486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:48.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:48.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:48.308929+0000) 2023-08-28T02:09:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:48 smithi093 ceph-mon[111335]: pgmap v9486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:48 smithi093 ceph-mon[118902]: pgmap v9486: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:49.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/362522249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:49.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:49.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:49.309175+0000) 2023-08-28T02:09:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/362522249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/362522249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:50.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:50 smithi136 ceph-mon[113531]: pgmap v9487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:50.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:50.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:50.309417+0000) 2023-08-28T02:09:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:50 smithi093 ceph-mon[111335]: pgmap v9487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:50 smithi093 ceph-mon[118902]: pgmap v9487: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:51.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/182522964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:51.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:51.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:51.309626+0000) 2023-08-28T02:09:51.485 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/182522964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:51.485 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/182522964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:51.486Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:51.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:51.487Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:52.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:52 smithi136 ceph-mon[113531]: pgmap v9488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:52.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:52.308+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:52.309832+0000) 2023-08-28T02:09:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:52 smithi093 ceph-mon[111335]: pgmap v9488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:52 smithi093 ceph-mon[118902]: pgmap v9488: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:52.917Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:09:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:09:52.917Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:09:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:53.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:53.310131+0000) 2023-08-28T02:09:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:54 smithi136 ceph-mon[113531]: pgmap v9489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1414584289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:54.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:54.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:54.310387+0000) 2023-08-28T02:09:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:54 smithi093 ceph-mon[111335]: pgmap v9489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1414584289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:54 smithi093 ceph-mon[118902]: pgmap v9489: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1414584289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:09:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:55.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:55.310634+0000) 2023-08-28T02:09:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:09:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:09:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:09:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:56 smithi136 ceph-mon[113531]: pgmap v9490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3910402001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:56.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:56.309+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:56.310830+0000) 2023-08-28T02:09:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:56 smithi093 ceph-mon[111335]: pgmap v9490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3910402001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:56 smithi093 ceph-mon[118902]: pgmap v9490: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3910402001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:57 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:57 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:57.309 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:57 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:09:57.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:57.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:57.311008+0000) 2023-08-28T02:09:58.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:58 smithi136 ceph-mon[113531]: pgmap v9491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:58.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:58.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:58.311180+0000) 2023-08-28T02:09:58.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:58 smithi093 ceph-mon[111335]: pgmap v9491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:58 smithi093 ceph-mon[118902]: pgmap v9491: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:09:59.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:09:59.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:09:59.311321+0000) 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3657879739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:09:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3657879739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3657879739' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:09:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:09:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:09:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:10:00.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:00.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:00.311527+0000) 2023-08-28T02:10:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:00 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2385M 2023-08-28T02:10:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:00 smithi136 ceph-mon[113531]: pgmap v9492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:00 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:10:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:10:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2385M 2023-08-28T02:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[111335]: pgmap v9492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:10:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:10:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2385M 2023-08-28T02:10:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[118902]: pgmap v9492: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:10:00.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:10:01.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2075881031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:01.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:01.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:01.311752+0000) 2023-08-28T02:10:01.485 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2075881031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:01.486 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2075881031' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:01.486Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:01.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:01.487Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:02 smithi136 ceph-mon[113531]: pgmap v9493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:02.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:02.310+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:02.312008+0000) 2023-08-28T02:10:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:02 smithi093 ceph-mon[111335]: pgmap v9493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:02 smithi093 ceph-mon[118902]: pgmap v9493: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3153680762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3153680762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:02.917Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:02.918Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3153680762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:03.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:03.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:03.312291+0000) 2023-08-28T02:10:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:04 smithi136 ceph-mon[113531]: pgmap v9494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:04.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:04.312578+0000) 2023-08-28T02:10:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:04 smithi093 ceph-mon[111335]: pgmap v9494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:04 smithi093 ceph-mon[118902]: pgmap v9494: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:05.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:05.311+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:05.312855+0000) 2023-08-28T02:10:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:10:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:10:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:10:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:06 smithi136 ceph-mon[113531]: pgmap v9495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4002316542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:06.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:06.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:06.313168+0000) 2023-08-28T02:10:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:06 smithi093 ceph-mon[118902]: pgmap v9495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4002316542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:06 smithi093 ceph-mon[111335]: pgmap v9495: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4002316542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:07.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:07.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:07.313465+0000) 2023-08-28T02:10:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:07 smithi093 ceph-mon[111335]: pgmap v9496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:07 smithi093 ceph-mon[118902]: pgmap v9496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:07 smithi136 ceph-mon[113531]: pgmap v9496: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:08.312+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:08.313768+0000) 2023-08-28T02:10:08.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2638156443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:08.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2638156443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:09.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2638156443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:09.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:09.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:09.314098+0000) 2023-08-28T02:10:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:09.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:09 smithi093 ceph-mon[118902]: pgmap v9497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:09.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:09 smithi093 ceph-mon[111335]: pgmap v9497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:10.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:09 smithi136 ceph-mon[113531]: pgmap v9497: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:10.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:10.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:10.314366+0000) 2023-08-28T02:10:10.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2279098987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:10.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2279098987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:11.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2279098987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:11.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:11.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:11.314587+0000) 2023-08-28T02:10:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:11 smithi093 ceph-mon[111335]: pgmap v9498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:11 smithi093 ceph-mon[118902]: pgmap v9498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:11.487Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:11.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:11.488Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:11 smithi136 ceph-mon[113531]: pgmap v9498: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:12.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:12.313+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:12.314795+0000) 2023-08-28T02:10:12.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1681800176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:12.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1681800176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:12.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:12.918Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:12.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:12.919Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:13.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1681800176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:13.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:13.315112+0000) 2023-08-28T02:10:13.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:13 smithi093 ceph-mon[118902]: pgmap v9499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:13.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:13 smithi093 ceph-mon[111335]: pgmap v9499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:13 smithi136 ceph-mon[113531]: pgmap v9499: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:14.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:14.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:14.315401+0000) 2023-08-28T02:10:15.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1536632422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1536632422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1536632422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:15.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:15.314+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:15.315615+0000) 2023-08-28T02:10:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:10:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:10:15] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T02:10:16.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:15 smithi136 ceph-mon[113531]: pgmap v9500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:15 smithi093 ceph-mon[111335]: pgmap v9500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:15 smithi093 ceph-mon[118902]: pgmap v9500: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:16.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:16.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:16.315836+0000) 2023-08-28T02:10:17.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:17.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:17.316065+0000) 2023-08-28T02:10:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:17 smithi093 ceph-mon[111335]: pgmap v9501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/706075717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:17 smithi093 ceph-mon[118902]: pgmap v9501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/706075717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:17 smithi136 ceph-mon[113531]: pgmap v9501: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/706075717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:18.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:18.316338+0000) 2023-08-28T02:10:19.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:19.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:19.316581+0000) 2023-08-28T02:10:20.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:20 smithi136 ceph-mon[113531]: pgmap v9502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1059832249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:20.315+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:20.316849+0000) 2023-08-28T02:10:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:20 smithi093 ceph-mon[111335]: pgmap v9502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1059832249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:20 smithi093 ceph-mon[118902]: pgmap v9502: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1059832249' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:21.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:21.317154+0000) 2023-08-28T02:10:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:21.488Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:21.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:21.489Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:22 smithi136 ceph-mon[113531]: pgmap v9503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1539556705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:22.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:22.316+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:22.317439+0000) 2023-08-28T02:10:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:22 smithi093 ceph-mon[111335]: pgmap v9503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1539556705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:22 smithi093 ceph-mon[118902]: pgmap v9503: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1539556705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:22.919Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:22.920Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:23.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:23.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:23.317750+0000) 2023-08-28T02:10:24.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:24 smithi136 ceph-mon[113531]: pgmap v9504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:24.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:24.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:24.317972+0000) 2023-08-28T02:10:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:24 smithi093 ceph-mon[111335]: pgmap v9504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:24 smithi093 ceph-mon[118902]: pgmap v9504: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:25.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1896537806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:25.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:25.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:25.318155+0000) 2023-08-28T02:10:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1896537806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:25.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:10:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:10:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:10:25.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1896537806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:26 smithi136 ceph-mon[113531]: pgmap v9505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:26.317+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:26.318407+0000) 2023-08-28T02:10:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:26 smithi093 ceph-mon[111335]: pgmap v9505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:26 smithi093 ceph-mon[118902]: pgmap v9505: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:27.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:27.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:27.318633+0000) 2023-08-28T02:10:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3779557263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:27.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:27 smithi093 ceph-mon[118902]: pgmap v9506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3779557263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:27 smithi093 ceph-mon[111335]: pgmap v9506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3779557263' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:27 smithi136 ceph-mon[113531]: pgmap v9506: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:28.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:28.318902+0000) 2023-08-28T02:10:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:29.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:29.319240+0000) 2023-08-28T02:10:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:30 smithi136 ceph-mon[113531]: pgmap v9507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/979392876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:30.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:30.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:30.319459+0000) 2023-08-28T02:10:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:30 smithi093 ceph-mon[111335]: pgmap v9507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/979392876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:30 smithi093 ceph-mon[118902]: pgmap v9507: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/979392876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:31.318+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:31.319748+0000) 2023-08-28T02:10:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:31.488Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:31.738 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:31.489Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:32.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:32.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:32.320055+0000) 2023-08-28T02:10:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:32 smithi136 ceph-mon[113531]: pgmap v9508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1227041032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:32 smithi093 ceph-mon[111335]: pgmap v9508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1227041032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:32 smithi093 ceph-mon[118902]: pgmap v9508: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1227041032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:32.920Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:32.921Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:33.319+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:33.320344+0000) 2023-08-28T02:10:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:34 smithi136 ceph-mon[113531]: pgmap v9509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3165364415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:34.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:34.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:34.320572+0000) 2023-08-28T02:10:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:34 smithi093 ceph-mon[111335]: pgmap v9509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3165364415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:34 smithi093 ceph-mon[118902]: pgmap v9509: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3165364415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:35.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:35.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:35.320770+0000) 2023-08-28T02:10:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:10:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:10:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:10:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:36 smithi136 ceph-mon[113531]: pgmap v9510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:36.320+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:36.321082+0000) 2023-08-28T02:10:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:36 smithi093 ceph-mon[111335]: pgmap v9510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:36 smithi093 ceph-mon[118902]: pgmap v9510: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:37.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2865584240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:37.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:37.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:37.321364+0000) 2023-08-28T02:10:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2865584240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:37.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2865584240' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:38.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:38 smithi136 ceph-mon[113531]: pgmap v9511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:38.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:38.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:38.321625+0000) 2023-08-28T02:10:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:38 smithi093 ceph-mon[111335]: pgmap v9511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:38.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:38 smithi093 ceph-mon[118902]: pgmap v9511: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:39.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1549035271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:39.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:39.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:39.321886+0000) 2023-08-28T02:10:39.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1549035271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:39.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1549035271' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:40.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:40 smithi136 ceph-mon[113531]: pgmap v9512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:40.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:40.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:40.322223+0000) 2023-08-28T02:10:40.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:40 smithi093 ceph-mon[111335]: pgmap v9512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:40.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:40 smithi093 ceph-mon[118902]: pgmap v9512: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:41.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/967927716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:41.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:41.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:41.322519+0000) 2023-08-28T02:10:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/967927716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:41.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/967927716' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:41.489Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:41.490Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:42 smithi136 ceph-mon[113531]: pgmap v9513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:42.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:42.321+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:42.322863+0000) 2023-08-28T02:10:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:42 smithi093 ceph-mon[111335]: pgmap v9513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:42 smithi093 ceph-mon[118902]: pgmap v9513: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:42.920Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:42.921Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:43.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:43.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:43.323138+0000) 2023-08-28T02:10:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:44 smithi136 ceph-mon[113531]: pgmap v9514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1193143374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:44.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:44.323331+0000) 2023-08-28T02:10:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:44 smithi093 ceph-mon[111335]: pgmap v9514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1193143374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:44 smithi093 ceph-mon[118902]: pgmap v9514: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1193143374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:45.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:45.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:45.323565+0000) 2023-08-28T02:10:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:10:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:10:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:10:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:46 smithi136 ceph-mon[113531]: pgmap v9515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:46.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3121847407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:46.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:46.322+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:46.323830+0000) 2023-08-28T02:10:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:46 smithi093 ceph-mon[118902]: pgmap v9515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3121847407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:46 smithi093 ceph-mon[111335]: pgmap v9515: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3121847407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:47.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:47.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:47.324068+0000) 2023-08-28T02:10:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:47 smithi093 ceph-mon[111335]: pgmap v9516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:47 smithi093 ceph-mon[118902]: pgmap v9516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:47 smithi136 ceph-mon[113531]: pgmap v9516: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:48.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:48.324305+0000) 2023-08-28T02:10:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1131148943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1131148943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:49.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1131148943' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:49.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:49.324560+0000) 2023-08-28T02:10:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:49 smithi093 ceph-mon[111335]: pgmap v9517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:49 smithi093 ceph-mon[118902]: pgmap v9517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:49 smithi136 ceph-mon[113531]: pgmap v9517: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:50.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:50.323+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:50.324813+0000) 2023-08-28T02:10:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3613465231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3613465231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:51.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3613465231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:51.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:51.325140+0000) 2023-08-28T02:10:51.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:51.490Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:51.741 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:51.491Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:51 smithi136 ceph-mon[113531]: pgmap v9518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:51 smithi093 ceph-mon[111335]: pgmap v9518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:51 smithi093 ceph-mon[118902]: pgmap v9518: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:52.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:52.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:52.325427+0000) 2023-08-28T02:10:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1416324398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1416324398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:52.921Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:10:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:10:52.922Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:10:53.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1416324398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:53.324+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:53.325689+0000) 2023-08-28T02:10:54.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:54 smithi136 ceph-mon[113531]: pgmap v9519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:54.324 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:54 smithi093 ceph-mon[111335]: pgmap v9519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:54 smithi093 ceph-mon[118902]: pgmap v9519: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:10:54.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:54.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:54.325946+0000) 2023-08-28T02:10:55.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:55.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:55.326233+0000) 2023-08-28T02:10:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:10:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:10:55] "GET /metrics HTTP/1.1" 200 239935 "" "Prometheus/2.43.0" 2023-08-28T02:10:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:56 smithi136 ceph-mon[113531]: pgmap v9520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1831583806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:56.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:56.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:56.326505+0000) 2023-08-28T02:10:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:56 smithi093 ceph-mon[118902]: pgmap v9520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1831583806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:56 smithi093 ceph-mon[111335]: pgmap v9520: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1831583806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:57.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:57.325+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:57.326762+0000) 2023-08-28T02:10:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:57 smithi093 ceph-mon[111335]: pgmap v9521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:57 smithi093 ceph-mon[118902]: pgmap v9521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:57 smithi136 ceph-mon[113531]: pgmap v9521: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:58.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:58.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:58.327002+0000) 2023-08-28T02:10:58.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1046523045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:58.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1046523045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:59.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1046523045' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:10:59.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:10:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:10:59.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:10:59.327221+0000) 2023-08-28T02:10:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:59 smithi093 ceph-mon[111335]: pgmap v9522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:59.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:10:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:10:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:59 smithi093 ceph-mon[118902]: pgmap v9522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:10:59.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:10:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:00.166 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:59 smithi136 ceph-mon[113531]: pgmap v9522: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:00.166 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:10:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:11:00.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:00.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:00.327405+0000) 2023-08-28T02:11:00.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3618448356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:00.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3618448356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:00.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3618448356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:01.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:01.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:01.327567+0000) 2023-08-28T02:11:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:01.490Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:01.491Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:02.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:02 smithi136 ceph-mon[113531]: pgmap v9523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:02.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:02 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[111335]: pgmap v9523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[118902]: pgmap v9523: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:02.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:02.326+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:02.327797+0000) 2023-08-28T02:11:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:02.921Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:02.922Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:03.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1761070968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1761070968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:03.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:11:03.326 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:11:03.327 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:11:03.327 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1761070968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:03.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:03.328007+0000) 2023-08-28T02:11:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:04 smithi136 ceph-mon[113531]: pgmap v9524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:04.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:04.328316+0000) 2023-08-28T02:11:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:04 smithi093 ceph-mon[111335]: pgmap v9524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:04 smithi093 ceph-mon[118902]: pgmap v9524: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3761664356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:05.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3761664356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:05.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3761664356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:05.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:05.327+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:05.328569+0000) 2023-08-28T02:11:05.438 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 02:11:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T02:11:05.317239091Z level=info msg="Completed cleanup jobs" duration=12.721463ms 2023-08-28T02:11:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:11:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:11:05] "GET /metrics HTTP/1.1" 200 239935 "" "Prometheus/2.43.0" 2023-08-28T02:11:06.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:06 smithi136 ceph-mon[113531]: pgmap v9525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:06.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:06.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:06.328800+0000) 2023-08-28T02:11:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:06 smithi093 ceph-mon[111335]: pgmap v9525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:06 smithi093 ceph-mon[118902]: pgmap v9525: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:07.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:07.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:07.329080+0000) 2023-08-28T02:11:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:07 smithi093 ceph-mon[111335]: pgmap v9526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3505626632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:07 smithi093 ceph-mon[118902]: pgmap v9526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3505626632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:07 smithi136 ceph-mon[113531]: pgmap v9526: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3505626632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:08.328+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:08.329304+0000) 2023-08-28T02:11:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:09.328 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:09.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:09.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:09.329560+0000) 2023-08-28T02:11:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:09 smithi093 ceph-mon[111335]: pgmap v9527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3849841174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:09 smithi093 ceph-mon[118902]: pgmap v9527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3849841174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:10.328 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:09 smithi136 ceph-mon[113531]: pgmap v9527: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:10.328 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3849841174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:10.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:10.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:10.329803+0000) 2023-08-28T02:11:11.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:11.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:11.330070+0000) 2023-08-28T02:11:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:11.491Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:11.990 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:11.492Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:12 smithi136 ceph-mon[113531]: pgmap v9528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/658397211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:12.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:12.329+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:12.330319+0000) 2023-08-28T02:11:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:12 smithi093 ceph-mon[111335]: pgmap v9528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/658397211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:12 smithi093 ceph-mon[118902]: pgmap v9528: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/658397211' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:12.923Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:12.923Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:13.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:13.330565+0000) 2023-08-28T02:11:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:14 smithi136 ceph-mon[113531]: pgmap v9529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:14.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:14.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:14.330795+0000) 2023-08-28T02:11:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:14 smithi093 ceph-mon[111335]: pgmap v9529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:14 smithi093 ceph-mon[118902]: pgmap v9529: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:15.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/532534239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:15.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:15.330+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:15.331074+0000) 2023-08-28T02:11:15.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/532534239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:15.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:11:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:11:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:11:15.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/532534239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:16 smithi136 ceph-mon[113531]: pgmap v9530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:16.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:16.331349+0000) 2023-08-28T02:11:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:16 smithi093 ceph-mon[111335]: pgmap v9530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:16 smithi093 ceph-mon[118902]: pgmap v9530: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:17.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:17.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:17.331608+0000) 2023-08-28T02:11:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/261739343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:17 smithi093 ceph-mon[111335]: pgmap v9531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/261739343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:17 smithi093 ceph-mon[118902]: pgmap v9531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/261739343' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:17 smithi136 ceph-mon[113531]: pgmap v9531: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:18.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:18.331877+0000) 2023-08-28T02:11:19.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:19.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:19.332153+0000) 2023-08-28T02:11:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:20 smithi136 ceph-mon[113531]: pgmap v9532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3389696251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:20.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:20.332353+0000) 2023-08-28T02:11:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:20 smithi093 ceph-mon[111335]: pgmap v9532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3389696251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:20 smithi093 ceph-mon[118902]: pgmap v9532: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3389696251' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:21.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:21.332576+0000) 2023-08-28T02:11:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:21.492Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:21.493Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:22.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:22 smithi136 ceph-mon[113531]: pgmap v9533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2571123528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:22.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:22.331+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:22.332865+0000) 2023-08-28T02:11:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:22 smithi093 ceph-mon[111335]: pgmap v9533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2571123528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:22 smithi093 ceph-mon[118902]: pgmap v9533: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2571123528' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:22.924Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:22.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:23.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:23.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:23.333108+0000) 2023-08-28T02:11:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:24.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:24.333309+0000) 2023-08-28T02:11:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:24 smithi136 ceph-mon[113531]: pgmap v9534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1673322512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:24 smithi093 ceph-mon[111335]: pgmap v9534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1673322512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:24 smithi093 ceph-mon[118902]: pgmap v9534: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1673322512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:25.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:25.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:25.333517+0000) 2023-08-28T02:11:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:11:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:11:25] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:11:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:26 smithi136 ceph-mon[113531]: pgmap v9535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:26.332+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:26.333807+0000) 2023-08-28T02:11:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:26 smithi093 ceph-mon[111335]: pgmap v9535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:26 smithi093 ceph-mon[118902]: pgmap v9535: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:27.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1031002259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:27.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:27.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:27.334104+0000) 2023-08-28T02:11:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1031002259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1031002259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:28 smithi136 ceph-mon[113531]: pgmap v9536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:28.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:28.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:28.334350+0000) 2023-08-28T02:11:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:28 smithi093 ceph-mon[111335]: pgmap v9536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:28.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:28 smithi093 ceph-mon[118902]: pgmap v9536: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/742170933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:29.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:29.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:29.334602+0000) 2023-08-28T02:11:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/742170933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/742170933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:30.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:30 smithi136 ceph-mon[113531]: pgmap v9537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:30.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:30.333+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:30.334850+0000) 2023-08-28T02:11:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:30 smithi093 ceph-mon[111335]: pgmap v9537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:30 smithi093 ceph-mon[118902]: pgmap v9537: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:31.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1822726268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:31.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:31.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:31.335133+0000) 2023-08-28T02:11:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1822726268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:31.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1822726268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:31.493Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:31.494Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:32.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:32 smithi136 ceph-mon[113531]: pgmap v9538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:32.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:32.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:32.335362+0000) 2023-08-28T02:11:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:32 smithi093 ceph-mon[111335]: pgmap v9538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:32 smithi093 ceph-mon[118902]: pgmap v9538: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:32.924Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:32.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:33.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:33.335627+0000) 2023-08-28T02:11:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:34 smithi136 ceph-mon[113531]: pgmap v9539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4071159194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:34.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:34.334+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:34.335831+0000) 2023-08-28T02:11:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:34 smithi093 ceph-mon[111335]: pgmap v9539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4071159194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:34 smithi093 ceph-mon[118902]: pgmap v9539: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4071159194' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:35.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:35.336132+0000) 2023-08-28T02:11:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:11:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:11:35] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:11:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:36 smithi136 ceph-mon[113531]: pgmap v9540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1306360205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:36.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:36.336337+0000) 2023-08-28T02:11:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:36 smithi093 ceph-mon[111335]: pgmap v9540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1306360205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:36 smithi093 ceph-mon[118902]: pgmap v9540: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1306360205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:37.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:37.335+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:37.336582+0000) 2023-08-28T02:11:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:37 smithi093 ceph-mon[111335]: pgmap v9541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:37 smithi093 ceph-mon[118902]: pgmap v9541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:37 smithi136 ceph-mon[113531]: pgmap v9541: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:38.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:38.336852+0000) 2023-08-28T02:11:38.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3142801281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3142801281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3142801281' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:39.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:39.337167+0000) 2023-08-28T02:11:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:39.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:39 smithi093 ceph-mon[111335]: pgmap v9542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:39.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:39 smithi093 ceph-mon[118902]: pgmap v9542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:39 smithi136 ceph-mon[113531]: pgmap v9542: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:40.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:40.337443+0000) 2023-08-28T02:11:40.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1617001054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:40.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1617001054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1617001054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:41.336+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:41.337675+0000) 2023-08-28T02:11:41.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:41 smithi093 ceph-mon[118902]: pgmap v9543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:41.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:41 smithi093 ceph-mon[111335]: pgmap v9543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:41.494Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:41.495Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:41 smithi136 ceph-mon[113531]: pgmap v9543: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:42.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:42.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:42.337901+0000) 2023-08-28T02:11:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/605235182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/605235182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:42.924Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:42.925Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:43.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/605235182' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:43.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:43.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:43.338172+0000) 2023-08-28T02:11:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:44 smithi136 ceph-mon[113531]: pgmap v9544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:44.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:44.338411+0000) 2023-08-28T02:11:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:44 smithi093 ceph-mon[111335]: pgmap v9544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:44 smithi093 ceph-mon[118902]: pgmap v9544: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:45.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:45.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:45.338693+0000) 2023-08-28T02:11:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:11:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:11:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:11:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:46 smithi136 ceph-mon[113531]: pgmap v9545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2653099531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:46.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:46.337+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:46.338944+0000) 2023-08-28T02:11:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:46 smithi093 ceph-mon[118902]: pgmap v9545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2653099531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:46 smithi093 ceph-mon[111335]: pgmap v9545: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2653099531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:47.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:47.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:47.339204+0000) 2023-08-28T02:11:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:47 smithi093 ceph-mon[111335]: pgmap v9546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:47 smithi093 ceph-mon[118902]: pgmap v9546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:47 smithi136 ceph-mon[113531]: pgmap v9546: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:48.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:48.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:48.339408+0000) 2023-08-28T02:11:48.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/113191953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:48.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/113191953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:49.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/113191953' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:49.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:49.339665+0000) 2023-08-28T02:11:49.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:49 smithi093 ceph-mon[111335]: pgmap v9547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:49.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:49 smithi093 ceph-mon[118902]: pgmap v9547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:50.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:49 smithi136 ceph-mon[113531]: pgmap v9547: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:50.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:50.338+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:50.339922+0000) 2023-08-28T02:11:50.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3886963441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:50.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3886963441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:51.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3886963441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:51.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:51.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:51.340146+0000) 2023-08-28T02:11:51.745 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:51.494Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:51.745 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:51.495Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:51 smithi136 ceph-mon[113531]: pgmap v9548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:51 smithi093 ceph-mon[111335]: pgmap v9548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:51 smithi093 ceph-mon[118902]: pgmap v9548: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:52.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:52.340383+0000) 2023-08-28T02:11:53.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1320076447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1320076447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1320076447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:53.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:52.925Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:53.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:11:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:11:52.926Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:11:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:53.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:53.340625+0000) 2023-08-28T02:11:54.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:53 smithi136 ceph-mon[113531]: pgmap v9549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:53 smithi093 ceph-mon[111335]: pgmap v9549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:53 smithi093 ceph-mon[118902]: pgmap v9549: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:54.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:54.339+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:54.340882+0000) 2023-08-28T02:11:55.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:11:55.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:55.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:55.341084+0000) 2023-08-28T02:11:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:11:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:11:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:11:56.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1705740215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:55 smithi136 ceph-mon[113531]: pgmap v9550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1705740215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:55 smithi093 ceph-mon[111335]: pgmap v9550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1705740215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:55 smithi093 ceph-mon[118902]: pgmap v9550: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:56.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:56.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:56.341268+0000) 2023-08-28T02:11:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:57.340+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:57.341466+0000) 2023-08-28T02:11:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:57 smithi093 ceph-mon[118902]: pgmap v9551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:57.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:11:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/995934027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:57 smithi093 ceph-mon[111335]: pgmap v9551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:57.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:11:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/995934027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:57 smithi136 ceph-mon[113531]: pgmap v9551: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:11:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:11:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/995934027' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:11:58.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:58.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:58.341744+0000) 2023-08-28T02:11:59.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:11:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:11:59.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:11:59.341996+0000) 2023-08-28T02:12:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:00 smithi136 ceph-mon[113531]: pgmap v9552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1739085641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:00.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:00.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:00.342245+0000) 2023-08-28T02:12:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:00 smithi093 ceph-mon[111335]: pgmap v9552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1739085641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:00 smithi093 ceph-mon[118902]: pgmap v9552: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1739085641' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:01.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:01.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:01.342525+0000) 2023-08-28T02:12:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:01.495Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:01.496Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:02.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:02 smithi136 ceph-mon[113531]: pgmap v9553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1064844692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:02.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:02.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:02.342809+0000) 2023-08-28T02:12:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:02 smithi093 ceph-mon[111335]: pgmap v9553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1064844692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:02 smithi093 ceph-mon[118902]: pgmap v9553: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1064844692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:03 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:03 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:02.926Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:02.927Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:03.342 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:03 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:12:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:03.341+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:03.343003+0000) 2023-08-28T02:12:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:04 smithi093 ceph-mon[111335]: pgmap v9554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:04 smithi093 ceph-mon[118902]: pgmap v9554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:04.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:04 smithi136 ceph-mon[113531]: pgmap v9554: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:04.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:04.343135+0000) 2023-08-28T02:12:05.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:12:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:12:05.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2302310948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[111335]: pgmap v9555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2302310948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:05 smithi093 ceph-mon[118902]: pgmap v9555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:05.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:05.342+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:05.343273+0000) 2023-08-28T02:12:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2302310948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:05 smithi136 ceph-mon[113531]: pgmap v9555: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:12:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:12:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:12:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:06.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:06.343494+0000) 2023-08-28T02:12:07.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:07.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:07.343724+0000) 2023-08-28T02:12:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/399230283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:07.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:07 smithi093 ceph-mon[118902]: pgmap v9556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:07.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/399230283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:07.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:07 smithi093 ceph-mon[111335]: pgmap v9556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/399230283' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:07 smithi136 ceph-mon[113531]: pgmap v9556: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:08.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:08.343972+0000) 2023-08-28T02:12:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:09.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:09.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:09.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:09.344225+0000) 2023-08-28T02:12:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:09 smithi093 ceph-mon[111335]: pgmap v9557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2218455842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:09 smithi093 ceph-mon[118902]: pgmap v9557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2218455842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:10.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:09 smithi136 ceph-mon[113531]: pgmap v9557: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:10.343 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2218455842' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:10.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:10.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:10.344442+0000) 2023-08-28T02:12:11.709 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:11.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:11.344736+0000) 2023-08-28T02:12:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:11.495Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:11.496Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:12.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:12 smithi136 ceph-mon[113531]: pgmap v9558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2574598603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:12.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:12.343+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:12.344992+0000) 2023-08-28T02:12:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:12 smithi093 ceph-mon[111335]: pgmap v9558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2574598603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:12 smithi093 ceph-mon[118902]: pgmap v9558: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2574598603' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:12.926Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:12.927Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:13.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:13.345223+0000) 2023-08-28T02:12:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:14 smithi136 ceph-mon[113531]: pgmap v9559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1513943621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:14.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:14.345525+0000) 2023-08-28T02:12:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:14 smithi093 ceph-mon[111335]: pgmap v9559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1513943621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:14 smithi093 ceph-mon[118902]: pgmap v9559: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1513943621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:15.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:15.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:15.345786+0000) 2023-08-28T02:12:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:12:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:12:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:12:16.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:16 smithi136 ceph-mon[113531]: pgmap v9560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:16.344+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:16.345997+0000) 2023-08-28T02:12:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:16 smithi093 ceph-mon[111335]: pgmap v9560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:16 smithi093 ceph-mon[118902]: pgmap v9560: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:17.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2942658044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:17.686 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:17.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:17.346260+0000) 2023-08-28T02:12:17.887 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2942658044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:17.888 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2942658044' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:18.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:18 smithi136 ceph-mon[113531]: pgmap v9561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:18.664 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:18.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:18.346529+0000) 2023-08-28T02:12:18.664 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:18 smithi093 ceph-mon[111335]: pgmap v9561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:18.665 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:18 smithi093 ceph-mon[118902]: pgmap v9561: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:19.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/568259968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:19.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:19.345+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:19.346781+0000) 2023-08-28T02:12:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/568259968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/568259968' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:20 smithi136 ceph-mon[113531]: pgmap v9562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:20.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:20.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:20.347058+0000) 2023-08-28T02:12:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:20 smithi093 ceph-mon[111335]: pgmap v9562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:20 smithi093 ceph-mon[118902]: pgmap v9562: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:21.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:21.347212+0000) 2023-08-28T02:12:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:21.496Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:21.497Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:22 smithi136 ceph-mon[113531]: pgmap v9563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1679254434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:22.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:22.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:22.347450+0000) 2023-08-28T02:12:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:22 smithi093 ceph-mon[118902]: pgmap v9563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1679254434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:22 smithi093 ceph-mon[111335]: pgmap v9563: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1679254434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:22.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:22.928Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:23.346+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:23.347738+0000) 2023-08-28T02:12:24.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:24.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:24.348003+0000) 2023-08-28T02:12:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:24 smithi136 ceph-mon[113531]: pgmap v9564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/313256741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:24 smithi093 ceph-mon[111335]: pgmap v9564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/313256741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:24 smithi093 ceph-mon[118902]: pgmap v9564: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/313256741' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:25.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:25.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:25.348292+0000) 2023-08-28T02:12:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:12:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:12:25] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:12:26.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:26 smithi136 ceph-mon[113531]: pgmap v9565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3347060082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:26.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:26.347+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:26.348508+0000) 2023-08-28T02:12:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:26 smithi093 ceph-mon[111335]: pgmap v9565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3347060082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:26 smithi093 ceph-mon[118902]: pgmap v9565: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3347060082' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:27.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:27.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:27.348788+0000) 2023-08-28T02:12:27.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:27 smithi093 ceph-mon[111335]: pgmap v9566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:27.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:27 smithi093 ceph-mon[118902]: pgmap v9566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:27 smithi136 ceph-mon[113531]: pgmap v9566: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:28.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:28.349056+0000) 2023-08-28T02:12:28.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/444907645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:28.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/444907645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/444907645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:29.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:29.349259+0000) 2023-08-28T02:12:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:29 smithi136 ceph-mon[113531]: pgmap v9567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:29 smithi093 ceph-mon[111335]: pgmap v9567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:29 smithi093 ceph-mon[118902]: pgmap v9567: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:30.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:30.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:30.349411+0000) 2023-08-28T02:12:31.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3266035071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:31.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3266035071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3266035071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:31.348+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:31.349667+0000) 2023-08-28T02:12:31.750 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:31.497Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:31.750 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:31.498Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:31 smithi136 ceph-mon[113531]: pgmap v9568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:31 smithi093 ceph-mon[111335]: pgmap v9568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:31 smithi093 ceph-mon[118902]: pgmap v9568: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:32.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:32.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:32.349903+0000) 2023-08-28T02:12:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2016233645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2016233645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:32.927Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:32.928Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:33.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2016233645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:33.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:33.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:33.350166+0000) 2023-08-28T02:12:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:34 smithi136 ceph-mon[113531]: pgmap v9569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:34.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:34.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:34.350393+0000) 2023-08-28T02:12:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:34 smithi093 ceph-mon[111335]: pgmap v9569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:34 smithi093 ceph-mon[118902]: pgmap v9569: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:35.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:35.349+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:35.350646+0000) 2023-08-28T02:12:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:12:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:12:35] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:12:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:36 smithi136 ceph-mon[113531]: pgmap v9570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3197563368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:36.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:36.350895+0000) 2023-08-28T02:12:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:36 smithi093 ceph-mon[111335]: pgmap v9570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3197563368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:36 smithi093 ceph-mon[118902]: pgmap v9570: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3197563368' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:37.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:37.351149+0000) 2023-08-28T02:12:37.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:37 smithi093 ceph-mon[111335]: pgmap v9571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:37.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:37 smithi093 ceph-mon[118902]: pgmap v9571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:37 smithi136 ceph-mon[113531]: pgmap v9571: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:38.350+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:38.351414+0000) 2023-08-28T02:12:38.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/545379793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:38.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/545379793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:39.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/545379793' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:39.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:39.351684+0000) 2023-08-28T02:12:40.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:39 smithi136 ceph-mon[113531]: pgmap v9572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:39 smithi093 ceph-mon[118902]: pgmap v9572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:39 smithi093 ceph-mon[111335]: pgmap v9572: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:40.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:40.351931+0000) 2023-08-28T02:12:41.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3216891205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3216891205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3216891205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:41.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:41.352170+0000) 2023-08-28T02:12:41.747 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:41.498Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:41.747 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:41.499Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:42.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:41 smithi136 ceph-mon[113531]: pgmap v9573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:41 smithi093 ceph-mon[111335]: pgmap v9573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:41 smithi093 ceph-mon[118902]: pgmap v9573: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:42.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:42.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:42.352566+0000) 2023-08-28T02:12:43.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4104728956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4104728956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4104728956' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:42.928Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:42.929Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:43.351+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:43.352838+0000) 2023-08-28T02:12:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:43 smithi136 ceph-mon[113531]: pgmap v9574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:44.439 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:43 smithi093 ceph-mon[111335]: pgmap v9574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:44.439 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:43 smithi093 ceph-mon[118902]: pgmap v9574: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:44.352+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:44.353104+0000) 2023-08-28T02:12:45.351 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/693319209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/693319209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:45.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/693319209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:45.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:45.353382+0000) 2023-08-28T02:12:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:12:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:12:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:12:46.352 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:46 smithi136 ceph-mon[113531]: pgmap v9575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:46 smithi093 ceph-mon[111335]: pgmap v9575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:46 smithi093 ceph-mon[118902]: pgmap v9575: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:46.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:46.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:46.353636+0000) 2023-08-28T02:12:47.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:47.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:47.353893+0000) 2023-08-28T02:12:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:47 smithi093 ceph-mon[111335]: pgmap v9576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:47.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/114270285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:47 smithi093 ceph-mon[118902]: pgmap v9576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:47.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/114270285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:47 smithi136 ceph-mon[113531]: pgmap v9576: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/114270285' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:48.353+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:48.354160+0000) 2023-08-28T02:12:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:49.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:49.354415+0000) 2023-08-28T02:12:50.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:50.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:50.354633+0000) 2023-08-28T02:12:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:50 smithi136 ceph-mon[113531]: pgmap v9577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2356564851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:50 smithi093 ceph-mon[111335]: pgmap v9577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2356564851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:50 smithi093 ceph-mon[118902]: pgmap v9577: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2356564851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:51.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:51.354835+0000) 2023-08-28T02:12:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:51.498Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:51.499Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:52 smithi136 ceph-mon[113531]: pgmap v9578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:52.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:52.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:52.355090+0000) 2023-08-28T02:12:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:52 smithi093 ceph-mon[111335]: pgmap v9578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:52.705 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:52 smithi093 ceph-mon[118902]: pgmap v9578: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3370207222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3370207222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:52.929Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:12:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:12:52.929Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:12:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3370207222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:53.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:53.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:53.355352+0000) 2023-08-28T02:12:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:54 smithi136 ceph-mon[113531]: pgmap v9579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:54.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:54.355614+0000) 2023-08-28T02:12:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:54 smithi093 ceph-mon[111335]: pgmap v9579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:54 smithi093 ceph-mon[118902]: pgmap v9579: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:12:55.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1488808746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:55.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:55.354+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:55.355879+0000) 2023-08-28T02:12:55.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1488808746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:55.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:12:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:12:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:12:55.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1488808746' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:56.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:56 smithi136 ceph-mon[113531]: pgmap v9580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:56.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:56.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:56.356166+0000) 2023-08-28T02:12:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:56 smithi093 ceph-mon[111335]: pgmap v9580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:56 smithi093 ceph-mon[118902]: pgmap v9580: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:57.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:57.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:57.356426+0000) 2023-08-28T02:12:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/715020645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:12:57 smithi136 ceph-mon[113531]: pgmap v9581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/715020645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:12:57 smithi093 ceph-mon[111335]: pgmap v9581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/715020645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:12:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:12:57 smithi093 ceph-mon[118902]: pgmap v9581: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:12:58.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:58.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:58.356745+0000) 2023-08-28T02:12:59.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:12:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:12:59.355+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:12:59.356976+0000) 2023-08-28T02:13:00.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:00 smithi136 ceph-mon[113531]: pgmap v9582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1835761021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:00.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:00.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:00.357228+0000) 2023-08-28T02:13:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:00 smithi093 ceph-mon[111335]: pgmap v9582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1835761021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:00 smithi093 ceph-mon[118902]: pgmap v9582: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1835761021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:01.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:01.357481+0000) 2023-08-28T02:13:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:01.499Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:01.500Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:02 smithi136 ceph-mon[113531]: pgmap v9583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/812774593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:02.439 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:02.356+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:02.357720+0000) 2023-08-28T02:13:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:02 smithi093 ceph-mon[111335]: pgmap v9583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/812774593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:02 smithi093 ceph-mon[118902]: pgmap v9583: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/812774593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:02.929Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:02.930Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:03.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:03.357958+0000) 2023-08-28T02:13:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:04 smithi136 ceph-mon[113531]: pgmap v9584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4240897958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:04.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:04.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:04.358154+0000) 2023-08-28T02:13:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:04 smithi093 ceph-mon[111335]: pgmap v9584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4240897958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:04 smithi093 ceph-mon[118902]: pgmap v9584: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4240897958' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:05.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:05.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:05.358365+0000) 2023-08-28T02:13:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:13:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:13:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:13:06.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:06 smithi136 ceph-mon[113531]: pgmap v9585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:06.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:06.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:06.337 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:06 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[111335]: pgmap v9585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:06.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[118902]: pgmap v9585: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:13:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:06.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:06 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:06.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:06.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:06.358571+0000) 2023-08-28T02:13:07.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2339752139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.426 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:07.357+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:07.358737+0000) 2023-08-28T02:13:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2339752139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2339752139' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:08.437 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:08.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:08.358967+0000) 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: pgmap v9586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: pgmap v9586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: pgmap v9586: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:13:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:13:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2150426548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:09.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:09.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:09.359221+0000) 2023-08-28T02:13:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2150426548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2150426548' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:10 smithi136 ceph-mon[113531]: pgmap v9587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:10.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:10.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:10.359405+0000) 2023-08-28T02:13:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:10 smithi093 ceph-mon[111335]: pgmap v9587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:10.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:10 smithi093 ceph-mon[118902]: pgmap v9587: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:11.358+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:11.359660+0000) 2023-08-28T02:13:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:11.500Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:11.501Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:12 smithi136 ceph-mon[113531]: pgmap v9588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2172218681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:12.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:12.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:12.359884+0000) 2023-08-28T02:13:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:12 smithi093 ceph-mon[111335]: pgmap v9588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2172218681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:12 smithi093 ceph-mon[118902]: pgmap v9588: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2172218681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:12.930Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:12.931Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:13.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:13.360156+0000) 2023-08-28T02:13:14.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:14 smithi136 ceph-mon[113531]: pgmap v9589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/85058218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:14.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:14.360439+0000) 2023-08-28T02:13:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:14 smithi093 ceph-mon[111335]: pgmap v9589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/85058218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:14 smithi093 ceph-mon[118902]: pgmap v9589: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/85058218' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:15.359+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:15.360660+0000) 2023-08-28T02:13:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:13:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:13:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:13:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:16 smithi093 ceph-mon[111335]: pgmap v9590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/202235944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:16 smithi093 ceph-mon[118902]: pgmap v9590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/202235944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:16 smithi136 ceph-mon[113531]: pgmap v9590: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/202235944' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:16.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:16.360893+0000) 2023-08-28T02:13:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:17.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:17.361158+0000) 2023-08-28T02:13:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:17 smithi136 ceph-mon[113531]: pgmap v9591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:17 smithi093 ceph-mon[111335]: pgmap v9591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:17 smithi093 ceph-mon[118902]: pgmap v9591: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:18.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:18.361423+0000) 2023-08-28T02:13:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1695890894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1695890894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1695890894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:19.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:19.361678+0000) 2023-08-28T02:13:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:19 smithi136 ceph-mon[113531]: pgmap v9592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:19 smithi093 ceph-mon[111335]: pgmap v9592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:19 smithi093 ceph-mon[118902]: pgmap v9592: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:20.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:20.360+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:20.361915+0000) 2023-08-28T02:13:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3819624164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:21.242 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3819624164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:21.285 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3819624164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:21.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:21.362176+0000) 2023-08-28T02:13:21.824 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:21.501Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:21.825 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:21.502Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:21 smithi136 ceph-mon[113531]: pgmap v9593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:21 smithi093 ceph-mon[111335]: pgmap v9593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:21 smithi093 ceph-mon[118902]: pgmap v9593: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:22.361+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:22.362412+0000) 2023-08-28T02:13:23.198 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:22.930Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:23.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:22.931Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:23 smithi093 ceph-mon[111335]: pgmap v9594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2384452989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:23 smithi093 ceph-mon[118902]: pgmap v9594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2384452989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:23 smithi136 ceph-mon[113531]: pgmap v9594: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2384452989' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:23.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:23.362671+0000) 2023-08-28T02:13:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:24.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:24.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:24.362937+0000) 2023-08-28T02:13:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:25 smithi093 ceph-mon[111335]: pgmap v9595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:25.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:25 smithi093 ceph-mon[118902]: pgmap v9595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:13:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:13:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:13:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:25 smithi136 ceph-mon[113531]: pgmap v9595: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:25.362+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:25.363156+0000) 2023-08-28T02:13:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/23446157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/23446157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:26.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/23446157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:26.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:26.363417+0000) 2023-08-28T02:13:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:27.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:27.363667+0000) 2023-08-28T02:13:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:27 smithi136 ceph-mon[113531]: pgmap v9596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:27 smithi093 ceph-mon[111335]: pgmap v9596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:27 smithi093 ceph-mon[118902]: pgmap v9596: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:28.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:28.363937+0000) 2023-08-28T02:13:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3536781522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3536781522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:29.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3536781522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:29.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:29.364167+0000) 2023-08-28T02:13:30.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:29 smithi136 ceph-mon[113531]: pgmap v9597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:29 smithi093 ceph-mon[111335]: pgmap v9597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:29 smithi093 ceph-mon[118902]: pgmap v9597: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:30.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:30.363+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:30.364413+0000) 2023-08-28T02:13:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1107862832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:31.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1107862832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1107862832' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:31.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:31.364637+0000) 2023-08-28T02:13:31.755 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:31.502Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:31.755 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:31.503Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:32.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:31 smithi136 ceph-mon[113531]: pgmap v9598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:31 smithi093 ceph-mon[111335]: pgmap v9598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:31 smithi093 ceph-mon[118902]: pgmap v9598: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:32.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:32.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:32.364856+0000) 2023-08-28T02:13:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/155662007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/155662007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/155662007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:32.930Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:32.932Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:33.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:33.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:33.365127+0000) 2023-08-28T02:13:34.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:33 smithi136 ceph-mon[113531]: pgmap v9599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:33 smithi093 ceph-mon[118902]: pgmap v9599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:33 smithi093 ceph-mon[111335]: pgmap v9599: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:34.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:34.365371+0000) 2023-08-28T02:13:35.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/610193172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:35.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:35.364+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:35.365540+0000) 2023-08-28T02:13:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/610193172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:35.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:13:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:13:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:13:35.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/610193172' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:36.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:36 smithi136 ceph-mon[113531]: pgmap v9600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:36.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:36.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:36.365797+0000) 2023-08-28T02:13:36.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:36 smithi093 ceph-mon[111335]: pgmap v9600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:36 smithi093 ceph-mon[118902]: pgmap v9600: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:37.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:37.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:37.366052+0000) 2023-08-28T02:13:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:37 smithi136 ceph-mon[113531]: pgmap v9601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3957005900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:37 smithi093 ceph-mon[111335]: pgmap v9601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3957005900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:37 smithi093 ceph-mon[118902]: pgmap v9601: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3957005900' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:38.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:38.366329+0000) 2023-08-28T02:13:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:39.365 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:39.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:39.366592+0000) 2023-08-28T02:13:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:39 smithi093 ceph-mon[111335]: pgmap v9602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2387891613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:39 smithi093 ceph-mon[118902]: pgmap v9602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2387891613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:40.366 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:39 smithi136 ceph-mon[113531]: pgmap v9602: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:40.366 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2387891613' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:40.365+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:40.366850+0000) 2023-08-28T02:13:41.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:41.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:41.367139+0000) 2023-08-28T02:13:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:41.503Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:41.504Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:42.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:42 smithi136 ceph-mon[113531]: pgmap v9603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:42.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:42.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:42.367434+0000) 2023-08-28T02:13:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:42 smithi093 ceph-mon[111335]: pgmap v9603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:42 smithi093 ceph-mon[118902]: pgmap v9603: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1716198565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1716198565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:42.931Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:42.933Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1716198565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:43.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:43.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:43.367675+0000) 2023-08-28T02:13:44.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:44 smithi136 ceph-mon[113531]: pgmap v9604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:44.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:44.366+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:44.367893+0000) 2023-08-28T02:13:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:44 smithi093 ceph-mon[111335]: pgmap v9604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:44 smithi093 ceph-mon[118902]: pgmap v9604: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:45.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/259800961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:45.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:13:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:13:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:13:45.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/259800961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/259800961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:45.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:45.368116+0000) 2023-08-28T02:13:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:46 smithi093 ceph-mon[111335]: pgmap v9605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:46 smithi093 ceph-mon[118902]: pgmap v9605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:46 smithi136 ceph-mon[113531]: pgmap v9605: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:46.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:46.368371+0000) 2023-08-28T02:13:47.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:47.367+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:47.368626+0000) 2023-08-28T02:13:48.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1195411954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:47 smithi136 ceph-mon[113531]: pgmap v9606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1195411954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:47 smithi093 ceph-mon[111335]: pgmap v9606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1195411954' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:47 smithi093 ceph-mon[118902]: pgmap v9606: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:48.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:48.368898+0000) 2023-08-28T02:13:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:49.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:49.369151+0000) 2023-08-28T02:13:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:50 smithi136 ceph-mon[113531]: pgmap v9607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/712768699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:50.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:50.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:50.369397+0000) 2023-08-28T02:13:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:50 smithi093 ceph-mon[111335]: pgmap v9607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/712768699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:50 smithi093 ceph-mon[118902]: pgmap v9607: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/712768699' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:51.368+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:51.369638+0000) 2023-08-28T02:13:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:51.504Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:51.505Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:52 smithi136 ceph-mon[113531]: pgmap v9608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1615540498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:52.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:52.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:52.369894+0000) 2023-08-28T02:13:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:52 smithi093 ceph-mon[111335]: pgmap v9608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1615540498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:52 smithi093 ceph-mon[118902]: pgmap v9608: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1615540498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:52.932Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:13:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:13:52.933Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:13:53.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:53.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:53.370193+0000) 2023-08-28T02:13:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:54 smithi136 ceph-mon[113531]: pgmap v9609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/847154093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:54.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:54.370444+0000) 2023-08-28T02:13:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:54 smithi093 ceph-mon[118902]: pgmap v9609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/847154093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:54 smithi093 ceph-mon[111335]: pgmap v9609: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:13:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/847154093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:55.369+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:55.370693+0000) 2023-08-28T02:13:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:13:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:13:55] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:13:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:56 smithi136 ceph-mon[113531]: pgmap v9610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:56.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:56.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:56.370915+0000) 2023-08-28T02:13:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:56 smithi093 ceph-mon[111335]: pgmap v9610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:56.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:56 smithi093 ceph-mon[118902]: pgmap v9610: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:57.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/991242261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:57.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:57.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:57.371200+0000) 2023-08-28T02:13:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/991242261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/991242261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:58.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:58 smithi093 ceph-mon[111335]: pgmap v9611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:58 smithi093 ceph-mon[118902]: pgmap v9611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:58 smithi136 ceph-mon[113531]: pgmap v9611: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:13:58.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:58.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:58.371416+0000) 2023-08-28T02:13:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:13:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/240747069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:13:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/240747069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:13:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/240747069' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:13:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:13:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:13:59.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:13:59.371681+0000) 2023-08-28T02:14:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:00 smithi093 ceph-mon[111335]: pgmap v9612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:00 smithi093 ceph-mon[118902]: pgmap v9612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:00.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:00 smithi136 ceph-mon[113531]: pgmap v9612: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:00.370+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:00.371941+0000) 2023-08-28T02:14:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:01.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:01.372165+0000) 2023-08-28T02:14:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:01.505Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:01.506Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:02 smithi093 ceph-mon[111335]: pgmap v9613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/103559120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:02 smithi093 ceph-mon[118902]: pgmap v9613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/103559120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:02.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:02 smithi136 ceph-mon[113531]: pgmap v9613: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/103559120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:02.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:02.372413+0000) 2023-08-28T02:14:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:02.932Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:02.933Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:03.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:03.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:03.372696+0000) 2023-08-28T02:14:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:04 smithi093 ceph-mon[111335]: pgmap v9614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1095137698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:04 smithi093 ceph-mon[118902]: pgmap v9614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1095137698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:04 smithi136 ceph-mon[113531]: pgmap v9614: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1095137698' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:04.371+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:04.372952+0000) 2023-08-28T02:14:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:05 smithi136 ceph-mon[113531]: pgmap v9615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:05.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:05.373114+0000) 2023-08-28T02:14:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:05 smithi093 ceph-mon[111335]: pgmap v9615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:14:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:14:05] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:14:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:05 smithi093 ceph-mon[118902]: pgmap v9615: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1886706448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:06.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:06.373347+0000) 2023-08-28T02:14:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1886706448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1886706448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:07.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:07.373621+0000) 2023-08-28T02:14:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:07 smithi136 ceph-mon[113531]: pgmap v9616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:07 smithi093 ceph-mon[111335]: pgmap v9616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:07 smithi093 ceph-mon[118902]: pgmap v9616: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:08.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:08.373830+0000) 2023-08-28T02:14:09.096 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:09.097 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2647867853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:09.097 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:09.097 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2647867853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:14:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2647867853' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:09.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:09.372+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:09.374002+0000) 2023-08-28T02:14:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:09.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:09 smithi093 ceph-mon[118902]: pgmap v9617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:09.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:09 smithi093 ceph-mon[111335]: pgmap v9617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:09 smithi136 ceph-mon[113531]: pgmap v9617: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:10.584 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:10.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:10.374183+0000) 2023-08-28T02:14:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3538051717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3538051717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3538051717' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:11.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:11.373+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:11.374353+0000) 2023-08-28T02:14:11.942 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:11.505Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:11.942 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:11.506Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[111335]: pgmap v9618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:12.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[118902]: pgmap v9618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:12.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:12.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:12.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:12.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:11 smithi136 ceph-mon[113531]: pgmap v9618: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:12.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:14:12.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:14:12.373 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:14:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:12.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:12.374582+0000) 2023-08-28T02:14:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:12.933Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:12.934Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:13.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:13.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:13.374845+0000) 2023-08-28T02:14:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:14 smithi136 ceph-mon[113531]: pgmap v9619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2304941398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:14.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:14.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:14.375115+0000) 2023-08-28T02:14:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:14 smithi093 ceph-mon[111335]: pgmap v9619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2304941398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:14 smithi093 ceph-mon[118902]: pgmap v9619: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2304941398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:15.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:15.375390+0000) 2023-08-28T02:14:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:14:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:14:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:14:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:16 smithi136 ceph-mon[113531]: pgmap v9620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3526445575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:16.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:16.374+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:16.375590+0000) 2023-08-28T02:14:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:16 smithi093 ceph-mon[111335]: pgmap v9620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3526445575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:16 smithi093 ceph-mon[118902]: pgmap v9620: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3526445575' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:17.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:17.375840+0000) 2023-08-28T02:14:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:17 smithi136 ceph-mon[113531]: pgmap v9621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:17 smithi093 ceph-mon[111335]: pgmap v9621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:17 smithi093 ceph-mon[118902]: pgmap v9621: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:18.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:18.376134+0000) 2023-08-28T02:14:19.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2774773652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2774773652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2774773652' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:19.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:19.376405+0000) 2023-08-28T02:14:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:19 smithi136 ceph-mon[113531]: pgmap v9622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:19 smithi093 ceph-mon[111335]: pgmap v9622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:19 smithi093 ceph-mon[118902]: pgmap v9622: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:20.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:20.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:20.376683+0000) 2023-08-28T02:14:21.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2291111552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2291111552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2291111552' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:21.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:21.375+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:21.376942+0000) 2023-08-28T02:14:21.768 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:21.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:21.769 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:21.507Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:22.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:21 smithi136 ceph-mon[113531]: pgmap v9623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:21 smithi093 ceph-mon[111335]: pgmap v9623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:21 smithi093 ceph-mon[118902]: pgmap v9623: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:22.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:22.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:22.377185+0000) 2023-08-28T02:14:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2579742338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2579742338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2579742338' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:22.934Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:22.935Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:23.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:23.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:23.377397+0000) 2023-08-28T02:14:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:23 smithi136 ceph-mon[113531]: pgmap v9624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:23 smithi093 ceph-mon[111335]: pgmap v9624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:23 smithi093 ceph-mon[118902]: pgmap v9624: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:24.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:24.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:24.377654+0000) 2023-08-28T02:14:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:25.376+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:25.377941+0000) 2023-08-28T02:14:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:14:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:14:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:14:26.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3258095780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:25 smithi136 ceph-mon[113531]: pgmap v9625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3258095780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:25 smithi093 ceph-mon[111335]: pgmap v9625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3258095780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:25 smithi093 ceph-mon[118902]: pgmap v9625: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:26.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:26.378163+0000) 2023-08-28T02:14:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:27.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:27.378420+0000) 2023-08-28T02:14:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:27 smithi136 ceph-mon[113531]: pgmap v9626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3564521736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:27 smithi093 ceph-mon[111335]: pgmap v9626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3564521736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:27 smithi093 ceph-mon[118902]: pgmap v9626: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3564521736' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:28.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:28.378679+0000) 2023-08-28T02:14:29.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:29.377+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:29.378915+0000) 2023-08-28T02:14:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:30 smithi136 ceph-mon[113531]: pgmap v9627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3782757228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:30.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:30.379202+0000) 2023-08-28T02:14:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:30 smithi093 ceph-mon[111335]: pgmap v9627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3782757228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:30 smithi093 ceph-mon[118902]: pgmap v9627: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3782757228' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:31.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:31.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:31.379451+0000) 2023-08-28T02:14:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:31.507Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:31.508Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:32 smithi093 ceph-mon[111335]: pgmap v9628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:32 smithi093 ceph-mon[118902]: pgmap v9628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:32.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:32 smithi136 ceph-mon[113531]: pgmap v9628: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:32.378+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:32.379696+0000) 2023-08-28T02:14:33.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:32.935Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:33.199 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:32.936Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3459521798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:33.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3459521798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3459521798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:33.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:33.379968+0000) 2023-08-28T02:14:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:34 smithi093 ceph-mon[111335]: pgmap v9629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:34 smithi093 ceph-mon[118902]: pgmap v9629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:34.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:34 smithi136 ceph-mon[113531]: pgmap v9629: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:34.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:34.380202+0000) 2023-08-28T02:14:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3298682478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:35.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:14:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:14:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:14:35.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3298682478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3298682478' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:35.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:35.380405+0000) 2023-08-28T02:14:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:36 smithi093 ceph-mon[111335]: pgmap v9630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:36 smithi093 ceph-mon[118902]: pgmap v9630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:36 smithi136 ceph-mon[113531]: pgmap v9630: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:36.379+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:36.380651+0000) 2023-08-28T02:14:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:37.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:37.380869+0000) 2023-08-28T02:14:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2141314845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:37 smithi136 ceph-mon[113531]: pgmap v9631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2141314845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:37 smithi093 ceph-mon[111335]: pgmap v9631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2141314845' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:37 smithi093 ceph-mon[118902]: pgmap v9631: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:38.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:38.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:38.381150+0000) 2023-08-28T02:14:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:39.380 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:39.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:39.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:39.381354+0000) 2023-08-28T02:14:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:39 smithi093 ceph-mon[111335]: pgmap v9632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/181820000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:39 smithi093 ceph-mon[118902]: pgmap v9632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/181820000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:40.380 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:39 smithi136 ceph-mon[113531]: pgmap v9632: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:40.380 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/181820000' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:40.380+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:40.381576+0000) 2023-08-28T02:14:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:41.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:41.381837+0000) 2023-08-28T02:14:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:41.508Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:41.509Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:42 smithi093 ceph-mon[111335]: pgmap v9633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2596642193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:42 smithi093 ceph-mon[118902]: pgmap v9633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2596642193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:42 smithi136 ceph-mon[113531]: pgmap v9633: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2596642193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:42.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:42.382102+0000) 2023-08-28T02:14:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:42.936Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:42.937Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:43.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:43.382345+0000) 2023-08-28T02:14:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:44 smithi093 ceph-mon[111335]: pgmap v9634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1123911946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:44 smithi093 ceph-mon[118902]: pgmap v9634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1123911946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:44.381+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:44.382592+0000) 2023-08-28T02:14:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:44 smithi136 ceph-mon[113531]: pgmap v9634: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1123911946' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:45.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:45.382879+0000) 2023-08-28T02:14:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:14:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:14:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:14:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:46 smithi093 ceph-mon[111335]: pgmap v9635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:46 smithi093 ceph-mon[118902]: pgmap v9635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:46 smithi136 ceph-mon[113531]: pgmap v9635: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:46.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:46.383146+0000) 2023-08-28T02:14:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2952783028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2952783028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2952783028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:47.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:47.383404+0000) 2023-08-28T02:14:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:48 smithi093 ceph-mon[111335]: pgmap v9636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:48 smithi093 ceph-mon[118902]: pgmap v9636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:48 smithi136 ceph-mon[113531]: pgmap v9636: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:48.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:48.383699+0000) 2023-08-28T02:14:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3491190604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3491190604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3491190604' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:49.382+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:49.383944+0000) 2023-08-28T02:14:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:50 smithi093 ceph-mon[111335]: pgmap v9637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:50 smithi093 ceph-mon[118902]: pgmap v9637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:50 smithi136 ceph-mon[113531]: pgmap v9637: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:50.383+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:50.384215+0000) 2023-08-28T02:14:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:51.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:51.384445+0000) 2023-08-28T02:14:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:51.509Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:51.510Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:52 smithi093 ceph-mon[111335]: pgmap v9638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/769958198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:52 smithi093 ceph-mon[118902]: pgmap v9638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/769958198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:52.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:52 smithi136 ceph-mon[113531]: pgmap v9638: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:52.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/769958198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:52.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:52.384639+0000) 2023-08-28T02:14:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:52.936Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:14:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:14:52.937Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:14:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:53.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:53.384859+0000) 2023-08-28T02:14:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:54 smithi093 ceph-mon[111335]: pgmap v9639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3714065102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:54 smithi093 ceph-mon[118902]: pgmap v9639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3714065102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:54 smithi136 ceph-mon[113531]: pgmap v9639: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3714065102' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:14:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:54.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:54.385126+0000) 2023-08-28T02:14:55.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:55 smithi093 ceph-mon[111335]: pgmap v9640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:55.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:55 smithi093 ceph-mon[118902]: pgmap v9640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:55.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:14:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:14:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:14:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:55 smithi136 ceph-mon[113531]: pgmap v9640: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:55.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:55.385405+0000) 2023-08-28T02:14:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3280588321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3280588321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3280588321' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:56.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:56.385693+0000) 2023-08-28T02:14:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:57.384+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:57.385969+0000) 2023-08-28T02:14:58.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:57 smithi136 ceph-mon[113531]: pgmap v9641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:57 smithi093 ceph-mon[111335]: pgmap v9641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:57 smithi093 ceph-mon[118902]: pgmap v9641: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:14:58.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:58.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:58.386226+0000) 2023-08-28T02:14:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1806120250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1806120250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1806120250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:14:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:14:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:14:59.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:14:59.386464+0000) 2023-08-28T02:15:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:14:59 smithi136 ceph-mon[113531]: pgmap v9642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:14:59 smithi093 ceph-mon[111335]: pgmap v9642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:14:59 smithi093 ceph-mon[118902]: pgmap v9642: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:00.385+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:00.386727+0000) 2023-08-28T02:15:01.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1848780200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1848780200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1848780200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:01.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:01.386983+0000) 2023-08-28T02:15:01.902 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:01.510Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:01.902 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:01.510Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:01 smithi136 ceph-mon[113531]: pgmap v9643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:01 smithi093 ceph-mon[111335]: pgmap v9643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:01 smithi093 ceph-mon[118902]: pgmap v9643: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:02.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:02.387223+0000) 2023-08-28T02:15:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:02.937Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:02.938Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:03.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:03.387496+0000) 2023-08-28T02:15:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:04 smithi093 ceph-mon[111335]: pgmap v9644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:04.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2497599455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:04 smithi093 ceph-mon[118902]: pgmap v9644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:04.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2497599455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:04 smithi136 ceph-mon[113531]: pgmap v9644: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2497599455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:04.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:04.387746+0000) 2023-08-28T02:15:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:05.386+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:05.387992+0000) 2023-08-28T02:15:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:15:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:15:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:15:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:06 smithi093 ceph-mon[111335]: pgmap v9645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3163311032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:06 smithi093 ceph-mon[118902]: pgmap v9645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3163311032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:06 smithi136 ceph-mon[113531]: pgmap v9645: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3163311032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:06.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:06.388247+0000) 2023-08-28T02:15:07.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:07.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:07.388458+0000) 2023-08-28T02:15:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:07 smithi136 ceph-mon[113531]: pgmap v9646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:07 smithi093 ceph-mon[111335]: pgmap v9646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:07 smithi093 ceph-mon[118902]: pgmap v9646: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:08.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:08.388717+0000) 2023-08-28T02:15:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1875525117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1875525117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1875525117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:09.387+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:09.388922+0000) 2023-08-28T02:15:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:09 smithi136 ceph-mon[113531]: pgmap v9647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:09 smithi093 ceph-mon[111335]: pgmap v9647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:09 smithi093 ceph-mon[118902]: pgmap v9647: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:10.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:10.389140+0000) 2023-08-28T02:15:11.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2519219125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2519219125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2519219125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:11.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:11.389318+0000) 2023-08-28T02:15:11.773 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:11.510Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:11.773 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:11.511Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:12.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:11 smithi136 ceph-mon[113531]: pgmap v9648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:12.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:11 smithi093 ceph-mon[118902]: pgmap v9648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:11 smithi093 ceph-mon[111335]: pgmap v9648: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:15:12.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:12.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:12.389502+0000) 2023-08-28T02:15:13.165 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2720513660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2720513660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2720513660' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:12.938Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:12.939Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:13.438 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:13.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:13.389659+0000) 2023-08-28T02:15:14.388 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.388 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: pgmap v9649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:14.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:14.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:14.389 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: pgmap v9649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: pgmap v9649: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:15:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:15:14.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:15:14.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:14.388+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:14.389916+0000) 2023-08-28T02:15:15.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3904723868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:15.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3904723868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:15:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:15:15] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:15:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3904723868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:15.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:15.390168+0000) 2023-08-28T02:15:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:16 smithi093 ceph-mon[111335]: pgmap v9650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:16 smithi093 ceph-mon[118902]: pgmap v9650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:16 smithi136 ceph-mon[113531]: pgmap v9650: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:16.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:16.390426+0000) 2023-08-28T02:15:17.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:17.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:17.390681+0000) 2023-08-28T02:15:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:17 smithi136 ceph-mon[113531]: pgmap v9651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2834076346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:17 smithi093 ceph-mon[111335]: pgmap v9651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2834076346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:17 smithi093 ceph-mon[118902]: pgmap v9651: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2834076346' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:18.389+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:18.390958+0000) 2023-08-28T02:15:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:19.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:19.391227+0000) 2023-08-28T02:15:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:20 smithi093 ceph-mon[111335]: pgmap v9652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/270671431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:20 smithi093 ceph-mon[118902]: pgmap v9652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/270671431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:20 smithi136 ceph-mon[113531]: pgmap v9652: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/270671431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:20.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:20.391468+0000) 2023-08-28T02:15:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:21.390+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:21.391668+0000) 2023-08-28T02:15:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:21.512Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:21.513Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:22 smithi093 ceph-mon[111335]: pgmap v9653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:22 smithi093 ceph-mon[118902]: pgmap v9653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:22.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:22 smithi136 ceph-mon[113531]: pgmap v9653: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:22.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:22.391876+0000) 2023-08-28T02:15:23.211 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:22.939Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:23.212 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:22.940Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3637580041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3637580041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3637580041' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:23.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:23.392112+0000) 2023-08-28T02:15:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:24 smithi093 ceph-mon[111335]: pgmap v9654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:24 smithi093 ceph-mon[118902]: pgmap v9654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:24 smithi136 ceph-mon[113531]: pgmap v9654: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:24.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:24.392383+0000) 2023-08-28T02:15:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3598104680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:25 smithi093 ceph-mon[111335]: pgmap v9655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:25.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:15:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:15:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:15:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3598104680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:25 smithi093 ceph-mon[118902]: pgmap v9655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3598104680' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:25 smithi136 ceph-mon[113531]: pgmap v9655: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:25.391+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:25.392658+0000) 2023-08-28T02:15:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:26.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:26.392925+0000) 2023-08-28T02:15:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:27.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:27.393183+0000) 2023-08-28T02:15:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3710122164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:27 smithi136 ceph-mon[113531]: pgmap v9656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3710122164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:27 smithi093 ceph-mon[111335]: pgmap v9656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3710122164' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:27 smithi093 ceph-mon[118902]: pgmap v9656: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:28.392+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:28.393451+0000) 2023-08-28T02:15:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:29.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:29.393689+0000) 2023-08-28T02:15:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:30 smithi093 ceph-mon[111335]: pgmap v9657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1177290206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:30 smithi093 ceph-mon[118902]: pgmap v9657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1177290206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:30 smithi136 ceph-mon[113531]: pgmap v9657: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1177290206' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:30.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:30.393908+0000) 2023-08-28T02:15:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:31.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:31.394147+0000) 2023-08-28T02:15:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:31.512Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:31.513Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:32 smithi093 ceph-mon[111335]: pgmap v9658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4043294497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:32 smithi093 ceph-mon[118902]: pgmap v9658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4043294497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:32 smithi136 ceph-mon[113531]: pgmap v9658: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4043294497' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:32.393+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:32.394392+0000) 2023-08-28T02:15:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:32.940Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:32.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:33.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:33.394645+0000) 2023-08-28T02:15:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:34 smithi093 ceph-mon[111335]: pgmap v9659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:34 smithi093 ceph-mon[118902]: pgmap v9659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:34 smithi136 ceph-mon[113531]: pgmap v9659: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:34.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:34.394889+0000) 2023-08-28T02:15:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3967649514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3967649514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:15:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:15:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:15:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3967649514' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:35.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:35.395163+0000) 2023-08-28T02:15:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:36 smithi093 ceph-mon[111335]: pgmap v9660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:36 smithi093 ceph-mon[118902]: pgmap v9660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:36 smithi136 ceph-mon[113531]: pgmap v9660: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:36.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:36.395384+0000) 2023-08-28T02:15:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/77311215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:37 smithi093 ceph-mon[111335]: pgmap v9661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/77311215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:37 smithi093 ceph-mon[118902]: pgmap v9661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/77311215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:37 smithi136 ceph-mon[113531]: pgmap v9661: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:37.394+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:37.395576+0000) 2023-08-28T02:15:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:38.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:38.395765+0000) 2023-08-28T02:15:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:39.394 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:39.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:39.396017+0000) 2023-08-28T02:15:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3455284250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:39 smithi093 ceph-mon[111335]: pgmap v9662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3455284250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:39 smithi093 ceph-mon[118902]: pgmap v9662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:40.395 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3455284250' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:40.395 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:39 smithi136 ceph-mon[113531]: pgmap v9662: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:40.395+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:40.396303+0000) 2023-08-28T02:15:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:41.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:41.396548+0000) 2023-08-28T02:15:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:41.513Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:41.514Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:42 smithi093 ceph-mon[111335]: pgmap v9663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1784424291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:42 smithi093 ceph-mon[118902]: pgmap v9663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1784424291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:42 smithi136 ceph-mon[113531]: pgmap v9663: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1784424291' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:42.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:42.396796+0000) 2023-08-28T02:15:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:42.941Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:42.941Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:43.396+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:43.397078+0000) 2023-08-28T02:15:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:44 smithi093 ceph-mon[111335]: pgmap v9664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2447954747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:44 smithi093 ceph-mon[118902]: pgmap v9664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2447954747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:44 smithi136 ceph-mon[113531]: pgmap v9664: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2447954747' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:44.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:44.397335+0000) 2023-08-28T02:15:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:45.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:45.397623+0000) 2023-08-28T02:15:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:15:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:15:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:46 smithi093 ceph-mon[111335]: pgmap v9665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2235288059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:46 smithi093 ceph-mon[118902]: pgmap v9665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2235288059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:46 smithi136 ceph-mon[113531]: pgmap v9665: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2235288059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:46.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:46.397871+0000) 2023-08-28T02:15:47.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:47.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:47.398119+0000) 2023-08-28T02:15:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:47 smithi136 ceph-mon[113531]: pgmap v9666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:47 smithi093 ceph-mon[111335]: pgmap v9666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:47 smithi093 ceph-mon[118902]: pgmap v9666: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:48.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:48.398370+0000) 2023-08-28T02:15:49.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2179507723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2179507723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2179507723' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:49.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:49.398617+0000) 2023-08-28T02:15:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:49 smithi136 ceph-mon[113531]: pgmap v9667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:49 smithi093 ceph-mon[118902]: pgmap v9667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:49 smithi093 ceph-mon[111335]: pgmap v9667: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:50.397+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:50.398864+0000) 2023-08-28T02:15:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2142416174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2142416174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:51.397 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2142416174' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:51.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:51.399135+0000) 2023-08-28T02:15:51.953 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:51.514Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:51.953 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:51.515Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:51 smithi093 ceph-mon[111335]: pgmap v9668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:51 smithi093 ceph-mon[118902]: pgmap v9668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:52.398 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:51 smithi136 ceph-mon[113531]: pgmap v9668: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:52.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:52.399410+0000) 2023-08-28T02:15:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:52.941Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:15:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:15:52.943Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:15:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:53.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:53.399660+0000) 2023-08-28T02:15:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:54 smithi093 ceph-mon[111335]: pgmap v9669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3041099998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:54 smithi093 ceph-mon[118902]: pgmap v9669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3041099998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:54 smithi136 ceph-mon[113531]: pgmap v9669: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3041099998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:15:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:54.398+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:54.399943+0000) 2023-08-28T02:15:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:55.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:55.400145+0000) 2023-08-28T02:15:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:15:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:15:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:15:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:56 smithi093 ceph-mon[111335]: pgmap v9670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3708346557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:56 smithi093 ceph-mon[118902]: pgmap v9670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:56.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3708346557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:56 smithi136 ceph-mon[113531]: pgmap v9670: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3708346557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:56.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:56.400382+0000) 2023-08-28T02:15:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:57.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:57.400603+0000) 2023-08-28T02:15:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:57 smithi136 ceph-mon[113531]: pgmap v9671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:57 smithi093 ceph-mon[111335]: pgmap v9671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:57 smithi093 ceph-mon[118902]: pgmap v9671: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:15:58.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:58.399+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:58.400889+0000) 2023-08-28T02:15:59.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2490436752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2490436752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:59.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2490436752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:15:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:15:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:15:59.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:15:59.401185+0000) 2023-08-28T02:16:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:15:59 smithi136 ceph-mon[113531]: pgmap v9672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:15:59 smithi093 ceph-mon[111335]: pgmap v9672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:15:59 smithi093 ceph-mon[118902]: pgmap v9672: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:00.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:00.401439+0000) 2023-08-28T02:16:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3253922100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3253922100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3253922100' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:01.400+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:01.401711+0000) 2023-08-28T02:16:01.785 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:01.515Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:01.786 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:01.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:02.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:01 smithi136 ceph-mon[113531]: pgmap v9673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:01 smithi093 ceph-mon[111335]: pgmap v9673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:01 smithi093 ceph-mon[118902]: pgmap v9673: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:02.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:02.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:02.401935+0000) 2023-08-28T02:16:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2663020738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2663020738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2663020738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:02.943Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:02.943Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:03.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:03.402198+0000) 2023-08-28T02:16:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:03 smithi136 ceph-mon[113531]: pgmap v9674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:03 smithi093 ceph-mon[111335]: pgmap v9674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:03 smithi093 ceph-mon[118902]: pgmap v9674: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:04.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:04.402442+0000) 2023-08-28T02:16:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:05 smithi136 ceph-mon[113531]: pgmap v9675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/822942412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:05.401+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:05.402654+0000) 2023-08-28T02:16:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:16:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:16:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:16:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:05 smithi093 ceph-mon[118902]: pgmap v9675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/822942412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:05 smithi093 ceph-mon[111335]: pgmap v9675: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/822942412' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:06.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:06.402900+0000) 2023-08-28T02:16:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:07.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:07.403152+0000) 2023-08-28T02:16:08.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:07 smithi136 ceph-mon[113531]: pgmap v9676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3212343215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:07 smithi093 ceph-mon[111335]: pgmap v9676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3212343215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:07 smithi093 ceph-mon[118902]: pgmap v9676: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3212343215' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:08.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:08.403378+0000) 2023-08-28T02:16:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:09.402 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:09.402+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:09.403619+0000) 2023-08-28T02:16:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:09 smithi093 ceph-mon[111335]: pgmap v9677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:10.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1609218424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:09 smithi093 ceph-mon[118902]: pgmap v9677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1609218424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:10.403 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:09 smithi136 ceph-mon[113531]: pgmap v9677: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:10.403 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1609218424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:10.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:10.403847+0000) 2023-08-28T02:16:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:11.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:11.404088+0000) 2023-08-28T02:16:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:11.515Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:11.516Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:12 smithi093 ceph-mon[111335]: pgmap v9678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:12 smithi093 ceph-mon[118902]: pgmap v9678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:12 smithi136 ceph-mon[113531]: pgmap v9678: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:12.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:12.404322+0000) 2023-08-28T02:16:13.217 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:12.944Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:13.217 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:12.945Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/100785720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/100785720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/100785720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:13.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:13.404568+0000) 2023-08-28T02:16:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:14 smithi093 ceph-mon[111335]: pgmap v9679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:14 smithi093 ceph-mon[118902]: pgmap v9679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:14.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:14 smithi136 ceph-mon[113531]: pgmap v9679: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:16:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:14.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:14.404742+0000) 2023-08-28T02:16:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/127055669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/127055669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:15.403 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/127055669' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:15.404 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:15.403+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:15.404918+0000) 2023-08-28T02:16:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:16:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:16:15] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:16:16.404 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:16 smithi136 ceph-mon[113531]: pgmap v9680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:16.404 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:16.404 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:16 smithi093 ceph-mon[111335]: pgmap v9680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:16 smithi093 ceph-mon[118902]: pgmap v9680: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:16.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:16.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:16.405150+0000) 2023-08-28T02:16:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:17.404+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:17.405358+0000) 2023-08-28T02:16:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4100245428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:17.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:17 smithi136 ceph-mon[113531]: pgmap v9681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4100245428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[111335]: pgmap v9681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:16:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:16:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:16:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4100245428' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:17.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:17 smithi093 ceph-mon[118902]: pgmap v9681: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:18.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:18.405609+0000) 2023-08-28T02:16:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:19.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:19.405860+0000) 2023-08-28T02:16:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:20 smithi093 ceph-mon[111335]: pgmap v9682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2661499976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:20 smithi093 ceph-mon[118902]: pgmap v9682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2661499976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:20 smithi136 ceph-mon[113531]: pgmap v9682: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2661499976' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:20.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:20.406143+0000) 2023-08-28T02:16:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:21.405+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:21.406362+0000) 2023-08-28T02:16:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:21.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:21.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:22 smithi093 ceph-mon[111335]: pgmap v9683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3182147530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:22 smithi093 ceph-mon[118902]: pgmap v9683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3182147530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:22 smithi136 ceph-mon[113531]: pgmap v9683: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:22.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3182147530' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:22.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:22.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:22.406609+0000) 2023-08-28T02:16:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:22.945Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:22.946Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:23.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:23.406807+0000) 2023-08-28T02:16:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:24 smithi093 ceph-mon[111335]: pgmap v9684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:24 smithi093 ceph-mon[118902]: pgmap v9684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:24 smithi136 ceph-mon[113531]: pgmap v9684: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:24.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:24.407096+0000) 2023-08-28T02:16:25.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4124461912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4124461912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:16:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:16:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:16:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4124461912' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:25.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:25.407422+0000) 2023-08-28T02:16:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:26 smithi093 ceph-mon[111335]: pgmap v9685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:26 smithi093 ceph-mon[118902]: pgmap v9685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:26 smithi136 ceph-mon[113531]: pgmap v9685: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:26.406+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:26.407669+0000) 2023-08-28T02:16:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2673571988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:27 smithi093 ceph-mon[111335]: pgmap v9686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2673571988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:27 smithi093 ceph-mon[118902]: pgmap v9686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2673571988' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:27 smithi136 ceph-mon[113531]: pgmap v9686: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:27.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:27.407889+0000) 2023-08-28T02:16:28.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:28.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:28.408137+0000) 2023-08-28T02:16:29.407 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3702251125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3702251125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3702251125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:29.407+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:29.408403+0000) 2023-08-28T02:16:30.407 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:30 smithi136 ceph-mon[113531]: pgmap v9687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:30 smithi093 ceph-mon[111335]: pgmap v9687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:30 smithi093 ceph-mon[118902]: pgmap v9687: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:30.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:30.408655+0000) 2023-08-28T02:16:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:31.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:31.408894+0000) 2023-08-28T02:16:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:31.516Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:31.517Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:32 smithi093 ceph-mon[111335]: pgmap v9688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1425467153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:32 smithi093 ceph-mon[118902]: pgmap v9688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:32.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1425467153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:32 smithi136 ceph-mon[113531]: pgmap v9688: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1425467153' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:32.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:32.409174+0000) 2023-08-28T02:16:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:32.946Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:32.947Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:33.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:33.409451+0000) 2023-08-28T02:16:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:34 smithi093 ceph-mon[111335]: pgmap v9689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/174149112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:34 smithi093 ceph-mon[118902]: pgmap v9689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/174149112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:34 smithi136 ceph-mon[113531]: pgmap v9689: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/174149112' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:34.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:34.409664+0000) 2023-08-28T02:16:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:35.408+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:35.409936+0000) 2023-08-28T02:16:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:16:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:16:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:16:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:36 smithi093 ceph-mon[111335]: pgmap v9690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3068419216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:36 smithi093 ceph-mon[118902]: pgmap v9690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:36.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3068419216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:36 smithi136 ceph-mon[113531]: pgmap v9690: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3068419216' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:36.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:36.410189+0000) 2023-08-28T02:16:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:37.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:37.410423+0000) 2023-08-28T02:16:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:37 smithi136 ceph-mon[113531]: pgmap v9691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:37 smithi093 ceph-mon[111335]: pgmap v9691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:37 smithi093 ceph-mon[118902]: pgmap v9691: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:38.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:38.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:38.410701+0000) 2023-08-28T02:16:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2100344971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2100344971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2100344971' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:39.409+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:39.410984+0000) 2023-08-28T02:16:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:39 smithi136 ceph-mon[113531]: pgmap v9692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:39 smithi093 ceph-mon[111335]: pgmap v9692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:39 smithi093 ceph-mon[118902]: pgmap v9692: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:40.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:40.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:40.411241+0000) 2023-08-28T02:16:41.410 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3778802443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:41.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3778802443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3778802443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:41.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:41.411473+0000) 2023-08-28T02:16:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:41.517Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 9 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:41.518Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:42.410 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:42 smithi136 ceph-mon[113531]: pgmap v9693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:42 smithi093 ceph-mon[111335]: pgmap v9693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:42 smithi093 ceph-mon[118902]: pgmap v9693: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:42.410+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:42.411729+0000) 2023-08-28T02:16:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:42.946Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:42.947Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:43.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:43.411990+0000) 2023-08-28T02:16:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:44 smithi093 ceph-mon[111335]: pgmap v9694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2038690198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:44 smithi093 ceph-mon[118902]: pgmap v9694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2038690198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:44 smithi136 ceph-mon[113531]: pgmap v9694: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2038690198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:44.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:44.412210+0000) 2023-08-28T02:16:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:45.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:45.412438+0000) 2023-08-28T02:16:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:16:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:16:45] "GET /metrics HTTP/1.1" 200 239939 "" "Prometheus/2.43.0" 2023-08-28T02:16:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:46 smithi093 ceph-mon[111335]: pgmap v9695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1245342666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:46 smithi093 ceph-mon[118902]: pgmap v9695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1245342666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:46 smithi136 ceph-mon[113531]: pgmap v9695: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1245342666' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:46.411+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:46.412716+0000) 2023-08-28T02:16:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:47.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:47.412936+0000) 2023-08-28T02:16:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:47 smithi136 ceph-mon[113531]: pgmap v9696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:47 smithi093 ceph-mon[111335]: pgmap v9696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:47 smithi093 ceph-mon[118902]: pgmap v9696: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:48.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:48.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:48.413202+0000) 2023-08-28T02:16:49.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3435706134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3435706134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3435706134' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:49.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:49.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:49.413372+0000) 2023-08-28T02:16:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:49 smithi136 ceph-mon[113531]: pgmap v9697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:49 smithi093 ceph-mon[111335]: pgmap v9697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:49 smithi093 ceph-mon[118902]: pgmap v9697: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:50.412+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:50.413625+0000) 2023-08-28T02:16:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2904787964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2904787964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2904787964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:51.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:51.413899+0000) 2023-08-28T02:16:51.798 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:51.518Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:51.798 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:51.519Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:52.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:51 smithi136 ceph-mon[113531]: pgmap v9698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:51 smithi093 ceph-mon[111335]: pgmap v9698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:51 smithi093 ceph-mon[118902]: pgmap v9698: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:52.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:52.414149+0000) 2023-08-28T02:16:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3300607322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3300607322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3300607322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:52.946Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:16:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:16:52.948Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:16:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:53.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:53.414357+0000) 2023-08-28T02:16:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:53 smithi136 ceph-mon[113531]: pgmap v9699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:53 smithi093 ceph-mon[111335]: pgmap v9699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:53 smithi093 ceph-mon[118902]: pgmap v9699: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:54.413+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:54.414612+0000) 2023-08-28T02:16:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:16:55.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:55.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:55.414903+0000) 2023-08-28T02:16:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:16:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:16:55] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:16:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:55 smithi093 ceph-mon[111335]: pgmap v9700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4254940570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:55 smithi093 ceph-mon[118902]: pgmap v9700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4254940570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:56.414 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:55 smithi136 ceph-mon[113531]: pgmap v9700: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:56.414 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4254940570' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:56.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:56.415188+0000) 2023-08-28T02:16:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:57.414+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:57.415446+0000) 2023-08-28T02:16:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:57 smithi136 ceph-mon[113531]: pgmap v9701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:16:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1321300057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:57 smithi093 ceph-mon[111335]: pgmap v9701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:58.280 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:16:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1321300057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:58.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:57 smithi093 ceph-mon[118902]: pgmap v9701: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:16:58.280 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:16:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1321300057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:16:58.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:58.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:58.415707+0000) 2023-08-28T02:16:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:16:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:16:59.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:16:59.415969+0000) 2023-08-28T02:17:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:00 smithi093 ceph-mon[111335]: pgmap v9702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/765249340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:00 smithi093 ceph-mon[118902]: pgmap v9702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:00.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/765249340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:00 smithi136 ceph-mon[113531]: pgmap v9702: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/765249340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:00.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:00.416222+0000) 2023-08-28T02:17:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:01.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:01.416441+0000) 2023-08-28T02:17:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:01.519Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:01.520Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:02 smithi093 ceph-mon[111335]: pgmap v9703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:02 smithi093 ceph-mon[118902]: pgmap v9703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:02 smithi136 ceph-mon[113531]: pgmap v9703: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:02.415+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:02.416630+0000) 2023-08-28T02:17:03.233 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:02.947Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:03.234 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:02.948Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2683511621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:03 smithi093 ceph-mon[111335]: pgmap v9704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2683511621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:03 smithi093 ceph-mon[118902]: pgmap v9704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:03.687 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2683511621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:03 smithi136 ceph-mon[113531]: pgmap v9704: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:03.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:03.416849+0000) 2023-08-28T02:17:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:04.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:04.417109+0000) 2023-08-28T02:17:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3060875307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3060875307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3060875307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:05.416+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:05.417379+0000) 2023-08-28T02:17:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:17:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:17:05] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:17:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:05 smithi136 ceph-mon[113531]: pgmap v9705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:05 smithi093 ceph-mon[111335]: pgmap v9705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:05 smithi093 ceph-mon[118902]: pgmap v9705: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:06.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:06.417640+0000) 2023-08-28T02:17:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:07.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:07.417909+0000) 2023-08-28T02:17:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1928925294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:07 smithi136 ceph-mon[113531]: pgmap v9706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1928925294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:07 smithi093 ceph-mon[111335]: pgmap v9706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1928925294' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:07 smithi093 ceph-mon[118902]: pgmap v9706: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:08.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:08.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:08.418165+0000) 2023-08-28T02:17:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:09.417 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:09.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:09.418421+0000) 2023-08-28T02:17:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:09 smithi093 ceph-mon[111335]: pgmap v9707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1912396939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:09 smithi093 ceph-mon[118902]: pgmap v9707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1912396939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:10.417 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:09 smithi136 ceph-mon[113531]: pgmap v9707: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:10.417 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1912396939' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:10.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:10.418669+0000) 2023-08-28T02:17:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:11.417+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:11.418941+0000) 2023-08-28T02:17:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:11.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:11.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:12 smithi093 ceph-mon[111335]: pgmap v9708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2773395481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:12 smithi093 ceph-mon[118902]: pgmap v9708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2773395481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:12 smithi136 ceph-mon[113531]: pgmap v9708: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2773395481' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:12.418+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:12.419208+0000) 2023-08-28T02:17:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:12.948Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:12.949Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:13 smithi136 ceph-mon[113531]: pgmap v9709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:13.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:13.419472+0000) 2023-08-28T02:17:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:13 smithi093 ceph-mon[111335]: pgmap v9709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:13 smithi093 ceph-mon[118902]: pgmap v9709: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2015147358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:14.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:14.419752+0000) 2023-08-28T02:17:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2015147358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2015147358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:15 smithi136 ceph-mon[113531]: pgmap v9710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:15.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:15.420000+0000) 2023-08-28T02:17:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:15 smithi093 ceph-mon[111335]: pgmap v9710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:17:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:17:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:17:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:15 smithi093 ceph-mon[118902]: pgmap v9710: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:16.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:16.420222+0000) 2023-08-28T02:17:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2119812908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:17.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2119812908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:17.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2119812908' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:17.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:17.420450+0000) 2023-08-28T02:17:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:17.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:17 smithi093 ceph-mon[118902]: pgmap v9711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:17 smithi093 ceph-mon[111335]: pgmap v9711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:18.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:17:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:17 smithi136 ceph-mon[113531]: pgmap v9711: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:18.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:18.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:18.420616+0000) 2023-08-28T02:17:19.687 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:19.419+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:19.420821+0000) 2023-08-28T02:17:19.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4082205129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[118902]: pgmap v9712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:19.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:19.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4082205129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[111335]: pgmap v9712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:19.990 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4082205129' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:19 smithi136 ceph-mon[113531]: pgmap v9712: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:20.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:20.421003+0000) 2023-08-28T02:17:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:20.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:17:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:17:21.691 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:21.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:21.421270+0000) 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:21 smithi093 ceph-mon[111335]: pgmap v9713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3691467631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:21 smithi093 ceph-mon[118902]: pgmap v9713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:21.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3691467631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:21.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:21.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:21 smithi136 ceph-mon[113531]: pgmap v9713: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3691467631' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:22.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:22.421502+0000) 2023-08-28T02:17:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:22.949Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:22.950Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:23.420+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:23.421732+0000) 2023-08-28T02:17:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:24 smithi093 ceph-mon[111335]: pgmap v9714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1950383297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:24 smithi093 ceph-mon[118902]: pgmap v9714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1950383297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:24 smithi136 ceph-mon[113531]: pgmap v9714: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1950383297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:24.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:24.421976+0000) 2023-08-28T02:17:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:25 smithi093 ceph-mon[111335]: pgmap v9715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:25.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:17:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:17:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:17:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:25 smithi093 ceph-mon[118902]: pgmap v9715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:25 smithi136 ceph-mon[113531]: pgmap v9715: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:25.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:25.422248+0000) 2023-08-28T02:17:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1246976974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:26.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:26.422495+0000) 2023-08-28T02:17:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1246976974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1246976974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:27.421+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:27.422740+0000) 2023-08-28T02:17:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:27 smithi136 ceph-mon[113531]: pgmap v9716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:27 smithi093 ceph-mon[111335]: pgmap v9716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:27 smithi093 ceph-mon[118902]: pgmap v9716: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:28.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:28.422995+0000) 2023-08-28T02:17:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2100350432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2100350432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:29.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2100350432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:29.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:29.423267+0000) 2023-08-28T02:17:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:29 smithi136 ceph-mon[113531]: pgmap v9717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:29 smithi093 ceph-mon[111335]: pgmap v9717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:29 smithi093 ceph-mon[118902]: pgmap v9717: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:30.422+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:30.423542+0000) 2023-08-28T02:17:31.422 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/179782565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/179782565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:31.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/179782565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:31.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:31.423802+0000) 2023-08-28T02:17:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:31.520Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:31.521Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:32.422 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:32 smithi136 ceph-mon[113531]: pgmap v9718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:32 smithi093 ceph-mon[111335]: pgmap v9718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:32 smithi093 ceph-mon[118902]: pgmap v9718: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:32.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:32.424057+0000) 2023-08-28T02:17:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:32.950Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:32.951Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:33 smithi136 ceph-mon[113531]: pgmap v9719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:33.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:33.424278+0000) 2023-08-28T02:17:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:33 smithi093 ceph-mon[111335]: pgmap v9719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:33 smithi093 ceph-mon[118902]: pgmap v9719: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3790773565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:34.423+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:34.424521+0000) 2023-08-28T02:17:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3790773565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3790773565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:35 smithi136 ceph-mon[113531]: pgmap v9720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:35.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:35.424734+0000) 2023-08-28T02:17:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:35 smithi093 ceph-mon[111335]: pgmap v9720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:35 smithi093 ceph-mon[118902]: pgmap v9720: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:17:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:17:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:17:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2275247297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:36.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:36.424980+0000) 2023-08-28T02:17:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2275247297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2275247297' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:37.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:37.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:37.425239+0000) 2023-08-28T02:17:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:37 smithi136 ceph-mon[113531]: pgmap v9721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:37 smithi093 ceph-mon[111335]: pgmap v9721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:37 smithi093 ceph-mon[118902]: pgmap v9721: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:38.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:38.424+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:38.425509+0000) 2023-08-28T02:17:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3743649742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3743649742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3743649742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:39.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:39.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:39.425778+0000) 2023-08-28T02:17:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:39 smithi136 ceph-mon[113531]: pgmap v9722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:39 smithi093 ceph-mon[111335]: pgmap v9722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:39 smithi093 ceph-mon[118902]: pgmap v9722: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:40.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:40.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:40.426070+0000) 2023-08-28T02:17:41.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2627989705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2627989705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2627989705' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:41.425+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:41.426340+0000) 2023-08-28T02:17:41.795 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:41.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:41.796 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:41.522Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:41 smithi136 ceph-mon[113531]: pgmap v9723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:41 smithi093 ceph-mon[111335]: pgmap v9723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:41 smithi093 ceph-mon[118902]: pgmap v9723: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:42.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:42.426618+0000) 2023-08-28T02:17:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/449156663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/449156663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:42.950Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:42.951Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:43.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/449156663' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:43.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:43.426860+0000) 2023-08-28T02:17:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:43 smithi093 ceph-mon[111335]: pgmap v9724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:43 smithi093 ceph-mon[118902]: pgmap v9724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:44.425 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:43 smithi136 ceph-mon[113531]: pgmap v9724: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:44.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:44.427121+0000) 2023-08-28T02:17:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:45 smithi136 ceph-mon[113531]: pgmap v9725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:45.426+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:45.427355+0000) 2023-08-28T02:17:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:45 smithi093 ceph-mon[111335]: pgmap v9725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:45 smithi093 ceph-mon[118902]: pgmap v9725: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:17:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:17:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:17:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/188856313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:46.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:46.427624+0000) 2023-08-28T02:17:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/188856313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:46.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/188856313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:47.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:47.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:47.427793+0000) 2023-08-28T02:17:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:47 smithi136 ceph-mon[113531]: pgmap v9726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/791061118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:47 smithi093 ceph-mon[111335]: pgmap v9726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/791061118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:47 smithi093 ceph-mon[118902]: pgmap v9726: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/791061118' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:48.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:48.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:48.428072+0000) 2023-08-28T02:17:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:49 smithi136 ceph-mon[113531]: pgmap v9727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:49.427+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:49.428316+0000) 2023-08-28T02:17:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:49 smithi093 ceph-mon[111335]: pgmap v9727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:49 smithi093 ceph-mon[118902]: pgmap v9727: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1177359430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:50.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:50.428532+0000) 2023-08-28T02:17:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1177359430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1177359430' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:51 smithi136 ceph-mon[113531]: pgmap v9728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:51.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:51.428770+0000) 2023-08-28T02:17:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:51 smithi093 ceph-mon[111335]: pgmap v9728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:51 smithi093 ceph-mon[118902]: pgmap v9728: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:51.521Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:51.522Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:52.428+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:52.429065+0000) 2023-08-28T02:17:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/51334210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/51334210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/51334210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:52.951Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:17:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:17:52.952Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:17:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:53.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:53.429300+0000) 2023-08-28T02:17:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:53 smithi136 ceph-mon[113531]: pgmap v9729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:53 smithi093 ceph-mon[111335]: pgmap v9729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:53 smithi093 ceph-mon[118902]: pgmap v9729: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:54.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:54.429582+0000) 2023-08-28T02:17:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:17:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:55.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:55.429822+0000) 2023-08-28T02:17:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:17:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:17:55] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:17:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/339851179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:55 smithi136 ceph-mon[113531]: pgmap v9730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/339851179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:55 smithi093 ceph-mon[111335]: pgmap v9730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/339851179' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:55 smithi093 ceph-mon[118902]: pgmap v9730: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:56.429+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:56.430080+0000) 2023-08-28T02:17:57.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:57.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:57.430281+0000) 2023-08-28T02:17:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:57 smithi136 ceph-mon[113531]: pgmap v9731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1791273337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:57 smithi093 ceph-mon[111335]: pgmap v9731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1791273337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:57 smithi093 ceph-mon[118902]: pgmap v9731: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1791273337' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:17:58.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:58.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:58.430506+0000) 2023-08-28T02:17:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:17:59 smithi136 ceph-mon[113531]: pgmap v9732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:17:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:17:59.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:17:59.430731+0000) 2023-08-28T02:17:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:17:59 smithi093 ceph-mon[111335]: pgmap v9732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:17:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:17:59 smithi093 ceph-mon[118902]: pgmap v9732: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1241413295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:00.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:00.430975+0000) 2023-08-28T02:18:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1241413295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1241413295' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:01 smithi136 ceph-mon[113531]: pgmap v9733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:01.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:01.431226+0000) 2023-08-28T02:18:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:01 smithi093 ceph-mon[111335]: pgmap v9733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:01 smithi093 ceph-mon[118902]: pgmap v9733: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:01.522Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:01.523Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:02.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:02.431450+0000) 2023-08-28T02:18:03.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/12929247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/12929247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/12929247' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:02.952Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:02.953Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:03.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:03.431700+0000) 2023-08-28T02:18:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:03 smithi136 ceph-mon[113531]: pgmap v9734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:03 smithi093 ceph-mon[111335]: pgmap v9734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:03 smithi093 ceph-mon[118902]: pgmap v9734: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:04.430+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:04.431987+0000) 2023-08-28T02:18:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3486207653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3486207653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3486207653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:05.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:05.432232+0000) 2023-08-28T02:18:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:18:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:18:05] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:18:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:05 smithi136 ceph-mon[113531]: pgmap v9735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:05 smithi093 ceph-mon[111335]: pgmap v9735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:05 smithi093 ceph-mon[118902]: pgmap v9735: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:06.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:06.432487+0000) 2023-08-28T02:18:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2969503414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2969503414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2969503414' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:07.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:07.431+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:07.432730+0000) 2023-08-28T02:18:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:07 smithi136 ceph-mon[113531]: pgmap v9736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:07 smithi093 ceph-mon[111335]: pgmap v9736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:07 smithi093 ceph-mon[118902]: pgmap v9736: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:08.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:08.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:08.432975+0000) 2023-08-28T02:18:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:09.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:09.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:09.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:09.433192+0000) 2023-08-28T02:18:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3601083013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:09 smithi093 ceph-mon[111335]: pgmap v9737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3601083013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:09 smithi093 ceph-mon[118902]: pgmap v9737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:10.432 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3601083013' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:10.432 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:09 smithi136 ceph-mon[113531]: pgmap v9737: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:10.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:10.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:10.433429+0000) 2023-08-28T02:18:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:11 smithi136 ceph-mon[113531]: pgmap v9738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:11.432+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:11.433704+0000) 2023-08-28T02:18:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:11 smithi093 ceph-mon[111335]: pgmap v9738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:11 smithi093 ceph-mon[118902]: pgmap v9738: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:11.523Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:11.524Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3767570108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:12.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:12.433925+0000) 2023-08-28T02:18:12.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3767570108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:12.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3767570108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:12.953Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:12.953Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:13 smithi136 ceph-mon[113531]: pgmap v9739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:13.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:13.434174+0000) 2023-08-28T02:18:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:13 smithi093 ceph-mon[111335]: pgmap v9739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:13 smithi093 ceph-mon[118902]: pgmap v9739: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4265993402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:14.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:14.434366+0000) 2023-08-28T02:18:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4265993402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4265993402' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:15 smithi136 ceph-mon[113531]: pgmap v9740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:15.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:15.434588+0000) 2023-08-28T02:18:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:15 smithi093 ceph-mon[111335]: pgmap v9740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:18:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:18:15] "GET /metrics HTTP/1.1" 200 239940 "" "Prometheus/2.43.0" 2023-08-28T02:18:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:15 smithi093 ceph-mon[118902]: pgmap v9740: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4147613199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:16.433+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:16.434872+0000) 2023-08-28T02:18:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4147613199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4147613199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:17.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:17.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:17.435148+0000) 2023-08-28T02:18:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:17 smithi136 ceph-mon[113531]: pgmap v9741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:17 smithi093 ceph-mon[111335]: pgmap v9741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:17 smithi093 ceph-mon[118902]: pgmap v9741: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:18.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:18.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:18.435395+0000) 2023-08-28T02:18:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/75130961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/75130961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/75130961' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:19.434+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:19.435602+0000) 2023-08-28T02:18:20.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:19 smithi136 ceph-mon[113531]: pgmap v9742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:19 smithi093 ceph-mon[111335]: pgmap v9742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:19 smithi093 ceph-mon[118902]: pgmap v9742: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:20.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:20.435820+0000) 2023-08-28T02:18:20.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:20.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:21.077 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:18:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:21.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:21.436024+0000) 2023-08-28T02:18:21.807 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:21.524Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:21.807 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:21.525Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3549329008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:21 smithi136 ceph-mon[113531]: pgmap v9743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3549329008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:21 smithi093 ceph-mon[111335]: pgmap v9743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3549329008' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:22.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:21 smithi093 ceph-mon[118902]: pgmap v9743: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:22.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:22.436169+0000) 2023-08-28T02:18:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:22.953Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:22.954Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:23.435+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:23.436385+0000) 2023-08-28T02:18:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:23 smithi136 ceph-mon[113531]: pgmap v9744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3336010719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[111335]: pgmap v9744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3336010719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:18:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:18:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:18:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[118902]: pgmap v9744: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3336010719' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:24.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:24.436641+0000) 2023-08-28T02:18:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:25.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:25.436953+0000) 2023-08-28T02:18:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:18:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:18:25] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T02:18:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:25 smithi136 ceph-mon[113531]: pgmap v9745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:25 smithi093 ceph-mon[111335]: pgmap v9745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:26.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:25 smithi093 ceph-mon[118902]: pgmap v9745: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:26.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:26.437247+0000) 2023-08-28T02:18:27.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1277634407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1277634407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:27.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1277634407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:27.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:27.436+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:27.437496+0000) 2023-08-28T02:18:28.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:27 smithi136 ceph-mon[113531]: pgmap v9746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:27 smithi093 ceph-mon[111335]: pgmap v9746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:27 smithi093 ceph-mon[118902]: pgmap v9746: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:28.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:28.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:28.437758+0000) 2023-08-28T02:18:29.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/329951253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/329951253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/329951253' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:29.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:29.438052+0000) 2023-08-28T02:18:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:29 smithi136 ceph-mon[113531]: pgmap v9747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:29 smithi093 ceph-mon[111335]: pgmap v9747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:29 smithi093 ceph-mon[118902]: pgmap v9747: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:30.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:30.438289+0000) 2023-08-28T02:18:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/95421115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:31.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/95421115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/95421115' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:31.437+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:31.438439+0000) 2023-08-28T02:18:31.834 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:31.525Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:31.834 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:31.527Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:31 smithi136 ceph-mon[113531]: pgmap v9748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:31 smithi093 ceph-mon[111335]: pgmap v9748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:31 smithi093 ceph-mon[118902]: pgmap v9748: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:32.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:32.438698+0000) 2023-08-28T02:18:33.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3141779372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3141779372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:32.954Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:33.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:32.955Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:33.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3141779372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:33.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:33.438926+0000) 2023-08-28T02:18:34.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:34 smithi136 ceph-mon[113531]: pgmap v9749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:34 smithi093 ceph-mon[111335]: pgmap v9749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:34 smithi093 ceph-mon[118902]: pgmap v9749: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:34.438+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:34.439167+0000) 2023-08-28T02:18:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:35 smithi136 ceph-mon[113531]: pgmap v9750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:35.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:35.439394+0000) 2023-08-28T02:18:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:35 smithi093 ceph-mon[111335]: pgmap v9750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:35 smithi093 ceph-mon[118902]: pgmap v9750: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:18:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:18:35] "GET /metrics HTTP/1.1" 200 239932 "" "Prometheus/2.43.0" 2023-08-28T02:18:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3385568093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:36.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:36.439647+0000) 2023-08-28T02:18:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3385568093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3385568093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:37.793 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:37.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:37.439876+0000) 2023-08-28T02:18:38.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:37 smithi136 ceph-mon[113531]: pgmap v9751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3777348639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:37 smithi093 ceph-mon[111335]: pgmap v9751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3777348639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:37 smithi093 ceph-mon[118902]: pgmap v9751: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3777348639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:38.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:38.440123+0000) 2023-08-28T02:18:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:39.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:39.440292+0000) 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:39 smithi093 ceph-mon[111335]: pgmap v9752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:39 smithi093 ceph-mon[118902]: pgmap v9752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:39 smithi136 ceph-mon[113531]: pgmap v9752: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:40.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:40.440488+0000) 2023-08-28T02:18:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3906425493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3906425493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:41.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3906425493' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:41.439+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:41.440752+0000) 2023-08-28T02:18:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:41.526Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:41.527Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:41 smithi136 ceph-mon[113531]: pgmap v9753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:41 smithi093 ceph-mon[111335]: pgmap v9753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:42 smithi093 ceph-mon[118902]: pgmap v9753: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:42.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:42.441007+0000) 2023-08-28T02:18:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1736223744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1736223744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:42.955Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:42.955Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1736223744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:43.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:43.441173+0000) 2023-08-28T02:18:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:44 smithi136 ceph-mon[113531]: pgmap v9754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:44 smithi093 ceph-mon[111335]: pgmap v9754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:44 smithi093 ceph-mon[118902]: pgmap v9754: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:44.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:44.441328+0000) 2023-08-28T02:18:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3290916207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:45.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3290916207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3290916207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:18:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:18:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:18:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:45.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:45.441546+0000) 2023-08-28T02:18:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:46 smithi136 ceph-mon[113531]: pgmap v9755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:46 smithi093 ceph-mon[111335]: pgmap v9755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:46 smithi093 ceph-mon[118902]: pgmap v9755: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:46.440+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:46.441838+0000) 2023-08-28T02:18:47.799 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:47.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:47.442135+0000) 2023-08-28T02:18:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:47 smithi136 ceph-mon[113531]: pgmap v9756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/296936325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:47 smithi093 ceph-mon[111335]: pgmap v9756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/296936325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:47 smithi093 ceph-mon[118902]: pgmap v9756: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/296936325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:48.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:48.442413+0000) 2023-08-28T02:18:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:49 smithi136 ceph-mon[113531]: pgmap v9757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:49.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:49.442666+0000) 2023-08-28T02:18:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:49 smithi093 ceph-mon[111335]: pgmap v9757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:49 smithi093 ceph-mon[118902]: pgmap v9757: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1789487935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:50.441+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:50.442921+0000) 2023-08-28T02:18:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1789487935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:50.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1789487935' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:51 smithi136 ceph-mon[113531]: pgmap v9758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:51.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:51.443155+0000) 2023-08-28T02:18:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:51 smithi093 ceph-mon[111335]: pgmap v9758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:51 smithi093 ceph-mon[118902]: pgmap v9758: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:51.527Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:51.529Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:52.794 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:52.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:52.443400+0000) 2023-08-28T02:18:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2398060807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2398060807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2398060807' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:52.956Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:18:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:18:52.957Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:18:53.794 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:53.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:53.443638+0000) 2023-08-28T02:18:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:53 smithi136 ceph-mon[113531]: pgmap v9759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:53 smithi093 ceph-mon[111335]: pgmap v9759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:53 smithi093 ceph-mon[118902]: pgmap v9759: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:54.800 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:54.442+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:54.443903+0000) 2023-08-28T02:18:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2819937157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2819937157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:18:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2819937157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:18:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:18:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:18:55.805 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:55.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:55.444124+0000) 2023-08-28T02:18:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:55 smithi136 ceph-mon[113531]: pgmap v9760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:55 smithi093 ceph-mon[111335]: pgmap v9760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:55 smithi093 ceph-mon[118902]: pgmap v9760: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:56.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:56.444398+0000) 2023-08-28T02:18:57.798 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:57.443+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:57.444662+0000) 2023-08-28T02:18:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/991851380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:57 smithi136 ceph-mon[113531]: pgmap v9761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/991851380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:57 smithi093 ceph-mon[111335]: pgmap v9761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/991851380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:57 smithi093 ceph-mon[118902]: pgmap v9761: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:58.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:58.444919+0000) 2023-08-28T02:18:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:59 smithi136 ceph-mon[113531]: pgmap v9762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:18:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3355391936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:18:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:18:59.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:18:59.445171+0000) 2023-08-28T02:18:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:59 smithi093 ceph-mon[111335]: pgmap v9762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:18:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3355391936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:18:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:59 smithi093 ceph-mon[118902]: pgmap v9762: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:18:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:18:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3355391936' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:00.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:00.445390+0000) 2023-08-28T02:19:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:01 smithi136 ceph-mon[113531]: pgmap v9763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:01.444+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:01.445646+0000) 2023-08-28T02:19:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:01 smithi093 ceph-mon[111335]: pgmap v9763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:01 smithi093 ceph-mon[118902]: pgmap v9763: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:01.528Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:01.529Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:02.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3437099942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:02.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:02.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:02.445905+0000) 2023-08-28T02:19:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3437099942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:02.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3437099942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:02.956Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:02.958Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:03 smithi136 ceph-mon[113531]: pgmap v9764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:03.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:03.446162+0000) 2023-08-28T02:19:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:03 smithi093 ceph-mon[111335]: pgmap v9764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:03 smithi093 ceph-mon[118902]: pgmap v9764: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/910622470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:04.445+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:04.446431+0000) 2023-08-28T02:19:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/910622470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/910622470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:05 smithi136 ceph-mon[113531]: pgmap v9765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:05.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:05.446697+0000) 2023-08-28T02:19:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:05 smithi093 ceph-mon[111335]: pgmap v9765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:19:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:19:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:19:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:05 smithi093 ceph-mon[118902]: pgmap v9765: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4284329116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:06.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:06.446990+0000) 2023-08-28T02:19:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4284329116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4284329116' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:07.802 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:07.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:07.447288+0000) 2023-08-28T02:19:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:07 smithi136 ceph-mon[113531]: pgmap v9766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:07 smithi093 ceph-mon[111335]: pgmap v9766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:07 smithi093 ceph-mon[118902]: pgmap v9766: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:08.803 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:08.446+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:08.447549+0000) 2023-08-28T02:19:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/537382411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/537382411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/537382411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:09.810 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:09.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:09.447812+0000) 2023-08-28T02:19:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:09 smithi136 ceph-mon[113531]: pgmap v9767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:09 smithi093 ceph-mon[111335]: pgmap v9767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:09 smithi093 ceph-mon[118902]: pgmap v9767: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:10.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:10.448048+0000) 2023-08-28T02:19:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2932662689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2932662689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2932662689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:11.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:11.448288+0000) 2023-08-28T02:19:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:11.529Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:11.530Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:12 smithi093 ceph-mon[111335]: pgmap v9768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:12 smithi093 ceph-mon[118902]: pgmap v9768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:12 smithi136 ceph-mon[113531]: pgmap v9768: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:12.447+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:12.448568+0000) 2023-08-28T02:19:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:12.957Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:12.958Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:13.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:13.448803+0000) 2023-08-28T02:19:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:14 smithi093 ceph-mon[118902]: pgmap v9769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3447710406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:14 smithi093 ceph-mon[111335]: pgmap v9769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:14.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3447710406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:14 smithi136 ceph-mon[113531]: pgmap v9769: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3447710406' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:14.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:14.449024+0000) 2023-08-28T02:19:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:19:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:19:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:19:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:15.448+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:15.449297+0000) 2023-08-28T02:19:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:16 smithi093 ceph-mon[118902]: pgmap v9770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4016145653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:16 smithi093 ceph-mon[111335]: pgmap v9770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:16.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4016145653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:16 smithi136 ceph-mon[113531]: pgmap v9770: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4016145653' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:16.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:16.449554+0000) 2023-08-28T02:19:17.804 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:17.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:17.449764+0000) 2023-08-28T02:19:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:17 smithi136 ceph-mon[113531]: pgmap v9771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:17 smithi093 ceph-mon[111335]: pgmap v9771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:17 smithi093 ceph-mon[118902]: pgmap v9771: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:18.806 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:18.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:18.450055+0000) 2023-08-28T02:19:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/352660553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/352660553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/352660553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:19.812 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:19.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:19.450320+0000) 2023-08-28T02:19:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:19 smithi136 ceph-mon[113531]: pgmap v9772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:20.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:19 smithi093 ceph-mon[111335]: pgmap v9772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:19 smithi093 ceph-mon[118902]: pgmap v9772: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:20.811 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:20.449+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:20.450578+0000) 2023-08-28T02:19:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/959396531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/959396531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/959396531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:21.818 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:21.530Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:21.818 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:21.531Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:21.820 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:21.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:21.450834+0000) 2023-08-28T02:19:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:21 smithi136 ceph-mon[113531]: pgmap v9773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:21 smithi093 ceph-mon[111335]: pgmap v9773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:21 smithi093 ceph-mon[118902]: pgmap v9773: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:22.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:22.451103+0000) 2023-08-28T02:19:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2845946023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2845946023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:22.957Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:22.958Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:23.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2845946023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:19:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:23.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:23.451273+0000) 2023-08-28T02:19:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:24 smithi136 ceph-mon[113531]: pgmap v9774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:24 smithi093 ceph-mon[111335]: pgmap v9774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:24 smithi093 ceph-mon[118902]: pgmap v9774: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:24.859 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:24.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:24.451413+0000) 2023-08-28T02:19:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:19:25] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:19:25.745 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:25.450+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:25.451600+0000) 2023-08-28T02:19:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: pgmap v9775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3447781932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:26.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: pgmap v9775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3447781932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: pgmap v9775: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3447781932' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:19:26.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:19:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:26.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:26.451809+0000) 2023-08-28T02:19:27.802 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:27.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:27.452075+0000) 2023-08-28T02:19:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:27 smithi136 ceph-mon[113531]: pgmap v9776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:27 smithi093 ceph-mon[111335]: pgmap v9776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:27 smithi093 ceph-mon[118902]: pgmap v9776: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:28.804 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:28.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:28.452317+0000) 2023-08-28T02:19:29.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/531297176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/531297176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:29.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/531297176' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:29.811 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:29.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:29.452568+0000) 2023-08-28T02:19:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:29 smithi136 ceph-mon[113531]: pgmap v9777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:29 smithi093 ceph-mon[111335]: pgmap v9777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:29 smithi093 ceph-mon[118902]: pgmap v9777: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:30.813 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:30.451+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:30.452814+0000) 2023-08-28T02:19:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2583462039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:31.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2583462039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2583462039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:31.817 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:31.530Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:31.817 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:31.531Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:31.817 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:31.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:31.453085+0000) 2023-08-28T02:19:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:31 smithi136 ceph-mon[113531]: pgmap v9778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:31 smithi093 ceph-mon[111335]: pgmap v9778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:31 smithi093 ceph-mon[118902]: pgmap v9778: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:32.819 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:32.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:32.453328+0000) 2023-08-28T02:19:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2320641568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2320641568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2320641568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:32.958Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:32.959Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:33.824 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:33.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:33.453564+0000) 2023-08-28T02:19:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:33 smithi136 ceph-mon[113531]: pgmap v9779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:33 smithi093 ceph-mon[111335]: pgmap v9779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:33 smithi093 ceph-mon[118902]: pgmap v9779: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:34.452+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:34.453862+0000) 2023-08-28T02:19:35.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4256324627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4256324627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4256324627' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:19:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:19:35] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:19:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:35.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:35.454153+0000) 2023-08-28T02:19:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:36 smithi136 ceph-mon[113531]: pgmap v9780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:36 smithi093 ceph-mon[111335]: pgmap v9780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:36 smithi093 ceph-mon[118902]: pgmap v9780: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:36.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:36.454373+0000) 2023-08-28T02:19:37.807 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:37.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:37.454614+0000) 2023-08-28T02:19:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:37 smithi136 ceph-mon[113531]: pgmap v9781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2684527079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:37 smithi093 ceph-mon[111335]: pgmap v9781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2684527079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:37 smithi093 ceph-mon[118902]: pgmap v9781: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2684527079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:38.453+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:38.454868+0000) 2023-08-28T02:19:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:39.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:39.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:39.455148+0000) 2023-08-28T02:19:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:39 smithi093 ceph-mon[111335]: pgmap v9782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1173639455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:39 smithi093 ceph-mon[118902]: pgmap v9782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1173639455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:39 smithi136 ceph-mon[113531]: pgmap v9782: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1173639455' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:40.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:40.455365+0000) 2023-08-28T02:19:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:41 smithi136 ceph-mon[113531]: pgmap v9783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:41.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:41.455575+0000) 2023-08-28T02:19:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:41 smithi093 ceph-mon[111335]: pgmap v9783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:41 smithi093 ceph-mon[118902]: pgmap v9783: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:41.531Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:41.532Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:42.801 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:42.454+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:42.455789+0000) 2023-08-28T02:19:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1812935001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1812935001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1812935001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:42.959Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:42.960Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:43.806 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:43.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:43.456027+0000) 2023-08-28T02:19:44.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:43 smithi136 ceph-mon[113531]: pgmap v9784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:44.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:43 smithi093 ceph-mon[111335]: pgmap v9784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:44.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:43 smithi093 ceph-mon[118902]: pgmap v9784: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:44.808 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:44.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:44.456252+0000) 2023-08-28T02:19:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3992113141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3992113141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3992113141' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:19:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:19:45] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:19:45.813 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:45.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:45.456489+0000) 2023-08-28T02:19:46.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:45 smithi136 ceph-mon[113531]: pgmap v9785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:45 smithi093 ceph-mon[111335]: pgmap v9785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:46.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:45 smithi093 ceph-mon[118902]: pgmap v9785: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:46.455+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:46.456687+0000) 2023-08-28T02:19:47.805 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:47.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:47.456935+0000) 2023-08-28T02:19:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2119944731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:47 smithi136 ceph-mon[113531]: pgmap v9786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2119944731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:47 smithi093 ceph-mon[111335]: pgmap v9786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2119944731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:47 smithi093 ceph-mon[118902]: pgmap v9786: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:48.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:48.457192+0000) 2023-08-28T02:19:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:49.456+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:49.457448+0000) 2023-08-28T02:19:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:49 smithi136 ceph-mon[113531]: pgmap v9787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3732615423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:49 smithi093 ceph-mon[111335]: pgmap v9787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3732615423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:49 smithi093 ceph-mon[118902]: pgmap v9787: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3732615423' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:50.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:50.457690+0000) 2023-08-28T02:19:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:51 smithi136 ceph-mon[113531]: pgmap v9788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:51.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:51.457950+0000) 2023-08-28T02:19:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:51 smithi093 ceph-mon[111335]: pgmap v9788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:51 smithi093 ceph-mon[118902]: pgmap v9788: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:51.532Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:51.533Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:52.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/342791499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:52.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:52.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:52.458203+0000) 2023-08-28T02:19:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/342791499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:52.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/342791499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:52.960Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:19:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:19:52.961Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:19:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:53 smithi136 ceph-mon[113531]: pgmap v9789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:53.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:53.458409+0000) 2023-08-28T02:19:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:53 smithi093 ceph-mon[111335]: pgmap v9789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:53 smithi093 ceph-mon[118902]: pgmap v9789: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:54.457+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:54.458648+0000) 2023-08-28T02:19:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2369021325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2369021325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:19:54.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2369021325' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:55 smithi136 ceph-mon[113531]: pgmap v9790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:55.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:55.458927+0000) 2023-08-28T02:19:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:55 smithi093 ceph-mon[111335]: pgmap v9790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:55 smithi093 ceph-mon[118902]: pgmap v9790: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:19:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:19:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:19:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1927470198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:56.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:56.459123+0000) 2023-08-28T02:19:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1927470198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:56.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1927470198' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:57.807 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:57.458+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:57.459383+0000) 2023-08-28T02:19:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:57 smithi136 ceph-mon[113531]: pgmap v9791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:57 smithi093 ceph-mon[111335]: pgmap v9791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:57 smithi093 ceph-mon[118902]: pgmap v9791: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:19:58.874 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:58.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:58.459583+0000) 2023-08-28T02:19:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1512873710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1512873710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:59.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1512873710' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:19:59.881 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:19:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:19:59.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:19:59.459846+0000) 2023-08-28T02:20:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:19:59 smithi136 ceph-mon[113531]: pgmap v9792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:19:59 smithi093 ceph-mon[111335]: pgmap v9792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:19:59 smithi093 ceph-mon[118902]: pgmap v9792: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:00.883 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:00.459+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:00.460152+0000) 2023-08-28T02:20:01.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:20:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:20:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:20:01.892 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:01.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:01.892 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:01.534Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:01.892 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:01.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:01.460391+0000) 2023-08-28T02:20:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1559705160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:01 smithi136 ceph-mon[113531]: pgmap v9793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1559705160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:01 smithi093 ceph-mon[118902]: pgmap v9793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1559705160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:02.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:01 smithi093 ceph-mon[111335]: pgmap v9793: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:02.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:02.460676+0000) 2023-08-28T02:20:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:02.961Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:02.962Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:03 smithi136 ceph-mon[113531]: pgmap v9794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:03.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:03.460933+0000) 2023-08-28T02:20:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:03 smithi093 ceph-mon[111335]: pgmap v9794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:03 smithi093 ceph-mon[118902]: pgmap v9794: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3613734485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:04.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:04.461190+0000) 2023-08-28T02:20:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3613734485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3613734485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:05 smithi136 ceph-mon[113531]: pgmap v9795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:05.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:05.461453+0000) 2023-08-28T02:20:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:05 smithi093 ceph-mon[111335]: pgmap v9795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:05 smithi093 ceph-mon[118902]: pgmap v9795: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:20:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:20:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:20:06.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1605233388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:06.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:06.460+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:06.461734+0000) 2023-08-28T02:20:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1605233388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1605233388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:07.809 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:07.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:07.461963+0000) 2023-08-28T02:20:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:07 smithi136 ceph-mon[113531]: pgmap v9796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:07 smithi093 ceph-mon[111335]: pgmap v9796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:07 smithi093 ceph-mon[118902]: pgmap v9796: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:08.810 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:08.461+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:08.462165+0000) 2023-08-28T02:20:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3281243006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3281243006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3281243006' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:09.816 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:09.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:09.462441+0000) 2023-08-28T02:20:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:09 smithi136 ceph-mon[113531]: pgmap v9797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:09 smithi093 ceph-mon[111335]: pgmap v9797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:09 smithi093 ceph-mon[118902]: pgmap v9797: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:10.817 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:10.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:10.462704+0000) 2023-08-28T02:20:11.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1490169824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:11.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1490169824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:11.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1490169824' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:11.822 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:11.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:11.822 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:11.534Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:11.823 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:11.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:11.462944+0000) 2023-08-28T02:20:12.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:11 smithi136 ceph-mon[113531]: pgmap v9798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:11 smithi093 ceph-mon[111335]: pgmap v9798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:11 smithi093 ceph-mon[118902]: pgmap v9798: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:12.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:12.463223+0000) 2023-08-28T02:20:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2969240024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2969240024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:12.961Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:12.962Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2969240024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:13.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:13.463462+0000) 2023-08-28T02:20:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:14 smithi136 ceph-mon[113531]: pgmap v9799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:14 smithi093 ceph-mon[111335]: pgmap v9799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:14 smithi093 ceph-mon[118902]: pgmap v9799: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:14.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:14.463694+0000) 2023-08-28T02:20:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:15 smithi136 ceph-mon[113531]: pgmap v9800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:15.462+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:15.463901+0000) 2023-08-28T02:20:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:15 smithi093 ceph-mon[111335]: pgmap v9800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:15 smithi093 ceph-mon[118902]: pgmap v9800: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:20:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:20:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:20:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1565262390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:16.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:16.464160+0000) 2023-08-28T02:20:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1565262390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1565262390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:17.812 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:17.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:17.464416+0000) 2023-08-28T02:20:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:17 smithi136 ceph-mon[113531]: pgmap v9801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:17 smithi093 ceph-mon[111335]: pgmap v9801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:17 smithi093 ceph-mon[118902]: pgmap v9801: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:18.812 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:18.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:18.464673+0000) 2023-08-28T02:20:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2774158974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2774158974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2774158974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:19.817 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:19.463+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:19.464926+0000) 2023-08-28T02:20:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:19 smithi136 ceph-mon[113531]: pgmap v9802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:19 smithi093 ceph-mon[111335]: pgmap v9802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:19 smithi093 ceph-mon[118902]: pgmap v9802: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:20.818 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:20.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:20.465180+0000) 2023-08-28T02:20:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/163336825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/163336825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/163336825' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:21.825 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:21.533Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:21.826 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:21.534Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:21.826 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:21.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:21.465435+0000) 2023-08-28T02:20:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:21 smithi136 ceph-mon[113531]: pgmap v9803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:21 smithi093 ceph-mon[111335]: pgmap v9803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:21 smithi093 ceph-mon[118902]: pgmap v9803: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:22.829 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:22.464+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:22.465664+0000) 2023-08-28T02:20:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2403098553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2403098553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2403098553' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:22.962Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:22.963Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:23.836 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:23.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:23.465893+0000) 2023-08-28T02:20:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:23 smithi136 ceph-mon[113531]: pgmap v9804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:23 smithi093 ceph-mon[111335]: pgmap v9804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:23 smithi093 ceph-mon[118902]: pgmap v9804: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:24.837 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:24.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:24.466112+0000) 2023-08-28T02:20:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:20:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:20:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:20:25.842 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:25.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:25.466353+0000) 2023-08-28T02:20:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1015752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:25 smithi136 ceph-mon[113531]: pgmap v9805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:26.189 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1015752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:26.189 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:25 smithi093 ceph-mon[111335]: pgmap v9805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:26.189 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1015752' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:26.189 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:25 smithi093 ceph-mon[118902]: pgmap v9805: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:26.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:26.466566+0000) 2023-08-28T02:20:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:26.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:27.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:20:27.622 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:27.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:27.466826+0000) 2023-08-28T02:20:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:27 smithi136 ceph-mon[113531]: pgmap v9806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2932763284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:27.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:27 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[111335]: pgmap v9806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2932763284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:28.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[118902]: pgmap v9806: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2932763284' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:27 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:28.465+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:28.466982+0000) 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:29.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:20:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:20:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:20:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:20:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:29.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:29.467156+0000) 2023-08-28T02:20:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:30 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2374M 2023-08-28T02:20:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:30 smithi136 ceph-mon[113531]: pgmap v9807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1684005498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:30 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2374M 2023-08-28T02:20:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:30 smithi093 ceph-mon[111335]: pgmap v9807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1684005498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:30 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2374M 2023-08-28T02:20:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:30 smithi093 ceph-mon[118902]: pgmap v9807: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1684005498' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:30.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:30.467405+0000) 2023-08-28T02:20:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:31 smithi136 ceph-mon[113531]: pgmap v9808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:31.466+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:31.467649+0000) 2023-08-28T02:20:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:31 smithi093 ceph-mon[111335]: pgmap v9808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:31 smithi093 ceph-mon[118902]: pgmap v9808: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:31.534Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:31.536Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:32.809 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:32.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:32.467908+0000) 2023-08-28T02:20:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4120881184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4120881184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4120881184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:32.963Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:32.964Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:33.813 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:33.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:33.468119+0000) 2023-08-28T02:20:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:33 smithi136 ceph-mon[113531]: pgmap v9809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:33 smithi093 ceph-mon[111335]: pgmap v9809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:34.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:33 smithi093 ceph-mon[118902]: pgmap v9809: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:34.815 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:34.467+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:34.468353+0000) 2023-08-28T02:20:35.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2425082039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:35.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2425082039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2425082039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:20:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:20:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:20:35.820 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:35.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:35.468612+0000) 2023-08-28T02:20:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:35 smithi136 ceph-mon[113531]: pgmap v9810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:35 smithi093 ceph-mon[111335]: pgmap v9810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:35 smithi093 ceph-mon[118902]: pgmap v9810: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:36.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:36.468855+0000) 2023-08-28T02:20:37.813 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:37.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:37.469023+0000) 2023-08-28T02:20:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/975100001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:37 smithi136 ceph-mon[113531]: pgmap v9811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/975100001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:37 smithi093 ceph-mon[111335]: pgmap v9811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/975100001' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:37 smithi093 ceph-mon[118902]: pgmap v9811: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:38.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:38.469273+0000) 2023-08-28T02:20:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:39.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:39.469519+0000) 2023-08-28T02:20:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:39 smithi136 ceph-mon[113531]: pgmap v9812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/371084387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:40.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:39 smithi093 ceph-mon[111335]: pgmap v9812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:40.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/371084387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:40.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:39 smithi093 ceph-mon[118902]: pgmap v9812: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/371084387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:40.468+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:40.469788+0000) 2023-08-28T02:20:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:41 smithi136 ceph-mon[113531]: pgmap v9813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:41.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:41.470079+0000) 2023-08-28T02:20:41.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:41 smithi093 ceph-mon[111335]: pgmap v9813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:41.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:41 smithi093 ceph-mon[118902]: pgmap v9813: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:41.536Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:41.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:41.537Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3607623690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:42.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:42.470340+0000) 2023-08-28T02:20:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3607623690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:42.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3607623690' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:42.964Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:42.965Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:43 smithi136 ceph-mon[113531]: pgmap v9814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:43.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:43.470600+0000) 2023-08-28T02:20:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:43 smithi093 ceph-mon[111335]: pgmap v9814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:43 smithi093 ceph-mon[118902]: pgmap v9814: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2331066601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:44.469+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:44.470769+0000) 2023-08-28T02:20:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2331066601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2331066601' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:45 smithi136 ceph-mon[113531]: pgmap v9815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:45.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:45.471000+0000) 2023-08-28T02:20:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:45 smithi093 ceph-mon[118902]: pgmap v9815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:45 smithi093 ceph-mon[111335]: pgmap v9815: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:20:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:20:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:20:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2022582883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:46.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:46.471274+0000) 2023-08-28T02:20:46.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2022582883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2022582883' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:47.823 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:47.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:47.471500+0000) 2023-08-28T02:20:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:47 smithi136 ceph-mon[113531]: pgmap v9816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:47 smithi093 ceph-mon[111335]: pgmap v9816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:47 smithi093 ceph-mon[118902]: pgmap v9816: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:48.916 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:48.470+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:48.471764+0000) 2023-08-28T02:20:49.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/383300048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/383300048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/383300048' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:49.923 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:49.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:49.472019+0000) 2023-08-28T02:20:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:49 smithi136 ceph-mon[113531]: pgmap v9817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:49 smithi093 ceph-mon[111335]: pgmap v9817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:49 smithi093 ceph-mon[118902]: pgmap v9817: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:50.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:50.472317+0000) 2023-08-28T02:20:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:51 smithi136 ceph-mon[113531]: pgmap v9818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/172283888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:51.471+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:51.472513+0000) 2023-08-28T02:20:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:51 smithi093 ceph-mon[111335]: pgmap v9818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/172283888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:51 smithi093 ceph-mon[118902]: pgmap v9818: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/172283888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:51.537Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:51.538Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:52.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:52.472782+0000) 2023-08-28T02:20:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:52.964Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:20:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:20:52.965Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:20:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:53 smithi136 ceph-mon[113531]: pgmap v9819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:53.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:53.473001+0000) 2023-08-28T02:20:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:53 smithi093 ceph-mon[111335]: pgmap v9819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:53 smithi093 ceph-mon[118902]: pgmap v9819: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/869152365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:54.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:54.473262+0000) 2023-08-28T02:20:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/869152365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/869152365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:20:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:55 smithi136 ceph-mon[113531]: pgmap v9820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:55.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:55.473499+0000) 2023-08-28T02:20:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:55 smithi093 ceph-mon[111335]: pgmap v9820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:20:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:20:55] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:20:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:55 smithi093 ceph-mon[118902]: pgmap v9820: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/573511093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:56.472+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:56.473766+0000) 2023-08-28T02:20:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/573511093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:56.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/573511093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:57.820 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:57.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:57.474062+0000) 2023-08-28T02:20:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:57 smithi136 ceph-mon[113531]: pgmap v9821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:57 smithi093 ceph-mon[111335]: pgmap v9821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:57 smithi093 ceph-mon[118902]: pgmap v9821: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:20:58.820 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:58.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:58.474310+0000) 2023-08-28T02:20:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/248171235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/248171235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:59.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/248171235' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:20:59.824 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:20:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:20:59.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:20:59.474554+0000) 2023-08-28T02:21:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:20:59 smithi136 ceph-mon[113531]: pgmap v9822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:20:59 smithi093 ceph-mon[111335]: pgmap v9822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:00.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:20:59 smithi093 ceph-mon[118902]: pgmap v9822: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:00.826 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:00.473+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:00.474835+0000) 2023-08-28T02:21:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1886548584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1886548584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1886548584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:01.832 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:01.537Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:01.832 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:01.538Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:01.834 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:01.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:01.475093+0000) 2023-08-28T02:21:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:01 smithi136 ceph-mon[113531]: pgmap v9823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:01 smithi093 ceph-mon[111335]: pgmap v9823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:01 smithi093 ceph-mon[118902]: pgmap v9823: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:02.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:02.475355+0000) 2023-08-28T02:21:03.215 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:02.965Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:03.215 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:02.966Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:03.474 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1078333279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1078333279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:03.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1078333279' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:03.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:03.475604+0000) 2023-08-28T02:21:04.474 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:04 smithi136 ceph-mon[113531]: pgmap v9824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:04 smithi093 ceph-mon[111335]: pgmap v9824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:04 smithi093 ceph-mon[118902]: pgmap v9824: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:04.474+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:04.475864+0000) 2023-08-28T02:21:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:05 smithi136 ceph-mon[113531]: pgmap v9825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:05.688 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 02:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T02:21:05.318028626Z level=info msg="Completed cleanup jobs" duration=12.787856ms 2023-08-28T02:21:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:05.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:05.476132+0000) 2023-08-28T02:21:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:05 smithi093 ceph-mon[111335]: pgmap v9825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:05.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:05 smithi093 ceph-mon[118902]: pgmap v9825: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:21:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:21:05] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:21:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1232376780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:06.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:06.476379+0000) 2023-08-28T02:21:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1232376780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1232376780' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:07.823 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:07.475+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:07.476642+0000) 2023-08-28T02:21:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:07 smithi136 ceph-mon[113531]: pgmap v9826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:07 smithi093 ceph-mon[111335]: pgmap v9826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:07 smithi093 ceph-mon[118902]: pgmap v9826: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:08.826 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:08.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:08.476896+0000) 2023-08-28T02:21:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3703018066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3703018066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3703018066' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:09.833 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:09.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:09.477188+0000) 2023-08-28T02:21:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:09 smithi136 ceph-mon[113531]: pgmap v9827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:09 smithi093 ceph-mon[118902]: pgmap v9827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:09 smithi093 ceph-mon[111335]: pgmap v9827: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:10.834 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:10.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:10.477407+0000) 2023-08-28T02:21:11.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2687593023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:11.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2687593023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2687593023' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:11.838 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:11.538Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:11.839 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:11.539Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:11.839 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:11.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:11.477654+0000) 2023-08-28T02:21:12.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:11 smithi136 ceph-mon[113531]: pgmap v9828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:11 smithi093 ceph-mon[111335]: pgmap v9828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:12.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:11 smithi093 ceph-mon[118902]: pgmap v9828: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:12.840 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:12.476+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:12.477853+0000) 2023-08-28T02:21:13.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3879668707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3879668707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3879668707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:12.966Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:12.967Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:13.847 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:13.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:13.478113+0000) 2023-08-28T02:21:14.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:13 smithi136 ceph-mon[113531]: pgmap v9829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:13 smithi093 ceph-mon[111335]: pgmap v9829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:13 smithi093 ceph-mon[118902]: pgmap v9829: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:14.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:14.478369+0000) 2023-08-28T02:21:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3476433691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:15.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3476433691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:15.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:21:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:21:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:21:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3476433691' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:15.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:15.478536+0000) 2023-08-28T02:21:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:16 smithi136 ceph-mon[113531]: pgmap v9830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:16 smithi093 ceph-mon[111335]: pgmap v9830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:16 smithi093 ceph-mon[118902]: pgmap v9830: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:16.477+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:16.478781+0000) 2023-08-28T02:21:17.825 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:17.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:17.479020+0000) 2023-08-28T02:21:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:17 smithi136 ceph-mon[113531]: pgmap v9831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3389789166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:17 smithi093 ceph-mon[111335]: pgmap v9831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3389789166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:17 smithi093 ceph-mon[118902]: pgmap v9831: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3389789166' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:18.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:18.479282+0000) 2023-08-28T02:21:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:19 smithi136 ceph-mon[113531]: pgmap v9832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:19.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:19.479501+0000) 2023-08-28T02:21:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:19 smithi093 ceph-mon[111335]: pgmap v9832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:19 smithi093 ceph-mon[118902]: pgmap v9832: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1939881658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:20.478+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:20.479771+0000) 2023-08-28T02:21:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1939881658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1939881658' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:21 smithi136 ceph-mon[113531]: pgmap v9833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:21.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:21.480028+0000) 2023-08-28T02:21:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:21 smithi093 ceph-mon[111335]: pgmap v9833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:21 smithi093 ceph-mon[118902]: pgmap v9833: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:21.539Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:21.540Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:22.818 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:22.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:22.480303+0000) 2023-08-28T02:21:23.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2928163503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2928163503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:23.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2928163503' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:22.967Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:22.968Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:23.822 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:23.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:23.480538+0000) 2023-08-28T02:21:24.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:23 smithi136 ceph-mon[113531]: pgmap v9834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:23 smithi093 ceph-mon[111335]: pgmap v9834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:23 smithi093 ceph-mon[118902]: pgmap v9834: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:24.824 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:24.479+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:24.480796+0000) 2023-08-28T02:21:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:25.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/685019892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/685019892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/685019892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:21:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:21:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:21:25.828 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:25.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:25.481099+0000) 2023-08-28T02:21:26.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:25 smithi136 ceph-mon[113531]: pgmap v9835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:25 smithi093 ceph-mon[111335]: pgmap v9835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:25 smithi093 ceph-mon[118902]: pgmap v9835: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:26.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:26.481334+0000) 2023-08-28T02:21:27.822 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:27.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:27.481575+0000) 2023-08-28T02:21:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3446039667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:28.189 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:27 smithi136 ceph-mon[113531]: pgmap v9836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3446039667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:27 smithi093 ceph-mon[111335]: pgmap v9836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3446039667' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:27 smithi093 ceph-mon[118902]: pgmap v9836: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:28.823 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:28.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:28.481820+0000) 2023-08-28T02:21:29.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:21:29.682 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:29.480+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:29.482000+0000) 2023-08-28T02:21:29.937 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:29 smithi136 ceph-mon[113531]: pgmap v9837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:29.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1072012727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:30.195 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:29 smithi093 ceph-mon[111335]: pgmap v9837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:30.196 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1072012727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:30.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:29 smithi093 ceph-mon[118902]: pgmap v9837: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:30.196 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1072012727' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:30.804 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:30.481+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:30.482150+0000) 2023-08-28T02:21:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:31 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:31 smithi136 ceph-mon[113531]: pgmap v9838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:31.689 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:31.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:31.482367+0000) 2023-08-28T02:21:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[111335]: pgmap v9838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:31.539Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:31.540Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:31.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:31 smithi093 ceph-mon[118902]: pgmap v9838: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:32.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:32.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:32.482573+0000) 2023-08-28T02:21:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2356410117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:32.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2356410117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:21:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:21:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:21:32.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2356410117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:32.968Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:32.969Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:33 smithi136 ceph-mon[113531]: pgmap v9839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:33.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:33.482842+0000) 2023-08-28T02:21:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:33 smithi093 ceph-mon[111335]: pgmap v9839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:33 smithi093 ceph-mon[118902]: pgmap v9839: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4038519694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:34.482+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:34.483118+0000) 2023-08-28T02:21:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4038519694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4038519694' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:35 smithi136 ceph-mon[113531]: pgmap v9840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:35.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:35.483304+0000) 2023-08-28T02:21:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:35 smithi093 ceph-mon[111335]: pgmap v9840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:35 smithi093 ceph-mon[118902]: pgmap v9840: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:21:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:21:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:21:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/346866591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:36.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:36.483560+0000) 2023-08-28T02:21:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/346866591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/346866591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:37.831 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:37.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:37.483816+0000) 2023-08-28T02:21:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:37 smithi136 ceph-mon[113531]: pgmap v9841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:37 smithi093 ceph-mon[111335]: pgmap v9841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:37 smithi093 ceph-mon[118902]: pgmap v9841: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:38.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:38.484088+0000) 2023-08-28T02:21:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2020719059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2020719059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2020719059' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:39.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:39.484409+0000) 2023-08-28T02:21:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:39 smithi093 ceph-mon[111335]: pgmap v9842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:39 smithi093 ceph-mon[118902]: pgmap v9842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:39 smithi136 ceph-mon[113531]: pgmap v9842: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:40.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:40.484659+0000) 2023-08-28T02:21:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:41 smithi136 ceph-mon[113531]: pgmap v9843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2649249197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:41.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:41.483+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:41.484918+0000) 2023-08-28T02:21:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:41 smithi093 ceph-mon[111335]: pgmap v9843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:41.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2649249197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:41 smithi093 ceph-mon[118902]: pgmap v9843: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:41.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2649249197' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:41.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:41.540Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:41.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:41.541Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:42.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:42.485189+0000) 2023-08-28T02:21:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:42.968Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:42.969Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:43 smithi136 ceph-mon[113531]: pgmap v9844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:43.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:43.485397+0000) 2023-08-28T02:21:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:43 smithi093 ceph-mon[111335]: pgmap v9844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:43 smithi093 ceph-mon[118902]: pgmap v9844: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4290163975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:44.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:44.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:44.485647+0000) 2023-08-28T02:21:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4290163975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:44.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4290163975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:45 smithi136 ceph-mon[113531]: pgmap v9845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:45.484+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:45.485922+0000) 2023-08-28T02:21:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:45 smithi093 ceph-mon[111335]: pgmap v9845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:45 smithi093 ceph-mon[118902]: pgmap v9845: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:21:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:21:45] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:21:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/611753836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:46.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:46.486223+0000) 2023-08-28T02:21:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/611753836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/611753836' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:47.827 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:47.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:47.486485+0000) 2023-08-28T02:21:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:47 smithi136 ceph-mon[113531]: pgmap v9846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:47 smithi093 ceph-mon[111335]: pgmap v9846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:47 smithi093 ceph-mon[118902]: pgmap v9846: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:48.830 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:48.485+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:48.486751+0000) 2023-08-28T02:21:49.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1199214415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1199214415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1199214415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:49.833 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:49.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:49.486982+0000) 2023-08-28T02:21:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:49 smithi136 ceph-mon[113531]: pgmap v9847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:49 smithi093 ceph-mon[111335]: pgmap v9847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:49 smithi093 ceph-mon[118902]: pgmap v9847: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:50.838 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:50.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:50.487231+0000) 2023-08-28T02:21:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1429411149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1429411149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1429411149' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:51.844 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:51.541Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:51.844 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:51.541Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:51.845 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:51.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:51.487505+0000) 2023-08-28T02:21:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:51 smithi136 ceph-mon[113531]: pgmap v9848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:51 smithi093 ceph-mon[111335]: pgmap v9848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:52.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:51 smithi093 ceph-mon[118902]: pgmap v9848: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:52.486+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:52.487758+0000) 2023-08-28T02:21:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:52.969Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:53.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:21:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:21:52.969Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:21:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3442036226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:53.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:53.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:53.488004+0000) 2023-08-28T02:21:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3442036226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3442036226' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:54 smithi136 ceph-mon[113531]: pgmap v9849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:54.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:54.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:54.488192+0000) 2023-08-28T02:21:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:54 smithi093 ceph-mon[111335]: pgmap v9849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:54 smithi093 ceph-mon[118902]: pgmap v9849: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:21:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:21:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:21:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:21:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:55.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:55.488431+0000) 2023-08-28T02:21:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:56 smithi136 ceph-mon[113531]: pgmap v9850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1756608806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:56.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:56.487+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:56.488716+0000) 2023-08-28T02:21:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:56 smithi093 ceph-mon[111335]: pgmap v9850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1756608806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:56.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:56 smithi093 ceph-mon[118902]: pgmap v9850: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:56.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1756608806' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:57.828 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:57.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:57.488966+0000) 2023-08-28T02:21:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:57 smithi136 ceph-mon[113531]: pgmap v9851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:57 smithi093 ceph-mon[111335]: pgmap v9851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:57 smithi093 ceph-mon[118902]: pgmap v9851: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:21:58.830 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:58.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:58.489270+0000) 2023-08-28T02:21:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1576970097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1576970097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1576970097' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:21:59.836 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:21:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:21:59.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:21:59.489510+0000) 2023-08-28T02:22:00.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:21:59 smithi136 ceph-mon[113531]: pgmap v9852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:00.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:21:59 smithi093 ceph-mon[111335]: pgmap v9852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:00.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:21:59 smithi093 ceph-mon[118902]: pgmap v9852: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:00.839 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:00.488+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:00.489759+0000) 2023-08-28T02:22:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1402402903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1402402903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:01.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1402402903' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:01.842 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:01.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:01.490018+0000) 2023-08-28T02:22:01.842 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:01.541Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:01.842 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:01.542Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:02.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:01 smithi136 ceph-mon[113531]: pgmap v9853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:01 smithi093 ceph-mon[111335]: pgmap v9853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:01 smithi093 ceph-mon[118902]: pgmap v9853: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:02.844 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:02.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:02.490282+0000) 2023-08-28T02:22:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2663120659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2663120659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2663120659' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:02.969Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:02.970Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:03.850 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:03.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:03.490518+0000) 2023-08-28T02:22:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:03 smithi136 ceph-mon[113531]: pgmap v9854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:03 smithi093 ceph-mon[111335]: pgmap v9854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:03 smithi093 ceph-mon[118902]: pgmap v9854: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:04.489+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:04.490800+0000) 2023-08-28T02:22:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1267806032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:05.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1267806032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1267806032' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:22:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:22:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:22:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:05.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:05.491089+0000) 2023-08-28T02:22:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:06 smithi136 ceph-mon[113531]: pgmap v9855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:06 smithi093 ceph-mon[111335]: pgmap v9855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:06 smithi093 ceph-mon[118902]: pgmap v9855: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:06.490+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:06.491363+0000) 2023-08-28T02:22:07.844 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:07.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:07.491606+0000) 2023-08-28T02:22:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:07 smithi136 ceph-mon[113531]: pgmap v9856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1794300418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:07 smithi093 ceph-mon[111335]: pgmap v9856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1794300418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:07 smithi093 ceph-mon[118902]: pgmap v9856: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1794300418' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:08.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:08.491872+0000) 2023-08-28T02:22:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:09.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:09.492161+0000) 2023-08-28T02:22:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:09 smithi136 ceph-mon[113531]: pgmap v9857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/245790147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:09 smithi093 ceph-mon[111335]: pgmap v9857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/245790147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:10.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:09 smithi093 ceph-mon[118902]: pgmap v9857: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:10.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/245790147' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:10.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:10.492411+0000) 2023-08-28T02:22:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:11 smithi136 ceph-mon[113531]: pgmap v9858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:11.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:11.492625+0000) 2023-08-28T02:22:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:11 smithi093 ceph-mon[111335]: pgmap v9858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:11 smithi093 ceph-mon[118902]: pgmap v9858: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:11.542Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:11.543Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:12.831 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:12.491+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:12.492865+0000) 2023-08-28T02:22:13.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4180059160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4180059160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4180059160' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:12.970Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:12.971Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:13.835 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:13.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:13.493114+0000) 2023-08-28T02:22:14.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:13 smithi136 ceph-mon[113531]: pgmap v9859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:13 smithi093 ceph-mon[111335]: pgmap v9859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:13 smithi093 ceph-mon[118902]: pgmap v9859: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:14.836 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:14.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:14.493382+0000) 2023-08-28T02:22:15.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2376509214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2376509214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2376509214' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:22:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:22:15] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:22:15.841 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:15.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:15.493573+0000) 2023-08-28T02:22:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:15 smithi136 ceph-mon[113531]: pgmap v9860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:15 smithi093 ceph-mon[111335]: pgmap v9860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:16.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:15 smithi093 ceph-mon[118902]: pgmap v9860: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:16.492+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:16.493836+0000) 2023-08-28T02:22:17.834 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:17.493+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:17.494114+0000) 2023-08-28T02:22:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3082640913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:17 smithi136 ceph-mon[113531]: pgmap v9861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3082640913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:17 smithi093 ceph-mon[111335]: pgmap v9861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3082640913' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:17 smithi093 ceph-mon[118902]: pgmap v9861: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:18.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:18.494388+0000) 2023-08-28T02:22:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:19 smithi136 ceph-mon[113531]: pgmap v9862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:19.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:19.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:19.494650+0000) 2023-08-28T02:22:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:19 smithi093 ceph-mon[111335]: pgmap v9862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:19.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:19 smithi093 ceph-mon[118902]: pgmap v9862: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3113823662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:20.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:20.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:20.494864+0000) 2023-08-28T02:22:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3113823662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:20.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3113823662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:21 smithi136 ceph-mon[113531]: pgmap v9863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:21.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:21.495097+0000) 2023-08-28T02:22:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:21 smithi093 ceph-mon[111335]: pgmap v9863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:21 smithi093 ceph-mon[118902]: pgmap v9863: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:21.543Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:21.543Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/336125303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:22.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:22.495278+0000) 2023-08-28T02:22:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/336125303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/336125303' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:22.971Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 9 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:22.972Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:23 smithi136 ceph-mon[113531]: pgmap v9864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:23.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:23.495532+0000) 2023-08-28T02:22:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:23 smithi093 ceph-mon[111335]: pgmap v9864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:23 smithi093 ceph-mon[118902]: pgmap v9864: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2853541722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:24.494+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:24.495808+0000) 2023-08-28T02:22:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2853541722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2853541722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:25 smithi136 ceph-mon[113531]: pgmap v9865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:25.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:25.496080+0000) 2023-08-28T02:22:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:25 smithi093 ceph-mon[111335]: pgmap v9865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:25 smithi093 ceph-mon[118902]: pgmap v9865: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:22:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:22:25] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:22:26.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3471490441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:26.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:26.496358+0000) 2023-08-28T02:22:26.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3471490441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:26.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3471490441' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:27.843 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:27.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:27.496626+0000) 2023-08-28T02:22:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:27 smithi136 ceph-mon[113531]: pgmap v9866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:27 smithi093 ceph-mon[111335]: pgmap v9866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:27 smithi093 ceph-mon[118902]: pgmap v9866: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:28.495+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:28.496836+0000) 2023-08-28T02:22:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2313069692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2313069692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2313069692' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:29.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:29.497086+0000) 2023-08-28T02:22:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:30 smithi136 ceph-mon[113531]: pgmap v9867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:30.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:30 smithi093 ceph-mon[111335]: pgmap v9867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:30.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:30 smithi093 ceph-mon[118902]: pgmap v9867: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:30.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:30.497351+0000) 2023-08-28T02:22:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:31 smithi136 ceph-mon[113531]: pgmap v9868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1365471802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:31.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:31.497609+0000) 2023-08-28T02:22:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:31 smithi093 ceph-mon[111335]: pgmap v9868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:31.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1365471802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:31 smithi093 ceph-mon[118902]: pgmap v9868: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:31.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1365471802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:31.543Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:31.544Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:32 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:32 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:32 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:22:32.874 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:32.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:32.497822+0000) 2023-08-28T02:22:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:32.971Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:32.972Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:33.574 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:33.496+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:33.498015+0000) 2023-08-28T02:22:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:34.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:34.498204+0000) 2023-08-28T02:22:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: pgmap v9869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2159140477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:34.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: pgmap v9869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2159140477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: pgmap v9869: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2159140477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:22:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:22:34.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:22:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:22:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:22:35] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:22:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:35.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:35.498460+0000) 2023-08-28T02:22:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:36 smithi136 ceph-mon[113531]: pgmap v9870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/173431550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:36.497+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:36.498676+0000) 2023-08-28T02:22:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:36 smithi093 ceph-mon[111335]: pgmap v9870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/173431550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:36 smithi093 ceph-mon[118902]: pgmap v9870: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/173431550' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:37.838 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:37.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:37.498913+0000) 2023-08-28T02:22:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:37 smithi136 ceph-mon[113531]: pgmap v9871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:37 smithi093 ceph-mon[111335]: pgmap v9871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:37 smithi093 ceph-mon[118902]: pgmap v9871: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:38.840 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:38.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:38.499149+0000) 2023-08-28T02:22:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2674904612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2674904612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2674904612' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:39.845 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:39.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:39.499408+0000) 2023-08-28T02:22:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:39 smithi136 ceph-mon[113531]: pgmap v9872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:39 smithi093 ceph-mon[111335]: pgmap v9872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:39 smithi093 ceph-mon[118902]: pgmap v9872: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:40.914 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:40.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:40.499660+0000) 2023-08-28T02:22:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2783746171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2783746171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2783746171' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:41.918 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:41.544Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:41.918 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:41.545Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:41.919 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:41.498+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:41.499906+0000) 2023-08-28T02:22:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:41 smithi136 ceph-mon[113531]: pgmap v9873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:42.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:41 smithi093 ceph-mon[118902]: pgmap v9873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:41 smithi093 ceph-mon[111335]: pgmap v9873: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:42.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:42.500166+0000) 2023-08-28T02:22:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:42.971Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:43.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:42.972Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4144907178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:43 smithi136 ceph-mon[113531]: pgmap v9874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:43.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:43.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:43.500408+0000) 2023-08-28T02:22:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4144907178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:43 smithi093 ceph-mon[111335]: pgmap v9874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4144907178' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:43.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:43 smithi093 ceph-mon[118902]: pgmap v9874: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:44.499+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:44.500653+0000) 2023-08-28T02:22:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:45 smithi136 ceph-mon[113531]: pgmap v9875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:45.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:45.500889+0000) 2023-08-28T02:22:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:45 smithi093 ceph-mon[111335]: pgmap v9875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:22:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:22:45] "GET /metrics HTTP/1.1" 200 239952 "" "Prometheus/2.43.0" 2023-08-28T02:22:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:45 smithi093 ceph-mon[118902]: pgmap v9875: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4216022871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:46.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:46.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:46.501149+0000) 2023-08-28T02:22:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4216022871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:46.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4216022871' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:47.849 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:47.500+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:47.501368+0000) 2023-08-28T02:22:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:47 smithi136 ceph-mon[113531]: pgmap v9876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:47 smithi093 ceph-mon[111335]: pgmap v9876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:47 smithi093 ceph-mon[118902]: pgmap v9876: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:48.851 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:48.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:48.501596+0000) 2023-08-28T02:22:49.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/129530877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/129530877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/129530877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:49.855 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:49.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:49.501849+0000) 2023-08-28T02:22:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:49 smithi136 ceph-mon[113531]: pgmap v9877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:49 smithi093 ceph-mon[111335]: pgmap v9877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:49 smithi093 ceph-mon[118902]: pgmap v9877: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:50.857 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:50.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:50.502148+0000) 2023-08-28T02:22:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2456625436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2456625436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2456625436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:51.860 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:51.545Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:51.860 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:51.546Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:51.861 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:51.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:51.502383+0000) 2023-08-28T02:22:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:51 smithi136 ceph-mon[113531]: pgmap v9878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:51 smithi093 ceph-mon[111335]: pgmap v9878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:51 smithi093 ceph-mon[118902]: pgmap v9878: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:52.862 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:52.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:52.502642+0000) 2023-08-28T02:22:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4192463342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4192463342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4192463342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:52.972Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:22:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:22:52.973Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:22:53.869 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:53.501+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:53.502895+0000) 2023-08-28T02:22:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:53 smithi136 ceph-mon[113531]: pgmap v9879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:53 smithi093 ceph-mon[118902]: pgmap v9879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:53 smithi093 ceph-mon[111335]: pgmap v9879: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:54.870 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:54.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:54.503185+0000) 2023-08-28T02:22:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:22:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:22:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:22:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:22:55.877 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:55.502+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:55.503424+0000) 2023-08-28T02:22:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/435298783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:55 smithi136 ceph-mon[113531]: pgmap v9880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/435298783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:55 smithi093 ceph-mon[111335]: pgmap v9880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/435298783' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:55 smithi093 ceph-mon[118902]: pgmap v9880: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:56.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:56.503627+0000) 2023-08-28T02:22:57.845 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:57.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:57.503854+0000) 2023-08-28T02:22:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:57 smithi136 ceph-mon[113531]: pgmap v9881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/761346537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:57 smithi093 ceph-mon[111335]: pgmap v9881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:58.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/761346537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:57 smithi093 ceph-mon[118902]: pgmap v9881: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/761346537' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:22:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:58.503+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:58.504133+0000) 2023-08-28T02:22:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:22:59 smithi136 ceph-mon[113531]: pgmap v9882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:22:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:22:59.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:22:59.504385+0000) 2023-08-28T02:22:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:22:59 smithi093 ceph-mon[111335]: pgmap v9882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:22:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:22:59 smithi093 ceph-mon[118902]: pgmap v9882: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2246050357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:00.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:00.504646+0000) 2023-08-28T02:23:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2246050357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2246050357' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:01 smithi136 ceph-mon[113531]: pgmap v9883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:01.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:01.504860+0000) 2023-08-28T02:23:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:01 smithi093 ceph-mon[111335]: pgmap v9883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:01 smithi093 ceph-mon[118902]: pgmap v9883: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:01.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:01.546Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:01.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:01.547Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:02.839 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:02.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:02.505145+0000) 2023-08-28T02:23:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/27453595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/27453595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/27453595' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:02.973Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:02.974Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:03.844 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:03.504+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:03.505315+0000) 2023-08-28T02:23:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:03 smithi136 ceph-mon[113531]: pgmap v9884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:03 smithi093 ceph-mon[111335]: pgmap v9884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:04.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:03 smithi093 ceph-mon[118902]: pgmap v9884: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:04.846 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:04.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:04.505551+0000) 2023-08-28T02:23:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/90365154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/90365154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/90365154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:23:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:23:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:23:05.850 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:05.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:05.505794+0000) 2023-08-28T02:23:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:05 smithi136 ceph-mon[113531]: pgmap v9885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:05 smithi093 ceph-mon[111335]: pgmap v9885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:06.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:05 smithi093 ceph-mon[118902]: pgmap v9885: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:06.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:06.506114+0000) 2023-08-28T02:23:07.845 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:07.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:07.506327+0000) 2023-08-28T02:23:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/188562628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:07 smithi136 ceph-mon[113531]: pgmap v9886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/188562628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:07 smithi093 ceph-mon[111335]: pgmap v9886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/188562628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:07 smithi093 ceph-mon[118902]: pgmap v9886: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:08.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:08.506553+0000) 2023-08-28T02:23:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:09.505+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:09.506804+0000) 2023-08-28T02:23:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:09 smithi136 ceph-mon[113531]: pgmap v9887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2434932183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:09 smithi093 ceph-mon[111335]: pgmap v9887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2434932183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:09 smithi093 ceph-mon[118902]: pgmap v9887: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2434932183' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:10.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:10.507094+0000) 2023-08-28T02:23:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:11 smithi136 ceph-mon[113531]: pgmap v9888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:11.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:11.507320+0000) 2023-08-28T02:23:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:11 smithi093 ceph-mon[111335]: pgmap v9888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:11 smithi093 ceph-mon[118902]: pgmap v9888: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:11.546Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:11.547Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:12.840 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:12.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:12.507522+0000) 2023-08-28T02:23:13.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2846594811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2846594811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2846594811' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:12.973Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:12.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:13.845 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:13.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:13.507692+0000) 2023-08-28T02:23:14.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:13 smithi136 ceph-mon[113531]: pgmap v9889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:13 smithi093 ceph-mon[111335]: pgmap v9889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:14.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:13 smithi093 ceph-mon[118902]: pgmap v9889: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:14.847 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:14.506+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:14.507894+0000) 2023-08-28T02:23:15.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3517424697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3517424697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3517424697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:23:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:23:15] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:23:15.852 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:15.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:15.508169+0000) 2023-08-28T02:23:16.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:15 smithi136 ceph-mon[113531]: pgmap v9890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:15 smithi093 ceph-mon[111335]: pgmap v9890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:15 smithi093 ceph-mon[118902]: pgmap v9890: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:16.855 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:16.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:16.508388+0000) 2023-08-28T02:23:17.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2670823744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:17.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2670823744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:17.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2670823744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:17.858 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:17.507+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:17.508656+0000) 2023-08-28T02:23:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:17 smithi136 ceph-mon[113531]: pgmap v9891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:17 smithi093 ceph-mon[111335]: pgmap v9891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:17 smithi093 ceph-mon[118902]: pgmap v9891: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:18.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:18.508925+0000) 2023-08-28T02:23:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2114024067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2114024067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2114024067' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:19.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:19.509158+0000) 2023-08-28T02:23:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:20 smithi136 ceph-mon[113531]: pgmap v9892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:20 smithi093 ceph-mon[111335]: pgmap v9892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:20 smithi093 ceph-mon[118902]: pgmap v9892: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:20.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:20.509381+0000) 2023-08-28T02:23:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:21 smithi136 ceph-mon[113531]: pgmap v9893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:21.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:21.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:21.509627+0000) 2023-08-28T02:23:21.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:21 smithi093 ceph-mon[111335]: pgmap v9893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:21.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:21 smithi093 ceph-mon[118902]: pgmap v9893: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:21.547Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:21.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:21.548Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/831523037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:22.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:22.508+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:22.509890+0000) 2023-08-28T02:23:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/831523037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:22.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/831523037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:22.974Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:22.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:23 smithi136 ceph-mon[113531]: pgmap v9894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:23.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:23.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:23.510151+0000) 2023-08-28T02:23:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:23 smithi093 ceph-mon[111335]: pgmap v9894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:23 smithi093 ceph-mon[118902]: pgmap v9894: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2096031831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:24.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:24.510421+0000) 2023-08-28T02:23:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2096031831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2096031831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:25 smithi136 ceph-mon[113531]: pgmap v9895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:25.509+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:25.510710+0000) 2023-08-28T02:23:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:25 smithi093 ceph-mon[111335]: pgmap v9895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:25 smithi093 ceph-mon[118902]: pgmap v9895: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:23:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:23:25] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:23:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2001733117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:26.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:26.510948+0000) 2023-08-28T02:23:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2001733117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2001733117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:27.851 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:27.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:27.511207+0000) 2023-08-28T02:23:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:27 smithi136 ceph-mon[113531]: pgmap v9896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:27 smithi093 ceph-mon[111335]: pgmap v9896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:27 smithi093 ceph-mon[118902]: pgmap v9896: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:28.852 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:28.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:28.511540+0000) 2023-08-28T02:23:29.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2762901162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:29.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2762901162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2762901162' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:29.857 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:29.510+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:29.511754+0000) 2023-08-28T02:23:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:29 smithi136 ceph-mon[113531]: pgmap v9897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:29 smithi093 ceph-mon[111335]: pgmap v9897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:29 smithi093 ceph-mon[118902]: pgmap v9897: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:30.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:30.512071+0000) 2023-08-28T02:23:31.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2849874788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:31.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2849874788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2849874788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:31.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:31.512353+0000) 2023-08-28T02:23:31.976 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:31.547Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:31.976 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:31.548Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:31 smithi093 ceph-mon[111335]: pgmap v9898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:32.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:31 smithi093 ceph-mon[118902]: pgmap v9898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:31 smithi136 ceph-mon[113531]: pgmap v9898: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:32.511+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:32.512605+0000) 2023-08-28T02:23:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:32.974Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:33.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:32.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:33 smithi136 ceph-mon[113531]: pgmap v9899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2806356591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:33.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:33.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:33.512864+0000) 2023-08-28T02:23:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:33 smithi093 ceph-mon[111335]: pgmap v9899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2806356591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:33 smithi093 ceph-mon[118902]: pgmap v9899: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2806356591' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:34 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:34.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:34.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:34.513059+0000) 2023-08-28T02:23:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:34 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:34 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:23:35.691 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:23:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:23:35] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:23:35.863 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:35.863 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:35 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:35.863 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:35 smithi136 ceph-mon[113531]: pgmap v9900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:35.863 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:35.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:35.513316+0000) 2023-08-28T02:23:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:35.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:35 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:35.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:35 smithi093 ceph-mon[111335]: pgmap v9900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:35 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:35.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:35 smithi093 ceph-mon[118902]: pgmap v9900: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:36.631 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:36.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:36.513480+0000) 2023-08-28T02:23:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2964680902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:36.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:36 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2964680902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:36.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:36 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2964680902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:36.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:36 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:37.854 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:37.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:37.513698+0000) 2023-08-28T02:23:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:37 smithi136 ceph-mon[113531]: pgmap v9901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[111335]: pgmap v9901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[118902]: pgmap v9901: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:23:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:23:38.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:23:38.863 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:38.512+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:38.513968+0000) 2023-08-28T02:23:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2422500650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2422500650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2422500650' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:39.870 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:39.513+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:39.514218+0000) 2023-08-28T02:23:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:39 smithi136 ceph-mon[113531]: pgmap v9902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:39 smithi093 ceph-mon[111335]: pgmap v9902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:39 smithi093 ceph-mon[118902]: pgmap v9902: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:40.871 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:40.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:40.514463+0000) 2023-08-28T02:23:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4156326556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4156326556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4156326556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:41.879 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:41.549Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:41.879 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:41.550Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:41.880 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:41.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:41.514705+0000) 2023-08-28T02:23:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:41 smithi136 ceph-mon[113531]: pgmap v9903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:41 smithi093 ceph-mon[111335]: pgmap v9903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:41 smithi093 ceph-mon[118902]: pgmap v9903: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:42.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:42.514954+0000) 2023-08-28T02:23:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2086275558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2086275558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:42.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:42.975Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2086275558' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:43.514+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:43.515209+0000) 2023-08-28T02:23:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:44 smithi136 ceph-mon[113531]: pgmap v9904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:44 smithi093 ceph-mon[111335]: pgmap v9904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:44 smithi093 ceph-mon[118902]: pgmap v9904: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:44.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:44.515487+0000) 2023-08-28T02:23:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:45.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:45.515755+0000) 2023-08-28T02:23:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:45 smithi136 ceph-mon[113531]: pgmap v9905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1763193655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:23:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:23:45] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:23:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:45 smithi093 ceph-mon[111335]: pgmap v9905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1763193655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:45 smithi093 ceph-mon[118902]: pgmap v9905: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1763193655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:46.515+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:46.516019+0000) 2023-08-28T02:23:47.851 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:47.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:47.516307+0000) 2023-08-28T02:23:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:47 smithi136 ceph-mon[113531]: pgmap v9906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3583267621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:47 smithi093 ceph-mon[111335]: pgmap v9906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3583267621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:47 smithi093 ceph-mon[118902]: pgmap v9906: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3583267621' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:48.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:48.516580+0000) 2023-08-28T02:23:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:49 smithi136 ceph-mon[113531]: pgmap v9907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:49.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:49.516832+0000) 2023-08-28T02:23:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:49 smithi093 ceph-mon[111335]: pgmap v9907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:49 smithi093 ceph-mon[118902]: pgmap v9907: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1848353021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:50.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:50.517079+0000) 2023-08-28T02:23:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1848353021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1848353021' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:51 smithi136 ceph-mon[113531]: pgmap v9908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:51.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:51.517231+0000) 2023-08-28T02:23:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:51 smithi093 ceph-mon[111335]: pgmap v9908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:51 smithi093 ceph-mon[118902]: pgmap v9908: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:51.549Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:51.550Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:52.845 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:52.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:52.517457+0000) 2023-08-28T02:23:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3889106039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3889106039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3889106039' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:52.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:23:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:23:52.976Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:23:53.850 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:53.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:53.517686+0000) 2023-08-28T02:23:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:53 smithi136 ceph-mon[113531]: pgmap v9909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:53 smithi093 ceph-mon[111335]: pgmap v9909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:53 smithi093 ceph-mon[118902]: pgmap v9909: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:54.851 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:54.516+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:54.517957+0000) 2023-08-28T02:23:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:55.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:23:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:23:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:23:55] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:23:55.858 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:55.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:55.518242+0000) 2023-08-28T02:23:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2098349895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:55 smithi136 ceph-mon[113531]: pgmap v9910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2098349895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:56.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:55 smithi093 ceph-mon[118902]: pgmap v9910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2098349895' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:55 smithi093 ceph-mon[111335]: pgmap v9910: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:56.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:56.518499+0000) 2023-08-28T02:23:57.852 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:57.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:57.518720+0000) 2023-08-28T02:23:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3808126767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:57 smithi136 ceph-mon[113531]: pgmap v9911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3808126767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:57 smithi093 ceph-mon[111335]: pgmap v9911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3808126767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:23:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:57 smithi093 ceph-mon[118902]: pgmap v9911: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:58.517+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:58.518984+0000) 2023-08-28T02:23:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:23:59 smithi136 ceph-mon[113531]: pgmap v9912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:23:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:23:59.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:23:59.519267+0000) 2023-08-28T02:23:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:23:59 smithi093 ceph-mon[111335]: pgmap v9912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:23:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:23:59 smithi093 ceph-mon[118902]: pgmap v9912: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/399860095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:00.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:00.519496+0000) 2023-08-28T02:24:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/399860095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/399860095' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:01 smithi136 ceph-mon[113531]: pgmap v9913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:01.518+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:01.519778+0000) 2023-08-28T02:24:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:01 smithi093 ceph-mon[111335]: pgmap v9913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:01 smithi093 ceph-mon[118902]: pgmap v9913: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:01.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:01.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:02.848 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:02.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:02.520075+0000) 2023-08-28T02:24:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2325421068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2325421068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2325421068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:02.975Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:02.976Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:03.854 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:03.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:03.520289+0000) 2023-08-28T02:24:04.187 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:03 smithi136 ceph-mon[113531]: pgmap v9914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:03 smithi093 ceph-mon[111335]: pgmap v9914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:03 smithi093 ceph-mon[118902]: pgmap v9914: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:04.854 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:04.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:04.520521+0000) 2023-08-28T02:24:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3695635470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3695635470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3695635470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:24:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:24:05] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:24:05.861 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:05.519+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:05.520755+0000) 2023-08-28T02:24:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:05 smithi136 ceph-mon[113531]: pgmap v9915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:05 smithi093 ceph-mon[111335]: pgmap v9915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:05 smithi093 ceph-mon[118902]: pgmap v9915: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:06.864 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:06.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:06.521047+0000) 2023-08-28T02:24:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/579280899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/579280899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/579280899' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:07.871 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:07.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:07.521331+0000) 2023-08-28T02:24:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:07 smithi136 ceph-mon[113531]: pgmap v9916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:07 smithi093 ceph-mon[111335]: pgmap v9916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:08.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:07 smithi093 ceph-mon[118902]: pgmap v9916: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:08.520+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:08.521566+0000) 2023-08-28T02:24:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:09.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:09.521808+0000) 2023-08-28T02:24:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2527481907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:09 smithi136 ceph-mon[113531]: pgmap v9917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2527481907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:09 smithi093 ceph-mon[111335]: pgmap v9917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2527481907' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:09 smithi093 ceph-mon[118902]: pgmap v9917: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:10.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:10.522101+0000) 2023-08-28T02:24:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:11 smithi136 ceph-mon[113531]: pgmap v9918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:11.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:11.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:11.522345+0000) 2023-08-28T02:24:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:11 smithi093 ceph-mon[111335]: pgmap v9918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:11.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:11 smithi093 ceph-mon[118902]: pgmap v9918: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:11.550Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:11.551Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:12.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1078457571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:12.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:12.521+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:12.522602+0000) 2023-08-28T02:24:12.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1078457571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:12.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1078457571' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:12.976Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:12.977Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:13 smithi136 ceph-mon[113531]: pgmap v9919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:13.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:13.522851+0000) 2023-08-28T02:24:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:13 smithi093 ceph-mon[111335]: pgmap v9919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:13 smithi093 ceph-mon[118902]: pgmap v9919: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1747282985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:14.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:14.523086+0000) 2023-08-28T02:24:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1747282985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1747282985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:15 smithi136 ceph-mon[113531]: pgmap v9920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:15.522+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:15.523336+0000) 2023-08-28T02:24:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:15 smithi093 ceph-mon[118902]: pgmap v9920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:15 smithi093 ceph-mon[111335]: pgmap v9920: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:24:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:24:15] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:24:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2095621940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:16.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:16.523594+0000) 2023-08-28T02:24:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2095621940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2095621940' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:17.855 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:17.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:17.523829+0000) 2023-08-28T02:24:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:17 smithi136 ceph-mon[113531]: pgmap v9921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:17 smithi093 ceph-mon[118902]: pgmap v9921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:18.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:17 smithi093 ceph-mon[111335]: pgmap v9921: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:18.854 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:18.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:18.524115+0000) 2023-08-28T02:24:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3206569986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3206569986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3206569986' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:19.861 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:19.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:19.524372+0000) 2023-08-28T02:24:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:19 smithi136 ceph-mon[113531]: pgmap v9922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:19 smithi093 ceph-mon[111335]: pgmap v9922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:19 smithi093 ceph-mon[118902]: pgmap v9922: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:20.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:20.524620+0000) 2023-08-28T02:24:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1886100737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1886100737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1886100737' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:21.523+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:21.524872+0000) 2023-08-28T02:24:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:21.551Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:21.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:22 smithi136 ceph-mon[113531]: pgmap v9923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:22 smithi093 ceph-mon[111335]: pgmap v9923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:22 smithi093 ceph-mon[118902]: pgmap v9923: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:22.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:22.525147+0000) 2023-08-28T02:24:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:22.977Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:23.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:22.978Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:23 smithi136 ceph-mon[113531]: pgmap v9924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:23.715 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:23.524+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:23.525421+0000) 2023-08-28T02:24:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:23 smithi093 ceph-mon[111335]: pgmap v9924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:23 smithi093 ceph-mon[118902]: pgmap v9924: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/487699751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:24.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:24.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:24.525645+0000) 2023-08-28T02:24:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/487699751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/487699751' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:25 smithi136 ceph-mon[113531]: pgmap v9925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:25.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:25.525901+0000) 2023-08-28T02:24:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:25 smithi093 ceph-mon[111335]: pgmap v9925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:25 smithi093 ceph-mon[118902]: pgmap v9925: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:24:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:24:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:24:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/98482637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:26.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:26.526160+0000) 2023-08-28T02:24:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/98482637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/98482637' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:27.854 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:27.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:27.526392+0000) 2023-08-28T02:24:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:27 smithi136 ceph-mon[113531]: pgmap v9926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:27 smithi093 ceph-mon[111335]: pgmap v9926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:28.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:27 smithi093 ceph-mon[118902]: pgmap v9926: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:28.856 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:28.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:28.526665+0000) 2023-08-28T02:24:29.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1468572349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:29.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1468572349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:29.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1468572349' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:29.862 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:29.525+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:29.526937+0000) 2023-08-28T02:24:30.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:29 smithi136 ceph-mon[113531]: pgmap v9927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:29 smithi093 ceph-mon[111335]: pgmap v9927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:29 smithi093 ceph-mon[118902]: pgmap v9927: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:30.863 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:30.526+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:30.527145+0000) 2023-08-28T02:24:31.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3549205673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:31.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3549205673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3549205673' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:31.866 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:31.551Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:31.866 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:31.552Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:31.866 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:31.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:31.527365+0000) 2023-08-28T02:24:32.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:31 smithi136 ceph-mon[113531]: pgmap v9928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:32.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:31 smithi093 ceph-mon[111335]: pgmap v9928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:31 smithi093 ceph-mon[118902]: pgmap v9928: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:32.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:32.527618+0000) 2023-08-28T02:24:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3803015099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3803015099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:32.978Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:32.979Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3803015099' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:33.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:33.527851+0000) 2023-08-28T02:24:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:34 smithi136 ceph-mon[113531]: pgmap v9929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:34 smithi093 ceph-mon[111335]: pgmap v9929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:34.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:34 smithi093 ceph-mon[118902]: pgmap v9929: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:34.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:34.528129+0000) 2023-08-28T02:24:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:35 smithi136 ceph-mon[113531]: pgmap v9930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:35.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:35.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:35.528358+0000) 2023-08-28T02:24:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:35 smithi093 ceph-mon[111335]: pgmap v9930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:24:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:24:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:24:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:35 smithi093 ceph-mon[118902]: pgmap v9930: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2604956593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:36.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:36.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:36.528629+0000) 2023-08-28T02:24:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2604956593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:36.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2604956593' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:37.856 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:37.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:37.528809+0000) 2023-08-28T02:24:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:37 smithi136 ceph-mon[113531]: pgmap v9931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:37 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2058179417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[111335]: pgmap v9931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:38.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2058179417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[118902]: pgmap v9931: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:24:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:38.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2058179417' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:38.685 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:38.527+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:38.529004+0000) 2023-08-28T02:24:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:39.782 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:39.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:39.529169+0000) 2023-08-28T02:24:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[111335]: pgmap v9932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[118902]: pgmap v9932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:40 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:40 smithi136 ceph-mon[113531]: pgmap v9932: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:40 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:40.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:40.529407+0000) 2023-08-28T02:24:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/181522608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/181522608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/181522608' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:41.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:41.529693+0000) 2023-08-28T02:24:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:41.552Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:41.554Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:42 smithi136 ceph-mon[113531]: pgmap v9933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:24:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:24:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:42 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[111335]: pgmap v9933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[118902]: pgmap v9933: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:24:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:24:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:42.528+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:42.529988+0000) 2023-08-28T02:24:43.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2677105833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2677105833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:42.978Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:43.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:42.980Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2677105833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:43.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:43.530302+0000) 2023-08-28T02:24:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:44 smithi136 ceph-mon[113531]: pgmap v9934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:44.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:44 smithi093 ceph-mon[118902]: pgmap v9934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:44.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:44 smithi093 ceph-mon[111335]: pgmap v9934: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:44.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:44.530584+0000) 2023-08-28T02:24:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3453808123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:45.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3453808123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3453808123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:24:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:24:45] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:24:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:45.529+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:45.530875+0000) 2023-08-28T02:24:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:46 smithi136 ceph-mon[113531]: pgmap v9935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:46 smithi093 ceph-mon[111335]: pgmap v9935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:46 smithi093 ceph-mon[118902]: pgmap v9935: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:46.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:46.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:46.531123+0000) 2023-08-28T02:24:47.858 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:47.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:47.531384+0000) 2023-08-28T02:24:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:47 smithi136 ceph-mon[113531]: pgmap v9936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/731389531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:47 smithi093 ceph-mon[111335]: pgmap v9936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/731389531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:47 smithi093 ceph-mon[118902]: pgmap v9936: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/731389531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:48.530+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:48.531693+0000) 2023-08-28T02:24:49.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:49.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:49.531921+0000) 2023-08-28T02:24:49.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:49 smithi136 ceph-mon[113531]: pgmap v9937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:49 smithi093 ceph-mon[111335]: pgmap v9937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:49 smithi093 ceph-mon[118902]: pgmap v9937: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3102965316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:50.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:50.532179+0000) 2023-08-28T02:24:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3102965316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3102965316' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:51 smithi136 ceph-mon[113531]: pgmap v9938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:51.531+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:51.532469+0000) 2023-08-28T02:24:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:51 smithi093 ceph-mon[111335]: pgmap v9938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:51.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:51 smithi093 ceph-mon[118902]: pgmap v9938: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:51.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:51.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:52.858 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:52.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:52.532784+0000) 2023-08-28T02:24:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3594933361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3594933361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3594933361' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:52.979Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:24:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:24:52.980Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:24:53.864 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:53.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:53.533049+0000) 2023-08-28T02:24:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:53 smithi136 ceph-mon[113531]: pgmap v9939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:53 smithi093 ceph-mon[111335]: pgmap v9939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:54.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:53 smithi093 ceph-mon[118902]: pgmap v9939: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:54.865 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:54.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:54.533293+0000) 2023-08-28T02:24:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3105599299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3105599299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:24:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3105599299' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:24:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:24:55] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:24:55.873 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:55.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:55.533454+0000) 2023-08-28T02:24:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:55 smithi136 ceph-mon[113531]: pgmap v9940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:55 smithi093 ceph-mon[118902]: pgmap v9940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:56.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:55 smithi093 ceph-mon[111335]: pgmap v9940: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:56.876 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:56.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:56.533710+0000) 2023-08-28T02:24:57.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3339287204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3339287204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:57.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3339287204' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:57.881 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:57.532+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:57.533901+0000) 2023-08-28T02:24:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:57 smithi136 ceph-mon[113531]: pgmap v9941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:57 smithi093 ceph-mon[111335]: pgmap v9941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:58.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:57 smithi093 ceph-mon[118902]: pgmap v9941: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:24:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:58.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:58.534235+0000) 2023-08-28T02:24:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:24:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2009767689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:59.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:24:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:24:59.533+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:24:59.534546+0000) 2023-08-28T02:24:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:24:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2009767689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:24:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:24:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2009767689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:00 smithi136 ceph-mon[113531]: pgmap v9942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:00.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:00.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:00.534821+0000) 2023-08-28T02:25:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:00 smithi093 ceph-mon[111335]: pgmap v9942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:00 smithi093 ceph-mon[118902]: pgmap v9942: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:01 smithi136 ceph-mon[113531]: pgmap v9943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:01.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:01.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:01.535018+0000) 2023-08-28T02:25:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:01 smithi093 ceph-mon[111335]: pgmap v9943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:01.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:01 smithi093 ceph-mon[118902]: pgmap v9943: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:01.554Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:01.555Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:02.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4108971307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:02.690 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:02.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:02.535294+0000) 2023-08-28T02:25:02.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4108971307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:02.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4108971307' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:02.980Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:02.981Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:03 smithi136 ceph-mon[113531]: pgmap v9944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:03.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:03.534+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:03.535560+0000) 2023-08-28T02:25:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:03 smithi093 ceph-mon[111335]: pgmap v9944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:03.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:03 smithi093 ceph-mon[118902]: pgmap v9944: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:04.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3529978978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:04.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:04.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:04.535812+0000) 2023-08-28T02:25:04.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3529978978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:04.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3529978978' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:05 smithi136 ceph-mon[113531]: pgmap v9945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:05.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:05.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:05.536130+0000) 2023-08-28T02:25:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:05 smithi093 ceph-mon[111335]: pgmap v9945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:05 smithi093 ceph-mon[118902]: pgmap v9945: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:25:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:25:05] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:25:06.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3456399159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:06.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:06.535+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:06.536383+0000) 2023-08-28T02:25:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3456399159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3456399159' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:07.862 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:07.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:07.536603+0000) 2023-08-28T02:25:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:07 smithi136 ceph-mon[113531]: pgmap v9946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:07 smithi093 ceph-mon[111335]: pgmap v9946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:07 smithi093 ceph-mon[118902]: pgmap v9946: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:08.864 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:08.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:08.536887+0000) 2023-08-28T02:25:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1816913880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1816913880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1816913880' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:09.869 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:09.536+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:09.537166+0000) 2023-08-28T02:25:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:09 smithi136 ceph-mon[113531]: pgmap v9947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:09 smithi093 ceph-mon[111335]: pgmap v9947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:09 smithi093 ceph-mon[118902]: pgmap v9947: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:10.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:10.537438+0000) 2023-08-28T02:25:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111456617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111456617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:11.536 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111456617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:11.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:11.537710+0000) 2023-08-28T02:25:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:11.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:11.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:12 smithi093 ceph-mon[111335]: pgmap v9948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:12.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:12 smithi093 ceph-mon[118902]: pgmap v9948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:12.536 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:12 smithi136 ceph-mon[113531]: pgmap v9948: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:12.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:12.537926+0000) 2023-08-28T02:25:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:12.981Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:12.982Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:13.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:13 smithi136 ceph-mon[113531]: pgmap v9949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:13.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:13.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:13.538185+0000) 2023-08-28T02:25:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:13 smithi093 ceph-mon[111335]: pgmap v9949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:13.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:13 smithi093 ceph-mon[118902]: pgmap v9949: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:14.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1858536898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:14.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:14.537+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:14.538436+0000) 2023-08-28T02:25:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1858536898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1858536898' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:15.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:15 smithi136 ceph-mon[113531]: pgmap v9950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:15.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:15.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:15.538668+0000) 2023-08-28T02:25:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:15 smithi093 ceph-mon[111335]: pgmap v9950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:15 smithi093 ceph-mon[118902]: pgmap v9950: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:25:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:25:15] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:25:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2969288744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:16.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:16.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:16.538923+0000) 2023-08-28T02:25:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2969288744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2969288744' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:17.864 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:17.538+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:17.539200+0000) 2023-08-28T02:25:18.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:17 smithi136 ceph-mon[113531]: pgmap v9951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:17 smithi093 ceph-mon[111335]: pgmap v9951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:18.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:17 smithi093 ceph-mon[118902]: pgmap v9951: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:18.865 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:18.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:18.539500+0000) 2023-08-28T02:25:19.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3797776075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3797776075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:19.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3797776075' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:19.872 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:19.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:19.539722+0000) 2023-08-28T02:25:20.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:19 smithi136 ceph-mon[113531]: pgmap v9952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:19 smithi093 ceph-mon[111335]: pgmap v9952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:20.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:19 smithi093 ceph-mon[118902]: pgmap v9952: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:20.873 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:20.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:20.540010+0000) 2023-08-28T02:25:21.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/934884755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/934884755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/934884755' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:21.877 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:21.556Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:21.877 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:21.557Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:21.877 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:21.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:21.540285+0000) 2023-08-28T02:25:22.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:21 smithi136 ceph-mon[113531]: pgmap v9953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:21 smithi093 ceph-mon[111335]: pgmap v9953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:22.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:21 smithi093 ceph-mon[118902]: pgmap v9953: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:22.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:22.540528+0000) 2023-08-28T02:25:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2697698328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2697698328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:22.981Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:22.982Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2697698328' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:23.539+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:23.540780+0000) 2023-08-28T02:25:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:24 smithi136 ceph-mon[113531]: pgmap v9954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:24 smithi093 ceph-mon[111335]: pgmap v9954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:24 smithi093 ceph-mon[118902]: pgmap v9954: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:24.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:24.541105+0000) 2023-08-28T02:25:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:25 smithi136 ceph-mon[113531]: pgmap v9955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:25.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:25.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:25.541374+0000) 2023-08-28T02:25:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:25 smithi093 ceph-mon[111335]: pgmap v9955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:25 smithi093 ceph-mon[118902]: pgmap v9955: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:25:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:25:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:25:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3492149788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:26.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:26.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:26.541582+0000) 2023-08-28T02:25:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3492149788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3492149788' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:27.861 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:27.540+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:27.541865+0000) 2023-08-28T02:25:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:27 smithi136 ceph-mon[113531]: pgmap v9956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:28.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2916255490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:27 smithi093 ceph-mon[111335]: pgmap v9956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2916255490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:27 smithi093 ceph-mon[118902]: pgmap v9956: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2916255490' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:28.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:28.542193+0000) 2023-08-28T02:25:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:29 smithi136 ceph-mon[113531]: pgmap v9957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:29.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:29.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:29.542468+0000) 2023-08-28T02:25:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:29 smithi093 ceph-mon[111335]: pgmap v9957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:29 smithi093 ceph-mon[118902]: pgmap v9957: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3286473231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:30.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:30.542664+0000) 2023-08-28T02:25:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3286473231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3286473231' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:31.541+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:31.542909+0000) 2023-08-28T02:25:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:31.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:31.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:32.861 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:32.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:32.543164+0000) 2023-08-28T02:25:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:32 smithi136 ceph-mon[113531]: pgmap v9958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3245723675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:32 smithi093 ceph-mon[111335]: pgmap v9958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3245723675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:32 smithi093 ceph-mon[118902]: pgmap v9958: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3245723675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:32.982Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:32.983Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:33.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:33.543410+0000) 2023-08-28T02:25:34.864 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:34.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:34.543626+0000) 2023-08-28T02:25:35.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:34 smithi136 ceph-mon[113531]: pgmap v9959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:35.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:34 smithi093 ceph-mon[111335]: pgmap v9959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:34 smithi093 ceph-mon[118902]: pgmap v9959: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:25:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:25:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:25:35.874 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:35.542+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:35.543904+0000) 2023-08-28T02:25:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2834960784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2834960784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2834960784' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:36.876 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:36.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:36.544161+0000) 2023-08-28T02:25:37.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:36 smithi136 ceph-mon[113531]: pgmap v9960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:36 smithi093 ceph-mon[111335]: pgmap v9960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:37.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:36 smithi093 ceph-mon[118902]: pgmap v9960: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:37.883 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:37.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:37.544393+0000) 2023-08-28T02:25:38.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3412455985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:38.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3412455985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:38.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3412455985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:38.883 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:38.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:38.544653+0000) 2023-08-28T02:25:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:38 smithi136 ceph-mon[113531]: pgmap v9961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:38 smithi093 ceph-mon[111335]: pgmap v9961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:39.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:38 smithi093 ceph-mon[118902]: pgmap v9961: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:39.888 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:39.543+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:39.544894+0000) 2023-08-28T02:25:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3967833359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3967833359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3967833359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:40.888 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:40.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:40.545169+0000) 2023-08-28T02:25:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:40 smithi136 ceph-mon[113531]: pgmap v9962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:40 smithi093 ceph-mon[111335]: pgmap v9962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:40 smithi093 ceph-mon[118902]: pgmap v9962: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:41.893 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:41.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:41.545437+0000) 2023-08-28T02:25:41.894 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:41 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:25:41.894 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:41 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:25:41.895 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:41.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:41.895 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:41.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:42.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:41 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:25:42.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:42.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:42.545659+0000) 2023-08-28T02:25:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:42 smithi136 ceph-mon[113531]: pgmap v9963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3360332327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:42 smithi093 ceph-mon[111335]: pgmap v9963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3360332327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:43.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:42 smithi093 ceph-mon[118902]: pgmap v9963: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3360332327' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:42.983Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:42.984Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:43.544+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:43.545916+0000) 2023-08-28T02:25:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:44.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:44.546105+0000) 2023-08-28T02:25:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:44 smithi136 ceph-mon[113531]: pgmap v9964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:44 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/888465525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[111335]: pgmap v9964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:44.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/888465525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[118902]: pgmap v9964: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:44.990 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/888465525' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:45.612 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:25:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:25:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:25:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:45.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:45.546322+0000) 2023-08-28T02:25:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:45.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:45.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:25:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:25:45.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:25:46.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:46 smithi136 ceph-mon[113531]: pgmap v9965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:46.545+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:46.546504+0000) 2023-08-28T02:25:46.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:46 smithi093 ceph-mon[111335]: pgmap v9965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:46.988 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:46 smithi093 ceph-mon[118902]: pgmap v9965: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:47.878 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:47.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:47.546719+0000) 2023-08-28T02:25:48.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3914830061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3914830061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:48.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3914830061' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:48.880 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:48.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:48.546973+0000) 2023-08-28T02:25:49.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:48 smithi136 ceph-mon[113531]: pgmap v9966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:48 smithi093 ceph-mon[111335]: pgmap v9966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:48 smithi093 ceph-mon[118902]: pgmap v9966: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:49.883 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:49.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:49.547172+0000) 2023-08-28T02:25:50.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2359342523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2359342523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:50.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2359342523' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:50.886 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:50.546+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:50.547409+0000) 2023-08-28T02:25:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:50 smithi136 ceph-mon[113531]: pgmap v9967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:50 smithi093 ceph-mon[111335]: pgmap v9967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:50 smithi093 ceph-mon[118902]: pgmap v9967: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:51.893 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:51.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:51.893 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:51.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:51.894 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:51.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:51.547637+0000) 2023-08-28T02:25:52.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2605193475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:52.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2605193475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2605193475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:52.895 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:52.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:52.547865+0000) 2023-08-28T02:25:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:52 smithi136 ceph-mon[113531]: pgmap v9968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:52 smithi093 ceph-mon[111335]: pgmap v9968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:52 smithi093 ceph-mon[118902]: pgmap v9968: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:52.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:25:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:25:52.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:25:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:53.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:53.548146+0000) 2023-08-28T02:25:54.898 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:54.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:54.548379+0000) 2023-08-28T02:25:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:54 smithi136 ceph-mon[113531]: pgmap v9969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/874906239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:54 smithi093 ceph-mon[111335]: pgmap v9969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/874906239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:54 smithi093 ceph-mon[118902]: pgmap v9969: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:25:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/874906239' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:25:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:25:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:25:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:55.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:55.548599+0000) 2023-08-28T02:25:56.903 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:56.547+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:56.548868+0000) 2023-08-28T02:25:57.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:56 smithi136 ceph-mon[113531]: pgmap v9970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:57.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3198128731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:56 smithi093 ceph-mon[111335]: pgmap v9970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3198128731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:56 smithi093 ceph-mon[118902]: pgmap v9970: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3198128731' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:57.548+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:57.549107+0000) 2023-08-28T02:25:58.906 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:58.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:58.549339+0000) 2023-08-28T02:25:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:58 smithi136 ceph-mon[113531]: pgmap v9971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:25:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3288199356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:58 smithi093 ceph-mon[111335]: pgmap v9971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:25:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3288199356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:58 smithi093 ceph-mon[118902]: pgmap v9971: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:25:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:25:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3288199356' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:25:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:25:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:25:59.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:25:59.549589+0000) 2023-08-28T02:26:00.910 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:00.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:00.549787+0000) 2023-08-28T02:26:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:00 smithi136 ceph-mon[113531]: pgmap v9972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:00 smithi093 ceph-mon[111335]: pgmap v9972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:00 smithi093 ceph-mon[118902]: pgmap v9972: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:01.914 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:01.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:01.914 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:01.558Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:01.914 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:01.549+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:01.550067+0000) 2023-08-28T02:26:02.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2309104821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2309104821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:02.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2309104821' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:02.916 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:02.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:02.550280+0000) 2023-08-28T02:26:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:02 smithi136 ceph-mon[113531]: pgmap v9973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:02 smithi093 ceph-mon[111335]: pgmap v9973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:02 smithi093 ceph-mon[118902]: pgmap v9973: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:03.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:02.984Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:02.985Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:03.921 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:03.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:03.550545+0000) 2023-08-28T02:26:04.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2497935876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:04.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2497935876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:04.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2497935876' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:04.923 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:04.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:04.550792+0000) 2023-08-28T02:26:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:04 smithi136 ceph-mon[113531]: pgmap v9974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:04 smithi093 ceph-mon[111335]: pgmap v9974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:04 smithi093 ceph-mon[118902]: pgmap v9974: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:26:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:26:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:26:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:05.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:05.551067+0000) 2023-08-28T02:26:06.926 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:06.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:06.551306+0000) 2023-08-28T02:26:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:06 smithi136 ceph-mon[113531]: pgmap v9975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/849176398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:06 smithi093 ceph-mon[111335]: pgmap v9975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:07.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/849176398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:06 smithi093 ceph-mon[118902]: pgmap v9975: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/849176398' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:07.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:07.551552+0000) 2023-08-28T02:26:08.929 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:08.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:08.551789+0000) 2023-08-28T02:26:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:08 smithi136 ceph-mon[113531]: pgmap v9976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2844033434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:08 smithi093 ceph-mon[118902]: pgmap v9976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:09.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2844033434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:08 smithi093 ceph-mon[111335]: pgmap v9976: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2844033434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:09.934 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:09.550+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:09.551987+0000) 2023-08-28T02:26:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:10.935 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:10.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:10.552236+0000) 2023-08-28T02:26:11.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:10 smithi136 ceph-mon[113531]: pgmap v9977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:11.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1493557757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:10 smithi093 ceph-mon[118902]: pgmap v9977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1493557757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:10 smithi093 ceph-mon[111335]: pgmap v9977: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1493557757' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:11.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:11.552431+0000) 2023-08-28T02:26:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:11.557Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:11.559Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:12.551+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:12.552678+0000) 2023-08-28T02:26:13.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:12 smithi093 ceph-mon[118902]: pgmap v9978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:12 smithi093 ceph-mon[111335]: pgmap v9978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:12.985Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:13.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:12.986Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:12 smithi136 ceph-mon[113531]: pgmap v9978: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:13.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:13.552955+0000) 2023-08-28T02:26:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2958516948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2958516948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2958516948' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:14.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:14.553206+0000) 2023-08-28T02:26:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:14 smithi093 ceph-mon[111335]: pgmap v9979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:15.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:14 smithi093 ceph-mon[118902]: pgmap v9979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:14 smithi136 ceph-mon[113531]: pgmap v9979: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:26:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:26:15] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:26:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:15.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:15.553461+0000) 2023-08-28T02:26:16.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2894847779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:16.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2894847779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2894847779' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:16.552+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:16.553691+0000) 2023-08-28T02:26:17.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:16 smithi093 ceph-mon[111335]: pgmap v9980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:17.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:16 smithi093 ceph-mon[118902]: pgmap v9980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:16 smithi136 ceph-mon[113531]: pgmap v9980: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:17.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:17.553940+0000) 2023-08-28T02:26:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1286815794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1286815794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1286815794' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:18.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:18.554201+0000) 2023-08-28T02:26:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:18 smithi093 ceph-mon[111335]: pgmap v9981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:18 smithi093 ceph-mon[118902]: pgmap v9981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:18 smithi136 ceph-mon[113531]: pgmap v9981: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:19.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:19.554444+0000) 2023-08-28T02:26:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:20.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:20.554722+0000) 2023-08-28T02:26:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:20 smithi093 ceph-mon[111335]: pgmap v9982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/460598185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:20 smithi093 ceph-mon[118902]: pgmap v9982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:21.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/460598185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:20 smithi136 ceph-mon[113531]: pgmap v9982: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/460598185' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:21.553+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:21.554990+0000) 2023-08-28T02:26:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:21.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:21.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:22.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:22.555261+0000) 2023-08-28T02:26:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:22 smithi093 ceph-mon[111335]: pgmap v9983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3721093778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:22 smithi093 ceph-mon[118902]: pgmap v9983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3721093778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:22.986Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:22.987Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:22 smithi136 ceph-mon[113531]: pgmap v9983: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3721093778' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:23.554+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:23.555511+0000) 2023-08-28T02:26:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:24.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:24.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:24.555772+0000) 2023-08-28T02:26:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:24 smithi093 ceph-mon[111335]: pgmap v9984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:25.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:24 smithi093 ceph-mon[118902]: pgmap v9984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:24 smithi136 ceph-mon[113531]: pgmap v9984: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:26:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:26:25] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:26:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:25.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:25.555972+0000) 2023-08-28T02:26:26.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2794920350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:26.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2794920350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2794920350' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:26.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:26.556191+0000) 2023-08-28T02:26:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:26 smithi093 ceph-mon[111335]: pgmap v9985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:26 smithi093 ceph-mon[118902]: pgmap v9985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:26 smithi136 ceph-mon[113531]: pgmap v9985: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:27.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:27.556432+0000) 2023-08-28T02:26:28.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2220917022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:28.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2220917022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2220917022' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:28.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:28.556641+0000) 2023-08-28T02:26:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:28 smithi093 ceph-mon[111335]: pgmap v9986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:28 smithi093 ceph-mon[118902]: pgmap v9986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:28 smithi136 ceph-mon[113531]: pgmap v9986: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:29.555+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:29.556909+0000) 2023-08-28T02:26:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1262161132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1262161132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:30.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1262161132' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:30.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:30.557171+0000) 2023-08-28T02:26:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:30 smithi136 ceph-mon[113531]: pgmap v9987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:30 smithi093 ceph-mon[111335]: pgmap v9987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:31.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:30 smithi093 ceph-mon[118902]: pgmap v9987: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:31.556+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:31.557414+0000) 2023-08-28T02:26:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:31.559Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:31.560Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:32.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:32.557655+0000) 2023-08-28T02:26:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:32 smithi093 ceph-mon[111335]: pgmap v9988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/13694829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:33 smithi093 ceph-mon[118902]: pgmap v9988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/13694829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:32.987Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:32.988Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:33 smithi136 ceph-mon[113531]: pgmap v9988: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/13694829' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:33.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:33.557909+0000) 2023-08-28T02:26:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:34.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:34.558113+0000) 2023-08-28T02:26:35.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:35 smithi093 ceph-mon[111335]: pgmap v9989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:35.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4283027742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:35 smithi093 ceph-mon[118902]: pgmap v9989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4283027742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:35 smithi136 ceph-mon[113531]: pgmap v9989: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4283027742' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:26:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:26:35] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:26:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:35.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:35.558364+0000) 2023-08-28T02:26:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:36.557+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:36.558591+0000) 2023-08-28T02:26:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:37 smithi136 ceph-mon[113531]: pgmap v9990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/805474103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:37 smithi093 ceph-mon[111335]: pgmap v9990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/805474103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:37 smithi093 ceph-mon[118902]: pgmap v9990: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/805474103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:37.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:37.558839+0000) 2023-08-28T02:26:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:38.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:38.559099+0000) 2023-08-28T02:26:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:39 smithi136 ceph-mon[113531]: pgmap v9991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:39 smithi093 ceph-mon[111335]: pgmap v9991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:39 smithi093 ceph-mon[118902]: pgmap v9991: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:39.558+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:39.559347+0000) 2023-08-28T02:26:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3651277494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:40.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3651277494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:40.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3651277494' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:40.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:40.559627+0000) 2023-08-28T02:26:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:41 smithi136 ceph-mon[113531]: pgmap v9992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:41 smithi093 ceph-mon[111335]: pgmap v9992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:41 smithi093 ceph-mon[118902]: pgmap v9992: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:41.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:41.559895+0000) 2023-08-28T02:26:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:41.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:41.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3195913258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3195913258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3195913258' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:42.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:42.560140+0000) 2023-08-28T02:26:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:43 smithi093 ceph-mon[111335]: pgmap v9993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:43 smithi093 ceph-mon[118902]: pgmap v9993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:42.988Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:42.989Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:43 smithi136 ceph-mon[113531]: pgmap v9993: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:43.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:43.560419+0000) 2023-08-28T02:26:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:44.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:44.560613+0000) 2023-08-28T02:26:45.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:45 smithi093 ceph-mon[118902]: pgmap v9994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4065311475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:45 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:45 smithi093 ceph-mon[111335]: pgmap v9994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4065311475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:45 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:45.405 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:45 smithi136 ceph-mon[113531]: pgmap v9994: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:45.405 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4065311475' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:45.405 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:45 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:26:45.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:45.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:45.560800+0000) 2023-08-28T02:26:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:26:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:26:45] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:26:46.840 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:46.559+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:46.560960+0000) 2023-08-28T02:26:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:47 smithi136 ceph-mon[113531]: pgmap v9995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1193520078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:47.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:47 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[118902]: pgmap v9995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1193520078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[111335]: pgmap v9995: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1193520078' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:47.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:47 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:47.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:47.561110+0000) 2023-08-28T02:26:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:26:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:26:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:26:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:26:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:26:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:26:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:26:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:48.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:48.561360+0000) 2023-08-28T02:26:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:49 smithi136 ceph-mon[113531]: pgmap v9996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/31937020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:49 smithi093 ceph-mon[111335]: pgmap v9996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/31937020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:49 smithi093 ceph-mon[118902]: pgmap v9996: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/31937020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:49.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:49.561614+0000) 2023-08-28T02:26:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:50.560+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:50.561874+0000) 2023-08-28T02:26:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:51 smithi136 ceph-mon[113531]: pgmap v9997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:51 smithi093 ceph-mon[111335]: pgmap v9997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:51 smithi093 ceph-mon[118902]: pgmap v9997: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:51.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:51.562174+0000) 2023-08-28T02:26:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:51.560Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:51.561Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3729942449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3729942449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:52.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3729942449' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:52.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:52.562442+0000) 2023-08-28T02:26:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:53 smithi093 ceph-mon[111335]: pgmap v9998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:53 smithi093 ceph-mon[118902]: pgmap v9998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:52.989Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:26:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:26:52.990Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:26:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:53 smithi136 ceph-mon[113531]: pgmap v9998: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:53.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:53.562678+0000) 2023-08-28T02:26:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2954285724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:54.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2954285724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:54.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2954285724' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:26:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:54.561+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:54.562931+0000) 2023-08-28T02:26:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:55 smithi093 ceph-mon[111335]: pgmap v9999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:55 smithi093 ceph-mon[118902]: pgmap v9999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:55 smithi136 ceph-mon[113531]: pgmap v9999: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:26:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:26:55] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:26:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:55.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:55.563214+0000) 2023-08-28T02:26:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3757483549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3757483549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:56.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3757483549' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:56.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:56.563455+0000) 2023-08-28T02:26:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:57 smithi136 ceph-mon[113531]: pgmap v10000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:57 smithi093 ceph-mon[111335]: pgmap v10000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:57 smithi093 ceph-mon[118902]: pgmap v10000: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:57.562+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:57.563710+0000) 2023-08-28T02:26:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:58.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:58.563968+0000) 2023-08-28T02:26:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:59 smithi136 ceph-mon[113531]: pgmap v10001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:26:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/592418720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:59 smithi093 ceph-mon[111335]: pgmap v10001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:26:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/592418720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:59 smithi093 ceph-mon[118902]: pgmap v10001: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:26:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:26:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/592418720' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:26:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:26:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:26:59.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:26:59.564217+0000) 2023-08-28T02:27:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:00.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:00.564467+0000) 2023-08-28T02:27:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:01 smithi136 ceph-mon[113531]: pgmap v10002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3309811980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:01 smithi093 ceph-mon[111335]: pgmap v10002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3309811980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:01 smithi093 ceph-mon[118902]: pgmap v10002: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3309811980' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:01.563+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:01.564742+0000) 2023-08-28T02:27:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:01.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:01.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:02.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:02.564981+0000) 2023-08-28T02:27:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:03 smithi136 ceph-mon[113531]: pgmap v10003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:03 smithi093 ceph-mon[111335]: pgmap v10003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:03.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:03 smithi093 ceph-mon[118902]: pgmap v10003: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:02.989Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:02.990Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:03.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:03.565158+0000) 2023-08-28T02:27:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3603031781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3603031781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3603031781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:04.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:04.565382+0000) 2023-08-28T02:27:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:05 smithi093 ceph-mon[111335]: pgmap v10004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:05 smithi093 ceph-mon[118902]: pgmap v10004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:05 smithi136 ceph-mon[113531]: pgmap v10004: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:27:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:27:05] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:27:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:05.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:05.565668+0000) 2023-08-28T02:27:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/108650199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/108650199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:06.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/108650199' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:06.564+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:06.565987+0000) 2023-08-28T02:27:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:07 smithi136 ceph-mon[113531]: pgmap v10005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:07 smithi093 ceph-mon[111335]: pgmap v10005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:07 smithi093 ceph-mon[118902]: pgmap v10005: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:07.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:07.566208+0000) 2023-08-28T02:27:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1611152508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1611152508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1611152508' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:08.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:08.566471+0000) 2023-08-28T02:27:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:09 smithi136 ceph-mon[113531]: pgmap v10006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:09 smithi093 ceph-mon[111335]: pgmap v10006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:09 smithi093 ceph-mon[118902]: pgmap v10006: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:09.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:09.566753+0000) 2023-08-28T02:27:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:10.565+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:10.566972+0000) 2023-08-28T02:27:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:11 smithi136 ceph-mon[113531]: pgmap v10007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2703872311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:11 smithi093 ceph-mon[111335]: pgmap v10007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2703872311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:11 smithi093 ceph-mon[118902]: pgmap v10007: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2703872311' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:11.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:11.567230+0000) 2023-08-28T02:27:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:11.561Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:11.562Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:12.566+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:12.567484+0000) 2023-08-28T02:27:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:13 smithi136 ceph-mon[113531]: pgmap v10008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/769547267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:13 smithi093 ceph-mon[111335]: pgmap v10008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/769547267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:13 smithi093 ceph-mon[118902]: pgmap v10008: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/769547267' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:12.990Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:13.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:12.991Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:13.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:13.567744+0000) 2023-08-28T02:27:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:14.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:14.567981+0000) 2023-08-28T02:27:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:15 smithi136 ceph-mon[113531]: pgmap v10009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/115954268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:15.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:15 smithi093 ceph-mon[118902]: pgmap v10009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:15.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/115954268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:15 smithi093 ceph-mon[111335]: pgmap v10009: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/115954268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:27:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:27:15] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:27:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:15.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:15.568226+0000) 2023-08-28T02:27:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:16.567+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:16.568457+0000) 2023-08-28T02:27:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:17 smithi136 ceph-mon[113531]: pgmap v10010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:17 smithi093 ceph-mon[111335]: pgmap v10010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:17 smithi093 ceph-mon[118902]: pgmap v10010: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:17.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:17.568709+0000) 2023-08-28T02:27:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1892481920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:18.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1892481920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1892481920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:18.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:18.568949+0000) 2023-08-28T02:27:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:19 smithi136 ceph-mon[113531]: pgmap v10011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:19 smithi093 ceph-mon[111335]: pgmap v10011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:19 smithi093 ceph-mon[118902]: pgmap v10011: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:19.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:19.569208+0000) 2023-08-28T02:27:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/492250516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/492250516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/492250516' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:20.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:20.569449+0000) 2023-08-28T02:27:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:21 smithi136 ceph-mon[113531]: pgmap v10012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:21.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:21 smithi093 ceph-mon[111335]: pgmap v10012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:21 smithi093 ceph-mon[118902]: pgmap v10012: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:21.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:21.569725+0000) 2023-08-28T02:27:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:21.562Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:21.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:22.568+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:22.570015+0000) 2023-08-28T02:27:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:23 smithi136 ceph-mon[113531]: pgmap v10013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/669890486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:23 smithi093 ceph-mon[111335]: pgmap v10013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/669890486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:23 smithi093 ceph-mon[118902]: pgmap v10013: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/669890486' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:22.991Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:22.992Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:23.569+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:23.570250+0000) 2023-08-28T02:27:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:24.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:24.570497+0000) 2023-08-28T02:27:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:25 smithi136 ceph-mon[113531]: pgmap v10014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2400599184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:25.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:27:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:27:25] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:27:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:25 smithi093 ceph-mon[111335]: pgmap v10014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:25.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2400599184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:25 smithi093 ceph-mon[118902]: pgmap v10014: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2400599184' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:25.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:25.570720+0000) 2023-08-28T02:27:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:26.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:26.570946+0000) 2023-08-28T02:27:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:27 smithi136 ceph-mon[113531]: pgmap v10015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/113396233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:27 smithi093 ceph-mon[111335]: pgmap v10015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/113396233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:27 smithi093 ceph-mon[118902]: pgmap v10015: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/113396233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:27.570+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:27.571197+0000) 2023-08-28T02:27:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:28.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:28.571420+0000) 2023-08-28T02:27:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:29 smithi136 ceph-mon[113531]: pgmap v10016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:29 smithi093 ceph-mon[111335]: pgmap v10016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:29 smithi093 ceph-mon[118902]: pgmap v10016: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:29.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:29.571641+0000) 2023-08-28T02:27:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/242380974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:30.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/242380974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:30.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/242380974' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:30.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:30.571891+0000) 2023-08-28T02:27:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:31 smithi136 ceph-mon[113531]: pgmap v10017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:31 smithi093 ceph-mon[111335]: pgmap v10017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:31.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:31 smithi093 ceph-mon[118902]: pgmap v10017: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:31.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:31.572150+0000) 2023-08-28T02:27:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:31.563Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:31.564Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1755779287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1755779287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1755779287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:32.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:32.572393+0000) 2023-08-28T02:27:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:33 smithi136 ceph-mon[113531]: pgmap v10018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:33 smithi093 ceph-mon[111335]: pgmap v10018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:33 smithi093 ceph-mon[118902]: pgmap v10018: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:32.992Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:32.993Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:33.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:33.572654+0000) 2023-08-28T02:27:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/277988103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/277988103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:34.571 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/277988103' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:34.571+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:34.572939+0000) 2023-08-28T02:27:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:35 smithi093 ceph-mon[111335]: pgmap v10019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:35 smithi093 ceph-mon[118902]: pgmap v10019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:27:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:27:35] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:27:35.572 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:35 smithi136 ceph-mon[113531]: pgmap v10019: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:35.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:35.573185+0000) 2023-08-28T02:27:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:36.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:36.573445+0000) 2023-08-28T02:27:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:37 smithi093 ceph-mon[111335]: pgmap v10020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1710951120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:37 smithi093 ceph-mon[118902]: pgmap v10020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1710951120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:37.572 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:37 smithi136 ceph-mon[113531]: pgmap v10020: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:37.572 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1710951120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:37.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:37.573714+0000) 2023-08-28T02:27:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:38.572+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:38.574000+0000) 2023-08-28T02:27:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:39 smithi093 ceph-mon[111335]: pgmap v10021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3330321436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:39 smithi093 ceph-mon[118902]: pgmap v10021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3330321436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:39.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:39 smithi136 ceph-mon[113531]: pgmap v10021: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:39.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3330321436' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:39.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:39.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:39.574251+0000) 2023-08-28T02:27:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:40.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:40.574467+0000) 2023-08-28T02:27:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:41 smithi093 ceph-mon[111335]: pgmap v10022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:41 smithi093 ceph-mon[118902]: pgmap v10022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:41.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:41 smithi136 ceph-mon[113531]: pgmap v10022: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:41.573+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:41.574731+0000) 2023-08-28T02:27:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:41.564Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:41.565Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1844952289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1844952289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:42.573 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1844952289' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:42.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:42.574948+0000) 2023-08-28T02:27:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:43 smithi093 ceph-mon[111335]: pgmap v10023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:43 smithi093 ceph-mon[118902]: pgmap v10023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:42.992Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:42.993Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:43.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:43 smithi136 ceph-mon[113531]: pgmap v10023: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:43.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:43.575164+0000) 2023-08-28T02:27:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/993251305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/993251305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:44.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/993251305' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:44.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:44.575377+0000) 2023-08-28T02:27:45.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:45 smithi093 ceph-mon[111335]: pgmap v10024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:27:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:27:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:27:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:45 smithi093 ceph-mon[118902]: pgmap v10024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:45.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:45 smithi136 ceph-mon[113531]: pgmap v10024: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:45.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:45.575551+0000) 2023-08-28T02:27:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3908335187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3908335187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:46.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3908335187' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:46.574+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:46.575818+0000) 2023-08-28T02:27:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:47 smithi093 ceph-mon[111335]: pgmap v10025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:47 smithi093 ceph-mon[118902]: pgmap v10025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:47.574 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:47 smithi136 ceph-mon[113531]: pgmap v10025: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:47.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:47.576092+0000) 2023-08-28T02:27:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:48 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:27:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:48 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:27:48.535 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:48 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:27:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:48.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:48.576324+0000) 2023-08-28T02:27:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:49 smithi093 ceph-mon[111335]: pgmap v10026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1426782495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:49 smithi093 ceph-mon[118902]: pgmap v10026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1426782495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:49.567 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:49 smithi136 ceph-mon[113531]: pgmap v10026: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:49.567 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1426782495' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:49.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:49.576503+0000) 2023-08-28T02:27:50.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:50.575+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:50.576753+0000) 2023-08-28T02:27:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: pgmap v10027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:50.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:50 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: pgmap v10027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:50.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:50.740 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:50.741 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: pgmap v10027: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.741 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:50.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:27:50.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:27:50.742 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:50 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:27:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/581866920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:51.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:51.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:51.577024+0000) 2023-08-28T02:27:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/581866920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/581866920' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:51.564Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:51.565Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:52.889 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:52.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:52.577325+0000) 2023-08-28T02:27:53.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:52 smithi136 ceph-mon[113531]: pgmap v10028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:52 smithi093 ceph-mon[111335]: pgmap v10028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:53.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:52 smithi093 ceph-mon[118902]: pgmap v10028: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:52.993Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:27:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:27:52.994Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:27:53.895 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:53.576+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:53.577574+0000) 2023-08-28T02:27:54.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/743447706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/743447706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/743447706' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:54.896 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:54.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:54.577850+0000) 2023-08-28T02:27:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:54 smithi136 ceph-mon[113531]: pgmap v10029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:55.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:54 smithi093 ceph-mon[111335]: pgmap v10029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:54 smithi093 ceph-mon[118902]: pgmap v10029: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:27:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:27:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:27:55] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:27:55.901 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:55.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:55.578072+0000) 2023-08-28T02:27:56.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3442451113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:56.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3442451113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:56.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3442451113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:56.902 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:56.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:56.578251+0000) 2023-08-28T02:27:57.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:56 smithi136 ceph-mon[113531]: pgmap v10030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:56 smithi093 ceph-mon[111335]: pgmap v10030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:56 smithi093 ceph-mon[118902]: pgmap v10030: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:57.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:57.578440+0000) 2023-08-28T02:27:58.905 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:58.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:58.578701+0000) 2023-08-28T02:27:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:58 smithi136 ceph-mon[113531]: pgmap v10031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:59.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:27:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/980822012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:58 smithi093 ceph-mon[111335]: pgmap v10031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:59.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:27:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/980822012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:58 smithi093 ceph-mon[118902]: pgmap v10031: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:27:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:27:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/980822012' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:27:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:27:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:27:59.577+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:27:59.578996+0000) 2023-08-28T02:28:00.907 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:00.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:00.579242+0000) 2023-08-28T02:28:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:00 smithi136 ceph-mon[113531]: pgmap v10032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:01.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3399954318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:00 smithi093 ceph-mon[111335]: pgmap v10032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:01.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3399954318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:00 smithi093 ceph-mon[118902]: pgmap v10032: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:01.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3399954318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:01.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:01.579501+0000) 2023-08-28T02:28:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:01.565Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:01.566Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:02.910 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:02.578+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:02.579769+0000) 2023-08-28T02:28:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:02 smithi136 ceph-mon[113531]: pgmap v10033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:03.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1937357834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:02 smithi093 ceph-mon[111335]: pgmap v10033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1937357834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:02 smithi093 ceph-mon[118902]: pgmap v10033: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1937357834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:02.994Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:02.995Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:03.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:03.580022+0000) 2023-08-28T02:28:04.913 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:04.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:04.580320+0000) 2023-08-28T02:28:05.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:04 smithi136 ceph-mon[113531]: pgmap v10034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:04 smithi093 ceph-mon[111335]: pgmap v10034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:04 smithi093 ceph-mon[118902]: pgmap v10034: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:28:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:28:05] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:28:05.920 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:05.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:05.580529+0000) 2023-08-28T02:28:06.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2315003802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2315003802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2315003802' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:06.923 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:06.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:06.580766+0000) 2023-08-28T02:28:07.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:06 smithi136 ceph-mon[113531]: pgmap v10035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:06 smithi093 ceph-mon[111335]: pgmap v10035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:06 smithi093 ceph-mon[118902]: pgmap v10035: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:07.930 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:07.579+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:07.580984+0000) 2023-08-28T02:28:08.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3914313796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3914313796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3914313796' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:08.931 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:08.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:08.581266+0000) 2023-08-28T02:28:09.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:08 smithi136 ceph-mon[113531]: pgmap v10036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:08 smithi093 ceph-mon[111335]: pgmap v10036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:08 smithi093 ceph-mon[118902]: pgmap v10036: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:09.936 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:09.580+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:09.581538+0000) 2023-08-28T02:28:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:10.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3558873007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3558873007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:10.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3558873007' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:10.936 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:10.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:10.581795+0000) 2023-08-28T02:28:11.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:10 smithi136 ceph-mon[113531]: pgmap v10037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:11.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:10 smithi093 ceph-mon[111335]: pgmap v10037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:11.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:10 smithi093 ceph-mon[118902]: pgmap v10037: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:11.581+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:11.582073+0000) 2023-08-28T02:28:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:11.565Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:11.567Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:12.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:12.582321+0000) 2023-08-28T02:28:13.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:12 smithi093 ceph-mon[111335]: pgmap v10038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3379558071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:12 smithi093 ceph-mon[118902]: pgmap v10038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3379558071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:12.994Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:13.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:12.995Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:12 smithi136 ceph-mon[113531]: pgmap v10038: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3379558071' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:13.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:13.582595+0000) 2023-08-28T02:28:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:14.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:14.582786+0000) 2023-08-28T02:28:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:14 smithi093 ceph-mon[111335]: pgmap v10039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:15.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3548944798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:14 smithi093 ceph-mon[118902]: pgmap v10039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3548944798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:14 smithi136 ceph-mon[113531]: pgmap v10039: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3548944798' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:28:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:28:15] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:28:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:15.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:15.583000+0000) 2023-08-28T02:28:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:16.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:16.583309+0000) 2023-08-28T02:28:17.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:16 smithi093 ceph-mon[111335]: pgmap v10040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:17.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:16 smithi093 ceph-mon[118902]: pgmap v10040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:16 smithi136 ceph-mon[113531]: pgmap v10040: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:17.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:17.583567+0000) 2023-08-28T02:28:18.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2295260374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:18.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2295260374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2295260374' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:18.582+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:18.583821+0000) 2023-08-28T02:28:19.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:18 smithi093 ceph-mon[111335]: pgmap v10041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:19.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:18 smithi093 ceph-mon[118902]: pgmap v10041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:18 smithi136 ceph-mon[113531]: pgmap v10041: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:19.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:19.584117+0000) 2023-08-28T02:28:20.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/289497392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:20.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/289497392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/289497392' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:20.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:20.584396+0000) 2023-08-28T02:28:21.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:20 smithi093 ceph-mon[111335]: pgmap v10042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:21.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:20 smithi093 ceph-mon[118902]: pgmap v10042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:20 smithi136 ceph-mon[113531]: pgmap v10042: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:21.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:21.584649+0000) 2023-08-28T02:28:21.971 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:21.566Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:21.971 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:21.567Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:22.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1369247330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:22.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1369247330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1369247330' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:22.583+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:22.584898+0000) 2023-08-28T02:28:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:22 smithi093 ceph-mon[111335]: pgmap v10043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:22 smithi093 ceph-mon[118902]: pgmap v10043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:22.994Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:22.995Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:22 smithi136 ceph-mon[113531]: pgmap v10043: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:23.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:23.585153+0000) 2023-08-28T02:28:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:24.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:24.585408+0000) 2023-08-28T02:28:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:24 smithi093 ceph-mon[111335]: pgmap v10044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3619530308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:24 smithi093 ceph-mon[118902]: pgmap v10044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3619530308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:24 smithi136 ceph-mon[113531]: pgmap v10044: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3619530308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:28:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:28:25] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:28:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:25.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:25.585645+0000) 2023-08-28T02:28:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:26.584+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:26.585903+0000) 2023-08-28T02:28:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:26 smithi093 ceph-mon[111335]: pgmap v10045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2576399910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:26 smithi093 ceph-mon[118902]: pgmap v10045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2576399910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:26 smithi136 ceph-mon[113531]: pgmap v10045: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2576399910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:27.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:27.586167+0000) 2023-08-28T02:28:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:28.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:28.586385+0000) 2023-08-28T02:28:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:28 smithi093 ceph-mon[111335]: pgmap v10046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1900210453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:28 smithi093 ceph-mon[118902]: pgmap v10046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1900210453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:28 smithi136 ceph-mon[113531]: pgmap v10046: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1900210453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:29.585+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:29.586607+0000) 2023-08-28T02:28:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:30.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:30.586809+0000) 2023-08-28T02:28:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:30 smithi136 ceph-mon[113531]: pgmap v10047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:30 smithi093 ceph-mon[111335]: pgmap v10047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:31.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:30 smithi093 ceph-mon[118902]: pgmap v10047: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:31.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:31.587097+0000) 2023-08-28T02:28:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:31.567Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:31.568Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3540866982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3540866982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3540866982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:32.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:32.587355+0000) 2023-08-28T02:28:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:32 smithi136 ceph-mon[113531]: pgmap v10048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:32 smithi093 ceph-mon[111335]: pgmap v10048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:32 smithi093 ceph-mon[118902]: pgmap v10048: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:32.995Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:32 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:32.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:33.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:33.587614+0000) 2023-08-28T02:28:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3649378106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:34.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3649378106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:34.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3649378106' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:34.586+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:34.587891+0000) 2023-08-28T02:28:35.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:35 smithi093 ceph-mon[111335]: pgmap v10049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:35 smithi093 ceph-mon[118902]: pgmap v10049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:35 smithi136 ceph-mon[113531]: pgmap v10049: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:28:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:28:35] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:28:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:35.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:35.588126+0000) 2023-08-28T02:28:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:36.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:36.588364+0000) 2023-08-28T02:28:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:37 smithi136 ceph-mon[113531]: pgmap v10050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/361333991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:37 smithi093 ceph-mon[111335]: pgmap v10050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/361333991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:37 smithi093 ceph-mon[118902]: pgmap v10050: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/361333991' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:37.587+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:37.588622+0000) 2023-08-28T02:28:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:38.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:38.588862+0000) 2023-08-28T02:28:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:39 smithi136 ceph-mon[113531]: pgmap v10051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4044736857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:39 smithi093 ceph-mon[111335]: pgmap v10051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4044736857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:39 smithi093 ceph-mon[118902]: pgmap v10051: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4044736857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:39.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:39.589099+0000) 2023-08-28T02:28:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:40.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:40.589357+0000) 2023-08-28T02:28:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:41 smithi136 ceph-mon[113531]: pgmap v10052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4132794148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:41 smithi093 ceph-mon[111335]: pgmap v10052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4132794148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:41 smithi093 ceph-mon[118902]: pgmap v10052: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4132794148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:41.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:41.589557+0000) 2023-08-28T02:28:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:41.568Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:41.569Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:42.588+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:42.589829+0000) 2023-08-28T02:28:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:43 smithi136 ceph-mon[113531]: pgmap v10053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:43 smithi093 ceph-mon[111335]: pgmap v10053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:43 smithi093 ceph-mon[118902]: pgmap v10053: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:42.995Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:42 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:42.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:43.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:43.590094+0000) 2023-08-28T02:28:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2491747632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2491747632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2491747632' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:44.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:44.590325+0000) 2023-08-28T02:28:45.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:45 smithi093 ceph-mon[111335]: pgmap v10054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:45 smithi093 ceph-mon[118902]: pgmap v10054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:45 smithi136 ceph-mon[113531]: pgmap v10054: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:28:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:28:45] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:28:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:45.589+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:45.590524+0000) 2023-08-28T02:28:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1050998833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1050998833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:46.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1050998833' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:46.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:46.590754+0000) 2023-08-28T02:28:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:47 smithi136 ceph-mon[113531]: pgmap v10055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:47 smithi093 ceph-mon[111335]: pgmap v10055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:47 smithi093 ceph-mon[118902]: pgmap v10055: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:47.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:47.591005+0000) 2023-08-28T02:28:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2809137393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2809137393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2809137393' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:48.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:48.591288+0000) 2023-08-28T02:28:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:49 smithi136 ceph-mon[113531]: pgmap v10056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:49 smithi093 ceph-mon[111335]: pgmap v10056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:49 smithi093 ceph-mon[118902]: pgmap v10056: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:49.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:49.591563+0000) 2023-08-28T02:28:50.929 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:50.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:50.591741+0000) 2023-08-28T02:28:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:51 smithi136 ceph-mon[113531]: pgmap v10057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:51 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:51.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/532695512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-mon[111335]: pgmap v10057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/532695512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-mon[118902]: pgmap v10057: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:28:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/532695512' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:51.590+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:51.591952+0000) 2023-08-28T02:28:51.974 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:51.569Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:51.974 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:51.571Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:52.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:52.592124+0000) 2023-08-28T02:28:53.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:52.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:28:52 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:28:52.996Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:28:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: pgmap v10058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4247158702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: pgmap v10058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4247158702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:53.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:53.491 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: pgmap v10058: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:53.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4247158702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:53.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:28:53.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:28:53.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:28:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:53.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:53.592358+0000) 2023-08-28T02:28:54.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:28:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:54.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:54.592615+0000) 2023-08-28T02:28:55.591 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:55 smithi136 ceph-mon[113531]: pgmap v10059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:55.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3524326758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:55 smithi093 ceph-mon[118902]: pgmap v10059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:55.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3524326758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:55 smithi093 ceph-mon[111335]: pgmap v10059: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:55.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3524326758' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:28:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:28:55] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:28:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:55.591+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:55.592789+0000) 2023-08-28T02:28:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:56.592+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:56.592993+0000) 2023-08-28T02:28:57.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:57 smithi136 ceph-mon[113531]: pgmap v10060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:57 smithi093 ceph-mon[111335]: pgmap v10060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:57 smithi093 ceph-mon[118902]: pgmap v10060: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:57.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:57.593232+0000) 2023-08-28T02:28:58.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2386710949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2386710949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2386710949' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:28:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:58.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:58.593495+0000) 2023-08-28T02:28:59.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:28:59 smithi136 ceph-mon[113531]: pgmap v10061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:28:59 smithi093 ceph-mon[111335]: pgmap v10061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:28:59 smithi093 ceph-mon[118902]: pgmap v10061: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:28:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:28:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:28:59.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:28:59.593765+0000) 2023-08-28T02:29:00.592 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3404494926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3404494926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3404494926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:00.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:00.593991+0000) 2023-08-28T02:29:01.570 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:01 smithi093 ceph-mon[111335]: pgmap v10062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:01.570 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:01 smithi093 ceph-mon[118902]: pgmap v10062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:01.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:01 smithi136 ceph-mon[113531]: pgmap v10062: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:01.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:01.594205+0000) 2023-08-28T02:29:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:01.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:01.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:02.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:02.594455+0000) 2023-08-28T02:29:03.266 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:02.996Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:03.267 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:02 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:02.997Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:03.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:03 smithi136 ceph-mon[113531]: pgmap v10063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:03.593 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/166892101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:03 smithi093 ceph-mon[111335]: pgmap v10063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/166892101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:03 smithi093 ceph-mon[118902]: pgmap v10063: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/166892101' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:03.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:03.594700+0000) 2023-08-28T02:29:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:04.593+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:04.594902+0000) 2023-08-28T02:29:05.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:05 smithi136 ceph-mon[113531]: pgmap v10064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:05.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3744716416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:29:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:29:05] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:29:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:05 smithi093 ceph-mon[111335]: pgmap v10064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3744716416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:05 smithi093 ceph-mon[118902]: pgmap v10064: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3744716416' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:05.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:05.595165+0000) 2023-08-28T02:29:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:06.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:06.595366+0000) 2023-08-28T02:29:07.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:07 smithi136 ceph-mon[113531]: pgmap v10065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:07.594 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4160979689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:07 smithi093 ceph-mon[111335]: pgmap v10065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4160979689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:07 smithi093 ceph-mon[118902]: pgmap v10065: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4160979689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:07.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:07.595584+0000) 2023-08-28T02:29:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:08.594+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:08.595803+0000) 2023-08-28T02:29:09.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:09 smithi136 ceph-mon[113531]: pgmap v10066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:09.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:09 smithi093 ceph-mon[111335]: pgmap v10066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:09 smithi093 ceph-mon[118902]: pgmap v10066: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:09.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:09.596089+0000) 2023-08-28T02:29:10.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2087298462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2087298462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2087298462' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:10.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:10.596324+0000) 2023-08-28T02:29:11.570 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:11 smithi093 ceph-mon[111335]: pgmap v10067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:11.570 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:11 smithi093 ceph-mon[118902]: pgmap v10067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:11.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:11 smithi136 ceph-mon[113531]: pgmap v10067: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:11.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:11.596564+0000) 2023-08-28T02:29:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:11.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:11.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:12.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2836561964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:12.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2836561964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2836561964' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:12.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:12.596775+0000) 2023-08-28T02:29:13.289 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:12.998Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:13.289 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:12 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:12.999Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:13.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:13 smithi136 ceph-mon[113531]: pgmap v10068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:13 smithi093 ceph-mon[111335]: pgmap v10068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:13 smithi093 ceph-mon[118902]: pgmap v10068: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:13.595+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:13.596972+0000) 2023-08-28T02:29:14.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/590015269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/590015269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/590015269' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:14.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:14.597224+0000) 2023-08-28T02:29:15.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:15 smithi136 ceph-mon[113531]: pgmap v10069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:15.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:15 smithi093 ceph-mon[118902]: pgmap v10069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:15 smithi093 ceph-mon[111335]: pgmap v10069: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:29:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:29:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:29:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:15.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:15.597400+0000) 2023-08-28T02:29:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:16.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:16.597585+0000) 2023-08-28T02:29:17.596 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:17 smithi136 ceph-mon[113531]: pgmap v10070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:17.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3537063233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:17 smithi093 ceph-mon[111335]: pgmap v10070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3537063233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:17 smithi093 ceph-mon[118902]: pgmap v10070: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3537063233' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:17.596+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:17.597815+0000) 2023-08-28T02:29:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:18.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:18.598088+0000) 2023-08-28T02:29:19.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:19 smithi136 ceph-mon[113531]: pgmap v10071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:19.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/299515874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:19 smithi093 ceph-mon[111335]: pgmap v10071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/299515874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:19 smithi093 ceph-mon[118902]: pgmap v10071: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/299515874' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:19.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:19.598309+0000) 2023-08-28T02:29:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:20.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:20.598502+0000) 2023-08-28T02:29:21.570 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:21 smithi093 ceph-mon[111335]: pgmap v10072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:21.571 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:21 smithi093 ceph-mon[118902]: pgmap v10072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:21.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:21 smithi136 ceph-mon[113531]: pgmap v10072: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:21.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:21.598730+0000) 2023-08-28T02:29:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:21.571Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:21.572Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:22.597 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1011764201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1011764201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:22.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1011764201' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:22.597+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:22.598933+0000) 2023-08-28T02:29:23.314 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:22.998Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:23.314 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:22 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:22.999Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:23.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:23 smithi136 ceph-mon[113531]: pgmap v10073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:23 smithi093 ceph-mon[111335]: pgmap v10073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:23 smithi093 ceph-mon[118902]: pgmap v10073: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:23.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:23.599122+0000) 2023-08-28T02:29:24.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3231049771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:24.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3231049771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3231049771' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:24.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:24.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:24.599396+0000) 2023-08-28T02:29:25.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:25 smithi136 ceph-mon[113531]: pgmap v10074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:25 smithi093 ceph-mon[118902]: pgmap v10074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:29:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:29:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:29:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:25 smithi093 ceph-mon[111335]: pgmap v10074: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:25.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:25.599611+0000) 2023-08-28T02:29:26.598 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/836246941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/836246941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:26.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/836246941' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:26.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:26.599807+0000) 2023-08-28T02:29:27.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:27 smithi136 ceph-mon[113531]: pgmap v10075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:27 smithi093 ceph-mon[111335]: pgmap v10075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:27 smithi093 ceph-mon[118902]: pgmap v10075: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:27.598+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:27.600005+0000) 2023-08-28T02:29:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:28.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:28.600282+0000) 2023-08-28T02:29:29.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:29 smithi136 ceph-mon[113531]: pgmap v10076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:29.599 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3651483767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:29 smithi093 ceph-mon[111335]: pgmap v10076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3651483767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:29 smithi093 ceph-mon[118902]: pgmap v10076: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3651483767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:29.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:29.600539+0000) 2023-08-28T02:29:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:30 smithi136 ceph-mon[113531]: pgmap v10077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:30.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:30.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:30.600746+0000) 2023-08-28T02:29:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:30 smithi093 ceph-mon[111335]: pgmap v10077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:30 smithi093 ceph-mon[118902]: pgmap v10077: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/427425985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:31.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:31.599+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:31.600954+0000) 2023-08-28T02:29:31.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/427425985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:31.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/427425985' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:31.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:31.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:31.573Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:32.908 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:32.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:32.601140+0000) 2023-08-28T02:29:33.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:32 smithi136 ceph-mon[113531]: pgmap v10078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:32 smithi093 ceph-mon[111335]: pgmap v10078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:33.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:32 smithi093 ceph-mon[118902]: pgmap v10078: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:32.999Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:33.000Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:33.912 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:33.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:33.601300+0000) 2023-08-28T02:29:34.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2782390020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:34.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2782390020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:34.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2782390020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:34.913 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:34.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:34.601466+0000) 2023-08-28T02:29:35.190 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:34 smithi136 ceph-mon[113531]: pgmap v10079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:35.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:34 smithi093 ceph-mon[111335]: pgmap v10079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:34 smithi093 ceph-mon[118902]: pgmap v10079: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:29:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:29:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:29:35.917 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:35.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:35.601669+0000) 2023-08-28T02:29:36.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2400951154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:36.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2400951154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:36.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2400951154' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:36.919 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:36.600+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:36.601885+0000) 2023-08-28T02:29:37.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:36 smithi136 ceph-mon[113531]: pgmap v10080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:36 smithi093 ceph-mon[111335]: pgmap v10080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:36 smithi093 ceph-mon[118902]: pgmap v10080: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:37.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:37.602136+0000) 2023-08-28T02:29:38.922 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:38.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:38.602381+0000) 2023-08-28T02:29:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:38 smithi136 ceph-mon[113531]: pgmap v10081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:39.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/757581500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:38 smithi093 ceph-mon[111335]: pgmap v10081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/757581500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:38 smithi093 ceph-mon[118902]: pgmap v10081: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/757581500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:39.929 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:39.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:39.602576+0000) 2023-08-28T02:29:40.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:40.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:40.928 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:40.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:40.602756+0000) 2023-08-28T02:29:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:40 smithi136 ceph-mon[113531]: pgmap v10082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:41.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1019564410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:40 smithi093 ceph-mon[111335]: pgmap v10082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1019564410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:40 smithi093 ceph-mon[118902]: pgmap v10082: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:41.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1019564410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:41.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:41.601+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:41.602998+0000) 2023-08-28T02:29:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:41.572Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:41.574Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:42.931 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:42.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:42.603263+0000) 2023-08-28T02:29:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:42 smithi136 ceph-mon[113531]: pgmap v10083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3900244931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:42 smithi093 ceph-mon[111335]: pgmap v10083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3900244931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:42 smithi093 ceph-mon[118902]: pgmap v10083: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3900244931' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:43.000Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:43.001Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:43.602+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:43.603501+0000) 2023-08-28T02:29:44.934 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:44.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:44.603704+0000) 2023-08-28T02:29:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:44 smithi136 ceph-mon[113531]: pgmap v10084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:44 smithi093 ceph-mon[111335]: pgmap v10084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:44 smithi093 ceph-mon[118902]: pgmap v10084: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:29:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:29:45] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:29:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:45.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:45.603893+0000) 2023-08-28T02:29:46.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3188501675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:46.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3188501675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3188501675' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:46.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:46.604150+0000) 2023-08-28T02:29:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:46 smithi093 ceph-mon[111335]: pgmap v10085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:46 smithi093 ceph-mon[118902]: pgmap v10085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:46 smithi136 ceph-mon[113531]: pgmap v10085: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:47.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:47.604376+0000) 2023-08-28T02:29:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3038072584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3038072584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3038072584' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:48.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:48.604667+0000) 2023-08-28T02:29:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:48 smithi093 ceph-mon[111335]: pgmap v10086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:48 smithi093 ceph-mon[118902]: pgmap v10086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:48 smithi136 ceph-mon[113531]: pgmap v10086: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:49.603+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:49.604945+0000) 2023-08-28T02:29:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:50.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:50.605191+0000) 2023-08-28T02:29:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:50 smithi093 ceph-mon[111335]: pgmap v10087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3054394266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:50 smithi093 ceph-mon[118902]: pgmap v10087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3054394266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:50 smithi136 ceph-mon[113531]: pgmap v10087: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3054394266' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:51.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:51.605393+0000) 2023-08-28T02:29:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:51.574Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:51.575Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:52.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:52.605659+0000) 2023-08-28T02:29:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:52 smithi093 ceph-mon[111335]: pgmap v10088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:53.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2683385024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:52 smithi093 ceph-mon[118902]: pgmap v10088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2683385024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:53.001Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:53.240 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:29:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:29:53.002Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:29:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:52 smithi136 ceph-mon[113531]: pgmap v10088: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2683385024' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:53.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:53.605847+0000) 2023-08-28T02:29:54.208 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:54.209 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:29:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:29:54.606 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:54.604+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:54.605999+0000) 2023-08-28T02:29:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[111335]: pgmap v10089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2264074238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:55.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[118902]: pgmap v10089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2264074238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:55.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:55.385 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:54 smithi136 ceph-mon[113531]: pgmap v10089: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:55.385 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2264074238' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:55.385 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:55.385 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:55.688 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:55.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:55.606205+0000) 2023-08-28T02:29:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:29:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:29:55] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:29:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:56.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:56.606373+0000) 2023-08-28T02:29:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-mon[113531]: pgmap v10090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.939 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:56 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[111335]: pgmap v10090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[118902]: pgmap v10090: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:56.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:56 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:57.919 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:57.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:57.606567+0000) 2023-08-28T02:29:58.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2505774037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:58.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2505774037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:58.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2505774037' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:29:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:58.605+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:58.606837+0000) 2023-08-28T02:29:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:59 smithi136 ceph-mon[113531]: pgmap v10091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:29:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:29:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:29:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[111335]: pgmap v10091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[118902]: pgmap v10091: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:29:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:29:59.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:29:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:29:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:29:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:29:59.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:29:59.607124+0000) 2023-08-28T02:30:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3082414905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:30:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3082414905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:30:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3082414905' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:00.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:30:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:00.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:00.607413+0000) 2023-08-28T02:30:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:01 smithi136 ceph-mon[113531]: pgmap v10092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:01 smithi093 ceph-mon[111335]: pgmap v10092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:01 smithi093 ceph-mon[118902]: pgmap v10092: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:01.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:01.607632+0000) 2023-08-28T02:30:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:01.574Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:01.576Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3813364395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3813364395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3813364395' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:02.606+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:02.607850+0000) 2023-08-28T02:30:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:03 smithi136 ceph-mon[113531]: pgmap v10093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:03 smithi093 ceph-mon[111335]: pgmap v10093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:03 smithi093 ceph-mon[118902]: pgmap v10093: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:03.002Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:03.004Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:03.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:03.608147+0000) 2023-08-28T02:30:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:04.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:04.608331+0000) 2023-08-28T02:30:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:05 smithi136 ceph-mon[113531]: pgmap v10094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1737809371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:05.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:30:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:30:05] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:30:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:05 smithi093 ceph-mon[111335]: pgmap v10094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:05.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1737809371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:05 smithi093 ceph-mon[118902]: pgmap v10094: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1737809371' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:05.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:05.608542+0000) 2023-08-28T02:30:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:06.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:06.608757+0000) 2023-08-28T02:30:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:07 smithi136 ceph-mon[113531]: pgmap v10095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1878419019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:07 smithi093 ceph-mon[118902]: pgmap v10095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1878419019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:07 smithi093 ceph-mon[111335]: pgmap v10095: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1878419019' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:07.607+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:07.608987+0000) 2023-08-28T02:30:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:08.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:08.609173+0000) 2023-08-28T02:30:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:09 smithi136 ceph-mon[113531]: pgmap v10096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/694515086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:09 smithi093 ceph-mon[118902]: pgmap v10096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/694515086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:09 smithi093 ceph-mon[111335]: pgmap v10096: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/694515086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:09.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:09.609407+0000) 2023-08-28T02:30:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:10.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:10.609700+0000) 2023-08-28T02:30:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:11 smithi136 ceph-mon[113531]: pgmap v10097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:11 smithi093 ceph-mon[111335]: pgmap v10097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:11 smithi093 ceph-mon[118902]: pgmap v10097: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:11.608+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:11.610004+0000) 2023-08-28T02:30:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:11.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:11.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3294348987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3294348987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:12.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3294348987' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:12.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:12.610303+0000) 2023-08-28T02:30:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:13 smithi093 ceph-mon[111335]: pgmap v10098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:13 smithi093 ceph-mon[118902]: pgmap v10098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:13.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:13.004Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:13.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:13 smithi136 ceph-mon[113531]: pgmap v10098: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:13.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:13.610561+0000) 2023-08-28T02:30:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1830294657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1830294657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:14.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1830294657' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:14.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:14.610821+0000) 2023-08-28T02:30:15.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:15 smithi093 ceph-mon[111335]: pgmap v10099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:15.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:30:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:30:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:30:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:15 smithi093 ceph-mon[118902]: pgmap v10099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:15.609 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:15 smithi136 ceph-mon[113531]: pgmap v10099: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:15.609+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:15.611006+0000) 2023-08-28T02:30:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1681870370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1681870370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:16.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1681870370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:16.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:16.611209+0000) 2023-08-28T02:30:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:17 smithi093 ceph-mon[111335]: pgmap v10100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:17 smithi093 ceph-mon[118902]: pgmap v10100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:17.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:17 smithi136 ceph-mon[113531]: pgmap v10100: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:17.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:17.611465+0000) 2023-08-28T02:30:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:18.610+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:18.611743+0000) 2023-08-28T02:30:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:19 smithi093 ceph-mon[111335]: pgmap v10101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:19.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/567299707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:19.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:19 smithi093 ceph-mon[118902]: pgmap v10101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:19.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/567299707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:19.610 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:19 smithi136 ceph-mon[113531]: pgmap v10101: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:19.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/567299707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:19.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:19.611991+0000) 2023-08-28T02:30:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:20.611+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:20.612196+0000) 2023-08-28T02:30:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:21 smithi093 ceph-mon[111335]: pgmap v10102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3622816840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:21 smithi093 ceph-mon[118902]: pgmap v10102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:21.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3622816840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:21.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:21 smithi136 ceph-mon[113531]: pgmap v10102: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:21.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3622816840' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:21.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:21.612439+0000) 2023-08-28T02:30:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:21.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:21.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:22.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:22.612625+0000) 2023-08-28T02:30:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:23 smithi093 ceph-mon[111335]: pgmap v10103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:23 smithi093 ceph-mon[118902]: pgmap v10103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:23.003Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:23.004Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:23.611 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:23 smithi136 ceph-mon[113531]: pgmap v10103: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:23.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:23.612844+0000) 2023-08-28T02:30:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1230050359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:24.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1230050359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:24.612 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1230050359' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:24.612 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:24.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:24.613087+0000) 2023-08-28T02:30:25.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:25 smithi093 ceph-mon[111335]: pgmap v10104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:25.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:25 smithi093 ceph-mon[118902]: pgmap v10104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:25.491 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:30:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:30:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:30:25.612 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:25 smithi136 ceph-mon[113531]: pgmap v10104: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:25.612+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:25.613301+0000) 2023-08-28T02:30:26.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1628310020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1628310020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:26.612 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1628310020' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:26.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:26.613502+0000) 2023-08-28T02:30:27.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:27 smithi093 ceph-mon[111335]: pgmap v10105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:27 smithi093 ceph-mon[118902]: pgmap v10105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:27.612 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:27 smithi136 ceph-mon[113531]: pgmap v10105: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:27.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:27.613727+0000) 2023-08-28T02:30:28.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3725547457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3725547457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:28.612 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3725547457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:28.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:28.613951+0000) 2023-08-28T02:30:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:29 smithi093 ceph-mon[111335]: pgmap v10106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:29 smithi093 ceph-mon[118902]: pgmap v10106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:29.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:29 smithi136 ceph-mon[113531]: pgmap v10106: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:29.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:29.614175+0000) 2023-08-28T02:30:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:30.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:30.614433+0000) 2023-08-28T02:30:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:31 smithi093 ceph-mon[111335]: pgmap v10107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:31.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2233948875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:31 smithi093 ceph-mon[118902]: pgmap v10107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2233948875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:31.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:31 smithi136 ceph-mon[113531]: pgmap v10107: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:31.613 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2233948875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:31.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:31.614680+0000) 2023-08-28T02:30:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:31.576Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:31.577Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:32.613+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:32.614935+0000) 2023-08-28T02:30:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:33 smithi093 ceph-mon[111335]: pgmap v10108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:33.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3536909470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:33 smithi093 ceph-mon[118902]: pgmap v10108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3536909470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:33.004Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:33.005Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:33.614 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:33 smithi136 ceph-mon[113531]: pgmap v10108: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:33.614 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3536909470' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:33.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:33.615189+0000) 2023-08-28T02:30:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:34.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:34.615447+0000) 2023-08-28T02:30:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:35 smithi093 ceph-mon[111335]: pgmap v10109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/234306965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:30:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:30:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:30:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:35 smithi093 ceph-mon[118902]: pgmap v10109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/234306965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:35.614 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:35 smithi136 ceph-mon[113531]: pgmap v10109: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:35.615 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/234306965' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:35.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:35.615693+0000) 2023-08-28T02:30:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:36.614+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:36.615947+0000) 2023-08-28T02:30:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:37 smithi093 ceph-mon[111335]: pgmap v10110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:37 smithi093 ceph-mon[118902]: pgmap v10110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:37.615 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:37 smithi136 ceph-mon[113531]: pgmap v10110: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:37.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:37.616193+0000) 2023-08-28T02:30:38.615 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3559336662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3559336662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3559336662' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:38.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:38.616446+0000) 2023-08-28T02:30:39.615 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:39 smithi136 ceph-mon[113531]: pgmap v10111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:39.616 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:39 smithi093 ceph-mon[111335]: pgmap v10111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:39 smithi093 ceph-mon[118902]: pgmap v10111: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:39.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:39.616677+0000) 2023-08-28T02:30:40.616 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/30062617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/30062617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:40.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/30062617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:40.615+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:40.616949+0000) 2023-08-28T02:30:41.576 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:41 smithi093 ceph-mon[111335]: pgmap v10112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:41.577 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:41 smithi093 ceph-mon[118902]: pgmap v10112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:41.616 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:41 smithi136 ceph-mon[113531]: pgmap v10112: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:41.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:41.617223+0000) 2023-08-28T02:30:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:41.577Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:41.578Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:42.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:42.617458+0000) 2023-08-28T02:30:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:43 smithi093 ceph-mon[111335]: pgmap v10113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3638268598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:43 smithi093 ceph-mon[118902]: pgmap v10113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3638268598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:43.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:43.006Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:43.616 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:43 smithi136 ceph-mon[113531]: pgmap v10113: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:43.616 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3638268598' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:43.616+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:43.617673+0000) 2023-08-28T02:30:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:44.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:44.617935+0000) 2023-08-28T02:30:45.617 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:45 smithi136 ceph-mon[113531]: pgmap v10114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:45.617 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2287382268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:30:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:30:45] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:30:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:45 smithi093 ceph-mon[111335]: pgmap v10114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:45.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2287382268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:45 smithi093 ceph-mon[118902]: pgmap v10114: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2287382268' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:45.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:45.618172+0000) 2023-08-28T02:30:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:46.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:46.618409+0000) 2023-08-28T02:30:47.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:47 smithi136 ceph-mon[113531]: pgmap v10115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:47.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1817815921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:47 smithi093 ceph-mon[111335]: pgmap v10115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1817815921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:47 smithi093 ceph-mon[118902]: pgmap v10115: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1817815921' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:47.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:47.618674+0000) 2023-08-28T02:30:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:48.617+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:48.618959+0000) 2023-08-28T02:30:49.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:49 smithi136 ceph-mon[113531]: pgmap v10116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:49 smithi093 ceph-mon[111335]: pgmap v10116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:49 smithi093 ceph-mon[118902]: pgmap v10116: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:49.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:49.619161+0000) 2023-08-28T02:30:50.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/374137529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:50.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/374137529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:50.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/374137529' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:50.618+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:50.619461+0000) 2023-08-28T02:30:51.577 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:51 smithi093 ceph-mon[111335]: pgmap v10117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:51.577 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:51 smithi093 ceph-mon[118902]: pgmap v10117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:51.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:51 smithi136 ceph-mon[113531]: pgmap v10117: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:51.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:51.619696+0000) 2023-08-28T02:30:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:51.578Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:51.579Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:52.618 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/588426892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:52.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/588426892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:52.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/588426892' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:52.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:52.619918+0000) 2023-08-28T02:30:53.272 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:53.005Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:53.272 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:30:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:30:53.006Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:30:53.619 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:53 smithi136 ceph-mon[113531]: pgmap v10118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:53 smithi093 ceph-mon[111335]: pgmap v10118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:53.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:53 smithi093 ceph-mon[118902]: pgmap v10118: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:53.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:53.620192+0000) 2023-08-28T02:30:54.619 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:30:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:54.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:54.620446+0000) 2023-08-28T02:30:55.619 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:55 smithi136 ceph-mon[113531]: pgmap v10119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:55.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2277908868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:55 smithi093 ceph-mon[111335]: pgmap v10119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2277908868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:30:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:30:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:30:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:55 smithi093 ceph-mon[118902]: pgmap v10119: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2277908868' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:55.619+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:55.620695+0000) 2023-08-28T02:30:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:56.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:56.620982+0000) 2023-08-28T02:30:57.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:57 smithi136 ceph-mon[113531]: pgmap v10120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:57.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/757324457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:57 smithi093 ceph-mon[111335]: pgmap v10120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:57.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/757324457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:57 smithi093 ceph-mon[118902]: pgmap v10120: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/757324457' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:57.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:57.621258+0000) 2023-08-28T02:30:58.890 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:58.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:58.621438+0000) 2023-08-28T02:30:59.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:59 smithi136 ceph-mon[113531]: pgmap v10121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:59.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:59 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:30:59.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:30:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1819870805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:59.627 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:59 smithi093 ceph-mon[111335]: pgmap v10121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:59.627 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:59 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:30:59.627 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:30:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1819870805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:59.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:59 smithi093 ceph-mon[118902]: pgmap v10121: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:30:59.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:59 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:30:59.627 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:30:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1819870805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:30:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:30:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:30:59.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:30:59.621577+0000) 2023-08-28T02:31:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:00.620+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:00.621742+0000) 2023-08-28T02:31:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: pgmap v10122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: pgmap v10122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:01.491 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: pgmap v10122: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:01.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.0", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.1", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.2", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config rm", "who": "osd.3", "name": "osd_memory_target"}]: dispatch 2023-08-28T02:31:01.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:31:01.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:31:01.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:31:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:01.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:01.621980+0000) 2023-08-28T02:31:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:01.579Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:01.580Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:02 smithi093 ceph-mon[111335]: Adjusting osd_memory_target on smithi093 to 2363M 2023-08-28T02:31:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1881996934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:02 smithi093 ceph-mon[118902]: Adjusting osd_memory_target on smithi093 to 2363M 2023-08-28T02:31:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1881996934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:02.620 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:02 smithi136 ceph-mon[113531]: Adjusting osd_memory_target on smithi093 to 2363M 2023-08-28T02:31:02.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1881996934' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:02.621+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:02.622166+0000) 2023-08-28T02:31:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:03 smithi093 ceph-mon[111335]: pgmap v10123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:03 smithi093 ceph-mon[118902]: pgmap v10123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:03.007Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:03.007Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:03.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:03 smithi136 ceph-mon[113531]: pgmap v10123: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:03.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:03.622437+0000) 2023-08-28T02:31:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/620845543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/620845543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:04.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/620845543' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:04.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:04.622660+0000) 2023-08-28T02:31:05.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:05 smithi093 ceph-mon[111335]: pgmap v10124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:31:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:31:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:31:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:05 smithi093 ceph-mon[118902]: pgmap v10124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:05.621 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:05 smithi136 ceph-mon[113531]: pgmap v10124: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:05.622 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 02:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T02:31:05.317979194Z level=info msg="Completed cleanup jobs" duration=12.934538ms 2023-08-28T02:31:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:05.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:05.622913+0000) 2023-08-28T02:31:06.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3208502969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:06.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3208502969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:06.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3208502969' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:06.622+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:06.623187+0000) 2023-08-28T02:31:07.622 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:07 smithi136 ceph-mon[113531]: pgmap v10125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:07 smithi093 ceph-mon[111335]: pgmap v10125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:07.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:07 smithi093 ceph-mon[118902]: pgmap v10125: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:07.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:07.623396+0000) 2023-08-28T02:31:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:08.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:08.623607+0000) 2023-08-28T02:31:09.623 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:09 smithi136 ceph-mon[113531]: pgmap v10126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:09.623 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2962392419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:09.623 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:09 smithi093 ceph-mon[111335]: pgmap v10126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2962392419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:09 smithi093 ceph-mon[118902]: pgmap v10126: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2962392419' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:09.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:09.623823+0000) 2023-08-28T02:31:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:10.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:10.624106+0000) 2023-08-28T02:31:11.579 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:11 smithi093 ceph-mon[111335]: pgmap v10127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:11.579 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1358902096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:11.580 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:11 smithi093 ceph-mon[118902]: pgmap v10127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:11.580 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1358902096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:11.623 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:11 smithi136 ceph-mon[113531]: pgmap v10127: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:11.623 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1358902096' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:11.623+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:11.624290+0000) 2023-08-28T02:31:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:11.580Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:11.581Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:12.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:12.624501+0000) 2023-08-28T02:31:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:13 smithi093 ceph-mon[111335]: pgmap v10128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:13 smithi093 ceph-mon[118902]: pgmap v10128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:13.008Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:13.009Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:13.623 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:13 smithi136 ceph-mon[113531]: pgmap v10128: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:13.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:13.624759+0000) 2023-08-28T02:31:14.624 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1174912146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:14.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1174912146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1174912146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:14.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:14.625020+0000) 2023-08-28T02:31:15.624 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:15 smithi136 ceph-mon[113531]: pgmap v10129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:15 smithi093 ceph-mon[111335]: pgmap v10129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:15 smithi093 ceph-mon[118902]: pgmap v10129: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:31:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:31:15] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:31:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:15.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:15.625315+0000) 2023-08-28T02:31:16.624 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2888832676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2888832676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:16.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2888832676' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:16.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:16.625496+0000) 2023-08-28T02:31:17.624 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:17 smithi136 ceph-mon[113531]: pgmap v10130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:17 smithi093 ceph-mon[111335]: pgmap v10130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:17 smithi093 ceph-mon[118902]: pgmap v10130: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:17.624+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:17.625754+0000) 2023-08-28T02:31:18.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/319333681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/319333681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/319333681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:18.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:18.626023+0000) 2023-08-28T02:31:19.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:19 smithi136 ceph-mon[113531]: pgmap v10131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:19 smithi093 ceph-mon[111335]: pgmap v10131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:19 smithi093 ceph-mon[118902]: pgmap v10131: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:19.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:19.626291+0000) 2023-08-28T02:31:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:20.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:20.626567+0000) 2023-08-28T02:31:21.579 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:21 smithi093 ceph-mon[111335]: pgmap v10132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:21.580 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3561244893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:21.580 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:21 smithi093 ceph-mon[118902]: pgmap v10132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:21.580 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3561244893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:21.625 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:21 smithi136 ceph-mon[113531]: pgmap v10132: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:21.626 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3561244893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:21.625+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:21.626824+0000) 2023-08-28T02:31:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:21.580Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:21.581Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:22.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:22.627118+0000) 2023-08-28T02:31:23.279 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:23.008Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:23.280 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:23.009Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:23.626 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:23 smithi136 ceph-mon[113531]: pgmap v10133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:23.626 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1428269390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:23 smithi093 ceph-mon[111335]: pgmap v10133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1428269390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:23 smithi093 ceph-mon[118902]: pgmap v10133: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1428269390' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:23.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:23.627330+0000) 2023-08-28T02:31:24.626 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:24.778 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:24.778 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:24.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:24.627537+0000) 2023-08-28T02:31:25.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:25 smithi136 ceph-mon[113531]: pgmap v10134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:25.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3183007261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:25 smithi093 ceph-mon[111335]: pgmap v10134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3183007261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:25 smithi093 ceph-mon[118902]: pgmap v10134: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3183007261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:31:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:31:25] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:31:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:25.626+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:25.627797+0000) 2023-08-28T02:31:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:26.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:26.628027+0000) 2023-08-28T02:31:27.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:27 smithi136 ceph-mon[113531]: pgmap v10135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:27 smithi093 ceph-mon[111335]: pgmap v10135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:27 smithi093 ceph-mon[118902]: pgmap v10135: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:27.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:27.628297+0000) 2023-08-28T02:31:28.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3314034743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3314034743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:28.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3314034743' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:28.627+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:28.628584+0000) 2023-08-28T02:31:29.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:29 smithi136 ceph-mon[113531]: pgmap v10136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:29 smithi093 ceph-mon[111335]: pgmap v10136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:29 smithi093 ceph-mon[118902]: pgmap v10136: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:29.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:29.628840+0000) 2023-08-28T02:31:30.627 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/128527140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/128527140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/128527140' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:30.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:30.629012+0000) 2023-08-28T02:31:31.581 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:31 smithi093 ceph-mon[111335]: pgmap v10137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:31.581 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:31 smithi093 ceph-mon[118902]: pgmap v10137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:31.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:31 smithi136 ceph-mon[113531]: pgmap v10137: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:31.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:31.629244+0000) 2023-08-28T02:31:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:31.581Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:31.582Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:32.628+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:32.629489+0000) 2023-08-28T02:31:33.305 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:33.009Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:33.305 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:33.010Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:33.628 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:33 smithi136 ceph-mon[113531]: pgmap v10138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:33.629 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1422695342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:33 smithi093 ceph-mon[118902]: pgmap v10138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:33.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1422695342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:33 smithi093 ceph-mon[111335]: pgmap v10138: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1422695342' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:33.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:33.629741+0000) 2023-08-28T02:31:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:34.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:34.629992+0000) 2023-08-28T02:31:35.629 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:35 smithi136 ceph-mon[113531]: pgmap v10139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:35.629 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/817939125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:35 smithi093 ceph-mon[111335]: pgmap v10139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/817939125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:31:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:31:35] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:31:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:35 smithi093 ceph-mon[118902]: pgmap v10139: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/817939125' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:35.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:35.630256+0000) 2023-08-28T02:31:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:36.629+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:36.630472+0000) 2023-08-28T02:31:37.629 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:37 smithi136 ceph-mon[113531]: pgmap v10140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:37.629 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3354642050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:37 smithi093 ceph-mon[111335]: pgmap v10140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3354642050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:37 smithi093 ceph-mon[118902]: pgmap v10140: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3354642050' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:37.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:37.630730+0000) 2023-08-28T02:31:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:38.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:38.631054+0000) 2023-08-28T02:31:39.630 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:39 smithi136 ceph-mon[113531]: pgmap v10141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:39.630 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:39 smithi093 ceph-mon[111335]: pgmap v10141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:39 smithi093 ceph-mon[118902]: pgmap v10141: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:39.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:39.631315+0000) 2023-08-28T02:31:40.630 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2590844169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2590844169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2590844169' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:40.630+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:40.631523+0000) 2023-08-28T02:31:41.581 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:41 smithi093 ceph-mon[111335]: pgmap v10142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:41.581 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:41 smithi093 ceph-mon[118902]: pgmap v10142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:41.630 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:41 smithi136 ceph-mon[113531]: pgmap v10142: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:41.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:41.631801+0000) 2023-08-28T02:31:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:41.582Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:41.583Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:42.933 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:42.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:42.632071+0000) 2023-08-28T02:31:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:42 smithi136 ceph-mon[113531]: pgmap v10143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:43.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3818349148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:42 smithi093 ceph-mon[111335]: pgmap v10143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3818349148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:42 smithi093 ceph-mon[118902]: pgmap v10143: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3818349148' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:43.010Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:43.011Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:43.631+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:43.632335+0000) 2023-08-28T02:31:44.936 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:44.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:44.632575+0000) 2023-08-28T02:31:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:44 smithi136 ceph-mon[113531]: pgmap v10144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:45.188 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2869226322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:44 smithi093 ceph-mon[111335]: pgmap v10144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2869226322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:45.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:44 smithi093 ceph-mon[118902]: pgmap v10144: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2869226322' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:31:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:31:45] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:31:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:45.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:45.632833+0000) 2023-08-28T02:31:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:46.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:46.633101+0000) 2023-08-28T02:31:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:46 smithi093 ceph-mon[111335]: pgmap v10145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3356680072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:46 smithi093 ceph-mon[118902]: pgmap v10145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3356680072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:46 smithi136 ceph-mon[113531]: pgmap v10145: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3356680072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:47.632+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:47.633331+0000) 2023-08-28T02:31:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:48.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:48.633597+0000) 2023-08-28T02:31:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:48 smithi093 ceph-mon[111335]: pgmap v10146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:49.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:48 smithi093 ceph-mon[118902]: pgmap v10146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:48 smithi136 ceph-mon[113531]: pgmap v10146: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:49.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:49.633850+0000) 2023-08-28T02:31:50.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3750191117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:50.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3750191117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3750191117' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:50.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:50.634094+0000) 2023-08-28T02:31:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:50 smithi093 ceph-mon[111335]: pgmap v10147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:51.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:50 smithi093 ceph-mon[118902]: pgmap v10147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:50 smithi136 ceph-mon[113531]: pgmap v10147: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:51.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:51.634352+0000) 2023-08-28T02:31:51.951 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:51.583Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:51.951 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:51.584Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:52.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1358989745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:52.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1358989745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1358989745' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:52.633+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:52.634597+0000) 2023-08-28T02:31:53.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:52 smithi093 ceph-mon[111335]: pgmap v10148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:53.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:52 smithi093 ceph-mon[118902]: pgmap v10148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:53.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:53.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:31:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:31:53.012Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:31:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:52 smithi136 ceph-mon[113531]: pgmap v10148: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:53.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:53.634871+0000) 2023-08-28T02:31:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2232174262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:54.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:53 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2232174262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:54.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:53 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2232174262' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:53 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:31:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:54.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:54.635154+0000) 2023-08-28T02:31:55.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:54 smithi093 ceph-mon[111335]: pgmap v10149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:55.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:54 smithi093 ceph-mon[118902]: pgmap v10149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:54 smithi136 ceph-mon[113531]: pgmap v10149: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:31:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:31:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:31:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:55.634+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:55.635353+0000) 2023-08-28T02:31:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:56.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:56.635554+0000) 2023-08-28T02:31:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:56 smithi093 ceph-mon[111335]: pgmap v10150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:57.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3231342168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:56 smithi093 ceph-mon[118902]: pgmap v10150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:57.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3231342168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:56 smithi136 ceph-mon[113531]: pgmap v10150: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3231342168' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:57.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:57.635795+0000) 2023-08-28T02:31:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:58.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:58.636067+0000) 2023-08-28T02:31:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:58 smithi093 ceph-mon[111335]: pgmap v10151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:59.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:31:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1962276157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:58 smithi093 ceph-mon[118902]: pgmap v10151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:59.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:31:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1962276157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:58 smithi136 ceph-mon[113531]: pgmap v10151: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:31:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:31:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1962276157' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:31:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:31:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:31:59.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:31:59.636281+0000) 2023-08-28T02:32:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:00.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:00.636531+0000) 2023-08-28T02:32:01.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:00 smithi093 ceph-mon[111335]: pgmap v10152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:01.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:00 smithi093 ceph-mon[118902]: pgmap v10152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:00 smithi136 ceph-mon[113531]: pgmap v10152: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:01.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:01.636693+0000) 2023-08-28T02:32:01.986 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:01.584Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:01.986 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:01.585Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3277844453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:02.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:01 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3277844453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:02.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:01 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3277844453' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:01 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:32:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:02.635+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:02.636901+0000) 2023-08-28T02:32:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:02 smithi093 ceph-mon[111335]: pgmap v10153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:03.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:02 smithi093 ceph-mon[118902]: pgmap v10153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:03.011Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:03.012Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:03.305 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:02 smithi136 ceph-mon[113531]: pgmap v10153: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:03.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:03.637052+0000) 2023-08-28T02:32:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1951035614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1951035614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:04.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:04 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1951035614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:04.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:32:04.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:32:04.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:32:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:04.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:04.637259+0000) 2023-08-28T02:32:05.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:05 smithi093 ceph-mon[111335]: pgmap v10154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:05.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:32:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:32:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:32:05.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:05 smithi093 ceph-mon[118902]: pgmap v10154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:05.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:05 smithi136 ceph-mon[113531]: pgmap v10154: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:05.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:05.637475+0000) 2023-08-28T02:32:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3480804415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3480804415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:06.636 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3480804415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:06.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:06.637668+0000) 2023-08-28T02:32:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:07 smithi093 ceph-mon[111335]: pgmap v10155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:07 smithi093 ceph-mon[118902]: pgmap v10155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:07.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:07 smithi136 ceph-mon[113531]: pgmap v10155: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:07.636+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:07.637902+0000) 2023-08-28T02:32:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1729121655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:08.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1729121655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:08.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1729121655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:08.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:08.638150+0000) 2023-08-28T02:32:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:09 smithi093 ceph-mon[111335]: pgmap v10156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:09 smithi093 ceph-mon[118902]: pgmap v10156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:09.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:09 smithi136 ceph-mon[113531]: pgmap v10156: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:09.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:09.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:09.638376+0000) 2023-08-28T02:32:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:10.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:10.638650+0000) 2023-08-28T02:32:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:11 smithi093 ceph-mon[111335]: pgmap v10157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1717176703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:11 smithi093 ceph-mon[118902]: pgmap v10157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1717176703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:11.637 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:11 smithi136 ceph-mon[113531]: pgmap v10157: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:11.638 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1717176703' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:11.637+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:11.638885+0000) 2023-08-28T02:32:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:11.585Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:11.586Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:12.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:12.639136+0000) 2023-08-28T02:32:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:13 smithi093 ceph-mon[111335]: pgmap v10158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3145117715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:13 smithi093 ceph-mon[118902]: pgmap v10158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3145117715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:13.012Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:13.013Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:13.638 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:13 smithi136 ceph-mon[113531]: pgmap v10158: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:13.638 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3145117715' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:13.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:13.639375+0000) 2023-08-28T02:32:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:14.638+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:14.639622+0000) 2023-08-28T02:32:15.638 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:15 smithi136 ceph-mon[113531]: pgmap v10159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:15.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:15 smithi093 ceph-mon[111335]: pgmap v10159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:15.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:32:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:32:15] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:32:15.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:15 smithi093 ceph-mon[118902]: pgmap v10159: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:15.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:15.639866+0000) 2023-08-28T02:32:16.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2760797600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:16.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2760797600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2760797600' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:16.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:16.640081+0000) 2023-08-28T02:32:17.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:17 smithi136 ceph-mon[113531]: pgmap v10160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:17 smithi093 ceph-mon[111335]: pgmap v10160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:17 smithi093 ceph-mon[118902]: pgmap v10160: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:17.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:17.640278+0000) 2023-08-28T02:32:18.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3472342028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:18.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3472342028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:18.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3472342028' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:18.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:18.640510+0000) 2023-08-28T02:32:19.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:19 smithi136 ceph-mon[113531]: pgmap v10161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:19 smithi093 ceph-mon[111335]: pgmap v10161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:19 smithi093 ceph-mon[118902]: pgmap v10161: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:19.639+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:19.640673+0000) 2023-08-28T02:32:20.639 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3693423677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:20.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3693423677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:20.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3693423677' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:20.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:20.640924+0000) 2023-08-28T02:32:21.585 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:21 smithi093 ceph-mon[111335]: pgmap v10162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:21.585 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:21 smithi093 ceph-mon[118902]: pgmap v10162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:21.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:21 smithi136 ceph-mon[113531]: pgmap v10162: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:21.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:21.641168+0000) 2023-08-28T02:32:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:21.586Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:21.586Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:22.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:22.641409+0000) 2023-08-28T02:32:23.265 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:23.012Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:23.265 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:23.013Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:23.642 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:23 smithi136 ceph-mon[113531]: pgmap v10163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:23.642 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1384342823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:23 smithi093 ceph-mon[111335]: pgmap v10163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1384342823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:23 smithi093 ceph-mon[118902]: pgmap v10163: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1384342823' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:23.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:23.641658+0000) 2023-08-28T02:32:24.640 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:24.640+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:24.641935+0000) 2023-08-28T02:32:25.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:25 smithi136 ceph-mon[113531]: pgmap v10164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:25.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2011702073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:25 smithi093 ceph-mon[111335]: pgmap v10164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2011702073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:32:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:32:25] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:32:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:25 smithi093 ceph-mon[118902]: pgmap v10164: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2011702073' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:25.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:25.642134+0000) 2023-08-28T02:32:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:26.641+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:26.642390+0000) 2023-08-28T02:32:27.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:27 smithi136 ceph-mon[113531]: pgmap v10165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:27.641 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1263923559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:27 smithi093 ceph-mon[111335]: pgmap v10165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1263923559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:27 smithi093 ceph-mon[118902]: pgmap v10165: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1263923559' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:27.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:27.642632+0000) 2023-08-28T02:32:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:28.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:28.642863+0000) 2023-08-28T02:32:29.642 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:29 smithi136 ceph-mon[113531]: pgmap v10166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:29 smithi093 ceph-mon[111335]: pgmap v10166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:29 smithi093 ceph-mon[118902]: pgmap v10166: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:29.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:29.643147+0000) 2023-08-28T02:32:30.642 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3879112477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3879112477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3879112477' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:30.642+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:30.643396+0000) 2023-08-28T02:32:31.586 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:31 smithi093 ceph-mon[111335]: pgmap v10167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:31.586 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:31 smithi093 ceph-mon[118902]: pgmap v10167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:31.642 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:31 smithi136 ceph-mon[113531]: pgmap v10167: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:31.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:31.643623+0000) 2023-08-28T02:32:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:31.586Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:31.587Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:32.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:32.643861+0000) 2023-08-28T02:32:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:32 smithi093 ceph-mon[111335]: pgmap v10168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/895511767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:32 smithi093 ceph-mon[118902]: pgmap v10168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/895511767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:33.013Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:33.014Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:32 smithi136 ceph-mon[113531]: pgmap v10168: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/895511767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:33.941 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:33.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:33.644136+0000) 2023-08-28T02:32:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:34.643+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:34.644307+0000) 2023-08-28T02:32:35.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:34 smithi093 ceph-mon[118902]: pgmap v10169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1072373938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:34 smithi093 ceph-mon[111335]: pgmap v10169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1072373938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:34 smithi136 ceph-mon[113531]: pgmap v10169: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1072373938' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:32:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:32:35] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:32:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:35.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:35.644559+0000) 2023-08-28T02:32:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:36.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:36.644820+0000) 2023-08-28T02:32:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:36 smithi093 ceph-mon[111335]: pgmap v10170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:37.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1660630086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:36 smithi093 ceph-mon[118902]: pgmap v10170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1660630086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:36 smithi136 ceph-mon[113531]: pgmap v10170: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1660630086' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:37.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:37.645091+0000) 2023-08-28T02:32:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:38.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:38.645284+0000) 2023-08-28T02:32:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:38 smithi093 ceph-mon[111335]: pgmap v10171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:38 smithi093 ceph-mon[118902]: pgmap v10171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:38 smithi136 ceph-mon[113531]: pgmap v10171: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:39.644+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:39.645536+0000) 2023-08-28T02:32:40.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2576907970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:40.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2576907970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2576907970' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:40.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:40.645807+0000) 2023-08-28T02:32:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:40 smithi093 ceph-mon[111335]: pgmap v10172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:40 smithi093 ceph-mon[118902]: pgmap v10172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:40 smithi136 ceph-mon[113531]: pgmap v10172: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:41.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:41.646075+0000) 2023-08-28T02:32:41.968 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:41.587Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:41.968 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:41.588Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:42.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4189932431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:42.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4189932431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4189932431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:42.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:42.646316+0000) 2023-08-28T02:32:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:42 smithi093 ceph-mon[111335]: pgmap v10173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:42 smithi093 ceph-mon[118902]: pgmap v10173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:43.014Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:43.015Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:42 smithi136 ceph-mon[113531]: pgmap v10173: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:43.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:43.646552+0000) 2023-08-28T02:32:44.240 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1440651272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:44.241 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1440651272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1440651272' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:44.645+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:44.646815+0000) 2023-08-28T02:32:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:44 smithi093 ceph-mon[111335]: pgmap v10174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:44 smithi093 ceph-mon[118902]: pgmap v10174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:44 smithi136 ceph-mon[113531]: pgmap v10174: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:32:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:32:45] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:32:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:45.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:45.647020+0000) 2023-08-28T02:32:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:46.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:46.647286+0000) 2023-08-28T02:32:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:46 smithi093 ceph-mon[111335]: pgmap v10175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:47.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2540866448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:46 smithi093 ceph-mon[118902]: pgmap v10175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2540866448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:46 smithi136 ceph-mon[113531]: pgmap v10175: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2540866448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:47.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:47.647524+0000) 2023-08-28T02:32:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:48.646+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:48.647762+0000) 2023-08-28T02:32:49.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:48 smithi093 ceph-mon[111335]: pgmap v10176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:49.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2256206945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:49.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:48 smithi093 ceph-mon[118902]: pgmap v10176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:49.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2256206945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:48 smithi136 ceph-mon[113531]: pgmap v10176: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2256206945' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:49.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:49.647987+0000) 2023-08-28T02:32:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:50.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:50.648208+0000) 2023-08-28T02:32:51.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:50 smithi093 ceph-mon[111335]: pgmap v10177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:51.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:50 smithi093 ceph-mon[118902]: pgmap v10177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:50 smithi136 ceph-mon[113531]: pgmap v10177: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:51.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:51.648415+0000) 2023-08-28T02:32:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:51.588Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:51.589Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3456449722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3456449722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3456449722' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:52.647+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:52.648649+0000) 2023-08-28T02:32:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:52 smithi136 ceph-mon[113531]: pgmap v10178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:52 smithi093 ceph-mon[111335]: pgmap v10178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:53.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:52 smithi093 ceph-mon[118902]: pgmap v10178: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:53.015Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:32:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:32:53.016Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:32:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:53.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:53.648848+0000) 2023-08-28T02:32:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1587464500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1587464500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1587464500' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:32:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:54.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:54.649125+0000) 2023-08-28T02:32:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:55 smithi093 ceph-mon[111335]: pgmap v10179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:55.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:55 smithi093 ceph-mon[118902]: pgmap v10179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:55 smithi136 ceph-mon[113531]: pgmap v10179: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:32:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:32:55] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:32:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:55.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:55.649385+0000) 2023-08-28T02:32:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3439132614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3439132614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3439132614' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:56.648+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:56.649641+0000) 2023-08-28T02:32:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:57 smithi136 ceph-mon[113531]: pgmap v10180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:57 smithi093 ceph-mon[111335]: pgmap v10180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:57 smithi093 ceph-mon[118902]: pgmap v10180: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:57.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:57.649867+0000) 2023-08-28T02:32:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:58.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:58.650137+0000) 2023-08-28T02:32:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:59 smithi136 ceph-mon[113531]: pgmap v10181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:32:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3429336145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:59 smithi093 ceph-mon[111335]: pgmap v10181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:32:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3429336145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:59 smithi093 ceph-mon[118902]: pgmap v10181: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:32:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:32:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3429336145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:32:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:32:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:32:59.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:32:59.650383+0000) 2023-08-28T02:33:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:00.649+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:00.650665+0000) 2023-08-28T02:33:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:01 smithi136 ceph-mon[113531]: pgmap v10182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3439883835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:01 smithi093 ceph-mon[111335]: pgmap v10182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3439883835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:01 smithi093 ceph-mon[118902]: pgmap v10182: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3439883835' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:01.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:01.650922+0000) 2023-08-28T02:33:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:01.589Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:01.590Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:02.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:02.651156+0000) 2023-08-28T02:33:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:03 smithi136 ceph-mon[113531]: pgmap v10183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1983258979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:03 smithi093 ceph-mon[111335]: pgmap v10183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1983258979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:03 smithi093 ceph-mon[118902]: pgmap v10183: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1983258979' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:03.016Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:03.017Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:03.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:03.651403+0000) 2023-08-28T02:33:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:04.650+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:04.651574+0000) 2023-08-28T02:33:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:05 smithi093 ceph-mon[111335]: pgmap v10184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:05.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:05 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:05 smithi093 ceph-mon[118902]: pgmap v10184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:05 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:05 smithi136 ceph-mon[113531]: pgmap v10184: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:05 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:33:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:33:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:33:05] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:33:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:05.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:05.651826+0000) 2023-08-28T02:33:06.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2111280093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:06.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2111280093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:06.291 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2111280093' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:06.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:06.652054+0000) 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: pgmap v10185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:33:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: pgmap v10185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: pgmap v10185: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:33:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:33:07.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:07 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:33:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:07.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:07.652266+0000) 2023-08-28T02:33:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/487185624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/487185624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:08.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/487185624' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:08.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:08.652464+0000) 2023-08-28T02:33:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:09 smithi136 ceph-mon[113531]: pgmap v10186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:09 smithi093 ceph-mon[111335]: pgmap v10186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:09 smithi093 ceph-mon[118902]: pgmap v10186: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:09.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:09.652721+0000) 2023-08-28T02:33:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1504494738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1504494738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:10.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1504494738' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:10.651+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:10.652995+0000) 2023-08-28T02:33:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:11 smithi136 ceph-mon[113531]: pgmap v10187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:11 smithi093 ceph-mon[111335]: pgmap v10187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:11 smithi093 ceph-mon[118902]: pgmap v10187: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:11.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:11.653282+0000) 2023-08-28T02:33:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:11.590Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:11.591Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:12.652+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:12.653516+0000) 2023-08-28T02:33:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:13 smithi136 ceph-mon[113531]: pgmap v10188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3658860348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:13 smithi093 ceph-mon[111335]: pgmap v10188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3658860348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:13 smithi093 ceph-mon[118902]: pgmap v10188: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3658860348' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:13.017Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:13.018Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:13.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:13.653711+0000) 2023-08-28T02:33:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:14.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:14.653921+0000) 2023-08-28T02:33:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:15 smithi136 ceph-mon[113531]: pgmap v10189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:15.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/138839644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:15.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:33:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:33:15] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:33:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:15 smithi093 ceph-mon[118902]: pgmap v10189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:15.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/138839644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:15 smithi093 ceph-mon[111335]: pgmap v10189: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:15.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/138839644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:15.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:15.654144+0000) 2023-08-28T02:33:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:16.653+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:16.654363+0000) 2023-08-28T02:33:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:17 smithi136 ceph-mon[113531]: pgmap v10190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:17 smithi093 ceph-mon[111335]: pgmap v10190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:17 smithi093 ceph-mon[118902]: pgmap v10190: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:17.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:17.654600+0000) 2023-08-28T02:33:18.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/316584909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:18.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/316584909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:18.653 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/316584909' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:18.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:18.654860+0000) 2023-08-28T02:33:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:19 smithi093 ceph-mon[111335]: pgmap v10191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:19 smithi093 ceph-mon[118902]: pgmap v10191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:19.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:19 smithi136 ceph-mon[113531]: pgmap v10191: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:19.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:19.655154+0000) 2023-08-28T02:33:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1004872718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1004872718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:20.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1004872718' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:20.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:20.655407+0000) 2023-08-28T02:33:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:21 smithi093 ceph-mon[111335]: pgmap v10192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:21 smithi093 ceph-mon[118902]: pgmap v10192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:21.654 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:21 smithi136 ceph-mon[113531]: pgmap v10192: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:21.654+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:21.655697+0000) 2023-08-28T02:33:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:21.591Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:21.592Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:22.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:22.655916+0000) 2023-08-28T02:33:23.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:22 smithi093 ceph-mon[111335]: pgmap v10193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:23.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2112840079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:22 smithi093 ceph-mon[118902]: pgmap v10193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:23.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2112840079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:23.018Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:23.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:23.019Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:22 smithi136 ceph-mon[113531]: pgmap v10193: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2112840079' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:23.655+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:23.656150+0000) 2023-08-28T02:33:24.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:24.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:24.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:24.656373+0000) 2023-08-28T02:33:25.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:24 smithi093 ceph-mon[111335]: pgmap v10194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:25.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1263628054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:24 smithi093 ceph-mon[118902]: pgmap v10194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:25.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1263628054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:24 smithi136 ceph-mon[113531]: pgmap v10194: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1263628054' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:33:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:33:25] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:33:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:25.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:25.656669+0000) 2023-08-28T02:33:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:26.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:26.656947+0000) 2023-08-28T02:33:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:26 smithi093 ceph-mon[111335]: pgmap v10195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2117662781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:26 smithi093 ceph-mon[118902]: pgmap v10195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2117662781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:26 smithi136 ceph-mon[113531]: pgmap v10195: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2117662781' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:27.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:27.657154+0000) 2023-08-28T02:33:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:28.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:28.657393+0000) 2023-08-28T02:33:29.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:28 smithi093 ceph-mon[111335]: pgmap v10196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:29.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:28 smithi093 ceph-mon[118902]: pgmap v10196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:28 smithi136 ceph-mon[113531]: pgmap v10196: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:29.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:29.657637+0000) 2023-08-28T02:33:30.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/992923491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:30.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/992923491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:30.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/992923491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:30.656+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:30.657911+0000) 2023-08-28T02:33:31.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:30 smithi093 ceph-mon[111335]: pgmap v10197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:31.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:30 smithi093 ceph-mon[118902]: pgmap v10197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:30 smithi136 ceph-mon[113531]: pgmap v10197: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:31.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:31.658164+0000) 2023-08-28T02:33:31.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:31.592Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:31.972 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:31.593Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:32.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3568339491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:32.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3568339491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3568339491' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:32.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:32.658425+0000) 2023-08-28T02:33:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:32 smithi093 ceph-mon[111335]: pgmap v10198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:32 smithi093 ceph-mon[118902]: pgmap v10198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:33.019Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:33.020Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:32 smithi136 ceph-mon[113531]: pgmap v10198: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:33.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:33.658673+0000) 2023-08-28T02:33:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:34.657+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:34.658941+0000) 2023-08-28T02:33:35.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:34 smithi093 ceph-mon[111335]: pgmap v10199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:35.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3162535145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:34 smithi093 ceph-mon[118902]: pgmap v10199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:35.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3162535145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:34 smithi136 ceph-mon[113531]: pgmap v10199: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3162535145' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:33:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:33:35] "GET /metrics HTTP/1.1" 200 239944 "" "Prometheus/2.43.0" 2023-08-28T02:33:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:35.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:35.659173+0000) 2023-08-28T02:33:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:36.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:36.659401+0000) 2023-08-28T02:33:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:36 smithi093 ceph-mon[111335]: pgmap v10200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:37.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1881202170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:36 smithi093 ceph-mon[118902]: pgmap v10200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:37.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1881202170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:36 smithi136 ceph-mon[113531]: pgmap v10200: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1881202170' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:37.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:37.659625+0000) 2023-08-28T02:33:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:38.658+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:38.659809+0000) 2023-08-28T02:33:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:38 smithi093 ceph-mon[111335]: pgmap v10201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3948320617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:39.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:38 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:38 smithi093 ceph-mon[118902]: pgmap v10201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3948320617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:39.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:38 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:38 smithi136 ceph-mon[113531]: pgmap v10201: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3948320617' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:38 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:39.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:39.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:39.660063+0000) 2023-08-28T02:33:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:40.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:40.660292+0000) 2023-08-28T02:33:41.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:40 smithi093 ceph-mon[111335]: pgmap v10202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:41.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:40 smithi093 ceph-mon[118902]: pgmap v10202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:40 smithi136 ceph-mon[113531]: pgmap v10202: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:41.659+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:41.660556+0000) 2023-08-28T02:33:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:41.593Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:41.593Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2142029403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2142029403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:42.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2142029403' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:42.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:42.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:42.660793+0000) 2023-08-28T02:33:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:42 smithi136 ceph-mon[113531]: pgmap v10203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:42 smithi093 ceph-mon[111335]: pgmap v10203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:42 smithi093 ceph-mon[118902]: pgmap v10203: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:43.020Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:43.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:43.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:43.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:43.661081+0000) 2023-08-28T02:33:44.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2026359431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:44.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2026359431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:44.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2026359431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:44.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:44.661294+0000) 2023-08-28T02:33:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:45 smithi093 ceph-mon[111335]: pgmap v10204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:45 smithi093 ceph-mon[118902]: pgmap v10204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:45 smithi136 ceph-mon[113531]: pgmap v10204: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:33:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:33:45] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:33:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:45.660+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:45.661520+0000) 2023-08-28T02:33:46.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2504257590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2504257590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2504257590' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:46.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:46.661786+0000) 2023-08-28T02:33:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:47 smithi136 ceph-mon[113531]: pgmap v10205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:47 smithi093 ceph-mon[111335]: pgmap v10205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:47 smithi093 ceph-mon[118902]: pgmap v10205: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:47.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:47.662061+0000) 2023-08-28T02:33:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:48.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:48.662287+0000) 2023-08-28T02:33:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:49 smithi136 ceph-mon[113531]: pgmap v10206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1438777693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:49 smithi093 ceph-mon[111335]: pgmap v10206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:49.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1438777693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:49 smithi093 ceph-mon[118902]: pgmap v10206: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:49.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1438777693' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:49.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:49.662488+0000) 2023-08-28T02:33:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:50.661+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:50.662751+0000) 2023-08-28T02:33:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:51 smithi136 ceph-mon[113531]: pgmap v10207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3216584254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:51 smithi093 ceph-mon[111335]: pgmap v10207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:51.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3216584254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:51 smithi093 ceph-mon[118902]: pgmap v10207: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3216584254' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:51.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:51.663026+0000) 2023-08-28T02:33:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:51.594Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:51.595Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:52.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:52.663256+0000) 2023-08-28T02:33:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:53 smithi136 ceph-mon[113531]: pgmap v10208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:53 smithi093 ceph-mon[111335]: pgmap v10208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:53 smithi093 ceph-mon[118902]: pgmap v10208: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:53.021Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:33:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:33:53.021Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:33:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:53.662+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:53.663551+0000) 2023-08-28T02:33:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2505533982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2505533982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2505533982' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:33:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:54.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:54.663783+0000) 2023-08-28T02:33:55.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:55 smithi093 ceph-mon[111335]: pgmap v10209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:55.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:55 smithi093 ceph-mon[118902]: pgmap v10209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:55 smithi136 ceph-mon[113531]: pgmap v10209: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:33:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:33:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:33:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:55.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:55.664008+0000) 2023-08-28T02:33:56.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1955609708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:56.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1955609708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:56.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1955609708' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:56.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:56.664243+0000) 2023-08-28T02:33:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:57 smithi136 ceph-mon[113531]: pgmap v10210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:57 smithi093 ceph-mon[111335]: pgmap v10210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:57 smithi093 ceph-mon[118902]: pgmap v10210: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:57.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:57.664483+0000) 2023-08-28T02:33:58.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2304282080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2304282080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:58.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2304282080' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:33:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:58.663+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:58.664700+0000) 2023-08-28T02:33:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:33:59 smithi136 ceph-mon[113531]: pgmap v10211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:33:59 smithi093 ceph-mon[111335]: pgmap v10211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:33:59 smithi093 ceph-mon[118902]: pgmap v10211: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:33:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:33:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:33:59.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:33:59.664973+0000) 2023-08-28T02:34:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:00.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:00.665241+0000) 2023-08-28T02:34:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:01 smithi136 ceph-mon[113531]: pgmap v10212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3907906104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:01 smithi093 ceph-mon[111335]: pgmap v10212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3907906104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:01 smithi093 ceph-mon[118902]: pgmap v10212: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3907906104' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:01.664+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:01.665470+0000) 2023-08-28T02:34:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:01.594Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:01.595Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:02.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:02.665698+0000) 2023-08-28T02:34:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:03 smithi136 ceph-mon[113531]: pgmap v10213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4282695065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:03 smithi093 ceph-mon[111335]: pgmap v10213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4282695065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:03 smithi093 ceph-mon[118902]: pgmap v10213: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4282695065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:03.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:03.021Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:03.490 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:03.022Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:03.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:03.665933+0000) 2023-08-28T02:34:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:04.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:04.666184+0000) 2023-08-28T02:34:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:05 smithi093 ceph-mon[111335]: pgmap v10214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2483524092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:05 smithi093 ceph-mon[118902]: pgmap v10214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2483524092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:05 smithi136 ceph-mon[113531]: pgmap v10214: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2483524092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:34:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:34:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:34:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:05.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:05.666352+0000) 2023-08-28T02:34:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:06.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:06.666552+0000) 2023-08-28T02:34:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:07 smithi136 ceph-mon[113531]: pgmap v10215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:07 smithi093 ceph-mon[111335]: pgmap v10215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:07 smithi093 ceph-mon[118902]: pgmap v10215: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:07.665+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:07.666738+0000) 2023-08-28T02:34:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:34:08.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1866723540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:08.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:34:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1866723540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:08.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:34:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1866723540' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:08.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:08.666927+0000) 2023-08-28T02:34:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:09 smithi093 ceph-mon[111335]: pgmap v10216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:09.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:09 smithi093 ceph-mon[118902]: pgmap v10216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:09.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:09.412 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:09 smithi136 ceph-mon[113531]: pgmap v10216: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:09.412 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:09.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:09.667116+0000) 2023-08-28T02:34:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/857414407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:10.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/857414407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/857414407' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:34:10.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:34:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:10.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:10.667289+0000) 2023-08-28T02:34:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:11 smithi136 ceph-mon[113531]: pgmap v10217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:11 smithi093 ceph-mon[111335]: pgmap v10217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:11 smithi093 ceph-mon[118902]: pgmap v10217: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:11.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:11.667578+0000) 2023-08-28T02:34:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:11.594Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:11.596Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:12.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:12.667788+0000) 2023-08-28T02:34:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:13 smithi136 ceph-mon[113531]: pgmap v10218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2418339113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:13.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:13 smithi093 ceph-mon[118902]: pgmap v10218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:13.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2418339113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:13 smithi093 ceph-mon[111335]: pgmap v10218: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:13.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2418339113' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:13.023Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:13.024Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:13.666+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:13.668011+0000) 2023-08-28T02:34:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:14.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:14.668196+0000) 2023-08-28T02:34:15.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:15 smithi093 ceph-mon[111335]: pgmap v10219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2486067531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:15 smithi093 ceph-mon[118902]: pgmap v10219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2486067531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:15 smithi136 ceph-mon[113531]: pgmap v10219: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2486067531' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:34:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:34:15] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:34:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:15.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:15.668431+0000) 2023-08-28T02:34:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:16.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:16.668644+0000) 2023-08-28T02:34:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:17 smithi136 ceph-mon[113531]: pgmap v10220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/716478556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:17 smithi093 ceph-mon[111335]: pgmap v10220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/716478556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:17 smithi093 ceph-mon[118902]: pgmap v10220: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/716478556' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:17.667+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:17.668860+0000) 2023-08-28T02:34:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:18.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:18.669103+0000) 2023-08-28T02:34:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:19 smithi136 ceph-mon[113531]: pgmap v10221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:19 smithi093 ceph-mon[111335]: pgmap v10221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:19 smithi093 ceph-mon[118902]: pgmap v10221: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:19.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:19.669308+0000) 2023-08-28T02:34:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4012974926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4012974926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4012974926' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:20.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:20.669576+0000) 2023-08-28T02:34:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:21 smithi136 ceph-mon[113531]: pgmap v10222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:21 smithi093 ceph-mon[111335]: pgmap v10222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:21 smithi093 ceph-mon[118902]: pgmap v10222: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:21.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:21.668+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:21.669818+0000) 2023-08-28T02:34:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:21.595Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:21.595Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2370439380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2370439380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2370439380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:22.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:22.670080+0000) 2023-08-28T02:34:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:23 smithi136 ceph-mon[113531]: pgmap v10223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:23 smithi093 ceph-mon[111335]: pgmap v10223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:23 smithi093 ceph-mon[118902]: pgmap v10223: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:23.023Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:23.024Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:23.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:23.670347+0000) 2023-08-28T02:34:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/333872906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/333872906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/333872906' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:24.669+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:24.670627+0000) 2023-08-28T02:34:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:25 smithi136 ceph-mon[113531]: pgmap v10224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:25.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:25 smithi093 ceph-mon[111335]: pgmap v10224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:25.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:34:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:34:25] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:34:25.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:25 smithi093 ceph-mon[118902]: pgmap v10224: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:25.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:25.670900+0000) 2023-08-28T02:34:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:26.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:26.671167+0000) 2023-08-28T02:34:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:27 smithi136 ceph-mon[113531]: pgmap v10225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1123682256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:27 smithi093 ceph-mon[111335]: pgmap v10225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1123682256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:27 smithi093 ceph-mon[118902]: pgmap v10225: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:27.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1123682256' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:27.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:27.671392+0000) 2023-08-28T02:34:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:28.670+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:28.671645+0000) 2023-08-28T02:34:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:29 smithi136 ceph-mon[113531]: pgmap v10226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4027128332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:29 smithi093 ceph-mon[111335]: pgmap v10226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4027128332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:29 smithi093 ceph-mon[118902]: pgmap v10226: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4027128332' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:29.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:29.671898+0000) 2023-08-28T02:34:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:30.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:30.672156+0000) 2023-08-28T02:34:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:31 smithi136 ceph-mon[113531]: pgmap v10227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:31 smithi093 ceph-mon[111335]: pgmap v10227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:31.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:31 smithi093 ceph-mon[118902]: pgmap v10227: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:31.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:31.672426+0000) 2023-08-28T02:34:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:31.595Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:31.596Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:32.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2550725434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:32.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2550725434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:32.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2550725434' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:32.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:32.672722+0000) 2023-08-28T02:34:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:33 smithi136 ceph-mon[113531]: pgmap v10228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:33 smithi093 ceph-mon[111335]: pgmap v10228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:33 smithi093 ceph-mon[118902]: pgmap v10228: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:33.024Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:33.024Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:33.671+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:33.673007+0000) 2023-08-28T02:34:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3655008025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3655008025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3655008025' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:34.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:34.673256+0000) 2023-08-28T02:34:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:35 smithi136 ceph-mon[113531]: pgmap v10229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:35 smithi093 ceph-mon[111335]: pgmap v10229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:35.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:34:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:34:35] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:34:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:35 smithi093 ceph-mon[118902]: pgmap v10229: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:35.672+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:35.673503+0000) 2023-08-28T02:34:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1679737502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1679737502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1679737502' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:36.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:36.673752+0000) 2023-08-28T02:34:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:37 smithi136 ceph-mon[113531]: pgmap v10230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:37 smithi093 ceph-mon[111335]: pgmap v10230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:37.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:37 smithi093 ceph-mon[118902]: pgmap v10230: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:37.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:37.673985+0000) 2023-08-28T02:34:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:38.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:38.674273+0000) 2023-08-28T02:34:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:39 smithi136 ceph-mon[113531]: pgmap v10231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4292091714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:39 smithi093 ceph-mon[111335]: pgmap v10231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4292091714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:39.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:39 smithi093 ceph-mon[118902]: pgmap v10231: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4292091714' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:39.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:39.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:39.674570+0000) 2023-08-28T02:34:40.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:40.673+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:40.674783+0000) 2023-08-28T02:34:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:41 smithi136 ceph-mon[113531]: pgmap v10232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1414194792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:41 smithi093 ceph-mon[111335]: pgmap v10232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1414194792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:41 smithi093 ceph-mon[118902]: pgmap v10232: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1414194792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:41.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:41.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:41.675065+0000) 2023-08-28T02:34:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:41.596Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:41.597Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:42.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:42.674+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:42.675350+0000) 2023-08-28T02:34:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:43 smithi093 ceph-mon[118902]: pgmap v10233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2509916767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:43 smithi093 ceph-mon[111335]: pgmap v10233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2509916767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:43.024Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:43.025Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:43.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:43 smithi136 ceph-mon[113531]: pgmap v10233: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:43.595 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2509916767' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:43.939 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:43.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:43.675607+0000) 2023-08-28T02:34:44.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:44.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:44.675834+0000) 2023-08-28T02:34:45.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:45 smithi093 ceph-mon[111335]: pgmap v10234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:34:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:34:45] "GET /metrics HTTP/1.1" 200 239941 "" "Prometheus/2.43.0" 2023-08-28T02:34:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:45 smithi093 ceph-mon[118902]: pgmap v10234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:45.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:45 smithi136 ceph-mon[113531]: pgmap v10234: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:45.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:45.675+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:45.676134+0000) 2023-08-28T02:34:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1873251568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1873251568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:46.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1873251568' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:46.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:46.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:46.676363+0000) 2023-08-28T02:34:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:47 smithi093 ceph-mon[111335]: pgmap v10235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:47 smithi093 ceph-mon[118902]: pgmap v10235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:47.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:47 smithi136 ceph-mon[113531]: pgmap v10235: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:47.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:47.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:47.676605+0000) 2023-08-28T02:34:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1237801847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1237801847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:48.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1237801847' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:48.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:48.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:48.676818+0000) 2023-08-28T02:34:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:49 smithi093 ceph-mon[111335]: pgmap v10236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:49 smithi093 ceph-mon[118902]: pgmap v10236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:49.675 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:49 smithi136 ceph-mon[113531]: pgmap v10236: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:49.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:49.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:49.677086+0000) 2023-08-28T02:34:50.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:50.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:50.677304+0000) 2023-08-28T02:34:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:51 smithi093 ceph-mon[111335]: pgmap v10237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/957472367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:51 smithi093 ceph-mon[118902]: pgmap v10237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/957472367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:51.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:51 smithi136 ceph-mon[113531]: pgmap v10237: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:51.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/957472367' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:51.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:51.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:51.677545+0000) 2023-08-28T02:34:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:51.597Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:51.598Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:52.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:52.676+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:52.677773+0000) 2023-08-28T02:34:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:53 smithi093 ceph-mon[111335]: pgmap v10238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4258356448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:53 smithi093 ceph-mon[118902]: pgmap v10238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4258356448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:53.025Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:34:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:34:53.026Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:34:53.676 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:53 smithi136 ceph-mon[113531]: pgmap v10238: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:53.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4258356448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:53.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:53.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:53.678065+0000) 2023-08-28T02:34:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:54.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:34:54.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:54.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:54.678303+0000) 2023-08-28T02:34:55.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:55 smithi093 ceph-mon[111335]: pgmap v10239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2358106411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:55.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:34:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:34:55] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:34:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:55 smithi093 ceph-mon[118902]: pgmap v10239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2358106411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:55.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:55 smithi136 ceph-mon[113531]: pgmap v10239: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:55.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2358106411' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:55.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:55.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:55.678564+0000) 2023-08-28T02:34:56.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:56.677+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:56.678819+0000) 2023-08-28T02:34:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:57 smithi093 ceph-mon[111335]: pgmap v10240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:57 smithi093 ceph-mon[118902]: pgmap v10240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:57.677 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:57 smithi136 ceph-mon[113531]: pgmap v10240: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:57.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:57.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:57.679098+0000) 2023-08-28T02:34:58.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2101581996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:58.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2101581996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:58.678 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2101581996' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:34:58.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:58.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:58.679392+0000) 2023-08-28T02:34:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:34:59 smithi093 ceph-mon[111335]: pgmap v10241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:34:59 smithi093 ceph-mon[118902]: pgmap v10241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:59.678 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:34:59 smithi136 ceph-mon[113531]: pgmap v10241: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:34:59.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:34:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:34:59.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:34:59.679609+0000) 2023-08-28T02:35:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1161931873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1161931873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:00.678 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1161931873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:00.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:00.678+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:00.679848+0000) 2023-08-28T02:35:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:01 smithi093 ceph-mon[111335]: pgmap v10242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:01 smithi093 ceph-mon[118902]: pgmap v10242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:01.679 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:01 smithi136 ceph-mon[113531]: pgmap v10242: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:01.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:01.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:01.680161+0000) 2023-08-28T02:35:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:01.597Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:01.598Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:02.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:02.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:02.680435+0000) 2023-08-28T02:35:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:02 smithi093 ceph-mon[111335]: pgmap v10243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:03.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2393622062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:02 smithi093 ceph-mon[118902]: pgmap v10243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:03.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2393622062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:03.026Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:03.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:03.027Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:02 smithi136 ceph-mon[113531]: pgmap v10243: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2393622062' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:03.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:03.679+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:03.680724+0000) 2023-08-28T02:35:04.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:04.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:04.680955+0000) 2023-08-28T02:35:05.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:04 smithi093 ceph-mon[118902]: pgmap v10244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:05.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1508275326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:04 smithi093 ceph-mon[111335]: pgmap v10244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:05.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1508275326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:04 smithi136 ceph-mon[113531]: pgmap v10244: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1508275326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:35:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:35:05] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:35:05.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:05.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:05.681211+0000) 2023-08-28T02:35:06.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:06.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:06.681481+0000) 2023-08-28T02:35:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:06 smithi093 ceph-mon[111335]: pgmap v10245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:07.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3708794681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:07.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:06 smithi093 ceph-mon[118902]: pgmap v10245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:07.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3708794681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:06 smithi136 ceph-mon[113531]: pgmap v10245: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3708794681' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:07.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:07.680+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:07.681729+0000) 2023-08-28T02:35:08.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:08.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:08.682012+0000) 2023-08-28T02:35:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:08 smithi093 ceph-mon[111335]: pgmap v10246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:09.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:08 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:08 smithi093 ceph-mon[118902]: pgmap v10246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:09.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:08 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:08 smithi136 ceph-mon[113531]: pgmap v10246: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:08 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:09.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:09.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:09.682230+0000) 2023-08-28T02:35:10.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3001191768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:10.238 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3001191768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:10.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3001191768' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:10.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:10.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:10.682411+0000) 2023-08-28T02:35:11.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[111335]: pgmap v10247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:11.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[118902]: pgmap v10247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:11.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:11.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:10 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:10 smithi136 ceph-mon[113531]: pgmap v10247: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:11.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:35:11.427 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.428 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:10 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:11.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:11.682574+0000) 2023-08-28T02:35:11.982 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2459985167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:11.982 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.983 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:11.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:11.598Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:11.983 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:11.599Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2459985167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:12.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:11 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:12.408 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2459985167' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:12.409 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:12.409 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:11 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:12.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:12.681+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:12.682766+0000) 2023-08-28T02:35:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:13.026Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:13.238 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:13.027Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:12 smithi093 ceph-mon[111335]: pgmap v10248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:13.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:12 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:13.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:12 smithi093 ceph-mon[118902]: pgmap v10248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:13.240 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:12 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:12 smithi136 ceph-mon[113531]: pgmap v10248: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:12 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:13.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:13.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:13.682947+0000) 2023-08-28T02:35:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:14.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:14.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:14.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:13 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:35:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:35:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:13 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:35:14.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:14.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:14.683154+0000) 2023-08-28T02:35:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:14 smithi093 ceph-mon[111335]: pgmap v10249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:15.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4165887562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:14 smithi093 ceph-mon[118902]: pgmap v10249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:15.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4165887562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:14 smithi136 ceph-mon[113531]: pgmap v10249: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4165887562' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:35:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:35:15] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:35:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:15.682+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:15.683418+0000) 2023-08-28T02:35:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:16.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:16.683633+0000) 2023-08-28T02:35:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:16 smithi136 ceph-mon[113531]: pgmap v10250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/987747259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:16 smithi093 ceph-mon[111335]: pgmap v10250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/987747259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:16 smithi093 ceph-mon[118902]: pgmap v10250: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/987747259' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:17.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:17.683885+0000) 2023-08-28T02:35:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:18.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:18.684177+0000) 2023-08-28T02:35:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:18 smithi136 ceph-mon[113531]: pgmap v10251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3204595410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:18 smithi093 ceph-mon[111335]: pgmap v10251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3204595410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:18 smithi093 ceph-mon[118902]: pgmap v10251: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3204595410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:19.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:19.683+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:19.684397+0000) 2023-08-28T02:35:20.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:20.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:20.684629+0000) 2023-08-28T02:35:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:20 smithi136 ceph-mon[113531]: pgmap v10252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:20 smithi093 ceph-mon[111335]: pgmap v10252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:20 smithi093 ceph-mon[118902]: pgmap v10252: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:21.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:21.684823+0000) 2023-08-28T02:35:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:21.600Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:21.601Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1448683092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1448683092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1448683092' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:22.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:22.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:22.685021+0000) 2023-08-28T02:35:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:23 smithi136 ceph-mon[113531]: pgmap v10253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:23 smithi093 ceph-mon[111335]: pgmap v10253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:23 smithi093 ceph-mon[118902]: pgmap v10253: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:23.028Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:23.029Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:23.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:23.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:23.685229+0000) 2023-08-28T02:35:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4251394707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4251394707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:24.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4251394707' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:24.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:24.685428+0000) 2023-08-28T02:35:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:25 smithi093 ceph-mon[111335]: pgmap v10254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:25.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:25 smithi093 ceph-mon[118902]: pgmap v10254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:25 smithi136 ceph-mon[113531]: pgmap v10254: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:35:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:35:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:35:25.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:25.684+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:25.685715+0000) 2023-08-28T02:35:26.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2940187448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:26.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2940187448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2940187448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:26.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:26.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:26.685998+0000) 2023-08-28T02:35:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:27 smithi136 ceph-mon[113531]: pgmap v10255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:27.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:27 smithi093 ceph-mon[111335]: pgmap v10255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:27.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:27 smithi093 ceph-mon[118902]: pgmap v10255: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:27.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:27.685+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:27.686280+0000) 2023-08-28T02:35:28.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:28.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:28.686541+0000) 2023-08-28T02:35:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:29 smithi136 ceph-mon[113531]: pgmap v10256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2200644937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:29 smithi093 ceph-mon[111335]: pgmap v10256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2200644937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:29.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:29 smithi093 ceph-mon[118902]: pgmap v10256: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2200644937' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:29.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:29.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:29.686776+0000) 2023-08-28T02:35:30.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:30.686+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:30.687094+0000) 2023-08-28T02:35:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:31 smithi136 ceph-mon[113531]: pgmap v10257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/944911697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:31 smithi093 ceph-mon[111335]: pgmap v10257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/944911697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:31 smithi093 ceph-mon[118902]: pgmap v10257: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/944911697' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:31.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:31.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:31.687359+0000) 2023-08-28T02:35:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:31.600Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:31.601Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:32.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:32.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:32.687608+0000) 2023-08-28T02:35:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:33 smithi136 ceph-mon[113531]: pgmap v10258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:33 smithi093 ceph-mon[111335]: pgmap v10258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:33 smithi093 ceph-mon[118902]: pgmap v10258: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:33.028Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:33.029Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:33.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:33.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:33.687878+0000) 2023-08-28T02:35:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4133501533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4133501533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4133501533' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:34.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:34.687+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:34.688073+0000) 2023-08-28T02:35:35.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:35 smithi093 ceph-mon[111335]: pgmap v10259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:35 smithi093 ceph-mon[118902]: pgmap v10259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:35 smithi136 ceph-mon[113531]: pgmap v10259: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:35:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:35:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:35:35.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:35.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:35.688328+0000) 2023-08-28T02:35:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2384846126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2384846126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2384846126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:36.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:36.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:36.688580+0000) 2023-08-28T02:35:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:37 smithi136 ceph-mon[113531]: pgmap v10260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:37 smithi093 ceph-mon[111335]: pgmap v10260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:37.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:37 smithi093 ceph-mon[118902]: pgmap v10260: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:37.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:37.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:37.688818+0000) 2023-08-28T02:35:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3561271950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3561271950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:38.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3561271950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:38.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:38.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:38.689079+0000) 2023-08-28T02:35:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:39 smithi136 ceph-mon[113531]: pgmap v10261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:39.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:39 smithi093 ceph-mon[111335]: pgmap v10261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:39 smithi093 ceph-mon[118902]: pgmap v10261: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:39.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:39.689274+0000) 2023-08-28T02:35:41.043 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:40.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:40.689557+0000) 2023-08-28T02:35:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:41 smithi136 ceph-mon[113531]: pgmap v10262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3050187287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:41 smithi093 ceph-mon[111335]: pgmap v10262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3050187287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:41 smithi093 ceph-mon[118902]: pgmap v10262: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3050187287' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:41.601Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:41.602Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:41.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:41.689802+0000) 2023-08-28T02:35:43.050 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:42.688+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:42.690005+0000) 2023-08-28T02:35:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:43 smithi136 ceph-mon[113531]: pgmap v10263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/548436975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:43 smithi093 ceph-mon[118902]: pgmap v10263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/548436975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:43 smithi093 ceph-mon[111335]: pgmap v10263: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:43.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/548436975' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:43.029Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:43.031Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:43.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:43.690234+0000) 2023-08-28T02:35:45.053 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:44.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:44.690431+0000) 2023-08-28T02:35:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:45 smithi093 ceph-mon[111335]: pgmap v10264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3731519998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:45 smithi093 ceph-mon[118902]: pgmap v10264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3731519998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:45 smithi136 ceph-mon[113531]: pgmap v10264: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3731519998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:35:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:35:45] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:35:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:45.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:45.690703+0000) 2023-08-28T02:35:47.055 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:46.689+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:46.690939+0000) 2023-08-28T02:35:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:47 smithi136 ceph-mon[113531]: pgmap v10265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:47 smithi093 ceph-mon[111335]: pgmap v10265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:47 smithi093 ceph-mon[118902]: pgmap v10265: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:48.060 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:47.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:47.691191+0000) 2023-08-28T02:35:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/423657205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:48.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/423657205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:48.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/423657205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:49.063 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:48.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:48.691480+0000) 2023-08-28T02:35:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:49 smithi136 ceph-mon[113531]: pgmap v10266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:49 smithi093 ceph-mon[111335]: pgmap v10266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:49 smithi093 ceph-mon[118902]: pgmap v10266: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:50.069 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:49.690+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:49.691715+0000) 2023-08-28T02:35:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/660959789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/660959789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/660959789' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:51.070 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:50.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:50.691963+0000) 2023-08-28T02:35:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:51 smithi136 ceph-mon[113531]: pgmap v10267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:51 smithi093 ceph-mon[111335]: pgmap v10267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:51.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:51 smithi093 ceph-mon[118902]: pgmap v10267: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:51.602Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:51.603Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:51.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:51.692200+0000) 2023-08-28T02:35:53.072 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:52.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:52.692428+0000) 2023-08-28T02:35:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:53 smithi136 ceph-mon[113531]: pgmap v10268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/568375098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:53 smithi093 ceph-mon[111335]: pgmap v10268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/568375098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:53 smithi093 ceph-mon[118902]: pgmap v10268: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/568375098' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:53.029Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:35:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:35:53.030Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:35:54.080 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:53.691+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:53.692644+0000) 2023-08-28T02:35:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:35:55.081 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:54.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:54.692848+0000) 2023-08-28T02:35:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:55 smithi093 ceph-mon[111335]: pgmap v10269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3803584910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:55.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:55 smithi093 ceph-mon[118902]: pgmap v10269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:55.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3803584910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:55 smithi136 ceph-mon[113531]: pgmap v10269: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3803584910' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:35:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:35:55] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:35:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:55.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:55.693111+0000) 2023-08-28T02:35:57.087 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:56.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:56.693363+0000) 2023-08-28T02:35:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:57 smithi136 ceph-mon[113531]: pgmap v10270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3203089644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:57 smithi093 ceph-mon[111335]: pgmap v10270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:57.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3203089644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:57 smithi093 ceph-mon[118902]: pgmap v10270: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3203089644' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:35:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:57.692+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:57.693596+0000) 2023-08-28T02:35:59.089 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:58.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:58.693875+0000) 2023-08-28T02:35:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:35:59 smithi136 ceph-mon[113531]: pgmap v10271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:35:59 smithi093 ceph-mon[111335]: pgmap v10271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:35:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:35:59 smithi093 ceph-mon[118902]: pgmap v10271: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:00.096 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:35:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:35:59.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:35:59.694122+0000) 2023-08-28T02:36:00.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4284263365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:00.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4284263365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:00.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4284263365' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:01.096 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:00.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:00.694394+0000) 2023-08-28T02:36:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:01 smithi136 ceph-mon[113531]: pgmap v10272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:01 smithi093 ceph-mon[111335]: pgmap v10272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:01 smithi093 ceph-mon[118902]: pgmap v10272: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:01.603Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:01.604Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:02.103 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:01.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:01.694684+0000) 2023-08-28T02:36:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2068696645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2068696645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:02.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2068696645' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:03.102 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:02.693+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:02.694928+0000) 2023-08-28T02:36:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:03 smithi136 ceph-mon[113531]: pgmap v10273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:03 smithi093 ceph-mon[111335]: pgmap v10273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:03 smithi093 ceph-mon[118902]: pgmap v10273: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:03.030Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:03.031Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:03.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:03.695178+0000) 2023-08-28T02:36:05.105 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:04.694+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:04.695485+0000) 2023-08-28T02:36:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:05 smithi093 ceph-mon[118902]: pgmap v10274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:05.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/61014364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:05 smithi093 ceph-mon[111335]: pgmap v10274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:05.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/61014364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:05 smithi136 ceph-mon[113531]: pgmap v10274: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/61014364' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:36:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:36:05] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:36:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:05.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:05.695704+0000) 2023-08-28T02:36:07.108 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:06.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:06.695933+0000) 2023-08-28T02:36:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:07 smithi136 ceph-mon[113531]: pgmap v10275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1630231220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:07 smithi093 ceph-mon[111335]: pgmap v10275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1630231220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:07 smithi093 ceph-mon[118902]: pgmap v10275: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1630231220' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:07.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:07.696167+0000) 2023-08-28T02:36:09.110 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:08.695+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:08.696449+0000) 2023-08-28T02:36:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:09 smithi136 ceph-mon[113531]: pgmap v10276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4224039380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:09 smithi093 ceph-mon[111335]: pgmap v10276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4224039380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:09 smithi093 ceph-mon[118902]: pgmap v10276: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4224039380' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:09.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:09.696733+0000) 2023-08-28T02:36:11.113 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:10.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:10.696980+0000) 2023-08-28T02:36:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:11 smithi136 ceph-mon[113531]: pgmap v10277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:11 smithi093 ceph-mon[111335]: pgmap v10277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:11.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:11 smithi093 ceph-mon[118902]: pgmap v10277: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:11.603Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:11.604Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:12.118 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:11.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:11.697266+0000) 2023-08-28T02:36:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2279371557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2279371557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2279371557' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:13.122 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:12.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:12.697525+0000) 2023-08-28T02:36:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:13 smithi136 ceph-mon[113531]: pgmap v10278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:13 smithi093 ceph-mon[111335]: pgmap v10278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:13 smithi093 ceph-mon[118902]: pgmap v10278: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:13.031Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:13.032Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:14.075 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:13.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:13.697736+0000) 2023-08-28T02:36:14.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:14 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:14.431 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2604994016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:14 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2604994016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:14.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:14 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:36:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2604994016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:15.130 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:14.696+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:14.697868+0000) 2023-08-28T02:36:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:15 smithi093 ceph-mon[111335]: pgmap v10279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:15 smithi093 ceph-mon[118902]: pgmap v10279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:15.437 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:15 smithi136 ceph-mon[113531]: pgmap v10279: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:36:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:36:15] "GET /metrics HTTP/1.1" 200 239942 "" "Prometheus/2.43.0" 2023-08-28T02:36:15.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:15.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:15.698065+0000) 2023-08-28T02:36:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:16 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1010509831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1010509831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:16.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1010509831' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:17.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:16.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:16.698226+0000) 2023-08-28T02:36:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:17 smithi136 ceph-mon[113531]: pgmap v10280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:36:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:36:17.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[111335]: pgmap v10280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:36:17.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:36:17.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[118902]: pgmap v10280: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:36:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:36:17.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:36:18.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:17.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:17.698421+0000) 2023-08-28T02:36:19.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:18.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:18.698646+0000) 2023-08-28T02:36:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:19 smithi136 ceph-mon[113531]: pgmap v10281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:19 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/709227065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:19 smithi093 ceph-mon[111335]: pgmap v10281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:19.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:19 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/709227065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:19 smithi093 ceph-mon[118902]: pgmap v10281: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:19.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:19 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/709227065' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:19.697+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:19.698878+0000) 2023-08-28T02:36:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:20.698+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:20.699165+0000) 2023-08-28T02:36:21.604 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:21 smithi093 ceph-mon[111335]: pgmap v10282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:21.604 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4211518123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:21.604 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:21 smithi093 ceph-mon[118902]: pgmap v10282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:21.604 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4211518123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:21 smithi136 ceph-mon[113531]: pgmap v10282: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4211518123' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:21.604Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:21.605Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:21.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:21.699415+0000) 2023-08-28T02:36:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:22.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:22.699652+0000) 2023-08-28T02:36:23.289 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:23.032Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:23.289 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:23.033Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:23 smithi136 ceph-mon[113531]: pgmap v10283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/443428004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:23 smithi093 ceph-mon[118902]: pgmap v10283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/443428004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:23 smithi093 ceph-mon[111335]: pgmap v10283: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/443428004' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:23.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:23.699922+0000) 2023-08-28T02:36:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:24.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:24.700153+0000) 2023-08-28T02:36:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:25 smithi136 ceph-mon[113531]: pgmap v10284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:25 smithi093 ceph-mon[111335]: pgmap v10284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:25 smithi093 ceph-mon[118902]: pgmap v10284: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:36:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:36:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:36:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:25.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:25.700350+0000) 2023-08-28T02:36:26.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:26 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3263795308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:26.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:26 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3263795308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:26.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:26 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3263795308' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:26.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:26.700590+0000) 2023-08-28T02:36:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:27 smithi136 ceph-mon[113531]: pgmap v10285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:27 smithi093 ceph-mon[111335]: pgmap v10285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:27 smithi093 ceph-mon[118902]: pgmap v10285: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:27.699+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:27.700803+0000) 2023-08-28T02:36:28.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3730714891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3730714891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3730714891' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:28.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:28.701056+0000) 2023-08-28T02:36:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:29 smithi136 ceph-mon[113531]: pgmap v10286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:29 smithi093 ceph-mon[118902]: pgmap v10286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:29.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:29 smithi093 ceph-mon[111335]: pgmap v10286: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:29.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:29.701281+0000) 2023-08-28T02:36:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:30.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:30.701530+0000) 2023-08-28T02:36:31.604 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:31 smithi093 ceph-mon[111335]: pgmap v10287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:31.604 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1069466877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:31.604 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:31 smithi093 ceph-mon[118902]: pgmap v10287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:31.604 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1069466877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:31 smithi136 ceph-mon[113531]: pgmap v10287: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1069466877' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:31.605Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:32.038 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:31.606Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:31.700+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:31.701820+0000) 2023-08-28T02:36:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:32.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:32.702061+0000) 2023-08-28T02:36:33.314 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:33.032Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:33.314 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:33.033Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:33 smithi136 ceph-mon[113531]: pgmap v10288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:33 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3669121893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:33 smithi093 ceph-mon[118902]: pgmap v10288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:33 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3669121893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:33 smithi093 ceph-mon[111335]: pgmap v10288: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:33.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:33 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3669121893' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:33.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:33.702270+0000) 2023-08-28T02:36:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:34.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:34.702505+0000) 2023-08-28T02:36:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:35 smithi136 ceph-mon[113531]: pgmap v10289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:35 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/649211762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:35 smithi093 ceph-mon[111335]: pgmap v10289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:35 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/649211762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:35.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:35 smithi093 ceph-mon[118902]: pgmap v10289: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:35 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/649211762' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:35.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:36:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:36:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:36:36.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:35.701+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:35.702768+0000) 2023-08-28T02:36:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:36.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:36.703019+0000) 2023-08-28T02:36:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:37 smithi136 ceph-mon[113531]: pgmap v10290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:37 smithi093 ceph-mon[111335]: pgmap v10290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:37 smithi093 ceph-mon[118902]: pgmap v10290: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:37.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:37.703280+0000) 2023-08-28T02:36:38.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3995278068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:38.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3995278068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:38.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3995278068' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:38.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:38.703541+0000) 2023-08-28T02:36:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:39 smithi136 ceph-mon[113531]: pgmap v10291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:39 smithi093 ceph-mon[111335]: pgmap v10291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:39 smithi093 ceph-mon[118902]: pgmap v10291: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:39.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:39.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:39.703771+0000) 2023-08-28T02:36:40.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3806651108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:40.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3806651108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:40.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3806651108' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:40.702+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:40.703990+0000) 2023-08-28T02:36:41.605 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:41 smithi093 ceph-mon[111335]: pgmap v10292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:41.605 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:41 smithi093 ceph-mon[118902]: pgmap v10292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:41 smithi136 ceph-mon[113531]: pgmap v10292: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:41.605Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:41.606Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:41.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:41.704239+0000) 2023-08-28T02:36:42.977 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:42.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:42.704479+0000) 2023-08-28T02:36:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:42 smithi093 ceph-mon[111335]: pgmap v10293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:43.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1458531210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:42 smithi093 ceph-mon[118902]: pgmap v10293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:43.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1458531210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:43.033Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:43.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:43.034Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:42 smithi136 ceph-mon[113531]: pgmap v10293: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1458531210' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:43.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:43.704680+0000) 2023-08-28T02:36:44.980 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:44.703+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:44.704938+0000) 2023-08-28T02:36:45.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:44 smithi093 ceph-mon[111335]: pgmap v10294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:45.239 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/972959857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:44 smithi093 ceph-mon[118902]: pgmap v10294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:45.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/972959857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:44 smithi136 ceph-mon[113531]: pgmap v10294: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/972959857' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:36:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:36:45] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:36:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:45.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:45.705196+0000) 2023-08-28T02:36:46.985 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:46.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:46.705359+0000) 2023-08-28T02:36:47.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:46 smithi093 ceph-mon[111335]: pgmap v10295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:47.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:46 smithi093 ceph-mon[118902]: pgmap v10295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:46 smithi136 ceph-mon[113531]: pgmap v10295: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:47.988 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:47.704+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:47.705578+0000) 2023-08-28T02:36:48.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2451123424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:48.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2451123424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:48.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2451123424' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:48.991 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:48.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:48.705861+0000) 2023-08-28T02:36:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:48 smithi136 ceph-mon[113531]: pgmap v10296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:48 smithi093 ceph-mon[111335]: pgmap v10296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:49.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:48 smithi093 ceph-mon[118902]: pgmap v10296: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:49.996 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:49.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:49.706087+0000) 2023-08-28T02:36:50.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2160136873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2160136873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:50.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2160136873' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:50.996 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:50.705+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:50.706364+0000) 2023-08-28T02:36:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:50 smithi136 ceph-mon[113531]: pgmap v10297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:50 smithi093 ceph-mon[111335]: pgmap v10297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:50 smithi093 ceph-mon[118902]: pgmap v10297: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:51.606Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:51.607Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:52.005 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:51.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:51.706613+0000) 2023-08-28T02:36:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2626865422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2626865422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2626865422' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:53.009 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:52.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:52.706847+0000) 2023-08-28T02:36:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:53 smithi136 ceph-mon[113531]: pgmap v10298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:53 smithi093 ceph-mon[111335]: pgmap v10298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:53 smithi093 ceph-mon[118902]: pgmap v10298: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:53.034Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:36:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:36:53.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:36:54.013 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:53.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:53.707127+0000) 2023-08-28T02:36:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:36:55.015 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:54.706+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:54.707394+0000) 2023-08-28T02:36:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:55 smithi093 ceph-mon[111335]: pgmap v10299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/163782639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:55 smithi093 ceph-mon[118902]: pgmap v10299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:55.359 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/163782639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:55 smithi136 ceph-mon[113531]: pgmap v10299: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/163782639' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:36:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:36:55] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:36:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:55.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:55.707648+0000) 2023-08-28T02:36:57.023 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:56.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:56.707847+0000) 2023-08-28T02:36:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:57 smithi136 ceph-mon[113531]: pgmap v10300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/254908344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:57 smithi093 ceph-mon[111335]: pgmap v10300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/254908344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:57 smithi093 ceph-mon[118902]: pgmap v10300: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/254908344' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:57.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:57.708107+0000) 2023-08-28T02:36:59.026 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:58.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:58.708363+0000) 2023-08-28T02:36:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:59 smithi136 ceph-mon[113531]: pgmap v10301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:36:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3934247499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:59 smithi093 ceph-mon[111335]: pgmap v10301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:36:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3934247499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:36:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:59 smithi093 ceph-mon[118902]: pgmap v10301: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:36:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:36:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3934247499' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:36:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:36:59.707+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:36:59.708576+0000) 2023-08-28T02:37:01.029 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:00.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:00.708777+0000) 2023-08-28T02:37:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:01 smithi136 ceph-mon[113531]: pgmap v10302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:01.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:01 smithi093 ceph-mon[111335]: pgmap v10302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:01.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:01 smithi093 ceph-mon[118902]: pgmap v10302: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:01.607Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:01.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:01.607Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:02.033 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:01.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:01.709109+0000) 2023-08-28T02:37:02.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:02 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3465570984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:02.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:02 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3465570984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:02.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:02 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3465570984' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:03.035 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:02.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:02.709315+0000) 2023-08-28T02:37:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:03 smithi136 ceph-mon[113531]: pgmap v10303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:03 smithi093 ceph-mon[111335]: pgmap v10303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:03 smithi093 ceph-mon[118902]: pgmap v10303: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:03.034Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:03.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:04.040 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:03.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:03.709543+0000) 2023-08-28T02:37:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2274235838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2274235838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2274235838' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:05.040 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:04.708+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:04.709727+0000) 2023-08-28T02:37:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:05 smithi093 ceph-mon[111335]: pgmap v10304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:05 smithi093 ceph-mon[118902]: pgmap v10304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:05 smithi136 ceph-mon[113531]: pgmap v10304: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:37:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:37:05] "GET /metrics HTTP/1.1" 200 239943 "" "Prometheus/2.43.0" 2023-08-28T02:37:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:05.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:05.709972+0000) 2023-08-28T02:37:07.043 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:06.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:06.710246+0000) 2023-08-28T02:37:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:07 smithi136 ceph-mon[113531]: pgmap v10305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/629353713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:07 smithi093 ceph-mon[111335]: pgmap v10305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/629353713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:07 smithi093 ceph-mon[118902]: pgmap v10305: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:07.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/629353713' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:07.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:07.710483+0000) 2023-08-28T02:37:09.046 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:08.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:08.710736+0000) 2023-08-28T02:37:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:09 smithi136 ceph-mon[113531]: pgmap v10306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:09 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2366413535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:09 smithi093 ceph-mon[118902]: pgmap v10306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:09 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2366413535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:09 smithi093 ceph-mon[111335]: pgmap v10306: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:09 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2366413535' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:09.709+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:09.710966+0000) 2023-08-28T02:37:11.050 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:10.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:10.711229+0000) 2023-08-28T02:37:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:11 smithi136 ceph-mon[113531]: pgmap v10307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1775674314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:11 smithi093 ceph-mon[118902]: pgmap v10307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1775674314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:11 smithi093 ceph-mon[111335]: pgmap v10307: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1775674314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:11.608Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:11.609Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:11.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:11.711480+0000) 2023-08-28T02:37:13.051 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:12.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:12.711689+0000) 2023-08-28T02:37:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:13 smithi136 ceph-mon[113531]: pgmap v10308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:13 smithi093 ceph-mon[111335]: pgmap v10308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:13 smithi093 ceph-mon[118902]: pgmap v10308: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:13.035Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:13.035Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:14.058 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:13.710+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:13.711951+0000) 2023-08-28T02:37:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1276625318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1276625318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1276625318' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:15.060 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:14.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:14.712243+0000) 2023-08-28T02:37:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:15 smithi093 ceph-mon[111335]: pgmap v10309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:15 smithi093 ceph-mon[118902]: pgmap v10309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:15 smithi136 ceph-mon[113531]: pgmap v10309: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:37:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:37:15] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:37:16.066 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:15.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:15.712507+0000) 2023-08-28T02:37:16.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2115117334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2115117334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:16.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2115117334' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:17.026 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:16.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:16.712679+0000) 2023-08-28T02:37:17.390 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:17 smithi136 ceph-mon[113531]: pgmap v10310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:17.390 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:17 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:37:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:17 smithi093 ceph-mon[118902]: pgmap v10310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:17 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:37:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:17 smithi093 ceph-mon[111335]: pgmap v10310: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:17.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:17 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:37:18.073 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:17.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:17.712830+0000) 2023-08-28T02:37:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4143483265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:18.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4143483265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4143483265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:18.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:18.711+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:18.712997+0000) 2023-08-28T02:37:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[111335]: pgmap v10311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[118902]: pgmap v10311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:19 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:19 smithi136 ceph-mon[113531]: pgmap v10311: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:19.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:19 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:20.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:19.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:19.713213+0000) 2023-08-28T02:37:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:37:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:37:20.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:37:21.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:20.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:20.713468+0000) 2023-08-28T02:37:21.608 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:21 smithi093 ceph-mon[111335]: pgmap v10312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:21.609 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:21 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1376285678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:21.609 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:21 smithi093 ceph-mon[118902]: pgmap v10312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:21.609 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:21 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1376285678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:21 smithi136 ceph-mon[113531]: pgmap v10312: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:21.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:21 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1376285678' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:21.608Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:21.610Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:22.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:21.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:21.713718+0000) 2023-08-28T02:37:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:22.712+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:22.713966+0000) 2023-08-28T02:37:23.295 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:23.035Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:23.295 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:23.036Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:23 smithi136 ceph-mon[113531]: pgmap v10313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/902280522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:23 smithi093 ceph-mon[118902]: pgmap v10313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:23.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/902280522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:23 smithi093 ceph-mon[111335]: pgmap v10313: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:23.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/902280522' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:23.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:23.714173+0000) 2023-08-28T02:37:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:24.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:24.714431+0000) 2023-08-28T02:37:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:25 smithi136 ceph-mon[113531]: pgmap v10314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/464773476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:25 smithi093 ceph-mon[111335]: pgmap v10314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/464773476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:25 smithi093 ceph-mon[118902]: pgmap v10314: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/464773476' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:37:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:37:25] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:37:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:25.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:25.714640+0000) 2023-08-28T02:37:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:26.713+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:26.714900+0000) 2023-08-28T02:37:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:27 smithi136 ceph-mon[113531]: pgmap v10315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:27.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:27 smithi093 ceph-mon[118902]: pgmap v10315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:27 smithi093 ceph-mon[111335]: pgmap v10315: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:27.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:27.715153+0000) 2023-08-28T02:37:28.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:28 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3074540689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:28.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:28 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3074540689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:28.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:28 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3074540689' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:28.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:28.715396+0000) 2023-08-28T02:37:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:29 smithi136 ceph-mon[113531]: pgmap v10316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:29 smithi093 ceph-mon[111335]: pgmap v10316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:29 smithi093 ceph-mon[118902]: pgmap v10316: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:29.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:29.715638+0000) 2023-08-28T02:37:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3078279261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3078279261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:30.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3078279261' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:30.714+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:30.715911+0000) 2023-08-28T02:37:31.609 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:31 smithi093 ceph-mon[111335]: pgmap v10317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:31.609 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:31 smithi093 ceph-mon[118902]: pgmap v10317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:31 smithi136 ceph-mon[113531]: pgmap v10317: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:31.610Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:31.611Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:31.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:31.716160+0000) 2023-08-28T02:37:32.987 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:32.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:32.716378+0000) 2023-08-28T02:37:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:32 smithi093 ceph-mon[111335]: pgmap v10318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:33.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2612501834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:32 smithi093 ceph-mon[118902]: pgmap v10318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:33.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2612501834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:33.035Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:33.239 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:33.036Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:32 smithi136 ceph-mon[113531]: pgmap v10318: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2612501834' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:33.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:33.716673+0000) 2023-08-28T02:37:34.990 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:34.715+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:34.716946+0000) 2023-08-28T02:37:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:34 smithi093 ceph-mon[118902]: pgmap v10319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:35.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/267630326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:34 smithi093 ceph-mon[111335]: pgmap v10319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:35.359 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/267630326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:34 smithi136 ceph-mon[113531]: pgmap v10319: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/267630326' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:37:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:37:35] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:37:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:35.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:35.717195+0000) 2023-08-28T02:37:36.994 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:36.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:36.717450+0000) 2023-08-28T02:37:37.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:36 smithi136 ceph-mon[113531]: pgmap v10320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:36 smithi093 ceph-mon[111335]: pgmap v10320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:37.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:36 smithi093 ceph-mon[118902]: pgmap v10320: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:38.004 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:37.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:37.717685+0000) 2023-08-28T02:37:38.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:38 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2951571369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:38.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:38 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2951571369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:38.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:38 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2951571369' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:39.005 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:38.716+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:38.717908+0000) 2023-08-28T02:37:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:39 smithi136 ceph-mon[113531]: pgmap v10321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:39.439 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:39 smithi093 ceph-mon[111335]: pgmap v10321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:39 smithi093 ceph-mon[118902]: pgmap v10321: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:39.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:40.009 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:39.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:39.718107+0000) 2023-08-28T02:37:40.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:40 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3348047193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:40.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:40 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3348047193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:40.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:40 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3348047193' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:41.011 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:40.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:40.718355+0000) 2023-08-28T02:37:41.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:41 smithi136 ceph-mon[113531]: pgmap v10322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:41 smithi093 ceph-mon[111335]: pgmap v10322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:41.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:41 smithi093 ceph-mon[118902]: pgmap v10322: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:41.611Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:41.612Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:42.015 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:41.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:41.718611+0000) 2023-08-28T02:37:42.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3717903313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:42.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3717903313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:42.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3717903313' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:43.017 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:42.717+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:42.718872+0000) 2023-08-28T02:37:43.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:43 smithi136 ceph-mon[113531]: pgmap v10323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:43 smithi093 ceph-mon[111335]: pgmap v10323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:43.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:43 smithi093 ceph-mon[118902]: pgmap v10323: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:43.036Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:43.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:43.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:43.719134+0000) 2023-08-28T02:37:45.019 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:44.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:44.719378+0000) 2023-08-28T02:37:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:45 smithi093 ceph-mon[111335]: pgmap v10324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:45.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:45 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3579952146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:45 smithi093 ceph-mon[118902]: pgmap v10324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:45.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:45 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3579952146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:45 smithi136 ceph-mon[113531]: pgmap v10324: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:45.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:45 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3579952146' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:37:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:37:45] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:37:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:45.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:45.719606+0000) 2023-08-28T02:37:47.023 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:46.718+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:46.719889+0000) 2023-08-28T02:37:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:47 smithi136 ceph-mon[113531]: pgmap v10325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:47.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:47 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4247853542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:47 smithi093 ceph-mon[111335]: pgmap v10325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:47 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4247853542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:47 smithi093 ceph-mon[118902]: pgmap v10325: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:47.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:47 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4247853542' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:47.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:47.720146+0000) 2023-08-28T02:37:49.028 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:48.719+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:48.720365+0000) 2023-08-28T02:37:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:49 smithi136 ceph-mon[113531]: pgmap v10326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:49.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4076997370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:49 smithi093 ceph-mon[111335]: pgmap v10326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4076997370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:49 smithi093 ceph-mon[118902]: pgmap v10326: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4076997370' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:49.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:49.720649+0000) 2023-08-28T02:37:51.029 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:50.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:50.720930+0000) 2023-08-28T02:37:51.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:51 smithi136 ceph-mon[113531]: pgmap v10327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:51 smithi093 ceph-mon[111335]: pgmap v10327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:51.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:51 smithi093 ceph-mon[118902]: pgmap v10327: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:51.611Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:51.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:51.612Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:52.032 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:51.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:51.721177+0000) 2023-08-28T02:37:52.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:52 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2305640431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:52.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:52 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2305640431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:52.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:52 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2305640431' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:53.034 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:52.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:52.721423+0000) 2023-08-28T02:37:53.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:53 smithi136 ceph-mon[113531]: pgmap v10328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:53 smithi093 ceph-mon[111335]: pgmap v10328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:53.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:53 smithi093 ceph-mon[118902]: pgmap v10328: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:53.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:37:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:37:53.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:37:54.036 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:53.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:53.721647+0000) 2023-08-28T02:37:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:54 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/525599131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:54.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:54 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/525599131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:54.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:54 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/525599131' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:37:55.042 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:54.720+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:54.721870+0000) 2023-08-28T02:37:55.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:55 smithi093 ceph-mon[111335]: pgmap v10329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:55.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:55 smithi093 ceph-mon[118902]: pgmap v10329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:55.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:55 smithi136 ceph-mon[113531]: pgmap v10329: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:55.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:37:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:37:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:37:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:55.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:55.722166+0000) 2023-08-28T02:37:57.047 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:56.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:56.722441+0000) 2023-08-28T02:37:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:57 smithi136 ceph-mon[113531]: pgmap v10330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:57.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:57 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2750112372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:57 smithi093 ceph-mon[111335]: pgmap v10330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:57 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2750112372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:57 smithi093 ceph-mon[118902]: pgmap v10330: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:57.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:57 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2750112372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:57.721+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:57.722727+0000) 2023-08-28T02:37:59.051 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:58.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:58.723008+0000) 2023-08-28T02:37:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:59 smithi136 ceph-mon[113531]: pgmap v10331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:59.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:37:59 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1763976655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:59 smithi093 ceph-mon[111335]: pgmap v10331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:59.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:37:59 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1763976655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:37:59.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:59 smithi093 ceph-mon[118902]: pgmap v10331: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:37:59.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:37:59 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1763976655' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:37:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:37:59.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:37:59.723287+0000) 2023-08-28T02:38:01.055 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:00.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:00.723541+0000) 2023-08-28T02:38:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:01 smithi136 ceph-mon[113531]: pgmap v10332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:01.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4011308016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:01 smithi093 ceph-mon[111335]: pgmap v10332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:01.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4011308016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:01 smithi093 ceph-mon[118902]: pgmap v10332: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:01.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4011308016' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:01.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:01.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:01.722+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:01.723801+0000) 2023-08-28T02:38:03.061 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:02.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:02.724097+0000) 2023-08-28T02:38:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:03 smithi136 ceph-mon[113531]: pgmap v10333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:03 smithi093 ceph-mon[111335]: pgmap v10333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:03.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:03 smithi093 ceph-mon[118902]: pgmap v10333: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:03.488 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:03.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:03.037Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:04.067 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:03.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:03.724310+0000) 2023-08-28T02:38:04.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:04 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3536164576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:04.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:04 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3536164576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:04.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:04 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3536164576' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:05.068 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:04.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:04.724556+0000) 2023-08-28T02:38:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:05 smithi093 ceph-mon[111335]: pgmap v10334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:05 smithi093 ceph-mon[118902]: pgmap v10334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:05.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:05 smithi136 ceph-mon[113531]: pgmap v10334: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:38:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:38:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:38:06.075 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:05.723+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:05.724810+0000) 2023-08-28T02:38:06.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:06 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/103634818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:06.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:06 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/103634818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:06.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:06 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/103634818' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:07.078 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:06.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:06.725075+0000) 2023-08-28T02:38:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:07 smithi136 ceph-mon[113531]: pgmap v10335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:07 smithi093 ceph-mon[111335]: pgmap v10335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:07 smithi093 ceph-mon[118902]: pgmap v10335: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:08.082 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:07.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:07.725334+0000) 2023-08-28T02:38:08.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3528127209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:08.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3528127209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:08.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3528127209' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:09.084 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:08.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:08.725570+0000) 2023-08-28T02:38:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:09 smithi136 ceph-mon[113531]: pgmap v10336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:09 smithi093 ceph-mon[118902]: pgmap v10336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:09.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:09 smithi093 ceph-mon[111335]: pgmap v10336: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:09.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:09.724+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:09.725837+0000) 2023-08-28T02:38:11.087 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:10.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:10.726107+0000) 2023-08-28T02:38:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:11 smithi136 ceph-mon[113531]: pgmap v10337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:11 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2994288651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:11 smithi093 ceph-mon[118902]: pgmap v10337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:11 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2994288651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:11 smithi093 ceph-mon[111335]: pgmap v10337: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:11.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:11 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2994288651' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:11.612Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:11.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:11.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:11.726365+0000) 2023-08-28T02:38:13.089 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:12.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:12.726643+0000) 2023-08-28T02:38:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:13 smithi136 ceph-mon[113531]: pgmap v10338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/874850812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:13 smithi093 ceph-mon[111335]: pgmap v10338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/874850812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:13.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:13 smithi093 ceph-mon[118902]: pgmap v10338: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/874850812' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:13.037Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:13.038Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:13.725+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:13.726880+0000) 2023-08-28T02:38:15.092 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:14.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:14.727125+0000) 2023-08-28T02:38:15.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:15 smithi093 ceph-mon[118902]: pgmap v10339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:15.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:15 smithi093 ceph-mon[111335]: pgmap v10339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:15 smithi136 ceph-mon[113531]: pgmap v10339: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:38:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:38:15] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:38:16.097 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:15.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:15.727382+0000) 2023-08-28T02:38:16.440 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:16 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/908547888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:16.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:16 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/908547888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:16.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:16 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/908547888' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:17.098 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:16.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:16.727658+0000) 2023-08-28T02:38:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:17 smithi136 ceph-mon[113531]: pgmap v10340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:17 smithi093 ceph-mon[111335]: pgmap v10340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:17.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:17 smithi093 ceph-mon[118902]: pgmap v10340: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:18.103 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:17.726+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:17.727912+0000) 2023-08-28T02:38:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2573980410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:18.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2573980410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:18.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2573980410' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:19.104 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:18.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:18.728198+0000) 2023-08-28T02:38:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:19 smithi136 ceph-mon[113531]: pgmap v10341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:19 smithi093 ceph-mon[111335]: pgmap v10341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:19 smithi093 ceph-mon[118902]: pgmap v10341: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:20.017 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:19.727+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:19.728396+0000) 2023-08-28T02:38:20.377 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:20 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:20.377 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1442558213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:20 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1442558213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:20 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:38:20.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1442558213' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:21.109 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:20.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:20.728611+0000) 2023-08-28T02:38:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:21 smithi136 ceph-mon[113531]: pgmap v10342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:21 smithi093 ceph-mon[111335]: pgmap v10342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:21 smithi093 ceph-mon[118902]: pgmap v10342: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:21.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:21.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:21.728839+0000) 2023-08-28T02:38:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:21.613Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:21.613Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:38:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:38:22.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:38:22.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:38:22.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:22 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:38:23.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:22.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:22.729083+0000) 2023-08-28T02:38:23.297 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:23.038Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:23.298 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:23.039Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:23 smithi136 ceph-mon[113531]: pgmap v10343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:23.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:23 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1402393487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:23 smithi093 ceph-mon[111335]: pgmap v10343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:23.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:23 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1402393487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:23 smithi093 ceph-mon[118902]: pgmap v10343: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:23.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:23 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1402393487' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:24.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:23.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:23.729347+0000) 2023-08-28T02:38:24.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:24.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:24.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:25.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:24.728+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:24.729594+0000) 2023-08-28T02:38:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:25 smithi136 ceph-mon[113531]: pgmap v10344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3644240482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:25 smithi093 ceph-mon[118902]: pgmap v10344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:25.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3644240482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:25.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:25 smithi093 ceph-mon[111335]: pgmap v10344: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3644240482' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:25.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:38:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:38:25] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:38:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:25.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:25.729856+0000) 2023-08-28T02:38:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:26.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:26.730094+0000) 2023-08-28T02:38:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:27 smithi136 ceph-mon[113531]: pgmap v10345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2567682894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:27 smithi093 ceph-mon[111335]: pgmap v10345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2567682894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:27 smithi093 ceph-mon[118902]: pgmap v10345: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2567682894' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:27.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:27.730298+0000) 2023-08-28T02:38:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:28.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:28.730549+0000) 2023-08-28T02:38:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:29 smithi136 ceph-mon[113531]: pgmap v10346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:29 smithi093 ceph-mon[111335]: pgmap v10346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:29 smithi093 ceph-mon[118902]: pgmap v10346: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:29.729+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:29.730797+0000) 2023-08-28T02:38:30.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:30 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/522441859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:30.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:30 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/522441859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:30.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:30 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/522441859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:30.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:30.731074+0000) 2023-08-28T02:38:31.613 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:31 smithi093 ceph-mon[111335]: pgmap v10347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:31.613 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:31 smithi093 ceph-mon[118902]: pgmap v10347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:31 smithi136 ceph-mon[113531]: pgmap v10347: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:31.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:31.615Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:31.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:31.731353+0000) 2023-08-28T02:38:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1884729035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1884729035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:32.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1884729035' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:32.730+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:32.731568+0000) 2023-08-28T02:38:33.320 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:33.039Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:33.321 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:33.040Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:33 smithi136 ceph-mon[113531]: pgmap v10348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:33 smithi093 ceph-mon[111335]: pgmap v10348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:33 smithi093 ceph-mon[118902]: pgmap v10348: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:33.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:33.731844+0000) 2023-08-28T02:38:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1174443057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1174443057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:34.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1174443057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:34.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:34.732101+0000) 2023-08-28T02:38:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:35 smithi136 ceph-mon[113531]: pgmap v10349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:35 smithi093 ceph-mon[111335]: pgmap v10349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:38:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:38:35] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:38:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:35 smithi093 ceph-mon[118902]: pgmap v10349: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:35.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:35.732367+0000) 2023-08-28T02:38:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:36.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:36.732627+0000) 2023-08-28T02:38:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:37 smithi136 ceph-mon[113531]: pgmap v10350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:37 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2579735485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:37 smithi093 ceph-mon[111335]: pgmap v10350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:37 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2579735485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:37 smithi093 ceph-mon[118902]: pgmap v10350: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:37.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:37 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2579735485' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:37.731+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:37.732862+0000) 2023-08-28T02:38:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:38.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:38.733156+0000) 2023-08-28T02:38:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:39 smithi136 ceph-mon[113531]: pgmap v10351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1368266017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:39 smithi093 ceph-mon[111335]: pgmap v10351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1368266017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:39 smithi093 ceph-mon[118902]: pgmap v10351: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1368266017' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:39.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:39.733359+0000) 2023-08-28T02:38:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:40.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:40.733549+0000) 2023-08-28T02:38:41.613 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:41 smithi093 ceph-mon[111335]: pgmap v10352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:41.613 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:41 smithi093 ceph-mon[118902]: pgmap v10352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:41 smithi136 ceph-mon[113531]: pgmap v10352: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:41.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:41.615Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:41.732+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:41.733809+0000) 2023-08-28T02:38:42.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:42 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2279684726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:42.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:42 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2279684726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:42.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:42 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2279684726' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:43.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:42.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:42.734078+0000) 2023-08-28T02:38:43.342 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:43.040Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:43.342 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:43.041Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:43 smithi136 ceph-mon[113531]: pgmap v10353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:43 smithi093 ceph-mon[111335]: pgmap v10353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:43 smithi093 ceph-mon[118902]: pgmap v10353: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:43.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:43.734353+0000) 2023-08-28T02:38:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1008234902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1008234902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:44.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1008234902' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:45.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:44.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:44.734593+0000) 2023-08-28T02:38:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:45 smithi136 ceph-mon[113531]: pgmap v10354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:45 smithi093 ceph-mon[111335]: pgmap v10354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:45.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:38:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:38:45] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:38:45.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:45 smithi093 ceph-mon[118902]: pgmap v10354: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:45.733+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:45.734832+0000) 2023-08-28T02:38:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1811388088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1811388088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:46.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1811388088' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:46.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:46.735139+0000) 2023-08-28T02:38:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:47 smithi136 ceph-mon[113531]: pgmap v10355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:47 smithi093 ceph-mon[111335]: pgmap v10355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:47.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:47 smithi093 ceph-mon[118902]: pgmap v10355: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:47.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:47.735365+0000) 2023-08-28T02:38:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:48.734+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:48.735668+0000) 2023-08-28T02:38:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:49 smithi136 ceph-mon[113531]: pgmap v10356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:49 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1433682443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:49 smithi093 ceph-mon[111335]: pgmap v10356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:49 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1433682443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:49 smithi093 ceph-mon[118902]: pgmap v10356: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:49.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:49 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1433682443' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:49.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:49.735912+0000) 2023-08-28T02:38:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:50.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:50.736195+0000) 2023-08-28T02:38:51.614 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:51 smithi093 ceph-mon[111335]: pgmap v10357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:51.614 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2417703511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:51.614 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:51 smithi093 ceph-mon[118902]: pgmap v10357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:51.615 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2417703511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:51.615 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:51.614Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:51.615 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:51.615Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:51 smithi136 ceph-mon[113531]: pgmap v10357: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2417703511' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:51.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:51.736411+0000) 2023-08-28T02:38:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:52.735+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:52.736626+0000) 2023-08-28T02:38:53.366 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:53.041Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:53.366 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:38:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:38:53.042Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:38:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:53 smithi136 ceph-mon[113531]: pgmap v10358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3892637981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:53 smithi093 ceph-mon[111335]: pgmap v10358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3892637981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:53 smithi093 ceph-mon[118902]: pgmap v10358: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3892637981' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:53.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:53.736890+0000) 2023-08-28T02:38:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:54.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:38:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:54.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:54.737125+0000) 2023-08-28T02:38:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:55 smithi136 ceph-mon[113531]: pgmap v10359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:55 smithi093 ceph-mon[111335]: pgmap v10359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:55 smithi093 ceph-mon[118902]: pgmap v10359: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:38:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:38:55] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:38:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:55.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:55.737372+0000) 2023-08-28T02:38:56.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:56 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2862429085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:56.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:56 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2862429085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:56.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:56 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2862429085' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:56.736+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:56.737629+0000) 2023-08-28T02:38:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:57 smithi136 ceph-mon[113531]: pgmap v10360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:57 smithi093 ceph-mon[111335]: pgmap v10360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:57.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:57 smithi093 ceph-mon[118902]: pgmap v10360: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:57.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:57.737865+0000) 2023-08-28T02:38:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1444261445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1444261445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:58.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1444261445' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:38:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:58.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:58.738114+0000) 2023-08-28T02:38:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:38:59 smithi136 ceph-mon[113531]: pgmap v10361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:38:59 smithi093 ceph-mon[111335]: pgmap v10361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:38:59.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:38:59 smithi093 ceph-mon[118902]: pgmap v10361: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:38:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:38:59.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:38:59.738331+0000) 2023-08-28T02:39:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:00.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:00.738577+0000) 2023-08-28T02:39:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:01 smithi136 ceph-mon[113531]: pgmap v10362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:01 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4125733859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:01 smithi093 ceph-mon[111335]: pgmap v10362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:01.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:01 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4125733859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:01 smithi093 ceph-mon[118902]: pgmap v10362: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:01 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4125733859' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:01.615Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:01.740 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:01.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:01.737+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:01.738826+0000) 2023-08-28T02:39:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:02.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:02.739099+0000) 2023-08-28T02:39:03.395 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:03.041Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:03.395 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:03.042Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:03 smithi136 ceph-mon[113531]: pgmap v10363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2282389648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:03 smithi093 ceph-mon[111335]: pgmap v10363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2282389648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:03 smithi093 ceph-mon[118902]: pgmap v10363: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2282389648' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:03.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:03.739357+0000) 2023-08-28T02:39:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:04.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:04.739598+0000) 2023-08-28T02:39:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:05 smithi136 ceph-mon[113531]: pgmap v10364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1950837851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:39:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:39:05] "GET /metrics HTTP/1.1" 200 239945 "" "Prometheus/2.43.0" 2023-08-28T02:39:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:05 smithi093 ceph-mon[111335]: pgmap v10364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:05.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1950837851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:05 smithi093 ceph-mon[118902]: pgmap v10364: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1950837851' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:05.738+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:05.739871+0000) 2023-08-28T02:39:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:06.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:06.740126+0000) 2023-08-28T02:39:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:07 smithi136 ceph-mon[113531]: pgmap v10365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:07 smithi093 ceph-mon[111335]: pgmap v10365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:07 smithi093 ceph-mon[118902]: pgmap v10365: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:07.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:07.740348+0000) 2023-08-28T02:39:08.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:08 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2461188110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:08.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:08 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2461188110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:08.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:08 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2461188110' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:08.739+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:08.740624+0000) 2023-08-28T02:39:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:09 smithi136 ceph-mon[113531]: pgmap v10366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:09 smithi093 ceph-mon[111335]: pgmap v10366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:09.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:09 smithi093 ceph-mon[118902]: pgmap v10366: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:09.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:09.740885+0000) 2023-08-28T02:39:10.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2723675628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2723675628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:10.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2723675628' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:11.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:10.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:10.741151+0000) 2023-08-28T02:39:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:11 smithi136 ceph-mon[113531]: pgmap v10367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:11 smithi093 ceph-mon[111335]: pgmap v10367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:11.768 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:11 smithi093 ceph-mon[118902]: pgmap v10367: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:11.768 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:11.615Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:11.768 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:11.616Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:11.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:11.741318+0000) 2023-08-28T02:39:13.008 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:12.740+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:12.741513+0000) 2023-08-28T02:39:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:13 smithi136 ceph-mon[113531]: pgmap v10368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:13 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3017197997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:13 smithi093 ceph-mon[111335]: pgmap v10368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:13 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3017197997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:13 smithi093 ceph-mon[118902]: pgmap v10368: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:13 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3017197997' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:13.042Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:13.043Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:13.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:13.741777+0000) 2023-08-28T02:39:15.011 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:14.741+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:14.742023+0000) 2023-08-28T02:39:15.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:15 smithi093 ceph-mon[111335]: pgmap v10369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:15.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:15 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3040116805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:15.357 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:15 smithi093 ceph-mon[118902]: pgmap v10369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:15.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:15 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3040116805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:15 smithi136 ceph-mon[113531]: pgmap v10369: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:15.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:15 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3040116805' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:39:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:39:15] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:39:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:15.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:15.742255+0000) 2023-08-28T02:39:17.014 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:16.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:16.742486+0000) 2023-08-28T02:39:17.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:17 smithi136 ceph-mon[113531]: pgmap v10370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:17.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:17 smithi093 ceph-mon[111335]: pgmap v10370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:17.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:17 smithi093 ceph-mon[118902]: pgmap v10370: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:18.018 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:17.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:17.742715+0000) 2023-08-28T02:39:18.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:18 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4042929750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:18.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:18 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4042929750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:18.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:18 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4042929750' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:19.020 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:18.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:18.742976+0000) 2023-08-28T02:39:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:19 smithi136 ceph-mon[113531]: pgmap v10371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:19 smithi093 ceph-mon[111335]: pgmap v10371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:19 smithi093 ceph-mon[118902]: pgmap v10371: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:20.023 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:19.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:19.743194+0000) 2023-08-28T02:39:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/404184933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/404184933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/404184933' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:21.024 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:20.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:20.743436+0000) 2023-08-28T02:39:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:21 smithi136 ceph-mon[113531]: pgmap v10372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:21 smithi093 ceph-mon[111335]: pgmap v10372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:21 smithi093 ceph-mon[118902]: pgmap v10372: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:21.616Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:21.617Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:22.027 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:21.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:21.743688+0000) 2023-08-28T02:39:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1217284605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1217284605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:22.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1217284605' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:23.029 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:22.742+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:22.743885+0000) 2023-08-28T02:39:23.336 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:23 smithi136 ceph-mon[113531]: pgmap v10373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:23.336 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:23 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:39:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:23 smithi093 ceph-mon[111335]: pgmap v10373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:23 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:39:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:23 smithi093 ceph-mon[118902]: pgmap v10373: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:23 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:39:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:23.043Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:23.044Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:24.033 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:23.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:23.744073+0000) 2023-08-28T02:39:24.112 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:24.112 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:24.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:24.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:24.744265+0000) 2023-08-28T02:39:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: pgmap v10374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2711226387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:25.689 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:39:25] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: pgmap v10374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2711226387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: pgmap v10374: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2711226387' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:39:25.740 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:25 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:39:26.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:25.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:25.744474+0000) 2023-08-28T02:39:27.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:26.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:26.744702+0000) 2023-08-28T02:39:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:27 smithi136 ceph-mon[113531]: pgmap v10375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:27.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/943496120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:27 smithi093 ceph-mon[111335]: pgmap v10375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:27.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/943496120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:27 smithi093 ceph-mon[118902]: pgmap v10375: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:27.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/943496120' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:28.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:27.743+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:27.744912+0000) 2023-08-28T02:39:29.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:28.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:28.745189+0000) 2023-08-28T02:39:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:29 smithi136 ceph-mon[113531]: pgmap v10376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:29.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2506882856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:29 smithi093 ceph-mon[111335]: pgmap v10376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:29.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2506882856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:29.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:29 smithi093 ceph-mon[118902]: pgmap v10376: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:29.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2506882856' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:29.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:29.745412+0000) 2023-08-28T02:39:31.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:30.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:30.745666+0000) 2023-08-28T02:39:31.616 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:31 smithi093 ceph-mon[111335]: pgmap v10377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:31.616 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:31 smithi093 ceph-mon[118902]: pgmap v10377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:31.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:31 smithi136 ceph-mon[113531]: pgmap v10377: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:31.616Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:31.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:31.618Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:31.744+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:31.745922+0000) 2023-08-28T02:39:32.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:32 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4206728447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:32.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:32 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4206728447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:32.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:32 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4206728447' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:33.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:32.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:32.746162+0000) 2023-08-28T02:39:33.335 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:33.044Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:33.335 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:33.045Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:33.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:33 smithi136 ceph-mon[113531]: pgmap v10378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:33.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:33 smithi093 ceph-mon[111335]: pgmap v10378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:33.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:33 smithi093 ceph-mon[118902]: pgmap v10378: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:34.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:33.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:33.746412+0000) 2023-08-28T02:39:34.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/492620569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:34.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/492620569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:34.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/492620569' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:35.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:34.745+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:34.746659+0000) 2023-08-28T02:39:35.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:35 smithi136 ceph-mon[113531]: pgmap v10379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:35.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:35 smithi093 ceph-mon[111335]: pgmap v10379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:35.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:39:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:39:35] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:39:35.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:35 smithi093 ceph-mon[118902]: pgmap v10379: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:36.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:35.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:35.746914+0000) 2023-08-28T02:39:36.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/37908942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:36.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/37908942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:36.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/37908942' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:36.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:36.747139+0000) 2023-08-28T02:39:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:37 smithi136 ceph-mon[113531]: pgmap v10380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:37.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:37 smithi093 ceph-mon[111335]: pgmap v10380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:37.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:37 smithi093 ceph-mon[118902]: pgmap v10380: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:37.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:37.747326+0000) 2023-08-28T02:39:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:38.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:38.747571+0000) 2023-08-28T02:39:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:39 smithi136 ceph-mon[113531]: pgmap v10381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1998948314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:39.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:39 smithi093 ceph-mon[111335]: pgmap v10381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1998948314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:39.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:39 smithi093 ceph-mon[118902]: pgmap v10381: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1998948314' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:39.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:39.746+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:39.747811+0000) 2023-08-28T02:39:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:40.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:40.748095+0000) 2023-08-28T02:39:41.617 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:41 smithi093 ceph-mon[111335]: pgmap v10382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:41.617 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3275888620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:41.617 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:41 smithi093 ceph-mon[118902]: pgmap v10382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:41.617 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3275888620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:41 smithi136 ceph-mon[113531]: pgmap v10382: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3275888620' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:41.618Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:41.619Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:41.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:41.748334+0000) 2023-08-28T02:39:43.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:42.747+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:42.748577+0000) 2023-08-28T02:39:43.359 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:43.045Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:43.359 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:43.046Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:43 smithi136 ceph-mon[113531]: pgmap v10383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:43.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:43 smithi093 ceph-mon[111335]: pgmap v10383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:43.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:43 smithi093 ceph-mon[118902]: pgmap v10383: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:43.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:43.748816+0000) 2023-08-28T02:39:44.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:44 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1455579399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:44.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:44 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1455579399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:44.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:44 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1455579399' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:45.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:44.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:44.749070+0000) 2023-08-28T02:39:45.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:45 smithi136 ceph-mon[113531]: pgmap v10384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:45.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:45 smithi093 ceph-mon[111335]: pgmap v10384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:45.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:45 smithi093 ceph-mon[118902]: pgmap v10384: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:45.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:39:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:39:45] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:39:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:45.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:45.749284+0000) 2023-08-28T02:39:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2220994432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:46.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2220994432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:46.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2220994432' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:46.748+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:46.749564+0000) 2023-08-28T02:39:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:47 smithi136 ceph-mon[113531]: pgmap v10385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:47.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:47 smithi093 ceph-mon[111335]: pgmap v10385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:47.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:47 smithi093 ceph-mon[118902]: pgmap v10385: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:47.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:47.749810+0000) 2023-08-28T02:39:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2899390126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:48.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2899390126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:48.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2899390126' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:48.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:48.750049+0000) 2023-08-28T02:39:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:49 smithi136 ceph-mon[113531]: pgmap v10386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:49.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:49 smithi093 ceph-mon[111335]: pgmap v10386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:49.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:49 smithi093 ceph-mon[118902]: pgmap v10386: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:49.749+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:49.750295+0000) 2023-08-28T02:39:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:50.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:50.750541+0000) 2023-08-28T02:39:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:51 smithi136 ceph-mon[113531]: pgmap v10387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:51 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3570106998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:51 smithi093 ceph-mon[111335]: pgmap v10387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:51.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:51 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3570106998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:51 smithi093 ceph-mon[118902]: pgmap v10387: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:51.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:51 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3570106998' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:51.619Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:51.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:51.620Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:51.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:51.750793+0000) 2023-08-28T02:39:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:52.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:52.751005+0000) 2023-08-28T02:39:53.384 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:53.046Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:53.384 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:39:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:39:53.047Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:39:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:53 smithi136 ceph-mon[113531]: pgmap v10388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3201343340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:53.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:53 smithi093 ceph-mon[111335]: pgmap v10388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:53.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3201343340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:53 smithi093 ceph-mon[118902]: pgmap v10388: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:53.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3201343340' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:53.750+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:53.751235+0000) 2023-08-28T02:39:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:54.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:54.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:39:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:54.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:54.751525+0000) 2023-08-28T02:39:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:55 smithi136 ceph-mon[113531]: pgmap v10389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3951351057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:55 smithi093 ceph-mon[111335]: pgmap v10389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:55.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3951351057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:55.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:39:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:39:55] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:39:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:55 smithi093 ceph-mon[118902]: pgmap v10389: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:55.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3951351057' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:55.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:55.751756+0000) 2023-08-28T02:39:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:56.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:56.751982+0000) 2023-08-28T02:39:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:57 smithi136 ceph-mon[113531]: pgmap v10390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:57.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:57 smithi093 ceph-mon[111335]: pgmap v10390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:57.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:57 smithi093 ceph-mon[118902]: pgmap v10390: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:57.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:57.752273+0000) 2023-08-28T02:39:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2853610265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2853610265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2853610265' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:39:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:58.751+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:58.752506+0000) 2023-08-28T02:39:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:39:59 smithi136 ceph-mon[113531]: pgmap v10391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:39:59 smithi093 ceph-mon[111335]: pgmap v10391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:39:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:39:59 smithi093 ceph-mon[118902]: pgmap v10391: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:39:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:39:59.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:39:59.752747+0000) 2023-08-28T02:40:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:00 smithi136 ceph-mon[113531]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:40:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4047603026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:00 smithi093 ceph-mon[111335]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:40:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4047603026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:00 smithi093 ceph-mon[118902]: overall HEALTH_WARN Upgrade: Need standby mgr daemon 2023-08-28T02:40:00.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4047603026' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:00.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:00.753052+0000) 2023-08-28T02:40:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:01 smithi136 ceph-mon[113531]: pgmap v10392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:01.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:01 smithi093 ceph-mon[111335]: pgmap v10392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:01.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:01 smithi093 ceph-mon[118902]: pgmap v10392: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:01.619Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:01.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:01.620Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:01.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:01.753290+0000) 2023-08-28T02:40:03.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:02.752+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:02.753483+0000) 2023-08-28T02:40:03.410 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:03.047Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:03.410 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:03.048Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:03 smithi136 ceph-mon[113531]: pgmap v10393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:03.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3706099875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:03 smithi093 ceph-mon[111335]: pgmap v10393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:03.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3706099875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:03 smithi093 ceph-mon[118902]: pgmap v10393: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:03.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3706099875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:03.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:03.753682+0000) 2023-08-28T02:40:05.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:04.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:04.753928+0000) 2023-08-28T02:40:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:05 smithi136 ceph-mon[113531]: pgmap v10394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:05.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/862662273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:05 smithi093 ceph-mon[111335]: pgmap v10394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:05.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/862662273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:05 smithi093 ceph-mon[118902]: pgmap v10394: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:05.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/862662273' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:05.739 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:40:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:40:05] "GET /metrics HTTP/1.1" 200 239947 "" "Prometheus/2.43.0" 2023-08-28T02:40:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:05.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:05.754195+0000) 2023-08-28T02:40:07.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:06.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:06.754384+0000) 2023-08-28T02:40:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:07 smithi136 ceph-mon[113531]: pgmap v10395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:07.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2943423930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:07 smithi093 ceph-mon[111335]: pgmap v10395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:07.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2943423930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:07 smithi093 ceph-mon[118902]: pgmap v10395: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:07.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2943423930' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:07.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:07.754613+0000) 2023-08-28T02:40:09.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:08.753+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:08.754877+0000) 2023-08-28T02:40:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:09 smithi136 ceph-mon[113531]: pgmap v10396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:09.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:09 smithi093 ceph-mon[111335]: pgmap v10396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:09.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:09 smithi093 ceph-mon[118902]: pgmap v10396: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:09.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:10.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:09.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:09.755150+0000) 2023-08-28T02:40:10.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1283654950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:10.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1283654950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:10.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1283654950' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:11.190 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:10.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:10.755364+0000) 2023-08-28T02:40:11.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:11 smithi136 ceph-mon[113531]: pgmap v10397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:11.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:11 smithi093 ceph-mon[111335]: pgmap v10397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:11.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:11 smithi093 ceph-mon[118902]: pgmap v10397: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:11.620Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:11.739 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:11.621Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:12.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:11.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:11.755612+0000) 2023-08-28T02:40:12.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3794429372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:12.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3794429372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:12.754 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3794429372' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:13.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:12.754+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:12.755856+0000) 2023-08-28T02:40:13.439 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:13.048Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:13.440 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:13.049Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:13.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:13 smithi093 ceph-mon[111335]: pgmap v10398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:13.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:13 smithi093 ceph-mon[118902]: pgmap v10398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:13.755 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:13 smithi136 ceph-mon[113531]: pgmap v10398: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:14.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:13.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:13.756128+0000) 2023-08-28T02:40:14.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:14 smithi093 ceph-mon[111335]: pgmap v10399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:14.739 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3643470072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:14 smithi093 ceph-mon[118902]: pgmap v10399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:14.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3643470072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:14.755 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:14 smithi136 ceph-mon[113531]: pgmap v10399: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:14.755 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3643470072' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:15.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:14 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:14.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:14.756380+0000) 2023-08-28T02:40:15.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:40:15 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:40:15] "GET /metrics HTTP/1.1" 200 239948 "" "Prometheus/2.43.0" 2023-08-28T02:40:16.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:15 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:15.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:15.756619+0000) 2023-08-28T02:40:16.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:16 smithi136 ceph-mon[113531]: pgmap v10400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:16.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:16 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:16.755+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:16.756809+0000) 2023-08-28T02:40:16.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:16 smithi093 ceph-mon[111335]: pgmap v10400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:16.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:16 smithi093 ceph-mon[118902]: pgmap v10400: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:17.938 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:17 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2265497602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:17.938 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:17 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:17.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:17.757024+0000) 2023-08-28T02:40:17.988 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:17 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2265497602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:17.989 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:17 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2265497602' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:19.021 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:18 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:18.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:18.757244+0000) 2023-08-28T02:40:19.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:19 smithi136 ceph-mon[113531]: pgmap v10401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:19.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:19 smithi093 ceph-mon[111335]: pgmap v10401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:19.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:19 smithi093 ceph-mon[118902]: pgmap v10401: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:20.026 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:19 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:19.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:19.757452+0000) 2023-08-28T02:40:20.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:20 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3105136960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:20.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:20 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3105136960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:20.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:20 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3105136960' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:21.027 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:20 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:20.756+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:20.757688+0000) 2023-08-28T02:40:21.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:21 smithi136 ceph-mon[113531]: pgmap v10402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:21.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:21 smithi093 ceph-mon[111335]: pgmap v10402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:21.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:21 smithi093 ceph-mon[118902]: pgmap v10402: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:21.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:21.621Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:21.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:21 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:21.622Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:22.032 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:21 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:21.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:21.757892+0000) 2023-08-28T02:40:22.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:22 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2716733792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:22.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:22 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2716733792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:22.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:22 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2716733792' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:23.033 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:22 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:22.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:22.758152+0000) 2023-08-28T02:40:23.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:23 smithi136 ceph-mon[113531]: pgmap v10403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:23.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:23 smithi093 ceph-mon[111335]: pgmap v10403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:23.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:23 smithi093 ceph-mon[118902]: pgmap v10403: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:23.049Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:23.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:23 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:23.050Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:24.038 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:23 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:23.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:23.758396+0000) 2023-08-28T02:40:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:24 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1068453728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:24.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:24 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:24 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1068453728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:24.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:24 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:24 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1068453728' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:24.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:24 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:25.038 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:24 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:24.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:24.758587+0000) 2023-08-28T02:40:25.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:25 smithi093 ceph-mon[111335]: pgmap v10404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:25.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:25 smithi093 ceph-mon[118902]: pgmap v10404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:25.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:25 smithi136 ceph-mon[113531]: pgmap v10404: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:25.710 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:40:25 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:40:25] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:40:26.042 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:25 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:25.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:25.758780+0000) 2023-08-28T02:40:26.331 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:26 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:26.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:26 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:26.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:26 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config dump", "format": "json"}]: dispatch 2023-08-28T02:40:27.043 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:26 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:26.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:26.758875+0000) 2023-08-28T02:40:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:27 smithi093 ceph-mon[111335]: pgmap v10405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:27.238 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:27 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/346088459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:27 smithi093 ceph-mon[118902]: pgmap v10405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:27.239 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:27 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/346088459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:27 smithi136 ceph-mon[113531]: pgmap v10405: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:27.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:27 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/346088459' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:28.157 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:27 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:27.757+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:27.759009+0000) 2023-08-28T02:40:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:28 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:28.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:28 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:29.163 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:28 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:28.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:28.759233+0000) 2023-08-28T02:40:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:29 smithi136 ceph-mon[113531]: pgmap v10406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:29 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:29.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:29 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1297125358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[111335]: pgmap v10406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:29.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:29.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1297125358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[118902]: pgmap v10406: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:29.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "config generate-minimal-conf"}]: dispatch 2023-08-28T02:40:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "auth get", "entity": "client.admin"}]: dispatch 2023-08-28T02:40:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' 2023-08-28T02:40:29.490 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:29 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1297125358' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:30.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:29 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:29.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:29.759471+0000) 2023-08-28T02:40:31.166 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:30 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:30.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:30.759716+0000) 2023-08-28T02:40:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:31 smithi136 ceph-mon[113531]: pgmap v10407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:31.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:31 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1226764565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:31 smithi093 ceph-mon[111335]: pgmap v10407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:31.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:31 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1226764565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:31 smithi093 ceph-mon[118902]: pgmap v10407: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:31.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:31 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1226764565' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:31.621Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:31.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:31 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:31.623Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:32.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:31 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:31.758+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:31.759989+0000) 2023-08-28T02:40:33.173 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:32 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:32.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:32.760206+0000) 2023-08-28T02:40:33.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:33 smithi136 ceph-mon[113531]: pgmap v10408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:33.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:33 smithi093 ceph-mon[111335]: pgmap v10408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:33.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:33 smithi093 ceph-mon[118902]: pgmap v10408: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:33.049Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:33.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:33 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:33.050Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:34.180 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:33 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:33.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:33.760399+0000) 2023-08-28T02:40:34.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:34 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/770067702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:34.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:34 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/770067702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:34.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:34 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/770067702' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:35.181 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:34 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:34.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:34.760689+0000) 2023-08-28T02:40:35.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:35 smithi136 ceph-mon[113531]: pgmap v10409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:35.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:35 smithi093 ceph-mon[111335]: pgmap v10409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:35.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:35 smithi093 ceph-mon[118902]: pgmap v10409: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:35.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:40:35 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:40:35] "GET /metrics HTTP/1.1" 200 239949 "" "Prometheus/2.43.0" 2023-08-28T02:40:36.187 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:35 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:35.759+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:35.760952+0000) 2023-08-28T02:40:36.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:36 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2904100886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:36.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:36 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2904100886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:36.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:36 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2904100886' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:37.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:36 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:36.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:36.761180+0000) 2023-08-28T02:40:37.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:37 smithi093 ceph-mon[111335]: pgmap v10410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:37.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:37 smithi093 ceph-mon[118902]: pgmap v10410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:37.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:37 smithi136 ceph-mon[113531]: pgmap v10410: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:38.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:37 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:37.760+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:37.761368+0000) 2023-08-28T02:40:39.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:38 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:38.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:38.761622+0000) 2023-08-28T02:40:39.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:39 smithi093 ceph-mon[111335]: pgmap v10411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:39 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/162151038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:39.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:39 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:39 smithi093 ceph-mon[118902]: pgmap v10411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:39 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/162151038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:39.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:39 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:39 smithi136 ceph-mon[113531]: pgmap v10411: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:39 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/162151038' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:39.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:39 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:40.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:39 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:39.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:39.761848+0000) 2023-08-28T02:40:41.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:40 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:40.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:40.762085+0000) 2023-08-28T02:40:41.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:41 smithi093 ceph-mon[111335]: pgmap v10412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:41.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:41 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/356576388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:41 smithi093 ceph-mon[118902]: pgmap v10412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:41.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:41 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/356576388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:41 smithi136 ceph-mon[113531]: pgmap v10412: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:41.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:41 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/356576388' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:41.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:41.622Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:41.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:41 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:41.623Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:42.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:41 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:41.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:41.762305+0000) 2023-08-28T02:40:43.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:42 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:42.761+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:42.762541+0000) 2023-08-28T02:40:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:43 smithi093 ceph-mon[111335]: pgmap v10413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:43.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:43 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2666807397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:43 smithi093 ceph-mon[118902]: pgmap v10413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:43.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:43 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2666807397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:43.050Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:43.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:43 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:43.051Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:43 smithi136 ceph-mon[113531]: pgmap v10413: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:43.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:43 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2666807397' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:44.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:43 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:43.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:43.762764+0000) 2023-08-28T02:40:45.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:44 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:44.762+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:44.762963+0000) 2023-08-28T02:40:45.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:45 smithi093 ceph-mon[111335]: pgmap v10414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:45.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:45 smithi093 ceph-mon[118902]: pgmap v10414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:45.489 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:40:45 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:40:45] "GET /metrics HTTP/1.1" 200 239950 "" "Prometheus/2.43.0" 2023-08-28T02:40:45.690 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:45 smithi136 ceph-mon[113531]: pgmap v10414: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:46.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:45 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:45.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:45.763188+0000) 2023-08-28T02:40:46.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:46 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/840681896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:46.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:46 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/840681896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:46.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:46 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/840681896' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:47.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:46 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:46.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:46.763451+0000) 2023-08-28T02:40:47.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:47 smithi093 ceph-mon[111335]: pgmap v10415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:47.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:47 smithi093 ceph-mon[118902]: pgmap v10415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:47.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:47 smithi136 ceph-mon[113531]: pgmap v10415: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:48.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:47 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:47.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:47.763686+0000) 2023-08-28T02:40:48.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:48 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/64863200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:48.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:48 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/64863200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:48.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:48 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/64863200' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:49.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:48 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:48.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:48.763919+0000) 2023-08-28T02:40:49.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:49 smithi093 ceph-mon[111335]: pgmap v10416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:49.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:49 smithi093 ceph-mon[118902]: pgmap v10416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:49.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:49 smithi136 ceph-mon[113531]: pgmap v10416: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:50.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:49 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:49.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:49.764174+0000) 2023-08-28T02:40:50.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:50 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3171151207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:50.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:50 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3171151207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:50.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:50 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3171151207' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:51.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:50 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:50.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:50.764418+0000) 2023-08-28T02:40:51.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:51 smithi093 ceph-mon[111335]: pgmap v10417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:51.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:51 smithi093 ceph-mon[118902]: pgmap v10417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:51.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:51 smithi136 ceph-mon[113531]: pgmap v10417: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:51.623Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:51.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:51 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:51.624Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:52.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:51 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:51.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:51.764640+0000) 2023-08-28T02:40:53.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:52 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:52.763+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:52.764912+0000) 2023-08-28T02:40:53.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:53 smithi093 ceph-mon[111335]: pgmap v10418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:53.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:53 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/4194307875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:53 smithi093 ceph-mon[118902]: pgmap v10418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:53.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:53 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/4194307875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:53.050Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:53.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:40:53 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:40:53.051Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:40:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:53 smithi136 ceph-mon[113531]: pgmap v10418: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:53.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:53 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/4194307875' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:54.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:53 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:53.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:53.765138+0000) 2023-08-28T02:40:54.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:54 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:54.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:54 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:54.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:54 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:40:55.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:54 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:54.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:54.765362+0000) 2023-08-28T02:40:55.488 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:40:55 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:40:55] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:40:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:55 smithi093 ceph-mon[111335]: pgmap v10419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:55.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:55 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1563239341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:55 smithi093 ceph-mon[118902]: pgmap v10419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:55.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:55 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1563239341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:55 smithi136 ceph-mon[113531]: pgmap v10419: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:55.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:55 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1563239341' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:56.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:55 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:55.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:55.765585+0000) 2023-08-28T02:40:57.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:56 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:56.764+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:56.765790+0000) 2023-08-28T02:40:57.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:57 smithi093 ceph-mon[111335]: pgmap v10420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:57.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:57 smithi093 ceph-mon[118902]: pgmap v10420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:57.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:57 smithi136 ceph-mon[113531]: pgmap v10420: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:58.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:57 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:57.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:57.766047+0000) 2023-08-28T02:40:58.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:58 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3393613205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:58.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:58 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3393613205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:58.739 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:58 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3393613205' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:40:59.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:58 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:58.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:58.766320+0000) 2023-08-28T02:40:59.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:40:59 smithi136 ceph-mon[113531]: pgmap v10421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:59.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:40:59 smithi093 ceph-mon[111335]: pgmap v10421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:40:59.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:40:59 smithi093 ceph-mon[118902]: pgmap v10421: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:00.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:40:59 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:40:59.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:40:59.766555+0000) 2023-08-28T02:41:00.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:00 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1288825513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:00.738 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:00 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1288825513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:00.738 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:00 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1288825513' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:01.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:00 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:00.765+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:00.766723+0000) 2023-08-28T02:41:01.624 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:01 smithi093 ceph-mon[111335]: pgmap v10422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:01.624 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:01 smithi093 ceph-mon[118902]: pgmap v10422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:01.688 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:01 smithi136 ceph-mon[113531]: pgmap v10422: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:01.624Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:01.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:01 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:01.625Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:02.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:01 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:01.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:01.766973+0000) 2023-08-28T02:41:03.023 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:02 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:02.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:02.767210+0000) 2023-08-28T02:41:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:03 smithi136 ceph-mon[113531]: pgmap v10423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:03.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:03 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1959172011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:03.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:03 smithi093 ceph-mon[111335]: pgmap v10423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:03.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:03 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1959172011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:03 smithi093 ceph-mon[118902]: pgmap v10423: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:03.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:03 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1959172011' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:03.051Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:03.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:03 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:03.052Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:04.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:03 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:03.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:03.767433+0000) 2023-08-28T02:41:05.026 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:04 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:04.766+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:04.767676+0000) 2023-08-28T02:41:05.316 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:05 smithi136 ceph-mon[113531]: pgmap v10424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:05.317 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:05 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/2728602878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:05.357 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:05 smithi093 ceph-mon[111335]: pgmap v10424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:05.358 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:05 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/2728602878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:05 smithi093 ceph-mon[118902]: pgmap v10424: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:05.358 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:05 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/2728602878' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:05.688 INFO:journalctl@ceph.grafana.a.smithi136.stdout:Aug 28 02:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-grafana-a[136614]: logger=cleanup t=2023-08-28T02:41:05.317925047Z level=info msg="Completed cleanup jobs" duration=12.853694ms 2023-08-28T02:41:05.738 INFO:journalctl@ceph.mgr.y.smithi093.stdout:Aug 28 02:41:05 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-y[111560]: ::ffff:172.21.15.136 - - [28/Aug/2023:02:41:05] "GET /metrics HTTP/1.1" 200 239946 "" "Prometheus/2.43.0" 2023-08-28T02:41:06.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:05 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:05.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:05.767897+0000) 2023-08-28T02:41:07.029 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:06 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:06.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:06.768149+0000) 2023-08-28T02:41:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:07 smithi136 ceph-mon[113531]: pgmap v10425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:07.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:07 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3716739448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:07 smithi093 ceph-mon[118902]: pgmap v10425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:07.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:07 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3716739448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:07.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:07 smithi093 ceph-mon[111335]: pgmap v10425: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:07.489 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:07 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3716739448' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:08.188 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:07 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:07.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:07.768335+0000) 2023-08-28T02:41:09.032 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:08 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:08.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:08.768602+0000) 2023-08-28T02:41:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:09 smithi136 ceph-mon[113531]: pgmap v10426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:09.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:09 smithi136 ceph-mon[113531]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:41:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:09 smithi093 ceph-mon[111335]: pgmap v10426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:09.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:09 smithi093 ceph-mon[111335]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:41:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:09 smithi093 ceph-mon[118902]: pgmap v10426: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:09.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:09 smithi093 ceph-mon[118902]: from='mgr.24881 172.21.15.93:0/2285617367' entity='mgr.y' cmd=[{"prefix": "osd blocklist ls", "format": "json"}]: dispatch 2023-08-28T02:41:10.036 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:09 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:09.767+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:09.768875+0000) 2023-08-28T02:41:10.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:10 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/1293779925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:10.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:10 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/1293779925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:10.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:10 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/1293779925' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:11.038 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:10 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:10.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:10.769110+0000) 2023-08-28T02:41:11.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:11 smithi136 ceph-mon[113531]: pgmap v10427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:11.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:11 smithi093 ceph-mon[111335]: pgmap v10427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:11.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:11 smithi093 ceph-mon[118902]: pgmap v10427: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:11.988 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:11.625Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 7 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:11.989 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:11 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:11.626Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:12.042 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:11 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:11.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:11.769343+0000) 2023-08-28T02:41:12.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:12 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/174782222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:12.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:12 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/174782222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:12.488 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:12 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/174782222' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:13.044 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:12 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:12.768+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:12.769553+0000) 2023-08-28T02:41:13.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:13 smithi136 ceph-mon[113531]: pgmap v10428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:13.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:13 smithi093 ceph-mon[111335]: pgmap v10428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:13.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:13 smithi093 ceph-mon[118902]: pgmap v10428: 73 pgs: 73 active+clean; 580 KiB data, 206 MiB used, 715 GiB / 715 GiB avail 2023-08-28T02:41:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:13.051Z caller=dispatch.go:352 level=error component=dispatcher msg="Notify for alerts failed" num_alerts=1 err="ceph-dashboard/webhook[1]: notify retry canceled after 8 attempts: Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:13.489 INFO:journalctl@ceph.alertmanager.a.smithi093.stdout:Aug 28 02:41:13 smithi093 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-alertmanager-a[146346]: ts=2023-08-28T02:41:13.052Z caller=notify.go:732 level=warn component=dispatcher receiver=ceph-dashboard integration=webhook[1] msg="Notify attempt failed, will retry later" attempts=1 err="Post \"https://smithi136.front.sepia.ceph.com:8443/api/prometheus_receiver\": dial tcp 172.21.15.136:8443: connect: connection refused" 2023-08-28T02:41:14.048 INFO:journalctl@ceph.mgr.x.smithi136.stdout:Aug 28 02:41:13 smithi136 ceph-b79a52c0-451a-11ee-817b-1b7a5f1ba315-mgr-x[140597]: 2023-08-28T02:41:13.769+0000 7fa0d06a8700 -1 monclient: _check_auth_rotating possible clock skew, rotating keys expired way too early (before 2023-08-28T01:41:13.769786+0000) 2023-08-28T02:41:14.438 INFO:journalctl@ceph.mon.b.smithi136.stdout:Aug 28 02:41:14 smithi136 ceph-mon[113531]: from='client.? 172.21.15.93:0/3690275415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:14.488 INFO:journalctl@ceph.mon.a.smithi093.stdout:Aug 28 02:41:14 smithi093 ceph-mon[111335]: from='client.? 172.21.15.93:0/3690275415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch 2023-08-28T02:41:14.489 INFO:journalctl@ceph.mon.c.smithi093.stdout:Aug 28 02:41:14 smithi093 ceph-mon[118902]: from='client.? 172.21.15.93:0/3690275415' entity='client.admin' cmd=[{"prefix": "mon versions"}]: dispatch